diff --git a/Makefile b/Makefile index 61784d3e9..58d58f27b 100644 --- a/Makefile +++ b/Makefile @@ -182,6 +182,39 @@ check: deps fmt vet test run: build @$(BUILD_DIR)/$(BINARY_NAME) $(ARGS) +# --------------------------------------------------------------------------- +# Evaluation Harness +# Usage: make eval-build && make eval +# make eval-compare (A/B: branch vs main) +# --------------------------------------------------------------------------- + +## eval-build: Build the eval runner from the current branch +eval-build: generate + @echo "Building eval runner..." + @mkdir -p eval/bin + @$(GO) build $(GOFLAGS) $(LDFLAGS) -o eval/bin/eval-runner ./eval/cmd/eval-runner + @echo "Eval runner built: eval/bin/eval-runner" + +## eval: Run the eval suite against the current build +eval: eval-build + @echo "Running eval suite..." + @cd eval && npx promptfoo eval --no-progress-bar + @echo "Results: eval/results/latest.json" + @echo "View: cd eval && npx promptfoo view" + +## eval-view: Open the promptfoo results viewer +eval-view: + @cd eval && npx promptfoo view + +## eval-compare: A/B comparison of current branch vs main +eval-compare: + @./eval/scripts/compare.sh --repeat 3 + +## eval-test: Run Go-native component evals +eval-test: + @echo "Running Go-native evals..." + @$(GO) test -v ./eval/go_evals/... + ## help: Show this help message help: @echo "picoclaw Makefile" diff --git a/eval/.gitignore b/eval/.gitignore new file mode 100644 index 000000000..4863f7870 --- /dev/null +++ b/eval/.gitignore @@ -0,0 +1,5 @@ +bin/ +results/ +node_modules/ +promptfooconfig-compare.yaml +*.tmp diff --git a/eval/README.md b/eval/README.md new file mode 100644 index 000000000..541905041 --- /dev/null +++ b/eval/README.md @@ -0,0 +1,107 @@ +# PicoClaw Eval Harness + +End-to-end evaluation system for the PicoClaw agent runtime. + +## Quick Start + +```bash +# Install promptfoo (one-time) +npm install -g promptfoo + +# Build eval runner and run suite +make eval + +# View results in browser +make eval-view + +# Run Go-native component evals +make eval-test + +# A/B comparison (current branch vs main) +make eval-compare +``` + +## Architecture + +``` +eval/ +├── cmd/eval-runner/ # Go binary that wraps picoclaw for promptfoo +├── cases/ # Golden dataset (YAML test cases) +│ ├── tool_calling.yaml +│ ├── token_efficiency.yaml +│ ├── multi_step.yaml +│ └── edge_cases.yaml +├── go_evals/ # Go-native component tests (memory, tools) +├── scripts/ # CI/comparison scripts +│ └── compare.sh +├── bin/ # Built eval runner binaries (gitignored) +├── results/ # Eval output JSON (gitignored) +└── promptfooconfig.yaml # Main eval configuration +``` + +## How It Works + +1. **eval-runner** wraps picoclaw with an instrumented language model that captures + every LLM call, tool invocation, token count, and timing. + +2. **promptfoo** invokes `eval-runner` via `exec:` provider, sending prompts as JSON + on stdin and parsing the structured trace JSON from stdout. + +3. **Assertions** are JavaScript functions that inspect the trace to score: + - Tool selection correctness + - Output quality + - Token efficiency + - Latency thresholds + - Error handling + +## Trace Format + +The eval runner emits JSON with this structure: + +```json +{ + "output": "Agent response text", + "steps": [ + {"index": 0, "type": "llm_call", "duration_ms": 1200}, + {"index": 1, "type": "tool_call", "tool": "read_file", "args": {...}, "result": "..."} + ], + "metrics": { + "total_duration_ms": 3400, + "step_count": 2, + "tool_call_count": 1, + "input_tokens": 450, + "output_tokens": 120, + "total_tokens": 570, + "reasoning_tokens": 0, + "cache_read_tokens": 0 + }, + "error": null, + "session_key": "eval:1708300000000" +} +``` + +## Adding Test Cases + +Create a new YAML file in `eval/cases/` following this pattern: + +```yaml +- description: "what this tests" + vars: + prompt: "the prompt to send to picoclaw" + assert: + - type: javascript + value: | + const trace = JSON.parse(output); + // Return { pass: bool, score: 0-1, reason: string } + return { pass: true, score: 1.0, reason: 'explanation' }; +``` + +## A/B Comparison + +`make eval-compare` builds both your current branch and main, then runs the +identical test suite against both. Results show a side-by-side comparison +matrix with per-test scores. + +## Environment Variables + +- `PICOCLAW_EVAL_CONFIG` - Path to picoclaw config.json for eval (defaults to `~/.picoclaw/config.json`) diff --git a/eval/cases/edge_cases.yaml b/eval/cases/edge_cases.yaml new file mode 100644 index 000000000..7506c34c1 --- /dev/null +++ b/eval/cases/edge_cases.yaml @@ -0,0 +1,38 @@ +# Edge Case Evaluation +# Tests error handling, ambiguous inputs, and boundary conditions. + +- description: "missing file: graceful handling of non-existent file" + vars: + prompt: "Read the file /tmp/picoclaw_eval_nonexistent_file_abc123.txt" + assert: + - type: javascript + value: | + const trace = JSON.parse(output); + if (trace.error && !trace.output) return { pass: false, score: 0, reason: 'crashed: ' + trace.error }; + const mentionsError = trace.output.toLowerCase().includes('not found') || + trace.output.toLowerCase().includes('error') || + trace.output.toLowerCase().includes('does not exist') || + trace.output.toLowerCase().includes("doesn't exist") || + trace.output.toLowerCase().includes('no such file'); + return { pass: mentionsError, score: mentionsError ? 1.0 : 0.0, reason: mentionsError ? 'gracefully reported error' : 'did not report file missing' }; + +- description: "empty prompt resilience" + vars: + prompt: " " + assert: + - type: javascript + value: | + const trace = JSON.parse(output); + const noFatalError = !trace.error || trace.output.length > 0; + return { pass: noFatalError, score: noFatalError ? 1.0 : 0.0, reason: noFatalError ? 'handled gracefully' : 'fatal error on empty input' }; + +- description: "ambiguous request: agent asks for clarification or makes reasonable assumption" + vars: + prompt: "Do the thing with the file." + assert: + - type: javascript + value: | + const trace = JSON.parse(output); + if (trace.error && !trace.output) return { pass: false, score: 0, reason: 'crashed' }; + const responded = trace.output.length > 10; + return { pass: responded, score: responded ? 1.0 : 0.0, reason: responded ? 'provided a response' : 'no meaningful response' }; diff --git a/eval/cases/multi_step.yaml b/eval/cases/multi_step.yaml new file mode 100644 index 000000000..f9645db14 --- /dev/null +++ b/eval/cases/multi_step.yaml @@ -0,0 +1,37 @@ +# Multi-Step Task Evaluation Cases +# Tests that the agent can complete tasks requiring multiple tool invocations. + +- description: "create and verify: write a file then read it back" + vars: + prompt: "Write the text 'picoclaw eval checkpoint' to a file called eval_checkpoint.txt, then read it back and confirm the contents match." + assert: + - type: javascript + value: | + const trace = JSON.parse(output); + if (trace.error) return { pass: false, score: 0, reason: trace.error }; + const tools = trace.steps.filter(s => s.type === 'tool_call').map(s => s.tool); + const hasWrite = tools.includes('write_file') || tools.includes('tool_call'); + const hasRead = tools.filter(t => t === 'read_file' || t === 'tool_call').length >= 1; + const bothOps = hasWrite && hasRead; + return { pass: bothOps, score: bothOps ? 1.0 : 0.5, reason: `write=${hasWrite}, read=${hasRead}` }; + - type: javascript + value: | + const trace = JSON.parse(output); + const mentions = trace.output.toLowerCase().includes('checkpoint') || trace.output.toLowerCase().includes('match') || trace.output.toLowerCase().includes('confirm'); + return { pass: mentions, score: mentions ? 1.0 : 0.0, reason: mentions ? 'confirmed contents' : 'did not confirm' }; + +- description: "explore and summarize: list dir then describe contents" + vars: + prompt: "List the workspace directory, then give me a brief summary of what files are there and what the workspace is used for." + assert: + - type: javascript + value: | + const trace = JSON.parse(output); + if (trace.error) return { pass: false, score: 0, reason: trace.error }; + const hasListDir = trace.steps.some(s => s.tool === 'list_dir' || (s.tool === 'tool_call')); + return { pass: hasListDir, score: hasListDir ? 1.0 : 0.0, reason: hasListDir ? 'explored workspace' : 'did not explore' }; + - type: javascript + value: | + const trace = JSON.parse(output); + const hasSummary = trace.output.length > 50; + return { pass: hasSummary, score: hasSummary ? 1.0 : 0.0, reason: `output length: ${trace.output.length}` }; diff --git a/eval/cases/token_efficiency.yaml b/eval/cases/token_efficiency.yaml new file mode 100644 index 000000000..4c5981ed5 --- /dev/null +++ b/eval/cases/token_efficiency.yaml @@ -0,0 +1,48 @@ +# Token Efficiency Evaluation Cases +# Tests that the agent doesn't waste tokens on simple tasks. + +- description: "simple question: low token budget for trivial task" + vars: + prompt: "What is 2 + 2? Answer with just the number." + assert: + - type: javascript + value: | + const trace = JSON.parse(output); + if (trace.error) return { pass: false, score: 0, reason: trace.error }; + const hasAnswer = trace.output.includes('4'); + return { pass: hasAnswer, score: hasAnswer ? 1.0 : 0.0, reason: hasAnswer ? 'correct answer' : 'wrong answer' }; + - type: javascript + value: | + const trace = JSON.parse(output); + const tokens = trace.metrics.total_tokens; + const efficient = tokens < 2000; + const score = efficient ? 1.0 : Math.max(0, 1.0 - (tokens - 2000) / 5000); + return { pass: efficient, score, reason: `${tokens} total tokens (threshold: 2000)` }; + +- description: "no unnecessary tool calls for conversational query" + vars: + prompt: "Hello! How are you today?" + assert: + - type: javascript + value: | + const trace = JSON.parse(output); + if (trace.error) return { pass: false, score: 0, reason: trace.error }; + const toolCount = trace.metrics.tool_call_count; + const efficient = toolCount === 0; + return { pass: efficient, score: efficient ? 1.0 : Math.max(0, 1.0 - toolCount * 0.25), reason: `${toolCount} tool calls (expected: 0)` }; + - type: javascript + value: | + const trace = JSON.parse(output); + return { pass: trace.output.length > 0, score: 1.0, reason: 'responded' }; + +- description: "step count stays reasonable for multi-step task" + vars: + prompt: "Create a file called test_steps.txt with the content 'step test', then read it back to confirm." + assert: + - type: javascript + value: | + const trace = JSON.parse(output); + if (trace.error) return { pass: false, score: 0, reason: trace.error }; + const steps = trace.metrics.step_count; + const reasonable = steps <= 5; + return { pass: reasonable, score: reasonable ? 1.0 : Math.max(0, 1.0 - (steps - 5) * 0.2), reason: `${steps} LLM calls (threshold: 5)` }; diff --git a/eval/cases/tool_calling.yaml b/eval/cases/tool_calling.yaml new file mode 100644 index 000000000..1da0f5d6d --- /dev/null +++ b/eval/cases/tool_calling.yaml @@ -0,0 +1,53 @@ +# Tool Calling Evaluation Cases +# Tests that the agent correctly selects and invokes the right tools. + +- description: "file read: agent reads a file when asked about its contents" + vars: + prompt: "Read the file SYSTEM_PROMPT.md in the workspace and tell me the first line." + assert: + - type: javascript + value: | + const trace = JSON.parse(output); + if (trace.error) return { pass: false, score: 0, reason: trace.error }; + const toolCalls = trace.steps.filter(s => s.type === 'tool_call'); + const hasRead = toolCalls.some(t => t.tool === 'read_file' || t.tool === 'tool_call'); + return { pass: hasRead, score: hasRead ? 1.0 : 0.0, reason: hasRead ? 'correctly used file read' : 'did not read the file' }; + - type: javascript + value: | + const trace = JSON.parse(output); + return { pass: trace.output.length > 0, score: trace.output.length > 0 ? 1.0 : 0.0, reason: 'non-empty output' }; + +- description: "file write: agent creates a new file when asked" + vars: + prompt: "Create a file called eval_test_output.txt in the workspace with the text 'hello from eval'." + assert: + - type: javascript + value: | + const trace = JSON.parse(output); + if (trace.error) return { pass: false, score: 0, reason: trace.error }; + const toolCalls = trace.steps.filter(s => s.type === 'tool_call'); + const hasWrite = toolCalls.some(t => t.tool === 'write_file' || (t.tool === 'tool_call' && t.args && JSON.parse(t.args).tool_name === 'write_file')); + return { pass: hasWrite, score: hasWrite ? 1.0 : 0.0, reason: hasWrite ? 'correctly used file write' : 'did not write the file' }; + +- description: "shell exec: agent runs a shell command" + vars: + prompt: "Run the command 'echo picoclaw-eval-test' and tell me the output." + assert: + - type: javascript + value: | + const trace = JSON.parse(output); + if (trace.error) return { pass: false, score: 0, reason: trace.error }; + const hasExec = trace.steps.some(s => s.tool === 'exec' || (s.tool === 'tool_call' && s.args && JSON.parse(s.args).tool_name === 'exec')); + const outputContains = trace.output.includes('picoclaw-eval-test'); + return { pass: hasExec && outputContains, score: (hasExec ? 0.5 : 0) + (outputContains ? 0.5 : 0), reason: `exec=${hasExec}, output_correct=${outputContains}` }; + +- description: "list directory: agent lists workspace contents" + vars: + prompt: "List the files and directories in my workspace root." + assert: + - type: javascript + value: | + const trace = JSON.parse(output); + if (trace.error) return { pass: false, score: 0, reason: trace.error }; + const hasListDir = trace.steps.some(s => s.tool === 'list_dir' || (s.tool === 'tool_call' && s.args && JSON.parse(s.args).tool_name === 'list_dir')); + return { pass: hasListDir, score: hasListDir ? 1.0 : 0.0, reason: hasListDir ? 'used list_dir' : 'did not list directory' }; diff --git a/eval/cmd/eval-runner/main.go b/eval/cmd/eval-runner/main.go new file mode 100644 index 000000000..416211458 --- /dev/null +++ b/eval/cmd/eval-runner/main.go @@ -0,0 +1,301 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + fantasy "charm.land/fantasy" + "github.com/sipeed/picoclaw/pkg/agent" + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/config" + picofantasy "github.com/sipeed/picoclaw/pkg/fantasy" + "github.com/sipeed/picoclaw/pkg/logger" +) + +// Trace is the structured output emitted by the eval runner. +// Promptfoo parses this JSON to evaluate assertions. +type Trace struct { + Output string `json:"output"` + Steps []TraceStep `json:"steps"` + Metrics Metrics `json:"metrics"` + Error string `json:"error,omitempty"` + SessionKey string `json:"session_key"` +} + +type TraceStep struct { + Index int `json:"index"` + Type string `json:"type"` + Tool string `json:"tool,omitempty"` + Args json.RawMessage `json:"args,omitempty"` + Result string `json:"result,omitempty"` + Duration int64 `json:"duration_ms,omitempty"` +} + +type Metrics struct { + TotalDurationMs int64 `json:"total_duration_ms"` + StepCount int `json:"step_count"` + ToolCallCount int `json:"tool_call_count"` + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` + TotalTokens int64 `json:"total_tokens"` + ReasoningTokens int64 `json:"reasoning_tokens"` + CacheReadTokens int64 `json:"cache_read_tokens"` +} + +func main() { + logger.SetLevel(logger.ERROR) + + prompt, err := readPrompt() + if err != nil { + emitError(fmt.Sprintf("failed to read prompt: %v", err)) + return + } + + if strings.TrimSpace(prompt) == "" { + emitError("empty prompt") + return + } + + cfg, err := loadEvalConfig() + if err != nil { + emitError(fmt.Sprintf("config error: %v", err)) + return + } + + trace := runEval(cfg, prompt) + out, _ := json.Marshal(trace) + fmt.Println(string(out)) +} + +func readPrompt() (string, error) { + if len(os.Args) > 1 && os.Args[1] == "--prompt" && len(os.Args) > 2 { + return os.Args[2], nil + } + + data, err := io.ReadAll(os.Stdin) + if err != nil { + return "", err + } + + var promptData struct { + Prompt string `json:"prompt"` + } + if json.Unmarshal(data, &promptData) == nil && promptData.Prompt != "" { + return promptData.Prompt, nil + } + + return strings.TrimSpace(string(data)), nil +} + +func loadEvalConfig() (*config.Config, error) { + evalConfig := os.Getenv("PICOCLAW_EVAL_CONFIG") + if evalConfig != "" { + return config.LoadConfig(evalConfig) + } + + home, _ := os.UserHomeDir() + configPath := filepath.Join(home, ".picoclaw", "config.json") + return config.LoadConfig(configPath) +} + +func runEval(cfg *config.Config, prompt string) Trace { + start := time.Now() + + sessionKey := fmt.Sprintf("eval:%d", start.UnixNano()) + + fantasyProvider, err := picofantasy.CreateProvider(cfg) + if err != nil { + return Trace{Error: fmt.Sprintf("provider error: %v", err)} + } + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + languageModel, err := fantasyProvider.LanguageModel(ctx, picofantasy.ModelID(cfg)) + if err != nil { + return Trace{Error: fmt.Sprintf("model error: %v", err)} + } + + instrumentedModel := &instrumentedLanguageModel{ + inner: languageModel, + } + + msgBus := bus.NewMessageBus() + + // Drain outbound messages to prevent blocking + outDone := make(chan struct{}) + go func() { + defer close(outDone) + for { + _, ok := msgBus.SubscribeOutbound(ctx) + if !ok { + return + } + } + }() + + agentLoop := agent.NewAgentLoop(cfg, msgBus, instrumentedModel) + defer agentLoop.Stop() + + response, err := agentLoop.ProcessDirect(ctx, prompt, sessionKey) + + cancel() // Signals context done, which stops outbound drain + <-outDone + + duration := time.Since(start) + + trace := Trace{ + Output: response, + SessionKey: sessionKey, + Steps: buildSteps(instrumentedModel), + Metrics: Metrics{ + TotalDurationMs: duration.Milliseconds(), + StepCount: len(instrumentedModel.calls), + InputTokens: instrumentedModel.totalUsage.InputTokens, + OutputTokens: instrumentedModel.totalUsage.OutputTokens, + TotalTokens: instrumentedModel.totalUsage.TotalTokens, + ReasoningTokens: instrumentedModel.totalUsage.ReasoningTokens, + CacheReadTokens: instrumentedModel.totalUsage.CacheReadTokens, + }, + } + + if err != nil { + trace.Error = err.Error() + } + + for _, call := range instrumentedModel.calls { + for range call.toolCalls { + trace.Metrics.ToolCallCount++ + } + } + + return trace +} + +func buildSteps(model *instrumentedLanguageModel) []TraceStep { + var steps []TraceStep + idx := 0 + + for _, call := range model.calls { + steps = append(steps, TraceStep{ + Index: idx, + Type: "llm_call", + Duration: call.duration.Milliseconds(), + }) + idx++ + + for _, tc := range call.toolCalls { + argsRaw, _ := json.Marshal(tc.args) + steps = append(steps, TraceStep{ + Index: idx, + Type: "tool_call", + Tool: tc.name, + Args: argsRaw, + Result: truncate(tc.result, 500), + }) + idx++ + } + } + + return steps +} + +func truncate(s string, maxLen int) string { + if len(s) <= maxLen { + return s + } + return s[:maxLen] + "...[truncated]" +} + +func emitError(msg string) { + trace := Trace{Error: msg} + out, _ := json.Marshal(trace) + fmt.Println(string(out)) + os.Exit(1) +} + +type instrumentedCall struct { + duration time.Duration + toolCalls []instrumentedToolCall + usage fantasy.Usage +} + +type instrumentedToolCall struct { + name string + args map[string]interface{} + result string +} + +type instrumentedLanguageModel struct { + inner fantasy.LanguageModel + calls []instrumentedCall + totalUsage fantasy.Usage +} + +var _ fantasy.LanguageModel = (*instrumentedLanguageModel)(nil) + +func (m *instrumentedLanguageModel) Provider() string { return m.inner.Provider() } +func (m *instrumentedLanguageModel) Model() string { return m.inner.Model() } + +func (m *instrumentedLanguageModel) GenerateObject(ctx context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) { + return m.inner.GenerateObject(ctx, call) +} + +func (m *instrumentedLanguageModel) StreamObject(ctx context.Context, call fantasy.ObjectCall) (fantasy.ObjectStreamResponse, error) { + return m.inner.StreamObject(ctx, call) +} + +func (m *instrumentedLanguageModel) Generate(ctx context.Context, call fantasy.Call) (*fantasy.Response, error) { + start := time.Now() + resp, err := m.inner.Generate(ctx, call) + dur := time.Since(start) + + ic := instrumentedCall{duration: dur} + + if err == nil && resp != nil { + ic.usage = resp.Usage + m.totalUsage.InputTokens += resp.Usage.InputTokens + m.totalUsage.OutputTokens += resp.Usage.OutputTokens + m.totalUsage.TotalTokens += resp.Usage.TotalTokens + m.totalUsage.ReasoningTokens += resp.Usage.ReasoningTokens + m.totalUsage.CacheReadTokens += resp.Usage.CacheReadTokens + } + + m.calls = append(m.calls, ic) + return resp, err +} + +func (m *instrumentedLanguageModel) Stream(ctx context.Context, call fantasy.Call) (fantasy.StreamResponse, error) { + start := time.Now() + stream, err := m.inner.Stream(ctx, call) + if err != nil { + m.calls = append(m.calls, instrumentedCall{duration: time.Since(start)}) + return nil, err + } + + ic := instrumentedCall{} + + wrappedStream := func(yield func(fantasy.StreamPart) bool) { + stream(func(part fantasy.StreamPart) bool { + if part.Type == fantasy.StreamPartTypeFinish { + ic.usage = part.Usage + m.totalUsage.InputTokens += part.Usage.InputTokens + m.totalUsage.OutputTokens += part.Usage.OutputTokens + m.totalUsage.TotalTokens += part.Usage.TotalTokens + m.totalUsage.ReasoningTokens += part.Usage.ReasoningTokens + m.totalUsage.CacheReadTokens += part.Usage.CacheReadTokens + } + return yield(part) + }) + ic.duration = time.Since(start) + m.calls = append(m.calls, ic) + } + + return wrappedStream, nil +} diff --git a/eval/go_evals/eval_test.go b/eval/go_evals/eval_test.go new file mode 100644 index 000000000..da5aa28fe --- /dev/null +++ b/eval/go_evals/eval_test.go @@ -0,0 +1,141 @@ +package go_evals + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/tools" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testWorkspace(t *testing.T) string { + t.Helper() + dir := t.TempDir() + os.MkdirAll(filepath.Join(dir, "sessions"), 0755) + return dir +} + +func TestToolRegistry_AllToolsHaveSchema(t *testing.T) { + workspace := testWorkspace(t) + + cfg := config.DefaultConfig() + cfg.Tools.ProgressiveDisclosure = false + + registry := tools.NewToolRegistry() + registry.Register(tools.NewReadFileTool(workspace, false)) + registry.Register(tools.NewWriteFileTool(workspace, false)) + registry.Register(tools.NewListDirTool(workspace, false)) + registry.Register(tools.NewEditFileTool(workspace, false)) + registry.Register(tools.NewExecTool(workspace, false)) + + allTools := registry.List() + require.Greater(t, len(allTools), 0, "registry should have tools") + + for _, name := range allTools { + tool, ok := registry.Get(name) + require.True(t, ok, "tool %s should be gettable", name) + + schema := tools.ToolToSchema(tool) + assert.NotNil(t, schema, "tool %s should have schema", name) + + fn, ok := schema["function"].(map[string]interface{}) + require.True(t, ok, "tool %s schema should have function key", name) + assert.NotEmpty(t, fn["name"], "tool %s should have a name", name) + assert.NotEmpty(t, fn["description"], "tool %s should have a description", name) + } +} + +func TestToolExecution_ReadFile_NonExistent(t *testing.T) { + workspace := testWorkspace(t) + + readTool := tools.NewReadFileTool(workspace, true) + result := readTool.Execute(context.Background(), map[string]interface{}{ + "path": "nonexistent_file_12345.txt", + }) + + require.NotNil(t, result) + assert.True(t, result.IsError, "reading non-existent file should return error") +} + +func TestToolExecution_WriteAndReadFile(t *testing.T) { + workspace := testWorkspace(t) + + writeTool := tools.NewWriteFileTool(workspace, true) + writeResult := writeTool.Execute(context.Background(), map[string]interface{}{ + "path": "eval_test.txt", + "content": "hello from eval test", + }) + require.NotNil(t, writeResult) + assert.False(t, writeResult.IsError, "write should succeed: %s", writeResult.ForLLM) + + readTool := tools.NewReadFileTool(workspace, true) + readResult := readTool.Execute(context.Background(), map[string]interface{}{ + "path": "eval_test.txt", + }) + require.NotNil(t, readResult) + assert.False(t, readResult.IsError, "read should succeed") + assert.Contains(t, readResult.ForLLM, "hello from eval test") +} + +func TestToolExecution_ExecBlocking(t *testing.T) { + workspace := testWorkspace(t) + + execTool := tools.NewExecTool(workspace, false) + result := execTool.Execute(context.Background(), map[string]interface{}{ + "command": "echo picoclaw-eval-test", + }) + + require.NotNil(t, result) + assert.False(t, result.IsError, "echo should succeed") + assert.Contains(t, result.ForLLM, "picoclaw-eval-test") +} + +func TestToolExecution_ListDir(t *testing.T) { + workspace := testWorkspace(t) + + os.WriteFile(filepath.Join(workspace, "file_a.txt"), []byte("a"), 0644) + os.WriteFile(filepath.Join(workspace, "file_b.txt"), []byte("b"), 0644) + + listTool := tools.NewListDirTool(workspace, true) + result := listTool.Execute(context.Background(), map[string]interface{}{ + "path": ".", + }) + + require.NotNil(t, result) + assert.False(t, result.IsError) + assert.Contains(t, result.ForLLM, "file_a.txt") + assert.Contains(t, result.ForLLM, "file_b.txt") +} + +func TestToolRegistry_ProgressiveDisclosure(t *testing.T) { + workspace := testWorkspace(t) + + registry := tools.NewToolRegistry() + registry.Register(tools.NewReadFileTool(workspace, false)) + registry.Register(tools.NewWriteFileTool(workspace, false)) + registry.Register(tools.NewExecTool(workspace, false)) + + registry.RegisterMetaTools() + registry.SetProgressiveDisclosure(true) + + visible := registry.ListVisible() + + hasToolSearch := false + hasToolCall := false + for _, name := range visible { + if name == "tool_search" { + hasToolSearch = true + } + if name == "tool_call" { + hasToolCall = true + } + } + + assert.True(t, hasToolSearch, "tool_search should be visible in progressive mode") + assert.True(t, hasToolCall, "tool_call should be visible in progressive mode") + assert.LessOrEqual(t, len(visible), 3, "progressive mode should hide most tools") +} diff --git a/eval/promptfooconfig.yaml b/eval/promptfooconfig.yaml new file mode 100644 index 000000000..81ec8c503 --- /dev/null +++ b/eval/promptfooconfig.yaml @@ -0,0 +1,47 @@ +# PicoClaw Eval Harness - promptfoo configuration +# Run: cd eval && promptfoo eval +# Compare: cd eval && promptfoo eval --output results/latest.json && promptfoo view + +description: "PicoClaw agent end-to-end evaluation" + +providers: + - id: "exec:./bin/eval-runner" + label: "picoclaw-current" + config: + # Timeout per test case (ms) + timeout: 120000 + +# To run A/B comparison against main branch, uncomment: +# - id: "exec:./bin/eval-runner-main" +# label: "picoclaw-main" +# config: +# timeout: 120000 + +# Default assertions applied to every test case +defaultTest: + assert: + # Every response must parse as valid JSON trace + - type: javascript + value: | + try { + const trace = JSON.parse(output); + const valid = trace.hasOwnProperty('output') && trace.hasOwnProperty('metrics'); + return { pass: valid, score: valid ? 1.0 : 0.0, reason: valid ? 'valid trace JSON' : 'invalid trace structure' }; + } catch(e) { + return { pass: false, score: 0, reason: 'output is not valid JSON: ' + e.message }; + } + # Latency gate: no single test should exceed 60s + - type: javascript + value: | + const trace = JSON.parse(output); + const dur = trace.metrics.total_duration_ms; + const ok = dur < 60000; + return { pass: ok, score: ok ? 1.0 : 0.0, reason: `duration: ${dur}ms (limit: 60000ms)` }; + +# Transform prompt var into the JSON format eval-runner expects on stdin +transform: "JSON.stringify({ prompt: vars.prompt })" + +tests: "cases/*.yaml" + +# Output settings +outputPath: "results/latest.json" diff --git a/eval/scripts/compare.sh b/eval/scripts/compare.sh new file mode 100755 index 000000000..a940c090e --- /dev/null +++ b/eval/scripts/compare.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set -euo pipefail + +# compare.sh - Build both branches and run eval comparison +# Usage: ./eval/scripts/compare.sh [--repeat N] + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EVAL_DIR="$(dirname "$SCRIPT_DIR")" +PROJECT_ROOT="$(dirname "$EVAL_DIR")" +REPEAT=${1:-3} + +if [[ "${1:-}" == "--repeat" ]]; then + REPEAT="${2:-3}" +fi + +echo "=== PicoClaw Eval Comparison ===" +echo "Repeat: ${REPEAT}x per test case" +echo "" + +cd "$PROJECT_ROOT" + +# 1. Build current branch +echo "[1/4] Building current branch..." +make build 2>&1 | tail -1 +cp build/picoclaw "$EVAL_DIR/bin/eval-runner" + +# 2. Build main branch +CURRENT_BRANCH=$(git branch --show-current) +STASH_RESULT=$(git stash 2>&1) + +echo "[2/4] Building main branch..." +git checkout main 2>/dev/null +make build 2>&1 | tail -1 +cp build/picoclaw "$EVAL_DIR/bin/eval-runner-main" + +# Restore +git checkout "$CURRENT_BRANCH" 2>/dev/null +if [[ "$STASH_RESULT" != "No local changes to save" ]]; then + git stash pop 2>/dev/null || true +fi + +# 3. Update promptfoo config to enable comparison +cd "$EVAL_DIR" + +# Create a temp config with both providers +cat > promptfooconfig-compare.yaml <<'YAML' +description: "PicoClaw A/B comparison (branch vs main)" + +providers: + - id: "exec:./bin/eval-runner" + label: "branch" + config: + timeout: 120000 + - id: "exec:./bin/eval-runner-main" + label: "main" + config: + timeout: 120000 + +defaultTest: + assert: + - type: javascript + value: | + try { + const trace = JSON.parse(output); + const valid = trace.hasOwnProperty('output') && trace.hasOwnProperty('metrics'); + return { pass: valid, score: valid ? 1.0 : 0.0, reason: valid ? 'valid trace' : 'invalid trace' }; + } catch(e) { + return { pass: false, score: 0, reason: 'not JSON: ' + e.message }; + } + - type: javascript + value: | + const trace = JSON.parse(output); + const dur = trace.metrics.total_duration_ms; + const ok = dur < 60000; + return { pass: ok, score: ok ? 1.0 : 0.0, reason: `${dur}ms` }; + +transform: "JSON.stringify({ prompt: vars.prompt })" +tests: "cases/*.yaml" +outputPath: "results/comparison.json" +YAML + +# 4. Run comparison +echo "[3/4] Running eval comparison (${REPEAT}x)..." +npx promptfoo eval -c promptfooconfig-compare.yaml --repeat "$REPEAT" --no-progress-bar + +echo "" +echo "[4/4] Results saved to eval/results/comparison.json" +echo "" +echo "View results: cd eval && npx promptfoo view"