feat(eval): add promptfoo-based evaluation harness

Evaluation framework using promptfoo for LLM quality regression testing.
Includes eval runner, test cases, A/B comparison script, Go-native
component evals, and Makefile targets (eval-build, eval, eval-compare,
eval-view, eval-test).

Makefile

@@ -182,6 +182,39 @@ check: deps fmt vet test
 run: build
 	@$(BUILD_DIR)/$(BINARY_NAME) $(ARGS)
 
+# ---------------------------------------------------------------------------
+# Evaluation Harness
+# Usage: make eval-build && make eval
+#        make eval-compare       (A/B: branch vs main)
+# ---------------------------------------------------------------------------
+
+## eval-build: Build the eval runner from the current branch
+eval-build: generate
+	@echo "Building eval runner..."
+	@mkdir -p eval/bin
+	@$(GO) build $(GOFLAGS) $(LDFLAGS) -o eval/bin/eval-runner ./eval/cmd/eval-runner
+	@echo "Eval runner built: eval/bin/eval-runner"
+
+## eval: Run the eval suite against the current build
+eval: eval-build
+	@echo "Running eval suite..."
+	@cd eval && npx promptfoo eval --no-progress-bar
+	@echo "Results: eval/results/latest.json"
+	@echo "View: cd eval && npx promptfoo view"
+
+## eval-view: Open the promptfoo results viewer
+eval-view:
+	@cd eval && npx promptfoo view
+
+## eval-compare: A/B comparison of current branch vs main
+eval-compare:
+	@./eval/scripts/compare.sh --repeat 3
+
+## eval-test: Run Go-native component evals
+eval-test:
+	@echo "Running Go-native evals..."
+	@$(GO) test -v ./eval/go_evals/...
+
 ## help: Show this help message
 help:
 	@echo "picoclaw Makefile"

eval/.gitignore

@@ -0,0 +1,5 @@
+bin/
+results/
+node_modules/
+promptfooconfig-compare.yaml
+*.tmp

eval/README.md

@@ -0,0 +1,107 @@
+# PicoClaw Eval Harness
+
+End-to-end evaluation system for the PicoClaw agent runtime.
+
+## Quick Start
+
+```bash
+# Install promptfoo (one-time)
+npm install -g promptfoo
+
+# Build eval runner and run suite
+make eval
+
+# View results in browser
+make eval-view
+
+# Run Go-native component evals
+make eval-test
+
+# A/B comparison (current branch vs main)
+make eval-compare
+```
+
+## Architecture
+
+```
+eval/
+├── cmd/eval-runner/     # Go binary that wraps picoclaw for promptfoo
+├── cases/               # Golden dataset (YAML test cases)
+│   ├── tool_calling.yaml
+│   ├── token_efficiency.yaml
+│   ├── multi_step.yaml
+│   └── edge_cases.yaml
+├── go_evals/            # Go-native component tests (memory, tools)
+├── scripts/             # CI/comparison scripts
+│   └── compare.sh
+├── bin/                 # Built eval runner binaries (gitignored)
+├── results/             # Eval output JSON (gitignored)
+└── promptfooconfig.yaml # Main eval configuration
+```
+
+## How It Works
+
+1. **eval-runner** wraps picoclaw with an instrumented language model that captures
+   every LLM call, tool invocation, token count, and timing.
+
+2. **promptfoo** invokes `eval-runner` via `exec:` provider, sending prompts as JSON
+   on stdin and parsing the structured trace JSON from stdout.
+
+3. **Assertions** are JavaScript functions that inspect the trace to score:
+   - Tool selection correctness
+   - Output quality
+   - Token efficiency
+   - Latency thresholds
+   - Error handling
+
+## Trace Format
+
+The eval runner emits JSON with this structure:
+
+```json
+{
+  "output": "Agent response text",
+  "steps": [
+    {"index": 0, "type": "llm_call", "duration_ms": 1200},
+    {"index": 1, "type": "tool_call", "tool": "read_file", "args": {...}, "result": "..."}
+  ],
+  "metrics": {
+    "total_duration_ms": 3400,
+    "step_count": 2,
+    "tool_call_count": 1,
+    "input_tokens": 450,
+    "output_tokens": 120,
+    "total_tokens": 570,
+    "reasoning_tokens": 0,
+    "cache_read_tokens": 0
+  },
+  "error": null,
+  "session_key": "eval:1708300000000"
+}
+```
+
+## Adding Test Cases
+
+Create a new YAML file in `eval/cases/` following this pattern:
+
+```yaml
+- description: "what this tests"
+  vars:
+    prompt: "the prompt to send to picoclaw"
+  assert:
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        // Return { pass: bool, score: 0-1, reason: string }
+        return { pass: true, score: 1.0, reason: 'explanation' };
+```
+
+## A/B Comparison
+
+`make eval-compare` builds both your current branch and main, then runs the
+identical test suite against both. Results show a side-by-side comparison
+matrix with per-test scores.
+
+## Environment Variables
+
+- `PICOCLAW_EVAL_CONFIG` - Path to picoclaw config.json for eval (defaults to `~/.picoclaw/config.json`)

eval/cases/edge_cases.yaml

@@ -0,0 +1,38 @@
+# Edge Case Evaluation
+# Tests error handling, ambiguous inputs, and boundary conditions.
+
+- description: "missing file: graceful handling of non-existent file"
+  vars:
+    prompt: "Read the file /tmp/picoclaw_eval_nonexistent_file_abc123.txt"
+  assert:
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        if (trace.error && !trace.output) return { pass: false, score: 0, reason: 'crashed: ' + trace.error };
+        const mentionsError = trace.output.toLowerCase().includes('not found') ||
+                              trace.output.toLowerCase().includes('error') ||
+                              trace.output.toLowerCase().includes('does not exist') ||
+                              trace.output.toLowerCase().includes("doesn't exist") ||
+                              trace.output.toLowerCase().includes('no such file');
+        return { pass: mentionsError, score: mentionsError ? 1.0 : 0.0, reason: mentionsError ? 'gracefully reported error' : 'did not report file missing' };
+
+- description: "empty prompt resilience"
+  vars:
+    prompt: "   "
+  assert:
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        const noFatalError = !trace.error || trace.output.length > 0;
+        return { pass: noFatalError, score: noFatalError ? 1.0 : 0.0, reason: noFatalError ? 'handled gracefully' : 'fatal error on empty input' };
+
+- description: "ambiguous request: agent asks for clarification or makes reasonable assumption"
+  vars:
+    prompt: "Do the thing with the file."
+  assert:
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        if (trace.error && !trace.output) return { pass: false, score: 0, reason: 'crashed' };
+        const responded = trace.output.length > 10;
+        return { pass: responded, score: responded ? 1.0 : 0.0, reason: responded ? 'provided a response' : 'no meaningful response' };

eval/cases/multi_step.yaml

@@ -0,0 +1,37 @@
+# Multi-Step Task Evaluation Cases
+# Tests that the agent can complete tasks requiring multiple tool invocations.
+
+- description: "create and verify: write a file then read it back"
+  vars:
+    prompt: "Write the text 'picoclaw eval checkpoint' to a file called eval_checkpoint.txt, then read it back and confirm the contents match."
+  assert:
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        if (trace.error) return { pass: false, score: 0, reason: trace.error };
+        const tools = trace.steps.filter(s => s.type === 'tool_call').map(s => s.tool);
+        const hasWrite = tools.includes('write_file') || tools.includes('tool_call');
+        const hasRead = tools.filter(t => t === 'read_file' || t === 'tool_call').length >= 1;
+        const bothOps = hasWrite && hasRead;
+        return { pass: bothOps, score: bothOps ? 1.0 : 0.5, reason: `write=${hasWrite}, read=${hasRead}` };
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        const mentions = trace.output.toLowerCase().includes('checkpoint') || trace.output.toLowerCase().includes('match') || trace.output.toLowerCase().includes('confirm');
+        return { pass: mentions, score: mentions ? 1.0 : 0.0, reason: mentions ? 'confirmed contents' : 'did not confirm' };
+
+- description: "explore and summarize: list dir then describe contents"
+  vars:
+    prompt: "List the workspace directory, then give me a brief summary of what files are there and what the workspace is used for."
+  assert:
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        if (trace.error) return { pass: false, score: 0, reason: trace.error };
+        const hasListDir = trace.steps.some(s => s.tool === 'list_dir' || (s.tool === 'tool_call'));
+        return { pass: hasListDir, score: hasListDir ? 1.0 : 0.0, reason: hasListDir ? 'explored workspace' : 'did not explore' };
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        const hasSummary = trace.output.length > 50;
+        return { pass: hasSummary, score: hasSummary ? 1.0 : 0.0, reason: `output length: ${trace.output.length}` };

eval/cases/token_efficiency.yaml

@@ -0,0 +1,48 @@
+# Token Efficiency Evaluation Cases
+# Tests that the agent doesn't waste tokens on simple tasks.
+
+- description: "simple question: low token budget for trivial task"
+  vars:
+    prompt: "What is 2 + 2? Answer with just the number."
+  assert:
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        if (trace.error) return { pass: false, score: 0, reason: trace.error };
+        const hasAnswer = trace.output.includes('4');
+        return { pass: hasAnswer, score: hasAnswer ? 1.0 : 0.0, reason: hasAnswer ? 'correct answer' : 'wrong answer' };
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        const tokens = trace.metrics.total_tokens;
+        const efficient = tokens < 2000;
+        const score = efficient ? 1.0 : Math.max(0, 1.0 - (tokens - 2000) / 5000);
+        return { pass: efficient, score, reason: `${tokens} total tokens (threshold: 2000)` };
+
+- description: "no unnecessary tool calls for conversational query"
+  vars:
+    prompt: "Hello! How are you today?"
+  assert:
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        if (trace.error) return { pass: false, score: 0, reason: trace.error };
+        const toolCount = trace.metrics.tool_call_count;
+        const efficient = toolCount === 0;
+        return { pass: efficient, score: efficient ? 1.0 : Math.max(0, 1.0 - toolCount * 0.25), reason: `${toolCount} tool calls (expected: 0)` };
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        return { pass: trace.output.length > 0, score: 1.0, reason: 'responded' };
+
+- description: "step count stays reasonable for multi-step task"
+  vars:
+    prompt: "Create a file called test_steps.txt with the content 'step test', then read it back to confirm."
+  assert:
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        if (trace.error) return { pass: false, score: 0, reason: trace.error };
+        const steps = trace.metrics.step_count;
+        const reasonable = steps <= 5;
+        return { pass: reasonable, score: reasonable ? 1.0 : Math.max(0, 1.0 - (steps - 5) * 0.2), reason: `${steps} LLM calls (threshold: 5)` };

eval/cases/tool_calling.yaml

@@ -0,0 +1,53 @@
+# Tool Calling Evaluation Cases
+# Tests that the agent correctly selects and invokes the right tools.
+
+- description: "file read: agent reads a file when asked about its contents"
+  vars:
+    prompt: "Read the file SYSTEM_PROMPT.md in the workspace and tell me the first line."
+  assert:
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        if (trace.error) return { pass: false, score: 0, reason: trace.error };
+        const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
+        const hasRead = toolCalls.some(t => t.tool === 'read_file' || t.tool === 'tool_call');
+        return { pass: hasRead, score: hasRead ? 1.0 : 0.0, reason: hasRead ? 'correctly used file read' : 'did not read the file' };
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        return { pass: trace.output.length > 0, score: trace.output.length > 0 ? 1.0 : 0.0, reason: 'non-empty output' };
+
+- description: "file write: agent creates a new file when asked"
+  vars:
+    prompt: "Create a file called eval_test_output.txt in the workspace with the text 'hello from eval'."
+  assert:
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        if (trace.error) return { pass: false, score: 0, reason: trace.error };
+        const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
+        const hasWrite = toolCalls.some(t => t.tool === 'write_file' || (t.tool === 'tool_call' && t.args && JSON.parse(t.args).tool_name === 'write_file'));
+        return { pass: hasWrite, score: hasWrite ? 1.0 : 0.0, reason: hasWrite ? 'correctly used file write' : 'did not write the file' };
+
+- description: "shell exec: agent runs a shell command"
+  vars:
+    prompt: "Run the command 'echo picoclaw-eval-test' and tell me the output."
+  assert:
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        if (trace.error) return { pass: false, score: 0, reason: trace.error };
+        const hasExec = trace.steps.some(s => s.tool === 'exec' || (s.tool === 'tool_call' && s.args && JSON.parse(s.args).tool_name === 'exec'));
+        const outputContains = trace.output.includes('picoclaw-eval-test');
+        return { pass: hasExec && outputContains, score: (hasExec ? 0.5 : 0) + (outputContains ? 0.5 : 0), reason: `exec=${hasExec}, output_correct=${outputContains}` };
+
+- description: "list directory: agent lists workspace contents"
+  vars:
+    prompt: "List the files and directories in my workspace root."
+  assert:
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        if (trace.error) return { pass: false, score: 0, reason: trace.error };
+        const hasListDir = trace.steps.some(s => s.tool === 'list_dir' || (s.tool === 'tool_call' && s.args && JSON.parse(s.args).tool_name === 'list_dir'));
+        return { pass: hasListDir, score: hasListDir ? 1.0 : 0.0, reason: hasListDir ? 'used list_dir' : 'did not list directory' };

eval/cmd/eval-runner/main.go

@@ -0,0 +1,301 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	fantasy "charm.land/fantasy"
+	"github.com/sipeed/picoclaw/pkg/agent"
+	"github.com/sipeed/picoclaw/pkg/bus"
+	"github.com/sipeed/picoclaw/pkg/config"
+	picofantasy "github.com/sipeed/picoclaw/pkg/fantasy"
+	"github.com/sipeed/picoclaw/pkg/logger"
+)
+
+// Trace is the structured output emitted by the eval runner.
+// Promptfoo parses this JSON to evaluate assertions.
+type Trace struct {
+	Output     string      `json:"output"`
+	Steps      []TraceStep `json:"steps"`
+	Metrics    Metrics     `json:"metrics"`
+	Error      string      `json:"error,omitempty"`
+	SessionKey string      `json:"session_key"`
+}
+
+type TraceStep struct {
+	Index    int             `json:"index"`
+	Type     string          `json:"type"`
+	Tool     string          `json:"tool,omitempty"`
+	Args     json.RawMessage `json:"args,omitempty"`
+	Result   string          `json:"result,omitempty"`
+	Duration int64           `json:"duration_ms,omitempty"`
+}
+
+type Metrics struct {
+	TotalDurationMs  int64 `json:"total_duration_ms"`
+	StepCount        int   `json:"step_count"`
+	ToolCallCount    int   `json:"tool_call_count"`
+	InputTokens      int64 `json:"input_tokens"`
+	OutputTokens     int64 `json:"output_tokens"`
+	TotalTokens      int64 `json:"total_tokens"`
+	ReasoningTokens  int64 `json:"reasoning_tokens"`
+	CacheReadTokens  int64 `json:"cache_read_tokens"`
+}
+
+func main() {
+	logger.SetLevel(logger.ERROR)
+
+	prompt, err := readPrompt()
+	if err != nil {
+		emitError(fmt.Sprintf("failed to read prompt: %v", err))
+		return
+	}
+
+	if strings.TrimSpace(prompt) == "" {
+		emitError("empty prompt")
+		return
+	}
+
+	cfg, err := loadEvalConfig()
+	if err != nil {
+		emitError(fmt.Sprintf("config error: %v", err))
+		return
+	}
+
+	trace := runEval(cfg, prompt)
+	out, _ := json.Marshal(trace)
+	fmt.Println(string(out))
+}
+
+func readPrompt() (string, error) {
+	if len(os.Args) > 1 && os.Args[1] == "--prompt" && len(os.Args) > 2 {
+		return os.Args[2], nil
+	}
+
+	data, err := io.ReadAll(os.Stdin)
+	if err != nil {
+		return "", err
+	}
+
+	var promptData struct {
+		Prompt string `json:"prompt"`
+	}
+	if json.Unmarshal(data, &promptData) == nil && promptData.Prompt != "" {
+		return promptData.Prompt, nil
+	}
+
+	return strings.TrimSpace(string(data)), nil
+}
+
+func loadEvalConfig() (*config.Config, error) {
+	evalConfig := os.Getenv("PICOCLAW_EVAL_CONFIG")
+	if evalConfig != "" {
+		return config.LoadConfig(evalConfig)
+	}
+
+	home, _ := os.UserHomeDir()
+	configPath := filepath.Join(home, ".picoclaw", "config.json")
+	return config.LoadConfig(configPath)
+}
+
+func runEval(cfg *config.Config, prompt string) Trace {
+	start := time.Now()
+
+	sessionKey := fmt.Sprintf("eval:%d", start.UnixNano())
+
+	fantasyProvider, err := picofantasy.CreateProvider(cfg)
+	if err != nil {
+		return Trace{Error: fmt.Sprintf("provider error: %v", err)}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	languageModel, err := fantasyProvider.LanguageModel(ctx, picofantasy.ModelID(cfg))
+	if err != nil {
+		return Trace{Error: fmt.Sprintf("model error: %v", err)}
+	}
+
+	instrumentedModel := &instrumentedLanguageModel{
+		inner: languageModel,
+	}
+
+	msgBus := bus.NewMessageBus()
+
+	// Drain outbound messages to prevent blocking
+	outDone := make(chan struct{})
+	go func() {
+		defer close(outDone)
+		for {
+			_, ok := msgBus.SubscribeOutbound(ctx)
+			if !ok {
+				return
+			}
+		}
+	}()
+
+	agentLoop := agent.NewAgentLoop(cfg, msgBus, instrumentedModel)
+	defer agentLoop.Stop()
+
+	response, err := agentLoop.ProcessDirect(ctx, prompt, sessionKey)
+
+	cancel() // Signals context done, which stops outbound drain
+	<-outDone
+
+	duration := time.Since(start)
+
+	trace := Trace{
+		Output:     response,
+		SessionKey: sessionKey,
+		Steps:      buildSteps(instrumentedModel),
+		Metrics: Metrics{
+			TotalDurationMs: duration.Milliseconds(),
+			StepCount:       len(instrumentedModel.calls),
+			InputTokens:     instrumentedModel.totalUsage.InputTokens,
+			OutputTokens:    instrumentedModel.totalUsage.OutputTokens,
+			TotalTokens:     instrumentedModel.totalUsage.TotalTokens,
+			ReasoningTokens: instrumentedModel.totalUsage.ReasoningTokens,
+			CacheReadTokens: instrumentedModel.totalUsage.CacheReadTokens,
+		},
+	}
+
+	if err != nil {
+		trace.Error = err.Error()
+	}
+
+	for _, call := range instrumentedModel.calls {
+		for range call.toolCalls {
+			trace.Metrics.ToolCallCount++
+		}
+	}
+
+	return trace
+}
+
+func buildSteps(model *instrumentedLanguageModel) []TraceStep {
+	var steps []TraceStep
+	idx := 0
+
+	for _, call := range model.calls {
+		steps = append(steps, TraceStep{
+			Index:    idx,
+			Type:     "llm_call",
+			Duration: call.duration.Milliseconds(),
+		})
+		idx++
+
+		for _, tc := range call.toolCalls {
+			argsRaw, _ := json.Marshal(tc.args)
+			steps = append(steps, TraceStep{
+				Index:  idx,
+				Type:   "tool_call",
+				Tool:   tc.name,
+				Args:   argsRaw,
+				Result: truncate(tc.result, 500),
+			})
+			idx++
+		}
+	}
+
+	return steps
+}
+
+func truncate(s string, maxLen int) string {
+	if len(s) <= maxLen {
+		return s
+	}
+	return s[:maxLen] + "...[truncated]"
+}
+
+func emitError(msg string) {
+	trace := Trace{Error: msg}
+	out, _ := json.Marshal(trace)
+	fmt.Println(string(out))
+	os.Exit(1)
+}
+
+type instrumentedCall struct {
+	duration  time.Duration
+	toolCalls []instrumentedToolCall
+	usage     fantasy.Usage
+}
+
+type instrumentedToolCall struct {
+	name   string
+	args   map[string]interface{}
+	result string
+}
+
+type instrumentedLanguageModel struct {
+	inner      fantasy.LanguageModel
+	calls      []instrumentedCall
+	totalUsage fantasy.Usage
+}
+
+var _ fantasy.LanguageModel = (*instrumentedLanguageModel)(nil)
+
+func (m *instrumentedLanguageModel) Provider() string { return m.inner.Provider() }
+func (m *instrumentedLanguageModel) Model() string    { return m.inner.Model() }
+
+func (m *instrumentedLanguageModel) GenerateObject(ctx context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
+	return m.inner.GenerateObject(ctx, call)
+}
+
+func (m *instrumentedLanguageModel) StreamObject(ctx context.Context, call fantasy.ObjectCall) (fantasy.ObjectStreamResponse, error) {
+	return m.inner.StreamObject(ctx, call)
+}
+
+func (m *instrumentedLanguageModel) Generate(ctx context.Context, call fantasy.Call) (*fantasy.Response, error) {
+	start := time.Now()
+	resp, err := m.inner.Generate(ctx, call)
+	dur := time.Since(start)
+
+	ic := instrumentedCall{duration: dur}
+
+	if err == nil && resp != nil {
+		ic.usage = resp.Usage
+		m.totalUsage.InputTokens += resp.Usage.InputTokens
+		m.totalUsage.OutputTokens += resp.Usage.OutputTokens
+		m.totalUsage.TotalTokens += resp.Usage.TotalTokens
+		m.totalUsage.ReasoningTokens += resp.Usage.ReasoningTokens
+		m.totalUsage.CacheReadTokens += resp.Usage.CacheReadTokens
+	}
+
+	m.calls = append(m.calls, ic)
+	return resp, err
+}
+
+func (m *instrumentedLanguageModel) Stream(ctx context.Context, call fantasy.Call) (fantasy.StreamResponse, error) {
+	start := time.Now()
+	stream, err := m.inner.Stream(ctx, call)
+	if err != nil {
+		m.calls = append(m.calls, instrumentedCall{duration: time.Since(start)})
+		return nil, err
+	}
+
+	ic := instrumentedCall{}
+
+	wrappedStream := func(yield func(fantasy.StreamPart) bool) {
+		stream(func(part fantasy.StreamPart) bool {
+			if part.Type == fantasy.StreamPartTypeFinish {
+				ic.usage = part.Usage
+				m.totalUsage.InputTokens += part.Usage.InputTokens
+				m.totalUsage.OutputTokens += part.Usage.OutputTokens
+				m.totalUsage.TotalTokens += part.Usage.TotalTokens
+				m.totalUsage.ReasoningTokens += part.Usage.ReasoningTokens
+				m.totalUsage.CacheReadTokens += part.Usage.CacheReadTokens
+			}
+			return yield(part)
+		})
+		ic.duration = time.Since(start)
+		m.calls = append(m.calls, ic)
+	}
+
+	return wrappedStream, nil
+}

eval/go_evals/eval_test.go

@@ -0,0 +1,141 @@
+package go_evals
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/sipeed/picoclaw/pkg/config"
+	"github.com/sipeed/picoclaw/pkg/tools"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func testWorkspace(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+	os.MkdirAll(filepath.Join(dir, "sessions"), 0755)
+	return dir
+}
+
+func TestToolRegistry_AllToolsHaveSchema(t *testing.T) {
+	workspace := testWorkspace(t)
+
+	cfg := config.DefaultConfig()
+	cfg.Tools.ProgressiveDisclosure = false
+
+	registry := tools.NewToolRegistry()
+	registry.Register(tools.NewReadFileTool(workspace, false))
+	registry.Register(tools.NewWriteFileTool(workspace, false))
+	registry.Register(tools.NewListDirTool(workspace, false))
+	registry.Register(tools.NewEditFileTool(workspace, false))
+	registry.Register(tools.NewExecTool(workspace, false))
+
+	allTools := registry.List()
+	require.Greater(t, len(allTools), 0, "registry should have tools")
+
+	for _, name := range allTools {
+		tool, ok := registry.Get(name)
+		require.True(t, ok, "tool %s should be gettable", name)
+
+		schema := tools.ToolToSchema(tool)
+		assert.NotNil(t, schema, "tool %s should have schema", name)
+
+		fn, ok := schema["function"].(map[string]interface{})
+		require.True(t, ok, "tool %s schema should have function key", name)
+		assert.NotEmpty(t, fn["name"], "tool %s should have a name", name)
+		assert.NotEmpty(t, fn["description"], "tool %s should have a description", name)
+	}
+}
+
+func TestToolExecution_ReadFile_NonExistent(t *testing.T) {
+	workspace := testWorkspace(t)
+
+	readTool := tools.NewReadFileTool(workspace, true)
+	result := readTool.Execute(context.Background(), map[string]interface{}{
+		"path": "nonexistent_file_12345.txt",
+	})
+
+	require.NotNil(t, result)
+	assert.True(t, result.IsError, "reading non-existent file should return error")
+}
+
+func TestToolExecution_WriteAndReadFile(t *testing.T) {
+	workspace := testWorkspace(t)
+
+	writeTool := tools.NewWriteFileTool(workspace, true)
+	writeResult := writeTool.Execute(context.Background(), map[string]interface{}{
+		"path":    "eval_test.txt",
+		"content": "hello from eval test",
+	})
+	require.NotNil(t, writeResult)
+	assert.False(t, writeResult.IsError, "write should succeed: %s", writeResult.ForLLM)
+
+	readTool := tools.NewReadFileTool(workspace, true)
+	readResult := readTool.Execute(context.Background(), map[string]interface{}{
+		"path": "eval_test.txt",
+	})
+	require.NotNil(t, readResult)
+	assert.False(t, readResult.IsError, "read should succeed")
+	assert.Contains(t, readResult.ForLLM, "hello from eval test")
+}
+
+func TestToolExecution_ExecBlocking(t *testing.T) {
+	workspace := testWorkspace(t)
+
+	execTool := tools.NewExecTool(workspace, false)
+	result := execTool.Execute(context.Background(), map[string]interface{}{
+		"command": "echo picoclaw-eval-test",
+	})
+
+	require.NotNil(t, result)
+	assert.False(t, result.IsError, "echo should succeed")
+	assert.Contains(t, result.ForLLM, "picoclaw-eval-test")
+}
+
+func TestToolExecution_ListDir(t *testing.T) {
+	workspace := testWorkspace(t)
+
+	os.WriteFile(filepath.Join(workspace, "file_a.txt"), []byte("a"), 0644)
+	os.WriteFile(filepath.Join(workspace, "file_b.txt"), []byte("b"), 0644)
+
+	listTool := tools.NewListDirTool(workspace, true)
+	result := listTool.Execute(context.Background(), map[string]interface{}{
+		"path": ".",
+	})
+
+	require.NotNil(t, result)
+	assert.False(t, result.IsError)
+	assert.Contains(t, result.ForLLM, "file_a.txt")
+	assert.Contains(t, result.ForLLM, "file_b.txt")
+}
+
+func TestToolRegistry_ProgressiveDisclosure(t *testing.T) {
+	workspace := testWorkspace(t)
+
+	registry := tools.NewToolRegistry()
+	registry.Register(tools.NewReadFileTool(workspace, false))
+	registry.Register(tools.NewWriteFileTool(workspace, false))
+	registry.Register(tools.NewExecTool(workspace, false))
+
+	registry.RegisterMetaTools()
+	registry.SetProgressiveDisclosure(true)
+
+	visible := registry.ListVisible()
+
+	hasToolSearch := false
+	hasToolCall := false
+	for _, name := range visible {
+		if name == "tool_search" {
+			hasToolSearch = true
+		}
+		if name == "tool_call" {
+			hasToolCall = true
+		}
+	}
+
+	assert.True(t, hasToolSearch, "tool_search should be visible in progressive mode")
+	assert.True(t, hasToolCall, "tool_call should be visible in progressive mode")
+	assert.LessOrEqual(t, len(visible), 3, "progressive mode should hide most tools")
+}

eval/promptfooconfig.yaml

@@ -0,0 +1,47 @@
+# PicoClaw Eval Harness - promptfoo configuration
+# Run: cd eval && promptfoo eval
+# Compare: cd eval && promptfoo eval --output results/latest.json && promptfoo view
+
+description: "PicoClaw agent end-to-end evaluation"
+
+providers:
+  - id: "exec:./bin/eval-runner"
+    label: "picoclaw-current"
+    config:
+      # Timeout per test case (ms)
+      timeout: 120000
+
+# To run A/B comparison against main branch, uncomment:
+# - id: "exec:./bin/eval-runner-main"
+#   label: "picoclaw-main"
+#   config:
+#     timeout: 120000
+
+# Default assertions applied to every test case
+defaultTest:
+  assert:
+    # Every response must parse as valid JSON trace
+    - type: javascript
+      value: |
+        try {
+          const trace = JSON.parse(output);
+          const valid = trace.hasOwnProperty('output') && trace.hasOwnProperty('metrics');
+          return { pass: valid, score: valid ? 1.0 : 0.0, reason: valid ? 'valid trace JSON' : 'invalid trace structure' };
+        } catch(e) {
+          return { pass: false, score: 0, reason: 'output is not valid JSON: ' + e.message };
+        }
+    # Latency gate: no single test should exceed 60s
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        const dur = trace.metrics.total_duration_ms;
+        const ok = dur < 60000;
+        return { pass: ok, score: ok ? 1.0 : 0.0, reason: `duration: ${dur}ms (limit: 60000ms)` };
+
+# Transform prompt var into the JSON format eval-runner expects on stdin
+transform: "JSON.stringify({ prompt: vars.prompt })"
+
+tests: "cases/*.yaml"
+
+# Output settings
+outputPath: "results/latest.json"

eval/scripts/compare.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# compare.sh - Build both branches and run eval comparison
+# Usage: ./eval/scripts/compare.sh [--repeat N]
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+EVAL_DIR="$(dirname "$SCRIPT_DIR")"
+PROJECT_ROOT="$(dirname "$EVAL_DIR")"
+REPEAT=${1:-3}
+
+if [[ "${1:-}" == "--repeat" ]]; then
+  REPEAT="${2:-3}"
+fi
+
+echo "=== PicoClaw Eval Comparison ==="
+echo "Repeat: ${REPEAT}x per test case"
+echo ""
+
+cd "$PROJECT_ROOT"
+
+# 1. Build current branch
+echo "[1/4] Building current branch..."
+make build 2>&1 | tail -1
+cp build/picoclaw "$EVAL_DIR/bin/eval-runner"
+
+# 2. Build main branch
+CURRENT_BRANCH=$(git branch --show-current)
+STASH_RESULT=$(git stash 2>&1)
+
+echo "[2/4] Building main branch..."
+git checkout main 2>/dev/null
+make build 2>&1 | tail -1
+cp build/picoclaw "$EVAL_DIR/bin/eval-runner-main"
+
+# Restore
+git checkout "$CURRENT_BRANCH" 2>/dev/null
+if [[ "$STASH_RESULT" != "No local changes to save" ]]; then
+  git stash pop 2>/dev/null || true
+fi
+
+# 3. Update promptfoo config to enable comparison
+cd "$EVAL_DIR"
+
+# Create a temp config with both providers
+cat > promptfooconfig-compare.yaml <<'YAML'
+description: "PicoClaw A/B comparison (branch vs main)"
+
+providers:
+  - id: "exec:./bin/eval-runner"
+    label: "branch"
+    config:
+      timeout: 120000
+  - id: "exec:./bin/eval-runner-main"
+    label: "main"
+    config:
+      timeout: 120000
+
+defaultTest:
+  assert:
+    - type: javascript
+      value: |
+        try {
+          const trace = JSON.parse(output);
+          const valid = trace.hasOwnProperty('output') && trace.hasOwnProperty('metrics');
+          return { pass: valid, score: valid ? 1.0 : 0.0, reason: valid ? 'valid trace' : 'invalid trace' };
+        } catch(e) {
+          return { pass: false, score: 0, reason: 'not JSON: ' + e.message };
+        }
+    - type: javascript
+      value: |
+        const trace = JSON.parse(output);
+        const dur = trace.metrics.total_duration_ms;
+        const ok = dur < 60000;
+        return { pass: ok, score: ok ? 1.0 : 0.0, reason: `${dur}ms` };
+
+transform: "JSON.stringify({ prompt: vars.prompt })"
+tests: "cases/*.yaml"
+outputPath: "results/comparison.json"
+YAML
+
+# 4. Run comparison
+echo "[3/4] Running eval comparison (${REPEAT}x)..."
+npx promptfoo eval -c promptfooconfig-compare.yaml --repeat "$REPEAT" --no-progress-bar
+
+echo ""
+echo "[4/4] Results saved to eval/results/comparison.json"
+echo ""
+echo "View results: cd eval && npx promptfoo view"