From 9dfd67555678348aa3c223aa0e6623661a52d1b4 Mon Sep 17 00:00:00 2001 From: ZanzyTHEbar Date: Thu, 19 Feb 2026 21:26:04 +0000 Subject: [PATCH] chore(eval): XDG fixture paths, --no-cache flag, wrapper scripts, eval-clean Makefile: - eval-fixtures: migrate sandbox/skills paths from ~/.picoclaw/workspace to XDG-standard ~/.local/share/picoclaw/sandbox and ~/.local/share/picoclaw/skills - eval / eval-matrix: add --no-cache to prevent stale promptfoo results - eval-build: generate eval-runner-{default,progressive,no-memory} wrapper shell scripts that set PICOCLAW_EVAL_CONFIG and exec the binary - Add eval-clean target to remove eval/results and eval/bin eval/configs/*.json: minor cleanup eval/promptfooconfig*.yaml: update provider/config references to match new overlay-based config loading --- Makefile | 29 +++++++++++++++++++---------- eval/configs/default.json | 3 ++- eval/configs/no-memory.json | 3 ++- eval/configs/progressive.json | 3 ++- eval/promptfooconfig-default.yaml | 8 +++++--- eval/promptfooconfig.yaml | 12 +++++++----- 6 files changed, 37 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 9e2ffb178..a850536e5 100644 --- a/Makefile +++ b/Makefile @@ -220,37 +220,46 @@ eval-build: generate @echo "Building eval runner..." @mkdir -p eval/bin @$(GO) build $(GOFLAGS) $(LDFLAGS) -o eval/bin/eval-runner ./eval/cmd/eval-runner + @printf '#!/bin/sh\nSCRIPT_DIR="$$(cd "$$(dirname "$$0")" && pwd)"\nPICOCLAW_EVAL_CONFIG="$$SCRIPT_DIR/../configs/default.json" exec "$$SCRIPT_DIR/eval-runner" "$$@"\n' > eval/bin/eval-runner-default + @printf '#!/bin/sh\nSCRIPT_DIR="$$(cd "$$(dirname "$$0")" && pwd)"\nPICOCLAW_EVAL_CONFIG="$$SCRIPT_DIR/../configs/progressive.json" exec "$$SCRIPT_DIR/eval-runner" "$$@"\n' > eval/bin/eval-runner-progressive + @printf '#!/bin/sh\nSCRIPT_DIR="$$(cd "$$(dirname "$$0")" && pwd)"\nPICOCLAW_EVAL_CONFIG="$$SCRIPT_DIR/../configs/no-memory.json" exec "$$SCRIPT_DIR/eval-runner" "$$@"\n' > eval/bin/eval-runner-no-memory + @chmod +x eval/bin/eval-runner-default eval/bin/eval-runner-progressive eval/bin/eval-runner-no-memory @echo "Eval runner built: eval/bin/eval-runner" ## eval: Run the eval suite (default config only) against the current build eval: eval-build eval-fixtures @echo "Running eval suite (default config)..." - @cd eval && npx promptfoo eval --config promptfooconfig-default.yaml --no-progress-bar + @cd eval && npx promptfoo eval --config promptfooconfig-default.yaml --no-cache --no-progress-bar @echo "Results: eval/results/latest.json" @echo "View: cd eval && npx promptfoo view" ## eval-matrix: Run eval suite against all config variants (default, progressive, no-memory) eval-matrix: eval-build eval-fixtures @echo "Running eval matrix (all config variants)..." - @cd eval && npx promptfoo eval --config promptfooconfig.yaml --no-progress-bar + @cd eval && npx promptfoo eval --config promptfooconfig.yaml --no-cache --no-progress-bar @echo "Results: eval/results/latest.json" @echo "View: cd eval && npx promptfoo view" ## eval-fixtures: Reset workspace to a known state and seed fixture files for eval +## Uses XDG paths: ~/.local/share/picoclaw/sandbox/ and ~/.local/share/picoclaw/skills/ eval-fixtures: - @mkdir -p $(HOME)/.picoclaw/workspace - @rm -f $(HOME)/.picoclaw/workspace/eval_test_output.txt \ - $(HOME)/.picoclaw/workspace/test_steps.txt \ - $(HOME)/.picoclaw/workspace/eval_checkpoint.txt - @printf 'picoclaw eval fixture — hello from the eval harness\nThis is line two of the fixture file.\n' > $(HOME)/.picoclaw/workspace/eval_fixture.txt - @cp -f eval/fixtures/sample_data.txt $(HOME)/.picoclaw/workspace/sample_data.txt - @mkdir -p $(HOME)/.picoclaw/workspace/skills - @cp -rf eval/fixtures/skills/* $(HOME)/.picoclaw/workspace/skills/ 2>/dev/null || true + @mkdir -p $(HOME)/.local/share/picoclaw/sandbox + @rm -f $(HOME)/.local/share/picoclaw/sandbox/eval_test_output.txt \ + $(HOME)/.local/share/picoclaw/sandbox/test_steps.txt \ + $(HOME)/.local/share/picoclaw/sandbox/eval_checkpoint.txt + @printf 'picoclaw eval fixture — hello from the eval harness\nThis is line two of the fixture file.\n' > $(HOME)/.local/share/picoclaw/sandbox/eval_fixture.txt + @cp -f eval/fixtures/sample_data.txt $(HOME)/.local/share/picoclaw/sandbox/sample_data.txt + @mkdir -p $(HOME)/.local/share/picoclaw/skills + @cp -rf eval/fixtures/skills/* $(HOME)/.local/share/picoclaw/skills/ 2>/dev/null || true ## eval-view: Open the promptfoo results viewer eval-view: @cd eval && npx promptfoo view +eval-clean: + @rm -rf eval/results + @rm -rf eval/bin + ## eval-compare: A/B comparison of current branch vs main eval-compare: @./eval/scripts/compare.sh --repeat 3 diff --git a/eval/configs/default.json b/eval/configs/default.json index 57e6c5b44..37c8c5dea 100644 --- a/eval/configs/default.json +++ b/eval/configs/default.json @@ -3,7 +3,8 @@ "progressive_disclosure": false }, "memory": { - "enabled": true + "enabled": true, + "db_path": ":memory:" }, "agents": { "defaults": { diff --git a/eval/configs/no-memory.json b/eval/configs/no-memory.json index 118d92d75..e3645f233 100644 --- a/eval/configs/no-memory.json +++ b/eval/configs/no-memory.json @@ -4,7 +4,8 @@ }, "memory": { "embedding_dims": 384, - "offload_threshold_tokens": 100000 + "offload_threshold_tokens": 100000, + "db_path": ":memory:" }, "agents": { "defaults": { diff --git a/eval/configs/progressive.json b/eval/configs/progressive.json index b4131cf6b..8d57521fb 100644 --- a/eval/configs/progressive.json +++ b/eval/configs/progressive.json @@ -3,7 +3,8 @@ "progressive_disclosure": true }, "memory": { - "enabled": true + "enabled": true, + "db_path": ":memory:" }, "agents": { "defaults": { diff --git a/eval/promptfooconfig-default.yaml b/eval/promptfooconfig-default.yaml index 040c6f56c..8cfbab68d 100644 --- a/eval/promptfooconfig-default.yaml +++ b/eval/promptfooconfig-default.yaml @@ -3,11 +3,13 @@ description: "PicoClaw agent evaluation (default config)" +maxConcurrency: 1 + providers: - id: "exec:./bin/eval-runner-default" label: "picoclaw-default" config: - timeout: 120000 + timeout: 180000 defaultTest: assert: @@ -24,8 +26,8 @@ defaultTest: value: | const trace = JSON.parse(output); const dur = trace.metrics.total_duration_ms; - const ok = dur < 60000; - return { pass: ok, score: ok ? 1.0 : 0.0, reason: `duration: ${dur}ms (limit: 60000ms)` }; + const ok = dur < 90000; + return { pass: ok, score: ok ? 1.0 : 0.0, reason: `duration: ${dur}ms (limit: 90000ms)` }; # Excludes progressive_disclosure.yaml (requires picoclaw-progressive provider) # Run `make eval-matrix` to test all providers including progressive disclosure. diff --git a/eval/promptfooconfig.yaml b/eval/promptfooconfig.yaml index c3d381d1c..8625ed386 100644 --- a/eval/promptfooconfig.yaml +++ b/eval/promptfooconfig.yaml @@ -5,21 +5,23 @@ description: "PicoClaw agent end-to-end evaluation" +maxConcurrency: 1 + providers: - id: "exec:./bin/eval-runner-default" label: "picoclaw-default" config: - timeout: 120000 + timeout: 180000 - id: "exec:./bin/eval-runner-progressive" label: "picoclaw-progressive" config: - timeout: 120000 + timeout: 180000 - id: "exec:./bin/eval-runner-no-memory" label: "picoclaw-no-memory" config: - timeout: 120000 + timeout: 180000 # Default assertions applied to every test case defaultTest: @@ -37,8 +39,8 @@ defaultTest: value: | const trace = JSON.parse(output); const dur = trace.metrics.total_duration_ms; - const ok = dur < 60000; - return { pass: ok, score: ok ? 1.0 : 0.0, reason: `duration: ${dur}ms (limit: 60000ms)` }; + const ok = dur < 90000; + return { pass: ok, score: ok ? 1.0 : 0.0, reason: `duration: ${dur}ms (limit: 90000ms)` }; tests: "cases/*.yaml"