opencode/picoclaw
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore(eval): XDG fixture paths, --no-cache flag, wrapper scripts, eval-clean

Browse source 
Makefile:
- eval-fixtures: migrate sandbox/skills paths from ~/.picoclaw/workspace
  to XDG-standard ~/.local/share/picoclaw/sandbox and
  ~/.local/share/picoclaw/skills
- eval / eval-matrix: add --no-cache to prevent stale promptfoo results
- eval-build: generate eval-runner-{default,progressive,no-memory} wrapper
  shell scripts that set PICOCLAW_EVAL_CONFIG and exec the binary
- Add eval-clean target to remove eval/results and eval/bin

eval/configs/*.json: minor cleanup
eval/promptfooconfig*.yaml: update provider/config references to match
  new overlay-based config loading
This commit is contained in:
ZanzyTHEbar 2026-02-19 21:26:04 +00:00
parent 3453e37706
commit 9dfd675556
6 changed files with 37 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
Makefile
View file

@ -220,37 +220,46 @@ eval-build: generate
@echo "Building eval runner..." @echo "Building eval runner..."
@mkdir -p eval/bin @mkdir -p eval/bin
@$(GO) build $(GOFLAGS) $(LDFLAGS) -o eval/bin/eval-runner ./eval/cmd/eval-runner @$(GO) build $(GOFLAGS) $(LDFLAGS) -o eval/bin/eval-runner ./eval/cmd/eval-runner
@printf '#!/bin/sh\nSCRIPT_DIR="$$(cd "$$(dirname "$$0")" && pwd)"\nPICOCLAW_EVAL_CONFIG="$$SCRIPT_DIR/../configs/default.json" exec "$$SCRIPT_DIR/eval-runner" "$$@"\n' > eval/bin/eval-runner-default
@printf '#!/bin/sh\nSCRIPT_DIR="$$(cd "$$(dirname "$$0")" && pwd)"\nPICOCLAW_EVAL_CONFIG="$$SCRIPT_DIR/../configs/progressive.json" exec "$$SCRIPT_DIR/eval-runner" "$$@"\n' > eval/bin/eval-runner-progressive
@printf '#!/bin/sh\nSCRIPT_DIR="$$(cd "$$(dirname "$$0")" && pwd)"\nPICOCLAW_EVAL_CONFIG="$$SCRIPT_DIR/../configs/no-memory.json" exec "$$SCRIPT_DIR/eval-runner" "$$@"\n' > eval/bin/eval-runner-no-memory
@chmod +x eval/bin/eval-runner-default eval/bin/eval-runner-progressive eval/bin/eval-runner-no-memory
@echo "Eval runner built: eval/bin/eval-runner" @echo "Eval runner built: eval/bin/eval-runner"
## eval: Run the eval suite (default config only) against the current build ## eval: Run the eval suite (default config only) against the current build
eval: eval-build eval-fixtures eval: eval-build eval-fixtures
@echo "Running eval suite (default config)..." @echo "Running eval suite (default config)..."
@cd eval && npx promptfoo eval --config promptfooconfig-default.yaml --no-progress-bar @cd eval && npx promptfoo eval --config promptfooconfig-default.yaml --no-cache --no-progress-bar
@echo "Results: eval/results/latest.json" @echo "Results: eval/results/latest.json"
@echo "View: cd eval && npx promptfoo view" @echo "View: cd eval && npx promptfoo view"
## eval-matrix: Run eval suite against all config variants (default, progressive, no-memory) ## eval-matrix: Run eval suite against all config variants (default, progressive, no-memory)
eval-matrix: eval-build eval-fixtures eval-matrix: eval-build eval-fixtures
@echo "Running eval matrix (all config variants)..." @echo "Running eval matrix (all config variants)..."
@cd eval && npx promptfoo eval --config promptfooconfig.yaml --no-progress-bar @cd eval && npx promptfoo eval --config promptfooconfig.yaml --no-cache --no-progress-bar
@echo "Results: eval/results/latest.json" @echo "Results: eval/results/latest.json"
@echo "View: cd eval && npx promptfoo view" @echo "View: cd eval && npx promptfoo view"
## eval-fixtures: Reset workspace to a known state and seed fixture files for eval ## eval-fixtures: Reset workspace to a known state and seed fixture files for eval
## Uses XDG paths: ~/.local/share/picoclaw/sandbox/ and ~/.local/share/picoclaw/skills/
eval-fixtures: eval-fixtures:
@mkdir -p $(HOME)/.picoclaw/workspace @mkdir -p $(HOME)/.local/share/picoclaw/sandbox
@rm -f $(HOME)/.picoclaw/workspace/eval_test_output.txt \ @rm -f $(HOME)/.local/share/picoclaw/sandbox/eval_test_output.txt \
$(HOME)/.picoclaw/workspace/test_steps.txt \ $(HOME)/.local/share/picoclaw/sandbox/test_steps.txt \
$(HOME)/.picoclaw/workspace/eval_checkpoint.txt $(HOME)/.local/share/picoclaw/sandbox/eval_checkpoint.txt
@printf 'picoclaw eval fixture — hello from the eval harness\nThis is line two of the fixture file.\n' > $(HOME)/.picoclaw/workspace/eval_fixture.txt @printf 'picoclaw eval fixture — hello from the eval harness\nThis is line two of the fixture file.\n' > $(HOME)/.local/share/picoclaw/sandbox/eval_fixture.txt
@cp -f eval/fixtures/sample_data.txt $(HOME)/.picoclaw/workspace/sample_data.txt @cp -f eval/fixtures/sample_data.txt $(HOME)/.local/share/picoclaw/sandbox/sample_data.txt
@mkdir -p $(HOME)/.picoclaw/workspace/skills @mkdir -p $(HOME)/.local/share/picoclaw/skills
@cp -rf eval/fixtures/skills/* $(HOME)/.picoclaw/workspace/skills/ 2>/dev/null || true @cp -rf eval/fixtures/skills/* $(HOME)/.local/share/picoclaw/skills/ 2>/dev/null || true
## eval-view: Open the promptfoo results viewer ## eval-view: Open the promptfoo results viewer
eval-view: eval-view:
@cd eval && npx promptfoo view @cd eval && npx promptfoo view
eval-clean:
@rm -rf eval/results
@rm -rf eval/bin
## eval-compare: A/B comparison of current branch vs main ## eval-compare: A/B comparison of current branch vs main
eval-compare: eval-compare:
@./eval/scripts/compare.sh --repeat 3 @./eval/scripts/compare.sh --repeat 3

3
eval/configs/default.json
View file

@ -3,7 +3,8 @@
"progressive_disclosure": false "progressive_disclosure": false
}, },
"memory": { "memory": {
"enabled": true "enabled": true,
"db_path": ":memory:"
}, },
"agents": { "agents": {
"defaults": { "defaults": {

3
eval/configs/no-memory.json
View file

@ -4,7 +4,8 @@
}, },
"memory": { "memory": {
"embedding_dims": 384, "embedding_dims": 384,
"offload_threshold_tokens": 100000 "offload_threshold_tokens": 100000,
"db_path": ":memory:"
}, },
"agents": { "agents": {
"defaults": { "defaults": {

3
eval/configs/progressive.json
View file

@ -3,7 +3,8 @@
"progressive_disclosure": true "progressive_disclosure": true
}, },
"memory": { "memory": {
"enabled": true "enabled": true,
"db_path": ":memory:"
}, },
"agents": { "agents": {
"defaults": { "defaults": {

8
eval/promptfooconfig-default.yaml
View file

@ -3,11 +3,13 @@
description: "PicoClaw agent evaluation (default config)" description: "PicoClaw agent evaluation (default config)"
maxConcurrency: 1
providers: providers:
- id: "exec:./bin/eval-runner-default" - id: "exec:./bin/eval-runner-default"
label: "picoclaw-default" label: "picoclaw-default"
config: config:
timeout: 120000 timeout: 180000
defaultTest: defaultTest:
assert: assert:
@ -24,8 +26,8 @@ defaultTest:
value: | value: |
const trace = JSON.parse(output); const trace = JSON.parse(output);
const dur = trace.metrics.total_duration_ms; const dur = trace.metrics.total_duration_ms;
const ok = dur < 60000; const ok = dur < 90000;
return { pass: ok, score: ok ? 1.0 : 0.0, reason: `duration: ${dur}ms (limit: 60000ms)` }; return { pass: ok, score: ok ? 1.0 : 0.0, reason: `duration: ${dur}ms (limit: 90000ms)` };
# Excludes progressive_disclosure.yaml (requires picoclaw-progressive provider) # Excludes progressive_disclosure.yaml (requires picoclaw-progressive provider)
# Run `make eval-matrix` to test all providers including progressive disclosure. # Run `make eval-matrix` to test all providers including progressive disclosure.

12
eval/promptfooconfig.yaml
View file

@ -5,21 +5,23 @@
description: "PicoClaw agent end-to-end evaluation" description: "PicoClaw agent end-to-end evaluation"
maxConcurrency: 1
providers: providers:
- id: "exec:./bin/eval-runner-default" - id: "exec:./bin/eval-runner-default"
label: "picoclaw-default" label: "picoclaw-default"
config: config:
timeout: 120000 timeout: 180000
- id: "exec:./bin/eval-runner-progressive" - id: "exec:./bin/eval-runner-progressive"
label: "picoclaw-progressive" label: "picoclaw-progressive"
config: config:
timeout: 120000 timeout: 180000
- id: "exec:./bin/eval-runner-no-memory" - id: "exec:./bin/eval-runner-no-memory"
label: "picoclaw-no-memory" label: "picoclaw-no-memory"
config: config:
timeout: 120000 timeout: 180000
# Default assertions applied to every test case # Default assertions applied to every test case
defaultTest: defaultTest:
@ -37,8 +39,8 @@ defaultTest:
value: | value: |
const trace = JSON.parse(output); const trace = JSON.parse(output);
const dur = trace.metrics.total_duration_ms; const dur = trace.metrics.total_duration_ms;
const ok = dur < 60000; const ok = dur < 90000;
return { pass: ok, score: ok ? 1.0 : 0.0, reason: `duration: ${dur}ms (limit: 60000ms)` }; return { pass: ok, score: ok ? 1.0 : 0.0, reason: `duration: ${dur}ms (limit: 90000ms)` };
tests: "cases/*.yaml" tests: "cases/*.yaml"