feat(eval): update promptfoo config and memory_ops case

- Updated promptfoo configuration to specify individual test cases, excluding progressive_disclosure.yaml.
- Modified memory_ops.yaml to change the description of a test case to reflect the agent's response behavior with memory active.
This commit is contained in:
ZanzyTHEbar 2026-02-19 18:38:27 +00:00
parent 3130540ab4
commit 215d8efcf7
2 changed files with 13 additions and 3 deletions

View file

@ -48,8 +48,7 @@
out.includes('no matching') || out.length > 10;
return { pass: graceful, score: graceful ? 1.0 : 0.0, reason: graceful ? 'handled empty search gracefully' : 'no meaningful response' };
- description: "reduced iterations provider: agent respects max_tool_iterations limit"
providers: ["picoclaw-no-memory"]
- description: "agent responds to greeting with memory system active"
vars:
prompt: "Hello, what can you help me with today?"
assert:

View file

@ -27,6 +27,17 @@ defaultTest:
const ok = dur < 60000;
return { pass: ok, score: ok ? 1.0 : 0.0, reason: `duration: ${dur}ms (limit: 60000ms)` };
tests: "cases/*.yaml"
# Excludes progressive_disclosure.yaml (requires picoclaw-progressive provider)
# Run `make eval-matrix` to test all providers including progressive disclosure.
tests:
- "cases/tool_calling.yaml"
- "cases/multi_step.yaml"
- "cases/edge_cases.yaml"
- "cases/token_efficiency.yaml"
- "cases/memory_ops.yaml"
- "cases/subagent.yaml"
- "cases/skills.yaml"
- "cases/reasoning.yaml"
- "cases/error_recovery.yaml"
outputPath: "results/latest.json"