feat(eval): update promptfoo config and memory_ops case
- Updated promptfoo configuration to specify individual test cases, excluding progressive_disclosure.yaml. - Modified memory_ops.yaml to change the description of a test case to reflect the agent's response behavior with memory active.
This commit is contained in:
parent
3130540ab4
commit
215d8efcf7
2 changed files with 13 additions and 3 deletions
|
|
@ -48,8 +48,7 @@
|
|||
out.includes('no matching') || out.length > 10;
|
||||
return { pass: graceful, score: graceful ? 1.0 : 0.0, reason: graceful ? 'handled empty search gracefully' : 'no meaningful response' };
|
||||
|
||||
- description: "reduced iterations provider: agent respects max_tool_iterations limit"
|
||||
providers: ["picoclaw-no-memory"]
|
||||
- description: "agent responds to greeting with memory system active"
|
||||
vars:
|
||||
prompt: "Hello, what can you help me with today?"
|
||||
assert:
|
||||
|
|
|
|||
|
|
@ -27,6 +27,17 @@ defaultTest:
|
|||
const ok = dur < 60000;
|
||||
return { pass: ok, score: ok ? 1.0 : 0.0, reason: `duration: ${dur}ms (limit: 60000ms)` };
|
||||
|
||||
tests: "cases/*.yaml"
|
||||
# Excludes progressive_disclosure.yaml (requires picoclaw-progressive provider)
|
||||
# Run `make eval-matrix` to test all providers including progressive disclosure.
|
||||
tests:
|
||||
- "cases/tool_calling.yaml"
|
||||
- "cases/multi_step.yaml"
|
||||
- "cases/edge_cases.yaml"
|
||||
- "cases/token_efficiency.yaml"
|
||||
- "cases/memory_ops.yaml"
|
||||
- "cases/subagent.yaml"
|
||||
- "cases/skills.yaml"
|
||||
- "cases/reasoning.yaml"
|
||||
- "cases/error_recovery.yaml"
|
||||
|
||||
outputPath: "results/latest.json"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue