diff --git a/eval/cases/memory_ops.yaml b/eval/cases/memory_ops.yaml index ffc034416..94ef7d266 100644 --- a/eval/cases/memory_ops.yaml +++ b/eval/cases/memory_ops.yaml @@ -48,8 +48,7 @@ out.includes('no matching') || out.length > 10; return { pass: graceful, score: graceful ? 1.0 : 0.0, reason: graceful ? 'handled empty search gracefully' : 'no meaningful response' }; -- description: "reduced iterations provider: agent respects max_tool_iterations limit" - providers: ["picoclaw-no-memory"] +- description: "agent responds to greeting with memory system active" vars: prompt: "Hello, what can you help me with today?" assert: diff --git a/eval/promptfooconfig-default.yaml b/eval/promptfooconfig-default.yaml index 0b77d1103..040c6f56c 100644 --- a/eval/promptfooconfig-default.yaml +++ b/eval/promptfooconfig-default.yaml @@ -27,6 +27,17 @@ defaultTest: const ok = dur < 60000; return { pass: ok, score: ok ? 1.0 : 0.0, reason: `duration: ${dur}ms (limit: 60000ms)` }; -tests: "cases/*.yaml" +# Excludes progressive_disclosure.yaml (requires picoclaw-progressive provider) +# Run `make eval-matrix` to test all providers including progressive disclosure. +tests: + - "cases/tool_calling.yaml" + - "cases/multi_step.yaml" + - "cases/edge_cases.yaml" + - "cases/token_efficiency.yaml" + - "cases/memory_ops.yaml" + - "cases/subagent.yaml" + - "cases/skills.yaml" + - "cases/reasoning.yaml" + - "cases/error_recovery.yaml" outputPath: "results/latest.json"