From 215d8efcf70aabbc58b6b1a5b09ebd26220be40b Mon Sep 17 00:00:00 2001 From: ZanzyTHEbar Date: Thu, 19 Feb 2026 18:38:27 +0000 Subject: [PATCH] feat(eval): update promptfoo config and memory_ops case - Updated promptfoo configuration to specify individual test cases, excluding progressive_disclosure.yaml. - Modified memory_ops.yaml to change the description of a test case to reflect the agent's response behavior with memory active. --- eval/cases/memory_ops.yaml | 3 +-- eval/promptfooconfig-default.yaml | 13 ++++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/eval/cases/memory_ops.yaml b/eval/cases/memory_ops.yaml index ffc034416..94ef7d266 100644 --- a/eval/cases/memory_ops.yaml +++ b/eval/cases/memory_ops.yaml @@ -48,8 +48,7 @@ out.includes('no matching') || out.length > 10; return { pass: graceful, score: graceful ? 1.0 : 0.0, reason: graceful ? 'handled empty search gracefully' : 'no meaningful response' }; -- description: "reduced iterations provider: agent respects max_tool_iterations limit" - providers: ["picoclaw-no-memory"] +- description: "agent responds to greeting with memory system active" vars: prompt: "Hello, what can you help me with today?" assert: diff --git a/eval/promptfooconfig-default.yaml b/eval/promptfooconfig-default.yaml index 0b77d1103..040c6f56c 100644 --- a/eval/promptfooconfig-default.yaml +++ b/eval/promptfooconfig-default.yaml @@ -27,6 +27,17 @@ defaultTest: const ok = dur < 60000; return { pass: ok, score: ok ? 1.0 : 0.0, reason: `duration: ${dur}ms (limit: 60000ms)` }; -tests: "cases/*.yaml" +# Excludes progressive_disclosure.yaml (requires picoclaw-progressive provider) +# Run `make eval-matrix` to test all providers including progressive disclosure. +tests: + - "cases/tool_calling.yaml" + - "cases/multi_step.yaml" + - "cases/edge_cases.yaml" + - "cases/token_efficiency.yaml" + - "cases/memory_ops.yaml" + - "cases/subagent.yaml" + - "cases/skills.yaml" + - "cases/reasoning.yaml" + - "cases/error_recovery.yaml" outputPath: "results/latest.json"