From eca1aa249e282ffc090a252d2621d10ab82b628c Mon Sep 17 00:00:00 2001 From: ZanzyTHEbar Date: Fri, 20 Feb 2026 18:40:05 +0000 Subject: [PATCH] feat(eval): add meta_tools.yaml and expand eval-fixtures cleanup eval/cases/meta_tools.yaml: four new cases covering the always-on gateway tools (tool_search, tool_call): - tool_search discovers file tools by keyword - tool_call dispatches read_file and returns fixture content - tool_call dispatches exec and returns command output - multi-step write + read via tool_call indirection Makefile eval-fixtures: add cleanup for additional sandbox files created by reasoning and meta_tools cases (chain_test.txt, current_year.txt, result.txt, progressive_test.txt, os_name.txt, project/ directory). Remove wrapper script generation from eval-build (single provider now). Collapse eval + eval-matrix into a single eval target. --- Makefile | 23 ++++------ eval/cases/meta_tools.yaml | 86 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 14 deletions(-) create mode 100644 eval/cases/meta_tools.yaml diff --git a/Makefile b/Makefile index a850536e5..c21f593f6 100644 --- a/Makefile +++ b/Makefile @@ -220,22 +220,11 @@ eval-build: generate @echo "Building eval runner..." @mkdir -p eval/bin @$(GO) build $(GOFLAGS) $(LDFLAGS) -o eval/bin/eval-runner ./eval/cmd/eval-runner - @printf '#!/bin/sh\nSCRIPT_DIR="$$(cd "$$(dirname "$$0")" && pwd)"\nPICOCLAW_EVAL_CONFIG="$$SCRIPT_DIR/../configs/default.json" exec "$$SCRIPT_DIR/eval-runner" "$$@"\n' > eval/bin/eval-runner-default - @printf '#!/bin/sh\nSCRIPT_DIR="$$(cd "$$(dirname "$$0")" && pwd)"\nPICOCLAW_EVAL_CONFIG="$$SCRIPT_DIR/../configs/progressive.json" exec "$$SCRIPT_DIR/eval-runner" "$$@"\n' > eval/bin/eval-runner-progressive - @printf '#!/bin/sh\nSCRIPT_DIR="$$(cd "$$(dirname "$$0")" && pwd)"\nPICOCLAW_EVAL_CONFIG="$$SCRIPT_DIR/../configs/no-memory.json" exec "$$SCRIPT_DIR/eval-runner" "$$@"\n' > eval/bin/eval-runner-no-memory - @chmod +x eval/bin/eval-runner-default eval/bin/eval-runner-progressive eval/bin/eval-runner-no-memory @echo "Eval runner built: eval/bin/eval-runner" -## eval: Run the eval suite (default config only) against the current build +## eval: Run the eval suite against the current build eval: eval-build eval-fixtures - @echo "Running eval suite (default config)..." - @cd eval && npx promptfoo eval --config promptfooconfig-default.yaml --no-cache --no-progress-bar - @echo "Results: eval/results/latest.json" - @echo "View: cd eval && npx promptfoo view" - -## eval-matrix: Run eval suite against all config variants (default, progressive, no-memory) -eval-matrix: eval-build eval-fixtures - @echo "Running eval matrix (all config variants)..." + @echo "Running eval suite..." @cd eval && npx promptfoo eval --config promptfooconfig.yaml --no-cache --no-progress-bar @echo "Results: eval/results/latest.json" @echo "View: cd eval && npx promptfoo view" @@ -246,7 +235,13 @@ eval-fixtures: @mkdir -p $(HOME)/.local/share/picoclaw/sandbox @rm -f $(HOME)/.local/share/picoclaw/sandbox/eval_test_output.txt \ $(HOME)/.local/share/picoclaw/sandbox/test_steps.txt \ - $(HOME)/.local/share/picoclaw/sandbox/eval_checkpoint.txt + $(HOME)/.local/share/picoclaw/sandbox/eval_checkpoint.txt \ + $(HOME)/.local/share/picoclaw/sandbox/chain_test.txt \ + $(HOME)/.local/share/picoclaw/sandbox/current_year.txt \ + $(HOME)/.local/share/picoclaw/sandbox/result.txt \ + $(HOME)/.local/share/picoclaw/sandbox/progressive_test.txt \ + $(HOME)/.local/share/picoclaw/sandbox/os_name.txt + @rm -rf $(HOME)/.local/share/picoclaw/sandbox/project @printf 'picoclaw eval fixture — hello from the eval harness\nThis is line two of the fixture file.\n' > $(HOME)/.local/share/picoclaw/sandbox/eval_fixture.txt @cp -f eval/fixtures/sample_data.txt $(HOME)/.local/share/picoclaw/sandbox/sample_data.txt @mkdir -p $(HOME)/.local/share/picoclaw/skills diff --git a/eval/cases/meta_tools.yaml b/eval/cases/meta_tools.yaml new file mode 100644 index 000000000..611e31f28 --- /dev/null +++ b/eval/cases/meta_tools.yaml @@ -0,0 +1,86 @@ +# Meta Tool Evaluation Cases +# tool_search and tool_call are always-on gateway tools. + +- description: "meta tools: tool_search discovers file tools" + vars: + prompt: "Search for a tool that can read files." + assert: + - type: javascript + value: | + const trace = JSON.parse(output); + if (trace.error) return { pass: false, score: 0, reason: trace.error }; + const toolCalls = trace.steps.filter(s => s.type === 'tool_call'); + const hasToolSearch = toolCalls.some(t => t.tool === 'tool_search'); + return { pass: hasToolSearch, score: hasToolSearch ? 1.0 : 0.0, reason: hasToolSearch ? 'used tool_search to discover tools' : `did not use tool_search (tools: ${toolCalls.map(t=>t.tool).join(', ')})` }; + - type: javascript + value: | + const trace = JSON.parse(output); + const out = (trace.output || '').toLowerCase(); + const mentionsRead = out.includes('read_file') || out.includes('read') || out.includes('file'); + return { pass: mentionsRead, score: mentionsRead ? 1.0 : 0.0, reason: mentionsRead ? 'mentioned file reading capability' : 'did not mention file tools' }; + +- description: "meta tools: tool_call dispatches read_file correctly" + vars: + prompt: "Read the file sample_data.txt in the workspace and tell me line 5." + assert: + - type: javascript + value: | + const trace = JSON.parse(output); + if (trace.error) return { pass: false, score: 0, reason: trace.error }; + const toolCalls = trace.steps.filter(s => s.type === 'tool_call'); + const hasToolCall = toolCalls.some(t => { + if (t.tool === 'tool_call') { + const _ta = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return _ta && _ta.tool_name === 'read_file'; + } + return false; + }); + return { pass: hasToolCall, score: hasToolCall ? 1.0 : 0.0, reason: hasToolCall ? 'used tool_call to dispatch read_file' : `no tool_call dispatch (tools: ${toolCalls.map(t=>t.tool).join(', ')})` }; + - type: javascript + value: | + const trace = JSON.parse(output); + const out = (trace.output || ''); + const hasMarker = out.includes('picoclaw-fixture-marker-abc123') || out.includes('fixture'); + return { pass: hasMarker, score: hasMarker ? 1.0 : 0.0, reason: hasMarker ? 'returned fixture content' : 'did not return expected file content' }; + +- description: "meta tools: tool_call dispatches exec correctly" + vars: + prompt: "Run the command 'echo progressive-test-marker' and tell me the output." + assert: + - type: javascript + value: | + const trace = JSON.parse(output); + if (trace.error) return { pass: false, score: 0, reason: trace.error }; + const toolCalls = trace.steps.filter(s => s.type === 'tool_call'); + const hasToolCall = toolCalls.some(t => { + if (t.tool === 'tool_call') { + const _ta = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return _ta && _ta.tool_name === 'exec'; + } + return false; + }); + return { pass: hasToolCall, score: hasToolCall ? 1.0 : 0.0, reason: hasToolCall ? 'used tool_call for exec' : `no tool_call dispatch to exec (tools: ${toolCalls.map(t=>t.tool).join(', ')})` }; + - type: javascript + value: | + const trace = JSON.parse(output); + const out = (trace.output || ''); + const hasMarker = out.includes('progressive-test-marker'); + return { pass: hasMarker, score: hasMarker ? 1.0 : 0.0, reason: hasMarker ? 'output contains marker' : 'marker not in output' }; + +- description: "meta tools: multi-step via tool_call indirection" + vars: + prompt: "Create a file called progressive_test.txt with 'hello progressive', then read it back to confirm." + assert: + - type: javascript + value: | + const trace = JSON.parse(output); + if (trace.error) return { pass: false, score: 0, reason: trace.error }; + const toolCalls = trace.steps.filter(s => s.type === 'tool_call'); + const getDispatchedName = (t) => { + if (t.tool === 'tool_call') { + try { const a = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return (a && a.tool_name) || 'unknown'; } catch(e) { return 'unknown'; } + } + return t.tool; + }; + const dispatched = toolCalls.map(getDispatchedName); + const hasWrite = dispatched.includes('write_file'); + const hasRead = dispatched.includes('read_file'); + return { pass: hasWrite && hasRead, score: (hasWrite ? 0.5 : 0) + (hasRead ? 0.5 : 0), reason: `write=${hasWrite}, read=${hasRead} (dispatched: ${dispatched.join(', ')})` };