feat(eval): add meta_tools.yaml and expand eval-fixtures cleanup
eval/cases/meta_tools.yaml: four new cases covering the always-on gateway tools (tool_search, tool_call): - tool_search discovers file tools by keyword - tool_call dispatches read_file and returns fixture content - tool_call dispatches exec and returns command output - multi-step write + read via tool_call indirection Makefile eval-fixtures: add cleanup for additional sandbox files created by reasoning and meta_tools cases (chain_test.txt, current_year.txt, result.txt, progressive_test.txt, os_name.txt, project/ directory). Remove wrapper script generation from eval-build (single provider now). Collapse eval + eval-matrix into a single eval target.
This commit is contained in:
ZanzyTHEbar
parent
fba23b7e22
commit
eca1aa249e
2 changed files with 95 additions and 14 deletions
23
Makefile
23
Makefile
|
|
@ -220,22 +220,11 @@ eval-build: generate
|
|||
@echo "Building eval runner..."
|
||||
@mkdir -p eval/bin
|
||||
@$(GO) build $(GOFLAGS) $(LDFLAGS) -o eval/bin/eval-runner ./eval/cmd/eval-runner
|
||||
@printf '#!/bin/sh\nSCRIPT_DIR="$$(cd "$$(dirname "$$0")" && pwd)"\nPICOCLAW_EVAL_CONFIG="$$SCRIPT_DIR/../configs/default.json" exec "$$SCRIPT_DIR/eval-runner" "$$@"\n' > eval/bin/eval-runner-default
|
||||
@printf '#!/bin/sh\nSCRIPT_DIR="$$(cd "$$(dirname "$$0")" && pwd)"\nPICOCLAW_EVAL_CONFIG="$$SCRIPT_DIR/../configs/progressive.json" exec "$$SCRIPT_DIR/eval-runner" "$$@"\n' > eval/bin/eval-runner-progressive
|
||||
@printf '#!/bin/sh\nSCRIPT_DIR="$$(cd "$$(dirname "$$0")" && pwd)"\nPICOCLAW_EVAL_CONFIG="$$SCRIPT_DIR/../configs/no-memory.json" exec "$$SCRIPT_DIR/eval-runner" "$$@"\n' > eval/bin/eval-runner-no-memory
|
||||
@chmod +x eval/bin/eval-runner-default eval/bin/eval-runner-progressive eval/bin/eval-runner-no-memory
|
||||
@echo "Eval runner built: eval/bin/eval-runner"
|
||||
|
||||
## eval: Run the eval suite (default config only) against the current build
|
||||
## eval: Run the eval suite against the current build
|
||||
eval: eval-build eval-fixtures
|
||||
@echo "Running eval suite (default config)..."
|
||||
@cd eval && npx promptfoo eval --config promptfooconfig-default.yaml --no-cache --no-progress-bar
|
||||
@echo "Results: eval/results/latest.json"
|
||||
@echo "View: cd eval && npx promptfoo view"
|
||||
|
||||
## eval-matrix: Run eval suite against all config variants (default, progressive, no-memory)
|
||||
eval-matrix: eval-build eval-fixtures
|
||||
@echo "Running eval matrix (all config variants)..."
|
||||
@echo "Running eval suite..."
|
||||
@cd eval && npx promptfoo eval --config promptfooconfig.yaml --no-cache --no-progress-bar
|
||||
@echo "Results: eval/results/latest.json"
|
||||
@echo "View: cd eval && npx promptfoo view"
|
||||
|
|
@ -246,7 +235,13 @@ eval-fixtures:
|
|||
@mkdir -p $(HOME)/.local/share/picoclaw/sandbox
|
||||
@rm -f $(HOME)/.local/share/picoclaw/sandbox/eval_test_output.txt \
|
||||
$(HOME)/.local/share/picoclaw/sandbox/test_steps.txt \
|
||||
$(HOME)/.local/share/picoclaw/sandbox/eval_checkpoint.txt
|
||||
$(HOME)/.local/share/picoclaw/sandbox/eval_checkpoint.txt \
|
||||
$(HOME)/.local/share/picoclaw/sandbox/chain_test.txt \
|
||||
$(HOME)/.local/share/picoclaw/sandbox/current_year.txt \
|
||||
$(HOME)/.local/share/picoclaw/sandbox/result.txt \
|
||||
$(HOME)/.local/share/picoclaw/sandbox/progressive_test.txt \
|
||||
$(HOME)/.local/share/picoclaw/sandbox/os_name.txt
|
||||
@rm -rf $(HOME)/.local/share/picoclaw/sandbox/project
|
||||
@printf 'picoclaw eval fixture — hello from the eval harness\nThis is line two of the fixture file.\n' > $(HOME)/.local/share/picoclaw/sandbox/eval_fixture.txt
|
||||
@cp -f eval/fixtures/sample_data.txt $(HOME)/.local/share/picoclaw/sandbox/sample_data.txt
|
||||
@mkdir -p $(HOME)/.local/share/picoclaw/skills
|
||||
|
|
|
|||
86
eval/cases/meta_tools.yaml
Normal file
86
eval/cases/meta_tools.yaml
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
# Meta Tool Evaluation Cases
|
||||
# tool_search and tool_call are always-on gateway tools.
|
||||
|
||||
- description: "meta tools: tool_search discovers file tools"
|
||||
vars:
|
||||
prompt: "Search for a tool that can read files."
|
||||
assert:
|
||||
- type: javascript
|
||||
value: |
|
||||
const trace = JSON.parse(output);
|
||||
if (trace.error) return { pass: false, score: 0, reason: trace.error };
|
||||
const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
|
||||
const hasToolSearch = toolCalls.some(t => t.tool === 'tool_search');
|
||||
return { pass: hasToolSearch, score: hasToolSearch ? 1.0 : 0.0, reason: hasToolSearch ? 'used tool_search to discover tools' : `did not use tool_search (tools: ${toolCalls.map(t=>t.tool).join(', ')})` };
|
||||
- type: javascript
|
||||
value: |
|
||||
const trace = JSON.parse(output);
|
||||
const out = (trace.output || '').toLowerCase();
|
||||
const mentionsRead = out.includes('read_file') || out.includes('read') || out.includes('file');
|
||||
return { pass: mentionsRead, score: mentionsRead ? 1.0 : 0.0, reason: mentionsRead ? 'mentioned file reading capability' : 'did not mention file tools' };
|
||||
|
||||
- description: "meta tools: tool_call dispatches read_file correctly"
|
||||
vars:
|
||||
prompt: "Read the file sample_data.txt in the workspace and tell me line 5."
|
||||
assert:
|
||||
- type: javascript
|
||||
value: |
|
||||
const trace = JSON.parse(output);
|
||||
if (trace.error) return { pass: false, score: 0, reason: trace.error };
|
||||
const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
|
||||
const hasToolCall = toolCalls.some(t => {
|
||||
if (t.tool === 'tool_call') {
|
||||
const _ta = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return _ta && _ta.tool_name === 'read_file';
|
||||
}
|
||||
return false;
|
||||
});
|
||||
return { pass: hasToolCall, score: hasToolCall ? 1.0 : 0.0, reason: hasToolCall ? 'used tool_call to dispatch read_file' : `no tool_call dispatch (tools: ${toolCalls.map(t=>t.tool).join(', ')})` };
|
||||
- type: javascript
|
||||
value: |
|
||||
const trace = JSON.parse(output);
|
||||
const out = (trace.output || '');
|
||||
const hasMarker = out.includes('picoclaw-fixture-marker-abc123') || out.includes('fixture');
|
||||
return { pass: hasMarker, score: hasMarker ? 1.0 : 0.0, reason: hasMarker ? 'returned fixture content' : 'did not return expected file content' };
|
||||
|
||||
- description: "meta tools: tool_call dispatches exec correctly"
|
||||
vars:
|
||||
prompt: "Run the command 'echo progressive-test-marker' and tell me the output."
|
||||
assert:
|
||||
- type: javascript
|
||||
value: |
|
||||
const trace = JSON.parse(output);
|
||||
if (trace.error) return { pass: false, score: 0, reason: trace.error };
|
||||
const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
|
||||
const hasToolCall = toolCalls.some(t => {
|
||||
if (t.tool === 'tool_call') {
|
||||
const _ta = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return _ta && _ta.tool_name === 'exec';
|
||||
}
|
||||
return false;
|
||||
});
|
||||
return { pass: hasToolCall, score: hasToolCall ? 1.0 : 0.0, reason: hasToolCall ? 'used tool_call for exec' : `no tool_call dispatch to exec (tools: ${toolCalls.map(t=>t.tool).join(', ')})` };
|
||||
- type: javascript
|
||||
value: |
|
||||
const trace = JSON.parse(output);
|
||||
const out = (trace.output || '');
|
||||
const hasMarker = out.includes('progressive-test-marker');
|
||||
return { pass: hasMarker, score: hasMarker ? 1.0 : 0.0, reason: hasMarker ? 'output contains marker' : 'marker not in output' };
|
||||
|
||||
- description: "meta tools: multi-step via tool_call indirection"
|
||||
vars:
|
||||
prompt: "Create a file called progressive_test.txt with 'hello progressive', then read it back to confirm."
|
||||
assert:
|
||||
- type: javascript
|
||||
value: |
|
||||
const trace = JSON.parse(output);
|
||||
if (trace.error) return { pass: false, score: 0, reason: trace.error };
|
||||
const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
|
||||
const getDispatchedName = (t) => {
|
||||
if (t.tool === 'tool_call') {
|
||||
try { const a = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return (a && a.tool_name) || 'unknown'; } catch(e) { return 'unknown'; }
|
||||
}
|
||||
return t.tool;
|
||||
};
|
||||
const dispatched = toolCalls.map(getDispatchedName);
|
||||
const hasWrite = dispatched.includes('write_file');
|
||||
const hasRead = dispatched.includes('read_file');
|
||||
return { pass: hasWrite && hasRead, score: (hasWrite ? 0.5 : 0) + (hasRead ? 0.5 : 0), reason: `write=${hasWrite}, read=${hasRead} (dispatched: ${dispatched.join(', ')})` };
|
||||
Loading…
Add table
Reference in a new issue