opencode/picoclaw
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(eval): improve eval runner with structured output

Browse source 
Refactor the eval runner for cleaner result reporting and
better error handling across evaluation scenarios.
This commit is contained in:
ZanzyTHEbar 2026-02-18 15:54:39 +00:00
parent 09678e50cf
commit 255ea1f084
1 changed files with 33 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
eval/cmd/eval-runner/main.go
View file

@ -38,14 +38,14 @@ type TraceStep struct {
} }
type Metrics struct { type Metrics struct {
TotalDurationMs int64 `json:"total_duration_ms"` TotalDurationMs int64 `json:"total_duration_ms"`
StepCount int `json:"step_count"` StepCount int `json:"step_count"`
ToolCallCount int `json:"tool_call_count"` ToolCallCount int `json:"tool_call_count"`
InputTokens int64 `json:"input_tokens"` InputTokens int64 `json:"input_tokens"`
OutputTokens int64 `json:"output_tokens"` OutputTokens int64 `json:"output_tokens"`
TotalTokens int64 `json:"total_tokens"` TotalTokens int64 `json:"total_tokens"`
ReasoningTokens int64 `json:"reasoning_tokens"` ReasoningTokens int64 `json:"reasoning_tokens"`
CacheReadTokens int64 `json:"cache_read_tokens"` CacheReadTokens int64 `json:"cache_read_tokens"`
} }
func main() { func main() {
@ -78,19 +78,35 @@ func readPrompt() (string, error) {
return os.Args[2], nil return os.Args[2], nil
} }
data, err := io.ReadAll(os.Stdin) // promptfoo exec: provider passes the prompt as the first positional argument
if err != nil { if len(os.Args) > 1 && !strings.HasPrefix(os.Args[1], "-") {
return "", err raw := os.Args[1]
var promptData struct {
Prompt string `json:"prompt"`
}
if json.Unmarshal([]byte(raw), &promptData) == nil && promptData.Prompt != "" {
return promptData.Prompt, nil
}
return strings.TrimSpace(raw), nil
} }
var promptData struct { // Fallback: read from stdin (for manual testing / piping)
Prompt string `json:"prompt"` stat, _ := os.Stdin.Stat()
} if (stat.Mode() & os.ModeCharDevice) == 0 {
if json.Unmarshal(data, &promptData) == nil && promptData.Prompt != "" { data, err := io.ReadAll(os.Stdin)
return promptData.Prompt, nil if err != nil {
return "", err
}
var promptData struct {
Prompt string `json:"prompt"`
}
if json.Unmarshal(data, &promptData) == nil && promptData.Prompt != "" {
return promptData.Prompt, nil
}
return strings.TrimSpace(string(data)), nil
} }
return strings.TrimSpace(string(data)), nil return "", fmt.Errorf("no prompt provided (use positional arg, --prompt, or pipe to stdin)")
} }
func loadEvalConfig() (*config.Config, error) { func loadEvalConfig() (*config.Config, error) {
@ -217,7 +233,6 @@ func emitError(msg string) {
trace := Trace{Error: msg} trace := Trace{Error: msg}
out, _ := json.Marshal(trace) out, _ := json.Marshal(trace)
fmt.Println(string(out)) fmt.Println(string(out))
os.Exit(1)
} }
type instrumentedCall struct { type instrumentedCall struct {