feat(eval): improve eval runner with structured output

Refactor the eval runner for cleaner result reporting and
better error handling across evaluation scenarios.
This commit is contained in:
ZanzyTHEbar 2026-02-18 15:54:39 +00:00
parent 09678e50cf
commit 255ea1f084

View file

@ -78,19 +78,35 @@ func readPrompt() (string, error) {
return os.Args[2], nil
}
// promptfoo exec: provider passes the prompt as the first positional argument
if len(os.Args) > 1 && !strings.HasPrefix(os.Args[1], "-") {
raw := os.Args[1]
var promptData struct {
Prompt string `json:"prompt"`
}
if json.Unmarshal([]byte(raw), &promptData) == nil && promptData.Prompt != "" {
return promptData.Prompt, nil
}
return strings.TrimSpace(raw), nil
}
// Fallback: read from stdin (for manual testing / piping)
stat, _ := os.Stdin.Stat()
if (stat.Mode() & os.ModeCharDevice) == 0 {
data, err := io.ReadAll(os.Stdin)
if err != nil {
return "", err
}
var promptData struct {
Prompt string `json:"prompt"`
}
if json.Unmarshal(data, &promptData) == nil && promptData.Prompt != "" {
return promptData.Prompt, nil
}
return strings.TrimSpace(string(data)), nil
}
return "", fmt.Errorf("no prompt provided (use positional arg, --prompt, or pipe to stdin)")
}
func loadEvalConfig() (*config.Config, error) {
@ -217,7 +233,6 @@ func emitError(msg string) {
trace := Trace{Error: msg}
out, _ := json.Marshal(trace)
fmt.Println(string(out))
os.Exit(1)
}
type instrumentedCall struct {