feat(eval): improve eval runner with structured output
Refactor the eval runner for cleaner result reporting and better error handling across evaluation scenarios.
This commit is contained in:
parent
09678e50cf
commit
255ea1f084
1 changed files with 33 additions and 18 deletions
|
|
@ -78,19 +78,35 @@ func readPrompt() (string, error) {
|
||||||
return os.Args[2], nil
|
return os.Args[2], nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// promptfoo exec: provider passes the prompt as the first positional argument
|
||||||
|
if len(os.Args) > 1 && !strings.HasPrefix(os.Args[1], "-") {
|
||||||
|
raw := os.Args[1]
|
||||||
|
var promptData struct {
|
||||||
|
Prompt string `json:"prompt"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal([]byte(raw), &promptData) == nil && promptData.Prompt != "" {
|
||||||
|
return promptData.Prompt, nil
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(raw), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: read from stdin (for manual testing / piping)
|
||||||
|
stat, _ := os.Stdin.Stat()
|
||||||
|
if (stat.Mode() & os.ModeCharDevice) == 0 {
|
||||||
data, err := io.ReadAll(os.Stdin)
|
data, err := io.ReadAll(os.Stdin)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
var promptData struct {
|
var promptData struct {
|
||||||
Prompt string `json:"prompt"`
|
Prompt string `json:"prompt"`
|
||||||
}
|
}
|
||||||
if json.Unmarshal(data, &promptData) == nil && promptData.Prompt != "" {
|
if json.Unmarshal(data, &promptData) == nil && promptData.Prompt != "" {
|
||||||
return promptData.Prompt, nil
|
return promptData.Prompt, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return strings.TrimSpace(string(data)), nil
|
return strings.TrimSpace(string(data)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return "", fmt.Errorf("no prompt provided (use positional arg, --prompt, or pipe to stdin)")
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadEvalConfig() (*config.Config, error) {
|
func loadEvalConfig() (*config.Config, error) {
|
||||||
|
|
@ -217,7 +233,6 @@ func emitError(msg string) {
|
||||||
trace := Trace{Error: msg}
|
trace := Trace{Error: msg}
|
||||||
out, _ := json.Marshal(trace)
|
out, _ := json.Marshal(trace)
|
||||||
fmt.Println(string(out))
|
fmt.Println(string(out))
|
||||||
os.Exit(1)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type instrumentedCall struct {
|
type instrumentedCall struct {
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue