diff --git a/cmd/opsctl/main.go b/cmd/opsctl/main.go index 629974550..9a1725391 100644 --- a/cmd/opsctl/main.go +++ b/cmd/opsctl/main.go @@ -125,6 +125,7 @@ func defaultRunEnvKeys() []string { "DRAGONSCALE_EVAL_BASE_CONFIG", "DRAGONSCALE_EVAL_CONFIG", "DRAGONSCALE_EVAL_DEBUG", + "DRAGONSCALE_PROMPTFOO_ARGS", "DEVCONTAINER_EXEC", "PLATFORM", "ARCH", diff --git a/cmd/opsctl/main_test.go b/cmd/opsctl/main_test.go index 6fa6f89de..142b79d45 100644 --- a/cmd/opsctl/main_test.go +++ b/cmd/opsctl/main_test.go @@ -136,6 +136,13 @@ func TestRunHelpIncludesGlobalOutputFlags(t *testing.T) { require.Contains(t, out.String(), "--timeout") } +func TestDefaultRunEnvKeysIncludesPromptfooArgs(t *testing.T) { + t.Parallel() + + keys := defaultRunEnvKeys() + require.Contains(t, keys, "DRAGONSCALE_PROMPTFOO_ARGS") +} + func TestMakefilePhonyTargetsAreAvailableInOpsctl(t *testing.T) { makeTargets := parseMakefilePhonyTargets(t) out := &bytes.Buffer{} diff --git a/internal/opsctl/tasks/tasks.go b/internal/opsctl/tasks/tasks.go index 876cce13f..ae3b69864 100644 --- a/internal/opsctl/tasks/tasks.go +++ b/internal/opsctl/tasks/tasks.go @@ -302,7 +302,7 @@ func NewRegistry(_ string) []app.Task { NewCommandTask("eval-fixtures", "Prepare eval fixture workspace", evalFixturesSpecs, nil, nil), NewCommandTask("eval-view", "Open the promptfoo results viewer", evalViewSpecs, nil, nil), NewShellTask("eval-clean", "Cleanup eval artifacts", simpleScript("rm -rf eval/results eval/bin"), nil), - NewShellTask("eval-compare", "Run A/B comparison of current branch vs main", simpleScript("cd eval && DEVCONTAINER_EXEC= EVAL_NPM_CMD=$(npx --yes) ./scripts/compare.sh --repeat 3"), nil), + NewShellTask("eval-compare", "Run A/B comparison of current branch vs main", simpleScript("cd eval && DEVCONTAINER_EXEC= EVAL_NPM_CMD=\"npx --yes\" ./scripts/compare.sh --repeat 3"), nil), NewShellTask("eval-test", "Run Go-native component evals", staticGoScript("-v ./eval/go_evals/..."), nil), } return tasks @@ -332,6 +332,7 @@ func defaultEnv(c *app.Context) []string { appendIfSet("DRAGONSCALE_EVAL_BASE_CONFIG", cEnv(c, "DRAGONSCALE_EVAL_BASE_CONFIG", "")) appendIfSet("DRAGONSCALE_EVAL_CONFIG", cEnv(c, "DRAGONSCALE_EVAL_CONFIG", "")) appendIfSet("DRAGONSCALE_EVAL_DEBUG", cEnv(c, "DRAGONSCALE_EVAL_DEBUG", "")) + appendIfSet("DRAGONSCALE_PROMPTFOO_ARGS", cEnv(c, "DRAGONSCALE_PROMPTFOO_ARGS", "")) appendIfSet("VERSION", cEnv(c, "VERSION", "")) appendIfSet("FANTASY_VERSION", cEnv(c, "FANTASY_VERSION", "")) appendIfSet("NAME", cEnv(c, "NAME", "")) diff --git a/internal/opsctl/tasks/tasks_test.go b/internal/opsctl/tasks/tasks_test.go index 806f65440..8dca0cdcc 100644 --- a/internal/opsctl/tasks/tasks_test.go +++ b/internal/opsctl/tasks/tasks_test.go @@ -238,6 +238,25 @@ func TestEvalRunSpecsUsesBaseConfigWhenSetAndDebugEnabled(t *testing.T) { require.Equal(t, []string{"--yes", "promptfoo", "eval", "--config", "promptfooconfig.yaml", "--no-cache", "--no-progress-bar"}, specs[2].Args) } +func TestEvalRunSpecsUsesPromptfooArgsOverride(t *testing.T) { + t.Parallel() + + specs := evalRunSpecs(&app.Context{ + Root: t.TempDir(), + ExtraEnv: map[string]string{ + "DRAGONSCALE_PROMPTFOO_ARGS": "--no-cache --max-concurrency 1", + }, + }) + + require.NotEmpty(t, specs) + last := specs[len(specs)-1] + require.Equal(t, "npx", last.Name) + require.Equal(t, []string{ + "--yes", "promptfoo", "eval", "--config", "promptfooconfig.yaml", + "--no-cache", "--max-concurrency", "1", + }, last.Args) +} + func TestEvalCompareTaskDisablesNestedDevcontainerExecution(t *testing.T) { t.Parallel() @@ -265,7 +284,7 @@ func TestEvalCompareTaskDisablesNestedDevcontainerExecution(t *testing.T) { require.NoError(t, err) require.Len(t, fake.Calls, 1) script := strings.Join(fake.Calls[0].Args, " ") - require.Contains(t, script, "cd eval && DEVCONTAINER_EXEC= EVAL_NPM_CMD=$(npx --yes) ./scripts/compare.sh --repeat 3") + require.Contains(t, script, "cd eval && DEVCONTAINER_EXEC= EVAL_NPM_CMD=\"npx --yes\" ./scripts/compare.sh --repeat 3") joinedEnv := strings.Join(fake.Calls[0].Env, " ") require.Contains(t, joinedEnv, "DEVCONTAINER_EXEC=npx --yes @devcontainers/cli exec --workspace-folder \"$PWD\" --")