refactor(opsctl): format, tasks, main enhancements and tests

This commit is contained in:
ZanzyTHEbar 2026-02-22 22:50:01 +00:00
parent c2719ff30f
commit e5e187bad1
6 changed files with 981 additions and 168 deletions

View file

@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"io" "io"
"os" "os"
"path/filepath"
"strings" "strings"
"time" "time"
@ -70,10 +71,43 @@ func parseOutputMode(raw, json bool, mode string) (format.OutputMode, error) {
} }
} }
func resolveRootPath(root string) (string, error) {
resolved := app.EnsureRoot(root)
if strings.TrimSpace(resolved) == "" {
return "", fmt.Errorf("invalid --root: resolved empty repository path")
}
info, err := os.Stat(resolved)
if err != nil {
return "", fmt.Errorf("invalid --root %q: %w", resolved, err)
}
if !info.IsDir() {
return "", fmt.Errorf("invalid --root %q: not a directory", resolved)
}
return resolved, nil
}
func resolveWorkingDirectory(root, cwd string) (string, error) {
clean := strings.TrimSpace(cwd)
if clean == "" {
return "", nil
}
if !filepath.IsAbs(clean) {
clean = filepath.Join(root, clean)
}
info, err := os.Stat(clean)
if err != nil {
return "", fmt.Errorf("invalid --cwd %q: %w", clean, err)
}
if !info.IsDir() {
return "", fmt.Errorf("invalid --cwd %q: not a directory", clean)
}
return clean, nil
}
func newOpsApp(root string) *app.App { func newOpsApp(root string) *app.App {
rootPath := app.EnsureRoot(root) ops := app.New(runner.OSRunner{}, root)
ops := app.New(runner.OSRunner{}, rootPath) for _, t := range tasks.NewRegistry(root) {
for _, t := range tasks.NewRegistry(rootPath) {
ops.Register(t) ops.Register(t)
} }
ops.Register(tasks.NewHelpTask(ops)) ops.Register(tasks.NewHelpTask(ops))
@ -126,7 +160,10 @@ func buildOpsctlCommand(stdout, stderr io.Writer, args []string) *cobra.Command
return &exitCodeError{code: 2, err: err} return &exitCodeError{code: 2, err: err}
} }
resolvedRoot := app.EnsureRoot(rootFlag) resolvedRoot, err := resolveRootPath(rootFlag)
if err != nil {
return &exitCodeError{code: 2, err: err}
}
ops := newOpsApp(resolvedRoot) ops := newOpsApp(resolvedRoot)
if len(cmdArgs) == 0 || cmdArgs[0] == "help" || cmdArgs[0] == "--help" || cmdArgs[0] == "-h" { if len(cmdArgs) == 0 || cmdArgs[0] == "help" || cmdArgs[0] == "--help" || cmdArgs[0] == "-h" {
@ -145,9 +182,14 @@ func buildOpsctlCommand(stdout, stderr io.Writer, args []string) *cobra.Command
return &exitCodeError{code: 2, err: fmt.Errorf("unknown command: %s", taskName)} return &exitCodeError{code: 2, err: fmt.Errorf("unknown command: %s", taskName)}
} }
resolvedCwd, err := resolveWorkingDirectory(resolvedRoot, cwd)
if err != nil {
return &exitCodeError{code: 2, err: err}
}
ctx := &app.Context{ ctx := &app.Context{
Root: resolvedRoot, Root: resolvedRoot,
Cwd: cwd, Cwd: resolvedCwd,
Format: mode, Format: mode,
Quiet: quiet, Quiet: quiet,
NoColor: noColor, NoColor: noColor,
@ -177,7 +219,7 @@ func buildOpsctlCommand(stdout, stderr io.Writer, args []string) *cobra.Command
root.PersistentFlags().StringVar(&formatValue, "format", string(format.OutputText), "output mode: text, json, raw") root.PersistentFlags().StringVar(&formatValue, "format", string(format.OutputText), "output mode: text, json, raw")
root.PersistentFlags().BoolVar(&jsonMode, "json", false, "shortcut for --format json") root.PersistentFlags().BoolVar(&jsonMode, "json", false, "shortcut for --format json")
root.PersistentFlags().BoolVar(&rawMode, "raw", false, "shortcut for --format raw") root.PersistentFlags().BoolVar(&rawMode, "raw", false, "shortcut for --format raw")
root.PersistentFlags().BoolVar(&noColor, "no-color", false, "disable color output (reserved)") root.PersistentFlags().BoolVar(&noColor, "no-color", false, "disable color output")
root.PersistentFlags().BoolVar(&quiet, "quiet", false, "suppress non-error output") root.PersistentFlags().BoolVar(&quiet, "quiet", false, "suppress non-error output")
root.PersistentFlags().StringVar(&rootFlag, "root", "", "repository root") root.PersistentFlags().StringVar(&rootFlag, "root", "", "repository root")
root.PersistentFlags().StringVar(&cwd, "cwd", "", "working directory for command execution") root.PersistentFlags().StringVar(&cwd, "cwd", "", "working directory for command execution")
@ -188,7 +230,11 @@ func buildOpsctlCommand(stdout, stderr io.Writer, args []string) *cobra.Command
_, _ = fmt.Fprintln(cobraCmd.ErrOrStderr(), err) _, _ = fmt.Fprintln(cobraCmd.ErrOrStderr(), err)
return return
} }
rootPath := app.EnsureRoot(rootFlag) rootPath, err := resolveRootPath(rootFlag)
if err != nil {
_, _ = fmt.Fprintln(cobraCmd.ErrOrStderr(), err)
return
}
printUsage(cobraCmd.OutOrStdout(), newOpsApp(rootPath), cobraCmd.Root().Name()) printUsage(cobraCmd.OutOrStdout(), newOpsApp(rootPath), cobraCmd.Root().Name())
}) })

View file

@ -3,10 +3,12 @@ package main
import ( import (
"bytes" "bytes"
"os" "os"
"os/exec"
"path/filepath" "path/filepath"
"strings" "strings"
"testing" "testing"
"github.com/ZanzyTHEbar/dragonscale/internal/opsctl/format"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -19,6 +21,10 @@ func TestParseOutputMode(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, "raw", string(mode)) require.Equal(t, "raw", string(mode))
mode, err = parseOutputMode(true, true, "text")
require.NoError(t, err)
require.Equal(t, "raw", string(mode))
mode, err = parseOutputMode(false, true, "text") mode, err = parseOutputMode(false, true, "text")
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, "json", string(mode)) require.Equal(t, "json", string(mode))
@ -27,6 +33,26 @@ func TestParseOutputMode(t *testing.T) {
require.Error(t, err) require.Error(t, err)
} }
func TestParseOutputModePriority(t *testing.T) {
t.Parallel()
mode, err := parseOutputMode(true, true, "json")
require.NoError(t, err)
require.Equal(t, format.OutputRaw, mode)
mode, err = parseOutputMode(false, true, "text")
require.NoError(t, err)
require.Equal(t, format.OutputJSON, mode)
mode, err = parseOutputMode(true, false, "json")
require.NoError(t, err)
require.Equal(t, format.OutputRaw, mode)
mode, err = parseOutputMode(false, false, "json")
require.NoError(t, err)
require.Equal(t, format.OutputJSON, mode)
}
func TestRunPrintsUsageWithoutArgs(t *testing.T) { func TestRunPrintsUsageWithoutArgs(t *testing.T) {
out := &bytes.Buffer{} out := &bytes.Buffer{}
errOut := &bytes.Buffer{} errOut := &bytes.Buffer{}
@ -54,6 +80,27 @@ func TestRunInvalidFormat(t *testing.T) {
require.Contains(t, errOut.String(), "invalid --format value \"bad\"") require.Contains(t, errOut.String(), "invalid --format value \"bad\"")
} }
func TestRunInvalidRoot(t *testing.T) {
out := &bytes.Buffer{}
errOut := &bytes.Buffer{}
root := filepath.Join(t.TempDir(), "missing")
code := run([]string{"--root", root, "help"}, out, errOut)
require.Equal(t, 2, code)
require.Contains(t, errOut.String(), "invalid --root")
require.Equal(t, "", out.String())
}
func TestRunInvalidCwd(t *testing.T) {
out := &bytes.Buffer{}
errOut := &bytes.Buffer{}
root := t.TempDir()
missing := filepath.Join(root, "does-not-exist")
code := run([]string{"--root", root, "--cwd", missing, "build"}, out, errOut)
require.Equal(t, 2, code)
require.Contains(t, errOut.String(), "invalid --cwd")
require.Equal(t, "", out.String())
}
func TestRunHelpAlias(t *testing.T) { func TestRunHelpAlias(t *testing.T) {
out := &bytes.Buffer{} out := &bytes.Buffer{}
errOut := &bytes.Buffer{} errOut := &bytes.Buffer{}
@ -73,6 +120,22 @@ func TestRunHelpIncludesAllAlias(t *testing.T) {
require.Contains(t, out.String(), "build") require.Contains(t, out.String(), "build")
} }
func TestRunHelpIncludesGlobalOutputFlags(t *testing.T) {
out := &bytes.Buffer{}
errOut := &bytes.Buffer{}
code := run([]string{"--help"}, out, errOut)
require.Equal(t, 0, code)
require.Equal(t, "", errOut.String())
require.Contains(t, out.String(), "--format")
require.Contains(t, out.String(), "--json")
require.Contains(t, out.String(), "--raw")
require.Contains(t, out.String(), "--no-color")
require.Contains(t, out.String(), "--quiet")
require.Contains(t, out.String(), "--root")
require.Contains(t, out.String(), "--cwd")
require.Contains(t, out.String(), "--timeout")
}
func TestMakefilePhonyTargetsAreAvailableInOpsctl(t *testing.T) { func TestMakefilePhonyTargetsAreAvailableInOpsctl(t *testing.T) {
makeTargets := parseMakefilePhonyTargets(t) makeTargets := parseMakefilePhonyTargets(t)
out := &bytes.Buffer{} out := &bytes.Buffer{}
@ -87,6 +150,25 @@ func TestMakefilePhonyTargetsAreAvailableInOpsctl(t *testing.T) {
} }
} }
func TestMakefileAllTargetIsAliasToBuild(t *testing.T) {
makefile, err := findProjectMakefile()
require.NoError(t, err)
require.Equal(t, []string{"build"}, parseMakeTargetPrereqs(t, makefile, "all"))
}
func TestMakeAllTargetRunsOpsctlBuild(t *testing.T) {
output := runMakeTargetWithFakeOpsctl(t, "all", nil)
require.Contains(t, output, "cmd:build")
}
func TestMakeEvalBuildForwardsEnvToOpsctl(t *testing.T) {
output := runMakeTargetWithFakeOpsctl(t, "eval-build", map[string]string{
"DRAGONSCALE_EVAL_CONFIG": "/tmp/eval/config.json",
})
require.Contains(t, output, "cmd:--no-color --format raw eval-build")
require.Contains(t, output, "eval-config:/tmp/eval/config.json")
}
func parseMakefilePhonyTargets(t *testing.T) []string { func parseMakefilePhonyTargets(t *testing.T) []string {
t.Helper() t.Helper()
@ -141,6 +223,70 @@ func parseMakefilePhonyTargets(t *testing.T) []string {
return targets return targets
} }
func parseMakeTargetPrereqs(t *testing.T, makefile, target string) []string {
t.Helper()
raw, err := os.ReadFile(makefile)
require.NoError(t, err)
lines := strings.Split(string(raw), "\n")
prefix := target + ":"
for _, line := range lines {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "#") || trimmed == "" {
continue
}
if strings.HasPrefix(trimmed, prefix) {
after := strings.TrimSpace(strings.TrimPrefix(trimmed, prefix))
if idx := strings.Index(after, "#"); idx != -1 {
after = strings.TrimSpace(after[:idx])
}
if after == "" {
return nil
}
return strings.Fields(after)
}
}
t.Fatalf("make target %q not found in %s", target, makefile)
return nil
}
func runMakeTargetWithFakeOpsctl(t *testing.T, target string, env map[string]string) string {
t.Helper()
makefile, err := findProjectMakefile()
require.NoError(t, err)
makeRoot := filepath.Dir(makefile)
fakeOpsctl := filepath.Join(t.TempDir(), "opsctl.sh")
fakeScript := []byte(`#!/usr/bin/env sh
printf 'cmd:%s\n' "$*"
printf 'eval-config:%s\n' "${DRAGONSCALE_EVAL_CONFIG-}"
`)
require.NoError(t, os.WriteFile(fakeOpsctl, fakeScript, 0o755))
dummyOpsctl := filepath.Join(t.TempDir(), "opsctl")
require.NoError(t, os.WriteFile(dummyOpsctl, []byte("#!/usr/bin/env sh\nexit 0\n"), 0o755))
cmd := exec.Command("make", "-s", "-f", makefile, target)
cmd.Dir = makeRoot
runEnv := append(
os.Environ(),
"OPSCTL="+fakeOpsctl,
"OPSCTL_BIN="+dummyOpsctl,
"OPSCTL_EVAL="+fakeOpsctl,
"OPSCTL_EVAL_ARGS=--no-color --format raw",
)
for key, value := range env {
runEnv = append(runEnv, key+"="+value)
}
cmd.Env = runEnv
output, err := cmd.CombinedOutput()
require.NoError(t, err, "make %s failed: %s", target, string(output))
return string(output)
}
func findProjectMakefile() (string, error) { func findProjectMakefile() (string, error) {
dir, err := os.Getwd() dir, err := os.Getwd()
if err != nil { if err != nil {

View file

@ -1,6 +1,7 @@
package format package format
import ( import (
"bytes"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
@ -55,41 +56,103 @@ func NewTaskResult(name string, cmd string, result runner.CommandResult, started
func Render(w io.Writer, mode OutputMode, res TaskResult, quiet bool) error { func Render(w io.Writer, mode OutputMode, res TaskResult, quiet bool) error {
res.Mode = mode res.Mode = mode
if quiet && res.Success {
return nil
}
switch mode { switch mode {
case OutputJSON: case OutputJSON:
if quiet {
return nil
}
enc := json.NewEncoder(w) enc := json.NewEncoder(w)
enc.SetIndent("", " ") enc.SetIndent("", " ")
return enc.Encode(res) return enc.Encode(res)
case OutputRaw: case OutputRaw:
if quiet {
return nil
}
prefix := statusPrefix(!res.NoColor, res.Success) prefix := statusPrefix(!res.NoColor, res.Success)
_, _ = fmt.Fprintf(w, "%s %s (%dms)\n", prefix, res.Task, res.DurationMS) _, _ = fmt.Fprintf(w, "%s %s (%dms)\n", prefix, res.Task, res.DurationMS)
return nil return nil
default: default:
if quiet {
return nil
}
status := "ok" status := "ok"
if !res.Success { if !res.Success {
status = "failed" status = "failed"
} }
status = formatStatus(!res.NoColor, res.Success) status = formatStatus(!res.NoColor, res.Success)
_, _ = fmt.Fprintf(w, "%s: %s (exit=%d, duration=%dms)\n", res.Task, status, res.ExitCode, res.DurationMS) _, _ = fmt.Fprintf(w, "%s: %s (exit=%d, duration=%dms)\n", res.Task, status, res.ExitCode, res.DurationMS)
if strings.TrimSpace(res.Command) != "" {
_, _ = fmt.Fprintf(w, " command: %s\n", normalizeCommand(res.Command))
}
if res.Error != "" { if res.Error != "" {
_, _ = fmt.Fprintf(w, " error: %s\n", res.Error) _, _ = fmt.Fprintf(w, " error: %s\n", res.Error)
} }
if strings.TrimSpace(res.Stdout) != "" { if strings.TrimSpace(res.Stdout) != "" {
_, _ = fmt.Fprintf(w, " stdout: %s\n", res.Stdout) _, _ = fmt.Fprintf(w, " stdout: %s\n", normalizeStdout(res.Stdout))
} }
if strings.TrimSpace(res.Stderr) != "" { if strings.TrimSpace(res.Stderr) != "" {
_, _ = fmt.Fprintf(w, " stderr: %s\n", res.Stderr) _, _ = fmt.Fprintf(w, " stderr: %s\n", normalizeStdout(res.Stderr))
} }
return nil return nil
} }
} }
func normalizeCommand(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return raw
}
lines := strings.Split(raw, "\n")
compact := make([]string, 0, len(lines))
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
continue
}
compact = append(compact, line)
}
return strings.Join(compact, " ")
}
func normalizeStdout(raw string) string {
return normalizeOutput(raw, true)
}
func normalizeOutput(raw string, includeStructured bool) string {
raw = strings.TrimRight(raw, "\n")
if raw == "" {
return raw
}
lines := strings.Split(raw, "\n")
normalized := make([]string, 0, len(lines))
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
continue
}
normalized = append(normalized, normalizeLine(line, includeStructured))
}
return strings.Join(normalized, "\n")
}
func normalizeLine(line string, includeStructured bool) string {
if !includeStructured {
return line
}
var payload any
if err := json.Unmarshal([]byte(line), &payload); err != nil {
return line
}
return normalizeJSONLine(payload)
}
func normalizeJSONLine(payload any) string {
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
enc.SetIndent("", " ")
if err := enc.Encode(payload); err != nil {
return fmt.Sprint(payload)
}
return strings.TrimRight(buf.String(), "\n")
}
func formatStatus(colored bool, success bool) string { func formatStatus(colored bool, success bool) string {
if !colored { if !colored {
if !success { if !success {

View file

@ -48,6 +48,21 @@ func TestRenderTextIncludesDetails(t *testing.T) {
require.NotContains(t, output, "\x1b[") require.NotContains(t, output, "\x1b[")
} }
func TestRenderTextRespectsNoColorFalse(t *testing.T) {
buf := &bytes.Buffer{}
result := TaskResult{
Task: "test",
ExitCode: 0,
DurationMS: 10,
Success: true,
NoColor: false,
}
require.NoError(t, Render(buf, OutputText, result, false))
output := buf.String()
require.Contains(t, output, "\x1b[32mok\x1b[0m")
}
func TestRenderRawHonorQuiet(t *testing.T) { func TestRenderRawHonorQuiet(t *testing.T) {
buf := &bytes.Buffer{} buf := &bytes.Buffer{}
result := TaskResult{Task: "run", ExitCode: 0, DurationMS: 42, Success: true} result := TaskResult{Task: "run", ExitCode: 0, DurationMS: 42, Success: true}
@ -59,3 +74,39 @@ func TestRenderRawHonorQuiet(t *testing.T) {
require.True(t, strings.Contains(buf.String(), "run")) require.True(t, strings.Contains(buf.String(), "run"))
require.True(t, strings.Contains(buf.String(), "(42ms)")) require.True(t, strings.Contains(buf.String(), "(42ms)"))
} }
func TestRenderTextIncludesCommandSummaryAndErrors(t *testing.T) {
buf := &bytes.Buffer{}
result := TaskResult{
Task: "eval",
Command: "if [ -n \"$X\" ]; then\necho ok\nfi",
ExitCode: 1,
Success: false,
DurationMS: 55,
Error: "command failed",
Stdout: "ok\n",
Stderr: `{"outcome":"success"}`,
}
require.NoError(t, Render(buf, OutputText, result, false))
output := buf.String()
require.Contains(t, output, "eval")
require.Contains(t, output, "command:")
require.Contains(t, output, "if [ -n")
require.Contains(t, output, "error: command failed")
require.Contains(t, output, "stdout: ok")
require.Contains(t, output, `"outcome"`)
}
func TestRenderTextHonorsQuietOnlyForSuccess(t *testing.T) {
buf := &bytes.Buffer{}
successResult := TaskResult{Task: "run", ExitCode: 0, DurationMS: 1, Success: true}
require.NoError(t, Render(buf, OutputText, successResult, true))
require.Equal(t, "", buf.String())
buf.Reset()
failureResult := TaskResult{Task: "run", ExitCode: 1, DurationMS: 1, Success: false, Error: "boom"}
require.NoError(t, Render(buf, OutputText, failureResult, true))
require.Contains(t, buf.String(), "run")
require.Contains(t, buf.String(), "exit=1")
require.Contains(t, buf.String(), "error: boom")
}

View file

@ -6,6 +6,7 @@ import (
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"runtime"
"strconv" "strconv"
"strings" "strings"
@ -25,6 +26,7 @@ type shellTask struct {
description string description string
script func(*app.Context) string script func(*app.Context) string
env func(*app.Context) []string env func(*app.Context) []string
prepare func(*app.Context) error
} }
func (t shellTask) Name() string { func (t shellTask) Name() string {
@ -36,6 +38,17 @@ func (t shellTask) Description() string {
} }
func (t shellTask) Run(ctx context.Context, rnr runner.Runner, c *app.Context) (app.Result, error) { func (t shellTask) Run(ctx context.Context, rnr runner.Runner, c *app.Context) (app.Result, error) {
if t.prepare != nil {
if err := t.prepare(c); err != nil {
return app.Result{
Task: t.name,
Command: t.script(c),
ExitCode: 1,
Err: err.Error(),
}, err
}
}
script := t.script(c) script := t.script(c)
fullScript := applyDevcontainerWrapper(script, c) fullScript := applyDevcontainerWrapper(script, c)
workDir := c.Root workDir := c.Root
@ -54,7 +67,7 @@ func (t shellTask) Run(ctx context.Context, rnr runner.Runner, c *app.Context) (
result, err := rnr.Run(ctx, spec) result, err := rnr.Run(ctx, spec)
res := app.Result{ res := app.Result{
Task: t.name, Task: t.name,
Command: script, Command: fullScript,
Stdout: result.Stdout, Stdout: result.Stdout,
Stderr: result.Stderr, Stderr: result.Stderr,
Err: func() string { Err: func() string {
@ -71,6 +84,117 @@ func (t shellTask) Run(ctx context.Context, rnr runner.Runner, c *app.Context) (
return res, err return res, err
} }
type commandTask struct {
name string
description string
specs func(*app.Context) []runner.CommandSpec
env func(*app.Context) []string
prepare func(*app.Context) error
}
func (t commandTask) Name() string {
return t.name
}
func (t commandTask) Description() string {
return t.description
}
func (t commandTask) Run(ctx context.Context, rnr runner.Runner, c *app.Context) (app.Result, error) {
if t.prepare != nil {
if err := t.prepare(c); err != nil {
return app.Result{
Task: t.name,
ExitCode: 1,
Err: err.Error(),
}, err
}
}
specs := t.specs(c)
workDir := c.Root
if c.Cwd != "" {
workDir = c.Cwd
}
baseEnv := mergeEnv(c, t.env)
result := app.Result{
Task: t.name,
}
commandParts := make([]string, 0, len(specs))
for i, spec := range specs {
spec.InheritOutput = c.Format == format.OutputRaw
if spec.Dir == "" {
spec.Dir = workDir
}
spec.Env = mergeEnvPairs(baseEnv, spec.Env)
commandParts = append(commandParts, describeCommandSpec(spec))
commandResult, err := rnr.Run(ctx, spec)
result.ExitCode = commandResult.ExitCode
result.Stdout += commandResult.Stdout
result.Stderr += commandResult.Stderr
if i == 0 {
result.Command = commandParts[i]
} else {
result.Command += " && " + commandParts[i]
}
if err != nil {
result.Err = err.Error()
return result, err
}
}
return result, nil
}
func describeCommandSpec(spec runner.CommandSpec) string {
if len(spec.Args) == 0 {
return spec.Name
}
quotedArgs := make([]string, 0, len(spec.Args))
for _, arg := range spec.Args {
quotedArgs = append(quotedArgs, strconv.Quote(arg))
}
return spec.Name + " " + strings.Join(quotedArgs, " ")
}
func mergeEnvPairs(base []string, extra []string) []string {
seen := make(map[string]int)
values := make(map[string]string)
keys := make([]string, 0, len(base)+len(extra))
merge := func(items []string) {
for _, item := range items {
eq := strings.Index(item, "=")
if eq <= 0 {
continue
}
key := item[:eq]
value := item[eq+1:]
if _, ok := seen[key]; !ok {
keys = append(keys, key)
seen[key] = len(keys) - 1
}
values[key] = value
}
}
merge(base)
merge(extra)
env := make([]string, 0, len(keys))
for _, key := range keys {
value := values[key]
if strings.TrimSpace(value) == "" {
continue
}
env = append(env, fmt.Sprintf("%s=%s", key, value))
}
return env
}
type helpTask struct { type helpTask struct {
app *app.App app *app.App
} }
@ -134,12 +258,20 @@ func NewShellTask(name, description string, script func(*app.Context) string, en
return shellTask{name: name, description: description, script: script, env: env} return shellTask{name: name, description: description, script: script, env: env}
} }
func NewCommandTask(name, description string, specs func(*app.Context) []runner.CommandSpec, env func(*app.Context) []string, prepare func(*app.Context) error) commandTask {
return commandTask{name: name, description: description, specs: specs, env: env, prepare: prepare}
}
func NewShellTaskWithPrepare(name, description string, script func(*app.Context) string, env func(*app.Context) []string, prepare func(*app.Context) error) shellTask {
return shellTask{name: name, description: description, script: script, env: env, prepare: prepare}
}
func NewRegistry(_ string) []app.Task { func NewRegistry(_ string) []app.Task {
tasks := []app.Task{ tasks := []app.Task{
NewShellTask("all", "Build the dragonscale binary for current platform", buildScript, nil), NewShellTask("all", "Build the dragonscale binary for current platform", buildScript, nil),
NewShellTask("generate", "Run generate", generateScript, nil), NewShellTask("generate", "Run generate", generateScript, nil),
NewShellTask("build", "Build the dragonscale binary for current platform", buildScript, nil), NewShellTaskWithPrepare("build", "Build the dragonscale binary for current platform", buildScript, nil, validateBuildTaskEnv),
NewShellTask("build-all", "Build dragonscale for all platforms", buildAllScript, nil), NewShellTaskWithPrepare("build-all", "Build dragonscale for linux target with CGO", buildAllScript, nil, validateBuildTaskEnv),
NewShellTask("install", "Install dragonscale to system and copy builtin skills", installScript, nil), NewShellTask("install", "Install dragonscale to system and copy builtin skills", installScript, nil),
NewShellTask("uninstall", "Uninstall dragonscale from system", uninstallScript, nil), NewShellTask("uninstall", "Uninstall dragonscale from system", uninstallScript, nil),
NewShellTask("uninstall-all", "Uninstall dragonscale and workspace data", uninstallAllScript, nil), NewShellTask("uninstall-all", "Uninstall dragonscale and workspace data", uninstallAllScript, nil),
@ -147,7 +279,7 @@ func NewRegistry(_ string) []app.Task {
NewShellTask("vet", "Run go vet for static analysis", staticGoScript("vet ./..."), nil), NewShellTask("vet", "Run go vet for static analysis", staticGoScript("vet ./..."), nil),
NewShellTask("test", "Run tests", staticGoScript("test ./..."), nil), NewShellTask("test", "Run tests", staticGoScript("test ./..."), nil),
NewShellTask("fmt", "Format Go code", staticGoScript("fmt ./..."), nil), NewShellTask("fmt", "Format Go code", staticGoScript("fmt ./..."), nil),
NewShellTask("lint", "Run all linting checks", lintScript, nil), NewCommandTask("lint", "Run all linting checks", lintSpecs, nil, nil),
NewShellTask("hooks", "Install git hooks", hooksScript, nil), NewShellTask("hooks", "Install git hooks", hooksScript, nil),
NewShellTask("deps", "Download dependencies", staticGoScript("mod download && mod verify"), nil), NewShellTask("deps", "Download dependencies", staticGoScript("mod download && mod verify"), nil),
NewShellTask("update-deps", "Update dependencies", staticGoScript("get -u ./... && mod tidy"), nil), NewShellTask("update-deps", "Update dependencies", staticGoScript("get -u ./... && mod tidy"), nil),
@ -159,16 +291,16 @@ func NewRegistry(_ string) []app.Task {
NewShellTask("fantasy-sync", "Full sync of vendored Fantasy SDK", fantasySyncScript, nil), NewShellTask("fantasy-sync", "Full sync of vendored Fantasy SDK", fantasySyncScript, nil),
NewShellTask("fantasy-patch", "Save local modifications as a patch", fantasyPatchScript, nil), NewShellTask("fantasy-patch", "Save local modifications as a patch", fantasyPatchScript, nil),
NewShellTask("test-integration", "Run integration tests", staticGoScript("-tags integration -count=1 -timeout 120s -v ./pkg/memory/..."), nil), NewShellTask("test-integration", "Run integration tests", staticGoScript("-tags integration -count=1 -timeout 120s -v ./pkg/memory/..."), nil),
NewShellTask("check", "Run deps, formatting, linting and tests", checkScript, nil), NewCommandTask("check", "Run deps, formatting, linting and tests", checkSpecs, nil, nil),
NewShellTask("run", "Build and run dragonscale", runScript, runTaskEnv), NewShellTask("run", "Build and run dragonscale", runScript, runTaskEnv),
NewShellTask("devcontainer-build", "Build the local devcontainer image via npx", devcontainerBuildScript, nil), NewCommandTask("devcontainer-build", "Build the local devcontainer image via npx", devcontainerBuildSpecs, nil, nil),
NewShellTask("devcontainer-up", "Start/update the local devcontainer", devcontainerUpScript, nil), NewCommandTask("devcontainer-up", "Start/update the local devcontainer", devcontainerUpSpecs, nil, nil),
NewShellTask("devcontainer-generate", "Run generation in devcontainer", devcontainerGenerateScript, nil), NewCommandTask("devcontainer-generate", "Run generation in devcontainer", devcontainerGenerateSpecs, nil, nil),
NewShellTask("devcontainer-verify", "Verify generated code in devcontainer", devcontainerVerifyScript, nil), NewCommandTask("devcontainer-verify", "Verify generated code in devcontainer", devcontainerVerifySpecs, nil, nil),
NewShellTask("eval-build", "Build the eval runner from current branch", evalBuildScript, nil), NewCommandTask("eval-build", "Build the eval runner from current branch", evalBuildSpecs, nil, nil),
NewShellTask("eval", "Run the eval suite", evalRunScript, nil), NewCommandTask("eval", "Run the eval suite", evalRunSpecs, nil, nil),
NewShellTask("eval-fixtures", "Prepare eval fixture workspace", evalFixturesScript, nil), NewCommandTask("eval-fixtures", "Prepare eval fixture workspace", evalFixturesSpecs, nil, nil),
NewShellTask("eval-view", "Open the promptfoo results viewer", evalViewScript, nil), NewCommandTask("eval-view", "Open the promptfoo results viewer", evalViewSpecs, nil, nil),
NewShellTask("eval-clean", "Cleanup eval artifacts", simpleScript("rm -rf eval/results eval/bin"), nil), NewShellTask("eval-clean", "Cleanup eval artifacts", simpleScript("rm -rf eval/results eval/bin"), nil),
NewShellTask("eval-compare", "Run A/B comparison of current branch vs main", simpleScript("cd eval && DEVCONTAINER_EXEC= EVAL_NPM_CMD=$(npx --yes) ./scripts/compare.sh --repeat 3"), nil), NewShellTask("eval-compare", "Run A/B comparison of current branch vs main", simpleScript("cd eval && DEVCONTAINER_EXEC= EVAL_NPM_CMD=$(npx --yes) ./scripts/compare.sh --repeat 3"), nil),
NewShellTask("eval-test", "Run Go-native component evals", staticGoScript("-v ./eval/go_evals/..."), nil), NewShellTask("eval-test", "Run Go-native component evals", staticGoScript("-v ./eval/go_evals/..."), nil),
@ -177,66 +309,54 @@ func NewRegistry(_ string) []app.Task {
} }
func defaultEnv(c *app.Context) []string { func defaultEnv(c *app.Context) []string {
env := []string{ pairs := map[string]string{
fmt.Sprintf("GO=%s", cEnv(c, "GO", "go")), "GO": cEnv(c, "GO", "go"),
fmt.Sprintf("GOFLAGS=%s", cEnv(c, "GOFLAGS", "-v -trimpath -tags stdjson")), "GOFLAGS": cEnv(c, "GOFLAGS", "-v -trimpath -tags=stdjson"),
fmt.Sprintf("CGO_ENABLED=%s", cEnv(c, "CGO_ENABLED", "1")), "CGO_ENABLED": cEnv(c, "CGO_ENABLED", "1"),
fmt.Sprintf("BINARY_NAME=%s", cEnv(c, "BINARY_NAME", defaultBinaryName)), "BINARY_NAME": cEnv(c, "BINARY_NAME", defaultBinaryName),
fmt.Sprintf("BUILD_DIR=%s", cEnv(c, "BUILD_DIR", defaultBuildDir)), "BUILD_DIR": cEnv(c, "BUILD_DIR", defaultBuildDir),
fmt.Sprintf("CMD_DIR=%s", cEnv(c, "CMD_DIR", defaultCmdDir)), "CMD_DIR": cEnv(c, "CMD_DIR", defaultCmdDir),
fmt.Sprintf("WORKSPACE_DIR=%s", cEnv(c, "WORKSPACE_DIR", filepath.Join(homeDir(), ".dragonscale", "workspace"))), "WORKSPACE_DIR": cEnv(c, "WORKSPACE_DIR", filepath.Join(homeDir(), ".dragonscale", "workspace")),
"GOOS": cEnv(c, "GOOS", "linux"),
"GOARCH": cEnv(c, "GOARCH", runtime.GOARCH),
} }
if value := cEnv(c, "GOOS", ""); value != "" { appendIfSet := func(key, value string) {
env = append(env, fmt.Sprintf("GOOS=%s", value)) if strings.TrimSpace(value) != "" {
pairs[key] = value
} }
if value := cEnv(c, "GOARCH", ""); value != "" {
env = append(env, fmt.Sprintf("GOARCH=%s", value))
}
if value := cEnv(c, "PLATFORM", ""); value != "" {
env = append(env, fmt.Sprintf("PLATFORM=%s", value))
}
if value := cEnv(c, "ARCH", ""); value != "" {
env = append(env, fmt.Sprintf("ARCH=%s", value))
}
if v := cEnv(c, "DEVCONTAINER_EXEC", ""); v != "" {
env = append(env, "DEVCONTAINER_EXEC="+v)
}
if value := cEnv(c, "DRAGONSCALE_EVAL_HOST_HOME", ""); value != "" {
env = append(env, fmt.Sprintf("DRAGONSCALE_EVAL_HOST_HOME=%s", value))
}
if value := cEnv(c, "DRAGONSCALE_EVAL_BASE_CONFIG", ""); value != "" {
env = append(env, fmt.Sprintf("DRAGONSCALE_EVAL_BASE_CONFIG=%s", value))
}
if value := cEnv(c, "DRAGONSCALE_EVAL_CONFIG", ""); value != "" {
env = append(env, fmt.Sprintf("DRAGONSCALE_EVAL_CONFIG=%s", value))
}
if value := cEnv(c, "DRAGONSCALE_EVAL_DEBUG", ""); value != "" {
env = append(env, fmt.Sprintf("DRAGONSCALE_EVAL_DEBUG=%s", value))
}
if value := cEnv(c, "VERSION", ""); value != "" {
env = append(env, fmt.Sprintf("VERSION=%s", value))
}
if value := cEnv(c, "FANTASY_VERSION", ""); value != "" {
env = append(env, fmt.Sprintf("FANTASY_VERSION=%s", value))
}
if value := cEnv(c, "NAME", ""); value != "" {
env = append(env, fmt.Sprintf("NAME=%s", value))
}
if value := cEnv(c, "ARGS", ""); value != "" {
env = append(env, fmt.Sprintf("ARGS=%s", value))
} }
appendIfSet("PLATFORM", cEnv(c, "PLATFORM", ""))
appendIfSet("ARCH", cEnv(c, "ARCH", ""))
appendIfSet("DEVCONTAINER_EXEC", cEnv(c, "DEVCONTAINER_EXEC", ""))
appendIfSet("DRAGONSCALE_EVAL_HOST_HOME", cEnv(c, "DRAGONSCALE_EVAL_HOST_HOME", ""))
appendIfSet("DRAGONSCALE_EVAL_BASE_CONFIG", cEnv(c, "DRAGONSCALE_EVAL_BASE_CONFIG", ""))
appendIfSet("DRAGONSCALE_EVAL_CONFIG", cEnv(c, "DRAGONSCALE_EVAL_CONFIG", ""))
appendIfSet("DRAGONSCALE_EVAL_DEBUG", cEnv(c, "DRAGONSCALE_EVAL_DEBUG", ""))
appendIfSet("VERSION", cEnv(c, "VERSION", ""))
appendIfSet("FANTASY_VERSION", cEnv(c, "FANTASY_VERSION", ""))
appendIfSet("NAME", cEnv(c, "NAME", ""))
appendIfSet("ARGS", cEnv(c, "ARGS", ""))
if c.Root != "" { if c.Root != "" {
env = append(env, fmt.Sprintf("DRAGONSCALE_HOME=%s", filepath.Join(homeDir(), ".dragonscale"))) appendIfSet("DRAGONSCALE_HOME", filepath.Join(homeDir(), ".dragonscale"))
}
env := make([]string, 0, len(pairs))
for key, value := range pairs {
if key == "" || value == "" {
continue
}
env = append(env, fmt.Sprintf("%s=%s", key, value))
} }
return env return env
} }
func cEnv(c *app.Context, key string, fallback string) string { func cEnv(c *app.Context, key string, fallback string) string {
if c != nil && c.ExtraEnv != nil {
if v, ok := c.ExtraEnv[key]; ok { if v, ok := c.ExtraEnv[key]; ok {
if strings.TrimSpace(v) != "" { if strings.TrimSpace(v) != "" {
return v return v
} }
} }
}
return fallback return fallback
} }
@ -270,15 +390,11 @@ func applyDevcontainerWrapper(script string, c *app.Context) string {
return script return script
} }
workDir := c.Root return fmt.Sprintf("%s -- bash -lc %s", execCmd, shellSingleQuote(script))
if c.Cwd != "" { }
workDir = c.Cwd
}
if strings.TrimSpace(workDir) != "" {
script = fmt.Sprintf("cd %s\n%s", strconv.Quote(workDir), script)
}
return fmt.Sprintf("%s -- bash -lc %s", execCmd, strconv.Quote(script)) func shellSingleQuote(input string) string {
return "'" + strings.ReplaceAll(input, "'", `'"'"'`) + "'"
} }
func detectDevcontainerExec(root string) string { func detectDevcontainerExec(root string) string {
@ -331,61 +447,116 @@ func staticGoScript(rest string) func(*app.Context) string {
} }
} }
func generateScript(*app.Context) string { func generateScript(c *app.Context) string {
cmdDir := cEnv(c, "CMD_DIR", defaultCmdDir)
return scriptHeader() + return scriptHeader() +
"rm -rf ./$(CMD_DIR)/workspace 2>/dev/null || true\n" + fmt.Sprintf("rm -rf ./%s/workspace 2>/dev/null || true\n", cmdDir) +
"$GO generate ./...\n" "$GO generate ./...\n"
} }
func buildScript(*app.Context) string { func buildScript(c *app.Context) string {
version := "$(git describe --tags --always --dirty 2>/dev/null || echo \"dev\")" version := "$(git describe --tags --always --dirty 2>/dev/null || echo \"dev\")"
gitCommit := "$(git rev-parse --short=8 HEAD 2>/dev/null || echo \"dev\")" gitCommit := "$(git rev-parse --short=8 HEAD 2>/dev/null || echo \"dev\")"
buildTime := "$(date +%FT%T%z)" buildTime := "$(date +%FT%T%z)"
goVersion := "$($GO version | awk '{print \"$3\"}')" goVersion := "$($GO version | awk '{print $3}')"
buildDir := cEnv(c, "BUILD_DIR", defaultBuildDir)
binaryName := cEnv(c, "BINARY_NAME", defaultBinaryName)
cmdDir := cEnv(c, "CMD_DIR", defaultCmdDir)
targetGOOS := cEnv(c, "GOOS", "linux")
targetGOARCH := cEnv(c, "GOARCH", runtime.GOARCH)
ldFlags := fmt.Sprintf("-X main.version=%s -X main.gitCommit=%s -X main.buildTime=%s -X main.goVersion=%s -s -w", version, gitCommit, buildTime, goVersion) ldFlags := fmt.Sprintf("-X main.version=%s -X main.gitCommit=%s -X main.buildTime=%s -X main.goVersion=%s -s -w", version, gitCommit, buildTime, goVersion)
return scriptHeader() + return scriptHeader() +
"mkdir -p $(BUILD_DIR)\n" + fmt.Sprintf("mkdir -p %s\n", buildDir) +
"GOOS=$($GO env GOOS)\n" + fmt.Sprintf("echo \"build: target=%s/%s/%s\"\n", targetGOOS, targetGOARCH, binaryName) +
"GOARCH=$($GO env GOARCH)\n" + fmt.Sprintf("GOOS=$GOOS GOARCH=$GOARCH CGO_ENABLED=$CGO_ENABLED $GO build $GOFLAGS -ldflags \"%s\" -o %s/%s-${GOOS}-${GOARCH} ./%s\n", ldFlags, buildDir, binaryName, cmdDir) +
fmt.Sprintf("$GO build $GOFLAGS -ldflags \"%s\" -o $(BUILD_DIR)/$(BINARY_NAME)-${GOOS}-${GOARCH} ./$(CMD_DIR)\n", ldFlags) + fmt.Sprintf("ln -sf %s-${GOOS}-${GOARCH} %s/%s\n", binaryName, buildDir, binaryName)
"ln -sf $(BINARY_NAME)-$(GOOS)-$(GOARCH) $(BUILD_DIR)/$(BINARY_NAME)\n"
} }
func buildAllScript(*app.Context) string { func buildAllScript(c *app.Context) string {
ldFlags := "-ldflags '-s -w'" ldFlags := "-ldflags '-s -w'"
return scriptHeader() + buildDir := cEnv(c, "BUILD_DIR", defaultBuildDir)
"mkdir -p $(BUILD_DIR)\n" + binaryName := cEnv(c, "BINARY_NAME", defaultBinaryName)
fmt.Sprintf("GOOS=linux GOARCH=amd64 $GO build $GOFLAGS %s -o $(BUILD_DIR)/$(BINARY_NAME)-linux-amd64 ./$(CMD_DIR)\n", ldFlags) + cmdDir := cEnv(c, "CMD_DIR", defaultCmdDir)
fmt.Sprintf("GOOS=linux GOARCH=arm64 $GO build $GOFLAGS %s -o $(BUILD_DIR)/$(BINARY_NAME)-linux-arm64 ./$(CMD_DIR)\n", ldFlags) + targetGOOS := cEnv(c, "GOOS", "linux")
fmt.Sprintf("GOOS=linux GOARCH=loong64 $GO build $GOFLAGS %s -o $(BUILD_DIR)/$(BINARY_NAME)-linux-loong64 ./$(CMD_DIR)\n", ldFlags) + targetGOARCH := cEnv(c, "GOARCH", runtime.GOARCH)
fmt.Sprintf("GOOS=linux GOARCH=riscv64 $GO build $GOFLAGS %s -o $(BUILD_DIR)/$(BINARY_NAME)-linux-riscv64 ./$(CMD_DIR)\n", ldFlags) + var script strings.Builder
fmt.Sprintf("GOOS=darwin GOARCH=arm64 $GO build $GOFLAGS %s -o $(BUILD_DIR)/$(BINARY_NAME)-darwin-arm64 ./$(CMD_DIR)\n", ldFlags) + script.WriteString(scriptHeader())
fmt.Sprintf("GOOS=windows GOARCH=amd64 $GO build $GOFLAGS %s -o $(BUILD_DIR)/$(BINARY_NAME)-windows-amd64.exe ./$(CMD_DIR)\n", ldFlags) fmt.Fprintf(&script, "mkdir -p %s\n", buildDir)
fmt.Fprintf(&script, "echo \"build-all: target=%s/%s output=%s/%s-%s-%s\"\n", targetGOOS, targetGOARCH, buildDir, binaryName, targetGOOS, targetGOARCH)
fmt.Fprintf(&script, "OUTPUT_NAME=%s/%s-%s-%s\n", buildDir, binaryName, targetGOOS, targetGOARCH)
script.WriteString("export CGO_ENABLED=1\n")
fmt.Fprintf(&script, "GOOS=$GOOS GOARCH=$GOARCH CGO_ENABLED=$CGO_ENABLED $GO build $GOFLAGS %s -o ${OUTPUT_NAME} ./%s\n", ldFlags, cmdDir)
fmt.Fprintf(&script, "ln -sf ./%s %s/%s\n", "${OUTPUT_NAME}", buildDir, binaryName)
return script.String()
}
func validateBuildTaskEnv(c *app.Context) error {
targetGOOS := cEnv(c, "GOOS", "linux")
if targetGOOS != "linux" {
return fmt.Errorf("unsupported GOOS=%s: this project only supports linux/gnu builds", targetGOOS)
}
if cgoEnabled := strings.TrimSpace(cEnv(c, "CGO_ENABLED", "1")); cgoEnabled != "" && cgoEnabled != "1" {
return fmt.Errorf("unsupported CGO_ENABLED=%s: builds require CGO_ENABLED=1", cgoEnabled)
}
if err := detectGlibc(); err != nil {
return err
}
return nil
}
var detectGlibc = detectGlibcAvailable
func detectGlibcAvailable() error {
output, err := exec.Command("ldd", "--version").Output()
if err != nil {
return fmt.Errorf("unable to verify libc: %w", err)
}
header := strings.ToLower(strings.SplitN(string(output), "\n", 2)[0])
if strings.Contains(header, "glibc") || strings.Contains(header, "gnu c library") {
return nil
}
return fmt.Errorf("unsupported libc: this project requires glibc")
} }
func installScript(*app.Context) string { func installScript(*app.Context) string {
prefix := filepath.Join(homeDir(), ".local") prefix := filepath.Join(homeDir(), ".local")
buildDir := cEnv(nil, "BUILD_DIR", defaultBuildDir)
binaryName := cEnv(nil, "BINARY_NAME", defaultBinaryName)
return scriptHeader() + return scriptHeader() +
"mkdir -p ${INSTALL_PREFIX:-" + prefix + "}/bin\n" + "mkdir -p ${INSTALL_PREFIX:-" + prefix + "}/bin\n" +
"cp $(BUILD_DIR)/$(BINARY_NAME) ${INSTALL_PREFIX:-" + prefix + "}/bin/$(BINARY_NAME)\n" + fmt.Sprintf("cp %s/%s ${INSTALL_PREFIX:-%s}/bin/%s\n", buildDir, binaryName, prefix, binaryName) +
"chmod +x ${INSTALL_PREFIX:-" + prefix + "}/bin/$(BINARY_NAME)\n" fmt.Sprintf("chmod +x ${INSTALL_PREFIX:-%s}/bin/%s\n", prefix, binaryName)
} }
func uninstallScript(*app.Context) string { func uninstallScript(*app.Context) string {
return "rm -f ${INSTALL_PREFIX:-" + filepath.Join(homeDir(), ".local") + "}/bin/$(BINARY_NAME)\n" binaryName := cEnv(nil, "BINARY_NAME", defaultBinaryName)
prefix := filepath.Join(homeDir(), ".local")
return "rm -f ${INSTALL_PREFIX:-" + prefix + "}/bin/" + binaryName + "\n"
} }
func uninstallAllScript(*app.Context) string { func uninstallAllScript(c *app.Context) string {
return "rm -rf $(DRAGONSCALE_HOME)\n" dHome := filepath.Join(homeDir(), ".dragonscale")
if v := cEnv(c, "DRAGONSCALE_HOME", ""); v != "" {
dHome = v
}
return "rm -rf " + dHome + "\n"
} }
func cleanScript(*app.Context) string { func cleanScript(*app.Context) string {
return "rm -rf $(BUILD_DIR)\n" buildDir := cEnv(nil, "BUILD_DIR", defaultBuildDir)
return "rm -rf " + buildDir + "\n"
} }
func lintScript(*app.Context) string { func lintSpecs(c *app.Context) []runner.CommandSpec {
return scriptHeader() + goBinary := cEnv(c, "GO", "go")
"$GO fmt ./...\n$GO vet ./...\n$GO build ./...\n" return []runner.CommandSpec{
{Name: goBinary, Args: []string{"fmt", "./..."}},
{Name: goBinary, Args: []string{"vet", "./..."}},
{Name: goBinary, Args: []string{"build", "./..."}},
}
} }
func hooksScript(*app.Context) string { func hooksScript(*app.Context) string {
@ -431,11 +602,15 @@ func fantasyPatchScript(c *app.Context) string {
return simpleScript("./scripts/sync-fantasy.sh --save-patch " + cEnv(c, "NAME", ""))(c) return simpleScript("./scripts/sync-fantasy.sh --save-patch " + cEnv(c, "NAME", ""))(c)
} }
func checkScript(*app.Context) string { func checkSpecs(c *app.Context) []runner.CommandSpec {
return staticGoScript("mod download && mod verify")(&app.Context{}) + goBinary := cEnv(c, "GO", "go")
staticGoScript("fmt ./...")(&app.Context{}) + return []runner.CommandSpec{
staticGoScript("vet ./...")(&app.Context{}) + {Name: goBinary, Args: []string{"mod", "download"}},
staticGoScript("test ./...")(&app.Context{}) {Name: goBinary, Args: []string{"mod", "verify"}},
{Name: goBinary, Args: []string{"fmt", "./..."}},
{Name: goBinary, Args: []string{"vet", "./..."}},
{Name: goBinary, Args: []string{"test", "./..."}},
}
} }
func runTaskEnv(c *app.Context) []string { func runTaskEnv(c *app.Context) []string {
@ -446,7 +621,9 @@ func runTaskEnv(c *app.Context) []string {
} }
func runScript(c *app.Context) string { func runScript(c *app.Context) string {
return scriptHeader() + "$(BUILD_DIR)/$(BINARY_NAME) " + strings.Join(quoteArgs(c.Argv), " ") + "\n" buildDir := cEnv(c, "BUILD_DIR", defaultBuildDir)
binaryName := cEnv(c, "BINARY_NAME", defaultBinaryName)
return scriptHeader() + buildDir + "/" + binaryName + " " + strings.Join(quoteArgs(c.Argv), " ") + "\n"
} }
func quoteArgs(args []string) []string { func quoteArgs(args []string) []string {
@ -457,52 +634,173 @@ func quoteArgs(args []string) []string {
return quoted return quoted
} }
func devcontainerBuildScript(*app.Context) string { func devcontainerRoot(c *app.Context) string {
return "npx --yes @devcontainers/cli build --workspace-folder \"$(pwd)\"\n" if c != nil && c.Root != "" {
return c.Root
}
workspace := cValue(c, "DEVCONTAINER_WORKSPACE", ".")
if workspace == "" {
return "."
}
return workspace
} }
func devcontainerUpScript(*app.Context) string { func devcontainerBuildSpecs(c *app.Context) []runner.CommandSpec {
return "npx --yes @devcontainers/cli up --workspace-folder \"$(pwd)\"\n" return []runner.CommandSpec{
{
Name: "npx",
Args: []string{"--yes", "@devcontainers/cli", "build", "--workspace-folder", devcontainerRoot(c)},
},
}
} }
func devcontainerGenerateScript(*app.Context) string { func devcontainerUpSpecs(c *app.Context) []runner.CommandSpec {
return "npx --yes @devcontainers/cli exec --workspace-folder \"$(pwd)\" -- bash -lc \"go generate ./pkg/itr ./pkg/tools && sqlc generate -f pkg/memory/sqlc/sqlc.yaml\"\n" return []runner.CommandSpec{
{
Name: "npx",
Args: []string{"--yes", "@devcontainers/cli", "up", "--workspace-folder", devcontainerRoot(c)},
},
}
} }
func devcontainerVerifyScript(*app.Context) string { func devcontainerGenerateSpecs(c *app.Context) []runner.CommandSpec {
return "npx --yes @devcontainers/cli exec --workspace-folder \"$(pwd)\" -- bash -lc \"make flatc-check sqlc-check\"\n" root := devcontainerRoot(c)
goBinary := cEnv(c, "GO", "go")
return []runner.CommandSpec{
{
Name: "npx",
Args: []string{
"--yes", "@devcontainers/cli", "exec",
"--workspace-folder", root, "--",
"bash", "-lc",
fmt.Sprintf("%s generate ./pkg/itr ./pkg/tools && sqlc generate -f pkg/memory/sqlc/sqlc.yaml", goBinary),
},
},
}
} }
func evalBuildScript(*app.Context) string { func devcontainerVerifySpecs(c *app.Context) []runner.CommandSpec {
ldFlags := "-X main.version=$(git describe --tags --always --dirty 2>/dev/null || echo \"dev\") -X main.gitCommit=$(git rev-parse --short=8 HEAD 2>/dev/null || echo \"dev\") -X main.buildTime=$(date +%FT%T%z) -X main.goVersion=\"$($GO version | awk '{print \"$3\"}')\" -s -w" root := devcontainerRoot(c)
return scriptHeader() + return []runner.CommandSpec{
"$GO generate ./...\n" + {
"mkdir -p eval/bin\n" + Name: "npx",
fmt.Sprintf("$GO build $GOFLAGS -ldflags \"%s\" -o eval/bin/eval-runner ./eval/cmd/eval-runner\n", ldFlags) Args: []string{
"--yes", "@devcontainers/cli", "exec",
"--workspace-folder", root, "--",
"bash", "-lc", "make flatc-check sqlc-check",
},
},
}
} }
func evalRunScript(*app.Context) string { func evalBuildSpecs(c *app.Context) []runner.CommandSpec {
return `if [ -n "${DRAGONSCALE_EVAL_BASE_CONFIG:-}" ] && [ -n "${DRAGONSCALE_EVAL_DEBUG:-}" ]; then goBinary := cEnv(c, "GO", "go")
echo "DRAGONSCALE_EVAL_BASE_CONFIG=${DRAGONSCALE_EVAL_BASE_CONFIG}" goFlags := strings.Fields(cEnv(c, "GOFLAGS", "-v -trimpath -tags=stdjson"))
fi goVersion := "unknown"
if [ -n "${DRAGONSCALE_EVAL_DEBUG:-}" ]; then goVersionParts := strings.Fields(outputOrDefault(goBinary + " version"))
echo "DRAGONSCALE_EVAL_CONFIG=${DRAGONSCALE_EVAL_CONFIG:-./configs/default.json}" if len(goVersionParts) >= 3 {
fi goVersion = goVersionParts[2]
DRAGONSCALE_EVAL_HOST_HOME=/host_home DRAGONSCALE_EVAL_CONFIG="${DRAGONSCALE_EVAL_CONFIG:-./configs/default.json}" npx --yes promptfoo eval --config promptfooconfig.yaml --no-cache --no-progress-bar }
` version := outputOrDefault("git describe --tags --always --dirty 2>/dev/null || echo \"dev\"")
commit := outputOrDefault("git rev-parse --short=8 HEAD 2>/dev/null || echo \"dev\"")
buildTime := outputOrDefault("date +%FT%T%z")
ldFlags := fmt.Sprintf(
"-X main.version=%s -X main.gitCommit=%s -X main.buildTime=%s -X main.goVersion=%s -s -w",
version, commit, buildTime, goVersion,
)
return []runner.CommandSpec{
{Name: goBinary, Args: []string{"generate", "./..."}},
{Name: "mkdir", Args: []string{"-p", "eval/bin"}},
{
Name: goBinary,
Args: append(
append(append([]string{"build"}, goFlags...), "-ldflags", ldFlags),
"-o", filepath.Join("eval", "bin", "eval-runner"), "./eval/cmd/eval-runner",
),
},
}
} }
func evalFixturesScript(*app.Context) string { func evalRunSpecs(c *app.Context) []runner.CommandSpec {
return scriptHeader() + cfgPath := cEnv(c, "DRAGONSCALE_EVAL_CONFIG", "./configs/default.json")
"mkdir -p \"$HOME/.local/share/dragonscale/sandbox\"\n" + baseCfg := cEnv(c, "DRAGONSCALE_EVAL_BASE_CONFIG", "")
"rm -f \"$HOME/.local/share/dragonscale/sandbox/eval_test_output.txt\" \"$HOME/.local/share/dragonscale/sandbox/test_steps.txt\" \"$HOME/.local/share/dragonscale/sandbox/eval_checkpoint.txt\" \"$HOME/.local/share/dragonscale/sandbox/chain_test.txt\" \"$HOME/.local/share/dragonscale/sandbox/current_year.txt\" \"$HOME/.local/share/dragonscale/sandbox/result.txt\" \"$HOME/.local/share/dragonscale/sandbox/progressive_test.txt\" \"$HOME/.local/share/dragonscale/sandbox/os_name.txt\"\n" + debug := cEnv(c, "DRAGONSCALE_EVAL_DEBUG", "") != ""
"rm -rf \"$HOME/.local/share/dragonscale/sandbox/project\"\n" + promptfooArgs := strings.Fields(cEnv(c, "DRAGONSCALE_PROMPTFOO_ARGS", "--no-cache --no-progress-bar"))
"printf 'dragonscale eval fixture — hello from the eval harness\\nThis is line two of the fixture file.\\n' > \"$HOME/.local/share/dragonscale/sandbox/eval_fixture.txt\"\n" + if len(promptfooArgs) == 0 {
"cp -f eval/fixtures/sample_data.txt \"$HOME/.local/share/dragonscale/sandbox/sample_data.txt\"\n" + promptfooArgs = []string{"--no-cache", "--no-progress-bar"}
"mkdir -p \"$HOME/.local/share/dragonscale/skills\"\n" + }
"cp -rf eval/fixtures/skills/* \"$HOME/.local/share/dragonscale/skills/\" 2>/dev/null || true\n"
var specs []runner.CommandSpec
if debug && strings.TrimSpace(baseCfg) != "" {
specs = append(specs, runner.CommandSpec{
Name: "echo",
Args: []string{fmt.Sprintf("DRAGONSCALE_EVAL_BASE_CONFIG=%s", baseCfg)},
})
}
if debug {
specs = append(specs, runner.CommandSpec{
Name: "echo",
Args: []string{fmt.Sprintf("DRAGONSCALE_EVAL_CONFIG=%s", cfgPath)},
})
}
args := append([]string{"--yes", "promptfoo", "eval", "--config", "promptfooconfig.yaml"}, promptfooArgs...)
specs = append(specs, runner.CommandSpec{
Name: "npx",
Args: args,
Dir: filepath.Join(c.Root, "eval"),
Env: []string{fmt.Sprintf("DRAGONSCALE_EVAL_CONFIG=%s", cfgPath)},
})
return specs
} }
func evalViewScript(*app.Context) string { func evalFixturesSpecs(c *app.Context) []runner.CommandSpec {
return "cd eval && npx --yes promptfoo view\n" sandbox := filepath.Join(homeDir(), ".local", "share", "dragonscale", "sandbox")
project := filepath.Join(sandbox, "project")
skills := filepath.Join(homeDir(), ".local", "share", "dragonscale", "skills")
shared := filepath.Join(sandbox, "sample_data.txt")
fixture := filepath.Join(sandbox, "eval_fixture.txt")
files := []string{
filepath.Join(sandbox, "eval_test_output.txt"),
filepath.Join(sandbox, "test_steps.txt"),
filepath.Join(sandbox, "eval_checkpoint.txt"),
filepath.Join(sandbox, "chain_test.txt"),
filepath.Join(sandbox, "current_year.txt"),
filepath.Join(sandbox, "result.txt"),
filepath.Join(sandbox, "progressive_test.txt"),
filepath.Join(sandbox, "os_name.txt"),
}
sourceFixture := filepath.Join("eval", "fixtures", "sample_data.txt")
specs := []runner.CommandSpec{
{Name: "mkdir", Args: []string{"-p", sandbox}},
{Name: "rm", Args: append([]string{"-f"}, files...)},
{Name: "rm", Args: []string{"-rf", project}},
{Name: "mkdir", Args: []string{"-p", skills}},
{Name: "bash", Args: []string{"-lc", fmt.Sprintf("printf '%%s\\n%%s\\n' \"dragonscale eval fixture — hello from the eval harness\" \"This is line two of the fixture file.\" > %q", fixture)}},
{Name: "cp", Args: []string{"-f", sourceFixture, shared}},
}
specs = append(specs, runner.CommandSpec{
Name: "bash",
Args: []string{"-lc", "if [ -d eval/fixtures/skills ]; then cp -rf eval/fixtures/skills/. " + strconv.Quote(skills) + "; fi"},
})
return specs
}
func evalViewSpecs(c *app.Context) []runner.CommandSpec {
return []runner.CommandSpec{
{
Name: "npx",
Args: []string{"--yes", "promptfoo", "view"},
Dir: filepath.Join(c.Root, "eval"),
},
}
}
func outputOrDefault(command string) string {
output, err := exec.Command("bash", "-lc", command).Output()
if err != nil {
return "dev"
}
return strings.TrimSpace(string(output))
} }

View file

@ -81,12 +81,14 @@ func TestQuoteArgs(t *testing.T) {
} }
func TestBuildAllTaskPreservesGoEnvironmentForwarding(t *testing.T) { func TestBuildAllTaskPreservesGoEnvironmentForwarding(t *testing.T) {
t.Parallel() origCheck := detectGlibc
detectGlibc = func() error { return nil }
t.Cleanup(func() { detectGlibc = origCheck })
ctx := &app.Context{ ctx := &app.Context{
Root: t.TempDir(), Root: t.TempDir(),
ExtraEnv: map[string]string{ ExtraEnv: map[string]string{
"GOOS": "freebsd", "GOOS": "linux",
"GOARCH": "sparc64", "GOARCH": "sparc64",
"GOFLAGS": "-mod=mod", "GOFLAGS": "-mod=mod",
"DEVCONTAINER_EXEC": "echo devcontainer exec", "DEVCONTAINER_EXEC": "echo devcontainer exec",
@ -110,31 +112,100 @@ func TestBuildAllTaskPreservesGoEnvironmentForwarding(t *testing.T) {
require.Len(t, fake.Calls, 1) require.Len(t, fake.Calls, 1)
script := strings.Join(fake.Calls[0].Args, " ") script := strings.Join(fake.Calls[0].Args, " ")
require.Contains(t, script, "GOOS=linux GOARCH=amd64 $GO build") require.Contains(t, script, "GOOS=$GOOS")
require.Contains(t, script, "GOOS=darwin GOARCH=arm64 $GO build") require.NotContains(t, script, "if [ \"$GOOS\" != \"linux\" ]; then")
require.Contains(t, script, "GOOS=windows GOARCH=amd64 $GO build") require.Contains(t, script, "CGO_ENABLED=1")
require.Contains(t, script, "GOOS=linux GOARCH=arm64 $GO build") require.NotContains(t, script, "CGO_BUILD=1")
joinedEnv := strings.Join(fake.Calls[0].Env, " ") joinedEnv := strings.Join(fake.Calls[0].Env, " ")
require.Contains(t, joinedEnv, "GOOS=freebsd") require.Contains(t, joinedEnv, "GOOS=linux")
require.Contains(t, joinedEnv, "GOARCH=sparc64") require.Contains(t, joinedEnv, "GOARCH=sparc64")
require.Contains(t, joinedEnv, "GOFLAGS=-mod=mod") require.Contains(t, joinedEnv, "GOFLAGS=-mod=mod")
require.Contains(t, joinedEnv, "DEVCONTAINER_EXEC=echo devcontainer exec") require.Contains(t, joinedEnv, "DEVCONTAINER_EXEC=echo devcontainer exec")
} }
func TestEvalRunScriptPreservesEvalConfig(t *testing.T) { func TestBuildAllTaskDefaultsToHostTarget(t *testing.T) {
origCheck := detectGlibc
detectGlibc = func() error { return nil }
t.Cleanup(func() { detectGlibc = origCheck })
ctx := &app.Context{
Root: t.TempDir(),
ExtraEnv: map[string]string{
"GOFLAGS": "-mod=mod",
"DEVCONTAINER_EXEC": "echo devcontainer exec",
},
}
fake := &runner.FakeRunner{
Result: runner.CommandResult{ExitCode: 0},
}
var buildAll app.Task
for _, task := range NewRegistry(ctx.Root) {
if task.Name() == "build-all" {
buildAll = task
break
}
}
require.NotNil(t, buildAll)
_, err := buildAll.Run(context.Background(), fake, ctx)
require.NoError(t, err)
require.Len(t, fake.Calls, 1)
script := strings.Join(fake.Calls[0].Args, " ")
require.Contains(t, script, "build-all: target=linux/")
require.NotContains(t, script, "GOOS=$($GO env GOOS)")
require.Contains(t, script, "CGO_ENABLED=1")
require.NotContains(t, script, "CGO_BUILD=1")
}
func TestBuildAllTaskRejectsNonLinuxTarget(t *testing.T) {
origCheck := detectGlibc
detectGlibc = func() error { return nil }
t.Cleanup(func() { detectGlibc = origCheck })
ctx := &app.Context{
Root: t.TempDir(),
ExtraEnv: map[string]string{
"GOOS": "freebsd",
},
}
fake := &runner.FakeRunner{
Result: runner.CommandResult{ExitCode: 0},
}
var buildAll app.Task
for _, task := range NewRegistry(ctx.Root) {
if task.Name() == "build-all" {
buildAll = task
break
}
}
require.NotNil(t, buildAll)
_, err := buildAll.Run(context.Background(), fake, ctx)
require.Error(t, err)
require.Len(t, fake.Calls, 0)
}
func TestEvalRunSpecsPreservesEvalConfig(t *testing.T) {
t.Parallel() t.Parallel()
script := evalRunScript(&app.Context{ specs := evalRunSpecs(&app.Context{
ExtraEnv: map[string]string{ ExtraEnv: map[string]string{
"DRAGONSCALE_EVAL_DEBUG": "1", "DRAGONSCALE_EVAL_DEBUG": "1",
"DRAGONSCALE_EVAL_CONFIG": "./configs/override.json", "DRAGONSCALE_EVAL_CONFIG": "./configs/override.json",
}, },
}) })
require.Contains(t, script, "DRAGONSCALE_EVAL_CONFIG=\"${DRAGONSCALE_EVAL_CONFIG:-./configs/default.json}\"") require.Len(t, specs, 2)
require.Contains(t, script, "npx --yes promptfoo eval --config promptfooconfig.yaml") require.Equal(t, "echo", specs[0].Name)
require.NotContains(t, script, "DRAGONSCALE_EVAL_CONFIG=\"./configs/default.json\"") require.Equal(t, []string{"DRAGONSCALE_EVAL_CONFIG=./configs/override.json"}, specs[0].Args)
require.Equal(t, "npx", specs[1].Name)
require.Contains(t, strings.Join(specs[1].Env, " "), "DRAGONSCALE_EVAL_CONFIG=./configs/override.json")
require.NotContains(t, strings.Join(specs[1].Env, " "), "DRAGONSCALE_EVAL_CONFIG=./configs/default.json")
require.Contains(t, strings.Join(specs[1].Args, " "), "--yes promptfoo eval --config promptfooconfig.yaml --no-cache --no-progress-bar")
compareScript, err := os.ReadFile(filepath.Clean(filepath.Join("..", "..", "..", "eval", "scripts", "compare.sh"))) compareScript, err := os.ReadFile(filepath.Clean(filepath.Join("..", "..", "..", "eval", "scripts", "compare.sh")))
require.NoError(t, err) require.NoError(t, err)
@ -142,18 +213,27 @@ func TestEvalRunScriptPreservesEvalConfig(t *testing.T) {
require.Contains(t, content, "EVAL_CONFIG=\"${DRAGONSCALE_EVAL_CONFIG:-./configs/default.json}\"") require.Contains(t, content, "EVAL_CONFIG=\"${DRAGONSCALE_EVAL_CONFIG:-./configs/default.json}\"")
require.Contains(t, content, "trap cleanup_compare_config EXIT INT TERM") require.Contains(t, content, "trap cleanup_compare_config EXIT INT TERM")
require.Contains(t, content, "DRAGONSCALE_EVAL_CONFIG: \"${EVAL_CONFIG}\"") require.Contains(t, content, "DRAGONSCALE_EVAL_CONFIG: \"${EVAL_CONFIG}\"")
require.NotContains(t, content, "DRAGONSCALE_EVAL_HOST_HOME: \"/host_home\"")
require.Contains(t, content, "TEMP_CONFIG") require.Contains(t, content, "TEMP_CONFIG")
} }
func TestEvalRunScriptUsesBaseConfigWhenSet(t *testing.T) { func TestEvalRunSpecsUsesBaseConfigWhenSetAndDebugEnabled(t *testing.T) {
t.Parallel() t.Parallel()
script := evalRunScript(&app.Context{ specs := evalRunSpecs(&app.Context{
ExtraEnv: map[string]string{ ExtraEnv: map[string]string{
"DRAGONSCALE_EVAL_BASE_CONFIG": "/tmp/base.json", "DRAGONSCALE_EVAL_BASE_CONFIG": "/tmp/base.json",
"DRAGONSCALE_EVAL_DEBUG": "1",
}, },
}) })
require.Contains(t, script, `if [ -n "${DRAGONSCALE_EVAL_BASE_CONFIG:-}" ] && [ -n "${DRAGONSCALE_EVAL_DEBUG:-}" ]; then`)
require.Len(t, specs, 3)
require.Equal(t, "echo", specs[0].Name)
require.Equal(t, []string{"DRAGONSCALE_EVAL_BASE_CONFIG=/tmp/base.json"}, specs[0].Args)
require.Equal(t, "echo", specs[1].Name)
require.Equal(t, []string{"DRAGONSCALE_EVAL_CONFIG=./configs/default.json"}, specs[1].Args)
require.Equal(t, "npx", specs[2].Name)
require.Equal(t, []string{"--yes", "promptfoo", "eval", "--config", "promptfooconfig.yaml", "--no-cache", "--no-progress-bar"}, specs[2].Args)
} }
func TestEvalCompareTaskDisablesNestedDevcontainerExecution(t *testing.T) { func TestEvalCompareTaskDisablesNestedDevcontainerExecution(t *testing.T) {
@ -248,6 +328,135 @@ func TestEvalCompareTaskPassesBaseConfigEnvToRunner(t *testing.T) {
} }
} }
func TestLintTaskUsesCommandSpecs(t *testing.T) {
ctx := &app.Context{
Root: t.TempDir(),
}
fake := &runner.FakeRunner{
Result: runner.CommandResult{ExitCode: 0},
}
task := findTaskByName(t, NewRegistry(ctx.Root), "lint")
_, err := task.Run(context.Background(), fake, ctx)
require.NoError(t, err)
require.Len(t, fake.Calls, 3)
require.Equal(t, "go", fake.Calls[0].Name)
require.Equal(t, []string{"fmt", "./..."}, fake.Calls[0].Args)
require.Equal(t, "go", fake.Calls[1].Name)
require.Equal(t, []string{"vet", "./..."}, fake.Calls[1].Args)
require.Equal(t, "go", fake.Calls[2].Name)
require.Equal(t, []string{"build", "./..."}, fake.Calls[2].Args)
require.NotContains(t, strings.Join(fake.Calls[0].Env, " "), "set -euo pipefail")
}
func TestCheckTaskUsesCommandSpecs(t *testing.T) {
ctx := &app.Context{
Root: t.TempDir(),
ExtraEnv: map[string]string{
"GO": "go",
},
}
fake := &runner.FakeRunner{
Result: runner.CommandResult{ExitCode: 0},
}
task := findTaskByName(t, NewRegistry(ctx.Root), "check")
_, err := task.Run(context.Background(), fake, ctx)
require.NoError(t, err)
require.Len(t, fake.Calls, 5)
require.Equal(t, []string{"mod", "download"}, fake.Calls[0].Args)
require.Equal(t, []string{"mod", "verify"}, fake.Calls[1].Args)
require.Equal(t, []string{"fmt", "./..."}, fake.Calls[2].Args)
require.Equal(t, []string{"vet", "./..."}, fake.Calls[3].Args)
require.Equal(t, []string{"test", "./..."}, fake.Calls[4].Args)
}
func TestDevcontainerGenerateTaskUsesNpxExecCommand(t *testing.T) {
ctx := &app.Context{
Root: t.TempDir(),
}
fake := &runner.FakeRunner{
Result: runner.CommandResult{ExitCode: 0},
}
task := findTaskByName(t, NewRegistry(ctx.Root), "devcontainer-generate")
_, err := task.Run(context.Background(), fake, ctx)
require.NoError(t, err)
require.Len(t, fake.Calls, 1)
require.Equal(t, "npx", fake.Calls[0].Name)
require.Contains(t, fake.Calls[0].Args, "@devcontainers/cli")
require.Contains(t, fake.Calls[0].Args, "exec")
require.Contains(t, fake.Calls[0].Args, "bash")
require.Contains(t, strings.Join(fake.Calls[0].Args, " "), "go generate ./pkg/itr ./pkg/tools")
}
func TestEvalTaskBuildsPromptfooCommandWithDefaultConfig(t *testing.T) {
ctx := &app.Context{
Root: t.TempDir(),
ExtraEnv: map[string]string{
"DRAGONSCALE_EVAL_DEBUG": "1",
},
}
fake := &runner.FakeRunner{
Result: runner.CommandResult{ExitCode: 0},
}
task := findTaskByName(t, NewRegistry(ctx.Root), "eval")
_, err := task.Run(context.Background(), fake, ctx)
require.NoError(t, err)
require.Len(t, fake.Calls, 2)
require.Equal(t, "echo", fake.Calls[0].Name)
require.Equal(t, "npx", fake.Calls[1].Name)
require.Equal(t, filepath.Join(ctx.Root, "eval"), fake.Calls[1].Dir)
joinedEnv := strings.Join(fake.Calls[1].Env, " ")
require.Contains(t, joinedEnv, "DRAGONSCALE_EVAL_CONFIG=./configs/default.json")
require.Contains(t, strings.Join(fake.Calls[1].Args, " "), "--no-progress-bar")
}
func TestEvalViewTaskRunsInEvalDirectory(t *testing.T) {
ctx := &app.Context{
Root: t.TempDir(),
}
fake := &runner.FakeRunner{
Result: runner.CommandResult{ExitCode: 0},
}
task := findTaskByName(t, NewRegistry(ctx.Root), "eval-view")
_, err := task.Run(context.Background(), fake, ctx)
require.NoError(t, err)
require.Len(t, fake.Calls, 1)
require.Equal(t, filepath.Join(ctx.Root, "eval"), fake.Calls[0].Dir)
require.Equal(t, "npx", fake.Calls[0].Name)
require.Equal(t, []string{"--yes", "promptfoo", "view"}, fake.Calls[0].Args)
}
func TestEvalFixturesTaskCreatesFixtureCommands(t *testing.T) {
ctx := &app.Context{
Root: t.TempDir(),
}
fake := &runner.FakeRunner{
Result: runner.CommandResult{ExitCode: 0},
}
task := findTaskByName(t, NewRegistry(ctx.Root), "eval-fixtures")
_, err := task.Run(context.Background(), fake, ctx)
require.NoError(t, err)
require.GreaterOrEqual(t, len(fake.Calls), 5)
require.Equal(t, "mkdir", fake.Calls[0].Name)
require.Equal(t, "rm", fake.Calls[1].Name)
require.Equal(t, "bash", fake.Calls[4].Name)
require.Equal(t, "cp", fake.Calls[5].Name)
}
func findTaskByName(t *testing.T, tasks []app.Task, name string) app.Task {
t.Helper()
for _, task := range tasks {
if task.Name() == name {
return task
}
}
t.Fatalf("task %q missing from registry", name)
return nil
}
func ptrString(value string) *string { func ptrString(value string) *string {
return &value return &value
} }