refactor(opsctl): format, tasks, main enhancements and tests
This commit is contained in:
parent
c2719ff30f
commit
e5e187bad1
6 changed files with 981 additions and 168 deletions
|
|
@ -5,6 +5,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
|
@ -70,10 +71,43 @@ func parseOutputMode(raw, json bool, mode string) (format.OutputMode, error) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolveRootPath(root string) (string, error) {
|
||||||
|
resolved := app.EnsureRoot(root)
|
||||||
|
if strings.TrimSpace(resolved) == "" {
|
||||||
|
return "", fmt.Errorf("invalid --root: resolved empty repository path")
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := os.Stat(resolved)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("invalid --root %q: %w", resolved, err)
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
return "", fmt.Errorf("invalid --root %q: not a directory", resolved)
|
||||||
|
}
|
||||||
|
return resolved, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveWorkingDirectory(root, cwd string) (string, error) {
|
||||||
|
clean := strings.TrimSpace(cwd)
|
||||||
|
if clean == "" {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
if !filepath.IsAbs(clean) {
|
||||||
|
clean = filepath.Join(root, clean)
|
||||||
|
}
|
||||||
|
info, err := os.Stat(clean)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("invalid --cwd %q: %w", clean, err)
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
return "", fmt.Errorf("invalid --cwd %q: not a directory", clean)
|
||||||
|
}
|
||||||
|
return clean, nil
|
||||||
|
}
|
||||||
|
|
||||||
func newOpsApp(root string) *app.App {
|
func newOpsApp(root string) *app.App {
|
||||||
rootPath := app.EnsureRoot(root)
|
ops := app.New(runner.OSRunner{}, root)
|
||||||
ops := app.New(runner.OSRunner{}, rootPath)
|
for _, t := range tasks.NewRegistry(root) {
|
||||||
for _, t := range tasks.NewRegistry(rootPath) {
|
|
||||||
ops.Register(t)
|
ops.Register(t)
|
||||||
}
|
}
|
||||||
ops.Register(tasks.NewHelpTask(ops))
|
ops.Register(tasks.NewHelpTask(ops))
|
||||||
|
|
@ -126,7 +160,10 @@ func buildOpsctlCommand(stdout, stderr io.Writer, args []string) *cobra.Command
|
||||||
return &exitCodeError{code: 2, err: err}
|
return &exitCodeError{code: 2, err: err}
|
||||||
}
|
}
|
||||||
|
|
||||||
resolvedRoot := app.EnsureRoot(rootFlag)
|
resolvedRoot, err := resolveRootPath(rootFlag)
|
||||||
|
if err != nil {
|
||||||
|
return &exitCodeError{code: 2, err: err}
|
||||||
|
}
|
||||||
ops := newOpsApp(resolvedRoot)
|
ops := newOpsApp(resolvedRoot)
|
||||||
|
|
||||||
if len(cmdArgs) == 0 || cmdArgs[0] == "help" || cmdArgs[0] == "--help" || cmdArgs[0] == "-h" {
|
if len(cmdArgs) == 0 || cmdArgs[0] == "help" || cmdArgs[0] == "--help" || cmdArgs[0] == "-h" {
|
||||||
|
|
@ -145,9 +182,14 @@ func buildOpsctlCommand(stdout, stderr io.Writer, args []string) *cobra.Command
|
||||||
return &exitCodeError{code: 2, err: fmt.Errorf("unknown command: %s", taskName)}
|
return &exitCodeError{code: 2, err: fmt.Errorf("unknown command: %s", taskName)}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
resolvedCwd, err := resolveWorkingDirectory(resolvedRoot, cwd)
|
||||||
|
if err != nil {
|
||||||
|
return &exitCodeError{code: 2, err: err}
|
||||||
|
}
|
||||||
|
|
||||||
ctx := &app.Context{
|
ctx := &app.Context{
|
||||||
Root: resolvedRoot,
|
Root: resolvedRoot,
|
||||||
Cwd: cwd,
|
Cwd: resolvedCwd,
|
||||||
Format: mode,
|
Format: mode,
|
||||||
Quiet: quiet,
|
Quiet: quiet,
|
||||||
NoColor: noColor,
|
NoColor: noColor,
|
||||||
|
|
@ -177,7 +219,7 @@ func buildOpsctlCommand(stdout, stderr io.Writer, args []string) *cobra.Command
|
||||||
root.PersistentFlags().StringVar(&formatValue, "format", string(format.OutputText), "output mode: text, json, raw")
|
root.PersistentFlags().StringVar(&formatValue, "format", string(format.OutputText), "output mode: text, json, raw")
|
||||||
root.PersistentFlags().BoolVar(&jsonMode, "json", false, "shortcut for --format json")
|
root.PersistentFlags().BoolVar(&jsonMode, "json", false, "shortcut for --format json")
|
||||||
root.PersistentFlags().BoolVar(&rawMode, "raw", false, "shortcut for --format raw")
|
root.PersistentFlags().BoolVar(&rawMode, "raw", false, "shortcut for --format raw")
|
||||||
root.PersistentFlags().BoolVar(&noColor, "no-color", false, "disable color output (reserved)")
|
root.PersistentFlags().BoolVar(&noColor, "no-color", false, "disable color output")
|
||||||
root.PersistentFlags().BoolVar(&quiet, "quiet", false, "suppress non-error output")
|
root.PersistentFlags().BoolVar(&quiet, "quiet", false, "suppress non-error output")
|
||||||
root.PersistentFlags().StringVar(&rootFlag, "root", "", "repository root")
|
root.PersistentFlags().StringVar(&rootFlag, "root", "", "repository root")
|
||||||
root.PersistentFlags().StringVar(&cwd, "cwd", "", "working directory for command execution")
|
root.PersistentFlags().StringVar(&cwd, "cwd", "", "working directory for command execution")
|
||||||
|
|
@ -188,7 +230,11 @@ func buildOpsctlCommand(stdout, stderr io.Writer, args []string) *cobra.Command
|
||||||
_, _ = fmt.Fprintln(cobraCmd.ErrOrStderr(), err)
|
_, _ = fmt.Fprintln(cobraCmd.ErrOrStderr(), err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
rootPath := app.EnsureRoot(rootFlag)
|
rootPath, err := resolveRootPath(rootFlag)
|
||||||
|
if err != nil {
|
||||||
|
_, _ = fmt.Fprintln(cobraCmd.ErrOrStderr(), err)
|
||||||
|
return
|
||||||
|
}
|
||||||
printUsage(cobraCmd.OutOrStdout(), newOpsApp(rootPath), cobraCmd.Root().Name())
|
printUsage(cobraCmd.OutOrStdout(), newOpsApp(rootPath), cobraCmd.Root().Name())
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,10 +3,12 @@ package main
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"os"
|
"os"
|
||||||
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/ZanzyTHEbar/dragonscale/internal/opsctl/format"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -19,6 +21,10 @@ func TestParseOutputMode(t *testing.T) {
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, "raw", string(mode))
|
require.Equal(t, "raw", string(mode))
|
||||||
|
|
||||||
|
mode, err = parseOutputMode(true, true, "text")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, "raw", string(mode))
|
||||||
|
|
||||||
mode, err = parseOutputMode(false, true, "text")
|
mode, err = parseOutputMode(false, true, "text")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, "json", string(mode))
|
require.Equal(t, "json", string(mode))
|
||||||
|
|
@ -27,6 +33,26 @@ func TestParseOutputMode(t *testing.T) {
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseOutputModePriority(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
mode, err := parseOutputMode(true, true, "json")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, format.OutputRaw, mode)
|
||||||
|
|
||||||
|
mode, err = parseOutputMode(false, true, "text")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, format.OutputJSON, mode)
|
||||||
|
|
||||||
|
mode, err = parseOutputMode(true, false, "json")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, format.OutputRaw, mode)
|
||||||
|
|
||||||
|
mode, err = parseOutputMode(false, false, "json")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, format.OutputJSON, mode)
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunPrintsUsageWithoutArgs(t *testing.T) {
|
func TestRunPrintsUsageWithoutArgs(t *testing.T) {
|
||||||
out := &bytes.Buffer{}
|
out := &bytes.Buffer{}
|
||||||
errOut := &bytes.Buffer{}
|
errOut := &bytes.Buffer{}
|
||||||
|
|
@ -54,6 +80,27 @@ func TestRunInvalidFormat(t *testing.T) {
|
||||||
require.Contains(t, errOut.String(), "invalid --format value \"bad\"")
|
require.Contains(t, errOut.String(), "invalid --format value \"bad\"")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunInvalidRoot(t *testing.T) {
|
||||||
|
out := &bytes.Buffer{}
|
||||||
|
errOut := &bytes.Buffer{}
|
||||||
|
root := filepath.Join(t.TempDir(), "missing")
|
||||||
|
code := run([]string{"--root", root, "help"}, out, errOut)
|
||||||
|
require.Equal(t, 2, code)
|
||||||
|
require.Contains(t, errOut.String(), "invalid --root")
|
||||||
|
require.Equal(t, "", out.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunInvalidCwd(t *testing.T) {
|
||||||
|
out := &bytes.Buffer{}
|
||||||
|
errOut := &bytes.Buffer{}
|
||||||
|
root := t.TempDir()
|
||||||
|
missing := filepath.Join(root, "does-not-exist")
|
||||||
|
code := run([]string{"--root", root, "--cwd", missing, "build"}, out, errOut)
|
||||||
|
require.Equal(t, 2, code)
|
||||||
|
require.Contains(t, errOut.String(), "invalid --cwd")
|
||||||
|
require.Equal(t, "", out.String())
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunHelpAlias(t *testing.T) {
|
func TestRunHelpAlias(t *testing.T) {
|
||||||
out := &bytes.Buffer{}
|
out := &bytes.Buffer{}
|
||||||
errOut := &bytes.Buffer{}
|
errOut := &bytes.Buffer{}
|
||||||
|
|
@ -73,6 +120,22 @@ func TestRunHelpIncludesAllAlias(t *testing.T) {
|
||||||
require.Contains(t, out.String(), "build")
|
require.Contains(t, out.String(), "build")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunHelpIncludesGlobalOutputFlags(t *testing.T) {
|
||||||
|
out := &bytes.Buffer{}
|
||||||
|
errOut := &bytes.Buffer{}
|
||||||
|
code := run([]string{"--help"}, out, errOut)
|
||||||
|
require.Equal(t, 0, code)
|
||||||
|
require.Equal(t, "", errOut.String())
|
||||||
|
require.Contains(t, out.String(), "--format")
|
||||||
|
require.Contains(t, out.String(), "--json")
|
||||||
|
require.Contains(t, out.String(), "--raw")
|
||||||
|
require.Contains(t, out.String(), "--no-color")
|
||||||
|
require.Contains(t, out.String(), "--quiet")
|
||||||
|
require.Contains(t, out.String(), "--root")
|
||||||
|
require.Contains(t, out.String(), "--cwd")
|
||||||
|
require.Contains(t, out.String(), "--timeout")
|
||||||
|
}
|
||||||
|
|
||||||
func TestMakefilePhonyTargetsAreAvailableInOpsctl(t *testing.T) {
|
func TestMakefilePhonyTargetsAreAvailableInOpsctl(t *testing.T) {
|
||||||
makeTargets := parseMakefilePhonyTargets(t)
|
makeTargets := parseMakefilePhonyTargets(t)
|
||||||
out := &bytes.Buffer{}
|
out := &bytes.Buffer{}
|
||||||
|
|
@ -87,6 +150,25 @@ func TestMakefilePhonyTargetsAreAvailableInOpsctl(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMakefileAllTargetIsAliasToBuild(t *testing.T) {
|
||||||
|
makefile, err := findProjectMakefile()
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, []string{"build"}, parseMakeTargetPrereqs(t, makefile, "all"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMakeAllTargetRunsOpsctlBuild(t *testing.T) {
|
||||||
|
output := runMakeTargetWithFakeOpsctl(t, "all", nil)
|
||||||
|
require.Contains(t, output, "cmd:build")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMakeEvalBuildForwardsEnvToOpsctl(t *testing.T) {
|
||||||
|
output := runMakeTargetWithFakeOpsctl(t, "eval-build", map[string]string{
|
||||||
|
"DRAGONSCALE_EVAL_CONFIG": "/tmp/eval/config.json",
|
||||||
|
})
|
||||||
|
require.Contains(t, output, "cmd:--no-color --format raw eval-build")
|
||||||
|
require.Contains(t, output, "eval-config:/tmp/eval/config.json")
|
||||||
|
}
|
||||||
|
|
||||||
func parseMakefilePhonyTargets(t *testing.T) []string {
|
func parseMakefilePhonyTargets(t *testing.T) []string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
|
@ -141,6 +223,70 @@ func parseMakefilePhonyTargets(t *testing.T) []string {
|
||||||
return targets
|
return targets
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func parseMakeTargetPrereqs(t *testing.T, makefile, target string) []string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
raw, err := os.ReadFile(makefile)
|
||||||
|
require.NoError(t, err)
|
||||||
|
lines := strings.Split(string(raw), "\n")
|
||||||
|
prefix := target + ":"
|
||||||
|
|
||||||
|
for _, line := range lines {
|
||||||
|
trimmed := strings.TrimSpace(line)
|
||||||
|
if strings.HasPrefix(trimmed, "#") || trimmed == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(trimmed, prefix) {
|
||||||
|
after := strings.TrimSpace(strings.TrimPrefix(trimmed, prefix))
|
||||||
|
if idx := strings.Index(after, "#"); idx != -1 {
|
||||||
|
after = strings.TrimSpace(after[:idx])
|
||||||
|
}
|
||||||
|
if after == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return strings.Fields(after)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("make target %q not found in %s", target, makefile)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func runMakeTargetWithFakeOpsctl(t *testing.T, target string, env map[string]string) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
makefile, err := findProjectMakefile()
|
||||||
|
require.NoError(t, err)
|
||||||
|
makeRoot := filepath.Dir(makefile)
|
||||||
|
|
||||||
|
fakeOpsctl := filepath.Join(t.TempDir(), "opsctl.sh")
|
||||||
|
fakeScript := []byte(`#!/usr/bin/env sh
|
||||||
|
printf 'cmd:%s\n' "$*"
|
||||||
|
printf 'eval-config:%s\n' "${DRAGONSCALE_EVAL_CONFIG-}"
|
||||||
|
`)
|
||||||
|
require.NoError(t, os.WriteFile(fakeOpsctl, fakeScript, 0o755))
|
||||||
|
|
||||||
|
dummyOpsctl := filepath.Join(t.TempDir(), "opsctl")
|
||||||
|
require.NoError(t, os.WriteFile(dummyOpsctl, []byte("#!/usr/bin/env sh\nexit 0\n"), 0o755))
|
||||||
|
|
||||||
|
cmd := exec.Command("make", "-s", "-f", makefile, target)
|
||||||
|
cmd.Dir = makeRoot
|
||||||
|
runEnv := append(
|
||||||
|
os.Environ(),
|
||||||
|
"OPSCTL="+fakeOpsctl,
|
||||||
|
"OPSCTL_BIN="+dummyOpsctl,
|
||||||
|
"OPSCTL_EVAL="+fakeOpsctl,
|
||||||
|
"OPSCTL_EVAL_ARGS=--no-color --format raw",
|
||||||
|
)
|
||||||
|
for key, value := range env {
|
||||||
|
runEnv = append(runEnv, key+"="+value)
|
||||||
|
}
|
||||||
|
cmd.Env = runEnv
|
||||||
|
|
||||||
|
output, err := cmd.CombinedOutput()
|
||||||
|
require.NoError(t, err, "make %s failed: %s", target, string(output))
|
||||||
|
return string(output)
|
||||||
|
}
|
||||||
|
|
||||||
func findProjectMakefile() (string, error) {
|
func findProjectMakefile() (string, error) {
|
||||||
dir, err := os.Getwd()
|
dir, err := os.Getwd()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
package format
|
package format
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
|
@ -55,41 +56,103 @@ func NewTaskResult(name string, cmd string, result runner.CommandResult, started
|
||||||
|
|
||||||
func Render(w io.Writer, mode OutputMode, res TaskResult, quiet bool) error {
|
func Render(w io.Writer, mode OutputMode, res TaskResult, quiet bool) error {
|
||||||
res.Mode = mode
|
res.Mode = mode
|
||||||
|
if quiet && res.Success {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
switch mode {
|
switch mode {
|
||||||
case OutputJSON:
|
case OutputJSON:
|
||||||
|
if quiet {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
enc := json.NewEncoder(w)
|
enc := json.NewEncoder(w)
|
||||||
enc.SetIndent("", " ")
|
enc.SetIndent("", " ")
|
||||||
return enc.Encode(res)
|
return enc.Encode(res)
|
||||||
case OutputRaw:
|
case OutputRaw:
|
||||||
if quiet {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
prefix := statusPrefix(!res.NoColor, res.Success)
|
prefix := statusPrefix(!res.NoColor, res.Success)
|
||||||
_, _ = fmt.Fprintf(w, "%s %s (%dms)\n", prefix, res.Task, res.DurationMS)
|
_, _ = fmt.Fprintf(w, "%s %s (%dms)\n", prefix, res.Task, res.DurationMS)
|
||||||
return nil
|
return nil
|
||||||
default:
|
default:
|
||||||
if quiet {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
status := "ok"
|
status := "ok"
|
||||||
if !res.Success {
|
if !res.Success {
|
||||||
status = "failed"
|
status = "failed"
|
||||||
}
|
}
|
||||||
status = formatStatus(!res.NoColor, res.Success)
|
status = formatStatus(!res.NoColor, res.Success)
|
||||||
_, _ = fmt.Fprintf(w, "%s: %s (exit=%d, duration=%dms)\n", res.Task, status, res.ExitCode, res.DurationMS)
|
_, _ = fmt.Fprintf(w, "%s: %s (exit=%d, duration=%dms)\n", res.Task, status, res.ExitCode, res.DurationMS)
|
||||||
|
if strings.TrimSpace(res.Command) != "" {
|
||||||
|
_, _ = fmt.Fprintf(w, " command: %s\n", normalizeCommand(res.Command))
|
||||||
|
}
|
||||||
if res.Error != "" {
|
if res.Error != "" {
|
||||||
_, _ = fmt.Fprintf(w, " error: %s\n", res.Error)
|
_, _ = fmt.Fprintf(w, " error: %s\n", res.Error)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(res.Stdout) != "" {
|
if strings.TrimSpace(res.Stdout) != "" {
|
||||||
_, _ = fmt.Fprintf(w, " stdout: %s\n", res.Stdout)
|
_, _ = fmt.Fprintf(w, " stdout: %s\n", normalizeStdout(res.Stdout))
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(res.Stderr) != "" {
|
if strings.TrimSpace(res.Stderr) != "" {
|
||||||
_, _ = fmt.Fprintf(w, " stderr: %s\n", res.Stderr)
|
_, _ = fmt.Fprintf(w, " stderr: %s\n", normalizeStdout(res.Stderr))
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func normalizeCommand(raw string) string {
|
||||||
|
raw = strings.TrimSpace(raw)
|
||||||
|
if raw == "" {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
lines := strings.Split(raw, "\n")
|
||||||
|
compact := make([]string, 0, len(lines))
|
||||||
|
for _, line := range lines {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
compact = append(compact, line)
|
||||||
|
}
|
||||||
|
return strings.Join(compact, " ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeStdout(raw string) string {
|
||||||
|
return normalizeOutput(raw, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeOutput(raw string, includeStructured bool) string {
|
||||||
|
raw = strings.TrimRight(raw, "\n")
|
||||||
|
if raw == "" {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
lines := strings.Split(raw, "\n")
|
||||||
|
normalized := make([]string, 0, len(lines))
|
||||||
|
for _, line := range lines {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
normalized = append(normalized, normalizeLine(line, includeStructured))
|
||||||
|
}
|
||||||
|
return strings.Join(normalized, "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeLine(line string, includeStructured bool) string {
|
||||||
|
if !includeStructured {
|
||||||
|
return line
|
||||||
|
}
|
||||||
|
var payload any
|
||||||
|
if err := json.Unmarshal([]byte(line), &payload); err != nil {
|
||||||
|
return line
|
||||||
|
}
|
||||||
|
return normalizeJSONLine(payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeJSONLine(payload any) string {
|
||||||
|
var buf bytes.Buffer
|
||||||
|
enc := json.NewEncoder(&buf)
|
||||||
|
enc.SetIndent("", " ")
|
||||||
|
if err := enc.Encode(payload); err != nil {
|
||||||
|
return fmt.Sprint(payload)
|
||||||
|
}
|
||||||
|
return strings.TrimRight(buf.String(), "\n")
|
||||||
|
}
|
||||||
|
|
||||||
func formatStatus(colored bool, success bool) string {
|
func formatStatus(colored bool, success bool) string {
|
||||||
if !colored {
|
if !colored {
|
||||||
if !success {
|
if !success {
|
||||||
|
|
|
||||||
|
|
@ -48,6 +48,21 @@ func TestRenderTextIncludesDetails(t *testing.T) {
|
||||||
require.NotContains(t, output, "\x1b[")
|
require.NotContains(t, output, "\x1b[")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRenderTextRespectsNoColorFalse(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
result := TaskResult{
|
||||||
|
Task: "test",
|
||||||
|
ExitCode: 0,
|
||||||
|
DurationMS: 10,
|
||||||
|
Success: true,
|
||||||
|
NoColor: false,
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, Render(buf, OutputText, result, false))
|
||||||
|
output := buf.String()
|
||||||
|
require.Contains(t, output, "\x1b[32mok\x1b[0m")
|
||||||
|
}
|
||||||
|
|
||||||
func TestRenderRawHonorQuiet(t *testing.T) {
|
func TestRenderRawHonorQuiet(t *testing.T) {
|
||||||
buf := &bytes.Buffer{}
|
buf := &bytes.Buffer{}
|
||||||
result := TaskResult{Task: "run", ExitCode: 0, DurationMS: 42, Success: true}
|
result := TaskResult{Task: "run", ExitCode: 0, DurationMS: 42, Success: true}
|
||||||
|
|
@ -59,3 +74,39 @@ func TestRenderRawHonorQuiet(t *testing.T) {
|
||||||
require.True(t, strings.Contains(buf.String(), "run"))
|
require.True(t, strings.Contains(buf.String(), "run"))
|
||||||
require.True(t, strings.Contains(buf.String(), "(42ms)"))
|
require.True(t, strings.Contains(buf.String(), "(42ms)"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRenderTextIncludesCommandSummaryAndErrors(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
result := TaskResult{
|
||||||
|
Task: "eval",
|
||||||
|
Command: "if [ -n \"$X\" ]; then\necho ok\nfi",
|
||||||
|
ExitCode: 1,
|
||||||
|
Success: false,
|
||||||
|
DurationMS: 55,
|
||||||
|
Error: "command failed",
|
||||||
|
Stdout: "ok\n",
|
||||||
|
Stderr: `{"outcome":"success"}`,
|
||||||
|
}
|
||||||
|
require.NoError(t, Render(buf, OutputText, result, false))
|
||||||
|
output := buf.String()
|
||||||
|
require.Contains(t, output, "eval")
|
||||||
|
require.Contains(t, output, "command:")
|
||||||
|
require.Contains(t, output, "if [ -n")
|
||||||
|
require.Contains(t, output, "error: command failed")
|
||||||
|
require.Contains(t, output, "stdout: ok")
|
||||||
|
require.Contains(t, output, `"outcome"`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderTextHonorsQuietOnlyForSuccess(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
successResult := TaskResult{Task: "run", ExitCode: 0, DurationMS: 1, Success: true}
|
||||||
|
require.NoError(t, Render(buf, OutputText, successResult, true))
|
||||||
|
require.Equal(t, "", buf.String())
|
||||||
|
|
||||||
|
buf.Reset()
|
||||||
|
failureResult := TaskResult{Task: "run", ExitCode: 1, DurationMS: 1, Success: false, Error: "boom"}
|
||||||
|
require.NoError(t, Render(buf, OutputText, failureResult, true))
|
||||||
|
require.Contains(t, buf.String(), "run")
|
||||||
|
require.Contains(t, buf.String(), "exit=1")
|
||||||
|
require.Contains(t, buf.String(), "error: boom")
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
|
@ -25,6 +26,7 @@ type shellTask struct {
|
||||||
description string
|
description string
|
||||||
script func(*app.Context) string
|
script func(*app.Context) string
|
||||||
env func(*app.Context) []string
|
env func(*app.Context) []string
|
||||||
|
prepare func(*app.Context) error
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t shellTask) Name() string {
|
func (t shellTask) Name() string {
|
||||||
|
|
@ -36,6 +38,17 @@ func (t shellTask) Description() string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t shellTask) Run(ctx context.Context, rnr runner.Runner, c *app.Context) (app.Result, error) {
|
func (t shellTask) Run(ctx context.Context, rnr runner.Runner, c *app.Context) (app.Result, error) {
|
||||||
|
if t.prepare != nil {
|
||||||
|
if err := t.prepare(c); err != nil {
|
||||||
|
return app.Result{
|
||||||
|
Task: t.name,
|
||||||
|
Command: t.script(c),
|
||||||
|
ExitCode: 1,
|
||||||
|
Err: err.Error(),
|
||||||
|
}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
script := t.script(c)
|
script := t.script(c)
|
||||||
fullScript := applyDevcontainerWrapper(script, c)
|
fullScript := applyDevcontainerWrapper(script, c)
|
||||||
workDir := c.Root
|
workDir := c.Root
|
||||||
|
|
@ -54,7 +67,7 @@ func (t shellTask) Run(ctx context.Context, rnr runner.Runner, c *app.Context) (
|
||||||
result, err := rnr.Run(ctx, spec)
|
result, err := rnr.Run(ctx, spec)
|
||||||
res := app.Result{
|
res := app.Result{
|
||||||
Task: t.name,
|
Task: t.name,
|
||||||
Command: script,
|
Command: fullScript,
|
||||||
Stdout: result.Stdout,
|
Stdout: result.Stdout,
|
||||||
Stderr: result.Stderr,
|
Stderr: result.Stderr,
|
||||||
Err: func() string {
|
Err: func() string {
|
||||||
|
|
@ -71,6 +84,117 @@ func (t shellTask) Run(ctx context.Context, rnr runner.Runner, c *app.Context) (
|
||||||
return res, err
|
return res, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type commandTask struct {
|
||||||
|
name string
|
||||||
|
description string
|
||||||
|
specs func(*app.Context) []runner.CommandSpec
|
||||||
|
env func(*app.Context) []string
|
||||||
|
prepare func(*app.Context) error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t commandTask) Name() string {
|
||||||
|
return t.name
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t commandTask) Description() string {
|
||||||
|
return t.description
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t commandTask) Run(ctx context.Context, rnr runner.Runner, c *app.Context) (app.Result, error) {
|
||||||
|
if t.prepare != nil {
|
||||||
|
if err := t.prepare(c); err != nil {
|
||||||
|
return app.Result{
|
||||||
|
Task: t.name,
|
||||||
|
ExitCode: 1,
|
||||||
|
Err: err.Error(),
|
||||||
|
}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
specs := t.specs(c)
|
||||||
|
workDir := c.Root
|
||||||
|
if c.Cwd != "" {
|
||||||
|
workDir = c.Cwd
|
||||||
|
}
|
||||||
|
baseEnv := mergeEnv(c, t.env)
|
||||||
|
|
||||||
|
result := app.Result{
|
||||||
|
Task: t.name,
|
||||||
|
}
|
||||||
|
commandParts := make([]string, 0, len(specs))
|
||||||
|
for i, spec := range specs {
|
||||||
|
spec.InheritOutput = c.Format == format.OutputRaw
|
||||||
|
if spec.Dir == "" {
|
||||||
|
spec.Dir = workDir
|
||||||
|
}
|
||||||
|
spec.Env = mergeEnvPairs(baseEnv, spec.Env)
|
||||||
|
|
||||||
|
commandParts = append(commandParts, describeCommandSpec(spec))
|
||||||
|
commandResult, err := rnr.Run(ctx, spec)
|
||||||
|
result.ExitCode = commandResult.ExitCode
|
||||||
|
result.Stdout += commandResult.Stdout
|
||||||
|
result.Stderr += commandResult.Stderr
|
||||||
|
if i == 0 {
|
||||||
|
result.Command = commandParts[i]
|
||||||
|
} else {
|
||||||
|
result.Command += " && " + commandParts[i]
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
result.Err = err.Error()
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func describeCommandSpec(spec runner.CommandSpec) string {
|
||||||
|
if len(spec.Args) == 0 {
|
||||||
|
return spec.Name
|
||||||
|
}
|
||||||
|
quotedArgs := make([]string, 0, len(spec.Args))
|
||||||
|
for _, arg := range spec.Args {
|
||||||
|
quotedArgs = append(quotedArgs, strconv.Quote(arg))
|
||||||
|
}
|
||||||
|
return spec.Name + " " + strings.Join(quotedArgs, " ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeEnvPairs(base []string, extra []string) []string {
|
||||||
|
seen := make(map[string]int)
|
||||||
|
values := make(map[string]string)
|
||||||
|
keys := make([]string, 0, len(base)+len(extra))
|
||||||
|
|
||||||
|
merge := func(items []string) {
|
||||||
|
for _, item := range items {
|
||||||
|
eq := strings.Index(item, "=")
|
||||||
|
if eq <= 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key := item[:eq]
|
||||||
|
value := item[eq+1:]
|
||||||
|
if _, ok := seen[key]; !ok {
|
||||||
|
keys = append(keys, key)
|
||||||
|
seen[key] = len(keys) - 1
|
||||||
|
}
|
||||||
|
values[key] = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
merge(base)
|
||||||
|
merge(extra)
|
||||||
|
|
||||||
|
env := make([]string, 0, len(keys))
|
||||||
|
for _, key := range keys {
|
||||||
|
value := values[key]
|
||||||
|
if strings.TrimSpace(value) == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
env = append(env, fmt.Sprintf("%s=%s", key, value))
|
||||||
|
}
|
||||||
|
return env
|
||||||
|
}
|
||||||
|
|
||||||
type helpTask struct {
|
type helpTask struct {
|
||||||
app *app.App
|
app *app.App
|
||||||
}
|
}
|
||||||
|
|
@ -134,12 +258,20 @@ func NewShellTask(name, description string, script func(*app.Context) string, en
|
||||||
return shellTask{name: name, description: description, script: script, env: env}
|
return shellTask{name: name, description: description, script: script, env: env}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func NewCommandTask(name, description string, specs func(*app.Context) []runner.CommandSpec, env func(*app.Context) []string, prepare func(*app.Context) error) commandTask {
|
||||||
|
return commandTask{name: name, description: description, specs: specs, env: env, prepare: prepare}
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewShellTaskWithPrepare(name, description string, script func(*app.Context) string, env func(*app.Context) []string, prepare func(*app.Context) error) shellTask {
|
||||||
|
return shellTask{name: name, description: description, script: script, env: env, prepare: prepare}
|
||||||
|
}
|
||||||
|
|
||||||
func NewRegistry(_ string) []app.Task {
|
func NewRegistry(_ string) []app.Task {
|
||||||
tasks := []app.Task{
|
tasks := []app.Task{
|
||||||
NewShellTask("all", "Build the dragonscale binary for current platform", buildScript, nil),
|
NewShellTask("all", "Build the dragonscale binary for current platform", buildScript, nil),
|
||||||
NewShellTask("generate", "Run generate", generateScript, nil),
|
NewShellTask("generate", "Run generate", generateScript, nil),
|
||||||
NewShellTask("build", "Build the dragonscale binary for current platform", buildScript, nil),
|
NewShellTaskWithPrepare("build", "Build the dragonscale binary for current platform", buildScript, nil, validateBuildTaskEnv),
|
||||||
NewShellTask("build-all", "Build dragonscale for all platforms", buildAllScript, nil),
|
NewShellTaskWithPrepare("build-all", "Build dragonscale for linux target with CGO", buildAllScript, nil, validateBuildTaskEnv),
|
||||||
NewShellTask("install", "Install dragonscale to system and copy builtin skills", installScript, nil),
|
NewShellTask("install", "Install dragonscale to system and copy builtin skills", installScript, nil),
|
||||||
NewShellTask("uninstall", "Uninstall dragonscale from system", uninstallScript, nil),
|
NewShellTask("uninstall", "Uninstall dragonscale from system", uninstallScript, nil),
|
||||||
NewShellTask("uninstall-all", "Uninstall dragonscale and workspace data", uninstallAllScript, nil),
|
NewShellTask("uninstall-all", "Uninstall dragonscale and workspace data", uninstallAllScript, nil),
|
||||||
|
|
@ -147,7 +279,7 @@ func NewRegistry(_ string) []app.Task {
|
||||||
NewShellTask("vet", "Run go vet for static analysis", staticGoScript("vet ./..."), nil),
|
NewShellTask("vet", "Run go vet for static analysis", staticGoScript("vet ./..."), nil),
|
||||||
NewShellTask("test", "Run tests", staticGoScript("test ./..."), nil),
|
NewShellTask("test", "Run tests", staticGoScript("test ./..."), nil),
|
||||||
NewShellTask("fmt", "Format Go code", staticGoScript("fmt ./..."), nil),
|
NewShellTask("fmt", "Format Go code", staticGoScript("fmt ./..."), nil),
|
||||||
NewShellTask("lint", "Run all linting checks", lintScript, nil),
|
NewCommandTask("lint", "Run all linting checks", lintSpecs, nil, nil),
|
||||||
NewShellTask("hooks", "Install git hooks", hooksScript, nil),
|
NewShellTask("hooks", "Install git hooks", hooksScript, nil),
|
||||||
NewShellTask("deps", "Download dependencies", staticGoScript("mod download && mod verify"), nil),
|
NewShellTask("deps", "Download dependencies", staticGoScript("mod download && mod verify"), nil),
|
||||||
NewShellTask("update-deps", "Update dependencies", staticGoScript("get -u ./... && mod tidy"), nil),
|
NewShellTask("update-deps", "Update dependencies", staticGoScript("get -u ./... && mod tidy"), nil),
|
||||||
|
|
@ -159,16 +291,16 @@ func NewRegistry(_ string) []app.Task {
|
||||||
NewShellTask("fantasy-sync", "Full sync of vendored Fantasy SDK", fantasySyncScript, nil),
|
NewShellTask("fantasy-sync", "Full sync of vendored Fantasy SDK", fantasySyncScript, nil),
|
||||||
NewShellTask("fantasy-patch", "Save local modifications as a patch", fantasyPatchScript, nil),
|
NewShellTask("fantasy-patch", "Save local modifications as a patch", fantasyPatchScript, nil),
|
||||||
NewShellTask("test-integration", "Run integration tests", staticGoScript("-tags integration -count=1 -timeout 120s -v ./pkg/memory/..."), nil),
|
NewShellTask("test-integration", "Run integration tests", staticGoScript("-tags integration -count=1 -timeout 120s -v ./pkg/memory/..."), nil),
|
||||||
NewShellTask("check", "Run deps, formatting, linting and tests", checkScript, nil),
|
NewCommandTask("check", "Run deps, formatting, linting and tests", checkSpecs, nil, nil),
|
||||||
NewShellTask("run", "Build and run dragonscale", runScript, runTaskEnv),
|
NewShellTask("run", "Build and run dragonscale", runScript, runTaskEnv),
|
||||||
NewShellTask("devcontainer-build", "Build the local devcontainer image via npx", devcontainerBuildScript, nil),
|
NewCommandTask("devcontainer-build", "Build the local devcontainer image via npx", devcontainerBuildSpecs, nil, nil),
|
||||||
NewShellTask("devcontainer-up", "Start/update the local devcontainer", devcontainerUpScript, nil),
|
NewCommandTask("devcontainer-up", "Start/update the local devcontainer", devcontainerUpSpecs, nil, nil),
|
||||||
NewShellTask("devcontainer-generate", "Run generation in devcontainer", devcontainerGenerateScript, nil),
|
NewCommandTask("devcontainer-generate", "Run generation in devcontainer", devcontainerGenerateSpecs, nil, nil),
|
||||||
NewShellTask("devcontainer-verify", "Verify generated code in devcontainer", devcontainerVerifyScript, nil),
|
NewCommandTask("devcontainer-verify", "Verify generated code in devcontainer", devcontainerVerifySpecs, nil, nil),
|
||||||
NewShellTask("eval-build", "Build the eval runner from current branch", evalBuildScript, nil),
|
NewCommandTask("eval-build", "Build the eval runner from current branch", evalBuildSpecs, nil, nil),
|
||||||
NewShellTask("eval", "Run the eval suite", evalRunScript, nil),
|
NewCommandTask("eval", "Run the eval suite", evalRunSpecs, nil, nil),
|
||||||
NewShellTask("eval-fixtures", "Prepare eval fixture workspace", evalFixturesScript, nil),
|
NewCommandTask("eval-fixtures", "Prepare eval fixture workspace", evalFixturesSpecs, nil, nil),
|
||||||
NewShellTask("eval-view", "Open the promptfoo results viewer", evalViewScript, nil),
|
NewCommandTask("eval-view", "Open the promptfoo results viewer", evalViewSpecs, nil, nil),
|
||||||
NewShellTask("eval-clean", "Cleanup eval artifacts", simpleScript("rm -rf eval/results eval/bin"), nil),
|
NewShellTask("eval-clean", "Cleanup eval artifacts", simpleScript("rm -rf eval/results eval/bin"), nil),
|
||||||
NewShellTask("eval-compare", "Run A/B comparison of current branch vs main", simpleScript("cd eval && DEVCONTAINER_EXEC= EVAL_NPM_CMD=$(npx --yes) ./scripts/compare.sh --repeat 3"), nil),
|
NewShellTask("eval-compare", "Run A/B comparison of current branch vs main", simpleScript("cd eval && DEVCONTAINER_EXEC= EVAL_NPM_CMD=$(npx --yes) ./scripts/compare.sh --repeat 3"), nil),
|
||||||
NewShellTask("eval-test", "Run Go-native component evals", staticGoScript("-v ./eval/go_evals/..."), nil),
|
NewShellTask("eval-test", "Run Go-native component evals", staticGoScript("-v ./eval/go_evals/..."), nil),
|
||||||
|
|
@ -177,64 +309,52 @@ func NewRegistry(_ string) []app.Task {
|
||||||
}
|
}
|
||||||
|
|
||||||
func defaultEnv(c *app.Context) []string {
|
func defaultEnv(c *app.Context) []string {
|
||||||
env := []string{
|
pairs := map[string]string{
|
||||||
fmt.Sprintf("GO=%s", cEnv(c, "GO", "go")),
|
"GO": cEnv(c, "GO", "go"),
|
||||||
fmt.Sprintf("GOFLAGS=%s", cEnv(c, "GOFLAGS", "-v -trimpath -tags stdjson")),
|
"GOFLAGS": cEnv(c, "GOFLAGS", "-v -trimpath -tags=stdjson"),
|
||||||
fmt.Sprintf("CGO_ENABLED=%s", cEnv(c, "CGO_ENABLED", "1")),
|
"CGO_ENABLED": cEnv(c, "CGO_ENABLED", "1"),
|
||||||
fmt.Sprintf("BINARY_NAME=%s", cEnv(c, "BINARY_NAME", defaultBinaryName)),
|
"BINARY_NAME": cEnv(c, "BINARY_NAME", defaultBinaryName),
|
||||||
fmt.Sprintf("BUILD_DIR=%s", cEnv(c, "BUILD_DIR", defaultBuildDir)),
|
"BUILD_DIR": cEnv(c, "BUILD_DIR", defaultBuildDir),
|
||||||
fmt.Sprintf("CMD_DIR=%s", cEnv(c, "CMD_DIR", defaultCmdDir)),
|
"CMD_DIR": cEnv(c, "CMD_DIR", defaultCmdDir),
|
||||||
fmt.Sprintf("WORKSPACE_DIR=%s", cEnv(c, "WORKSPACE_DIR", filepath.Join(homeDir(), ".dragonscale", "workspace"))),
|
"WORKSPACE_DIR": cEnv(c, "WORKSPACE_DIR", filepath.Join(homeDir(), ".dragonscale", "workspace")),
|
||||||
|
"GOOS": cEnv(c, "GOOS", "linux"),
|
||||||
|
"GOARCH": cEnv(c, "GOARCH", runtime.GOARCH),
|
||||||
}
|
}
|
||||||
if value := cEnv(c, "GOOS", ""); value != "" {
|
appendIfSet := func(key, value string) {
|
||||||
env = append(env, fmt.Sprintf("GOOS=%s", value))
|
if strings.TrimSpace(value) != "" {
|
||||||
}
|
pairs[key] = value
|
||||||
if value := cEnv(c, "GOARCH", ""); value != "" {
|
}
|
||||||
env = append(env, fmt.Sprintf("GOARCH=%s", value))
|
|
||||||
}
|
|
||||||
if value := cEnv(c, "PLATFORM", ""); value != "" {
|
|
||||||
env = append(env, fmt.Sprintf("PLATFORM=%s", value))
|
|
||||||
}
|
|
||||||
if value := cEnv(c, "ARCH", ""); value != "" {
|
|
||||||
env = append(env, fmt.Sprintf("ARCH=%s", value))
|
|
||||||
}
|
|
||||||
if v := cEnv(c, "DEVCONTAINER_EXEC", ""); v != "" {
|
|
||||||
env = append(env, "DEVCONTAINER_EXEC="+v)
|
|
||||||
}
|
|
||||||
if value := cEnv(c, "DRAGONSCALE_EVAL_HOST_HOME", ""); value != "" {
|
|
||||||
env = append(env, fmt.Sprintf("DRAGONSCALE_EVAL_HOST_HOME=%s", value))
|
|
||||||
}
|
|
||||||
if value := cEnv(c, "DRAGONSCALE_EVAL_BASE_CONFIG", ""); value != "" {
|
|
||||||
env = append(env, fmt.Sprintf("DRAGONSCALE_EVAL_BASE_CONFIG=%s", value))
|
|
||||||
}
|
|
||||||
if value := cEnv(c, "DRAGONSCALE_EVAL_CONFIG", ""); value != "" {
|
|
||||||
env = append(env, fmt.Sprintf("DRAGONSCALE_EVAL_CONFIG=%s", value))
|
|
||||||
}
|
|
||||||
if value := cEnv(c, "DRAGONSCALE_EVAL_DEBUG", ""); value != "" {
|
|
||||||
env = append(env, fmt.Sprintf("DRAGONSCALE_EVAL_DEBUG=%s", value))
|
|
||||||
}
|
|
||||||
if value := cEnv(c, "VERSION", ""); value != "" {
|
|
||||||
env = append(env, fmt.Sprintf("VERSION=%s", value))
|
|
||||||
}
|
|
||||||
if value := cEnv(c, "FANTASY_VERSION", ""); value != "" {
|
|
||||||
env = append(env, fmt.Sprintf("FANTASY_VERSION=%s", value))
|
|
||||||
}
|
|
||||||
if value := cEnv(c, "NAME", ""); value != "" {
|
|
||||||
env = append(env, fmt.Sprintf("NAME=%s", value))
|
|
||||||
}
|
|
||||||
if value := cEnv(c, "ARGS", ""); value != "" {
|
|
||||||
env = append(env, fmt.Sprintf("ARGS=%s", value))
|
|
||||||
}
|
}
|
||||||
|
appendIfSet("PLATFORM", cEnv(c, "PLATFORM", ""))
|
||||||
|
appendIfSet("ARCH", cEnv(c, "ARCH", ""))
|
||||||
|
appendIfSet("DEVCONTAINER_EXEC", cEnv(c, "DEVCONTAINER_EXEC", ""))
|
||||||
|
appendIfSet("DRAGONSCALE_EVAL_HOST_HOME", cEnv(c, "DRAGONSCALE_EVAL_HOST_HOME", ""))
|
||||||
|
appendIfSet("DRAGONSCALE_EVAL_BASE_CONFIG", cEnv(c, "DRAGONSCALE_EVAL_BASE_CONFIG", ""))
|
||||||
|
appendIfSet("DRAGONSCALE_EVAL_CONFIG", cEnv(c, "DRAGONSCALE_EVAL_CONFIG", ""))
|
||||||
|
appendIfSet("DRAGONSCALE_EVAL_DEBUG", cEnv(c, "DRAGONSCALE_EVAL_DEBUG", ""))
|
||||||
|
appendIfSet("VERSION", cEnv(c, "VERSION", ""))
|
||||||
|
appendIfSet("FANTASY_VERSION", cEnv(c, "FANTASY_VERSION", ""))
|
||||||
|
appendIfSet("NAME", cEnv(c, "NAME", ""))
|
||||||
|
appendIfSet("ARGS", cEnv(c, "ARGS", ""))
|
||||||
if c.Root != "" {
|
if c.Root != "" {
|
||||||
env = append(env, fmt.Sprintf("DRAGONSCALE_HOME=%s", filepath.Join(homeDir(), ".dragonscale")))
|
appendIfSet("DRAGONSCALE_HOME", filepath.Join(homeDir(), ".dragonscale"))
|
||||||
|
}
|
||||||
|
env := make([]string, 0, len(pairs))
|
||||||
|
for key, value := range pairs {
|
||||||
|
if key == "" || value == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
env = append(env, fmt.Sprintf("%s=%s", key, value))
|
||||||
}
|
}
|
||||||
return env
|
return env
|
||||||
}
|
}
|
||||||
|
|
||||||
func cEnv(c *app.Context, key string, fallback string) string {
|
func cEnv(c *app.Context, key string, fallback string) string {
|
||||||
if v, ok := c.ExtraEnv[key]; ok {
|
if c != nil && c.ExtraEnv != nil {
|
||||||
if strings.TrimSpace(v) != "" {
|
if v, ok := c.ExtraEnv[key]; ok {
|
||||||
return v
|
if strings.TrimSpace(v) != "" {
|
||||||
|
return v
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return fallback
|
return fallback
|
||||||
|
|
@ -270,15 +390,11 @@ func applyDevcontainerWrapper(script string, c *app.Context) string {
|
||||||
return script
|
return script
|
||||||
}
|
}
|
||||||
|
|
||||||
workDir := c.Root
|
return fmt.Sprintf("%s -- bash -lc %s", execCmd, shellSingleQuote(script))
|
||||||
if c.Cwd != "" {
|
}
|
||||||
workDir = c.Cwd
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(workDir) != "" {
|
|
||||||
script = fmt.Sprintf("cd %s\n%s", strconv.Quote(workDir), script)
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Sprintf("%s -- bash -lc %s", execCmd, strconv.Quote(script))
|
func shellSingleQuote(input string) string {
|
||||||
|
return "'" + strings.ReplaceAll(input, "'", `'"'"'`) + "'"
|
||||||
}
|
}
|
||||||
|
|
||||||
func detectDevcontainerExec(root string) string {
|
func detectDevcontainerExec(root string) string {
|
||||||
|
|
@ -331,61 +447,116 @@ func staticGoScript(rest string) func(*app.Context) string {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func generateScript(*app.Context) string {
|
func generateScript(c *app.Context) string {
|
||||||
|
cmdDir := cEnv(c, "CMD_DIR", defaultCmdDir)
|
||||||
return scriptHeader() +
|
return scriptHeader() +
|
||||||
"rm -rf ./$(CMD_DIR)/workspace 2>/dev/null || true\n" +
|
fmt.Sprintf("rm -rf ./%s/workspace 2>/dev/null || true\n", cmdDir) +
|
||||||
"$GO generate ./...\n"
|
"$GO generate ./...\n"
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildScript(*app.Context) string {
|
func buildScript(c *app.Context) string {
|
||||||
version := "$(git describe --tags --always --dirty 2>/dev/null || echo \"dev\")"
|
version := "$(git describe --tags --always --dirty 2>/dev/null || echo \"dev\")"
|
||||||
gitCommit := "$(git rev-parse --short=8 HEAD 2>/dev/null || echo \"dev\")"
|
gitCommit := "$(git rev-parse --short=8 HEAD 2>/dev/null || echo \"dev\")"
|
||||||
buildTime := "$(date +%FT%T%z)"
|
buildTime := "$(date +%FT%T%z)"
|
||||||
goVersion := "$($GO version | awk '{print \"$3\"}')"
|
goVersion := "$($GO version | awk '{print $3}')"
|
||||||
|
buildDir := cEnv(c, "BUILD_DIR", defaultBuildDir)
|
||||||
|
binaryName := cEnv(c, "BINARY_NAME", defaultBinaryName)
|
||||||
|
cmdDir := cEnv(c, "CMD_DIR", defaultCmdDir)
|
||||||
|
targetGOOS := cEnv(c, "GOOS", "linux")
|
||||||
|
targetGOARCH := cEnv(c, "GOARCH", runtime.GOARCH)
|
||||||
ldFlags := fmt.Sprintf("-X main.version=%s -X main.gitCommit=%s -X main.buildTime=%s -X main.goVersion=%s -s -w", version, gitCommit, buildTime, goVersion)
|
ldFlags := fmt.Sprintf("-X main.version=%s -X main.gitCommit=%s -X main.buildTime=%s -X main.goVersion=%s -s -w", version, gitCommit, buildTime, goVersion)
|
||||||
return scriptHeader() +
|
return scriptHeader() +
|
||||||
"mkdir -p $(BUILD_DIR)\n" +
|
fmt.Sprintf("mkdir -p %s\n", buildDir) +
|
||||||
"GOOS=$($GO env GOOS)\n" +
|
fmt.Sprintf("echo \"build: target=%s/%s/%s\"\n", targetGOOS, targetGOARCH, binaryName) +
|
||||||
"GOARCH=$($GO env GOARCH)\n" +
|
fmt.Sprintf("GOOS=$GOOS GOARCH=$GOARCH CGO_ENABLED=$CGO_ENABLED $GO build $GOFLAGS -ldflags \"%s\" -o %s/%s-${GOOS}-${GOARCH} ./%s\n", ldFlags, buildDir, binaryName, cmdDir) +
|
||||||
fmt.Sprintf("$GO build $GOFLAGS -ldflags \"%s\" -o $(BUILD_DIR)/$(BINARY_NAME)-${GOOS}-${GOARCH} ./$(CMD_DIR)\n", ldFlags) +
|
fmt.Sprintf("ln -sf %s-${GOOS}-${GOARCH} %s/%s\n", binaryName, buildDir, binaryName)
|
||||||
"ln -sf $(BINARY_NAME)-$(GOOS)-$(GOARCH) $(BUILD_DIR)/$(BINARY_NAME)\n"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildAllScript(*app.Context) string {
|
func buildAllScript(c *app.Context) string {
|
||||||
ldFlags := "-ldflags '-s -w'"
|
ldFlags := "-ldflags '-s -w'"
|
||||||
return scriptHeader() +
|
buildDir := cEnv(c, "BUILD_DIR", defaultBuildDir)
|
||||||
"mkdir -p $(BUILD_DIR)\n" +
|
binaryName := cEnv(c, "BINARY_NAME", defaultBinaryName)
|
||||||
fmt.Sprintf("GOOS=linux GOARCH=amd64 $GO build $GOFLAGS %s -o $(BUILD_DIR)/$(BINARY_NAME)-linux-amd64 ./$(CMD_DIR)\n", ldFlags) +
|
cmdDir := cEnv(c, "CMD_DIR", defaultCmdDir)
|
||||||
fmt.Sprintf("GOOS=linux GOARCH=arm64 $GO build $GOFLAGS %s -o $(BUILD_DIR)/$(BINARY_NAME)-linux-arm64 ./$(CMD_DIR)\n", ldFlags) +
|
targetGOOS := cEnv(c, "GOOS", "linux")
|
||||||
fmt.Sprintf("GOOS=linux GOARCH=loong64 $GO build $GOFLAGS %s -o $(BUILD_DIR)/$(BINARY_NAME)-linux-loong64 ./$(CMD_DIR)\n", ldFlags) +
|
targetGOARCH := cEnv(c, "GOARCH", runtime.GOARCH)
|
||||||
fmt.Sprintf("GOOS=linux GOARCH=riscv64 $GO build $GOFLAGS %s -o $(BUILD_DIR)/$(BINARY_NAME)-linux-riscv64 ./$(CMD_DIR)\n", ldFlags) +
|
var script strings.Builder
|
||||||
fmt.Sprintf("GOOS=darwin GOARCH=arm64 $GO build $GOFLAGS %s -o $(BUILD_DIR)/$(BINARY_NAME)-darwin-arm64 ./$(CMD_DIR)\n", ldFlags) +
|
script.WriteString(scriptHeader())
|
||||||
fmt.Sprintf("GOOS=windows GOARCH=amd64 $GO build $GOFLAGS %s -o $(BUILD_DIR)/$(BINARY_NAME)-windows-amd64.exe ./$(CMD_DIR)\n", ldFlags)
|
fmt.Fprintf(&script, "mkdir -p %s\n", buildDir)
|
||||||
|
fmt.Fprintf(&script, "echo \"build-all: target=%s/%s output=%s/%s-%s-%s\"\n", targetGOOS, targetGOARCH, buildDir, binaryName, targetGOOS, targetGOARCH)
|
||||||
|
fmt.Fprintf(&script, "OUTPUT_NAME=%s/%s-%s-%s\n", buildDir, binaryName, targetGOOS, targetGOARCH)
|
||||||
|
script.WriteString("export CGO_ENABLED=1\n")
|
||||||
|
fmt.Fprintf(&script, "GOOS=$GOOS GOARCH=$GOARCH CGO_ENABLED=$CGO_ENABLED $GO build $GOFLAGS %s -o ${OUTPUT_NAME} ./%s\n", ldFlags, cmdDir)
|
||||||
|
fmt.Fprintf(&script, "ln -sf ./%s %s/%s\n", "${OUTPUT_NAME}", buildDir, binaryName)
|
||||||
|
return script.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateBuildTaskEnv(c *app.Context) error {
|
||||||
|
targetGOOS := cEnv(c, "GOOS", "linux")
|
||||||
|
if targetGOOS != "linux" {
|
||||||
|
return fmt.Errorf("unsupported GOOS=%s: this project only supports linux/gnu builds", targetGOOS)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cgoEnabled := strings.TrimSpace(cEnv(c, "CGO_ENABLED", "1")); cgoEnabled != "" && cgoEnabled != "1" {
|
||||||
|
return fmt.Errorf("unsupported CGO_ENABLED=%s: builds require CGO_ENABLED=1", cgoEnabled)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := detectGlibc(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var detectGlibc = detectGlibcAvailable
|
||||||
|
|
||||||
|
func detectGlibcAvailable() error {
|
||||||
|
output, err := exec.Command("ldd", "--version").Output()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("unable to verify libc: %w", err)
|
||||||
|
}
|
||||||
|
header := strings.ToLower(strings.SplitN(string(output), "\n", 2)[0])
|
||||||
|
if strings.Contains(header, "glibc") || strings.Contains(header, "gnu c library") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("unsupported libc: this project requires glibc")
|
||||||
}
|
}
|
||||||
|
|
||||||
func installScript(*app.Context) string {
|
func installScript(*app.Context) string {
|
||||||
prefix := filepath.Join(homeDir(), ".local")
|
prefix := filepath.Join(homeDir(), ".local")
|
||||||
|
buildDir := cEnv(nil, "BUILD_DIR", defaultBuildDir)
|
||||||
|
binaryName := cEnv(nil, "BINARY_NAME", defaultBinaryName)
|
||||||
return scriptHeader() +
|
return scriptHeader() +
|
||||||
"mkdir -p ${INSTALL_PREFIX:-" + prefix + "}/bin\n" +
|
"mkdir -p ${INSTALL_PREFIX:-" + prefix + "}/bin\n" +
|
||||||
"cp $(BUILD_DIR)/$(BINARY_NAME) ${INSTALL_PREFIX:-" + prefix + "}/bin/$(BINARY_NAME)\n" +
|
fmt.Sprintf("cp %s/%s ${INSTALL_PREFIX:-%s}/bin/%s\n", buildDir, binaryName, prefix, binaryName) +
|
||||||
"chmod +x ${INSTALL_PREFIX:-" + prefix + "}/bin/$(BINARY_NAME)\n"
|
fmt.Sprintf("chmod +x ${INSTALL_PREFIX:-%s}/bin/%s\n", prefix, binaryName)
|
||||||
}
|
}
|
||||||
|
|
||||||
func uninstallScript(*app.Context) string {
|
func uninstallScript(*app.Context) string {
|
||||||
return "rm -f ${INSTALL_PREFIX:-" + filepath.Join(homeDir(), ".local") + "}/bin/$(BINARY_NAME)\n"
|
binaryName := cEnv(nil, "BINARY_NAME", defaultBinaryName)
|
||||||
|
prefix := filepath.Join(homeDir(), ".local")
|
||||||
|
return "rm -f ${INSTALL_PREFIX:-" + prefix + "}/bin/" + binaryName + "\n"
|
||||||
}
|
}
|
||||||
|
|
||||||
func uninstallAllScript(*app.Context) string {
|
func uninstallAllScript(c *app.Context) string {
|
||||||
return "rm -rf $(DRAGONSCALE_HOME)\n"
|
dHome := filepath.Join(homeDir(), ".dragonscale")
|
||||||
|
if v := cEnv(c, "DRAGONSCALE_HOME", ""); v != "" {
|
||||||
|
dHome = v
|
||||||
|
}
|
||||||
|
return "rm -rf " + dHome + "\n"
|
||||||
}
|
}
|
||||||
|
|
||||||
func cleanScript(*app.Context) string {
|
func cleanScript(*app.Context) string {
|
||||||
return "rm -rf $(BUILD_DIR)\n"
|
buildDir := cEnv(nil, "BUILD_DIR", defaultBuildDir)
|
||||||
|
return "rm -rf " + buildDir + "\n"
|
||||||
}
|
}
|
||||||
|
|
||||||
func lintScript(*app.Context) string {
|
func lintSpecs(c *app.Context) []runner.CommandSpec {
|
||||||
return scriptHeader() +
|
goBinary := cEnv(c, "GO", "go")
|
||||||
"$GO fmt ./...\n$GO vet ./...\n$GO build ./...\n"
|
return []runner.CommandSpec{
|
||||||
|
{Name: goBinary, Args: []string{"fmt", "./..."}},
|
||||||
|
{Name: goBinary, Args: []string{"vet", "./..."}},
|
||||||
|
{Name: goBinary, Args: []string{"build", "./..."}},
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func hooksScript(*app.Context) string {
|
func hooksScript(*app.Context) string {
|
||||||
|
|
@ -431,11 +602,15 @@ func fantasyPatchScript(c *app.Context) string {
|
||||||
return simpleScript("./scripts/sync-fantasy.sh --save-patch " + cEnv(c, "NAME", ""))(c)
|
return simpleScript("./scripts/sync-fantasy.sh --save-patch " + cEnv(c, "NAME", ""))(c)
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkScript(*app.Context) string {
|
func checkSpecs(c *app.Context) []runner.CommandSpec {
|
||||||
return staticGoScript("mod download && mod verify")(&app.Context{}) +
|
goBinary := cEnv(c, "GO", "go")
|
||||||
staticGoScript("fmt ./...")(&app.Context{}) +
|
return []runner.CommandSpec{
|
||||||
staticGoScript("vet ./...")(&app.Context{}) +
|
{Name: goBinary, Args: []string{"mod", "download"}},
|
||||||
staticGoScript("test ./...")(&app.Context{})
|
{Name: goBinary, Args: []string{"mod", "verify"}},
|
||||||
|
{Name: goBinary, Args: []string{"fmt", "./..."}},
|
||||||
|
{Name: goBinary, Args: []string{"vet", "./..."}},
|
||||||
|
{Name: goBinary, Args: []string{"test", "./..."}},
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func runTaskEnv(c *app.Context) []string {
|
func runTaskEnv(c *app.Context) []string {
|
||||||
|
|
@ -446,7 +621,9 @@ func runTaskEnv(c *app.Context) []string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func runScript(c *app.Context) string {
|
func runScript(c *app.Context) string {
|
||||||
return scriptHeader() + "$(BUILD_DIR)/$(BINARY_NAME) " + strings.Join(quoteArgs(c.Argv), " ") + "\n"
|
buildDir := cEnv(c, "BUILD_DIR", defaultBuildDir)
|
||||||
|
binaryName := cEnv(c, "BINARY_NAME", defaultBinaryName)
|
||||||
|
return scriptHeader() + buildDir + "/" + binaryName + " " + strings.Join(quoteArgs(c.Argv), " ") + "\n"
|
||||||
}
|
}
|
||||||
|
|
||||||
func quoteArgs(args []string) []string {
|
func quoteArgs(args []string) []string {
|
||||||
|
|
@ -457,52 +634,173 @@ func quoteArgs(args []string) []string {
|
||||||
return quoted
|
return quoted
|
||||||
}
|
}
|
||||||
|
|
||||||
func devcontainerBuildScript(*app.Context) string {
|
func devcontainerRoot(c *app.Context) string {
|
||||||
return "npx --yes @devcontainers/cli build --workspace-folder \"$(pwd)\"\n"
|
if c != nil && c.Root != "" {
|
||||||
|
return c.Root
|
||||||
|
}
|
||||||
|
workspace := cValue(c, "DEVCONTAINER_WORKSPACE", ".")
|
||||||
|
if workspace == "" {
|
||||||
|
return "."
|
||||||
|
}
|
||||||
|
return workspace
|
||||||
}
|
}
|
||||||
|
|
||||||
func devcontainerUpScript(*app.Context) string {
|
func devcontainerBuildSpecs(c *app.Context) []runner.CommandSpec {
|
||||||
return "npx --yes @devcontainers/cli up --workspace-folder \"$(pwd)\"\n"
|
return []runner.CommandSpec{
|
||||||
|
{
|
||||||
|
Name: "npx",
|
||||||
|
Args: []string{"--yes", "@devcontainers/cli", "build", "--workspace-folder", devcontainerRoot(c)},
|
||||||
|
},
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func devcontainerGenerateScript(*app.Context) string {
|
func devcontainerUpSpecs(c *app.Context) []runner.CommandSpec {
|
||||||
return "npx --yes @devcontainers/cli exec --workspace-folder \"$(pwd)\" -- bash -lc \"go generate ./pkg/itr ./pkg/tools && sqlc generate -f pkg/memory/sqlc/sqlc.yaml\"\n"
|
return []runner.CommandSpec{
|
||||||
|
{
|
||||||
|
Name: "npx",
|
||||||
|
Args: []string{"--yes", "@devcontainers/cli", "up", "--workspace-folder", devcontainerRoot(c)},
|
||||||
|
},
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func devcontainerVerifyScript(*app.Context) string {
|
func devcontainerGenerateSpecs(c *app.Context) []runner.CommandSpec {
|
||||||
return "npx --yes @devcontainers/cli exec --workspace-folder \"$(pwd)\" -- bash -lc \"make flatc-check sqlc-check\"\n"
|
root := devcontainerRoot(c)
|
||||||
|
goBinary := cEnv(c, "GO", "go")
|
||||||
|
return []runner.CommandSpec{
|
||||||
|
{
|
||||||
|
Name: "npx",
|
||||||
|
Args: []string{
|
||||||
|
"--yes", "@devcontainers/cli", "exec",
|
||||||
|
"--workspace-folder", root, "--",
|
||||||
|
"bash", "-lc",
|
||||||
|
fmt.Sprintf("%s generate ./pkg/itr ./pkg/tools && sqlc generate -f pkg/memory/sqlc/sqlc.yaml", goBinary),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func evalBuildScript(*app.Context) string {
|
func devcontainerVerifySpecs(c *app.Context) []runner.CommandSpec {
|
||||||
ldFlags := "-X main.version=$(git describe --tags --always --dirty 2>/dev/null || echo \"dev\") -X main.gitCommit=$(git rev-parse --short=8 HEAD 2>/dev/null || echo \"dev\") -X main.buildTime=$(date +%FT%T%z) -X main.goVersion=\"$($GO version | awk '{print \"$3\"}')\" -s -w"
|
root := devcontainerRoot(c)
|
||||||
return scriptHeader() +
|
return []runner.CommandSpec{
|
||||||
"$GO generate ./...\n" +
|
{
|
||||||
"mkdir -p eval/bin\n" +
|
Name: "npx",
|
||||||
fmt.Sprintf("$GO build $GOFLAGS -ldflags \"%s\" -o eval/bin/eval-runner ./eval/cmd/eval-runner\n", ldFlags)
|
Args: []string{
|
||||||
|
"--yes", "@devcontainers/cli", "exec",
|
||||||
|
"--workspace-folder", root, "--",
|
||||||
|
"bash", "-lc", "make flatc-check sqlc-check",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func evalRunScript(*app.Context) string {
|
func evalBuildSpecs(c *app.Context) []runner.CommandSpec {
|
||||||
return `if [ -n "${DRAGONSCALE_EVAL_BASE_CONFIG:-}" ] && [ -n "${DRAGONSCALE_EVAL_DEBUG:-}" ]; then
|
goBinary := cEnv(c, "GO", "go")
|
||||||
echo "DRAGONSCALE_EVAL_BASE_CONFIG=${DRAGONSCALE_EVAL_BASE_CONFIG}"
|
goFlags := strings.Fields(cEnv(c, "GOFLAGS", "-v -trimpath -tags=stdjson"))
|
||||||
fi
|
goVersion := "unknown"
|
||||||
if [ -n "${DRAGONSCALE_EVAL_DEBUG:-}" ]; then
|
goVersionParts := strings.Fields(outputOrDefault(goBinary + " version"))
|
||||||
echo "DRAGONSCALE_EVAL_CONFIG=${DRAGONSCALE_EVAL_CONFIG:-./configs/default.json}"
|
if len(goVersionParts) >= 3 {
|
||||||
fi
|
goVersion = goVersionParts[2]
|
||||||
DRAGONSCALE_EVAL_HOST_HOME=/host_home DRAGONSCALE_EVAL_CONFIG="${DRAGONSCALE_EVAL_CONFIG:-./configs/default.json}" npx --yes promptfoo eval --config promptfooconfig.yaml --no-cache --no-progress-bar
|
}
|
||||||
`
|
version := outputOrDefault("git describe --tags --always --dirty 2>/dev/null || echo \"dev\"")
|
||||||
|
commit := outputOrDefault("git rev-parse --short=8 HEAD 2>/dev/null || echo \"dev\"")
|
||||||
|
buildTime := outputOrDefault("date +%FT%T%z")
|
||||||
|
|
||||||
|
ldFlags := fmt.Sprintf(
|
||||||
|
"-X main.version=%s -X main.gitCommit=%s -X main.buildTime=%s -X main.goVersion=%s -s -w",
|
||||||
|
version, commit, buildTime, goVersion,
|
||||||
|
)
|
||||||
|
return []runner.CommandSpec{
|
||||||
|
{Name: goBinary, Args: []string{"generate", "./..."}},
|
||||||
|
{Name: "mkdir", Args: []string{"-p", "eval/bin"}},
|
||||||
|
{
|
||||||
|
Name: goBinary,
|
||||||
|
Args: append(
|
||||||
|
append(append([]string{"build"}, goFlags...), "-ldflags", ldFlags),
|
||||||
|
"-o", filepath.Join("eval", "bin", "eval-runner"), "./eval/cmd/eval-runner",
|
||||||
|
),
|
||||||
|
},
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func evalFixturesScript(*app.Context) string {
|
func evalRunSpecs(c *app.Context) []runner.CommandSpec {
|
||||||
return scriptHeader() +
|
cfgPath := cEnv(c, "DRAGONSCALE_EVAL_CONFIG", "./configs/default.json")
|
||||||
"mkdir -p \"$HOME/.local/share/dragonscale/sandbox\"\n" +
|
baseCfg := cEnv(c, "DRAGONSCALE_EVAL_BASE_CONFIG", "")
|
||||||
"rm -f \"$HOME/.local/share/dragonscale/sandbox/eval_test_output.txt\" \"$HOME/.local/share/dragonscale/sandbox/test_steps.txt\" \"$HOME/.local/share/dragonscale/sandbox/eval_checkpoint.txt\" \"$HOME/.local/share/dragonscale/sandbox/chain_test.txt\" \"$HOME/.local/share/dragonscale/sandbox/current_year.txt\" \"$HOME/.local/share/dragonscale/sandbox/result.txt\" \"$HOME/.local/share/dragonscale/sandbox/progressive_test.txt\" \"$HOME/.local/share/dragonscale/sandbox/os_name.txt\"\n" +
|
debug := cEnv(c, "DRAGONSCALE_EVAL_DEBUG", "") != ""
|
||||||
"rm -rf \"$HOME/.local/share/dragonscale/sandbox/project\"\n" +
|
promptfooArgs := strings.Fields(cEnv(c, "DRAGONSCALE_PROMPTFOO_ARGS", "--no-cache --no-progress-bar"))
|
||||||
"printf 'dragonscale eval fixture — hello from the eval harness\\nThis is line two of the fixture file.\\n' > \"$HOME/.local/share/dragonscale/sandbox/eval_fixture.txt\"\n" +
|
if len(promptfooArgs) == 0 {
|
||||||
"cp -f eval/fixtures/sample_data.txt \"$HOME/.local/share/dragonscale/sandbox/sample_data.txt\"\n" +
|
promptfooArgs = []string{"--no-cache", "--no-progress-bar"}
|
||||||
"mkdir -p \"$HOME/.local/share/dragonscale/skills\"\n" +
|
}
|
||||||
"cp -rf eval/fixtures/skills/* \"$HOME/.local/share/dragonscale/skills/\" 2>/dev/null || true\n"
|
|
||||||
|
var specs []runner.CommandSpec
|
||||||
|
if debug && strings.TrimSpace(baseCfg) != "" {
|
||||||
|
specs = append(specs, runner.CommandSpec{
|
||||||
|
Name: "echo",
|
||||||
|
Args: []string{fmt.Sprintf("DRAGONSCALE_EVAL_BASE_CONFIG=%s", baseCfg)},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if debug {
|
||||||
|
specs = append(specs, runner.CommandSpec{
|
||||||
|
Name: "echo",
|
||||||
|
Args: []string{fmt.Sprintf("DRAGONSCALE_EVAL_CONFIG=%s", cfgPath)},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
args := append([]string{"--yes", "promptfoo", "eval", "--config", "promptfooconfig.yaml"}, promptfooArgs...)
|
||||||
|
specs = append(specs, runner.CommandSpec{
|
||||||
|
Name: "npx",
|
||||||
|
Args: args,
|
||||||
|
Dir: filepath.Join(c.Root, "eval"),
|
||||||
|
Env: []string{fmt.Sprintf("DRAGONSCALE_EVAL_CONFIG=%s", cfgPath)},
|
||||||
|
})
|
||||||
|
return specs
|
||||||
}
|
}
|
||||||
|
|
||||||
func evalViewScript(*app.Context) string {
|
func evalFixturesSpecs(c *app.Context) []runner.CommandSpec {
|
||||||
return "cd eval && npx --yes promptfoo view\n"
|
sandbox := filepath.Join(homeDir(), ".local", "share", "dragonscale", "sandbox")
|
||||||
|
project := filepath.Join(sandbox, "project")
|
||||||
|
skills := filepath.Join(homeDir(), ".local", "share", "dragonscale", "skills")
|
||||||
|
shared := filepath.Join(sandbox, "sample_data.txt")
|
||||||
|
fixture := filepath.Join(sandbox, "eval_fixture.txt")
|
||||||
|
files := []string{
|
||||||
|
filepath.Join(sandbox, "eval_test_output.txt"),
|
||||||
|
filepath.Join(sandbox, "test_steps.txt"),
|
||||||
|
filepath.Join(sandbox, "eval_checkpoint.txt"),
|
||||||
|
filepath.Join(sandbox, "chain_test.txt"),
|
||||||
|
filepath.Join(sandbox, "current_year.txt"),
|
||||||
|
filepath.Join(sandbox, "result.txt"),
|
||||||
|
filepath.Join(sandbox, "progressive_test.txt"),
|
||||||
|
filepath.Join(sandbox, "os_name.txt"),
|
||||||
|
}
|
||||||
|
sourceFixture := filepath.Join("eval", "fixtures", "sample_data.txt")
|
||||||
|
|
||||||
|
specs := []runner.CommandSpec{
|
||||||
|
{Name: "mkdir", Args: []string{"-p", sandbox}},
|
||||||
|
{Name: "rm", Args: append([]string{"-f"}, files...)},
|
||||||
|
{Name: "rm", Args: []string{"-rf", project}},
|
||||||
|
{Name: "mkdir", Args: []string{"-p", skills}},
|
||||||
|
{Name: "bash", Args: []string{"-lc", fmt.Sprintf("printf '%%s\\n%%s\\n' \"dragonscale eval fixture — hello from the eval harness\" \"This is line two of the fixture file.\" > %q", fixture)}},
|
||||||
|
{Name: "cp", Args: []string{"-f", sourceFixture, shared}},
|
||||||
|
}
|
||||||
|
specs = append(specs, runner.CommandSpec{
|
||||||
|
Name: "bash",
|
||||||
|
Args: []string{"-lc", "if [ -d eval/fixtures/skills ]; then cp -rf eval/fixtures/skills/. " + strconv.Quote(skills) + "; fi"},
|
||||||
|
})
|
||||||
|
return specs
|
||||||
|
}
|
||||||
|
|
||||||
|
func evalViewSpecs(c *app.Context) []runner.CommandSpec {
|
||||||
|
return []runner.CommandSpec{
|
||||||
|
{
|
||||||
|
Name: "npx",
|
||||||
|
Args: []string{"--yes", "promptfoo", "view"},
|
||||||
|
Dir: filepath.Join(c.Root, "eval"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func outputOrDefault(command string) string {
|
||||||
|
output, err := exec.Command("bash", "-lc", command).Output()
|
||||||
|
if err != nil {
|
||||||
|
return "dev"
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(string(output))
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -81,12 +81,14 @@ func TestQuoteArgs(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildAllTaskPreservesGoEnvironmentForwarding(t *testing.T) {
|
func TestBuildAllTaskPreservesGoEnvironmentForwarding(t *testing.T) {
|
||||||
t.Parallel()
|
origCheck := detectGlibc
|
||||||
|
detectGlibc = func() error { return nil }
|
||||||
|
t.Cleanup(func() { detectGlibc = origCheck })
|
||||||
|
|
||||||
ctx := &app.Context{
|
ctx := &app.Context{
|
||||||
Root: t.TempDir(),
|
Root: t.TempDir(),
|
||||||
ExtraEnv: map[string]string{
|
ExtraEnv: map[string]string{
|
||||||
"GOOS": "freebsd",
|
"GOOS": "linux",
|
||||||
"GOARCH": "sparc64",
|
"GOARCH": "sparc64",
|
||||||
"GOFLAGS": "-mod=mod",
|
"GOFLAGS": "-mod=mod",
|
||||||
"DEVCONTAINER_EXEC": "echo devcontainer exec",
|
"DEVCONTAINER_EXEC": "echo devcontainer exec",
|
||||||
|
|
@ -110,31 +112,100 @@ func TestBuildAllTaskPreservesGoEnvironmentForwarding(t *testing.T) {
|
||||||
require.Len(t, fake.Calls, 1)
|
require.Len(t, fake.Calls, 1)
|
||||||
|
|
||||||
script := strings.Join(fake.Calls[0].Args, " ")
|
script := strings.Join(fake.Calls[0].Args, " ")
|
||||||
require.Contains(t, script, "GOOS=linux GOARCH=amd64 $GO build")
|
require.Contains(t, script, "GOOS=$GOOS")
|
||||||
require.Contains(t, script, "GOOS=darwin GOARCH=arm64 $GO build")
|
require.NotContains(t, script, "if [ \"$GOOS\" != \"linux\" ]; then")
|
||||||
require.Contains(t, script, "GOOS=windows GOARCH=amd64 $GO build")
|
require.Contains(t, script, "CGO_ENABLED=1")
|
||||||
require.Contains(t, script, "GOOS=linux GOARCH=arm64 $GO build")
|
require.NotContains(t, script, "CGO_BUILD=1")
|
||||||
|
|
||||||
joinedEnv := strings.Join(fake.Calls[0].Env, " ")
|
joinedEnv := strings.Join(fake.Calls[0].Env, " ")
|
||||||
require.Contains(t, joinedEnv, "GOOS=freebsd")
|
require.Contains(t, joinedEnv, "GOOS=linux")
|
||||||
require.Contains(t, joinedEnv, "GOARCH=sparc64")
|
require.Contains(t, joinedEnv, "GOARCH=sparc64")
|
||||||
require.Contains(t, joinedEnv, "GOFLAGS=-mod=mod")
|
require.Contains(t, joinedEnv, "GOFLAGS=-mod=mod")
|
||||||
require.Contains(t, joinedEnv, "DEVCONTAINER_EXEC=echo devcontainer exec")
|
require.Contains(t, joinedEnv, "DEVCONTAINER_EXEC=echo devcontainer exec")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEvalRunScriptPreservesEvalConfig(t *testing.T) {
|
func TestBuildAllTaskDefaultsToHostTarget(t *testing.T) {
|
||||||
|
origCheck := detectGlibc
|
||||||
|
detectGlibc = func() error { return nil }
|
||||||
|
t.Cleanup(func() { detectGlibc = origCheck })
|
||||||
|
|
||||||
|
ctx := &app.Context{
|
||||||
|
Root: t.TempDir(),
|
||||||
|
ExtraEnv: map[string]string{
|
||||||
|
"GOFLAGS": "-mod=mod",
|
||||||
|
"DEVCONTAINER_EXEC": "echo devcontainer exec",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
fake := &runner.FakeRunner{
|
||||||
|
Result: runner.CommandResult{ExitCode: 0},
|
||||||
|
}
|
||||||
|
|
||||||
|
var buildAll app.Task
|
||||||
|
for _, task := range NewRegistry(ctx.Root) {
|
||||||
|
if task.Name() == "build-all" {
|
||||||
|
buildAll = task
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
require.NotNil(t, buildAll)
|
||||||
|
|
||||||
|
_, err := buildAll.Run(context.Background(), fake, ctx)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, fake.Calls, 1)
|
||||||
|
|
||||||
|
script := strings.Join(fake.Calls[0].Args, " ")
|
||||||
|
require.Contains(t, script, "build-all: target=linux/")
|
||||||
|
require.NotContains(t, script, "GOOS=$($GO env GOOS)")
|
||||||
|
require.Contains(t, script, "CGO_ENABLED=1")
|
||||||
|
require.NotContains(t, script, "CGO_BUILD=1")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildAllTaskRejectsNonLinuxTarget(t *testing.T) {
|
||||||
|
origCheck := detectGlibc
|
||||||
|
detectGlibc = func() error { return nil }
|
||||||
|
t.Cleanup(func() { detectGlibc = origCheck })
|
||||||
|
|
||||||
|
ctx := &app.Context{
|
||||||
|
Root: t.TempDir(),
|
||||||
|
ExtraEnv: map[string]string{
|
||||||
|
"GOOS": "freebsd",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
fake := &runner.FakeRunner{
|
||||||
|
Result: runner.CommandResult{ExitCode: 0},
|
||||||
|
}
|
||||||
|
|
||||||
|
var buildAll app.Task
|
||||||
|
for _, task := range NewRegistry(ctx.Root) {
|
||||||
|
if task.Name() == "build-all" {
|
||||||
|
buildAll = task
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
require.NotNil(t, buildAll)
|
||||||
|
|
||||||
|
_, err := buildAll.Run(context.Background(), fake, ctx)
|
||||||
|
require.Error(t, err)
|
||||||
|
require.Len(t, fake.Calls, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvalRunSpecsPreservesEvalConfig(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
script := evalRunScript(&app.Context{
|
specs := evalRunSpecs(&app.Context{
|
||||||
ExtraEnv: map[string]string{
|
ExtraEnv: map[string]string{
|
||||||
"DRAGONSCALE_EVAL_DEBUG": "1",
|
"DRAGONSCALE_EVAL_DEBUG": "1",
|
||||||
"DRAGONSCALE_EVAL_CONFIG": "./configs/override.json",
|
"DRAGONSCALE_EVAL_CONFIG": "./configs/override.json",
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
require.Contains(t, script, "DRAGONSCALE_EVAL_CONFIG=\"${DRAGONSCALE_EVAL_CONFIG:-./configs/default.json}\"")
|
require.Len(t, specs, 2)
|
||||||
require.Contains(t, script, "npx --yes promptfoo eval --config promptfooconfig.yaml")
|
require.Equal(t, "echo", specs[0].Name)
|
||||||
require.NotContains(t, script, "DRAGONSCALE_EVAL_CONFIG=\"./configs/default.json\"")
|
require.Equal(t, []string{"DRAGONSCALE_EVAL_CONFIG=./configs/override.json"}, specs[0].Args)
|
||||||
|
require.Equal(t, "npx", specs[1].Name)
|
||||||
|
require.Contains(t, strings.Join(specs[1].Env, " "), "DRAGONSCALE_EVAL_CONFIG=./configs/override.json")
|
||||||
|
require.NotContains(t, strings.Join(specs[1].Env, " "), "DRAGONSCALE_EVAL_CONFIG=./configs/default.json")
|
||||||
|
require.Contains(t, strings.Join(specs[1].Args, " "), "--yes promptfoo eval --config promptfooconfig.yaml --no-cache --no-progress-bar")
|
||||||
|
|
||||||
compareScript, err := os.ReadFile(filepath.Clean(filepath.Join("..", "..", "..", "eval", "scripts", "compare.sh")))
|
compareScript, err := os.ReadFile(filepath.Clean(filepath.Join("..", "..", "..", "eval", "scripts", "compare.sh")))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
@ -142,18 +213,27 @@ func TestEvalRunScriptPreservesEvalConfig(t *testing.T) {
|
||||||
require.Contains(t, content, "EVAL_CONFIG=\"${DRAGONSCALE_EVAL_CONFIG:-./configs/default.json}\"")
|
require.Contains(t, content, "EVAL_CONFIG=\"${DRAGONSCALE_EVAL_CONFIG:-./configs/default.json}\"")
|
||||||
require.Contains(t, content, "trap cleanup_compare_config EXIT INT TERM")
|
require.Contains(t, content, "trap cleanup_compare_config EXIT INT TERM")
|
||||||
require.Contains(t, content, "DRAGONSCALE_EVAL_CONFIG: \"${EVAL_CONFIG}\"")
|
require.Contains(t, content, "DRAGONSCALE_EVAL_CONFIG: \"${EVAL_CONFIG}\"")
|
||||||
|
require.NotContains(t, content, "DRAGONSCALE_EVAL_HOST_HOME: \"/host_home\"")
|
||||||
require.Contains(t, content, "TEMP_CONFIG")
|
require.Contains(t, content, "TEMP_CONFIG")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEvalRunScriptUsesBaseConfigWhenSet(t *testing.T) {
|
func TestEvalRunSpecsUsesBaseConfigWhenSetAndDebugEnabled(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
script := evalRunScript(&app.Context{
|
specs := evalRunSpecs(&app.Context{
|
||||||
ExtraEnv: map[string]string{
|
ExtraEnv: map[string]string{
|
||||||
"DRAGONSCALE_EVAL_BASE_CONFIG": "/tmp/base.json",
|
"DRAGONSCALE_EVAL_BASE_CONFIG": "/tmp/base.json",
|
||||||
|
"DRAGONSCALE_EVAL_DEBUG": "1",
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
require.Contains(t, script, `if [ -n "${DRAGONSCALE_EVAL_BASE_CONFIG:-}" ] && [ -n "${DRAGONSCALE_EVAL_DEBUG:-}" ]; then`)
|
|
||||||
|
require.Len(t, specs, 3)
|
||||||
|
require.Equal(t, "echo", specs[0].Name)
|
||||||
|
require.Equal(t, []string{"DRAGONSCALE_EVAL_BASE_CONFIG=/tmp/base.json"}, specs[0].Args)
|
||||||
|
require.Equal(t, "echo", specs[1].Name)
|
||||||
|
require.Equal(t, []string{"DRAGONSCALE_EVAL_CONFIG=./configs/default.json"}, specs[1].Args)
|
||||||
|
require.Equal(t, "npx", specs[2].Name)
|
||||||
|
require.Equal(t, []string{"--yes", "promptfoo", "eval", "--config", "promptfooconfig.yaml", "--no-cache", "--no-progress-bar"}, specs[2].Args)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEvalCompareTaskDisablesNestedDevcontainerExecution(t *testing.T) {
|
func TestEvalCompareTaskDisablesNestedDevcontainerExecution(t *testing.T) {
|
||||||
|
|
@ -248,6 +328,135 @@ func TestEvalCompareTaskPassesBaseConfigEnvToRunner(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLintTaskUsesCommandSpecs(t *testing.T) {
|
||||||
|
ctx := &app.Context{
|
||||||
|
Root: t.TempDir(),
|
||||||
|
}
|
||||||
|
fake := &runner.FakeRunner{
|
||||||
|
Result: runner.CommandResult{ExitCode: 0},
|
||||||
|
}
|
||||||
|
|
||||||
|
task := findTaskByName(t, NewRegistry(ctx.Root), "lint")
|
||||||
|
_, err := task.Run(context.Background(), fake, ctx)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, fake.Calls, 3)
|
||||||
|
require.Equal(t, "go", fake.Calls[0].Name)
|
||||||
|
require.Equal(t, []string{"fmt", "./..."}, fake.Calls[0].Args)
|
||||||
|
require.Equal(t, "go", fake.Calls[1].Name)
|
||||||
|
require.Equal(t, []string{"vet", "./..."}, fake.Calls[1].Args)
|
||||||
|
require.Equal(t, "go", fake.Calls[2].Name)
|
||||||
|
require.Equal(t, []string{"build", "./..."}, fake.Calls[2].Args)
|
||||||
|
require.NotContains(t, strings.Join(fake.Calls[0].Env, " "), "set -euo pipefail")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckTaskUsesCommandSpecs(t *testing.T) {
|
||||||
|
ctx := &app.Context{
|
||||||
|
Root: t.TempDir(),
|
||||||
|
ExtraEnv: map[string]string{
|
||||||
|
"GO": "go",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
fake := &runner.FakeRunner{
|
||||||
|
Result: runner.CommandResult{ExitCode: 0},
|
||||||
|
}
|
||||||
|
|
||||||
|
task := findTaskByName(t, NewRegistry(ctx.Root), "check")
|
||||||
|
_, err := task.Run(context.Background(), fake, ctx)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, fake.Calls, 5)
|
||||||
|
require.Equal(t, []string{"mod", "download"}, fake.Calls[0].Args)
|
||||||
|
require.Equal(t, []string{"mod", "verify"}, fake.Calls[1].Args)
|
||||||
|
require.Equal(t, []string{"fmt", "./..."}, fake.Calls[2].Args)
|
||||||
|
require.Equal(t, []string{"vet", "./..."}, fake.Calls[3].Args)
|
||||||
|
require.Equal(t, []string{"test", "./..."}, fake.Calls[4].Args)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDevcontainerGenerateTaskUsesNpxExecCommand(t *testing.T) {
|
||||||
|
ctx := &app.Context{
|
||||||
|
Root: t.TempDir(),
|
||||||
|
}
|
||||||
|
fake := &runner.FakeRunner{
|
||||||
|
Result: runner.CommandResult{ExitCode: 0},
|
||||||
|
}
|
||||||
|
|
||||||
|
task := findTaskByName(t, NewRegistry(ctx.Root), "devcontainer-generate")
|
||||||
|
_, err := task.Run(context.Background(), fake, ctx)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, fake.Calls, 1)
|
||||||
|
require.Equal(t, "npx", fake.Calls[0].Name)
|
||||||
|
require.Contains(t, fake.Calls[0].Args, "@devcontainers/cli")
|
||||||
|
require.Contains(t, fake.Calls[0].Args, "exec")
|
||||||
|
require.Contains(t, fake.Calls[0].Args, "bash")
|
||||||
|
require.Contains(t, strings.Join(fake.Calls[0].Args, " "), "go generate ./pkg/itr ./pkg/tools")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvalTaskBuildsPromptfooCommandWithDefaultConfig(t *testing.T) {
|
||||||
|
ctx := &app.Context{
|
||||||
|
Root: t.TempDir(),
|
||||||
|
ExtraEnv: map[string]string{
|
||||||
|
"DRAGONSCALE_EVAL_DEBUG": "1",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
fake := &runner.FakeRunner{
|
||||||
|
Result: runner.CommandResult{ExitCode: 0},
|
||||||
|
}
|
||||||
|
|
||||||
|
task := findTaskByName(t, NewRegistry(ctx.Root), "eval")
|
||||||
|
_, err := task.Run(context.Background(), fake, ctx)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, fake.Calls, 2)
|
||||||
|
require.Equal(t, "echo", fake.Calls[0].Name)
|
||||||
|
require.Equal(t, "npx", fake.Calls[1].Name)
|
||||||
|
require.Equal(t, filepath.Join(ctx.Root, "eval"), fake.Calls[1].Dir)
|
||||||
|
joinedEnv := strings.Join(fake.Calls[1].Env, " ")
|
||||||
|
require.Contains(t, joinedEnv, "DRAGONSCALE_EVAL_CONFIG=./configs/default.json")
|
||||||
|
require.Contains(t, strings.Join(fake.Calls[1].Args, " "), "--no-progress-bar")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvalViewTaskRunsInEvalDirectory(t *testing.T) {
|
||||||
|
ctx := &app.Context{
|
||||||
|
Root: t.TempDir(),
|
||||||
|
}
|
||||||
|
fake := &runner.FakeRunner{
|
||||||
|
Result: runner.CommandResult{ExitCode: 0},
|
||||||
|
}
|
||||||
|
task := findTaskByName(t, NewRegistry(ctx.Root), "eval-view")
|
||||||
|
_, err := task.Run(context.Background(), fake, ctx)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, fake.Calls, 1)
|
||||||
|
require.Equal(t, filepath.Join(ctx.Root, "eval"), fake.Calls[0].Dir)
|
||||||
|
require.Equal(t, "npx", fake.Calls[0].Name)
|
||||||
|
require.Equal(t, []string{"--yes", "promptfoo", "view"}, fake.Calls[0].Args)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvalFixturesTaskCreatesFixtureCommands(t *testing.T) {
|
||||||
|
ctx := &app.Context{
|
||||||
|
Root: t.TempDir(),
|
||||||
|
}
|
||||||
|
fake := &runner.FakeRunner{
|
||||||
|
Result: runner.CommandResult{ExitCode: 0},
|
||||||
|
}
|
||||||
|
task := findTaskByName(t, NewRegistry(ctx.Root), "eval-fixtures")
|
||||||
|
_, err := task.Run(context.Background(), fake, ctx)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.GreaterOrEqual(t, len(fake.Calls), 5)
|
||||||
|
require.Equal(t, "mkdir", fake.Calls[0].Name)
|
||||||
|
require.Equal(t, "rm", fake.Calls[1].Name)
|
||||||
|
require.Equal(t, "bash", fake.Calls[4].Name)
|
||||||
|
require.Equal(t, "cp", fake.Calls[5].Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func findTaskByName(t *testing.T, tasks []app.Task, name string) app.Task {
|
||||||
|
t.Helper()
|
||||||
|
for _, task := range tasks {
|
||||||
|
if task.Name() == name {
|
||||||
|
return task
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("task %q missing from registry", name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func ptrString(value string) *string {
|
func ptrString(value string) *string {
|
||||||
return &value
|
return &value
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue