diff --git a/.gitignore b/.gitignore index 3b4f8a2f..6dc8a3fa 100644 --- a/.gitignore +++ b/.gitignore @@ -37,5 +37,6 @@ xgen/v1.0/* !xgen/v1.0/index.html !xgen/v1.0/umi.js !xgen/v1.0/layouts__index.async.js +!pipe/ui *-unit-test docker/build/test \ No newline at end of file diff --git a/pipe/context.go b/pipe/context.go index 8544c59c..ae7a46aa 100644 --- a/pipe/context.go +++ b/pipe/context.go @@ -14,8 +14,18 @@ var contexts = sync.Map{} // Create create new context func (pipe *Pipe) Create() *Context { id := uuid.NewString() - ctx := &Context{Pipe: pipe, id: uuid.NewString()} - ctx.current = 0 + ctx := &Context{ + id: id, + Pipe: pipe, + in: map[string][]any{}, + out: map[string]any{}, + input: map[string][]any{}, + output: map[string]any{}, + } + + if pipe.Nodes != nil { + ctx.current = pipe.Nodes[0].Namespace() + } contexts.Store(id, ctx) return ctx } @@ -48,15 +58,210 @@ func (ctx *Context) ID() string { return ctx.id } -// Exec and return error +// Current the current node +func (ctx *Context) Current() (*Node, error) { + node, has := ctx.mapping[ctx.current] + if !has { + return nil, fmt.Errorf("pipe: %s %s", ctx.Name, "node not found") + } + return node, nil +} + +// Next the next node +func (ctx *Context) Next() (*Node, error) { + node, err := ctx.Current() + if err != nil { + return nil, err + } + + if node.Goto != "" { + next, err := ctx.replaceString(node.Goto) + if err != nil { + return nil, err + } + + if next == "EOF" { + return nil, fmt.Errorf("EOF") + } + + ctx.current = next + return ctx.Current() + } + + next := node.index[len(node.index)-1] + 1 + if next < len(ctx.Nodes) { + ctx.current = ctx.Nodes[next].Namespace() + return ctx.Current() + } + return nil, fmt.Errorf("EOF") +} + +// IsEOF check if the error is EOF +func IsEOF(err error) bool { + return err != nil && err.Error() == "EOF" +} + +// Exec the pipe func (ctx *Context) Exec(args ...any) (any, error) { - fmt.Printf("name: %v\n", ctx.Name) - fmt.Printf("global: %v\n", ctx.global) - fmt.Printf("sid: %v\n", ctx.sid) - fmt.Printf("whitelist: %v\n", ctx.Whitelist) + node, err := ctx.Current() + if err != nil { + return nil, err + } + return ctx.exec(node, args...) +} + +// Exec and return error +func (ctx *Context) exec(node *Node, args ...any) (any, error) { + + switch node.Type { + + case "process": + err := node.ExecProcess(ctx, args) + if err != nil { + return nil, err + } + + case "request": + err := node.ExecRequest(ctx, args) + if err != nil { + return nil, err + } + + case "ai": + err := node.ExecAI(ctx, args) + if err != nil { + return nil, err + } + + case "switch": + err := node.ExecSwitch(ctx, args) + if err != nil { + return nil, err + } + + case "user-input": + err := node.Render(ctx, args) + if err != nil { + return nil, err + } + + default: + return nil, fmt.Errorf("pipe: %s %s", ctx.Name, "node type error") + } + return nil, nil } +func (ctx *Context) replace(value any) (any, error) { + + switch v := value.(type) { + case string: + return ctx.replaceAny(v) + + case []any: + return ctx.replaceArray(v) + + case map[string]any: + return ctx.replaceMap(v) + + case Input: + return ctx.replaceArray(v) + } + + return value, nil +} + +func (ctx *Context) replaceAny(value string) (any, error) { + + if !IsExpression(value) { + return value, nil + } + + data, err := ctx.data() + if err != nil { + return "", err + } + + v, err := data.Exec(value) + if err != nil { + return "", err + } + return v, nil +} + +// replaceString replace the string +func (ctx *Context) replaceString(value string) (string, error) { + + if !IsExpression(value) { + return value, nil + } + + data, err := ctx.data() + if err != nil { + return "", err + } + + v, err := data.ExecString(value) + if err != nil { + return "", err + } + return v, nil +} + +func (ctx *Context) replaceMap(value map[string]any) (map[string]any, error) { + newValue := map[string]any{} + for k, v := range value { + res, err := ctx.replace(v) + if err != nil { + return nil, err + } + newValue[k] = res + } + return newValue, nil +} + +func (ctx *Context) replaceArray(value []any) ([]any, error) { + newValue := []any{} + for _, v := range value { + res, err := ctx.replace(v) + if err != nil { + return nil, err + } + newValue = append(newValue, res) + } + + return newValue, nil +} + +func (ctx *Context) replaceInput(value Input) (Input, error) { + return ctx.replaceArray(value) +} + +func (ctx *Context) data() (Data, error) { + node, err := ctx.Current() + if err != nil { + return Data{}, err + } + + name := node.Namespace() + data := map[string]any{ + "$sid": ctx.sid, + "$global": ctx.global, + "$in": ctx.in[name], + "$out": ctx.out[name], + "$input": ctx.input, + "$output": ctx.output, + } + + if ctx.output != nil { + for k, v := range ctx.output { + data[k] = v + } + } + return data, nil + +} + // With with the context func (ctx *Context) With(context context.Context) *Context { ctx.context = context diff --git a/pipe/expression.go b/pipe/expression.go new file mode 100644 index 00000000..153764e9 --- /dev/null +++ b/pipe/expression.go @@ -0,0 +1,74 @@ +package pipe + +import ( + "fmt" + "regexp" + "strings" + + "github.com/expr-lang/expr" + "github.com/expr-lang/expr/vm" + "github.com/yaoapp/kun/log" +) + +// If set the map value, should keep the space at the end of the statement +var stmtRe = regexp.MustCompile(`\{\{([\s\S]*?)\}\}`) +var options = []expr.Option{ + expr.AllowUndefinedVariables(), +} + +// New create a new expression +func (data Data) New(stmt string) (*vm.Program, error) { + + stmt = stmtRe.ReplaceAllStringFunc(stmt, func(stmt string) string { + matches := stmtRe.FindStringSubmatch(stmt) + if len(matches) > 0 { + stmt = strings.ReplaceAll(stmt, matches[0], matches[1]) + } + return stmt + }) + + stmt = strings.TrimSpace(stmt) + // ' => ' " => " + stmt = strings.ReplaceAll(stmt, "'", "'") + stmt = strings.ReplaceAll(stmt, """, "\"") + return expr.Compile(stmt, append([]expr.Option{expr.Env(data)}, options...)...) +} + +// Exec exec statement for the template +func (data Data) Exec(stmt string) (interface{}, error) { + program, err := data.New(stmt) + if err != nil { + log.Warn("pipe: %s %s", stmt, err) + return nil, nil + } + + v, err := expr.Run(program, data) + if err != nil { + log.Warn("pipe: %s %s", stmt, err) + return nil, nil + } + return v, nil +} + +// ExecString exec statement for the template +func (data Data) ExecString(stmt string) (string, error) { + + res, err := data.Exec(stmt) + if err != nil { + return "", nil + } + + if res == nil { + return "", nil + } + + if v, ok := res.(string); ok { + return v, nil + } + return fmt.Sprintf("%v", res), nil +} + +// IsExpression check if the statement is an expression +func IsExpression(stmt string) bool { + return stmtRe.MatchString(stmt) +} diff --git a/pipe/interface.go b/pipe/interface.go deleted file mode 100644 index a7bad566..00000000 --- a/pipe/interface.go +++ /dev/null @@ -1 +0,0 @@ -package pipe diff --git a/pipe/json.go b/pipe/json.go index 777c6836..465bcf68 100644 --- a/pipe/json.go +++ b/pipe/json.go @@ -100,6 +100,32 @@ func (args Args) UnmarshalJSON(data []byte) error { return fmt.Errorf("input type error: %#v", v) } + return nil +} + +// UnmarshalJSON Custom JSON unmarshal function +func (autoFill *AutoFill) UnmarshalJSON(data []byte) error { + + var res any + err := jsoniter.Unmarshal(data, &res) + if err != nil { + return err + } + + switch v := res.(type) { + + case map[string]interface{}: + if value, has := v["value"]; has { + autoFill.Value = fmt.Sprint(value) + } + if action, has := v["action"]; has { + autoFill.Action = fmt.Sprint(action) + } + + default: + autoFill.Value = v + } + return nil } diff --git a/pipe/node.go b/pipe/node.go new file mode 100644 index 00000000..bae23015 --- /dev/null +++ b/pipe/node.go @@ -0,0 +1,276 @@ +package pipe + +import ( + "fmt" + "strings" + + "github.com/yaoapp/kun/log" + "github.com/yaoapp/kun/utils" + "github.com/yaoapp/yao/pipe/ui/cli" +) + +// ExecProcess Execute the process +func (node Node) ExecProcess(ctx *Context, args []any) error { + var err error + name := node.Namespace() + ctx.in[name] = args + if node.Input != nil { + ctx.in[name], err = ctx.replaceInput(node.Input) + if err != nil { + return err + } + } + + ctx.input[name] = ctx.in[name] + res := true + + ctx.out[name] = res + ctx.output[name] = res + if node.Output != nil { + ctx.output[name], err = ctx.replace(node.Output) + if err != nil { + return err + } + } + + next, err := ctx.Next() + if err != nil { + if IsEOF(err) { + return nil + } + return err + } + + // Execute the next node + _, err = ctx.exec(next, ctx.output[name]) + if err != nil { + return err + } + return nil +} + +// ExecRequest Execute the request +func (node Node) ExecRequest(ctx *Context, args []any) error { + return nil +} + +// ExecAI Execute the AI +func (node Node) ExecAI(ctx *Context, args []any) error { + var err error + name := node.Namespace() + ctx.in[name] = args + if node.Input != nil { + ctx.in[name], err = ctx.replaceInput(node.Input) + if err != nil { + return err + } + } + ctx.input[name] = ctx.in[name] + + res := map[string]any{"args": args, "Chinese": "你好", "Arabic": "مرحبا"} + ctx.out[name] = res + ctx.output[name] = res + if node.Output != nil { + ctx.output[name], err = ctx.replace(node.Output) + if err != nil { + return err + } + } + + next, err := ctx.Next() + if err != nil { + if IsEOF(err) { + return nil + } + return err + } + + // Execute the next node + _, err = ctx.exec(next, ctx.output[name]) + if err != nil { + return err + } + return nil +} + +// ExecSwitch Execute the switch +func (node Node) ExecSwitch(ctx *Context, args []any) error { + var err error + name := node.Namespace() + + ctx.in[name] = args + if node.Input != nil { + ctx.in[name], err = ctx.replaceInput(node.Input) + if err != nil { + return err + } + } + ctx.input[name] = ctx.in[name] + + data, err := ctx.data() + if err != nil { + return err + } + + section, _ := node.Case["default"] + for stmt := range node.Case { + if stmt == "default" { + continue + } + + v, err := data.Exec(stmt) + if err != nil { + log.Warn("pipe: %s %s", ctx.Name, err) + continue + } + + // If the result is true, then break the loop + if match, ok := v.(bool); ok && match { + section = node.Case[stmt] + break + } + } + + // Execute the next node + if section == nil { + return fmt.Errorf("pipe: %s %s", ctx.Name, "node case not matched") + } + + // Execute The Pipe + subCtx := section.Create(). + With(ctx.context). + WithGlobal(ctx.global). + WithSid(ctx.sid) + + // Copy the input and output + for k, v := range ctx.in { + subCtx.in[k] = v + } + + for k, v := range ctx.input { + subCtx.input[k] = v + } + + for k, v := range ctx.out { + subCtx.out[k] = v + } + + for k, v := range ctx.output { + subCtx.output[k] = v + } + + _, err = subCtx.Exec(ctx.in[name]) + if err != nil { + return err + } + + // Merge the output + for k, v := range subCtx.out { + ctx.out[k] = v + } + + for k, v := range subCtx.output { + ctx.output[k] = v + } + + utils.Dump(name, ctx.output) + + return nil +} + +// Render Execute the user input +func (node Node) Render(ctx *Context, args []any) error { + + switch node.UI { + + case "cli": + return node.renderCli(ctx, args) + + case "web": + + default: + return fmt.Errorf("pipe: %s %s", ctx.Name, "node ui not supported") + } + + return nil +} + +// Namespace the node namespace +func (node Node) Namespace() string { + name := node.Name + if node.namespace != "" { + name = fmt.Sprintf("%s.%s", node.namespace, name) + } + return name +} + +func (node Node) renderCli(ctx *Context, args []any) error { + + var err error + name := node.Namespace() + + ctx.in[name] = args + if node.Input != nil { + ctx.in[name], err = ctx.replaceInput(node.Input) + if err != nil { + return err + } + } + ctx.input[name] = ctx.in[name] + + // Set option + label, err := ctx.replaceString(node.Label) + if err != nil { + return err + } + + option := &cli.Option{Label: label} + if node.AutoFill != nil { + + value := fmt.Sprintf("%v", node.AutoFill.Value) + value, err = ctx.replaceString(value) + if value != "" { + if err != nil { + fmt.Println("cmd", err) + return err + } + + if node.AutoFill.Action == "exit" { + value = fmt.Sprintf("%s\nexit()\n", value) + } + option.Reader = strings.NewReader(value) + } + } + + userDataLines, err := cli.New(option).Render(args) + if err != nil { + return err + } + + ctx.out[name] = userDataLines + ctx.output[name] = userDataLines + if node.Output != nil { + ctx.output[name], err = ctx.replace(node.Output) + if err != nil { + return err + } + } + + // Execute the next node + next, err := ctx.Next() + if err != nil { + if IsEOF(err) { + return nil + } + return err + } + + // Execute the next node + _, err = ctx.exec(next, ctx.output[name]) + if err != nil { + return err + } + + // Next node + return nil +} diff --git a/pipe/pipe.go b/pipe/pipe.go index 0aae8971..76d05933 100644 --- a/pipe/pipe.go +++ b/pipe/pipe.go @@ -3,6 +3,7 @@ package pipe import ( "errors" "fmt" + "strings" "github.com/yaoapp/gou/application" "github.com/yaoapp/yao/config" @@ -46,6 +47,12 @@ func New(source []byte) (*Pipe, error) { if err != nil { return nil, err } + + err = (&pipe).build() + if err != nil { + return nil, err + } + return &pipe, nil } @@ -63,6 +70,11 @@ func NewFile(file string, root string) (*Pipe, error) { return nil, err } + err = (&pipe).build() + if err != nil { + return nil, err + } + return &pipe, nil } @@ -85,3 +97,91 @@ func Get(id string) (*Pipe, error) { } return nil, fmt.Errorf("pipe %s not found", id) } + +// Build the pipe +func (pipe *Pipe) build() error { + pipe.mapping = map[string]*Node{} + if pipe.Nodes == nil || len(pipe.Nodes) == 0 { + return fmt.Errorf("pipe: %s nodes is required", pipe.Name) + } + + return pipe._build("", pipe.Nodes) +} + +func (pipe *Pipe) _build(namespace string, nodes []Node) error { + + for i, node := range nodes { + if node.Name == "" { + return fmt.Errorf("pipe: %s nodes[%d] name is required", pipe.Name, i) + } + + name := node.Name + if namespace != "" { + name = namespace + "." + name + } + + // Set the index of the node + if nodes[i].index == nil { + nodes[i].index = []int{} + } + + nodes[i].index = append(nodes[i].index, i) + nodes[i].namespace = namespace + pipe.mapping[name] = &nodes[i] + + // Set the label of the node + if node.Label == "" { + nodes[i].Label = strings.ToUpper(node.Name) + } + + // Set the type of the node + if node.Process != nil { + nodes[i].Type = "process" + + // Validate the process + if node.Process.Name == "" { + return fmt.Errorf("pipe: %s nodes[%d] process name is required", pipe.Name, i) + } + + // Security check + if pipe.Whitelist != nil { + if _, has := pipe.Whitelist[node.Process.Name]; !has { + return fmt.Errorf("pipe: %s nodes[%d] process %s is not in the whitelist", pipe.Name, i, node.Process.Name) + } + } + + } else if node.Request != nil { + nodes[i].Type = "request" + + } else if node.Prompts != nil { + nodes[i].Type = "ai" + + } else if node.Case != nil { + nodes[i].Type = "switch" + for _, sub := range node.Case { + // Copy the whitelist to the sub pipe + sub.Name = fmt.Sprintf("%s.%s", pipe.Name, node.Name) + sub.Whitelist = pipe.Whitelist + sub.mapping = map[string]*Node{} + sub.namespace = node.Name + if sub.Nodes != nil && len(sub.Nodes) > 0 { + err := sub._build("", sub.Nodes) + if err != nil { + return err + } + } + } + + } else if node.UI != "" { + nodes[i].Type = "user-input" + if node.UI != "cli" && node.UI != "web" && node.UI != "app" && node.UI != "wxapp" { // Vaildate the UI type + return fmt.Errorf("pipe: %s nodes[%d] the type of the UI must be cli, web, app, wxapp", pipe.Name, i) + } + + } else { + return fmt.Errorf("pipe: %s nodes[%d] process, request, case, prompts or ui is required at least one", pipe.Name, i) + } + } + + return nil +} diff --git a/pipe/pipe_test.go b/pipe/pipe_test.go index 7bcf1ce8..a6d2e237 100644 --- a/pipe/pipe_test.go +++ b/pipe/pipe_test.go @@ -22,15 +22,15 @@ func TestRun(t *testing.T) { sid := session.ID() context, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - ctx := translator. Create(). With(context). WithGlobal(map[string]interface{}{"foo": "bar"}). WithSid(sid) defer Close(ctx.ID()) - - assert.NotPanics(t, func() { ctx.Run() }) + assert.NotPanics(t, func() { + ctx.Run(map[string]interface{}{"placeholder": "translate\nhello world"}) + }) } func prepare(t *testing.T) { diff --git a/pipe/process.go b/pipe/process.go new file mode 100644 index 00000000..b088ac5f --- /dev/null +++ b/pipe/process.go @@ -0,0 +1,28 @@ +package pipe + +import ( + "github.com/yaoapp/gou/process" + "github.com/yaoapp/kun/exception" +) + +func init() { + process.Register("pipes", processPipes) +} + +// processScripts +func processPipes(process *process.Process) interface{} { + + pipe, err := Get(process.ID) + if err != nil { + exception.New("pipes.%s not loaded", 404, process.ID).Throw() + return nil + } + + ctx := pipe.Create().WithGlobal(process.Global).WithSid(process.Sid) + res, err := ctx.Exec(process.Args...) + if err != nil { + exception.New(err.Error(), 500).Throw() + } + + return res +} diff --git a/pipe/types.go b/pipe/types.go index 1254666a..de0e0d00 100644 --- a/pipe/types.go +++ b/pipe/types.go @@ -1,6 +1,8 @@ package pipe -import "context" +import ( + "context" +) // Pipe the pipe type Pipe struct { @@ -9,10 +11,13 @@ type Pipe struct { Nodes []Node `json:"nodes"` Label string `json:"label,omitempty"` Hooks *Hooks `json:"hooks,omitempty"` - Output any `json:"output,omitempty"` // $output - Input Input `json:"input,omitempty"` // $input + Output any `json:"output,omitempty"` + Input Input `json:"input,omitempty"` Whitelist Whitelist `json:"whitelist,omitempty"` // the process whitelist + Goto string `json:"goto,omitempty"` // goto node name / EOF + namespace string // the namespace of the pipe + mapping map[string]*Node // the mapping of the nodes Key:namespace.name Value:index } // Context the Context @@ -22,7 +27,11 @@ type Context struct { context context.Context global map[string]interface{} // $global sid string // $sid - current int // current position + current string // current position + in map[string][]any // $in the node input key:namespace.name Value:[] + out map[string]any // $out the node output key:namespace.name Value:any + input map[string][]any // $input the pipe input key:namespace.name Value:[] + output map[string]any // $output the pipe output key:namespace.name Value:any } // Hooks the Hooks @@ -32,16 +41,22 @@ type Hooks struct { // Node the pip node type Node struct { - Name string `json:"name"` - Type string `json:"type,omitempty"` // user-input, ai, process, switch, request - Label string `json:"label,omitempty"` // Display - Process *Process `json:"process,omitempty"` // Yao Process - Prompts []Prompt `json:"prompts,omitempty"` // AI prompts - Request *Request `json:"request,omitempty"` // Http Request - Interface string `json:"interface,omitempty"` // User Interface command-line, web, app, wxapp ... - Case map[string]CaseSection `json:"case,omitempty"` // Switch - Input Input `json:"input,omitempty"` // $in - Output any `json:"output,omitempty"` // $out + Name string `json:"name"` + Type string `json:"type,omitempty"` // user-input, ai, process, switch, request + Label string `json:"label,omitempty"` // Display + Process *Process `json:"process,omitempty"` // Yao Process + Prompts []Prompt `json:"prompts,omitempty"` // AI prompts + Request *Request `json:"request,omitempty"` // Http Request + UI string `json:"ui,omitempty"` // The User Interface cli, web, app, wxapp ... + AutoFill *AutoFill `json:"autofill,omitempty"` // Autofill the user input with the expression + Case map[string]*Pipe `json:"case,omitempty"` // Switch + Input Input `json:"input,omitempty"` // + Output any `json:"output,omitempty"` // + Goto string `json:"goto,omitempty"` // goto node name / EOF + + index []int // the index of the node + namespace string // the namespace of the node + history []Prompt // history of prompts, this is for the AI and auto merge to the prompts } // Whitelist the Whitelist @@ -53,12 +68,20 @@ type Input []any // Args the args type Args []any -// CaseSection the switch case section -type CaseSection struct { +// Data data for the template +type Data map[string]interface{} + +// AutoFill the autofill +type AutoFill struct { + Value any `json:"value"` + Action string `json:"action,omitempty"` +} + +// Case the switch case section +type Case struct { Input Input `json:"input,omitempty"` // $in Output any `json:"output,omitempty"` // $out Nodes []Node `json:"nodes,omitempty"` // $out - Goto string `json:"goto,omitempty"` // goto node name / EOF } // Prompt the switch diff --git a/pipe/ui/cli/cli.go b/pipe/ui/cli/cli.go new file mode 100644 index 00000000..372757d2 --- /dev/null +++ b/pipe/ui/cli/cli.go @@ -0,0 +1,62 @@ +package cli + +import ( + "bufio" + "fmt" + "io" + "os" + + "github.com/fatih/color" +) + +// Cli the CLI +type Cli struct { + option *Option +} + +// In the input stream +var reader io.Reader = os.Stdin + +// Option the CLI option +type Option struct { + Label string + Reader io.Reader +} + +// SetReader set the reader +func SetReader(r io.Reader) { + reader = r +} + +// New create a new CLI +func New(option *Option) *Cli { + if option.Reader == nil { + option.Reader = reader + } + return &Cli{ + option: option, + } +} + +// Render the CLI UI +func (cli *Cli) Render(args []any) ([]string, error) { + + scanner := bufio.NewScanner(cli.option.Reader) + var lines []string + color.Green("%s", cli.option.Label) + fmt.Printf("%s", color.WhiteString("> ")) + for scanner.Scan() { + line := scanner.Text() + if line == "exit()" { + break + } + lines = append(lines, line) + fmt.Printf("%s", color.WhiteString("> ")) + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return lines, nil +} diff --git a/pipe/ui/web/web.go b/pipe/ui/web/web.go new file mode 100644 index 00000000..e41d899f --- /dev/null +++ b/pipe/ui/web/web.go @@ -0,0 +1,12 @@ +package web + +// Web the web UI +type Web struct{} + +// Option the web option +type Option struct{} + +// Render the Web UI +func (web *Web) Render(args []any, option Option) error { + return nil +}