[add] pipe widget (dev)

This commit is contained in:
Max 2024-02-13 20:54:25 +08:00
parent ae599ec2e7
commit 35bf78b08d
12 changed files with 834 additions and 28 deletions

1
.gitignore vendored
View file

@ -37,5 +37,6 @@ xgen/v1.0/*
!xgen/v1.0/index.html !xgen/v1.0/index.html
!xgen/v1.0/umi.js !xgen/v1.0/umi.js
!xgen/v1.0/layouts__index.async.js !xgen/v1.0/layouts__index.async.js
!pipe/ui
*-unit-test *-unit-test
docker/build/test docker/build/test

View file

@ -14,8 +14,18 @@ var contexts = sync.Map{}
// Create create new context // Create create new context
func (pipe *Pipe) Create() *Context { func (pipe *Pipe) Create() *Context {
id := uuid.NewString() id := uuid.NewString()
ctx := &Context{Pipe: pipe, id: uuid.NewString()} ctx := &Context{
ctx.current = 0 id: id,
Pipe: pipe,
in: map[string][]any{},
out: map[string]any{},
input: map[string][]any{},
output: map[string]any{},
}
if pipe.Nodes != nil {
ctx.current = pipe.Nodes[0].Namespace()
}
contexts.Store(id, ctx) contexts.Store(id, ctx)
return ctx return ctx
} }
@ -48,15 +58,210 @@ func (ctx *Context) ID() string {
return ctx.id return ctx.id
} }
// Exec and return error // Current the current node
func (ctx *Context) Current() (*Node, error) {
node, has := ctx.mapping[ctx.current]
if !has {
return nil, fmt.Errorf("pipe: %s %s", ctx.Name, "node not found")
}
return node, nil
}
// Next the next node
func (ctx *Context) Next() (*Node, error) {
node, err := ctx.Current()
if err != nil {
return nil, err
}
if node.Goto != "" {
next, err := ctx.replaceString(node.Goto)
if err != nil {
return nil, err
}
if next == "EOF" {
return nil, fmt.Errorf("EOF")
}
ctx.current = next
return ctx.Current()
}
next := node.index[len(node.index)-1] + 1
if next < len(ctx.Nodes) {
ctx.current = ctx.Nodes[next].Namespace()
return ctx.Current()
}
return nil, fmt.Errorf("EOF")
}
// IsEOF check if the error is EOF
func IsEOF(err error) bool {
return err != nil && err.Error() == "EOF"
}
// Exec the pipe
func (ctx *Context) Exec(args ...any) (any, error) { func (ctx *Context) Exec(args ...any) (any, error) {
fmt.Printf("name: %v\n", ctx.Name) node, err := ctx.Current()
fmt.Printf("global: %v\n", ctx.global) if err != nil {
fmt.Printf("sid: %v\n", ctx.sid) return nil, err
fmt.Printf("whitelist: %v\n", ctx.Whitelist) }
return ctx.exec(node, args...)
}
// Exec and return error
func (ctx *Context) exec(node *Node, args ...any) (any, error) {
switch node.Type {
case "process":
err := node.ExecProcess(ctx, args)
if err != nil {
return nil, err
}
case "request":
err := node.ExecRequest(ctx, args)
if err != nil {
return nil, err
}
case "ai":
err := node.ExecAI(ctx, args)
if err != nil {
return nil, err
}
case "switch":
err := node.ExecSwitch(ctx, args)
if err != nil {
return nil, err
}
case "user-input":
err := node.Render(ctx, args)
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("pipe: %s %s", ctx.Name, "node type error")
}
return nil, nil return nil, nil
} }
func (ctx *Context) replace(value any) (any, error) {
switch v := value.(type) {
case string:
return ctx.replaceAny(v)
case []any:
return ctx.replaceArray(v)
case map[string]any:
return ctx.replaceMap(v)
case Input:
return ctx.replaceArray(v)
}
return value, nil
}
func (ctx *Context) replaceAny(value string) (any, error) {
if !IsExpression(value) {
return value, nil
}
data, err := ctx.data()
if err != nil {
return "", err
}
v, err := data.Exec(value)
if err != nil {
return "", err
}
return v, nil
}
// replaceString replace the string
func (ctx *Context) replaceString(value string) (string, error) {
if !IsExpression(value) {
return value, nil
}
data, err := ctx.data()
if err != nil {
return "", err
}
v, err := data.ExecString(value)
if err != nil {
return "", err
}
return v, nil
}
func (ctx *Context) replaceMap(value map[string]any) (map[string]any, error) {
newValue := map[string]any{}
for k, v := range value {
res, err := ctx.replace(v)
if err != nil {
return nil, err
}
newValue[k] = res
}
return newValue, nil
}
func (ctx *Context) replaceArray(value []any) ([]any, error) {
newValue := []any{}
for _, v := range value {
res, err := ctx.replace(v)
if err != nil {
return nil, err
}
newValue = append(newValue, res)
}
return newValue, nil
}
func (ctx *Context) replaceInput(value Input) (Input, error) {
return ctx.replaceArray(value)
}
func (ctx *Context) data() (Data, error) {
node, err := ctx.Current()
if err != nil {
return Data{}, err
}
name := node.Namespace()
data := map[string]any{
"$sid": ctx.sid,
"$global": ctx.global,
"$in": ctx.in[name],
"$out": ctx.out[name],
"$input": ctx.input,
"$output": ctx.output,
}
if ctx.output != nil {
for k, v := range ctx.output {
data[k] = v
}
}
return data, nil
}
// With with the context // With with the context
func (ctx *Context) With(context context.Context) *Context { func (ctx *Context) With(context context.Context) *Context {
ctx.context = context ctx.context = context

74
pipe/expression.go Normal file
View file

@ -0,0 +1,74 @@
package pipe
import (
"fmt"
"regexp"
"strings"
"github.com/expr-lang/expr"
"github.com/expr-lang/expr/vm"
"github.com/yaoapp/kun/log"
)
// If set the map value, should keep the space at the end of the statement
var stmtRe = regexp.MustCompile(`\{\{([\s\S]*?)\}\}`)
var options = []expr.Option{
expr.AllowUndefinedVariables(),
}
// New create a new expression
func (data Data) New(stmt string) (*vm.Program, error) {
stmt = stmtRe.ReplaceAllStringFunc(stmt, func(stmt string) string {
matches := stmtRe.FindStringSubmatch(stmt)
if len(matches) > 0 {
stmt = strings.ReplaceAll(stmt, matches[0], matches[1])
}
return stmt
})
stmt = strings.TrimSpace(stmt)
// &#39; => ' &#34; => "
stmt = strings.ReplaceAll(stmt, "&#39;", "'")
stmt = strings.ReplaceAll(stmt, "&#34;", "\"")
return expr.Compile(stmt, append([]expr.Option{expr.Env(data)}, options...)...)
}
// Exec exec statement for the template
func (data Data) Exec(stmt string) (interface{}, error) {
program, err := data.New(stmt)
if err != nil {
log.Warn("pipe: %s %s", stmt, err)
return nil, nil
}
v, err := expr.Run(program, data)
if err != nil {
log.Warn("pipe: %s %s", stmt, err)
return nil, nil
}
return v, nil
}
// ExecString exec statement for the template
func (data Data) ExecString(stmt string) (string, error) {
res, err := data.Exec(stmt)
if err != nil {
return "", nil
}
if res == nil {
return "", nil
}
if v, ok := res.(string); ok {
return v, nil
}
return fmt.Sprintf("%v", res), nil
}
// IsExpression check if the statement is an expression
func IsExpression(stmt string) bool {
return stmtRe.MatchString(stmt)
}

View file

@ -1 +0,0 @@
package pipe

View file

@ -100,6 +100,32 @@ func (args Args) UnmarshalJSON(data []byte) error {
return fmt.Errorf("input type error: %#v", v) return fmt.Errorf("input type error: %#v", v)
} }
return nil
}
// UnmarshalJSON Custom JSON unmarshal function
func (autoFill *AutoFill) UnmarshalJSON(data []byte) error {
var res any
err := jsoniter.Unmarshal(data, &res)
if err != nil {
return err
}
switch v := res.(type) {
case map[string]interface{}:
if value, has := v["value"]; has {
autoFill.Value = fmt.Sprint(value)
}
if action, has := v["action"]; has {
autoFill.Action = fmt.Sprint(action)
}
default:
autoFill.Value = v
}
return nil return nil
} }

276
pipe/node.go Normal file
View file

@ -0,0 +1,276 @@
package pipe
import (
"fmt"
"strings"
"github.com/yaoapp/kun/log"
"github.com/yaoapp/kun/utils"
"github.com/yaoapp/yao/pipe/ui/cli"
)
// ExecProcess Execute the process
func (node Node) ExecProcess(ctx *Context, args []any) error {
var err error
name := node.Namespace()
ctx.in[name] = args
if node.Input != nil {
ctx.in[name], err = ctx.replaceInput(node.Input)
if err != nil {
return err
}
}
ctx.input[name] = ctx.in[name]
res := true
ctx.out[name] = res
ctx.output[name] = res
if node.Output != nil {
ctx.output[name], err = ctx.replace(node.Output)
if err != nil {
return err
}
}
next, err := ctx.Next()
if err != nil {
if IsEOF(err) {
return nil
}
return err
}
// Execute the next node
_, err = ctx.exec(next, ctx.output[name])
if err != nil {
return err
}
return nil
}
// ExecRequest Execute the request
func (node Node) ExecRequest(ctx *Context, args []any) error {
return nil
}
// ExecAI Execute the AI
func (node Node) ExecAI(ctx *Context, args []any) error {
var err error
name := node.Namespace()
ctx.in[name] = args
if node.Input != nil {
ctx.in[name], err = ctx.replaceInput(node.Input)
if err != nil {
return err
}
}
ctx.input[name] = ctx.in[name]
res := map[string]any{"args": args, "Chinese": "你好", "Arabic": "مرحبا"}
ctx.out[name] = res
ctx.output[name] = res
if node.Output != nil {
ctx.output[name], err = ctx.replace(node.Output)
if err != nil {
return err
}
}
next, err := ctx.Next()
if err != nil {
if IsEOF(err) {
return nil
}
return err
}
// Execute the next node
_, err = ctx.exec(next, ctx.output[name])
if err != nil {
return err
}
return nil
}
// ExecSwitch Execute the switch
func (node Node) ExecSwitch(ctx *Context, args []any) error {
var err error
name := node.Namespace()
ctx.in[name] = args
if node.Input != nil {
ctx.in[name], err = ctx.replaceInput(node.Input)
if err != nil {
return err
}
}
ctx.input[name] = ctx.in[name]
data, err := ctx.data()
if err != nil {
return err
}
section, _ := node.Case["default"]
for stmt := range node.Case {
if stmt == "default" {
continue
}
v, err := data.Exec(stmt)
if err != nil {
log.Warn("pipe: %s %s", ctx.Name, err)
continue
}
// If the result is true, then break the loop
if match, ok := v.(bool); ok && match {
section = node.Case[stmt]
break
}
}
// Execute the next node
if section == nil {
return fmt.Errorf("pipe: %s %s", ctx.Name, "node case not matched")
}
// Execute The Pipe
subCtx := section.Create().
With(ctx.context).
WithGlobal(ctx.global).
WithSid(ctx.sid)
// Copy the input and output
for k, v := range ctx.in {
subCtx.in[k] = v
}
for k, v := range ctx.input {
subCtx.input[k] = v
}
for k, v := range ctx.out {
subCtx.out[k] = v
}
for k, v := range ctx.output {
subCtx.output[k] = v
}
_, err = subCtx.Exec(ctx.in[name])
if err != nil {
return err
}
// Merge the output
for k, v := range subCtx.out {
ctx.out[k] = v
}
for k, v := range subCtx.output {
ctx.output[k] = v
}
utils.Dump(name, ctx.output)
return nil
}
// Render Execute the user input
func (node Node) Render(ctx *Context, args []any) error {
switch node.UI {
case "cli":
return node.renderCli(ctx, args)
case "web":
default:
return fmt.Errorf("pipe: %s %s", ctx.Name, "node ui not supported")
}
return nil
}
// Namespace the node namespace
func (node Node) Namespace() string {
name := node.Name
if node.namespace != "" {
name = fmt.Sprintf("%s.%s", node.namespace, name)
}
return name
}
func (node Node) renderCli(ctx *Context, args []any) error {
var err error
name := node.Namespace()
ctx.in[name] = args
if node.Input != nil {
ctx.in[name], err = ctx.replaceInput(node.Input)
if err != nil {
return err
}
}
ctx.input[name] = ctx.in[name]
// Set option
label, err := ctx.replaceString(node.Label)
if err != nil {
return err
}
option := &cli.Option{Label: label}
if node.AutoFill != nil {
value := fmt.Sprintf("%v", node.AutoFill.Value)
value, err = ctx.replaceString(value)
if value != "" {
if err != nil {
fmt.Println("cmd", err)
return err
}
if node.AutoFill.Action == "exit" {
value = fmt.Sprintf("%s\nexit()\n", value)
}
option.Reader = strings.NewReader(value)
}
}
userDataLines, err := cli.New(option).Render(args)
if err != nil {
return err
}
ctx.out[name] = userDataLines
ctx.output[name] = userDataLines
if node.Output != nil {
ctx.output[name], err = ctx.replace(node.Output)
if err != nil {
return err
}
}
// Execute the next node
next, err := ctx.Next()
if err != nil {
if IsEOF(err) {
return nil
}
return err
}
// Execute the next node
_, err = ctx.exec(next, ctx.output[name])
if err != nil {
return err
}
// Next node
return nil
}

View file

@ -3,6 +3,7 @@ package pipe
import ( import (
"errors" "errors"
"fmt" "fmt"
"strings"
"github.com/yaoapp/gou/application" "github.com/yaoapp/gou/application"
"github.com/yaoapp/yao/config" "github.com/yaoapp/yao/config"
@ -46,6 +47,12 @@ func New(source []byte) (*Pipe, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
err = (&pipe).build()
if err != nil {
return nil, err
}
return &pipe, nil return &pipe, nil
} }
@ -63,6 +70,11 @@ func NewFile(file string, root string) (*Pipe, error) {
return nil, err return nil, err
} }
err = (&pipe).build()
if err != nil {
return nil, err
}
return &pipe, nil return &pipe, nil
} }
@ -85,3 +97,91 @@ func Get(id string) (*Pipe, error) {
} }
return nil, fmt.Errorf("pipe %s not found", id) return nil, fmt.Errorf("pipe %s not found", id)
} }
// Build the pipe
func (pipe *Pipe) build() error {
pipe.mapping = map[string]*Node{}
if pipe.Nodes == nil || len(pipe.Nodes) == 0 {
return fmt.Errorf("pipe: %s nodes is required", pipe.Name)
}
return pipe._build("", pipe.Nodes)
}
func (pipe *Pipe) _build(namespace string, nodes []Node) error {
for i, node := range nodes {
if node.Name == "" {
return fmt.Errorf("pipe: %s nodes[%d] name is required", pipe.Name, i)
}
name := node.Name
if namespace != "" {
name = namespace + "." + name
}
// Set the index of the node
if nodes[i].index == nil {
nodes[i].index = []int{}
}
nodes[i].index = append(nodes[i].index, i)
nodes[i].namespace = namespace
pipe.mapping[name] = &nodes[i]
// Set the label of the node
if node.Label == "" {
nodes[i].Label = strings.ToUpper(node.Name)
}
// Set the type of the node
if node.Process != nil {
nodes[i].Type = "process"
// Validate the process
if node.Process.Name == "" {
return fmt.Errorf("pipe: %s nodes[%d] process name is required", pipe.Name, i)
}
// Security check
if pipe.Whitelist != nil {
if _, has := pipe.Whitelist[node.Process.Name]; !has {
return fmt.Errorf("pipe: %s nodes[%d] process %s is not in the whitelist", pipe.Name, i, node.Process.Name)
}
}
} else if node.Request != nil {
nodes[i].Type = "request"
} else if node.Prompts != nil {
nodes[i].Type = "ai"
} else if node.Case != nil {
nodes[i].Type = "switch"
for _, sub := range node.Case {
// Copy the whitelist to the sub pipe
sub.Name = fmt.Sprintf("%s.%s", pipe.Name, node.Name)
sub.Whitelist = pipe.Whitelist
sub.mapping = map[string]*Node{}
sub.namespace = node.Name
if sub.Nodes != nil && len(sub.Nodes) > 0 {
err := sub._build("", sub.Nodes)
if err != nil {
return err
}
}
}
} else if node.UI != "" {
nodes[i].Type = "user-input"
if node.UI != "cli" && node.UI != "web" && node.UI != "app" && node.UI != "wxapp" { // Vaildate the UI type
return fmt.Errorf("pipe: %s nodes[%d] the type of the UI must be cli, web, app, wxapp", pipe.Name, i)
}
} else {
return fmt.Errorf("pipe: %s nodes[%d] process, request, case, prompts or ui is required at least one", pipe.Name, i)
}
}
return nil
}

View file

@ -22,15 +22,15 @@ func TestRun(t *testing.T) {
sid := session.ID() sid := session.ID()
context, cancel := context.WithTimeout(context.Background(), 5*time.Second) context, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel() defer cancel()
ctx := translator. ctx := translator.
Create(). Create().
With(context). With(context).
WithGlobal(map[string]interface{}{"foo": "bar"}). WithGlobal(map[string]interface{}{"foo": "bar"}).
WithSid(sid) WithSid(sid)
defer Close(ctx.ID()) defer Close(ctx.ID())
assert.NotPanics(t, func() {
assert.NotPanics(t, func() { ctx.Run() }) ctx.Run(map[string]interface{}{"placeholder": "translate\nhello world"})
})
} }
func prepare(t *testing.T) { func prepare(t *testing.T) {

28
pipe/process.go Normal file
View file

@ -0,0 +1,28 @@
package pipe
import (
"github.com/yaoapp/gou/process"
"github.com/yaoapp/kun/exception"
)
func init() {
process.Register("pipes", processPipes)
}
// processScripts
func processPipes(process *process.Process) interface{} {
pipe, err := Get(process.ID)
if err != nil {
exception.New("pipes.%s not loaded", 404, process.ID).Throw()
return nil
}
ctx := pipe.Create().WithGlobal(process.Global).WithSid(process.Sid)
res, err := ctx.Exec(process.Args...)
if err != nil {
exception.New(err.Error(), 500).Throw()
}
return res
}

View file

@ -1,6 +1,8 @@
package pipe package pipe
import "context" import (
"context"
)
// Pipe the pipe // Pipe the pipe
type Pipe struct { type Pipe struct {
@ -9,10 +11,13 @@ type Pipe struct {
Nodes []Node `json:"nodes"` Nodes []Node `json:"nodes"`
Label string `json:"label,omitempty"` Label string `json:"label,omitempty"`
Hooks *Hooks `json:"hooks,omitempty"` Hooks *Hooks `json:"hooks,omitempty"`
Output any `json:"output,omitempty"` // $output Output any `json:"output,omitempty"`
Input Input `json:"input,omitempty"` // $input Input Input `json:"input,omitempty"`
Whitelist Whitelist `json:"whitelist,omitempty"` // the process whitelist Whitelist Whitelist `json:"whitelist,omitempty"` // the process whitelist
Goto string `json:"goto,omitempty"` // goto node name / EOF
namespace string // the namespace of the pipe
mapping map[string]*Node // the mapping of the nodes Key:namespace.name Value:index
} }
// Context the Context // Context the Context
@ -22,7 +27,11 @@ type Context struct {
context context.Context context context.Context
global map[string]interface{} // $global global map[string]interface{} // $global
sid string // $sid sid string // $sid
current int // current position current string // current position
in map[string][]any // $in the node input key:namespace.name Value:[]
out map[string]any // $out the node output key:namespace.name Value:any
input map[string][]any // $input the pipe input key:namespace.name Value:[]
output map[string]any // $output the pipe output key:namespace.name Value:any
} }
// Hooks the Hooks // Hooks the Hooks
@ -38,10 +47,16 @@ type Node struct {
Process *Process `json:"process,omitempty"` // Yao Process Process *Process `json:"process,omitempty"` // Yao Process
Prompts []Prompt `json:"prompts,omitempty"` // AI prompts Prompts []Prompt `json:"prompts,omitempty"` // AI prompts
Request *Request `json:"request,omitempty"` // Http Request Request *Request `json:"request,omitempty"` // Http Request
Interface string `json:"interface,omitempty"` // User Interface command-line, web, app, wxapp ... UI string `json:"ui,omitempty"` // The User Interface cli, web, app, wxapp ...
Case map[string]CaseSection `json:"case,omitempty"` // Switch AutoFill *AutoFill `json:"autofill,omitempty"` // Autofill the user input with the expression
Input Input `json:"input,omitempty"` // $in Case map[string]*Pipe `json:"case,omitempty"` // Switch
Output any `json:"output,omitempty"` // $out Input Input `json:"input,omitempty"` //
Output any `json:"output,omitempty"` //
Goto string `json:"goto,omitempty"` // goto node name / EOF
index []int // the index of the node
namespace string // the namespace of the node
history []Prompt // history of prompts, this is for the AI and auto merge to the prompts
} }
// Whitelist the Whitelist // Whitelist the Whitelist
@ -53,12 +68,20 @@ type Input []any
// Args the args // Args the args
type Args []any type Args []any
// CaseSection the switch case section // Data data for the template
type CaseSection struct { type Data map[string]interface{}
// AutoFill the autofill
type AutoFill struct {
Value any `json:"value"`
Action string `json:"action,omitempty"`
}
// Case the switch case section
type Case struct {
Input Input `json:"input,omitempty"` // $in Input Input `json:"input,omitempty"` // $in
Output any `json:"output,omitempty"` // $out Output any `json:"output,omitempty"` // $out
Nodes []Node `json:"nodes,omitempty"` // $out Nodes []Node `json:"nodes,omitempty"` // $out
Goto string `json:"goto,omitempty"` // goto node name / EOF
} }
// Prompt the switch // Prompt the switch

62
pipe/ui/cli/cli.go Normal file
View file

@ -0,0 +1,62 @@
package cli
import (
"bufio"
"fmt"
"io"
"os"
"github.com/fatih/color"
)
// Cli the CLI
type Cli struct {
option *Option
}
// In the input stream
var reader io.Reader = os.Stdin
// Option the CLI option
type Option struct {
Label string
Reader io.Reader
}
// SetReader set the reader
func SetReader(r io.Reader) {
reader = r
}
// New create a new CLI
func New(option *Option) *Cli {
if option.Reader == nil {
option.Reader = reader
}
return &Cli{
option: option,
}
}
// Render the CLI UI
func (cli *Cli) Render(args []any) ([]string, error) {
scanner := bufio.NewScanner(cli.option.Reader)
var lines []string
color.Green("%s", cli.option.Label)
fmt.Printf("%s", color.WhiteString("> "))
for scanner.Scan() {
line := scanner.Text()
if line == "exit()" {
break
}
lines = append(lines, line)
fmt.Printf("%s", color.WhiteString("> "))
}
if err := scanner.Err(); err != nil {
return nil, err
}
return lines, nil
}

12
pipe/ui/web/web.go Normal file
View file

@ -0,0 +1,12 @@
package web
// Web the web UI
type Web struct{}
// Option the web option
type Option struct{}
// Render the Web UI
func (web *Web) Render(args []any, option Option) error {
return nil
}