opencode/yao
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[add] migrate studio

Browse source
This commit is contained in:
Max 2023-04-01 13:28:31 +08:00
parent 82dfe75b2d
commit 5d2c4349fd
13 changed files with 104 additions and 351 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Makefile
View file

@ -9,7 +9,7 @@ COMMIT := $(shell git log | head -n 1 | awk '{print substr($$2, 0, 12)}')
NOW := $(shell date +"%FT%T%z")
# ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
TESTFOLDER := $(shell $(GO) list ./... | grep -E 'api|model|flow|script|fs|i18n|connector|query|plugin|cert|crypto|task|schedule|runtime|helper|utils|widget|importer|store|widgets|engine|service' | grep -vE 'examples|tests*|config')
TESTFOLDER := $(shell $(GO) list ./... | grep -vE 'examples|tests|share*')
TESTTAGS ?= ""
# TESTWIDGETS := $(shell $(GO) list ./widgets/...)

17
cmd/start.go
View file

@ -25,6 +25,7 @@ import (
"github.com/yaoapp/yao/service"
"github.com/yaoapp/yao/setup"
"github.com/yaoapp/yao/share"
"github.com/yaoapp/yao/studio"
)
var startDebug = false
@ -115,14 +116,14 @@ var startCmd = &cobra.Command{
if mode == "development" {
// Start Studio Server
// go func() {
// err := studio.Start(config.Conf)
// if err != nil {
// fmt.Println(color.RedString(L("Fatal: %s"), err.Error()))
// os.Exit(2)
// }
// }()
// defer studio.Stop()
go func() {
err := studio.Start(config.Conf)
if err != nil {
fmt.Println(color.RedString(L("Fatal: %s"), err.Error()))
os.Exit(2)
}
}()
defer studio.Stop()
printApis(false)
printTasks(false)

13
cmd/studio/run.go
View file

@ -14,6 +14,7 @@ import (
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/engine"
"github.com/yaoapp/yao/share"
"github.com/yaoapp/yao/studio"
)
// RunCmd command
@ -34,13 +35,23 @@ var RunCmd = &cobra.Command{
Boot()
cfg := config.Conf
cfg.Session.IsCLI = true
engine.Load(cfg)
if len(args) < 1 {
fmt.Println(color.RedString(L("Not enough arguments")))
fmt.Println(color.WhiteString(share.BUILDNAME + " help"))
return
}
err := engine.Load(cfg)
if err != nil {
fmt.Println(color.RedString(L("Engine: %s"), err.Error()))
}
err = studio.Load(cfg)
if err != nil {
fmt.Println(color.RedString(L("Studio: %s"), err.Error()))
}
name := strings.Split(args[0], ".")
service := strings.Join(name[0:len(name)-1], ".")
method := name[len(name)-1]

24
config/config.go
View file

@ -31,6 +31,7 @@ func init() {
Conf = Load()
return
}
Conf = LoadFrom(filename)
if Conf.Mode == "production" {
Production()
@ -56,14 +57,21 @@ func LoadFrom(envfile string) Config {
return Load()
}
// Load 加载配置
// Load the config
func Load() Config {
cfg := Config{}
if err := env.Parse(&cfg); err != nil {
exception.New("Can't read config %s", 500, err.Error()).Throw()
}
// Root path
cfg.Root, _ = filepath.Abs(cfg.Root)
// App Root
if cfg.AppSource == "" {
cfg.AppSource = cfg.Root
}
// Studio Secret
if cfg.Studio.Secret == nil {
v, err := crypto.Hash(crypto.HashTypes["SHA256"], uuid.New().String())
@ -74,6 +82,20 @@ func Load() Config {
cfg.Studio.Auto = true
}
// DataRoot
if cfg.DataRoot == "" {
cfg.DataRoot = filepath.Join(cfg.Root, "data")
if !filepath.IsAbs(cfg.DataRoot) {
cfg.DataRoot, _ = filepath.Abs(cfg.DataRoot)
}
if _, err := os.Stat(cfg.DataRoot); errors.Is(err, os.ErrNotExist) {
if err := os.MkdirAll(cfg.DataRoot, os.ModePerm); err != nil {
exception.New("Can't create data root %s", 500, err.Error()).Throw()
}
}
}
return cfg
}

5
config/config_test.go
View file

@ -1,6 +1,7 @@
package config
import (
"fmt"
"os"
"path/filepath"
"testing"
@ -56,11 +57,11 @@ func TestLoadFrom(t *testing.T) {
assert.Equal(t, cfg.Root, root)
assert.Equal(t, cfg.Mode, os.Getenv("YAO_ENV"))
assert.Equal(t, cfg.Host, os.Getenv("YAO_HOST"))
assert.Equal(t, cfg.Port, os.Getenv("YAO_PORT"))
assert.Equal(t, fmt.Sprintf("%d", cfg.Port), os.Getenv("YAO_PORT"))
assert.Equal(t, cfg.JWTSecret, os.Getenv("YAO_JWT_SECRET"))
assert.Equal(t, cfg.Log, os.Getenv("YAO_LOG"))
assert.Equal(t, cfg.LogMode, os.Getenv("YAO_LOG_MODE"))
assert.Equal(t, cfg.DB.Driver, os.Getenv("YAO_DB_DRIVER"))
assert.Equal(t, cfg.DB.Primary[0], os.Getenv("YAO_DB_PRIMARY"))
assert.Equal(t, cfg.DB.Secondary[0], os.Getenv("YAO_DB_SECONDARY"))
// assert.Equal(t, cfg.DB.Secondary[0], os.Getenv("YAO_DB_SECONDARY"))
}

23
config/types.go
View file

@ -2,20 +2,21 @@ package config
// Config 象传应用引擎配置
type Config struct {
Mode string `json:"mode,omitempty" env:"YAO_ENV" envDefault:"production"` // 象传引擎启动模式 production/development
Root string `json:"root,omitempty" env:"YAO_ROOT" envDefault:"."` // 应用根目录
Mode string `json:"mode,omitempty" env:"YAO_ENV" envDefault:"production"` // The start mode production/development
AppSource string `json:"app,omitempty" env:"YAO_APP_SOURCE"` // The Application Source Root Path default same as Root
Root string `json:"root,omitempty" env:"YAO_ROOT" envDefault:"."` // The Application Root Path
Lang string `json:"lang,omitempty" env:"YAO_LANG" envDefault:"en-us"` // Default language setting
TimeZone string `json:"timezone,omitempty" env:"YAO_TIMEZONE"` // Default TimeZone
DataRoot string `json:"data_root,omitempty" env:"YAO_DATA_ROOT" envDefault:""` // DATA PATH
DataRoot string `json:"data_root,omitempty" env:"YAO_DATA_ROOT" envDefault:""` // The data root path
ExtensionRoot string `json:"extension_root,omitempty" env:"YAO_EXTENSION_ROOT" envDefault:""` // Plugin, Wasm root PATH, Default is <YAO_ROOT> (<YAO_ROOT>/plugins <YAO_ROOT>/wasms)
Host string `json:"host,omitempty" env:"YAO_HOST" envDefault:"0.0.0.0"` // 服务监听地址
Port int `json:"port,omitempty" env:"YAO_PORT" envDefault:"5099"` // 服务监听端口
Cert string `json:"cert,omitempty" env:"YAO_CERT"` // HTTPS 证书文件地址
Key string `json:"key,omitempty" env:"YAO_KEY"` // HTTPS 证书密钥地址
Log string `json:"log,omitempty" env:"YAO_LOG"` // 服务日志地址
LogMode string `json:"log_mode,omitempty" env:"YAO_LOG_MODE" envDefault:"TEXT"` // 服务日志模式 JSON|TEXT
JWTSecret string `json:"jwt_secret,omitempty" env:"YAO_JWT_SECRET"` // JWT 密钥
DB Database `json:"db,omitempty"` // 数据库配置
Host string `json:"host,omitempty" env:"YAO_HOST" envDefault:"0.0.0.0"` // The server host
Port int `json:"port,omitempty" env:"YAO_PORT" envDefault:"5099"` // The server port
Cert string `json:"cert,omitempty" env:"YAO_CERT"` // The HTTPS certificate path
Key string `json:"key,omitempty" env:"YAO_KEY"` // The HTTPS certificate key path
Log string `json:"log,omitempty" env:"YAO_LOG"` // The log file path
LogMode string `json:"log_mode,omitempty" env:"YAO_LOG_MODE" envDefault:"TEXT"` // The log mode TEXT|JSON
JWTSecret string `json:"jwt_secret,omitempty" env:"YAO_JWT_SECRET"` // The JWT Secret
DB Database `json:"db,omitempty"` // The database config
AllowFrom []string `json:"allowfrom,omitempty" envSeparator:"|" env:"YAO_ALLOW_FROM"` // Domain list the separator is |
Session Session `json:"session,omitempty"` // Session Config
Studio Studio `json:"studio,omitempty"` // Studio config

230
engine/load.go
View file

@ -46,7 +46,7 @@ func Load(cfg config.Config) (err error) {
os.Setenv("XGEN_BASE", adminRoot)
// load the application
err = loadApp(cfg.Root)
err = loadApp(cfg.AppSource)
if err != nil {
printErr(cfg.Mode, "Load Application", err)
panic(err)
@ -281,231 +281,3 @@ func printErr(mode, widget string, err error) {
color.Red(message)
}
}
// // LoadDeprecated 根据配置加载 API, FLow, Model, Plugin
// func LoadDeprecated(cfg config.Config) (err error) {
// defer func() { err = exception.Catch(recover()) }()
// // // Load Runtime
// // err = runtime.Load(cfg)
// // if err != nil {
// // printErr(cfg.Mode, "Runtime", err)
// // }
// // // 加载应用信息
// // // 第一步: 加载应用信息
// // app.Load(cfg)
// // 加密密钥函数
// // model.LoadCrypt(fmt.Sprintf(`{"key":"%s"}`, cfg.DB.AESKey), "AES")
// // gou.LoadCrypt(`{}`, "PASSWORD")
// // Load Certs
// err = cert.Load(cfg)
// if err != nil {
// printErr(cfg.Mode, "Cert", err)
// }
// // Load connectors
// err = connector.Load(cfg)
// if err != nil {
// printErr(cfg.Mode, "Connector", err)
// }
// // Load FileSystem
// err = fs.Load(cfg)
// if err != nil {
// printErr(cfg.Mode, "FileSystem", err)
// }
// // Load i18n
// err = i18n.Load(cfg)
// if err != nil {
// printErr(cfg.Mode, "i18n", err)
// }
// // Load Studio development mode only
// if cfg.Mode == "development" {
// err = studio.Load(cfg)
// if err != nil {
// printErr(cfg.Mode, "Studio", err)
// }
// }
// // 第二步: 建立数据库 & 会话连接
// err = share.DBConnect(cfg.DB) // 创建数据库连接
// if err != nil {
// printErr(cfg.Mode, "DB", err)
// }
// // share.SessionConnect(cfg.Session) // 创建会话服务器链接
// // 加载应用引擎
// if os.Getenv("YAO_DEV") != "" {
// LoadEngine(filepath.Join(os.Getenv("YAO_DEV"), "/yao"))
// } else {
// LoadEngine()
// }
// // 第三步: 加载数据分析引擎
// query.Load(cfg) // 加载数据分析引擎
// // 第四步: 加载共享库 & JS 处理器
// err = share.Load(cfg) // 加载共享库 lib
// if err != nil {
// printErr(cfg.Mode, "Lib", err)
// }
// err = script.Load(cfg) // 加载JS处理器 script
// if err != nil {
// printErr(cfg.Mode, "Script", err)
// }
// // 第五步: 加载数据模型等
// err = model.Load(cfg) // 加载数据模型 model
// if err != nil {
// printErr(cfg.Mode, "Model", err)
// }
// err = flow.Load(cfg) // 加载业务逻辑 Flow
// if err != nil {
// printErr(cfg.Mode, "Flow", err)
// }
// err = store.Load(cfg) // Load stores
// if err != nil {
// printErr(cfg.Mode, "Store", err)
// }
// err = plugin.Load(cfg) // 加载业务插件 plugin
// if err != nil {
// printErr(cfg.Mode, "Plugin", err)
// }
// // // XGEN 1.0
// // if share.App.XGen == "1.0" {
// // // SET XGEN_BASE
// // // adminRoot := "yao"
// // // if share.App.Optional != nil {
// // // if root, has := share.App.Optional["adminRoot"]; has {
// // // adminRoot = fmt.Sprintf("%v", root)
// // // }
// // // }
// // // os.Setenv("XGEN_BASE", adminRoot)
// // // Load build-in widgets
// // err = widgets.Load(cfg)
// // if err != nil {
// // printErr(cfg.Mode, "Widgets", err)
// // }
// // delete(gou.APIs, "xiang.table")
// // delete(gou.APIs, "xiang.page")
// // delete(gou.APIs, "xiang.chart")
// // delete(gou.APIs, "xiang.xiang")
// // delete(gou.APIs, "xiang.user")
// // delete(gou.APIs, "xiang.storage")
// // } else { // old version
// // err = table.Load(cfg) // 加载数据表格 table
// // if err != nil {
// // printErr(cfg.Mode, "Table", err)
// // }
// // err = chart.Load(cfg) // 加载分析图表 chart
// // if err != nil {
// // printErr(cfg.Mode, "Chart", err)
// // }
// // err = page.Load(cfg) // 加载页面 page 忽略错误
// // if err != nil {
// // printErr(cfg.Mode, "Page", err)
// // }
// // }
// importer.Load(cfg) // 加载数据导入 imports
// err = api.Load(cfg) // 加载业务接口 API
// if err != nil {
// printErr(cfg.Mode, "API", err)
// }
// err = socket.Load(cfg) // Load sockets
// if err != nil {
// printErr(cfg.Mode, "Socket", err)
// }
// err = websocket.Load(cfg) // Load websockets (client)
// if err != nil {
// printErr(cfg.Mode, "WebSocket", err)
// }
// err = task.Load(cfg) // Load tasks
// if err != nil {
// printErr(cfg.Mode, "Task", err)
// }
// err = schedule.Load(cfg) // Load schedules
// if err != nil {
// printErr(cfg.Mode, "Schedule", err)
// }
// err = widget.Load(cfg) // Load widgets
// if err != nil {
// printErr(cfg.Mode, "Widget", err)
// }
// return nil
// }
// Reload 根据配置重新加载 API, FLow, Model, Plugin
// func Reload(cfg config.Config) {
// gou.APIs = map[string]*gou.API{}
// gou.Models = map[string]*gou.Model{}
// gou.Flows = map[string]*gou.Flow{}
// gou.Plugins = map[string]*gou.Plugin{}
// Load(cfg)
// }
// LoadEngine 加载引擎的 API, Flow, Model 配置
// func LoadEngine(from ...string) {
// var scripts []share.Script
// if len(from) > 0 {
// scripts = share.GetFilesFS(from[0], ".json")
// } else {
// scripts = share.GetFilesBin("yao", ".json")
// }
// if scripts == nil {
// exception.New("读取文件失败", 500, from).Throw()
// }
// if len(scripts) == 0 {
// exception.New("读取文件失败, 未找到任何可执行脚本", 500, from).Throw()
// }
// // 加载 API, Flow, Models, Table, Chart, Screens
// for _, script := range scripts {
// switch script.Type {
// case "models":
// gou.LoadModel(string(script.Content), "xiang."+script.Name)
// break
// case "flows":
// gou.LoadFlow(string(script.Content), "xiang."+script.Name)
// break
// case "apis":
// gou.LoadAPI(string(script.Content), "xiang."+script.Name)
// break
// }
// }
// // 加载数据应用
// for _, script := range scripts {
// switch script.Type {
// case "tables":
// table.LoadTable(string(script.Content), "xiang."+script.Name)
// break
// }
// }
// }

44
fs/fs.go
View file

@ -1,7 +1,6 @@
package fs
import (
"os"
"path/filepath"
"github.com/yaoapp/gou/fs"
@ -13,44 +12,11 @@ import (
// Load system fs
func Load(cfg config.Config) error {
root, err := filepath.Abs(cfg.Root)
if err != nil {
return err
}
scriptRoot := filepath.Join(cfg.AppSource, "scripts")
dslDenyList := []string{scriptRoot, cfg.DataRoot}
dataRoot, err := Root(cfg)
if err != nil {
return err
}
scriptRoot := filepath.Join(root, "scripts")
dslDenyList := []string{scriptRoot, dataRoot}
if _, err := os.Stat(dataRoot); os.IsNotExist(err) {
err := os.MkdirAll(dataRoot, os.ModePerm)
if err != nil {
return err
}
}
fs.Register("system", system.New(dataRoot))
// fs.Register("binary", system.New(root)) // Next
fs.RootRegister("dsl", dsl.New(root).DenyAbs(dslDenyList...)) // DSL
fs.RootRegister("script", system.New(scriptRoot)) // Script
fs.Register("system", system.New(cfg.DataRoot))
fs.RootRegister("dsl", dsl.New(cfg.AppSource).DenyAbs(dslDenyList...)) // DSL
fs.RootRegister("script", system.New(scriptRoot)) // Script
return nil
}
// Root return data root
func Root(cfg config.Config) (string, error) {
root := cfg.DataRoot
if root == "" {
root = filepath.Join(cfg.Root, "data")
}
root, err := filepath.Abs(root)
if err != nil {
return "", err
}
return root, nil
}

3
fs/fs_test.go
View file

@ -18,8 +18,7 @@ func TestLoad(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, 2, size)
root, err := Root(config.Conf)
assert.Nil(t, err)
root := config.Conf.DataRoot
info, err := os.Stat(filepath.Join(root, "test.file"))
assert.Nil(t, err)

10
runtime/runtime.go
View file

@ -3,17 +3,11 @@ package runtime
import (
v8 "github.com/yaoapp/gou/runtime/v8"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/fs"
)
// Start v8 runtime
func Start(cfg config.Config) error {
dataRoot, err := fs.Root(cfg)
if err != nil {
return err
}
option := &v8.Option{
MinSize: cfg.Runtime.MinSize,
MaxSize: cfg.Runtime.MaxSize,
@ -21,10 +15,10 @@ func Start(cfg config.Config) error {
HeapAvailableSize: cfg.Runtime.HeapAvailableSize,
HeapSizeRelease: cfg.Runtime.HeapSizeRelease,
Precompile: cfg.Runtime.Precompile,
DataRoot: dataRoot,
DataRoot: cfg.DataRoot,
}
err = v8.Start(option)
err := v8.Start(option)
if err != nil {
return err
}

1
studio/router.go
View file

@ -206,6 +206,7 @@ func setRouter(router *gin.Engine) {
script, err := v8.SelectRoot(service)
if err != nil {
throw(c, 500, err.Error())
return
}
ctx, err := script.NewContext(fmt.Sprintf("%v", sid), nil)

56
studio/studio.go
View file

@ -6,11 +6,10 @@ import (
"net/http"
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"github.com/gin-gonic/gin"
"github.com/yaoapp/gou/application"
"github.com/yaoapp/gou/fs"
"github.com/yaoapp/gou/fs/dsl"
v8 "github.com/yaoapp/gou/runtime/v8"
@ -93,56 +92,27 @@ func Stop() {
// Load studio config
func Load(cfg config.Config) error {
err := loadDSL(cfg)
if err != nil {
return err
}
return loadScripts(cfg)
return loadScripts()
}
func loadDSL(cfg config.Config) error {
root, err := filepath.Abs(cfg.Root)
if err != nil {
return err
}
scriptRoot := filepath.Join(root, "scripts")
dataRoot := filepath.Join(root, "data")
dslDenyList := []string{scriptRoot, dataRoot}
dfs = dsl.New(root).DenyAbs(dslDenyList...)
dslDenyList := []string{cfg.DataRoot}
dfs = dsl.New(cfg.AppSource).DenyAbs(dslDenyList...)
return nil
}
func loadScripts(cfg config.Config) error {
root, err := filepath.Abs(cfg.Root)
if err != nil {
return err
}
studioRoot := filepath.Join(root, "studio")
return loadScriptFrom(studioRoot)
}
// Load script From dir
func loadScriptFrom(dir string) error {
if share.DirNotExists(dir) {
log.Warn("[Studio] Load %s does not exists", dir)
return nil
}
messages := []string{}
err := share.Walk(dir, ".js", func(root, filename string) {
name := share.SpecName(root, filename)
_, err := v8.LoadRoot(filename, name)
if err != nil {
messages = append(messages, err.Error())
func loadScripts() error {
exts := []string{"*.js"}
return application.App.Walk("studio", func(root, file string, isdir bool) error {
if isdir {
return nil
}
})
if len(messages) > 0 {
return fmt.Errorf("[Studio] Load %s", strings.Join(messages, ";"))
}
return err
_, err := v8.LoadRoot(file, share.ID(root, file))
return err
}, exts...)
}

27
studio/studio_test.go
View file

@ -12,17 +12,21 @@ import (
"github.com/stretchr/testify/assert"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/helper"
"github.com/yaoapp/yao/test"
)
type kv map[string]interface{}
type arr []interface{}
func TestLoad(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
err := Load(config.Conf)
if err != nil {
t.Fatal(err)
}
assert.NotNil(t, dfs)
// res, err := gou.Yao.Engine.RootCall(map[string]interface{}{}, "table", "Ping")
// assert.Nil(t, err)
@ -34,6 +38,10 @@ func TestLoad(t *testing.T) {
}
func TestStartStop(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
var err error
go func() { err = Start(config.Conf) }()
if err != nil {
@ -46,6 +54,10 @@ func TestStartStop(t *testing.T) {
}
func TestStartStopError(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
var err error
go func() { err = Start(config.Conf) }()
if err != nil {
@ -63,6 +75,9 @@ func TestStartStopError(t *testing.T) {
func TestAPI(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
Load(config.Conf)
var err error
@ -73,23 +88,23 @@ func TestAPI(t *testing.T) {
defer Stop()
time.Sleep(500 * time.Millisecond)
code, row := httpGet[kv]("/dsl/ReadFile?name=/models/user.json", t)
code, row := httpGet[kv]("/dsl/ReadFile?name=/models/user.mod.yao", t)
assert.Equal(t, 200, code)
assert.Equal(t, "用户", row["name"])
assert.Equal(t, "User", row["name"])
code, rows := httpGet[arr]("/dsl/ReadDir?name=/models", t)
assert.Equal(t, 200, code)
assert.Equal(t, 11, len(rows))
assert.Equal(t, 8, len(rows))
code, rows = httpGet[arr]("/dsl/ReadDir?name=/models&recursive=1", t)
assert.Equal(t, 200, code)
assert.Equal(t, 12, len(rows))
code, length := httpPost[int]("/dsl/WriteFile?name=/models/foo.mod.json", []byte(`{"name":"foo"}`), t)
code, length := httpPost[int]("/dsl/WriteFile?name=/models/foo.mod.yao", []byte(`{"name":"foo"}`), t)
assert.Equal(t, 200, code)
assert.Equal(t, 19, length)
code, _ = httpPost[kv]("/dsl/Remove?name=/models/foo.mod.json", nil, t)
code, _ = httpPost[kv]("/dsl/Remove?name=/models/foo.mod.yao", nil, t)
assert.Equal(t, 200, code)
code, _ = httpPost[kv]("/dsl/Mkdir?name=/models/bar", nil, t)