From 5faf62a7895270a511a2c62e3dcbf64da1934e23 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 3 Feb 2023 14:43:00 +0800 Subject: [PATCH] [add] migrate importer --- Makefile | 2 +- config/config.go | 2 +- importer/importer.go | 32 +++++++++----- importer/importer_test.go | 87 ++++++++++++++++++++++++++++++++++----- importer/init_test.go | 23 ----------- importer/process_test.go | 42 +++++++++++++++---- 6 files changed, 133 insertions(+), 55 deletions(-) delete mode 100644 importer/init_test.go diff --git a/Makefile b/Makefile index 69d21293..cb166dd8 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ COMMIT := $(shell git log | head -n 1 | awk '{print substr($$2, 0, 12)}') NOW := $(shell date +"%FT%T%z") # ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) -TESTFOLDER := $(shell $(GO) list ./... | grep -E 'api|model|flow|script|fs|i18n|connector|query|plugin|cert|crypto|task|schedule|runtime|helper|utils|widget' | grep -vE 'examples|tests*|config|widgets') +TESTFOLDER := $(shell $(GO) list ./... | grep -E 'api|model|flow|script|fs|i18n|connector|query|plugin|cert|crypto|task|schedule|runtime|helper|utils|widget|importer' | grep -vE 'examples|tests*|config|widgets') TESTTAGS ?= "" # TESTWIDGETS := $(shell $(GO) list ./widgets/...) diff --git a/config/config.go b/config/config.go index e87e52ab..6a816d9b 100644 --- a/config/config.go +++ b/config/config.go @@ -22,7 +22,7 @@ var Conf Config var LogOutput *os.File // 日志文件 // DSLExtensions the dsl file Extensions -var DSLExtensions = []string{".yao", ".json", ".jsonc"} +var DSLExtensions = []string{"*.yao", "*.json", "*.jsonc"} func init() { filename, _ := filepath.Abs(filepath.Join(".", ".env")) diff --git a/importer/importer.go b/importer/importer.go index bb76d111..e0c15f48 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -14,6 +14,7 @@ import ( "github.com/yaoapp/kun/exception" "github.com/yaoapp/kun/log" "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/fs" "github.com/yaoapp/yao/importer/from" "github.com/yaoapp/yao/importer/xlsx" "github.com/yaoapp/yao/share" @@ -22,9 +23,23 @@ import ( // Importers 导入器 var Importers = map[string]*Importer{} +// DataRoot data file root +var DataRoot string = "" + // Load 加载导入器 func Load(cfg config.Config) error { - return application.App.Walk("apis", func(root, file string, isdir bool) error { + + root, err := fs.Root(cfg) + if err != nil { + return err + } + DataRoot = root + + exts := []string{"*.imp.yao", "*.imp.json", "*.imp.jsonc"} + return application.App.Walk("imports", func(root, file string, isdir bool) error { + if isdir { + return nil + } id := share.ID(root, file) data, err := application.App.Read(file) @@ -37,10 +52,10 @@ func Load(cfg config.Config) error { if err != nil { return fmt.Errorf("%s 导入配置错误. %s", id, err.Error()) } - Importers[id] = &importer - return err - }, config.DSLExtensions...) + Importers[id] = &importer + return nil + }, exts...) } // Select 选择已加载导入器 @@ -57,13 +72,8 @@ func Open(name string) from.Source { ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), ".")) switch ext { case "xlsx": - // Refector ***** - // fullpath := name - // fullpath := filepath.Join(xfs.Stor.Root, name) - // if !strings.HasPrefix(fullpath, "/") { - // fullpath = filepath.Join(xfs.Stor.Root, name) - // } - return xlsx.Open(name) + file := filepath.Join(DataRoot, name) + return xlsx.Open(file) } exception.New("暂不支持: %s 文件导入", 400, ext).Throw() return nil diff --git a/importer/importer_test.go b/importer/importer_test.go index faac3084..656fd819 100644 --- a/importer/importer_test.go +++ b/importer/importer_test.go @@ -5,25 +5,41 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/yaoapp/kun/utils" "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/fs" "github.com/yaoapp/yao/importer/xlsx" + "github.com/yaoapp/yao/script" + "github.com/yaoapp/yao/test" ) func TestLoad(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + prepare(t, config.Conf) assert.IsType(t, &Importer{}, Select("order")) } func TestFingerprintSimple(t *testing.T) { - simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx") + test.Prepare(t, config.Conf) + defer test.Clean() + + root := prepare(t, config.Conf) + simple := filepath.Join(root, "assets", "simple.xlsx") file := xlsx.Open(simple) defer file.Close() + imp := Select("order") fingerprint := imp.Fingerprint(file) assert.Equal(t, "3451ca87d71801687abba8993e5a69af79482914435d7cc064236fd93160f999", fingerprint) } func TestAutoMappingSimple(t *testing.T) { - simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx") + + test.Prepare(t, config.Conf) + defer test.Clean() + + root := prepare(t, config.Conf) + simple := filepath.Join(root, "assets", "simple.xlsx") file := xlsx.Open(simple) defer file.Close() @@ -47,7 +63,12 @@ func TestAutoMappingSimple(t *testing.T) { } func TestDataGetSimple(t *testing.T) { - simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx") + + test.Prepare(t, config.Conf) + defer test.Clean() + + root := prepare(t, config.Conf) + simple := filepath.Join(root, "assets", "simple.xlsx") file := xlsx.Open(simple) defer file.Close() @@ -68,7 +89,11 @@ func TestDataGetSimple(t *testing.T) { } func TestDataChunkSimple(t *testing.T) { - simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx") + test.Prepare(t, config.Conf) + defer test.Clean() + + root := prepare(t, config.Conf) + simple := filepath.Join(root, "assets", "simple.xlsx") file := xlsx.Open(simple) defer file.Close() @@ -82,7 +107,11 @@ func TestDataChunkSimple(t *testing.T) { } func TestRunSimple(t *testing.T) { - simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx") + test.Prepare(t, config.Conf) + defer test.Clean() + + root := prepare(t, config.Conf) + simple := filepath.Join(root, "assets", "simple.xlsx") file := xlsx.Open(simple) defer file.Close() @@ -96,7 +125,11 @@ func TestRunSimple(t *testing.T) { } func TestDataPreviewSimple(t *testing.T) { - simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx") + test.Prepare(t, config.Conf) + defer test.Clean() + + root := prepare(t, config.Conf) + simple := filepath.Join(root, "assets", "simple.xlsx") file := xlsx.Open(simple) defer file.Close() imp := Select("order") @@ -115,7 +148,11 @@ func TestDataPreviewSimple(t *testing.T) { } func TestMappingPreviewSimple(t *testing.T) { - simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx") + test.Prepare(t, config.Conf) + defer test.Clean() + + root := prepare(t, config.Conf) + simple := filepath.Join(root, "assets", "simple.xlsx") file := xlsx.Open(simple) defer file.Close() @@ -139,19 +176,47 @@ func TestMappingPreviewSimple(t *testing.T) { } func TestMappingSetting(t *testing.T) { - simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx") + test.Prepare(t, config.Conf) + defer test.Clean() + + root := prepare(t, config.Conf) + simple := filepath.Join(root, "assets", "simple.xlsx") file := xlsx.Open(simple) defer file.Close() imp := Select("order") setting := imp.MappingSetting(file) - utils.Dump(setting) + // utils.Dump(setting) assert.NotNil(t, setting) } func TestDataSetting(t *testing.T) { + + test.Prepare(t, config.Conf) + defer test.Clean() + prepare(t, config.Conf) + imp := Select("order") setting := imp.DataSetting() - utils.Dump(setting) + // utils.Dump(setting) assert.NotNil(t, setting) } + +func prepare(t *testing.T, cfg config.Config) string { + err := Load(cfg) + if err != nil { + t.Fatal(err) + } + + dataRoot, err := fs.Root(cfg) + if err != nil { + t.Fatal(err) + } + + err = script.Load(cfg) + if err != nil { + t.Fatal(err) + } + + return dataRoot +} diff --git a/importer/init_test.go b/importer/init_test.go deleted file mode 100644 index f2976cf3..00000000 --- a/importer/init_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package importer - -import ( - "os" - "testing" - - "github.com/yaoapp/yao/config" - "github.com/yaoapp/yao/model" - "github.com/yaoapp/yao/query" - "github.com/yaoapp/yao/script" - "github.com/yaoapp/yao/share" -) - -func TestMain(m *testing.M) { - share.DBConnect(config.Conf.DB) - model.Load(config.Conf) - query.Load(config.Conf) - share.Load(config.Conf) - script.Load(config.Conf) - Load(config.Conf) - code := m.Run() - os.Exit(code) -} diff --git a/importer/process_test.go b/importer/process_test.go index 318c08ef..37f5e793 100644 --- a/importer/process_test.go +++ b/importer/process_test.go @@ -6,52 +6,78 @@ import ( "github.com/stretchr/testify/assert" "github.com/yaoapp/gou/process" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" ) func TestProcessMapping(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + prepare(t, config.Conf) + simple := filepath.Join("assets", "simple.xlsx") args := []interface{}{"order", simple} - response := process.New("xiang.import.Mapping", args...).Run() + response := process.New("yao.import.Mapping", args...).Run() _, ok := response.(*Mapping) assert.True(t, ok) } func TestProcessMappingSetting(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + prepare(t, config.Conf) + simple := filepath.Join("assets", "simple.xlsx") args := []interface{}{"order", simple} - response := process.New("xiang.import.MappingSetting", args...).Run() + response := process.New("yao.import.MappingSetting", args...).Run() _, ok := response.(map[string]interface{}) assert.True(t, ok) } func TestProcessData(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + prepare(t, config.Conf) + simple := filepath.Join("assets", "simple.xlsx") - mapping := process.New("xiang.import.Mapping", "order", simple).Run() + mapping := process.New("yao.import.Mapping", "order", simple).Run() args := []interface{}{"order", simple, 1, 2, mapping} - response := process.New("xiang.import.Data", args...).Run() + response := process.New("yao.import.Data", args...).Run() _, ok := response.(map[string]interface{}) assert.True(t, ok) } func TestProcessDataSetting(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + prepare(t, config.Conf) + args := []interface{}{"order"} - response := process.New("xiang.import.DataSetting", args...).Run() + response := process.New("yao.import.DataSetting", args...).Run() _, ok := response.(map[string]interface{}) assert.True(t, ok) } func TestProcessSetting(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + prepare(t, config.Conf) + args := []interface{}{"order"} - response := process.New("xiang.import.Setting", args...).Run() + response := process.New("yao.import.Setting", args...).Run() _, ok := response.(map[string]interface{}) assert.True(t, ok) } func TestProcessRun(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + prepare(t, config.Conf) + simple := filepath.Join("assets", "simple.xlsx") - mapping := process.New("xiang.import.Mapping", "order", simple).Run() + mapping := process.New("yao.import.Mapping", "order", simple).Run() args := []interface{}{"order", simple, mapping} - response := process.New("xiang.import.Run", args...).Run() + response := process.New("yao.import.Run", args...).Run() _, ok := response.(map[string]int) assert.True(t, ok) }