[add] migrate importer

Makefile

@@ -9,7 +9,7 @@ COMMIT := $(shell git log | head -n 1 | awk '{print substr($$2, 0, 12)}')
 NOW := $(shell date +"%FT%T%z")
 
 # ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
-TESTFOLDER := $(shell $(GO) list ./... | grep -E 'api|model|flow|script|fs|i18n|connector|query|plugin|cert|crypto|task|schedule|runtime|helper|utils|widget' | grep -vE 'examples|tests*|config|widgets')
+TESTFOLDER := $(shell $(GO) list ./... | grep -E 'api|model|flow|script|fs|i18n|connector|query|plugin|cert|crypto|task|schedule|runtime|helper|utils|widget|importer' | grep -vE 'examples|tests*|config|widgets')
 TESTTAGS ?= ""
 
 # TESTWIDGETS := $(shell $(GO) list ./widgets/...)

config/config.go

@@ -22,7 +22,7 @@ var Conf Config
 var LogOutput *os.File // 日志文件
 
 // DSLExtensions the dsl file Extensions
-var DSLExtensions = []string{".yao", ".json", ".jsonc"}
+var DSLExtensions = []string{"*.yao", "*.json", "*.jsonc"}
 
 func init() {
 	filename, _ := filepath.Abs(filepath.Join(".", ".env"))

importer/importer.go

@@ -14,6 +14,7 @@ import (
 	"github.com/yaoapp/kun/exception"
 	"github.com/yaoapp/kun/log"
 	"github.com/yaoapp/yao/config"
+	"github.com/yaoapp/yao/fs"
 	"github.com/yaoapp/yao/importer/from"
 	"github.com/yaoapp/yao/importer/xlsx"
 	"github.com/yaoapp/yao/share"
@@ -22,9 +23,23 @@ import (
 // Importers 导入器
 var Importers = map[string]*Importer{}
 
+// DataRoot data file root
+var DataRoot string = ""
+
 // Load 加载导入器
 func Load(cfg config.Config) error {
-	return application.App.Walk("apis", func(root, file string, isdir bool) error {
+
+	root, err := fs.Root(cfg)
+	if err != nil {
+		return err
+	}
+	DataRoot = root
+
+	exts := []string{"*.imp.yao", "*.imp.json", "*.imp.jsonc"}
+	return application.App.Walk("imports", func(root, file string, isdir bool) error {
+		if isdir {
+			return nil
+		}
 
 		id := share.ID(root, file)
 		data, err := application.App.Read(file)
@@ -37,10 +52,10 @@ func Load(cfg config.Config) error {
 		if err != nil {
 			return fmt.Errorf("%s 导入配置错误. %s", id, err.Error())
 		}
-		Importers[id] = &importer
 
-		return err
+		Importers[id] = &importer
-	}, config.DSLExtensions...)
+		return nil
+	}, exts...)
 }
 
 // Select 选择已加载导入器
@@ -57,13 +72,8 @@ func Open(name string) from.Source {
 	ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), "."))
 	switch ext {
 	case "xlsx":
-		// Refector *****
+		file := filepath.Join(DataRoot, name)
-		// fullpath := name
+		return xlsx.Open(file)
-		// fullpath := filepath.Join(xfs.Stor.Root, name)
-		// if !strings.HasPrefix(fullpath, "/") {
-		// 	fullpath = filepath.Join(xfs.Stor.Root, name)
-		// }
-		return xlsx.Open(name)
 	}
 	exception.New("暂不支持: %s 文件导入", 400, ext).Throw()
 	return nil

importer/importer_test.go

@@ -5,25 +5,41 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/yaoapp/kun/utils"
 	"github.com/yaoapp/yao/config"
+	"github.com/yaoapp/yao/fs"
 	"github.com/yaoapp/yao/importer/xlsx"
+	"github.com/yaoapp/yao/script"
+	"github.com/yaoapp/yao/test"
 )
 
 func TestLoad(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	prepare(t, config.Conf)
 	assert.IsType(t, &Importer{}, Select("order"))
 }
 func TestFingerprintSimple(t *testing.T) {
-	simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx")
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	root := prepare(t, config.Conf)
+	simple := filepath.Join(root, "assets", "simple.xlsx")
 	file := xlsx.Open(simple)
 	defer file.Close()
+
 	imp := Select("order")
 	fingerprint := imp.Fingerprint(file)
 	assert.Equal(t, "3451ca87d71801687abba8993e5a69af79482914435d7cc064236fd93160f999", fingerprint)
 }
 
 func TestAutoMappingSimple(t *testing.T) {
-	simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx")
+
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	root := prepare(t, config.Conf)
+	simple := filepath.Join(root, "assets", "simple.xlsx")
 	file := xlsx.Open(simple)
 	defer file.Close()
 
@@ -47,7 +63,12 @@ func TestAutoMappingSimple(t *testing.T) {
 }
 
 func TestDataGetSimple(t *testing.T) {
-	simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx")
+
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	root := prepare(t, config.Conf)
+	simple := filepath.Join(root, "assets", "simple.xlsx")
 	file := xlsx.Open(simple)
 	defer file.Close()
 
@@ -68,7 +89,11 @@ func TestDataGetSimple(t *testing.T) {
 }
 
 func TestDataChunkSimple(t *testing.T) {
-	simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx")
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	root := prepare(t, config.Conf)
+	simple := filepath.Join(root, "assets", "simple.xlsx")
 	file := xlsx.Open(simple)
 	defer file.Close()
 
@@ -82,7 +107,11 @@ func TestDataChunkSimple(t *testing.T) {
 }
 
 func TestRunSimple(t *testing.T) {
-	simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx")
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	root := prepare(t, config.Conf)
+	simple := filepath.Join(root, "assets", "simple.xlsx")
 	file := xlsx.Open(simple)
 	defer file.Close()
 
@@ -96,7 +125,11 @@ func TestRunSimple(t *testing.T) {
 }
 
 func TestDataPreviewSimple(t *testing.T) {
-	simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx")
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	root := prepare(t, config.Conf)
+	simple := filepath.Join(root, "assets", "simple.xlsx")
 	file := xlsx.Open(simple)
 	defer file.Close()
 	imp := Select("order")
@@ -115,7 +148,11 @@ func TestDataPreviewSimple(t *testing.T) {
 }
 
 func TestMappingPreviewSimple(t *testing.T) {
-	simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx")
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	root := prepare(t, config.Conf)
+	simple := filepath.Join(root, "assets", "simple.xlsx")
 	file := xlsx.Open(simple)
 	defer file.Close()
 
@@ -139,19 +176,47 @@ func TestMappingPreviewSimple(t *testing.T) {
 }
 
 func TestMappingSetting(t *testing.T) {
-	simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx")
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	root := prepare(t, config.Conf)
+	simple := filepath.Join(root, "assets", "simple.xlsx")
 	file := xlsx.Open(simple)
 	defer file.Close()
 
 	imp := Select("order")
 	setting := imp.MappingSetting(file)
-	utils.Dump(setting)
+	// utils.Dump(setting)
 	assert.NotNil(t, setting)
 }
 
 func TestDataSetting(t *testing.T) {
+
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+	prepare(t, config.Conf)
+
 	imp := Select("order")
 	setting := imp.DataSetting()
-	utils.Dump(setting)
+	// utils.Dump(setting)
 	assert.NotNil(t, setting)
 }
+
+func prepare(t *testing.T, cfg config.Config) string {
+	err := Load(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dataRoot, err := fs.Root(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = script.Load(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return dataRoot
+}

importer/init_test.go

@@ -1,23 +0,0 @@
-package importer
-
-import (
-	"os"
-	"testing"
-
-	"github.com/yaoapp/yao/config"
-	"github.com/yaoapp/yao/model"
-	"github.com/yaoapp/yao/query"
-	"github.com/yaoapp/yao/script"
-	"github.com/yaoapp/yao/share"
-)
-
-func TestMain(m *testing.M) {
-	share.DBConnect(config.Conf.DB)
-	model.Load(config.Conf)
-	query.Load(config.Conf)
-	share.Load(config.Conf)
-	script.Load(config.Conf)
-	Load(config.Conf)
-	code := m.Run()
-	os.Exit(code)
-}

importer/process_test.go

@@ -6,52 +6,78 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/yaoapp/gou/process"
+	"github.com/yaoapp/yao/config"
+	"github.com/yaoapp/yao/test"
 )
 
 func TestProcessMapping(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+	prepare(t, config.Conf)
+
 	simple := filepath.Join("assets", "simple.xlsx")
 	args := []interface{}{"order", simple}
-	response := process.New("xiang.import.Mapping", args...).Run()
+	response := process.New("yao.import.Mapping", args...).Run()
 	_, ok := response.(*Mapping)
 	assert.True(t, ok)
 }
 
 func TestProcessMappingSetting(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+	prepare(t, config.Conf)
+
 	simple := filepath.Join("assets", "simple.xlsx")
 	args := []interface{}{"order", simple}
-	response := process.New("xiang.import.MappingSetting", args...).Run()
+	response := process.New("yao.import.MappingSetting", args...).Run()
 	_, ok := response.(map[string]interface{})
 	assert.True(t, ok)
 }
 
 func TestProcessData(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+	prepare(t, config.Conf)
+
 	simple := filepath.Join("assets", "simple.xlsx")
-	mapping := process.New("xiang.import.Mapping", "order", simple).Run()
+	mapping := process.New("yao.import.Mapping", "order", simple).Run()
 	args := []interface{}{"order", simple, 1, 2, mapping}
-	response := process.New("xiang.import.Data", args...).Run()
+	response := process.New("yao.import.Data", args...).Run()
 	_, ok := response.(map[string]interface{})
 	assert.True(t, ok)
 }
 
 func TestProcessDataSetting(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+	prepare(t, config.Conf)
+
 	args := []interface{}{"order"}
-	response := process.New("xiang.import.DataSetting", args...).Run()
+	response := process.New("yao.import.DataSetting", args...).Run()
 	_, ok := response.(map[string]interface{})
 	assert.True(t, ok)
 }
 
 func TestProcessSetting(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+	prepare(t, config.Conf)
+
 	args := []interface{}{"order"}
-	response := process.New("xiang.import.Setting", args...).Run()
+	response := process.New("yao.import.Setting", args...).Run()
 	_, ok := response.(map[string]interface{})
 	assert.True(t, ok)
 }
 
 func TestProcessRun(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+	prepare(t, config.Conf)
+
 	simple := filepath.Join("assets", "simple.xlsx")
-	mapping := process.New("xiang.import.Mapping", "order", simple).Run()
+	mapping := process.New("yao.import.Mapping", "order", simple).Run()
 	args := []interface{}{"order", simple, mapping}
-	response := process.New("xiang.import.Run", args...).Run()
+	response := process.New("yao.import.Run", args...).Run()
 	_, ok := response.(map[string]int)
 	assert.True(t, ok)
 }