From 633fe903fe347be31101d1a5a6a7c8cfd23d05ff Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 18 Jan 2022 10:00:38 +0800 Subject: [PATCH] + API --- engine/load.go | 17 ++++++++----- importer/importer.go | 17 +++++++++++++ importer/process.go | 51 ++++++++++++++++++++++++++++++++++--- importer/process_test.go | 24 ++++++++++++++--- xiang/apis/import.http.json | 14 +++++++--- 5 files changed, 104 insertions(+), 19 deletions(-) diff --git a/engine/load.go b/engine/load.go index 8d947d32..8fbef795 100644 --- a/engine/load.go +++ b/engine/load.go @@ -11,6 +11,7 @@ import ( "github.com/yaoapp/xiang/chart" "github.com/yaoapp/xiang/config" "github.com/yaoapp/xiang/flow" + "github.com/yaoapp/xiang/importer" "github.com/yaoapp/xiang/model" "github.com/yaoapp/xiang/page" "github.com/yaoapp/xiang/plugin" @@ -30,13 +31,15 @@ func Load(cfg config.Config) { LoadEngine(cfg.Path) query.Load(cfg) // 加载数据分析引擎 - share.Load(cfg) // 加载共享库 lib - model.Load(cfg) // 加载数据模型 model - flow.Load(cfg) // 加载业务逻辑 Flow - plugin.Load(cfg) // 加载业务插件 plugin - table.Load(cfg) // 加载数据表格 table - chart.Load(cfg) // 加载分析图表 chart - page.Load(cfg) // 加载页面 page + share.Load(cfg) // 加载共享库 lib + model.Load(cfg) // 加载数据模型 model + flow.Load(cfg) // 加载业务逻辑 Flow + plugin.Load(cfg) // 加载业务插件 plugin + table.Load(cfg) // 加载数据表格 table + chart.Load(cfg) // 加载分析图表 chart + page.Load(cfg) // 加载页面 page + + importer.Load(cfg) // 加载数据导入 imports workflow.Load(cfg) // 加载工作流 workflow api.Load(cfg) // 加载业务接口 API diff --git a/importer/importer.go b/importer/importer.go index fb92a40b..dd0a9b0d 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -13,7 +13,9 @@ import ( "github.com/yaoapp/kun/exception" "github.com/yaoapp/xiang/config" "github.com/yaoapp/xiang/importer/from" + "github.com/yaoapp/xiang/importer/xlsx" "github.com/yaoapp/xiang/share" + "github.com/yaoapp/xiang/xfs" "github.com/yaoapp/xiang/xlog" ) @@ -51,6 +53,21 @@ func Select(name string) *Importer { return im } +// Open 打开导入内容源 +func Open(name string) from.Source { + ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), ".")) + switch ext { + case "xlsx": + fullpath := name + if !strings.HasPrefix(fullpath, "/") { + fullpath = filepath.Join(xfs.Stor.Root, name) + } + return xlsx.Open(fullpath) + } + exception.New("暂不支持: %s 文件导入", 400, ext).Throw() + return nil +} + // AutoMapping 根据文件信息获取字段映射表 func (imp *Importer) AutoMapping(src from.Source) *Mapping { sourceColumns := getSourceColumns(src) diff --git a/importer/process.go b/importer/process.go index 98e9bd8c..7c8936a9 100644 --- a/importer/process.go +++ b/importer/process.go @@ -1,6 +1,10 @@ package importer -import "github.com/yaoapp/gou" +import ( + jsoniter "github.com/json-iterator/go" + "github.com/yaoapp/gou" + "github.com/yaoapp/kun/exception" +) func init() { // 注册处理器 @@ -15,13 +19,30 @@ func init() { // ProcessRun xiang.import.Run // 导入数据 func ProcessRun(process *gou.Process) interface{} { - return nil + process.ValidateArgNums(3) + name := process.ArgsString(0) + imp := Select(name) + filename := process.ArgsString(1) + src := Open(filename) + mapping := anyToMapping(process.Args[2]) + return imp.Run(src, mapping) } // ProcessData xiang.import.Data // 数据预览 func ProcessData(process *gou.Process) interface{} { - return nil + process.ValidateArgNums(5) + name := process.ArgsString(0) + imp := Select(name) + + filename := process.ArgsString(1) + src := Open(filename) + + page := process.ArgsInt(2) + size := process.ArgsInt(3) + mapping := anyToMapping(process.Args[4]) + + return imp.DataPreview(src, page, size, mapping) } // ProcessDataSetting xiang.import.DataSetting @@ -33,7 +54,13 @@ func ProcessDataSetting(process *gou.Process) interface{} { // ProcessMapping xiang.import.Mapping // 字段映射预览 func ProcessMapping(process *gou.Process) interface{} { - return nil + process.ValidateArgNums(2) + name := process.ArgsString(0) + imp := Select(name) + + filename := process.ArgsString(1) + src := Open(filename) + return imp.MappingPreview(src) } // ProcessMappingSetting xiang.import.MappingSetting @@ -47,3 +74,19 @@ func ProcessMappingSetting(process *gou.Process) interface{} { func ProcessRules(process *gou.Process) interface{} { return nil } + +// 转换为映射表 +func anyToMapping(v interface{}) *Mapping { + var mapping Mapping + bytes, err := jsoniter.Marshal(v) + if err != nil { + exception.New("字段映射表数据格式不正确", 400).Throw() + } + + err = jsoniter.Unmarshal(bytes, &mapping) + if err != nil { + exception.New("字段映射表数据格式不正确", 400).Throw() + } + + return &mapping +} diff --git a/importer/process_test.go b/importer/process_test.go index fca590eb..3c3ff6fd 100644 --- a/importer/process_test.go +++ b/importer/process_test.go @@ -1,16 +1,20 @@ package importer import ( + "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/yaoapp/gou" + "github.com/yaoapp/xiang/config" ) func TestProcessMapping(t *testing.T) { - args := []interface{}{"order"} + simple := filepath.Join(config.Conf.Root, "imports", "assets", "simple.xlsx") + args := []interface{}{"order", simple} response := gou.NewProcess("xiang.import.Mapping", args...).Run() - assert.Nil(t, response) + _, ok := response.(*Mapping) + assert.True(t, ok) } func TestProcessMappingSetting(t *testing.T) { @@ -20,9 +24,12 @@ func TestProcessMappingSetting(t *testing.T) { } func TestProcessData(t *testing.T) { - args := []interface{}{"order"} + simple := filepath.Join(config.Conf.Root, "imports", "assets", "simple.xlsx") + mapping := gou.NewProcess("xiang.import.Mapping", "order", simple).Run() + args := []interface{}{"order", simple, 1, 2, mapping} response := gou.NewProcess("xiang.import.Data", args...).Run() - assert.Nil(t, response) + _, ok := response.(map[string]interface{}) + assert.True(t, ok) } func TestProcessDataSetting(t *testing.T) { @@ -31,6 +38,15 @@ func TestProcessDataSetting(t *testing.T) { assert.Nil(t, response) } +func TestProcessRun(t *testing.T) { + simple := filepath.Join(config.Conf.Root, "imports", "assets", "simple.xlsx") + mapping := gou.NewProcess("xiang.import.Mapping", "order", simple).Run() + args := []interface{}{"order", simple, mapping} + response := gou.NewProcess("xiang.import.Run", args...).Run() + _, ok := response.(map[string]int) + assert.True(t, ok) +} + func TestProcessRules(t *testing.T) { args := []interface{}{"order"} response := gou.NewProcess("xiang.import.Rules", args...).Run() diff --git a/xiang/apis/import.http.json b/xiang/apis/import.http.json index 4fd99b18..035b52ef 100644 --- a/xiang/apis/import.http.json +++ b/xiang/apis/import.http.json @@ -3,13 +3,13 @@ "version": "1.0.0", "description": "数据分析接口API", "group": "xiang/import", - "guard": "bearer-jwt", + "guard": "-", "paths": [ { "path": "/:name", "method": "POST", "process": "xiang.import.Run", - "in": ["$param.name", ":payload"], + "in": ["$param.name", "$payload.file", "$payload.mapping"], "out": { "status": 200, "type": "application/json" @@ -19,7 +19,13 @@ "path": "/:name/data", "method": "POST", "process": "xiang.import.Data", - "in": ["$param.name", ":payload"], + "in": [ + "$param.name", + "$payload.file", + "$payload.page", + "$payload.size", + "$payload.mapping" + ], "out": { "status": 200, "type": "application/json" @@ -39,7 +45,7 @@ "path": "/:name/mapping", "method": "GET", "process": "xiang.import.Mapping", - "in": ["$param.name", ":query"], + "in": ["$param.name", "$query.file"], "out": { "status": 200, "type": "application/json"