From 23ffcf13484bcb49edee06a7a9412b1a2a23f172 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 17 Jan 2022 20:37:18 +0800 Subject: [PATCH] + Imporer.DataGet --- importer/from/source.go | 2 +- importer/importer.go | 70 ++++++++++++++++++++++++++++++++++++--- importer/importer_test.go | 21 ++++++++++++ importer/init_test.go | 1 + importer/types.go | 1 + importer/xlsx/xlsx.go | 25 ++++++++++++-- tests/libs/rules.js | 21 ++++++++++++ 7 files changed, 134 insertions(+), 7 deletions(-) create mode 100644 tests/libs/rules.js diff --git a/importer/from/source.go b/importer/from/source.go index 966ac955..f9f0bcee 100644 --- a/importer/from/source.go +++ b/importer/from/source.go @@ -17,7 +17,7 @@ const ( // Source 导入文件接口 type Source interface { - Data(page int, size int) []map[string]interface{} + Data(row int, size int, cols []int) [][]interface{} Columns() []Column Inspect() Inspect Bind() diff --git a/importer/importer.go b/importer/importer.go index a2f6206b..508a3d94 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -8,10 +8,12 @@ import ( "strings" jsoniter "github.com/json-iterator/go" + "github.com/yaoapp/gou" "github.com/yaoapp/kun/exception" "github.com/yaoapp/xiang/config" "github.com/yaoapp/xiang/importer/from" "github.com/yaoapp/xiang/share" + "github.com/yaoapp/xiang/xlog" ) // Importers 导入器 @@ -75,6 +77,7 @@ func (imp *Importer) AutoMapping(src from.Source) *Mapping { Field: name, Name: srcCol.Name, Axis: srcCol.Axis, + Col: srcCol.Col, Rules: imp.Columns[i].Rules, } continue @@ -86,17 +89,76 @@ func (imp *Importer) AutoMapping(src from.Source) *Mapping { return mapping } +// DataGet 预览数据 +func (imp *Importer) DataGet(src from.Source, page int, size int, mapping *Mapping) ([]string, [][]interface{}) { + + row := (page-1)*size + mapping.RowStart + if row < 0 { + row = mapping.RowStart + } + cols := []int{} + for _, d := range mapping.Columns { + cols = append(cols, d.Col) + } + data := src.Data(row, size, cols) + return imp.DataClean(data, mapping.Columns) +} + +// DataClean 清洗数据 +func (imp *Importer) DataClean(data [][]interface{}, bindings []*Binding) ([]string, [][]interface{}) { + columns := []string{} + new := [][]interface{}{} + + for _, binding := range bindings { + columns = append(columns, binding.Field) + } + // 清洗数据 + for _, row := range data { + success := true + for i, binding := range bindings { // 调用字段清洗处理器 + for _, rule := range binding.Rules { + if !DataValidate(row, row[i], rule) { + success = false + } + } + } + row = append(row, success) + new = append(new, row) + } + + columns = append(columns, "__effected") + return columns, new +} + +// DataValidate 数值校验 +func DataValidate(row []interface{}, value interface{}, rule string) bool { + process, err := gou.ProcessOf(rule, value, row) + if err != nil { + xlog.Printf("DataValidate: %s %s", rule, err.Error()) + return true + } + res, err := process.Exec() + if err != nil { + xlog.Printf("DataValidate: %s %s", rule, err.Error()) + return true + } + + if update, ok := res.([]interface{}); ok { + row = update + return true + } + return false +} + // MappingPreview 预览字段映射关系 func (imp *Importer) MappingPreview(src from.Source) *Mapping { - // 读取文件结构指纹 + // 模板匹配(下一版实现) // tpl := imp.Fingerprint(src) // 查找已有模板 // 自动匹配 - imp.AutoMapping(src) - - return nil + return imp.AutoMapping(src) } // DataSetting 预览数据表格配置 diff --git a/importer/importer_test.go b/importer/importer_test.go index 3f340fd3..9dff36a1 100644 --- a/importer/importer_test.go +++ b/importer/importer_test.go @@ -43,3 +43,24 @@ func TestAutoMappingSimple(t *testing.T) { assert.NotEmpty(t, col.Axis) } } + +func TestDataGetSimple(t *testing.T) { + simple := filepath.Join(config.Conf.Root, "imports", "assets", "simple.xlsx") + file := xlsx.Open(simple) + defer file.Close() + + imp := Select("order") + mapping := imp.AutoMapping(file) + columns, data := imp.DataGet(file, 1, 2, mapping) + + assert.Equal(t, []string{ + "order_sn", "user.name", "user.sex", "user.age", "mobile", "skus[*].name", + "skus[*].amount", "skus[*].price", "total", "remark", "__effected", + }, columns) + + assert.Equal(t, [][]interface{}{ + {"SN202101120018", "张三", "男", "26", "13211000011", "彩绘湖北地图", "3", "65.5", "196.5", "", true}, + {"", "李四", "男", "42", "13211000011", "景祐遁甲符应经", "1", "34.8", "34.8", "", false}, + }, data) + +} diff --git a/importer/init_test.go b/importer/init_test.go index e090849a..219355a1 100644 --- a/importer/init_test.go +++ b/importer/init_test.go @@ -14,6 +14,7 @@ func TestMain(m *testing.M) { share.DBConnect(config.Conf.Database) model.Load(config.Conf) query.Load(config.Conf) + share.Load(config.Conf) Load(config.Conf) code := m.Run() os.Exit(code) diff --git a/importer/types.go b/importer/types.go index 3e55ebc3..d9782556 100644 --- a/importer/types.go +++ b/importer/types.go @@ -55,5 +55,6 @@ type Binding struct { Field string `json:"field"` // 目标字段名称 Name string `json:"name"` // 源关联字段名称 Axis string `json:"axis"` // 源关联字段坐标 + Col int `json:"col"` // 源关联字段列号 Rules []string `json:"rules"` // 清洗规则 } diff --git a/importer/xlsx/xlsx.go b/importer/xlsx/xlsx.go index 0528ef90..8d35c6eb 100644 --- a/importer/xlsx/xlsx.go +++ b/importer/xlsx/xlsx.go @@ -71,8 +71,29 @@ func (xlsx *Xlsx) Inspect() from.Inspect { } // Data 读取数据 -func (xlsx *Xlsx) Data(page int, size int) []map[string]interface{} { - return nil +func (xlsx *Xlsx) Data(row int, size int, cols []int) [][]interface{} { + data := [][]interface{}{} + for r := row; r < row+size; r++ { + row := []interface{}{} + end := true + for _, c := range cols { + axis := positionToAxis(r, c) + value, err := xlsx.File.GetCellValue(xlsx.SheetName, axis) + if err != nil { + xlog.Printf("读取数据出错 %s %s %s", xlsx.SheetName, axis, err.Error()) + value = "" + } + row = append(row, value) + if value != "" { + end = false + } + } + if end { + break + } + data = append(data, row) + } + return data } // Columns 读取列 diff --git a/tests/libs/rules.js b/tests/libs/rules.js new file mode 100644 index 00000000..e78b1012 --- /dev/null +++ b/tests/libs/rules.js @@ -0,0 +1,21 @@ +// 校验数值 +function order_sn(value, row) { + value = value || null; + row = row || {}; + if (value == null) { + return false; + } + return row; +} + +function FmtUser(value, row) { + return row; +} + +function FmtGoods(value, row) { + return row; +} + +function mobile(value, row) { + return row; +}