+ Imporer.DataGet

importer/from/source.go

@@ -17,7 +17,7 @@ const (
 
 // Source 导入文件接口
 type Source interface {
-	Data(page int, size int) []map[string]interface{}
+	Data(row int, size int, cols []int) [][]interface{}
 	Columns() []Column
 	Inspect() Inspect
 	Bind()

importer/importer.go

@@ -8,10 +8,12 @@ import (
 	"strings"
 
 	jsoniter "github.com/json-iterator/go"
+	"github.com/yaoapp/gou"
 	"github.com/yaoapp/kun/exception"
 	"github.com/yaoapp/xiang/config"
 	"github.com/yaoapp/xiang/importer/from"
 	"github.com/yaoapp/xiang/share"
+	"github.com/yaoapp/xiang/xlog"
 )
 
 // Importers 导入器
@@ -75,6 +77,7 @@ func (imp *Importer) AutoMapping(src from.Source) *Mapping {
 					Field: name,
 					Name:  srcCol.Name,
 					Axis:  srcCol.Axis,
+					Col:   srcCol.Col,
 					Rules: imp.Columns[i].Rules,
 				}
 				continue
@@ -86,17 +89,76 @@ func (imp *Importer) AutoMapping(src from.Source) *Mapping {
 	return mapping
 }
 
+// DataGet 预览数据
+func (imp *Importer) DataGet(src from.Source, page int, size int, mapping *Mapping) ([]string, [][]interface{}) {
+
+	row := (page-1)*size + mapping.RowStart
+	if row < 0 {
+		row = mapping.RowStart
+	}
+	cols := []int{}
+	for _, d := range mapping.Columns {
+		cols = append(cols, d.Col)
+	}
+	data := src.Data(row, size, cols)
+	return imp.DataClean(data, mapping.Columns)
+}
+
+// DataClean 清洗数据
+func (imp *Importer) DataClean(data [][]interface{}, bindings []*Binding) ([]string, [][]interface{}) {
+	columns := []string{}
+	new := [][]interface{}{}
+
+	for _, binding := range bindings {
+		columns = append(columns, binding.Field)
+	}
+	// 清洗数据
+	for _, row := range data {
+		success := true
+		for i, binding := range bindings { // 调用字段清洗处理器
+			for _, rule := range binding.Rules {
+				if !DataValidate(row, row[i], rule) {
+					success = false
+				}
+			}
+		}
+		row = append(row, success)
+		new = append(new, row)
+	}
+
+	columns = append(columns, "__effected")
+	return columns, new
+}
+
+// DataValidate 数值校验
+func DataValidate(row []interface{}, value interface{}, rule string) bool {
+	process, err := gou.ProcessOf(rule, value, row)
+	if err != nil {
+		xlog.Printf("DataValidate: %s %s", rule, err.Error())
+		return true
+	}
+	res, err := process.Exec()
+	if err != nil {
+		xlog.Printf("DataValidate: %s %s", rule, err.Error())
+		return true
+	}
+
+	if update, ok := res.([]interface{}); ok {
+		row = update
+		return true
+	}
+	return false
+}
+
 // MappingPreview 预览字段映射关系
 func (imp *Importer) MappingPreview(src from.Source) *Mapping {
 
-	// 读取文件结构指纹
+	// 模板匹配(下一版实现)
 	// tpl := imp.Fingerprint(src)
 	// 查找已有模板
 
 	// 自动匹配
-	imp.AutoMapping(src)
+	return imp.AutoMapping(src)
-
-	return nil
 }
 
 // DataSetting 预览数据表格配置

importer/importer_test.go

@@ -43,3 +43,24 @@ func TestAutoMappingSimple(t *testing.T) {
 		assert.NotEmpty(t, col.Axis)
 	}
 }
+
+func TestDataGetSimple(t *testing.T) {
+	simple := filepath.Join(config.Conf.Root, "imports", "assets", "simple.xlsx")
+	file := xlsx.Open(simple)
+	defer file.Close()
+
+	imp := Select("order")
+	mapping := imp.AutoMapping(file)
+	columns, data := imp.DataGet(file, 1, 2, mapping)
+
+	assert.Equal(t, []string{
+		"order_sn", "user.name", "user.sex", "user.age", "mobile", "skus[*].name",
+		"skus[*].amount", "skus[*].price", "total", "remark", "__effected",
+	}, columns)
+
+	assert.Equal(t, [][]interface{}{
+		{"SN202101120018", "张三", "男", "26", "13211000011", "彩绘湖北地图", "3", "65.5", "196.5", "", true},
+		{"", "李四", "男", "42", "13211000011", "景祐遁甲符应经", "1", "34.8", "34.8", "", false},
+	}, data)
+
+}

importer/init_test.go

@@ -14,6 +14,7 @@ func TestMain(m *testing.M) {
 	share.DBConnect(config.Conf.Database)
 	model.Load(config.Conf)
 	query.Load(config.Conf)
+	share.Load(config.Conf)
 	Load(config.Conf)
 	code := m.Run()
 	os.Exit(code)

importer/types.go

@@ -55,5 +55,6 @@ type Binding struct {
 	Field string   `json:"field"` // 目标字段名称
 	Name  string   `json:"name"`  // 源关联字段名称
 	Axis  string   `json:"axis"`  // 源关联字段坐标
+	Col   int      `json:"col"`   // 源关联字段列号
 	Rules []string `json:"rules"` // 清洗规则
 }

importer/xlsx/xlsx.go

@@ -71,8 +71,29 @@ func (xlsx *Xlsx) Inspect() from.Inspect {
 }
 
 // Data 读取数据
-func (xlsx *Xlsx) Data(page int, size int) []map[string]interface{} {
+func (xlsx *Xlsx) Data(row int, size int, cols []int) [][]interface{} {
-	return nil
+	data := [][]interface{}{}
+	for r := row; r < row+size; r++ {
+		row := []interface{}{}
+		end := true
+		for _, c := range cols {
+			axis := positionToAxis(r, c)
+			value, err := xlsx.File.GetCellValue(xlsx.SheetName, axis)
+			if err != nil {
+				xlog.Printf("读取数据出错 %s %s %s", xlsx.SheetName, axis, err.Error())
+				value = ""
+			}
+			row = append(row, value)
+			if value != "" {
+				end = false
+			}
+		}
+		if end {
+			break
+		}
+		data = append(data, row)
+	}
+	return data
 }
 
 // Columns 读取列

tests/libs/rules.js

@@ -0,0 +1,21 @@
+// 校验数值
+function order_sn(value, row) {
+  value = value || null;
+  row = row || {};
+  if (value == null) {
+    return false;
+  }
+  return row;
+}
+
+function FmtUser(value, row) {
+  return row;
+}
+
+function FmtGoods(value, row) {
+  return row;
+}
+
+function mobile(value, row) {
+  return row;
+}