From 2a457033cb3d9df3d2cba96a17ce740c540d4e88 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 19 Oct 2021 23:52:10 +0800 Subject: [PATCH] =?UTF-8?q?+=20Chart=20=E5=A4=84=E7=90=86=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- chart/api.go | 60 ++++++++++++++++++++++ chart/chart.go | 28 ++++++++++- chart/process.go | 82 +++++++++++++++++++++++++++++++ chart/process_test.go | 77 +++++++++++++++++++++++++++++ chart/types.go | 9 +--- helper/array.go | 78 +++++++++++++++++++++++++++++ helper/array_test.go | 23 +++++++++ helper/process.go | 36 ++++++++++++++ helper/process_test.go | 31 ++++++++++++ share/types.go | 1 + tests/charts/service/compare.json | 81 ++++++++++++++++++++++-------- tests/models/service.json | 15 ++++++ xiang/apis/chart.http.json | 29 +++++++++++ 13 files changed, 520 insertions(+), 30 deletions(-) create mode 100644 chart/api.go create mode 100644 chart/process.go create mode 100644 chart/process_test.go create mode 100644 helper/array.go create mode 100644 helper/array_test.go create mode 100644 helper/process.go create mode 100644 helper/process_test.go create mode 100644 xiang/apis/chart.http.json diff --git a/chart/api.go b/chart/api.go new file mode 100644 index 00000000..ad895c1d --- /dev/null +++ b/chart/api.go @@ -0,0 +1,60 @@ +package chart + +import ( + "github.com/yaoapp/xiang/share" +) + +// SetupAPIs 设定API数据 +func (chart *Chart) SetupAPIs() { + + defaults := map[string]share.API{ + "data": apiDataDefault(), + "setting": apiSettingDefault(), + } + + // 开发者填写的规则 + for name := range chart.APIs { + if _, has := defaults[name]; !has { + delete(chart.APIs, name) + continue + } + + api := defaults[name] + api.Name = name + if chart.APIs[name].Process != "" { + api.Process = chart.APIs[name].Process + } + + if chart.APIs[name].Guard != "" { + api.Guard = chart.APIs[name].Guard + } + + if chart.APIs[name].Default != nil { + api.Default = chart.APIs[name].Default + } + + defaults[name] = api + } + + chart.APIs = defaults +} + +// apiSearchDefault data 接口默认值 +func apiDataDefault() share.API { + param := map[string]interface{}{} + return share.API{ + Name: "data", + Guard: "bearer-jwt", + Process: "xiang.chart.data", + Default: []interface{}{param}, + } +} + +// apiSettingDefault setting 接口默认值 +func apiSettingDefault() share.API { + return share.API{ + Name: "setting", + Guard: "bearer-jwt", + Process: "xiang.chart.setting", + } +} diff --git a/chart/chart.go b/chart/chart.go index 0e180941..af0c15d9 100644 --- a/chart/chart.go +++ b/chart/chart.go @@ -1,7 +1,10 @@ package chart import ( + "fmt" + jsoniter "github.com/json-iterator/go" + "github.com/yaoapp/gou" "github.com/yaoapp/kun/exception" "github.com/yaoapp/xiang/config" "github.com/yaoapp/xiang/share" @@ -36,7 +39,11 @@ func LoadFrom(dir string, prefix string) { // LoadChart 载入数据表格 func LoadChart(source []byte, name string) (*Chart, error) { - chart := Chart{} + chart := Chart{ + Flow: gou.Flow{ + Name: name, + }, + } err := jsoniter.Unmarshal(source, &chart) if err != nil { xlog.Println(name) @@ -45,5 +52,24 @@ func LoadChart(source []byte, name string) (*Chart, error) { return nil, err } chart.Prepare() + chart.SetupAPIs() + return &chart, nil } + +// Select 读取已加载图表 +func Select(name string) *Chart { + chart, has := Charts[name] + if !has { + exception.New( + fmt.Sprintf("Chart:%s; 尚未加载", name), + 400, + ).Throw() + } + return chart +} + +// GetData 运行 flow 返回数值 +func (chart Chart) GetData(params map[string]interface{}) interface{} { + return chart.Flow.Exec(params) +} diff --git a/chart/process.go b/chart/process.go new file mode 100644 index 00000000..5a6c0509 --- /dev/null +++ b/chart/process.go @@ -0,0 +1,82 @@ +package chart + +import ( + "strings" + + "github.com/yaoapp/gou" + "github.com/yaoapp/kun/maps" +) + +// 注册处理器 +func init() { + gou.RegisterProcessHandler("xiang.chart.data", ProcessData) + gou.RegisterProcessHandler("xiang.chart.setting", ProcessSetting) +} + +// ProcessData xiang.chart.data +// 查询数据分析图表中定义的数据 +func ProcessData(process *gou.Process) interface{} { + + process.ValidateArgNums(3) + name := process.ArgsString(0) + params := process.ArgsMap(1) + chart := Select(name) + api := chart.APIs["data"] + if process.NumOfArgsIs(4) && api.IsAllow(process.Args[3]) { + return nil + } + + if len(api.Default) > 0 { + if defaults, ok := api.Default[0].(map[string]interface{}); ok { + for key, value := range defaults { + if !params.Has(key) { + params.Set(key, value) + } + } + } + } + + return chart.GetData(params) +} + +// ProcessSetting xiang.chart.setting +// 查询数据分析图表中定义的数据 +func ProcessSetting(process *gou.Process) interface{} { + + process.ValidateArgNums(2) + name := process.ArgsString(0) + field := process.ArgsString(1) + chart := Select(name) + api := chart.APIs["setting"] + if process.NumOfArgsIs(2) && api.IsAllow(process.Args[1]) { + return nil + } + + fields := strings.Split(field, ",") + setting := maps.Map{ + "name": chart.Name, + "label": chart.Label, + "version": chart.Version, + "description": chart.Description, + "filters": chart.Filters, + "page": chart.Page, + } + + if len(fields) == 1 && setting.Has(fields[0]) { + field := strings.TrimSpace(fields[0]) + return setting.Get(field) + } + + if len(fields) > 1 { + res := maps.Map{} + for _, field := range fields { + field = strings.TrimSpace(field) + if setting.Has(field) { + res.Set(field, setting.Get(field)) + } + } + return res + } + return setting + +} diff --git a/chart/process_test.go b/chart/process_test.go new file mode 100644 index 00000000..7d0a744b --- /dev/null +++ b/chart/process_test.go @@ -0,0 +1,77 @@ +package chart + +import ( + "net/url" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/yaoapp/gou" + "github.com/yaoapp/kun/any" + "github.com/yaoapp/kun/utils" + "github.com/yaoapp/xiang/config" + _ "github.com/yaoapp/xiang/helper" + "github.com/yaoapp/xiang/model" + "github.com/yaoapp/xiang/query" + "github.com/yaoapp/xiang/share" +) + +func init() { + share.DBConnect(config.Conf.Database) + model.Load(config.Conf) + query.Load(config.Conf) + Load(config.Conf) +} +func TestProcessSetting(t *testing.T) { + + args := []interface{}{ + "service.compare", + nil, + &gin.Context{}, + } + process := gou.NewProcess("xiang.chart.Setting", args...) + response := ProcessSetting(process) + assert.NotNil(t, response) + res := any.Of(response).Map() + assert.True(t, res.Has("name")) + assert.True(t, res.Has("label")) + assert.True(t, res.Has("description")) + assert.True(t, res.Has("page")) + assert.True(t, res.Has("version")) + + args = []interface{}{ + "service.compare", + "page,name", + &gin.Context{}, + } + process = gou.NewProcess("xiang.chart.Setting", args...) + response = ProcessSetting(process) + assert.NotNil(t, response) + + res = any.Of(response).Map() + assert.True(t, res.Has("name")) + assert.True(t, res.Has("page")) + assert.False(t, res.Has("label")) +} + +func TestProcessData(t *testing.T) { + + params := url.Values{ + "from": []string{"1981-01-01", "1990-01-01"}, + } + params.Set("to", "2049-12-31") + + args := []interface{}{ + "service.compare", + params, + &gin.Context{}, + } + process := gou.NewProcess("xiang.chart.Data", args...) + response := ProcessData(process) + utils.Dump(response) + + assert.NotNil(t, response) + + res := any.Of(response).Map().Dot() + assert.Equal(t, "北京", res.Get("合并.0.城市")) +} diff --git a/chart/types.go b/chart/types.go index 6fd8e4aa..30514724 100644 --- a/chart/types.go +++ b/chart/types.go @@ -8,14 +8,7 @@ import ( // Chart 图表格式 type Chart struct { gou.Flow - APIs map[string]API `json:"apis,omitempty"` + APIs map[string]share.API `json:"apis,omitempty"` Filters map[string]share.Filter `json:"filters,omitempty"` Page share.Page `json:"page,omitempty"` } - -// API 图表 API -type API struct { - Disable bool `json:"disable,omitempty"` - Guard string `json:"guard,omitempty"` - Default interface{} `json:"default,omitempty"` -} diff --git a/helper/array.go b/helper/array.go new file mode 100644 index 00000000..0712b69f --- /dev/null +++ b/helper/array.go @@ -0,0 +1,78 @@ +package helper + +import ( + "fmt" + + jsoniter "github.com/json-iterator/go" + "github.com/yaoapp/kun/exception" +) + +// ArrayPluckValue ArrayPluck 参数 +type ArrayPluckValue struct { + Key string `json:"key"` + Value string `json:"value"` + Items []map[string]interface{} `json:"items"` +} + +// ArrayPluck 将多个数据记录集合，合并为一个数据记录集合 +// columns: ["城市", "行业", "计费"] +// pluck: { +// "行业":{"key":"city", "value":"数量", "items":[{"city":"北京", "数量":32},{"city":"上海", "数量":20}]}, +// "计费":{"key":"city", "value":"计费种类", "items":[{"city":"北京", "计费种类":6},{"city":"西安", "计费种类":3}]}, +// } +// return: [ +// {"城市":"北京", "行业":32, "计费":6}, +// {"城市":"上海", "行业":20, "计费":null}, +// {"城市":"西安", "行业":null, "计费":6} +// ] +func ArrayPluck(columns []string, pluck map[string]interface{}) []map[string]interface{} { + if len(columns) < 2 { + exception.New("ArrayPluck 参数错误, 应至少包含两列。", 400).Ctx(columns).Throw() + } + + primary := columns[0] + data := map[string]map[string]interface{}{} + + // 解析数据 + for name, value := range pluck { // name="行业", value={"key":"city", "value":"数量", "items":[{"city":"北京", "数量":32},{"city":"上海", "数量":20}]}, + arg := OfArrayPluckValue(value) + for _, item := range arg.Items { // item = [{"city":"北京", "数量":32},{"city":"上海", "数量":20}] + if v, has := item[arg.Key]; has { // arg.Key = "city" + key := fmt.Sprintf("%#v", v) // key = `"北京"` + val := item[arg.Value] // arg.Value = "数量", val = 32 + if _, has := data[key]; !has { + data[key] = map[string]interface{}{} // {`"北京"`: {}} + data[key][primary] = v // {`"北京"`: {"城市":"北京"}} + } + data[key][name] = val // {`"北京"`: {"城市":"北京", "行业":32}} + } + } + } + + // 空值处理 + res := []map[string]interface{}{} + for key := range data { // key = `"北京"` + for name := range pluck { // name = "行业" + if _, has := data[key][name]; !has { + data[key][name] = nil + } + } + res = append(res, data[key]) + } + + return res +} + +// OfArrayPluckValue Any 转 ArrayPluckValue +func OfArrayPluckValue(any interface{}) ArrayPluckValue { + content, err := jsoniter.Marshal(any) + if err != nil { + exception.New("ArrayPluck 参数错误", 400).Ctx(err.Error()).Throw() + } + value := ArrayPluckValue{Items: []map[string]interface{}{}} + err = jsoniter.Unmarshal(content, &value) + if err != nil { + exception.New("ArrayPluck 参数错误", 400).Ctx(err.Error()).Throw() + } + return value +} diff --git a/helper/array_test.go b/helper/array_test.go new file mode 100644 index 00000000..586101c1 --- /dev/null +++ b/helper/array_test.go @@ -0,0 +1,23 @@ +package helper + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/kun/maps" +) + +func TestArrayPluck(t *testing.T) { + columns := []string{"城市", "行业", "计费"} + pluck := map[string]interface{}{ + "行业": map[string]interface{}{"key": "city", "value": "数量", "items": []map[string]interface{}{{"city": "北京", "数量": 32}, {"city": "上海", "数量": 20}}}, + "计费": map[string]interface{}{"key": "city", "value": "计费种类", "items": []map[string]interface{}{{"city": "北京", "计费种类": 6}, {"city": "西安", "计费种类": 3}}}, + } + items := ArrayPluck(columns, pluck) + assert.Equal(t, 3, len(items)) + for _, item := range items { + maps.Of(item).Has("城市") + maps.Of(item).Has("行业") + maps.Of(item).Has("计费") + } +} diff --git a/helper/process.go b/helper/process.go new file mode 100644 index 00000000..98a9216a --- /dev/null +++ b/helper/process.go @@ -0,0 +1,36 @@ +package helper + +import ( + "github.com/yaoapp/gou" + "github.com/yaoapp/kun/exception" +) + +func init() { + // 注册处理器 + gou.RegisterProcessHandler("xiang.helper.ArrayPluck", ProcessArrayPluck) + +} + +// ProcessArrayPluck xiang.helper.ArrayPluck 将多个数据记录集合，合并为一个数据记录集合 +func ProcessArrayPluck(process *gou.Process) interface{} { + process.ValidateArgNums(2) + columnsAny := process.Args[0] + columns := []string{} + switch columnsAny.(type) { + case []interface{}: + for _, v := range columnsAny.([]interface{}) { + value, ok := v.(string) + if ok { + columns = append(columns, value) + continue + } + exception.New("参数错误: 第1个参数不是字符串数组", 400).Ctx(process.Args[0]).Throw() + } + case []string: + default: + exception.New("参数错误: 第1个参数不是字符串数组", 400).Ctx(process.Args[0]).Throw() + break + } + pluck := process.ArgsMap(1) + return ArrayPluck(columns, pluck) +} diff --git a/helper/process_test.go b/helper/process_test.go new file mode 100644 index 00000000..c6af9f33 --- /dev/null +++ b/helper/process_test.go @@ -0,0 +1,31 @@ +package helper + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/gou" + "github.com/yaoapp/kun/maps" +) + +func TestProcessArrayPluck(t *testing.T) { + args := []interface{}{ + []interface{}{"城市", "行业", "计费"}, + map[string]interface{}{ + "行业": map[string]interface{}{"key": "city", "value": "数量", "items": []map[string]interface{}{{"city": "北京", "数量": 32}, {"city": "上海", "数量": 20}}}, + "计费": map[string]interface{}{"key": "city", "value": "计费种类", "items": []map[string]interface{}{{"city": "北京", "计费种类": 6}, {"city": "西安", "计费种类": 3}}}, + }, + } + process := gou.NewProcess("xiang.helper.ArrayPluck", args...) + response := ProcessArrayPluck(process) + assert.NotNil(t, response) + items, ok := response.([]map[string]interface{}) + assert.True(t, ok) + + assert.Equal(t, 3, len(items)) + for _, item := range items { + maps.Of(item).Has("城市") + maps.Of(item).Has("行业") + maps.Of(item).Has("计费") + } +} diff --git a/share/types.go b/share/types.go index 293e2ec2..a5858334 100644 --- a/share/types.go +++ b/share/types.go @@ -6,6 +6,7 @@ import "github.com/yaoapp/kun/maps" type API struct { Name string `json:"-"` Source string `json:"-"` + Disable bool `json:"disable,omitempty"` Process string `json:"process,omitempty"` Guard string `json:"guard,omitempty"` Default []interface{} `json:"default,omitempty"` diff --git a/tests/charts/service/compare.json b/tests/charts/service/compare.json index d88956af..96557046 100644 --- a/tests/charts/service/compare.json +++ b/tests/charts/service/compare.json @@ -1,53 +1,92 @@ { - "engine": "gou", "label": "指标对比", "version": "1.0.0", - "decription": "指标对比", + "description": "指标对比用于测试", "nodes": [ { - "name": "行业分布", + "name": "行业", "engine": "xiang", "query": { - "select": ["city", ":COUNT(id) as cnt", "industries[*] as industry"], + "debug": true, + "select": [ + "city", + ":COUNT(id) as 数量", + "industries[*](string 50) as industry" + ], "from": "$service", "wheres": [ - { "field": "created_at", ">": "?:$in.from" }, - { "field": "created_at", "<": "?:$in.to" } + { "field": "created_at", ">": "?:$from.0" }, + { "field": "created_at", "<": "?:$to" } ], - "order": "cnt desc", - "limit": 100, - "group": ["industries[*]", "city"] + "orders": "数量 desc", + "limit": 100 } }, { - "name": "计费方式", + "name": "计费", "engine": "xiang", "query": { - "select": ["city", ":COUNT(id) as cnt", "price_options[*] as option"], + "select": ["city", ":COUNT(id) as 数量", "price_options[*] as option"], "from": "$service", "wheres": [ - { "field": "created_at", ">": "?:$in.from" }, - { "field": "created_at", "<": "?:$in.to" } + { "field": "created_at", ">": "?:$from.0" }, + { "field": "created_at", "<": "?:$to" } ], - "order": "cnt desc", - "limit": 100, - "group": ["price_options[*]", "city"] + "orders": "数量 desc", + "limit": 100 } }, { "name": "合并结果", - "process": "xiang.chart.MergeData", + "process": "xiang.helper.ArrayPluck", "args": [ ["城市", "行业", "计费"], { - "行业": { "key": "city", "values": "{{$res.行业分布[*].industry}}" }, - "计费": { "key": "city", "values": "{{$res.计费方式[*].option}}" } + "行业": { "key": "city", "value": "数量", "items": "{{$res.行业}}" }, + "计费": { "key": "city", "value": "数量", "items": "{{$res.计费}}" } } ] } ], - "output": "{{$res.合并结果}}", - "filters": {}, + "output": { + "合并": "{{$res.合并结果}}", + "计费": "{{$res.计费}}", + "行业": "{{$res.行业}}", + "参数": "{{$in}}" + }, + "apis": { + "data": { + "disable": false, + "guard": "-", + "default": [{ "from": "1980-01-02", "to": "2050-12-31" }] + }, + "setting": { + "disable": false, + "guard": "-" + } + }, + "filters": { + "开始时间": { + "label": "开始时间", + "bind": "from", + "input": { + "type": "date", + "props": { + "placeholder": "请选择开始时间" + } + } + }, + "结束时间": { + "label": "结束时间", + "bind": "to", + "input": { + "type": "date", + "props": { + "placeholder": "请选择结束时间" + } + } + } + }, "page": { "primary": "城市", "layout": { diff --git a/tests/models/service.json b/tests/models/service.json index c2e5c18b..f624af76 100644 --- a/tests/models/service.json +++ b/tests/models/service.json @@ -22,6 +22,13 @@ "comment": "服务名称", "index": true }, + { + "name": "city", + "type": "string", + "length": 50, + "comment": "城市", + "index": true + }, { "name": "kind_id", "type": "bigInteger", @@ -123,30 +130,38 @@ "values": [ { "name": "腾讯云主机CVM", + "city": "北京", "short_name": "云主机", "kind_id": 3, "manu_id": 1, + "industries": ["教育", "能源"], "price_options": ["按月订阅"] }, { "name": "腾讯云磁盘", + "city": "广州", "short_name": "云磁盘", "kind_id": 6, "manu_id": 1, + "industries": ["政府", "能源"], "price_options": ["按月订阅"] }, { "name": "阿里主机", + "city": "上海", "short_name": "云主机", "kind_id": 3, "manu_id": 2, + "industries": ["教育", "医疗"], "price_options": ["按月订阅"] }, { "name": "UCloud云主机", "short_name": "云主机", + "city": "上海", "kind_id": 3, "manu_id": 3, + "industries": ["软件", "医疗"], "price_options": ["按月订阅"] } ] diff --git a/xiang/apis/chart.http.json b/xiang/apis/chart.http.json new file mode 100644 index 00000000..2aedb3fe --- /dev/null +++ b/xiang/apis/chart.http.json @@ -0,0 +1,29 @@ +{ + "name": "数据分析接口", + "version": "1.0.0", + "description": "数据分析接口API", + "group": "xiang/chart", + "guard": "in-process", + "paths": [ + { + "path": "/:name/data", + "method": "GET", + "process": "xiang.chart.Data", + "in": ["$param.name", ":query-param"], + "out": { + "status": 200, + "type": "application/json" + } + }, + { + "path": "/:name/setting", + "method": "GET", + "process": "xiang.chart.Setting", + "in": ["$param.name", "$query.select"], + "out": { + "status": 200, + "type": "application/json" + } + } + ] +}