From bdd7c0865e59cda5ffb4b75b6aa294d3a1b18cb7 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 15 Jul 2025 16:58:41 +0800 Subject: [PATCH] Refactor DSL list retrieval to support conditional data source selection - Enhanced the List method in the DSL struct to conditionally retrieve data from either the database or file system based on the StoreType specified in ListOptions. - Updated the Info struct to include a new Source field for optional source content retrieval. - Modified the List methods in both DB and FS to include source information when requested, improving data accessibility and flexibility. --- dsl/dsl.go | 23 ++++++++++++++------- dsl/io/db.go | 10 ++++++++-- dsl/io/fs.go | 16 +++++++++++++++ dsl/types/types.go | 36 ++++++++++++++++++--------------- model/model.go | 50 +++++++++++++++++++++++++++++++++++++++------- 5 files changed, 103 insertions(+), 32 deletions(-) diff --git a/dsl/dsl.go b/dsl/dsl.go index a1c94197..4377e070 100644 --- a/dsl/dsl.go +++ b/dsl/dsl.go @@ -127,15 +127,24 @@ func (dsl *DSL) Source(ctx context.Context, id string) (string, error) { // List DSLs func (dsl *DSL) List(ctx context.Context, opts *types.ListOptions) ([]*types.Info, error) { // Get the list from the db - dbList, err := dsl.db.List(opts) - if err != nil { - return nil, err + var dbList []*types.Info + var fileList []*types.Info + var err error + + // If StoreType is not specified or is DB, get from db + if opts.Store == "" || opts.Store == types.StoreTypeDB { + dbList, err = dsl.db.List(opts) + if err != nil { + return nil, err + } } - // Get the list from the file - fileList, err := dsl.fs.List(opts) - if err != nil { - return nil, err + // If StoreType is not specified or is File, get from file + if opts.Store == "" || opts.Store == types.StoreTypeFile { + fileList, err = dsl.fs.List(opts) + if err != nil { + return nil, err + } } // Merge the list and unique diff --git a/dsl/io/db.go b/dsl/io/db.go index 84bb445c..332f9f5d 100644 --- a/dsl/io/db.go +++ b/dsl/io/db.go @@ -118,10 +118,16 @@ func (db *DB) List(options *types.ListOptions) ([]*types.Info, error) { wheres = append(wheres, model.QueryWhere{Wheres: orwheres}) } + // Select fields + fields := []interface{}{"dsl_id", "label", "path", "sort", "tags", "description", "status", "store", "mtime", "ctime"} + if options.Source { + fields = append(fields, "source") + } + // Get the list rows, err := m.Get(model.QueryParam{ - Wheres: []model.QueryWhere{{Column: "type", Value: db.Type}}, - Select: []interface{}{"dsl_id", "label", "path", "sort", "tags", "description", "status", "store", "mtime", "ctime"}, + Wheres: wheres, + Select: fields, Orders: orders, }) if err != nil { diff --git a/dsl/io/fs.go b/dsl/io/fs.go index 22bdd4ae..0b566465 100644 --- a/dsl/io/fs.go +++ b/dsl/io/fs.go @@ -101,6 +101,14 @@ func (fs *FS) List(options *types.ListOptions) ([]*types.Info, error) { for _, tag := range options.Tags { for _, t := range info.Tags { if t == tag { + if options.Source { + source, _, err := fs.Source(id) + if err != nil { + errs = append(errs, err) + return nil + } + info.Source = source + } infos = append(infos, info) return nil } @@ -109,6 +117,14 @@ func (fs *FS) List(options *types.ListOptions) ([]*types.Info, error) { } // Add to the list + if options.Source { + source, _, err := fs.Source(id) + if err != nil { + errs = append(errs, err) + return nil + } + info.Source = source + } infos = append(infos, info) return err }, patterns...) diff --git a/dsl/types/types.go b/dsl/types/types.go index ec4bbdb2..9d05680f 100644 --- a/dsl/types/types.go +++ b/dsl/types/types.go @@ -78,30 +78,34 @@ const ( // Info for DSL type Info struct { - ID string + ID string `json:"id" yaml:"id"` // Unique identifier for the DSL instance - Type Type - Label string - Description string - Tags []string + Type Type `json:"type" yaml:"type"` // DSL type (model, api, table, form, list, chart, dashboard, etc.) + Label string `json:"label,omitempty" yaml:"label,omitempty"` // Display name for the DSL + Description string `json:"description,omitempty" yaml:"description,omitempty"` // Detailed description of the DSL + Tags []string `json:"tags,omitempty" yaml:"tags,omitempty"` // Tags for categorization and filtering - Sort int - Path string - Store StoreType + Sort int `json:"sort,omitempty" yaml:"sort,omitempty"` // Sort order for display, default is 0 + Path string `json:"path" yaml:"path"` // File system path or identifier + Store StoreType `json:"store" yaml:"store"` // Storage type (file or database) - Readable bool - Builtin bool + Readable bool `json:"readable,omitempty" yaml:"readable,omitempty"` // Whether the DSL is readable + Builtin bool `json:"built_in,omitempty" yaml:"built_in,omitempty"` // Whether this is a built-in DSL - Status Status - Mtime time.Time - Ctime time.Time + Status Status `json:"status,omitempty" yaml:"status,omitempty"` // Current status (loading, loaded, error) + Mtime time.Time `json:"mtime" yaml:"mtime"` // Last modification timestamp + Ctime time.Time `json:"ctime" yaml:"ctime"` // Creation timestamp + + Source string `json:"source,omitempty" yaml:"source,omitempty"` // Source content, only available when explicitly requested } // ListOptions for DSL list type ListOptions struct { - Sort string - Order string - Tags []string + Sort string + Order string + Store StoreType + Source bool + Tags []string } // CreateOptions for DSL upsert diff --git a/model/model.go b/model/model.go index 9989a5eb..bffbff8f 100644 --- a/model/model.go +++ b/model/model.go @@ -1,9 +1,11 @@ package model import ( + "context" "fmt" "path/filepath" "strings" + "time" jsoniter "github.com/json-iterator/go" "github.com/yaoapp/gou/application" @@ -11,6 +13,8 @@ import ( "github.com/yaoapp/kun/log" "github.com/yaoapp/yao/config" "github.com/yaoapp/yao/data" + "github.com/yaoapp/yao/dsl" + "github.com/yaoapp/yao/dsl/types" "github.com/yaoapp/yao/share" ) @@ -54,18 +58,23 @@ func Load(cfg config.Config) error { }, exts...) if len(messages) > 0 { + for _, message := range messages { + log.Error("Load filesystem models error: %s", message) + } return fmt.Errorf(strings.Join(messages, ";\n")) } // Load database models ( ignore error) - err = loadDatabaseModels() - if err != nil { - log.Error("load database models error: %s", err.Error()) + errs := loadDatabaseModels() + if len(errs) > 0 { + for _, err := range errs { + log.Error("Load database models error: %s", err.Error()) + } } - return err } +// LoadSystemModels load system models func loadSystemModels() error { for id, path := range systemModels { content, err := data.Read(path) @@ -100,7 +109,7 @@ func loadSystemModels() error { } // Auto migrate - err = mod.Migrate(true, model.WithDonotInsertValues(true)) + err = mod.Migrate(false, model.WithDonotInsertValues(true)) if err != nil { log.Error("migrate system model %s error: %s", id, err.Error()) return err @@ -110,6 +119,33 @@ func loadSystemModels() error { return nil } -func loadDatabaseModels() error { - return nil +// LoadDatabaseModels load database models +func loadDatabaseModels() []error { + + var errs []error = []error{} + manager, err := dsl.New(types.TypeModel) + if err != nil { + errs = append(errs, err) + return errs + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + models, err := manager.List(ctx, &types.ListOptions{Store: types.StoreTypeDB, Source: true}) + if err != nil { + errs = append(errs, err) + return errs + } + + // Load models + for _, info := range models { + _, err := model.LoadSource([]byte(info.Source), info.ID, info.Path) + if err != nil { + errs = append(errs, err) + continue + } + } + + return errs }