diff --git a/cmd/start.go b/cmd/start.go index d5793c09..62f688f6 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" "syscall" + "time" "github.com/fatih/color" "github.com/spf13/cobra" @@ -77,13 +78,32 @@ var startCmd = &cobra.Command{ config.Development() } + startTime := time.Now() + // load the application engine - loadWarnings, err := engine.Load(config.Conf, engine.LoadOption{Action: "start"}) + var progressCallback func(string, string) + if config.Conf.Mode == "development" { + fmt.Println(color.CyanString("Loading application engine...")) + progressCallback = func(name string, duration string) { + fmt.Printf(" %s %s %s\n", color.GreenString("✓"), name, color.GreenString("(%s)", duration)) + } + } + + loadWarnings, err := engine.Load(config.Conf, engine.LoadOption{ + Action: "start", + }, progressCallback) if err != nil { fmt.Println(color.RedString(L("Load: %s"), err.Error())) os.Exit(1) } + loadDuration := time.Since(startTime) + if config.Conf.Mode == "development" { + fmt.Printf("\n%s Engine loaded successfully in %s\n\n", + color.GreenString("✓"), + color.CyanString("%v", loadDuration)) + } + port := fmt.Sprintf(":%d", config.Conf.Port) if port == ":80" { port = "" diff --git a/engine/load.go b/engine/load.go index 5eb5c95e..873de786 100644 --- a/engine/load.go +++ b/engine/load.go @@ -5,6 +5,7 @@ import ( "os" "regexp" "strings" + "time" "github.com/fatih/color" "github.com/yaoapp/gou/application" @@ -70,10 +71,28 @@ type Warning struct { Error error } +// loadStep wraps a loading function with timing and progress reporting +func loadStep(name string, loadFunc func() error, callback func(string, string)) error { + start := time.Now() + err := loadFunc() + duration := time.Since(start) + + if callback != nil { + callback(name, duration.String()) + } + + return err +} + // Load application engine -func Load(cfg config.Config, options LoadOption) (warnings []Warning, err error) { +func Load(cfg config.Config, options LoadOption, progressCallback ...func(string, string)) (warnings []Warning, err error) { defer func() { err = exception.Catch(recover()) }() + + var callback func(string, string) + if len(progressCallback) > 0 { + callback = progressCallback[0] + } exception.Mode = cfg.Mode // SET XGEN_BASE @@ -86,116 +105,133 @@ func Load(cfg config.Config, options LoadOption) (warnings []Warning, err error) os.Setenv("XGEN_BASE", adminRoot) // load the application - err = loadApp(cfg.AppSource) + err = loadStep("Load Application", func() error { + return loadApp(cfg.AppSource) + }, callback) if err != nil { printErr(cfg.Mode, "Load Application", err) warnings = append(warnings, Warning{Widget: "Load Application", Error: err}) } // Make Database connections - err = share.DBConnect(cfg.DB) + err = loadStep("DB", func() error { + return share.DBConnect(cfg.DB) + }, callback) if err != nil { - // printErr(cfg.Mode, "DB", err) warnings = append(warnings, Warning{Widget: "DB", Error: err}) } // Load Certs - err = cert.Load(cfg) + err = loadStep("Cert", func() error { + return cert.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Cert", err) warnings = append(warnings, Warning{Widget: "Cert", Error: err}) } // Load Connectors - err = connector.Load(cfg) + err = loadStep("Connector", func() error { + return connector.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Connector", err) warnings = append(warnings, Warning{Widget: "Connector", Error: err}) } // Load FileSystem - err = fs.Load(cfg) + err = loadStep("FileSystem", func() error { + return fs.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "FileSystem", err) warnings = append(warnings, Warning{Widget: "FileSystem", Error: err}) } // Load i18n - err = i18n.Load(cfg) + err = loadStep("i18n", func() error { + return i18n.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "i18n", err) warnings = append(warnings, Warning{Widget: "i18n", Error: err}) } // start v8 runtime - err = runtime.Start(cfg) + err = loadStep("Runtime", func() error { + return runtime.Start(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Runtime", err) warnings = append(warnings, Warning{Widget: "Runtime", Error: err}) } // Load Query Engine - err = query.Load(cfg) + err = loadStep("Query Engine", func() error { + return query.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Query Engine", err) warnings = append(warnings, Warning{Widget: "Query Engine", Error: err}) } // Load Scripts - err = script.Load(cfg) + err = loadStep("Script", func() error { + return script.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Script", err) warnings = append(warnings, Warning{Widget: "Script", Error: err}) } // Load Models - err = model.Load(cfg) + err = loadStep("Model", func() error { + return model.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Model", err) warnings = append(warnings, Warning{Widget: "Model", Error: err}) } // Load Data flows - err = flow.Load(cfg) + err = loadStep("Flow", func() error { + return flow.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Flow", err) warnings = append(warnings, Warning{Widget: "Flow", Error: err}) } // Load Stores - err = store.Load(cfg) + err = loadStep("Store", func() error { + return store.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Store", err) warnings = append(warnings, Warning{Widget: "Store", Error: err}) } // Load Uploaders - err = attachment.Load(cfg) + err = loadStep("Uploader", func() error { + return attachment.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Uploader", err) warnings = append(warnings, Warning{Widget: "Uploader", Error: err}) } // Load Messengers - err = messenger.Load(cfg) + err = loadStep("Messenger", func() error { + return messenger.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Messenger", err) warnings = append(warnings, Warning{Widget: "Messenger", Error: err}) } // Load Plugins - err = plugin.Load(cfg) + err = loadStep("Plugin", func() error { + return plugin.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Plugin", err) warnings = append(warnings, Warning{Widget: "Plugin", Error: err}) } // Load WASM Application (experimental) // Load build-in widgets (table / form / chart / ...) - err = widgets.Load(cfg) + err = loadStep("Widgets", func() error { + return widgets.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Widgets", err) warnings = append(warnings, Warning{Widget: "Widgets", Error: err}) } @@ -207,100 +243,115 @@ func Load(cfg config.Config, options LoadOption) (warnings []Warning, err error) // } // Load Apis - err = api.Load(cfg) // 加载业务接口 API + err = loadStep("API", func() error { + return api.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "API", err) warnings = append(warnings, Warning{Widget: "API", Error: err}) } // Load Sockets - err = socket.Load(cfg) // Load sockets + err = loadStep("Socket", func() error { + return socket.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Socket", err) warnings = append(warnings, Warning{Widget: "Socket", Error: err}) } // Load websockets (client mode) - err = websocket.Load(cfg) + err = loadStep("WebSocket", func() error { + return websocket.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "WebSocket", err) warnings = append(warnings, Warning{Widget: "WebSocket", Error: err}) } // Load tasks - err = task.Load(cfg) + err = loadStep("Task", func() error { + return task.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Task", err) warnings = append(warnings, Warning{Widget: "Task", Error: err}) } // Load schedules - err = schedule.Load(cfg) + err = loadStep("Schedule", func() error { + return schedule.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Schedule", err) warnings = append(warnings, Warning{Widget: "Schedule", Error: err}) } // Load AIGC - err = aigc.Load(cfg) + err = loadStep("AIGC", func() error { + return aigc.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "AIGC", err) warnings = append(warnings, Warning{Widget: "AIGC", Error: err}) } // Load Custom Widget - err = widget.Load(cfg) + err = loadStep("Widget", func() error { + return widget.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Widget", err) warnings = append(warnings, Warning{Widget: "Widget", Error: err}) } // Load Custom Widget Instances - err = widget.LoadInstances() + err = loadStep("Widget Instances", func() error { + return widget.LoadInstances() + }, callback) if err != nil { - // printErr(cfg.Mode, "Widget", err) warnings = append(warnings, Warning{Widget: "Widget", Error: err}) } // Load SUI - err = sui.Load(cfg) + err = loadStep("SUI", func() error { + return sui.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "SUI", err) warnings = append(warnings, Warning{Widget: "SUI", Error: err}) } // Load Moapi - err = moapi.Load(cfg) + err = loadStep("Moapi", func() error { + return moapi.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Moapi", err) warnings = append(warnings, Warning{Widget: "Moapi", Error: err}) } // Load Pipe - err = pipe.Load(cfg) + err = loadStep("Pipe", func() error { + return pipe.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Pipe", err) warnings = append(warnings, Warning{Widget: "Pipe", Error: err}) } // Load MCP Clients - err = mcp.Load(cfg) + err = loadStep("MCP", func() error { + return mcp.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "MCP", err) warnings = append(warnings, Warning{Widget: "MCP", Error: err}) } // Load Knowledge Base - _, err = kb.Load(cfg) + err = loadStep("Knowledge Base", func() error { + _, err := kb.Load(cfg) + return err + }, callback) if err != nil { - // printErr(cfg.Mode, "Knowledge Base", err) warnings = append(warnings, Warning{Widget: "Knowledge Base", Error: err}) } // Load Agent - err = agent.Load(cfg) + err = loadStep("Agent", func() error { + return agent.Load(cfg) + }, callback) if err != nil { - // printErr(cfg.Mode, "Agent", err) warnings = append(warnings, Warning{Widget: "Agent", Error: err}) } @@ -313,22 +364,24 @@ func Load(cfg config.Config, options LoadOption) (warnings []Warning, err error) } // Load OpenAPI - _, err = openapi.Load(cfg) + err = loadStep("OpenAPI", func() error { + _, err := openapi.Load(cfg) + return err + }, callback) if err != nil { - // printErr(cfg.Mode, "OpenAPI", err) warnings = append(warnings, Warning{Widget: "OpenAPI", Error: err}) } // Execute AfterLoad Process if exists if share.App.AfterLoad != "" && !options.IgnoredAfterLoad { - p, err := process.Of(share.App.AfterLoad, options) - if err != nil { - printErr(cfg.Mode, "AfterLoad", err) - warnings = append(warnings, Warning{Widget: "AfterLoad", Error: err}) - return warnings, err - } - - _, err = p.Exec() + err = loadStep("AfterLoad", func() error { + p, err := process.Of(share.App.AfterLoad, options) + if err != nil { + return err + } + _, err = p.Exec() + return err + }, callback) if err != nil { printErr(cfg.Mode, "AfterLoad", err) warnings = append(warnings, Warning{Widget: "AfterLoad", Error: err}) diff --git a/model/migrate.go b/model/migrate.go new file mode 100644 index 00000000..2a785d30 --- /dev/null +++ b/model/migrate.go @@ -0,0 +1,86 @@ +package model + +import ( + "fmt" + "os" + "time" + + "github.com/fatih/color" + "github.com/yaoapp/gou/model" + "github.com/yaoapp/gou/schema" + "github.com/yaoapp/kun/log" +) + +// BatchMigrate batch migrate models after checking which tables are missing +// This optimizes the migration process by querying database only once +func BatchMigrate(models map[string]*model.Model) error { + if len(models) == 0 { + return nil + } + + start := time.Now() + + // Get the connector (assume all system/agent models use default connector) + connector := "default" + sch := schema.Use(connector) + + // Step 1: Get all existing tables in one query + existingTables, err := sch.Tables() + if err != nil { + return fmt.Errorf("failed to get existing tables: %w", err) + } + + // Build a map for fast lookup + tableExists := make(map[string]bool) + for _, table := range existingTables { + tableExists[table] = true + } + + // Step 2: Identify models that need creation (skip existing tables) + needCreate := make(map[string]*model.Model) + + for id, mod := range models { + tableName := mod.MetaData.Table.Name + if tableName == "" { + log.Warn("Model %s has no table name, skipping", id) + continue + } + + if !tableExists[tableName] { + needCreate[id] = mod + } + } + + // Step 3: Create missing tables only + if len(needCreate) > 0 { + isDevelopment := os.Getenv("YAO_ENV") == "development" + + if isDevelopment { + fmt.Printf(" %s Creating %d tables...\n", color.CyanString("→"), len(needCreate)) + } + + for id, mod := range needCreate { + createStart := time.Now() + err := mod.CreateTable() + if err != nil { + log.Error("Failed to create table for model %s: %s", id, err.Error()) + return fmt.Errorf("failed to create table for %s: %w", id, err) + } + + duration := time.Since(createStart) + if isDevelopment { + fmt.Printf(" %s %s %s\n", + color.GreenString("✓"), + mod.MetaData.Table.Name, + color.GreenString("(%v)", duration)) + } else { + log.Info("Created table: %s (%v)", mod.MetaData.Table.Name, duration) + } + } + } + + log.Trace("Batch migrate completed: %d models checked, %d tables created (%v)", + len(models), len(needCreate), time.Since(start)) + + return nil +} diff --git a/model/migrate_test.go b/model/migrate_test.go new file mode 100644 index 00000000..1319f726 --- /dev/null +++ b/model/migrate_test.go @@ -0,0 +1,73 @@ +package model + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/gou/model" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +func TestBatchMigrate(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + t.Run("LoadSystemModels", func(t *testing.T) { + models, err := loadSystemModels() + assert.NoError(t, err, "Should load system models without error") + assert.NotEmpty(t, models, "Should have loaded system models") + + // Check that all models have table names + for id, mod := range models { + assert.NotEmpty(t, mod.MetaData.Table.Name, "Model %s should have table name", id) + } + + t.Logf("Loaded %d system models", len(models)) + }) + + t.Run("LoadAssistantModels", func(t *testing.T) { + models, errs := loadAssistantModels() + assert.Empty(t, errs, "Should load assistant models without critical errors") + + t.Logf("Loaded %d assistant models", len(models)) + }) + + t.Run("BatchMigrateAllModels", func(t *testing.T) { + // Load all models + systemModels, err := loadSystemModels() + assert.NoError(t, err) + + assistantModels, _ := loadAssistantModels() + + // Combine all models + allModels := make(map[string]*model.Model) + for id, mod := range systemModels { + allModels[id] = mod + } + for id, mod := range assistantModels { + allModels[id] = mod + } + + // Run batch migrate + err = BatchMigrate(allModels) + assert.NoError(t, err, "Batch migrate should succeed") + + t.Logf("Batch migrated %d models", len(allModels)) + }) + + t.Run("BatchMigrateIdempotent", func(t *testing.T) { + // Load models + systemModels, err := loadSystemModels() + assert.NoError(t, err) + + // Run batch migrate twice - should be idempotent + err = BatchMigrate(systemModels) + assert.NoError(t, err, "First batch migrate should succeed") + + err = BatchMigrate(systemModels) + assert.NoError(t, err, "Second batch migrate should also succeed (idempotent)") + + t.Logf("Batch migrate is idempotent") + }) +} diff --git a/model/model.go b/model/model.go index 3b71629a..d10cccef 100644 --- a/model/model.go +++ b/model/model.go @@ -50,8 +50,8 @@ func Load(cfg config.Config) error { model.WithCrypt([]byte(fmt.Sprintf(`{"key":"%s"}`, cfg.DB.AESKey)), "AES") model.WithCrypt([]byte(`{}`), "PASSWORD") - // Load system models - err := loadSystemModels() + // Load system models (without migrate) + systemModels, err := loadSystemModels() if err != nil { return err } @@ -76,14 +76,28 @@ func Load(cfg config.Config) error { return fmt.Errorf("%s", strings.Join(messages, ";\n")) } - // Load models from assistants - errsAssistants := loadAssistantModels() + // Load models from assistants (without migrate) + assistantModels, errsAssistants := loadAssistantModels() if len(errsAssistants) > 0 { for _, err := range errsAssistants { log.Error("Load assistant models error: %s", err.Error()) } } + // Batch migrate all system and assistant models + allModels := make(map[string]*model.Model) + for id, mod := range systemModels { + allModels[id] = mod + } + for id, mod := range assistantModels { + allModels[id] = mod + } + + err = BatchMigrate(allModels) + if err != nil { + return err + } + // Load database models ( ignore error) errs := loadDatabaseModels() if len(errs) > 0 { @@ -94,19 +108,21 @@ func Load(cfg config.Config) error { return err } -// LoadSystemModels load system models -func loadSystemModels() error { +// LoadSystemModels load system models (without migration) +func loadSystemModels() (map[string]*model.Model, error) { + models := make(map[string]*model.Model) + for id, path := range systemModels { content, err := data.Read(path) if err != nil { - return err + return nil, err } // Parse model var data map[string]interface{} err = application.Parse(path, content, &data) if err != nil { - return err + return nil, err } // Set prefix @@ -116,38 +132,34 @@ func loadSystemModels() error { content, err = jsoniter.Marshal(data) if err != nil { log.Error("failed to marshal model data: %v", err) - return fmt.Errorf("failed to marshal model data: %v", err) + return nil, fmt.Errorf("failed to marshal model data: %v", err) } } } - // Load Model + // Load Model (just parse, no migration) mod, err := model.LoadSource(content, id, filepath.Join("__system", path)) if err != nil { log.Error("load system model %s error: %s", id, err.Error()) - return err + return nil, err } - // Auto migrate - err = mod.Migrate(false, model.WithDonotInsertValues(true)) - if err != nil { - log.Error("migrate system model %s error: %s", id, err.Error()) - return err - } + models[id] = mod } - return nil + return models, nil } -// loadAssistantModels load models from assistants directory -func loadAssistantModels() []error { +// loadAssistantModels load models from assistants directory (without migration) +func loadAssistantModels() (map[string]*model.Model, []error) { + models := make(map[string]*model.Model) var errs []error = []error{} // Check if assistants directory exists exists, err := application.App.Exists("assistants") if err != nil || !exists { log.Trace("Assistants directory not found or not accessible") - return errs + return models, errs } log.Trace("Loading models from assistants directory...") @@ -247,7 +259,7 @@ func loadAssistantModels() []error { } } - // Load model with modified content + // Load model with modified content (just parse, no migration) mod, err := model.LoadSource(content, modelID, modelFile) if err != nil { log.Error("Failed to load model %s from assistant %s: %s", modelID, assistantID, err.Error()) @@ -255,15 +267,8 @@ func loadAssistantModels() []error { return nil // Continue loading other models } - // Auto migrate the model (like system models) - err = mod.Migrate(false, model.WithDonotInsertValues(true)) - if err != nil { - log.Error("Failed to migrate model %s from assistant %s: %s", modelID, assistantID, err.Error()) - errs = append(errs, fmt.Errorf("failed to migrate model %s: %w", modelID, err)) - return nil - } - - log.Info("Loaded and migrated model: %s", modelID) + models[modelID] = mod + log.Trace("Loaded model: %s", modelID) return nil }, exts...) @@ -278,7 +283,7 @@ func loadAssistantModels() []error { errs = append(errs, fmt.Errorf("failed to walk assistants directory: %w", err)) } - return errs + return models, errs } // LoadDatabaseModels load database models