Enhance engine loading process with progress reporting and timing

- Introduced a `loadStep` function to wrap loading operations, providing timing and progress callback functionality.
- Updated the `Load` function to utilize `loadStep` for various components, improving visibility into loading durations.
- Enhanced the `start` command to display loading progress and duration in development mode, improving user experience during application startup.
- Refactored model loading functions to return models without migration, streamlining the loading process and improving performance.

cmd/start.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/fatih/color"
 	"github.com/spf13/cobra"
@@ -77,13 +78,32 @@ var startCmd = &cobra.Command{
 			config.Development()
 		}
 
+		startTime := time.Now()
+
 		// load the application engine
-		loadWarnings, err := engine.Load(config.Conf, engine.LoadOption{Action: "start"})
+		var progressCallback func(string, string)
+		if config.Conf.Mode == "development" {
+			fmt.Println(color.CyanString("Loading application engine..."))
+			progressCallback = func(name string, duration string) {
+				fmt.Printf("  %s %s %s\n", color.GreenString("✓"), name, color.GreenString("(%s)", duration))
+			}
+		}
+
+		loadWarnings, err := engine.Load(config.Conf, engine.LoadOption{
+			Action: "start",
+		}, progressCallback)
 		if err != nil {
 			fmt.Println(color.RedString(L("Load: %s"), err.Error()))
 			os.Exit(1)
 		}
 
+		loadDuration := time.Since(startTime)
+		if config.Conf.Mode == "development" {
+			fmt.Printf("\n%s Engine loaded successfully in %s\n\n",
+				color.GreenString("✓"),
+				color.CyanString("%v", loadDuration))
+		}
+
 		port := fmt.Sprintf(":%d", config.Conf.Port)
 		if port == ":80" {
 			port = ""

engine/load.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/fatih/color"
 	"github.com/yaoapp/gou/application"
@@ -70,10 +71,28 @@ type Warning struct {
 	Error  error
 }
 
+// loadStep wraps a loading function with timing and progress reporting
+func loadStep(name string, loadFunc func() error, callback func(string, string)) error {
+	start := time.Now()
+	err := loadFunc()
+	duration := time.Since(start)
+
+	if callback != nil {
+		callback(name, duration.String())
+	}
+
+	return err
+}
+
 // Load application engine
-func Load(cfg config.Config, options LoadOption) (warnings []Warning, err error) {
+func Load(cfg config.Config, options LoadOption, progressCallback ...func(string, string)) (warnings []Warning, err error) {
 
 	defer func() { err = exception.Catch(recover()) }()
+
+	var callback func(string, string)
+	if len(progressCallback) > 0 {
+		callback = progressCallback[0]
+	}
 	exception.Mode = cfg.Mode
 
 	// SET XGEN_BASE
@@ -86,116 +105,133 @@ func Load(cfg config.Config, options LoadOption) (warnings []Warning, err error)
 	os.Setenv("XGEN_BASE", adminRoot)
 
 	// load the application
-	err = loadApp(cfg.AppSource)
+	err = loadStep("Load Application", func() error {
+		return loadApp(cfg.AppSource)
+	}, callback)
 	if err != nil {
 		printErr(cfg.Mode, "Load Application", err)
 		warnings = append(warnings, Warning{Widget: "Load Application", Error: err})
 	}
 
 	// Make Database connections
-	err = share.DBConnect(cfg.DB)
+	err = loadStep("DB", func() error {
+		return share.DBConnect(cfg.DB)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "DB", err)
 		warnings = append(warnings, Warning{Widget: "DB", Error: err})
 	}
 
 	// Load Certs
-	err = cert.Load(cfg)
+	err = loadStep("Cert", func() error {
+		return cert.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Cert", err)
 		warnings = append(warnings, Warning{Widget: "Cert", Error: err})
 	}
 
 	// Load Connectors
-	err = connector.Load(cfg)
+	err = loadStep("Connector", func() error {
+		return connector.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Connector", err)
 		warnings = append(warnings, Warning{Widget: "Connector", Error: err})
 	}
 
 	// Load FileSystem
-	err = fs.Load(cfg)
+	err = loadStep("FileSystem", func() error {
+		return fs.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "FileSystem", err)
 		warnings = append(warnings, Warning{Widget: "FileSystem", Error: err})
 	}
 
 	// Load i18n
-	err = i18n.Load(cfg)
+	err = loadStep("i18n", func() error {
+		return i18n.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "i18n", err)
 		warnings = append(warnings, Warning{Widget: "i18n", Error: err})
 	}
 
 	// start v8 runtime
-	err = runtime.Start(cfg)
+	err = loadStep("Runtime", func() error {
+		return runtime.Start(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Runtime", err)
 		warnings = append(warnings, Warning{Widget: "Runtime", Error: err})
 	}
 
 	// Load Query Engine
-	err = query.Load(cfg)
+	err = loadStep("Query Engine", func() error {
+		return query.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Query Engine", err)
 		warnings = append(warnings, Warning{Widget: "Query Engine", Error: err})
 	}
 
 	// Load Scripts
-	err = script.Load(cfg)
+	err = loadStep("Script", func() error {
+		return script.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Script", err)
 		warnings = append(warnings, Warning{Widget: "Script", Error: err})
 	}
 
 	// Load Models
-	err = model.Load(cfg)
+	err = loadStep("Model", func() error {
+		return model.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Model", err)
 		warnings = append(warnings, Warning{Widget: "Model", Error: err})
 	}
 
 	// Load Data flows
-	err = flow.Load(cfg)
+	err = loadStep("Flow", func() error {
+		return flow.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Flow", err)
 		warnings = append(warnings, Warning{Widget: "Flow", Error: err})
 	}
 
 	// Load Stores
-	err = store.Load(cfg)
+	err = loadStep("Store", func() error {
+		return store.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Store", err)
 		warnings = append(warnings, Warning{Widget: "Store", Error: err})
 	}
 
 	// Load Uploaders
-	err = attachment.Load(cfg)
+	err = loadStep("Uploader", func() error {
+		return attachment.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Uploader", err)
 		warnings = append(warnings, Warning{Widget: "Uploader", Error: err})
 	}
 
 	// Load Messengers
-	err = messenger.Load(cfg)
+	err = loadStep("Messenger", func() error {
+		return messenger.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Messenger", err)
 		warnings = append(warnings, Warning{Widget: "Messenger", Error: err})
 	}
 
 	// Load Plugins
-	err = plugin.Load(cfg)
+	err = loadStep("Plugin", func() error {
+		return plugin.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Plugin", err)
 		warnings = append(warnings, Warning{Widget: "Plugin", Error: err})
 	}
 
 	// Load WASM Application (experimental)
 
 	// Load build-in widgets (table / form / chart / ...)
-	err = widgets.Load(cfg)
+	err = loadStep("Widgets", func() error {
+		return widgets.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Widgets", err)
 		warnings = append(warnings, Warning{Widget: "Widgets", Error: err})
 	}
 
@@ -207,100 +243,115 @@ func Load(cfg config.Config, options LoadOption) (warnings []Warning, err error)
 	// }
 
 	// Load Apis
-	err = api.Load(cfg) // 加载业务接口 API
+	err = loadStep("API", func() error {
+		return api.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "API", err)
 		warnings = append(warnings, Warning{Widget: "API", Error: err})
 	}
 
 	// Load Sockets
-	err = socket.Load(cfg) // Load sockets
+	err = loadStep("Socket", func() error {
+		return socket.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Socket", err)
 		warnings = append(warnings, Warning{Widget: "Socket", Error: err})
 	}
 
 	// Load websockets (client mode)
-	err = websocket.Load(cfg)
+	err = loadStep("WebSocket", func() error {
+		return websocket.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "WebSocket", err)
 		warnings = append(warnings, Warning{Widget: "WebSocket", Error: err})
 	}
 
 	// Load tasks
-	err = task.Load(cfg)
+	err = loadStep("Task", func() error {
+		return task.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Task", err)
 		warnings = append(warnings, Warning{Widget: "Task", Error: err})
 	}
 
 	// Load schedules
-	err = schedule.Load(cfg)
+	err = loadStep("Schedule", func() error {
+		return schedule.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Schedule", err)
 		warnings = append(warnings, Warning{Widget: "Schedule", Error: err})
 	}
 
 	// Load AIGC
-	err = aigc.Load(cfg)
+	err = loadStep("AIGC", func() error {
+		return aigc.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "AIGC", err)
 		warnings = append(warnings, Warning{Widget: "AIGC", Error: err})
 	}
 
 	// Load Custom Widget
-	err = widget.Load(cfg)
+	err = loadStep("Widget", func() error {
+		return widget.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Widget", err)
 		warnings = append(warnings, Warning{Widget: "Widget", Error: err})
 	}
 
 	// Load Custom Widget Instances
-	err = widget.LoadInstances()
+	err = loadStep("Widget Instances", func() error {
+		return widget.LoadInstances()
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Widget", err)
 		warnings = append(warnings, Warning{Widget: "Widget", Error: err})
 	}
 
 	// Load SUI
-	err = sui.Load(cfg)
+	err = loadStep("SUI", func() error {
+		return sui.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "SUI", err)
 		warnings = append(warnings, Warning{Widget: "SUI", Error: err})
 	}
 
 	// Load Moapi
-	err = moapi.Load(cfg)
+	err = loadStep("Moapi", func() error {
+		return moapi.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Moapi", err)
 		warnings = append(warnings, Warning{Widget: "Moapi", Error: err})
 	}
 
 	// Load Pipe
-	err = pipe.Load(cfg)
+	err = loadStep("Pipe", func() error {
+		return pipe.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Pipe", err)
 		warnings = append(warnings, Warning{Widget: "Pipe", Error: err})
 	}
 
 	// Load MCP Clients
-	err = mcp.Load(cfg)
+	err = loadStep("MCP", func() error {
+		return mcp.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "MCP", err)
 		warnings = append(warnings, Warning{Widget: "MCP", Error: err})
 	}
 
 	// Load Knowledge Base
-	_, err = kb.Load(cfg)
+	err = loadStep("Knowledge Base", func() error {
+		_, err := kb.Load(cfg)
+		return err
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Knowledge Base", err)
 		warnings = append(warnings, Warning{Widget: "Knowledge Base", Error: err})
 	}
 
 	// Load Agent
-	err = agent.Load(cfg)
+	err = loadStep("Agent", func() error {
+		return agent.Load(cfg)
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "Agent", err)
 		warnings = append(warnings, Warning{Widget: "Agent", Error: err})
 	}
 
@@ -313,22 +364,24 @@ func Load(cfg config.Config, options LoadOption) (warnings []Warning, err error)
 	}
 
 	// Load OpenAPI
-	_, err = openapi.Load(cfg)
+	err = loadStep("OpenAPI", func() error {
+		_, err := openapi.Load(cfg)
+		return err
+	}, callback)
 	if err != nil {
-		// printErr(cfg.Mode, "OpenAPI", err)
 		warnings = append(warnings, Warning{Widget: "OpenAPI", Error: err})
 	}
 
 	// Execute AfterLoad Process if exists
 	if share.App.AfterLoad != "" && !options.IgnoredAfterLoad {
-		p, err := process.Of(share.App.AfterLoad, options)
-		if err != nil {
-			printErr(cfg.Mode, "AfterLoad", err)
-			warnings = append(warnings, Warning{Widget: "AfterLoad", Error: err})
-			return warnings, err
-		}
-
-		_, err = p.Exec()
+		err = loadStep("AfterLoad", func() error {
+			p, err := process.Of(share.App.AfterLoad, options)
+			if err != nil {
+				return err
+			}
+			_, err = p.Exec()
+			return err
+		}, callback)
 		if err != nil {
 			printErr(cfg.Mode, "AfterLoad", err)
 			warnings = append(warnings, Warning{Widget: "AfterLoad", Error: err})

model/migrate.go

@@ -0,0 +1,86 @@
+package model
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/fatih/color"
+	"github.com/yaoapp/gou/model"
+	"github.com/yaoapp/gou/schema"
+	"github.com/yaoapp/kun/log"
+)
+
+// BatchMigrate batch migrate models after checking which tables are missing
+// This optimizes the migration process by querying database only once
+func BatchMigrate(models map[string]*model.Model) error {
+	if len(models) == 0 {
+		return nil
+	}
+
+	start := time.Now()
+
+	// Get the connector (assume all system/agent models use default connector)
+	connector := "default"
+	sch := schema.Use(connector)
+
+	// Step 1: Get all existing tables in one query
+	existingTables, err := sch.Tables()
+	if err != nil {
+		return fmt.Errorf("failed to get existing tables: %w", err)
+	}
+
+	// Build a map for fast lookup
+	tableExists := make(map[string]bool)
+	for _, table := range existingTables {
+		tableExists[table] = true
+	}
+
+	// Step 2: Identify models that need creation (skip existing tables)
+	needCreate := make(map[string]*model.Model)
+
+	for id, mod := range models {
+		tableName := mod.MetaData.Table.Name
+		if tableName == "" {
+			log.Warn("Model %s has no table name, skipping", id)
+			continue
+		}
+
+		if !tableExists[tableName] {
+			needCreate[id] = mod
+		}
+	}
+
+	// Step 3: Create missing tables only
+	if len(needCreate) > 0 {
+		isDevelopment := os.Getenv("YAO_ENV") == "development"
+
+		if isDevelopment {
+			fmt.Printf("  %s Creating %d tables...\n", color.CyanString("→"), len(needCreate))
+		}
+
+		for id, mod := range needCreate {
+			createStart := time.Now()
+			err := mod.CreateTable()
+			if err != nil {
+				log.Error("Failed to create table for model %s: %s", id, err.Error())
+				return fmt.Errorf("failed to create table for %s: %w", id, err)
+			}
+
+			duration := time.Since(createStart)
+			if isDevelopment {
+				fmt.Printf("    %s %s %s\n",
+					color.GreenString("✓"),
+					mod.MetaData.Table.Name,
+					color.GreenString("(%v)", duration))
+			} else {
+				log.Info("Created table: %s (%v)", mod.MetaData.Table.Name, duration)
+			}
+		}
+	}
+
+	log.Trace("Batch migrate completed: %d models checked, %d tables created (%v)",
+		len(models), len(needCreate), time.Since(start))
+
+	return nil
+}

model/migrate_test.go

@@ -0,0 +1,73 @@
+package model
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/yaoapp/gou/model"
+	"github.com/yaoapp/yao/config"
+	"github.com/yaoapp/yao/test"
+)
+
+func TestBatchMigrate(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	t.Run("LoadSystemModels", func(t *testing.T) {
+		models, err := loadSystemModels()
+		assert.NoError(t, err, "Should load system models without error")
+		assert.NotEmpty(t, models, "Should have loaded system models")
+
+		// Check that all models have table names
+		for id, mod := range models {
+			assert.NotEmpty(t, mod.MetaData.Table.Name, "Model %s should have table name", id)
+		}
+
+		t.Logf("Loaded %d system models", len(models))
+	})
+
+	t.Run("LoadAssistantModels", func(t *testing.T) {
+		models, errs := loadAssistantModels()
+		assert.Empty(t, errs, "Should load assistant models without critical errors")
+
+		t.Logf("Loaded %d assistant models", len(models))
+	})
+
+	t.Run("BatchMigrateAllModels", func(t *testing.T) {
+		// Load all models
+		systemModels, err := loadSystemModels()
+		assert.NoError(t, err)
+
+		assistantModels, _ := loadAssistantModels()
+
+		// Combine all models
+		allModels := make(map[string]*model.Model)
+		for id, mod := range systemModels {
+			allModels[id] = mod
+		}
+		for id, mod := range assistantModels {
+			allModels[id] = mod
+		}
+
+		// Run batch migrate
+		err = BatchMigrate(allModels)
+		assert.NoError(t, err, "Batch migrate should succeed")
+
+		t.Logf("Batch migrated %d models", len(allModels))
+	})
+
+	t.Run("BatchMigrateIdempotent", func(t *testing.T) {
+		// Load models
+		systemModels, err := loadSystemModels()
+		assert.NoError(t, err)
+
+		// Run batch migrate twice - should be idempotent
+		err = BatchMigrate(systemModels)
+		assert.NoError(t, err, "First batch migrate should succeed")
+
+		err = BatchMigrate(systemModels)
+		assert.NoError(t, err, "Second batch migrate should also succeed (idempotent)")
+
+		t.Logf("Batch migrate is idempotent")
+	})
+}

model/model.go

@@ -50,8 +50,8 @@ func Load(cfg config.Config) error {
 	model.WithCrypt([]byte(fmt.Sprintf(`{"key":"%s"}`, cfg.DB.AESKey)), "AES")
 	model.WithCrypt([]byte(`{}`), "PASSWORD")
 
-	// Load system models
-	err := loadSystemModels()
+	// Load system models (without migrate)
+	systemModels, err := loadSystemModels()
 	if err != nil {
 		return err
 	}
@@ -76,14 +76,28 @@ func Load(cfg config.Config) error {
 		return fmt.Errorf("%s", strings.Join(messages, ";\n"))
 	}
 
-	// Load models from assistants
-	errsAssistants := loadAssistantModels()
+	// Load models from assistants (without migrate)
+	assistantModels, errsAssistants := loadAssistantModels()
 	if len(errsAssistants) > 0 {
 		for _, err := range errsAssistants {
 			log.Error("Load assistant models error: %s", err.Error())
 		}
 	}
 
+	// Batch migrate all system and assistant models
+	allModels := make(map[string]*model.Model)
+	for id, mod := range systemModels {
+		allModels[id] = mod
+	}
+	for id, mod := range assistantModels {
+		allModels[id] = mod
+	}
+
+	err = BatchMigrate(allModels)
+	if err != nil {
+		return err
+	}
+
 	// Load database models ( ignore error)
 	errs := loadDatabaseModels()
 	if len(errs) > 0 {
@@ -94,19 +108,21 @@ func Load(cfg config.Config) error {
 	return err
 }
 
-// LoadSystemModels load system models
-func loadSystemModels() error {
+// LoadSystemModels load system models (without migration)
+func loadSystemModels() (map[string]*model.Model, error) {
+	models := make(map[string]*model.Model)
+
 	for id, path := range systemModels {
 		content, err := data.Read(path)
 		if err != nil {
-			return err
+			return nil, err
 		}
 
 		// Parse model
 		var data map[string]interface{}
 		err = application.Parse(path, content, &data)
 		if err != nil {
-			return err
+			return nil, err
 		}
 
 		// Set prefix
@@ -116,38 +132,34 @@ func loadSystemModels() error {
 				content, err = jsoniter.Marshal(data)
 				if err != nil {
 					log.Error("failed to marshal model data: %v", err)
-					return fmt.Errorf("failed to marshal model data: %v", err)
+					return nil, fmt.Errorf("failed to marshal model data: %v", err)
 				}
 			}
 		}
 
-		// Load Model
+		// Load Model (just parse, no migration)
 		mod, err := model.LoadSource(content, id, filepath.Join("__system", path))
 		if err != nil {
 			log.Error("load system model %s error: %s", id, err.Error())
-			return err
+			return nil, err
 		}
 
-		// Auto migrate
-		err = mod.Migrate(false, model.WithDonotInsertValues(true))
-		if err != nil {
-			log.Error("migrate system model %s error: %s", id, err.Error())
-			return err
-		}
+		models[id] = mod
 	}
 
-	return nil
+	return models, nil
 }
 
-// loadAssistantModels load models from assistants directory
-func loadAssistantModels() []error {
+// loadAssistantModels load models from assistants directory (without migration)
+func loadAssistantModels() (map[string]*model.Model, []error) {
+	models := make(map[string]*model.Model)
 	var errs []error = []error{}
 
 	// Check if assistants directory exists
 	exists, err := application.App.Exists("assistants")
 	if err != nil || !exists {
 		log.Trace("Assistants directory not found or not accessible")
-		return errs
+		return models, errs
 	}
 
 	log.Trace("Loading models from assistants directory...")
@@ -247,7 +259,7 @@ func loadAssistantModels() []error {
 				}
 			}
 
-			// Load model with modified content
+			// Load model with modified content (just parse, no migration)
 			mod, err := model.LoadSource(content, modelID, modelFile)
 			if err != nil {
 				log.Error("Failed to load model %s from assistant %s: %s", modelID, assistantID, err.Error())
@@ -255,15 +267,8 @@ func loadAssistantModels() []error {
 				return nil // Continue loading other models
 			}
 
-			// Auto migrate the model (like system models)
-			err = mod.Migrate(false, model.WithDonotInsertValues(true))
-			if err != nil {
-				log.Error("Failed to migrate model %s from assistant %s: %s", modelID, assistantID, err.Error())
-				errs = append(errs, fmt.Errorf("failed to migrate model %s: %w", modelID, err))
-				return nil
-			}
-
-			log.Info("Loaded and migrated model: %s", modelID)
+			models[modelID] = mod
+			log.Trace("Loaded model: %s", modelID)
 			return nil
 		}, exts...)
 
@@ -278,7 +283,7 @@ func loadAssistantModels() []error {
 		errs = append(errs, fmt.Errorf("failed to walk assistants directory: %w", err))
 	}
 
-	return errs
+	return models, errs
 }
 
 // LoadDatabaseModels load database models