Enhance job and category management with unique ID generation

- Introduced unique ID generation for jobs, categories, and executions using gonanoid, improving ID handling and reducing collisions.
- Updated SaveJob and SaveCategory functions to utilize new ID generation methods, ensuring consistent and unique identifiers.
- Enhanced job creation logic to allow for category name handling, enabling automatic category ID assignment based on provided names.
- Refactored ensureCategoryExists to check for existing categories by name, streamlining category management during job operations.
- Improved error handling and logging for ID generation and category retrieval processes, ensuring better traceability and reliability.

job/data.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/google/uuid"
 	jsoniter "github.com/json-iterator/go"
+	gonanoid "github.com/matoous/go-nanoid/v2"
 	"github.com/yaoapp/gou/model"
 	"github.com/yaoapp/kun/log"
 	"github.com/yaoapp/kun/maps"
@@ -104,12 +104,13 @@ func SaveJob(job *Job) error {
 		return fmt.Errorf("job model not found")
 	}
 
-	// Ensure category exists before saving job
-	if job.CategoryID != "" {
-		_, err := ensureCategoryExists(job.CategoryID)
+	// If no CategoryID but CategoryName is provided, get or create category ID
+	if job.CategoryID == "" && job.CategoryName != "" {
+		categoryID, err := getCategoryIDByName(job.CategoryName)
 		if err != nil {
-			return fmt.Errorf("failed to ensure category exists: %w", err)
+			return fmt.Errorf("failed to get category ID by name '%s': %w", job.CategoryName, err)
 		}
+		job.CategoryID = categoryID
 	}
 
 	data := structToMap(job)
@@ -118,7 +119,11 @@ func SaveJob(job *Job) error {
 	if job.ID == 0 {
 		// Create new job
 		if job.JobID == "" {
-			job.JobID = uuid.New().String()
+			var err error
+			job.JobID, err = generateJobID()
+			if err != nil {
+				return fmt.Errorf("failed to generate job ID: %w", err)
+			}
 		}
 
 		// Remove ID field from data to let database auto-increment
@@ -298,9 +303,35 @@ func SaveCategory(category *Category) error {
 	now := time.Now()
 
 	if category.ID == 0 {
-		// Create new category
+		// Create new category - but first check if name already exists
+		if category.Name != "" {
+			// Check if category with same name already exists
+			param := model.QueryParam{
+				Wheres: []model.QueryWhere{
+					{Column: "name", Value: category.Name},
+				},
+				Limit: 1,
+			}
+			results, err := mod.Get(param)
+			if err != nil {
+				return fmt.Errorf("failed to check existing category: %w", err)
+			}
+			if len(results) > 0 {
+				// Category with same name exists, update current category with existing data
+				if err := mapToStruct(results[0], category); err != nil {
+					return fmt.Errorf("failed to map existing category: %w", err)
+				}
+				return nil // Return the existing category
+			}
+		}
+
+		// No existing category found, create new one
 		if category.CategoryID == "" {
-			category.CategoryID = uuid.New().String()
+			var err error
+			category.CategoryID, err = generateCategoryID()
+			if err != nil {
+				return fmt.Errorf("failed to generate category ID: %w", err)
+			}
 		}
 
 		// Remove ID field from data to let database auto-increment
@@ -311,6 +342,22 @@ func SaveCategory(category *Category) error {
 
 		id, err := mod.Create(data)
 		if err != nil {
+			// If creation failed due to duplicate name, try to find existing category
+			if category.Name != "" {
+				param := model.QueryParam{
+					Wheres: []model.QueryWhere{
+						{Column: "name", Value: category.Name},
+					},
+					Limit: 1,
+				}
+				results, findErr := mod.Get(param)
+				if findErr == nil && len(results) > 0 {
+					// Found existing category, use it
+					if mapErr := mapToStruct(results[0], category); mapErr == nil {
+						return nil // Successfully found and mapped existing category
+					}
+				}
+			}
 			return fmt.Errorf("failed to create category: %w", err)
 		}
 		category.ID = uint(id)
@@ -367,8 +414,13 @@ func GetOrCreateCategory(name, description string) (*Category, error) {
 	}
 
 	// Create new category
+	categoryID, err := generateCategoryID()
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate category ID: %w", err)
+	}
+
 	category := &Category{
-		CategoryID:  uuid.New().String(),
+		CategoryID:  categoryID,
 		Name:        name,
 		Description: &description,
 		Sort:        0,
@@ -386,17 +438,26 @@ func GetOrCreateCategory(name, description string) (*Category, error) {
 	return category, nil
 }
 
-// ensureCategoryExists ensures a category exists by category_id, creates default if needed
-func ensureCategoryExists(categoryID string) (*Category, error) {
+// getCategoryIDByName gets category ID by name, creates category if not exists
+func getCategoryIDByName(categoryName string) (string, error) {
+	category, err := ensureCategoryExists(categoryName)
+	if err != nil {
+		return "", err
+	}
+	return category.CategoryID, nil
+}
+
+// ensureCategoryExists ensures a category exists by name, creates if needed
+func ensureCategoryExists(categoryName string) (*Category, error) {
 	mod := model.Select("__yao.job.category")
 	if mod == nil {
 		return nil, fmt.Errorf("job category model not found")
 	}
 
-	// Try to find existing category by category_id
+	// Try to find existing category by name (since external calls pass category name)
 	param := model.QueryParam{
 		Wheres: []model.QueryWhere{
-			{Column: "category_id", Value: categoryID},
+			{Column: "name", Value: categoryName},
 		},
 		Limit: 1,
 	}
@@ -407,7 +468,7 @@ func ensureCategoryExists(categoryID string) (*Category, error) {
 	}
 
 	if len(results) > 0 {
-		// Category exists
+		// Category exists, return it
 		category := &Category{}
 		if err := mapToStruct(results[0], category); err != nil {
 			return nil, err
@@ -415,14 +476,21 @@ func ensureCategoryExists(categoryID string) (*Category, error) {
 		return category, nil
 	}
 
-	// Create default category if it doesn't exist
-	var categoryName, categoryDesc string
-	if categoryID == "default" {
-		categoryName = "Default"
+	// Category doesn't exist, create it
+	var categoryID, categoryDesc string
+
+	if categoryName == "Default" {
+		// Keep "default" as the category ID for the default category
+		categoryID = "default"
 		categoryDesc = "Default job category"
 	} else {
-		categoryName = categoryID
-		categoryDesc = fmt.Sprintf("Auto-created category: %s", categoryID)
+		// For other categories, generate a new unique ID
+		var err error
+		categoryID, err = generateCategoryID()
+		if err != nil {
+			return nil, fmt.Errorf("failed to generate category ID: %w", err)
+		}
+		categoryDesc = fmt.Sprintf("Auto-created category: %s", categoryName)
 	}
 
 	category := &Category{
@@ -430,7 +498,7 @@ func ensureCategoryExists(categoryID string) (*Category, error) {
 		Name:        categoryName,
 		Description: &categoryDesc,
 		Sort:        0,
-		System:      categoryID == "default", // Mark default as system category
+		System:      categoryName == "Default", // Mark default as system category
 		Enabled:     true,
 		Readonly:    false,
 		CreatedAt:   time.Now(),
@@ -708,7 +776,11 @@ func SaveExecution(execution *Execution) error {
 	if execution.ID == 0 {
 		// Create new execution
 		if execution.ExecutionID == "" {
-			execution.ExecutionID = uuid.New().String()
+			var err error
+			execution.ExecutionID, err = generateExecutionID()
+			if err != nil {
+				return fmt.Errorf("failed to generate execution ID: %w", err)
+			}
 		}
 
 		// Remove ID field from data to let database auto-increment
@@ -781,6 +853,136 @@ func GetProgress(executionID string, cb func(progress *Progress)) (*Progress, er
 	return progress, nil
 }
 
+// ========================
+// ID Generation methods
+// ========================
+
+// generateJobID generates a unique job_id using nanoid with duplicate checking
+func generateJobID() (string, error) {
+	const maxRetries = 10
+	const alphabet = "23456789ABCDEFGHJKMNPQRSTUVWXYZabcdefghijkmnpqrstuvwxyz"
+	const length = 12
+
+	mod := model.Select("__yao.job")
+	if mod == nil {
+		return "", fmt.Errorf("job model not found")
+	}
+
+	for i := 0; i < maxRetries; i++ {
+		// Generate new ID using nanoid
+		id, err := gonanoid.Generate(alphabet, length)
+		if err != nil {
+			return "", fmt.Errorf("failed to generate nanoid: %w", err)
+		}
+
+		// Check if ID already exists
+		param := model.QueryParam{
+			Select: []interface{}{"id"}, // Just get primary key, minimal data
+			Wheres: []model.QueryWhere{
+				{Column: "job_id", Value: id},
+			},
+			Limit: 1,
+		}
+
+		results, err := mod.Get(param)
+		if err != nil {
+			return "", fmt.Errorf("failed to check job_id existence: %w", err)
+		}
+
+		if len(results) == 0 {
+			return id, nil // Found unique ID
+		}
+
+		// ID exists, retry with new generation
+	}
+
+	return "", fmt.Errorf("failed to generate unique job_id after %d retries", maxRetries)
+}
+
+// generateCategoryID generates a unique category_id using nanoid with duplicate checking
+func generateCategoryID() (string, error) {
+	const maxRetries = 10
+	const alphabet = "23456789ABCDEFGHJKMNPQRSTUVWXYZabcdefghijkmnpqrstuvwxyz"
+	const length = 12
+
+	mod := model.Select("__yao.job.category")
+	if mod == nil {
+		return "", fmt.Errorf("job category model not found")
+	}
+
+	for i := 0; i < maxRetries; i++ {
+		// Generate new ID using nanoid
+		id, err := gonanoid.Generate(alphabet, length)
+		if err != nil {
+			return "", fmt.Errorf("failed to generate nanoid: %w", err)
+		}
+
+		// Check if ID already exists
+		param := model.QueryParam{
+			Select: []interface{}{"id"}, // Just get primary key, minimal data
+			Wheres: []model.QueryWhere{
+				{Column: "category_id", Value: id},
+			},
+			Limit: 1,
+		}
+
+		results, err := mod.Get(param)
+		if err != nil {
+			return "", fmt.Errorf("failed to check category_id existence: %w", err)
+		}
+
+		if len(results) == 0 {
+			return id, nil // Found unique ID
+		}
+
+		// ID exists, retry with new generation
+	}
+
+	return "", fmt.Errorf("failed to generate unique category_id after %d retries", maxRetries)
+}
+
+// generateExecutionID generates a unique execution_id using nanoid with duplicate checking
+func generateExecutionID() (string, error) {
+	const maxRetries = 10
+	const alphabet = "23456789ABCDEFGHJKMNPQRSTUVWXYZabcdefghijkmnpqrstuvwxyz"
+	const length = 16 // Slightly longer for executions as they are more frequent
+
+	mod := model.Select("__yao.job.execution")
+	if mod == nil {
+		return "", fmt.Errorf("job execution model not found")
+	}
+
+	for i := 0; i < maxRetries; i++ {
+		// Generate new ID using nanoid
+		id, err := gonanoid.Generate(alphabet, length)
+		if err != nil {
+			return "", fmt.Errorf("failed to generate nanoid: %w", err)
+		}
+
+		// Check if ID already exists
+		param := model.QueryParam{
+			Select: []interface{}{"id"}, // Just get primary key, minimal data
+			Wheres: []model.QueryWhere{
+				{Column: "execution_id", Value: id},
+			},
+			Limit: 1,
+		}
+
+		results, err := mod.Get(param)
+		if err != nil {
+			return "", fmt.Errorf("failed to check execution_id existence: %w", err)
+		}
+
+		if len(results) == 0 {
+			return id, nil // Found unique ID
+		}
+
+		// ID exists, retry with new generation
+	}
+
+	return "", fmt.Errorf("failed to generate unique execution_id after %d retries", maxRetries)
+}
+
 // ========================
 // Helper methods
 // ========================

job/job.go

@@ -303,9 +303,9 @@ func makeJob(data []byte) (*Job, error) {
 		return nil, err
 	}
 
-	// Set default values if not provided
-	if job.CategoryID == "" {
-		job.CategoryID = "default"
+	// Set default CategoryName if both CategoryID and CategoryName are empty
+	if job.CategoryID == "" && job.CategoryName == "" {
+		job.CategoryName = "Default"
 	}
 	if job.Status == "" {
 		job.Status = "draft"

job/types.go

@@ -114,6 +114,7 @@ type Job struct {
 	Icon               *string                `json:"icon,omitempty"`        // nullable: true
 	Description        *string                `json:"description,omitempty"` // nullable: true
 	CategoryID         string                 `json:"category_id"`
+	CategoryName       string                 `json:"category_name,omitempty"`
 	MaxWorkerNums      int                    `json:"max_worker_nums"`               // default: 1
 	Status             string                 `json:"status"`                        // default: "draft"
 	Mode               ModeType               `json:"mode"`                          // default: "goroutine"

openapi/kb/addfile.go

@@ -304,11 +304,26 @@ func AddFileAsync(c *gin.Context) {
 
 	log.Info("AddFileAsync: Generated doc_id: %s", req.DocID)
 
-	// Step 1: Create and save Job in one step to get JobID
-	j, err := job.OnceAndSave(job.GOROUTINE, map[string]interface{}{
-		"name":        "KB Add File",
-		"description": "Add file to knowledge base collection",
-	})
+	// Step 1: Get job options with defaults
+	jobName, jobDescription, jobIcon, jobCategory := req.GetJobOptions(
+		"Knowledge Base File Processing",                                 // default name
+		"Processing and indexing file content for knowledge base search", // default description
+		"library_add",    // default icon (Material Icon)
+		"Knowledge Base", // default category
+	)
+
+	// Create job data
+	jobCreateData := map[string]interface{}{
+		"name":          jobName,
+		"description":   jobDescription,
+		"category_name": jobCategory, // Pass category name directly, let SaveJob handle it
+	}
+	if jobIcon != "" {
+		jobCreateData["icon"] = jobIcon
+	}
+
+	// Create and save Job in one step to get JobID
+	j, err := job.OnceAndSave(job.GOROUTINE, jobCreateData)
 	if err != nil {
 		log.Error("AddFileAsync: Job creation and save failed: %v", err)
 		errorResp := &response.ErrorResponse{
@@ -562,5 +577,23 @@ func parseAddFileRequest(reqMap map[string]interface{}) *AddFileRequest {
 		req.Converter = converter
 	}
 
+	// Handle job options
+	if jobMap, ok := reqMap["job"].(map[string]interface{}); ok {
+		job := &JobOptions{}
+		if name, ok := jobMap["name"].(string); ok {
+			job.Name = name
+		}
+		if description, ok := jobMap["description"].(string); ok {
+			job.Description = description
+		}
+		if icon, ok := jobMap["icon"].(string); ok {
+			job.Icon = icon
+		}
+		if category, ok := jobMap["category"].(string); ok {
+			job.Category = category
+		}
+		req.Job = job
+	}
+
 	return req
 }

openapi/kb/addtext.go

@@ -261,11 +261,26 @@ func AddTextAsync(c *gin.Context) {
 
 	log.Info("AddTextAsync: Generated doc_id: %s", req.DocID)
 
-	// Step 1: Create and save Job in one step to get JobID
-	j, err := job.OnceAndSave(job.GOROUTINE, map[string]interface{}{
-		"name":        "KB Add Text",
-		"description": "Add text to knowledge base collection",
-	})
+	// Step 1: Get job options with defaults
+	jobName, jobDescription, jobIcon, jobCategory := req.GetJobOptions(
+		"Knowledge Base Text Processing",                                 // default name
+		"Processing and indexing text content for knowledge base search", // default description
+		"library_add",    // default icon (Material Icon)
+		"Knowledge Base", // default category
+	)
+
+	// Create job data
+	jobCreateData := map[string]interface{}{
+		"name":          jobName,
+		"description":   jobDescription,
+		"category_name": jobCategory, // Pass category name directly, let SaveJob handle it
+	}
+	if jobIcon != "" {
+		jobCreateData["icon"] = jobIcon
+	}
+
+	// Create and save Job in one step to get JobID
+	j, err := job.OnceAndSave(job.GOROUTINE, jobCreateData)
 	if err != nil {
 		log.Error("AddTextAsync: Job creation and save failed: %v", err)
 		errorResp := &response.ErrorResponse{
@@ -512,5 +527,23 @@ func parseAddTextRequest(reqMap map[string]interface{}) *AddTextRequest {
 		req.Converter = converter
 	}
 
+	// Handle job options
+	if jobMap, ok := reqMap["job"].(map[string]interface{}); ok {
+		job := &JobOptions{}
+		if name, ok := jobMap["name"].(string); ok {
+			job.Name = name
+		}
+		if description, ok := jobMap["description"].(string); ok {
+			job.Description = description
+		}
+		if icon, ok := jobMap["icon"].(string); ok {
+			job.Icon = icon
+		}
+		if category, ok := jobMap["category"].(string); ok {
+			job.Category = category
+		}
+		req.Job = job
+	}
+
 	return req
 }

openapi/kb/addurl.go

@@ -261,11 +261,26 @@ func AddURLAsync(c *gin.Context) {
 
 	log.Info("AddURLAsync: Generated doc_id: %s", req.DocID)
 
-	// Step 1: Create and save Job in one step to get JobID
-	j, err := job.OnceAndSave(job.GOROUTINE, map[string]interface{}{
-		"name":        "KB Add URL",
-		"description": "Add URL to knowledge base collection",
-	})
+	// Step 1: Get job options with defaults
+	jobName, jobDescription, jobIcon, jobCategory := req.GetJobOptions(
+		"Knowledge Base Web Content Processing",                       // default name
+		"Fetching and indexing web content for knowledge base search", // default description
+		"library_add",    // default icon (Material Icon)
+		"Knowledge Base", // default category
+	)
+
+	// Create job data
+	jobCreateData := map[string]interface{}{
+		"name":          jobName,
+		"description":   jobDescription,
+		"category_name": jobCategory, // Pass category name directly, let SaveJob handle it
+	}
+	if jobIcon != "" {
+		jobCreateData["icon"] = jobIcon
+	}
+
+	// Create and save Job in one step to get JobID
+	j, err := job.OnceAndSave(job.GOROUTINE, jobCreateData)
 	if err != nil {
 		log.Error("AddURLAsync: Job creation and save failed: %v", err)
 		errorResp := &response.ErrorResponse{
@@ -512,5 +527,23 @@ func parseAddURLRequest(reqMap map[string]interface{}) *AddURLRequest {
 		req.Converter = converter
 	}
 
+	// Handle job options
+	if jobMap, ok := reqMap["job"].(map[string]interface{}); ok {
+		job := &JobOptions{}
+		if name, ok := jobMap["name"].(string); ok {
+			job.Name = name
+		}
+		if description, ok := jobMap["description"].(string); ok {
+			job.Description = description
+		}
+		if icon, ok := jobMap["icon"].(string); ok {
+			job.Icon = icon
+		}
+		if category, ok := jobMap["category"].(string); ok {
+			job.Category = category
+		}
+		req.Job = job
+	}
+
 	return req
 }

openapi/kb/types.go

@@ -78,6 +78,14 @@ type ProviderConfig struct {
 	Option     *kbtypes.ProviderOption `json:"option,omitempty"`
 }
 
+// JobOptions contains job options for async operations
+type JobOptions struct {
+	Name        string `json:"name,omitempty"`        // Job name (optional, defaults will be used)
+	Description string `json:"description,omitempty"` // Job description (optional, defaults will be used)
+	Icon        string `json:"icon,omitempty"`        // Job icon (optional, Material Icon name)
+	Category    string `json:"category,omitempty"`    // Job category (optional, defaults will be used)
+}
+
 // BaseUpsertRequest contains common fields for all upsert operations
 type BaseUpsertRequest struct {
 	// Collection ID - this will be mapped to UpsertOptions.CollectionID
@@ -96,6 +104,9 @@ type BaseUpsertRequest struct {
 	// Upsert options
 	DocID    string                 `json:"doc_id,omitempty"`
 	Metadata map[string]interface{} `json:"metadata,omitempty"`
+
+	// Job options for async operations
+	Job *JobOptions `json:"job,omitempty"`
 }
 
 // AddFileRequest represents the request for AddFile API
@@ -451,6 +462,31 @@ func (r *UpdateWeightsRequest) Validate() error {
 	return nil
 }
 
+// GetJobOptions returns job options with defaults
+func (r *BaseUpsertRequest) GetJobOptions(defaultName, defaultDescription, defaultIcon, defaultCategory string) (string, string, string, string) {
+	name := defaultName
+	description := defaultDescription
+	icon := defaultIcon
+	category := defaultCategory
+
+	if r.Job != nil {
+		if r.Job.Name != "" {
+			name = r.Job.Name
+		}
+		if r.Job.Description != "" {
+			description = r.Job.Description
+		}
+		if r.Job.Icon != "" {
+			icon = r.Job.Icon
+		}
+		if r.Job.Category != "" {
+			category = r.Job.Category
+		}
+	}
+
+	return name, description, icon, category
+}
+
 // AddBaseFields adds common fields from BaseUpsertRequest to data map
 func (r *BaseUpsertRequest) AddBaseFields(data map[string]interface{}) {
 	if r.Locale != "" {

yao/models/job/category.mod.yao

@@ -33,7 +33,7 @@
       "comment": "Display name of the category",
       "length": 255,
       "nullable": false,
-      "index": true
+      "unique": true
     },
     {
       "name": "icon",