From 2e2abf1d5fcf6f20110e98c23ff6f9c4fa4de749 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 20 Oct 2025 10:52:04 +0800 Subject: [PATCH] Implement JSON field parsing for CSV and XLSX imports in seed module - Added tests to verify correct parsing of JSON fields from both CSV and XLSX files during data import. - Enhanced import functions to build a column type map for detecting JSON fields and parse them appropriately. - Introduced helper functions for building column type maps and parsing JSON fields, improving data integrity during imports. - Updated existing tests to ensure successful imports and correct handling of JSON data structures. --- seed/seed.go | 58 +++++++++++++++++-- seed/seed_test.go | 141 ++++++++++++++++++++++++++++++++++++++++++++++ seed/types.go | 1 - 3 files changed, 195 insertions(+), 5 deletions(-) diff --git a/seed/seed.go b/seed/seed.go index 2621ea2b..bb091657 100644 --- a/seed/seed.go +++ b/seed/seed.go @@ -67,6 +67,9 @@ func importDataFromCSV(filename string, mod *model.Model, options ImportOption, return fmt.Errorf("failed to read CSV header: %v", err) } + // Build column type map for JSON field detection + columnTypes := buildColumnTypeMap(mod, header) + // Prepare handler handler := createImportHandler(mod, header, options, result) @@ -91,10 +94,10 @@ func importDataFromCSV(filename string, mod *model.Model, options ImportOption, continue } - // Convert to interface slice + // Convert to interface slice and parse JSON fields row := make([]interface{}, len(record)) for i, v := range record { - row[i] = v + row[i] = parseJSONField(v, columnTypes[i]) } chunk = append(chunk, row) @@ -154,6 +157,9 @@ func importDataFromXLSX(filename string, mod *model.Model, options ImportOption, return fmt.Errorf("failed to read header: %v", err) } + // Build column type map for JSON field detection + columnTypes := buildColumnTypeMap(mod, header) + // Prepare handler handler := createImportHandler(mod, header, options, result) @@ -188,10 +194,10 @@ func importDataFromXLSX(filename string, mod *model.Model, options ImportOption, continue } - // Convert to interface slice + // Convert to interface slice and parse JSON fields row := make([]interface{}, len(record)) for i, v := range record { - row[i] = v + row[i] = parseJSONField(v, columnTypes[i]) } chunk = append(chunk, row) @@ -501,3 +507,47 @@ func handleDuplicate(mod *model.Model, row maps.MapStrAny, line int, duplicateMo return nil } +// buildColumnTypeMap builds a map of column index to column type +// Returns a slice where index matches the CSV/XLSX column position +func buildColumnTypeMap(mod *model.Model, header []string) []string { + columnTypes := make([]string, len(header)) + for i, colName := range header { + if col, exists := mod.Columns[colName]; exists { + columnTypes[i] = strings.ToLower(col.Type) + } else { + columnTypes[i] = "" + } + } + return columnTypes +} + +// parseJSONField attempts to parse a value as JSON if the column type is json +// Returns the parsed JSON object if successful, otherwise returns the original value +func parseJSONField(value interface{}, columnType string) interface{} { + // Check if column type is JSON + if columnType != "json" && columnType != "jsonb" { + return value + } + + // Try to parse string value as JSON + strValue, ok := value.(string) + if !ok || strValue == "" { + return value + } + + // Trim whitespace + strValue = strings.TrimSpace(strValue) + if strValue == "" { + return value + } + + // Try to parse as JSON + var jsonValue interface{} + if err := json.Unmarshal([]byte(strValue), &jsonValue); err != nil { + // If parsing fails, return original value (might be empty or malformed) + // Don't log error as this is expected for non-JSON strings + return value + } + + return jsonValue +} diff --git a/seed/seed_test.go b/seed/seed_test.go index 83bde7f6..33020a34 100644 --- a/seed/seed_test.go +++ b/seed/seed_test.go @@ -322,3 +322,144 @@ func TestSeedImportChunkSize(t *testing.T) { assert.Nil(t, err) assert.Greater(t, len(roles), 0) } + +// TestSeedImportJSONFields tests that JSON fields are correctly parsed from CSV +func TestSeedImportJSONFields(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Clear existing roles + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // Import CSV file + p := process.New("seeds.import", "roles.csv", "__yao.role") + result := p.Run() + + // Verify result + assert.NotNil(t, result) + resultMap, ok := result.(*ImportResult) + assert.True(t, ok, "Result should be ImportResult") + assert.Greater(t, resultMap.Success, 0, "Should have successful imports") + + // Get imported data + roles, err := mod.Get(model.QueryParam{ + Wheres: []model.QueryWhere{ + {Column: "role_id", Value: "admin"}, + }, + }) + assert.Nil(t, err) + assert.Equal(t, 1, len(roles), "Should find admin role") + + // Verify JSON fields are parsed as objects, not strings + adminRole := roles[0] + + // Check permissions field (should be a map, not a string) + permissions := adminRole.Get("permissions") + assert.NotNil(t, permissions, "Permissions should not be nil") + permissionsMap, ok := permissions.(map[string]interface{}) + assert.True(t, ok, "Permissions should be parsed as map[string]interface{}, got %T", permissions) + assert.NotNil(t, permissionsMap["users"], "Should have users permissions") + + // Check metadata field (should be a map, not a string) + metadata := adminRole.Get("metadata") + assert.NotNil(t, metadata, "Metadata should not be nil") + metadataMap, ok := metadata.(map[string]interface{}) + assert.True(t, ok, "Metadata should be parsed as map[string]interface{}, got %T", metadata) + + // Verify nested values + if usersPerms, ok := permissionsMap["users"].([]interface{}); ok { + assert.Greater(t, len(usersPerms), 0, "Should have user permissions") + assert.Contains(t, usersPerms, "create", "Should have create permission") + } + + if maxUsers, ok := metadataMap["max_users"].(float64); ok { + assert.Equal(t, float64(5), maxUsers, "Max users should be 5") + } +} + +// TestSeedImportXLSXJSONFields tests that JSON fields are correctly parsed from XLSX +func TestSeedImportXLSXJSONFields(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Clear existing roles + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // Import XLSX file + p := process.New("seeds.import", "roles.xlsx", "__yao.role") + result := p.Run() + + // Verify result + assert.NotNil(t, result) + resultMap, ok := result.(*ImportResult) + assert.True(t, ok, "Result should be ImportResult") + assert.Greater(t, resultMap.Success, 0, "Should have successful imports") + + // Get imported data + roles, err := mod.Get(model.QueryParam{ + Wheres: []model.QueryWhere{ + {Column: "role_id", Value: "admin"}, + }, + }) + assert.Nil(t, err) + assert.Equal(t, 1, len(roles), "Should find admin role") + + // Verify JSON fields are parsed as objects, not strings + adminRole := roles[0] + + // Check permissions field (should be a map, not a string) + permissions := adminRole.Get("permissions") + assert.NotNil(t, permissions, "Permissions should not be nil") + permissionsMap, ok := permissions.(map[string]interface{}) + assert.True(t, ok, "Permissions should be parsed as map[string]interface{}, got %T", permissions) + assert.NotNil(t, permissionsMap["users"], "Should have users permissions") + + // Check metadata field (should be a map, not a string) + metadata := adminRole.Get("metadata") + assert.NotNil(t, metadata, "Metadata should not be nil") + metadataMap, ok := metadata.(map[string]interface{}) + assert.True(t, ok, "Metadata should be parsed as map[string]interface{}, got %T", metadata) + + // Verify nested values from XLSX + if usersPerms, ok := permissionsMap["users"].([]interface{}); ok { + assert.Greater(t, len(usersPerms), 0, "Should have user permissions") + assert.Contains(t, usersPerms, "create", "Should have create permission") + } + + if maxUsers, ok := metadataMap["max_users"].(float64); ok { + assert.Equal(t, float64(5), maxUsers, "Max users should be 5") + } + + // Also verify other roles to ensure all JSON fields are parsed + allRoles, err := mod.Get(model.QueryParam{}) + assert.Nil(t, err) + assert.Greater(t, len(allRoles), 1, "Should have multiple roles") + + // Check that all roles have properly parsed JSON fields + for _, role := range allRoles { + roleID := role.Get("role_id") + permissions := role.Get("permissions") + if permissions != nil { + _, ok := permissions.(map[string]interface{}) + assert.True(t, ok, "Role %s permissions should be parsed as map, got %T", roleID, permissions) + } + + metadata := role.Get("metadata") + if metadata != nil { + _, ok := metadata.(map[string]interface{}) + assert.True(t, ok, "Role %s metadata should be parsed as map, got %T", roleID, metadata) + } + } +} diff --git a/seed/types.go b/seed/types.go index 424cb0a5..49533417 100644 --- a/seed/types.go +++ b/seed/types.go @@ -54,4 +54,3 @@ type ImportError struct { Code int `json:"code,omitempty"` Data []interface{} `json:"data,omitempty"` } -