From b89a39b1929262ee328d986bf94ccbe381da38cf Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 7 Mar 2025 16:57:58 +0800 Subject: [PATCH 001/247] Refactor error handling and logging in chat streaming and API methods - Update error handling in handleChat to return and log errors - Modify streamChat to remove explicit error logging and improve retry mechanism - Adjust HookRetry method to return nil for more flexible error handling - Improve error handling in retryMessages method with better index tracking - Remove redundant error logging in requestMessages method --- neo/api.go | 9 ++++++++- neo/assistant/api.go | 29 ++++++++++++++++++----------- neo/assistant/call.go | 5 ++--- neo/assistant/hooks.go | 10 +++++----- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/neo/api.go b/neo/api.go index 71e866da..af765e56 100644 --- a/neo/api.go +++ b/neo/api.go @@ -232,7 +232,14 @@ func (neo *DSL) handleChat(c *gin.Context) { defer cancel() defer ctx.Release() // Release the context after the request is done - neo.Answer(ctx, content, c) + err := neo.Answer(ctx, content, c) + + // Error handling + if err != nil { + message.New().Done().Error(err).Write(c.Writer) + c.Done() + return + } } // handleChatList handles the chat list request diff --git a/neo/assistant/api.go b/neo/assistant/api.go index 64ffc10a..facab077 100644 --- a/neo/assistant/api.go +++ b/neo/assistant/api.go @@ -323,10 +323,6 @@ func (ast *Assistant) handleChatStream(c *gin.Context, ctx chatctx.Context, mess go func() { var res interface{} = nil res, err = ast.streamChat(c, ctx, messages, options, clientBreak, contents, callback...) - if err != nil { - chatMessage.New().Error(err).Done().Write(c.Writer) - err = fmt.Errorf("stream chat error %s", err.Error()) - } result = res done <- true }() @@ -622,16 +618,22 @@ func (ast *Assistant) streamChat( ctx.RetryTimes = ctx.RetryTimes + 1 // Increment the retry times ctx.Retry = true // Set the retry mode + // The maximum retry times is 9 + if ctx.RetryTimes > 9 { + color.Red("Maximum retry times is 9, please check the error and fix it") + // chatMessage.New().Error(retry.Error()).Done().Callback(cb).Write(c.Writer) + return nil, retry + } + // Hook retry promptAny, retryErr := ast.HookRetry(c, ctx, messages, contents, exception.Trim(retry)) if retryErr != nil { color.Red("%s, try to fix the error %d times, but failed with %s", exception.Trim(retry), ctx.RetryTimes, exception.Trim(retryErr)) - chatMessage.New().Error(retry.Error()).Done().Callback(cb).Write(c.Writer) + // chatMessage.New().Error(retry.Error()).Done().Callback(cb).Write(c.Writer) return nil, retry } if promptAny == nil { - chatMessage.New().Error(retry.Error()).Done().Callback(cb).Write(c.Writer) return nil, retry } @@ -640,7 +642,7 @@ func (ast *Assistant) streamChat( case NextAction: result, err := v.Execute(c, ctx, contents, cb) if err != nil { - chatMessage.New().Error(err.Error()).Done().Callback(cb).Write(c.Writer) + // chatMessage.New().Error(err.Error()).Done().Callback(cb).Write(c.Writer) return nil, retry } return result, nil @@ -653,7 +655,7 @@ func (ast *Assistant) streamChat( retryMessages, retryErr := ast.retryMessages(messages, prompt) if retryErr != nil { color.Red("%s, try to fix the error %d times, but failed with %s", exception.Trim(retry), ctx.RetryTimes, exception.Trim(retryErr)) - chatMessage.New().Error(retry.Error()).Done().Callback(cb).Write(c.Writer) + // chatMessage.New().Error(retry.Error()).Done().Callback(cb).Write(c.Writer) return nil, retry } @@ -690,7 +692,7 @@ func (ast *Assistant) streamChat( func (ast *Assistant) retryMessages(messages []chatMessage.Message, prompt string) ([]chatMessage.Message, error) { // Get the last user message - var lastIndex int + var lastIndex int = -1 for i := len(messages) - 1; i >= 0; i-- { if messages[i].Role == "user" { messages[i].Text = prompt @@ -699,7 +701,7 @@ func (ast *Assistant) retryMessages(messages []chatMessage.Message, prompt strin } } - if lastIndex == 0 { + if lastIndex == -1 { return nil, fmt.Errorf("no user message found") } @@ -1089,7 +1091,12 @@ func (ast *Assistant) requestMessages(ctx context.Context, messages []chatMessag content := message.String() if content == "" { - return nil, fmt.Errorf("content must be string") + // fmt.Println("--------------------------------") + // fmt.Println("Request Message Error") + // utils.Dump(message) + // fmt.Println("--------------------------------") + // return nil, fmt.Errorf("content must be string") + continue } newMessage := map[string]interface{}{ diff --git a/neo/assistant/call.go b/neo/assistant/call.go index cc17194e..fd3df996 100644 --- a/neo/assistant/call.go +++ b/neo/assistant/call.go @@ -246,7 +246,7 @@ func (obj *objectCall) run(info *v8go.FunctionCallbackInfo) *v8go.Value { var chatCtx chatctx.Context = global.ChatContext chatCtx.AssistantID = assistantID chatCtx.ChatID = fmt.Sprintf("call_%s", uuid.New().String()) // New chat id - chatCtx.Silent = options.Silent // Check the silent mode + chatCtx.Silent = options.Silent // Define the callback function var cb func(msg *chatMessage.Message) = nil @@ -346,7 +346,7 @@ func (obj *objectCall) retry(jsArgs []v8go.Valuer, err error, input interface{}, delay = options.Retry.DelayMax } - // Retry delay (millisecond) + // Wait for the delay if delay > 0 { time.Sleep(time.Duration(delay) * time.Millisecond) } @@ -448,7 +448,6 @@ func (obj *objectCall) retry(jsArgs []v8go.Valuer, err error, input interface{}, return nil, fmt.Errorf("%s occurred but failed to get the run function: %s", errmsg, fnErr.Error()) } - // Call the run function result, resErr := fn.Call(this, jsArgs...) if resErr != nil { return nil, fmt.Errorf("%s (%d)", exception.Trim(resErr), times-1) diff --git a/neo/assistant/hooks.go b/neo/assistant/hooks.go index 31aaec2c..90e84069 100644 --- a/neo/assistant/hooks.go +++ b/neo/assistant/hooks.go @@ -153,9 +153,9 @@ func (ast *Assistant) HookRetry(c *gin.Context, context chatctx.Context, input [ v, err := ast.call(ctx, "Retry", c, contents, context, lastInput.String(), output, errmsg) if err != nil { if err.Error() == HookErrorMethodNotFound { - return "", nil + return nil, nil } - return "", err + return nil, err } switch v := v.(type) { @@ -166,12 +166,12 @@ func (ast *Assistant) HookRetry(c *gin.Context, context chatctx.Context, input [ raw, _ := jsoniter.MarshalToString(v) err := jsoniter.UnmarshalFromString(raw, &next) if err != nil { - return "", err + return nil, err } - return next, nil + return &next, nil } - return "", nil + return nil, nil } // HookDone Handle completion of assistant response From d3734cf1d044886680b0a93cd4dbfa16c9a51ff1 Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 12 Mar 2025 17:24:36 +0800 Subject: [PATCH 002/247] Add support for passing arguments in assistant method calls - Extend Context struct to include Args field for storing call arguments - Modify objectCall.run to capture and store arguments in chat context - Update context mapping to include arguments when present --- neo/assistant/call.go | 1 + neo/context/context.go | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/neo/assistant/call.go b/neo/assistant/call.go index fd3df996..ef086cfe 100644 --- a/neo/assistant/call.go +++ b/neo/assistant/call.go @@ -247,6 +247,7 @@ func (obj *objectCall) run(info *v8go.FunctionCallbackInfo) *v8go.Value { chatCtx.AssistantID = assistantID chatCtx.ChatID = fmt.Sprintf("call_%s", uuid.New().String()) // New chat id chatCtx.Silent = options.Silent + chatCtx.Args = goArgs // Arguments for call // Define the callback function var cb func(msg *chatMessage.Message) = nil diff --git a/neo/context/context.go b/neo/context/context.go index 77d1c43f..929dfeb2 100644 --- a/neo/context/context.go +++ b/neo/context/context.go @@ -28,6 +28,7 @@ type Context struct { Upload *FileUpload `json:"upload,omitempty"` Version bool `json:"version,omitempty"` // Version support RAG bool `json:"rag,omitempty"` // RAG support + Args []interface{} `json:"args,omitempty"` // Arguments for call SharedSpace plan.Space `json:"-"` // Shared space } @@ -125,6 +126,11 @@ func (ctx *Context) Map() map[string]interface{} { data["retry"] = ctx.Retry } + // Arguments for call + if ctx.Args != nil && len(ctx.Args) > 0 { + data["args"] = ctx.Args + } + // Retry times data["retry_times"] = ctx.RetryTimes From 97a469bc038a5cb31166101c6fa4a0b5afbf23a6 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 15 Mar 2025 17:57:35 +0800 Subject: [PATCH 003/247] Add type filtering for assistants in API and data loading - Introduce 'Type' field in AssistantFilter for filtering assistant lists - Update handleAssistantList and handleAssistantDetail methods to set 'Type' to 'assistant' - Modify LoadPath function to ensure 'type' is set if not already present - Adjust GetAssistants and GetAssistantTags methods to apply type filtering in queries --- neo/api.go | 2 ++ neo/assistant/load.go | 20 ++++---------------- neo/store/types.go | 1 + neo/store/xun.go | 7 ++++++- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/neo/api.go b/neo/api.go index af765e56..2aa14591 100644 --- a/neo/api.go +++ b/neo/api.go @@ -962,6 +962,7 @@ func (neo *DSL) handleGenerateCustom(c *gin.Context) { func (neo *DSL) handleAssistantList(c *gin.Context) { // Parse filter parameters filter := store.AssistantFilter{ + Type: "assistant", Page: 1, PageSize: 20, } @@ -1106,6 +1107,7 @@ func (neo *DSL) handleAssistantDetail(c *gin.Context) { filter := store.AssistantFilter{ AssistantID: assistantID, + Type: "assistant", Page: 1, PageSize: 1, } diff --git a/neo/assistant/load.go b/neo/assistant/load.go index 9f162349..2b81df4a 100644 --- a/neo/assistant/load.go +++ b/neo/assistant/load.go @@ -78,21 +78,7 @@ func LoadBuiltIn() error { assistant.Sort = sort } if assistant.Tags == nil { - assistant.Tags = []string{"Built-in"} - } - - // Check if the assistant has Built-in tag - hasBuiltIn := false - for _, tag := range assistant.Tags { - if tag == "Built-in" { - hasBuiltIn = true - break - } - } - - // add Built-in tag if not exists - if !hasBuiltIn { - assistant.Tags = append(assistant.Tags, "Built-in") + assistant.Tags = []string{} } // Save the assistant @@ -254,8 +240,10 @@ func LoadPath(path string) (*Assistant, error) { // assistant_id id := strings.ReplaceAll(strings.TrimPrefix(path, "/assistants/"), "/", ".") data["assistant_id"] = id - data["type"] = "assistant" data["path"] = path + if _, has := data["type"]; !has { + data["type"] = "assistant" + } updatedAt := int64(0) diff --git a/neo/store/types.go b/neo/store/types.go index 22cab473..104cfcf7 100644 --- a/neo/store/types.go +++ b/neo/store/types.go @@ -48,6 +48,7 @@ type ChatGroupResponse struct { // Used for filtering and pagination when retrieving assistant lists type AssistantFilter struct { Tags []string `json:"tags,omitempty"` // Filter by tags + Type string `json:"type,omitempty"` // Filter by type Keywords string `json:"keywords,omitempty"` // Search in name and description Connector string `json:"connector,omitempty"` // Filter by connector AssistantID string `json:"assistant_id,omitempty"` // Filter by assistant ID diff --git a/neo/store/xun.go b/neo/store/xun.go index ceae2f0a..8c87bb62 100644 --- a/neo/store/xun.go +++ b/neo/store/xun.go @@ -1058,6 +1058,11 @@ func (conv *Xun) GetAssistants(filter AssistantFilter) (*AssistantResponse, erro }) } + // Apply type filter if provided + if filter.Type != "" { + qb.Where("type", filter.Type) + } + // Apply connector filter if provided if filter.Connector != "" { qb.Where("connector", filter.Connector) @@ -1259,7 +1264,7 @@ func (conv *Xun) DeleteAssistants(filter AssistantFilter) (int64, error) { // GetAssistantTags retrieves all unique tags from assistants func (conv *Xun) GetAssistantTags() ([]string, error) { q := conv.newQuery().Table(conv.getAssistantTable()) - rows, err := q.Select("tags").GroupBy("tags").Get() + rows, err := q.Select("tags").Where("type", "assistant").GroupBy("tags").Get() if err != nil { return nil, err } From 38b910108bdc46580d55951cf626c199772453ed Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 16 Mar 2025 11:49:22 +0800 Subject: [PATCH 004/247] Add connector support for AI assistants in processXgen - Import connector package to access AIConnectors - Update processXgen function to include 'connectors' field in agent map - Add 'placeholder' field to the assistant data structure for enhanced context --- widgets/app/app.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/widgets/app/app.go b/widgets/app/app.go index 98115beb..ed8bd164 100644 --- a/widgets/app/app.go +++ b/widgets/app/app.go @@ -12,6 +12,7 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/yaoapp/gou/api" "github.com/yaoapp/gou/application" + "github.com/yaoapp/gou/connector" "github.com/yaoapp/gou/process" v8 "github.com/yaoapp/gou/runtime/v8" "github.com/yaoapp/gou/session" @@ -67,10 +68,6 @@ func Load(cfg config.Config) error { return err } - if err != nil { - return err - } - if data == nil { return fmt.Errorf("app.yao not found") } @@ -574,8 +571,10 @@ func processXgen(process *process.Process) interface{} { "assistant_name": ast.Name, "assistant_avatar": ast.Avatar, "assistant_deleteable": false, + "placeholder": ast.Placeholder, } } + agent["connectors"] = connector.AIConnectors } xgenSetting := map[string]interface{}{ From 586ad9866c429b3b1d4a48a5c302ad25effb7f35 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 17 Mar 2025 18:28:02 +0800 Subject: [PATCH 005/247] Enhance assistant loading and ID generation - Update loadMap function to handle prompts as both string and array types, improving flexibility in data loading. - Refactor assistant ID generation in SaveAssistant method to use a dedicated GenerateAssistantID function, ensuring unique IDs are created with error handling. - Add GenerateAssistantID method to create a random 6-digit ID while checking for uniqueness in the database. --- neo/assistant/load.go | 32 ++++++++++++++++++++++++++------ neo/assistant/utils.go | 5 ++++- neo/store/xun.go | 36 +++++++++++++++++++++++++++++++++++- 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/neo/assistant/load.go b/neo/assistant/load.go index 2b81df4a..9dc80b61 100644 --- a/neo/assistant/load.go +++ b/neo/assistant/load.go @@ -424,13 +424,33 @@ func loadMap(data map[string]interface{}) (*Assistant, error) { } // prompts - if v, ok := data["prompts"].(string); ok { - var prompts []Prompt - err := yaml.Unmarshal([]byte(v), &prompts) - if err != nil { - return nil, err + if prompts, has := data["prompts"]; has { + + switch v := prompts.(type) { + case []Prompt: + assistant.Prompts = v + + case string: + var prompts []Prompt + err := yaml.Unmarshal([]byte(v), &prompts) + if err != nil { + return nil, err + } + assistant.Prompts = prompts + + default: + raw, err := jsoniter.Marshal(v) + if err != nil { + return nil, err + } + + var prompts []Prompt + err = jsoniter.Unmarshal(raw, &prompts) + if err != nil { + return nil, err + } + assistant.Prompts = prompts } - assistant.Prompts = prompts } // tools diff --git a/neo/assistant/utils.go b/neo/assistant/utils.go index c1f3ce95..c931bf19 100644 --- a/neo/assistant/utils.go +++ b/neo/assistant/utils.go @@ -33,8 +33,11 @@ func getTimestamp(v interface{}) (int64, error) { return ts, nil } + case nil: + return 0, nil } - return 0, fmt.Errorf("invalid timestamp type") + + return 0, fmt.Errorf("invalid timestamp type %T", v) } func stringToTimestamp(v string) (int64, error) { diff --git a/neo/store/xun.go b/neo/store/xun.go index 8c87bb62..26862877 100644 --- a/neo/store/xun.go +++ b/neo/store/xun.go @@ -963,7 +963,11 @@ func (conv *Xun) SaveAssistant(assistant map[string]interface{}) (interface{}, e // Generate assistant_id if not provided if _, ok := assistantCopy["assistant_id"]; !ok { - assistantCopy["assistant_id"] = uuid.New().String() + var err error + assistantCopy["assistant_id"], err = conv.GenerateAssistantID() + if err != nil { + return nil, err + } } // Check if assistant exists @@ -1359,3 +1363,33 @@ func (conv *Xun) GetHistoryWithFilter(sid string, cid string, filter ChatFilter) return res, nil } + +// GenerateAssistantID generates a random-looking 6-digit ID +func (conv *Xun) GenerateAssistantID() (string, error) { + maxAttempts := 10 // Maximum number of attempts to generate a unique ID + for i := 0; i < maxAttempts; i++ { + // Generate a random number using timestamp and some bit operations + timestamp := time.Now().UnixNano() + random := (timestamp ^ (timestamp >> 12)) % 1000000 + hash := fmt.Sprintf("%06d", random) + + // Check if this ID already exists + exists, err := conv.query.New(). + Table(conv.getAssistantTable()). + Where("assistant_id", hash). + Exists() + + if err != nil { + return "", err + } + + if !exists { + return hash, nil + } + + // If ID exists, wait a bit and try again + time.Sleep(time.Millisecond) + } + + return "", fmt.Errorf("failed to generate unique ID after %d attempts", maxAttempts) +} From f820e376ab26a3bfc10fe3ecedc47dd2308f15b4 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 18 Mar 2025 15:36:44 +0800 Subject: [PATCH 006/247] Refactor assistant save and delete methods for cache management - Rename variable for clarity in handleAssistantSave method. - Implement cache removal and reloading for assistants after save and delete operations to ensure data consistency. - Add GetCache method to retrieve the loaded cache for better cache management. --- neo/api.go | 31 +++++++++++++++++++++++++------ neo/assistant/load.go | 5 +++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/neo/api.go b/neo/api.go index 2aa14591..fc300421 100644 --- a/neo/api.go +++ b/neo/api.go @@ -1131,14 +1131,14 @@ func (neo *DSL) handleAssistantDetail(c *gin.Context) { // handleAssistantSave handles creating or updating an assistant func (neo *DSL) handleAssistantSave(c *gin.Context) { - var assistant map[string]interface{} - if err := c.BindJSON(&assistant); err != nil { + var assistantData map[string]interface{} + if err := c.BindJSON(&assistantData); err != nil { c.JSON(400, gin.H{"message": "invalid request body", "code": 400}) c.Done() return } - id, err := neo.Store.SaveAssistant(assistant) + id, err := neo.Store.SaveAssistant(assistantData) if err != nil { c.JSON(500, gin.H{"message": err.Error(), "code": 500}) c.Done() @@ -1146,11 +1146,24 @@ func (neo *DSL) handleAssistantSave(c *gin.Context) { } // Update the assistant map with the returned ID if it's not already set - if _, ok := assistant["assistant_id"]; !ok { - assistant["assistant_id"] = id + if _, ok := assistantData["assistant_id"]; !ok { + assistantData["assistant_id"] = id } - c.JSON(200, gin.H{"message": "ok", "data": assistant}) + // Remove the assistant from cache to ensure fresh data on next load + cache := assistant.GetCache() + if cache != nil { + cache.Remove(id.(string)) + } + + // Reload the assistant to ensure it's available in cache with updated data + _, err = assistant.Get(id.(string)) + if err != nil { + // Just log the error, don't fail the request + fmt.Printf("Error reloading assistant %s: %v\n", id, err) + } + + c.JSON(200, gin.H{"message": "ok", "data": assistantData}) c.Done() } @@ -1170,6 +1183,12 @@ func (neo *DSL) handleAssistantDelete(c *gin.Context) { return } + // Remove the assistant from cache to ensure it's fully deleted + cache := assistant.GetCache() + if cache != nil { + cache.Remove(assistantID) + } + c.JSON(200, gin.H{"message": "ok"}) c.Done() } diff --git a/neo/assistant/load.go b/neo/assistant/load.go index 9dc80b61..052b8de6 100644 --- a/neo/assistant/load.go +++ b/neo/assistant/load.go @@ -148,6 +148,11 @@ func ClearCache() { } } +// GetCache returns the loaded cache +func GetCache() *Cache { + return loaded +} + // LoadStore create a new assistant from store func LoadStore(id string) (*Assistant, error) { From 6edaa712df1edaa4e4b17fa9e54c25ff23349356 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 21 Mar 2025 18:36:59 +0800 Subject: [PATCH 007/247] Update error message for missing assistant data in GetAssistant method - Change error message from "assistant not found" to "the assistant is empty" for improved clarity in the Xun store. --- neo/store/xun.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/store/xun.go b/neo/store/xun.go index 26862877..3b897b30 100644 --- a/neo/store/xun.go +++ b/neo/store/xun.go @@ -1194,7 +1194,7 @@ func (conv *Xun) GetAssistant(assistantID string) (map[string]interface{}, error data := row.ToMap() if data == nil || len(data) == 0 { - return nil, fmt.Errorf("assistant %s not found", assistantID) + return nil, fmt.Errorf("the assistant %s is empty", assistantID) } // Parse JSON fields From 211e523759ab286582701c1d8cad783dc53fe61e Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 21 Mar 2025 19:57:46 +0800 Subject: [PATCH 008/247] Implement built-in assistant deletion and loading improvements - Retrieve existing built-in assistants before deletion to ensure accurate removal. - Update LoadBuiltIn function to track and delete assistants that are no longer present. - Enhance error handling during the deletion process for better reliability. --- neo/assistant/load.go | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/neo/assistant/load.go b/neo/assistant/load.go index 052b8de6..2c90726d 100644 --- a/neo/assistant/load.go +++ b/neo/assistant/load.go @@ -41,13 +41,23 @@ func LoadBuiltIn() error { return err } + // Get all existing built-in assistants + deletedBuiltIn := map[string]bool{} + // Remove the built-in assistants if storage != nil { + builtIn := true - _, err := storage.DeleteAssistants(store.AssistantFilter{BuiltIn: &builtIn}) + res, err := storage.GetAssistants(store.AssistantFilter{BuiltIn: &builtIn, Select: []string{"assistant_id", "id"}}) if err != nil { return err } + + // Get all existing built-in assistants + for _, assistant := range res.Data { + assistantID := assistant["assistant_id"].(string) + deletedBuiltIn[assistantID] = true + } } // Check if the assistant is built-in @@ -96,6 +106,22 @@ func LoadBuiltIn() error { sort++ loaded.Put(assistant) + // Remove the built-in assistant from the store + if _, ok := deletedBuiltIn[assistant.ID]; ok { + delete(deletedBuiltIn, assistant.ID) + } + } + + // Remove deleted built-in assistants + if len(deletedBuiltIn) > 0 { + assistantIDs := []string{} + for assistantID := range deletedBuiltIn { + assistantIDs = append(assistantIDs, assistantID) + } + _, err := storage.DeleteAssistants(store.AssistantFilter{AssistantIDs: assistantIDs}) + if err != nil { + return err + } } return nil From f0da60f467e52716a55efe36f757bc7eab89109f Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 21 Mar 2025 20:56:35 +0800 Subject: [PATCH 009/247] feat: Migrate Excel plugin to built-in functionality Move Excel plugin from external plugin to built-in functionality, including: - Core Excel file operations (open, close, save) - Sheet manipulation and cell operations - Row and column iterators - Process handlers for Excel operations - Comprehensive test coverage This change internalizes Excel manipulation capabilities: - Read/write operations on cells, rows and columns - Sheet management and styling - Cell merging and formula handling - Coordinate conversion utilities --- excel/each.go | 104 ++++++++++ excel/each_test.go | 64 +++++++ excel/excel.go | 102 ++++++++++ excel/excel_test.go | 252 ++++++++++++++++++++++++ excel/process.go | 261 +++++++++++++++++++++++++ excel/process_test.go | 432 ++++++++++++++++++++++++++++++++++++++++++ excel/write.go | 75 ++++++++ main.go | 1 + 8 files changed, 1291 insertions(+) create mode 100644 excel/each.go create mode 100644 excel/each_test.go create mode 100644 excel/excel.go create mode 100644 excel/excel_test.go create mode 100644 excel/process.go create mode 100644 excel/process_test.go create mode 100644 excel/write.go diff --git a/excel/each.go b/excel/each.go new file mode 100644 index 00000000..0d4ea904 --- /dev/null +++ b/excel/each.go @@ -0,0 +1,104 @@ +package excel + +import ( + "fmt" + "sync" + "time" + + "github.com/google/uuid" + "github.com/xuri/excelize/v2" +) + +// Cols defines an iterator to a sheet +type Cols struct { + id string + *excelize.Cols + create int64 +} + +// Rows defines an iterator to a sheet +type Rows struct { + id string + *excelize.Rows + create int64 +} + +var openCols = sync.Map{} +var openRows = sync.Map{} + +// OpenRow each row of the sheet +func (excel *Excel) OpenRow(sheet string) (string, error) { + id := uuid.NewString() + rows, err := excel.Rows(sheet) + if err != nil { + return "", err + } + openRows.Store(id, &Rows{id: id, Rows: rows, create: time.Now().Unix()}) + return id, nil +} + +// NextRow next row of the sheet +func NextRow(id string) ([]string, error) { + value, ok := openRows.Load(id) + if !ok { + return nil, fmt.Errorf("rows %s not found", id) + } + + if value.(*Rows).Next() { + row, err := value.(*Rows).Columns() + // fmt.Printf("DEBUG: %#v %v %v\n", row, err, row == nil) + if err != nil { + return nil, err + } + + if row == nil { + return []string{}, nil + } + return row, nil + } + return nil, nil +} + +// CloseRow done the sheet +func CloseRow(id string) { + openRows.Delete(id) +} + +// OpenColumn each cols of the sheet +func (excel *Excel) OpenColumn(sheet string) (string, error) { + id := uuid.NewString() + cols, err := excel.Cols(sheet) + if err != nil { + return "", err + } + openCols.Store(id, &Cols{id: id, Cols: cols, create: time.Now().Unix()}) + return id, nil +} + +// NextColumn next col of the sheet +func NextColumn(id string) ([]string, error) { + value, ok := openCols.Load(id) + if !ok { + return nil, fmt.Errorf("cols %s not found", id) + } + + if value.(*Cols).Next() { + col, err := value.(*Cols).Rows() + if err != nil { + return nil, err + } + + if col == nil { + return []string{}, nil + } + + return col, nil + } + + return nil, nil +} + +// CloseColumn done the sheet +func CloseColumn(id string) { + openCols.Delete(id) +} diff --git a/excel/each_test.go b/excel/each_test.go new file mode 100644 index 00000000..5d29e86a --- /dev/null +++ b/excel/each_test.go @@ -0,0 +1,64 @@ +package excel + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEachCols(t *testing.T) { + files := testFiles(t) + h1, err := Open(files["test-01"], false) + if err != nil { + t.Fatal(err) + } + defer Close(h1) + + xls, err := Get(h1) + if err != nil { + t.Fatal(err) + } + + id, err := xls.OpenColumn("供销存管理表格") + if err != nil { + t.Fatal(err) + } + defer CloseColumn(id) + + res := []string{} + for col, err := NextColumn(id); err == nil && col != nil; col, err = NextColumn(id) { + res = append(res, col...) + } + + assert.Contains(t, strings.Join(res, ""), "供销存管理表格产品查询") + assert.Contains(t, strings.Join(res, ""), "刘大大") +} + +func TestEachRows(t *testing.T) { + files := testFiles(t) + h1, err := Open(files["test-01"], false) + if err != nil { + t.Fatal(err) + } + defer Close(h1) + + xls, err := Get(h1) + if err != nil { + t.Fatal(err) + } + + id, err := xls.OpenRow("供销存管理表格") + if err != nil { + t.Fatal(err) + } + defer CloseRow(id) + + res := []string{} + for row, err := NextRow(id); err == nil && row != nil; row, err = NextRow(id) { + res = append(res, row...) + } + + assert.Contains(t, strings.Join(res, ""), "供销存管理表格产品查询") + assert.Contains(t, strings.Join(res, ""), "刘大大") +} diff --git a/excel/excel.go b/excel/excel.go new file mode 100644 index 00000000..e60cd646 --- /dev/null +++ b/excel/excel.go @@ -0,0 +1,102 @@ +package excel + +import ( + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/google/uuid" + "github.com/xuri/excelize/v2" + "github.com/yaoapp/yao/config" +) + +// Excel the excel file +type Excel struct { + id string + path string + create int64 + *excelize.File +} + +// openFiles the open files +var openFiles = sync.Map{} + +// Open open the excel file +func Open(path string, writable bool) (string, error) { + + excel := &Excel{path: path} + // GET DATA ROOT + root := config.Conf.DataRoot + path, err := filepath.Abs(filepath.Join(root, path)) + if err != nil { + return "", err + } + + if writable { + + // if the file not exists, create it + if _, err := os.Stat(path); os.IsNotExist(err) { + create := excelize.NewFile() + err := create.SaveAs(path) + if err != nil { + return "", err + } + create.Close() + } + + excelFile, err := excelize.OpenFile(path) + if err != nil { + return "", err + } + id := uuid.NewString() + excel.File = excelFile + excel.id = id + excel.create = time.Now().Unix() + openFiles.Store(id, excel) + return id, nil + } + + file, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("open file %s failed: %w", path, err) + } + + excelFile, err := excelize.OpenReader(file) + if err != nil { + return "", err + } + + id := uuid.NewString() + excel.File = excelFile + excel.id = id + excel.create = time.Now().Unix() + openFiles.Store(id, excel) + return id, nil +} + +// Close close the excel file +func Close(handler string) error { + excel, ok := openFiles.Load(handler) + if !ok { + return fmt.Errorf("file not found") + } + + err := excel.(*Excel).Close() + if err != nil { + return err + } + + openFiles.Delete(handler) + return nil +} + +// Get get the excel file +func Get(handler string) (*Excel, error) { + excel, ok := openFiles.Load(handler) + if !ok { + return nil, fmt.Errorf("%s not found", handler) + } + return excel.(*Excel), nil +} diff --git a/excel/excel_test.go b/excel/excel_test.go new file mode 100644 index 00000000..e9a883a5 --- /dev/null +++ b/excel/excel_test.go @@ -0,0 +1,252 @@ +package excel + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +func TestOpenClose(t *testing.T) { + files := testFiles(t) + + h1, err := Open(files["test-01"], false) + if err != nil { + t.Fatal(err) + } + + if _, ok := openFiles.Load(h1); !ok { + t.Fatal("open file failed") + } + + h2, err := Open(files["test-02"], true) + if err != nil { + t.Fatal(err) + } + if _, ok := openFiles.Load(h2); !ok { + t.Fatal("open file failed") + } + + h3, err := Open(files["test-03"], false) + if err != nil { + t.Fatal(err) + } + if _, ok := openFiles.Load(h3); !ok { + t.Fatal("open file failed") + } + + _, err = Open(files["test-04"], false) + assert.Error(t, err) + + err = Close(h1) + if err != nil { + t.Fatal(err) + } + if _, ok := openFiles.Load(h1); ok { + t.Fatal("close file failed") + } +} + +func TestGetSheetList(t *testing.T) { + + files := testFiles(t) + h1, err := Open(files["test-01"], false) + if err != nil { + t.Fatal(err) + } + defer Close(h1) + + xls, err := Get(h1) + if err != nil { + t.Fatal(err) + } + + sheets := xls.GetSheetList() + assert.Equal(t, []string{"供销存管理表格", "使用说明"}, sheets) + + _, err = Get("not found") + assert.Error(t, err) +} + +func TestOpenInvalidFile(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Create an invalid excel file in the data root + root := "excel" + invalidFile := filepath.Join(root, "invalid.xlsx") + + // Ensure cleanup after test + defer func() { + if err := os.Remove(filepath.Join(config.Conf.DataRoot, invalidFile)); err != nil { + t.Logf("Failed to cleanup test file: %v", err) + } + }() + + err := os.WriteFile(filepath.Join(config.Conf.DataRoot, invalidFile), []byte("invalid content"), 0644) + if err != nil { + t.Fatal(err) + } + + _, err = Open(invalidFile, false) + assert.Error(t, err, "should fail to open invalid excel file") +} + +func TestCloseErrors(t *testing.T) { + // Test closing non-existent handler + err := Close("non-existent-handler") + assert.Error(t, err, "should fail to close non-existent file") + + // Test double close + files := testFiles(t) + h1, err := Open(files["test-01"], false) + if err != nil { + t.Fatal(err) + } + + // First close + err = Close(h1) + assert.NoError(t, err) + + // Second close should fail + err = Close(h1) + assert.Error(t, err, "should fail on second close") +} + +func TestOpenWithInvalidPath(t *testing.T) { + // Test with invalid path + _, err := Open("../invalid/path/file.xlsx", false) + assert.Error(t, err, "should fail with invalid path") + + // Test with path trying to escape data root + _, err = Open("../../../../etc/file.xlsx", false) + assert.Error(t, err, "should fail with path trying to escape data root") +} + +func TestWrite(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Create a new writable file for testing + root := "excel" + testFile := filepath.Join(root, "write-test.xlsx") + + // Ensure cleanup after test + defer func() { + if err := os.Remove(filepath.Join(config.Conf.DataRoot, testFile)); err != nil { + t.Logf("Failed to cleanup test file: %v", err) + } + }() + + h1, err := Open(testFile, true) + if err != nil { + t.Fatal(err) + } + defer Close(h1) + + xls, err := Get(h1) + if err != nil { + t.Fatal(err) + } + + // Test WriteCell + err = xls.WriteCell("Sheet1", "A1", "Hello") + assert.NoError(t, err) + err = xls.WriteCell("Sheet1", "B1", 123) + assert.NoError(t, err) + err = xls.WriteCell("Sheet1", "C1", true) + assert.NoError(t, err) + + // Test WriteRow + row := []interface{}{"Row1", 456, false} + err = xls.WriteRow("Sheet1", "A2", row) + assert.NoError(t, err) + + // Test WriteColumn + col := []interface{}{"Col1", 789, true} + err = xls.WriteColumn("Sheet1", "D1", col) + assert.NoError(t, err) + + // Test WriteAll + data := [][]interface{}{ + {"Name", "Age", "City"}, + {"John", 30, "New York"}, + {"Alice", 25, "London"}, + } + err = xls.WriteAll("Sheet2", "A1", data) + assert.NoError(t, err) + + // Test error cases + // Invalid cell reference + err = xls.WriteCell("Sheet1", "invalid", "test") + assert.Error(t, err) + + // Save the file to verify changes + err = xls.SaveAs(filepath.Join(config.Conf.DataRoot, testFile)) + assert.NoError(t, err) + + // Verify written data + val, err := xls.GetCellValue("Sheet1", "A1") + assert.NoError(t, err) + assert.Equal(t, "Hello", val) + + val, err = xls.GetCellValue("Sheet2", "A1") + assert.NoError(t, err) + assert.Equal(t, "Name", val) +} + +func TestSetSheet(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + root := "excel" + testFile := filepath.Join(root, "sheet-test.xlsx") + + // Ensure cleanup after test + defer func() { + if err := os.Remove(filepath.Join(config.Conf.DataRoot, testFile)); err != nil { + t.Logf("Failed to cleanup test file: %v", err) + } + }() + + h1, err := Open(testFile, true) + if err != nil { + t.Fatal(err) + } + defer Close(h1) + + xls, err := Get(h1) + if err != nil { + t.Fatal(err) + } + + // Test creating new sheet + idx, err := xls.SetSheet("NewSheet") + assert.NoError(t, err) + assert.Greater(t, idx, 0) + + // Test getting existing sheet + idx2, err := xls.SetSheet("NewSheet") + assert.NoError(t, err) + assert.Equal(t, idx, idx2) + + // Verify sheet exists + sheets := xls.GetSheetList() + assert.Contains(t, sheets, "NewSheet") +} + +func testFiles(t *testing.T) map[string]string { + test.Prepare(t, config.Conf) + defer test.Clean() + + // test data root path + root := "excel" + return map[string]string{ + "test-01": filepath.Join(root, "test-01.xlsx"), + "test-02": filepath.Join(root, "test-02.xlsx"), + "test-03": filepath.Join(root, "test-03.xlsx"), + } +} diff --git a/excel/process.go b/excel/process.go new file mode 100644 index 00000000..bb616925 --- /dev/null +++ b/excel/process.go @@ -0,0 +1,261 @@ +package excel + +import ( + "github.com/xuri/excelize/v2" + "github.com/yaoapp/gou/process" + "github.com/yaoapp/kun/exception" +) + +func init() { + process.RegisterGroup("excel", map[string]process.Handler{ + "open": processOpen, + "close": processClose, + "save": processSave, + "sheets": processSheets, + + "read.cell": processReadCell, + "write.cell": processWriteCell, + + "set.style": processSetStyle, + "set.formula": processSetFormula, + "set.link": processSetLink, + "set.mergecell": processMergeCell, + "set.unmergecell": processUnmergeCell, + + "convert.columnnametonumber": processColumnNameToNumber, + "convert.columnnumbertoname": processColumnNumberToName, + "convert.cellnametocoordinates": processCellNameToCoordinates, + "convert.coordinatestocellname": processCoordinatesToCellName, + }) +} + +// processOpen process the excel.open +func processOpen(process *process.Process) interface{} { + process.ValidateArgNums(1) + file := process.ArgsString(0) + readonly := false + if len(process.Args) > 1 { + readonly = process.ArgsBool(1) + } + + handle, err := Open(file, readonly) + if err != nil { + exception.New("excel.open %s error: %s", 500, file, err.Error()).Throw() + } + return handle +} + +// processClose process the excel.close +func processClose(process *process.Process) interface{} { + process.ValidateArgNums(1) + handle := process.ArgsString(0) + err := Close(handle) + if err != nil { + exception.New("excel.close %s error: %s", 500, handle, err.Error()).Throw() + } + return nil +} + +// processSave process the excel.save +func processSave(process *process.Process) interface{} { + process.ValidateArgNums(1) + handle := process.ArgsString(0) + xls, err := Get(handle) + if err != nil { + exception.New("excel.save %s error: %s", 500, handle, err.Error()).Throw() + } + err = xls.Save() + if err != nil { + exception.New("excel.save %s error: %s", 500, handle, err.Error()).Throw() + } + return nil +} + +// processSheets process the excel.sheets +func processSheets(process *process.Process) interface{} { + process.ValidateArgNums(1) + handle := process.ArgsString(0) + xls, err := Get(handle) + if err != nil { + exception.New("excel.sheets %s error: %s", 500, handle, err.Error()).Throw() + } + return xls.GetSheetList() +} + +// processReadCell process the excel.read.cell +func processReadCell(process *process.Process) interface{} { + process.ValidateArgNums(3) + handle := process.ArgsString(0) + sheet := process.ArgsString(1) + cell := process.ArgsString(2) + + xls, err := Get(handle) + if err != nil { + exception.New("excel.read.cell %s error: %s", 500, handle, err.Error()).Throw() + } + value, err := xls.GetCellValue(sheet, cell) + if err != nil { + exception.New("excel.read.cell %s:%s:%s error: %s", 500, handle, sheet, cell, err.Error()).Throw() + } + return value +} + +// processWriteCell process the excel.write.cell +func processWriteCell(process *process.Process) interface{} { + process.ValidateArgNums(4) + handle := process.ArgsString(0) + sheet := process.ArgsString(1) + cell := process.ArgsString(2) + value := process.Args[3] + + xls, err := Get(handle) + if err != nil { + exception.New("excel.write.cell %s error: %s", 500, handle, err.Error()).Throw() + } + err = xls.SetCellValue(sheet, cell, value) + if err != nil { + exception.New("excel.write.cell %s:%s:%s error: %s", 500, handle, sheet, cell, err.Error()).Throw() + } + return nil +} + +// processSetStyle process the excel.set.style