From 238347c834ea0bb0126c8f61af2fd3d0a82f5145 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 31 Dec 2024 16:01:48 +0800 Subject: [PATCH 1/8] Refactor message handling and enhance file reading capabilities in Neo API - Updated the message handling logic to improve error reporting and content management, ensuring more robust communication with clients. - Introduced a new ReadBase64 method in both Local and OpenAI assistant implementations to read files and return their base64 encoded content, enhancing file handling capabilities. - Removed the deprecated JSON message handling code, streamlining the message processing structure. - Refactored message struct to include new fields and methods for better data management and response handling. - Improved the chat functionality to handle attachments and user messages more effectively, ensuring a smoother user experience. --- neo/assistant/local/file.go | 28 ++++ neo/assistant/openai/chat.go | 90 ++++++++++- neo/assistant/openai/file.go | 28 ++++ neo/assistant/types.go | 1 + neo/message/json.go | 266 ------------------------------- neo/message/message.go | 294 ++++++++++++++++++++++++++++++++++- neo/message/types.go | 27 ---- neo/neo.go | 52 ++++--- 8 files changed, 466 insertions(+), 320 deletions(-) delete mode 100644 neo/message/json.go delete mode 100644 neo/message/types.go diff --git a/neo/assistant/local/file.go b/neo/assistant/local/file.go index 79d40b6f..cb7a692a 100644 --- a/neo/assistant/local/file.go +++ b/neo/assistant/local/file.go @@ -3,6 +3,7 @@ package local import ( "context" "crypto/sha256" + "encoding/base64" "fmt" "io" "mime/multipart" @@ -132,3 +133,30 @@ func (ast *Local) Download(ctx context.Context, fileID string) (*assistant.FileR Extension: ext, }, nil } + +// ReadBase64 reads a file and returns its base64 encoded content +func (ast *Local) ReadBase64(ctx context.Context, fileID string) (string, error) { + // Get the data filesystem + data, err := fs.Get("data") + if err != nil { + return "", fmt.Errorf("get filesystem error: %s", err.Error()) + } + + // Check if file exists + exists, err := data.Exists(fileID) + if err != nil { + return "", fmt.Errorf("check file error: %s", err.Error()) + } + if !exists { + return "", fmt.Errorf("file %s not found", fileID) + } + + // Read file content + content, err := data.ReadFile(fileID) + if err != nil { + return "", fmt.Errorf("read file error: %s", err.Error()) + } + + // Encode to base64 + return base64.StdEncoding.EncodeToString(content), nil +} diff --git a/neo/assistant/openai/chat.go b/neo/assistant/openai/chat.go index 765c6f3a..1bf80a8c 100644 --- a/neo/assistant/openai/chat.go +++ b/neo/assistant/openai/chat.go @@ -3,6 +3,9 @@ package openai import ( "context" "fmt" + "strings" + + chatMessage "github.com/yaoapp/yao/neo/message" ) // Chat the chat struct @@ -21,10 +24,95 @@ func (ast *OpenAI) Chat(ctx context.Context, messages []map[string]interface{}, return fmt.Errorf("openai is not initialized") } - _, ext := ast.openai.ChatCompletionsWith(ctx, messages, option, cb) + requestMessages, err := ast.requestMessages(ctx, messages) + if err != nil { + return fmt.Errorf("request messages error: %s", err.Error()) + } + + _, ext := ast.openai.ChatCompletionsWith(ctx, requestMessages, option, cb) if ext != nil { return fmt.Errorf("openai chat completions with error: %s", ext.Message) } return nil } + +func (ast *OpenAI) requestMessages(ctx context.Context, messages []map[string]interface{}) ([]map[string]interface{}, error) { + newMessages := []map[string]interface{}{} + length := len(messages) + for index, message := range messages { + role, ok := message["role"].(string) + if !ok { + return nil, fmt.Errorf("role must be string") + } + + content, ok := message["content"].(string) + if !ok { + return nil, fmt.Errorf("content must be string") + } + + newMessage := map[string]interface{}{ + "role": role, + "content": content, + } + + // Handle name if present + if name, ok := message["name"].(string); ok { + newMessage["name"] = name + } + + newMessage["content"] = content + + // Special handling for user messages with JSON content last message + if role == "user" && index == length-1 { + content = strings.TrimSpace(content) + msg, err := chatMessage.NewString(content) + if err != nil { + return nil, fmt.Errorf("new string error: %s", err.Error()) + } + + newMessage["content"] = msg.Text + if msg.Attachments != nil { + content, err := ast.withAttachments(ctx, msg) + if err != nil { + return nil, fmt.Errorf("with attachments error: %s", err.Error()) + } + newMessage["content"] = content + } + } + + newMessages = append(newMessages, newMessage) + } + return newMessages, nil +} + +func (ast *OpenAI) withAttachments(ctx context.Context, msg *chatMessage.Message) ([]map[string]interface{}, error) { + contents := []map[string]interface{}{{"type": "text", "text": msg.Text}} + images := []string{} + for _, attachment := range msg.Attachments { + if strings.HasPrefix(attachment.ContentType, "image/") { + images = append(images, attachment.FileID) + } + } + + if len(images) == 0 { + return contents, nil + } + + for _, image := range images { + bytes64, err := ast.ReadBase64(ctx, image) + if err != nil { + return nil, fmt.Errorf("read base64 error: %s", err.Error()) + } + + contents = append(contents, map[string]interface{}{ + "type": "image_url", + "image_url": map[string]string{ + "url": fmt.Sprintf("data:image/jpeg;base64,%s", bytes64), + }, + }, + ) + } + + return contents, nil +} diff --git a/neo/assistant/openai/file.go b/neo/assistant/openai/file.go index 44892d49..062a688b 100644 --- a/neo/assistant/openai/file.go +++ b/neo/assistant/openai/file.go @@ -3,6 +3,7 @@ package openai import ( "context" "crypto/sha256" + "encoding/base64" "fmt" "io" "mime/multipart" @@ -137,3 +138,30 @@ func (ast *OpenAI) Download(ctx context.Context, fileID string) (*assistant.File Extension: ext, }, nil } + +// ReadBase64 reads a file and returns its base64 encoded content +func (ast *OpenAI) ReadBase64(ctx context.Context, fileID string) (string, error) { + // Get the data filesystem + data, err := fs.Get("data") + if err != nil { + return "", fmt.Errorf("get filesystem error: %s", err.Error()) + } + + // Check if file exists + exists, err := data.Exists(fileID) + if err != nil { + return "", fmt.Errorf("check file error: %s", err.Error()) + } + if !exists { + return "", fmt.Errorf("file %s not found", fileID) + } + + // Read file content + content, err := data.ReadFile(fileID) + if err != nil { + return "", fmt.Errorf("read file error: %s", err.Error()) + } + + // Encode to base64 + return base64.StdEncoding.EncodeToString(content), nil +} diff --git a/neo/assistant/types.go b/neo/assistant/types.go index 35ad239d..a122ed06 100644 --- a/neo/assistant/types.go +++ b/neo/assistant/types.go @@ -11,6 +11,7 @@ type API interface { Chat(ctx context.Context, messages []map[string]interface{}, option map[string]interface{}, cb func(data []byte) int) error Upload(ctx context.Context, file *multipart.FileHeader, reader io.Reader, option map[string]interface{}) (*File, error) Download(ctx context.Context, fileID string) (*FileResponse, error) + ReadBase64(ctx context.Context, fileID string) (string, error) } // Prompt a prompt diff --git a/neo/message/json.go b/neo/message/json.go deleted file mode 100644 index 1ebda8a1..00000000 --- a/neo/message/json.go +++ /dev/null @@ -1,266 +0,0 @@ -package message - -import ( - "fmt" - "strings" - - "github.com/fatih/color" - "github.com/gin-gonic/gin" - jsoniter "github.com/json-iterator/go" - "github.com/yaoapp/gou/helper" - "github.com/yaoapp/kun/exception" - "github.com/yaoapp/kun/log" - "github.com/yaoapp/kun/maps" - "github.com/yaoapp/yao/openai" -) - -// JSON the JSON message -type JSON struct{ *Message } - -// New create a new JSON message -func New() *JSON { - return &JSON{makeMessage()} -} - -// NewOpenAI create a new JSON message -func NewOpenAI(data []byte) *JSON { - - if data == nil || len(data) == 0 { - return nil - } - - msg := makeMessage() - text := string(data) - data = []byte(strings.TrimPrefix(text, "data: ")) - switch { - case strings.Contains(text, `"delta":{`) && strings.Contains(text, `"content":`): - var message openai.Message - err := jsoniter.Unmarshal(data, &message) - if err != nil { - msg.Text = err.Error() - return &JSON{msg} - } - - if len(message.Choices) > 0 { - msg.Text = message.Choices[0].Delta.Content - } - break - - case strings.Contains(text, `[DONE]`): - msg.Done = true - break - - case strings.Contains(text, `"finish_reason":"stop"`): - msg.Done = true - break - - default: - - str := string(data) - // Remove "data: " and " - str = strings.TrimPrefix(str, "data: ") - str = strings.Trim(str, "\"") - msg.Type = "error" - msg.Text = str - } - - return &JSON{msg} -} - -func (json *JSON) String() string { - if json.Message == nil { - return "" - } - return json.Message.Text -} - -// Text set the text -func (json *JSON) Text(text string) *JSON { - - json.Message.Text = text - if json.Message.Data != nil { - replaced := helper.Bind(text, json.Message.Data) - if replacedText, ok := replaced.(string); ok { - json.Message.Text = replacedText - } - } - - return json -} - -// Error set the error -func (json *JSON) Error(message interface{}) *JSON { - json.Message.Type = "error" - if err, ok := message.(error); ok { - json.Message.Text = err.Error() - } else if msg, ok := message.(string); ok { - json.Message.Text = msg - } else { - json.Message.Text = fmt.Sprintf("%v", message) - } - return json -} - -// Map set from map -func (json *JSON) Map(msg map[string]interface{}) *JSON { - if msg == nil { - return json - } - - if text, ok := msg["text"].(string); ok { - json.Message.Text = text - } - - if typ, ok := msg["type"].(string); ok { - json.Message.Text = typ - } - - if done, ok := msg["done"].(bool); ok { - json.Message.Done = done - } - - if confirm, ok := msg["confirm"].(bool); ok { - json.Message.Confirm = confirm - } - - if command, ok := msg["command"].(map[string]interface{}); ok { - json.Message.Command = &Command{} - if id, ok := command["id"].(string); ok { - json.Message.Command.ID = id - } - if name, ok := command["name"].(string); ok { - json.Message.Command.Name = name - } - if request, ok := command["request"].(string); ok { - json.Message.Command.Reqeust = request - } - } - - if actions, ok := msg["actions"].([]interface{}); ok { - for _, action := range actions { - if v, ok := action.(map[string]interface{}); ok { - action := Action{} - if name, ok := v["name"].(string); ok { - action.Name = name - } - if t, ok := v["type"].(string); ok { - action.Type = t - } - if payload, ok := v["payload"].(map[string]interface{}); ok { - action.Payload = payload - } - - if next, ok := v["next"].(string); ok { - action.Next = next - } - json.Message.Actions = append(json.Message.Actions, action) - } - } - } - - if data, ok := msg["data"].(map[string]interface{}); ok { - json.Message.Data = data - } - - return json -} - -// Done set the done -func (json *JSON) Done() *JSON { - json.Message.Done = true - return json -} - -// Confirm set the confirm -func (json *JSON) Confirm() *JSON { - json.Message.Confirm = true - return json -} - -// Command set the command -func (json *JSON) Command(name, id, request string) *JSON { - json.Message.Command = &Command{ - ID: id, - Name: name, - Reqeust: request, - } - return json -} - -// Action set the action -func (json *JSON) Action(name string, t string, payload interface{}, next string) *JSON { - - if json.Message.Data != nil { - payload = helper.Bind(payload, json.Message.Data) - } - - json.Message.Actions = append(json.Message.Actions, Action{ - Name: name, - Type: t, - Payload: payload, - Next: next, - }) - return json -} - -// Bind replace with data -func (json *JSON) Bind(data map[string]interface{}) *JSON { - if data == nil { - return json - } - - json.Message.Data = maps.Of(data).Dot() - return json -} - -// IsDone check if the message is done -func (json *JSON) IsDone() bool { - return json.Message.Done -} - -// Write the message -func (json *JSON) Write(w gin.ResponseWriter) bool { - - defer func() { - if r := recover(); r != nil { - message := "Write Response Exception: (if clinet close the connection, it's normal) \n %s\n\n" - color.Red(message, r) - } - }() - - data, err := jsoniter.Marshal(json.Message) - if err != nil { - log.Error("%s", err.Error()) - return false - } - - data = append([]byte("data: "), data...) - data = append(data, []byte("\n\n")...) - - _, err = w.Write(data) - if err != nil { - color.Red("Write JSON Message Error: %s", err.Error()) - return false - } - w.Flush() - return true -} - -// Append the message -func (json *JSON) Append(content []byte) []byte { - return append(content, []byte(json.Message.Text)...) -} - -func (json *JSON) writeError(w gin.ResponseWriter, message string) { - data := []byte(`{"text":"` + strings.Trim(exception.New(message, 500).Message, "\"") + `","type":"error"}`) - if json.Message.Done { - data = []byte(`{"text":"` + strings.Trim(exception.New(message, 500).Message, "\"") + `","type":"error", "done":true}`) - } - data = append([]byte("data: "), data...) - data = append(data, []byte("\n\n")...) - _, err := w.Write(data) - if err != nil { - color.Red("Write JSON Message Error: %s", message) - } - w.Flush() -} diff --git a/neo/message/message.go b/neo/message/message.go index 7726dd26..48c3a770 100644 --- a/neo/message/message.go +++ b/neo/message/message.go @@ -1,6 +1,296 @@ package message -// makeMessage create a new message -func makeMessage() *Message { +import ( + "fmt" + "strings" + + "github.com/fatih/color" + "github.com/gin-gonic/gin" + jsoniter "github.com/json-iterator/go" + "github.com/yaoapp/gou/helper" + "github.com/yaoapp/kun/exception" + "github.com/yaoapp/kun/log" + "github.com/yaoapp/kun/maps" + "github.com/yaoapp/yao/openai" +) + +// Message the message +type Message struct { + Text string `json:"text,omitempty"` // text content + Type string `json:"type,omitempty"` // error, text, plan, table, form, page, file, video, audio, image, markdown, json ... + Props map[string]interface{} `json:"props,omitempty"` // props for the types + IsDone bool `json:"done,omitempty"` + Actions []Action `json:"actions,omitempty"` // Conversation Actions for frontend + Attachments []Attachment `json:"attachments,omitempty"` // File attachments + Data map[string]interface{} `json:"-"` +} + +// Attachment represents a file attachment +type Attachment struct { + Name string `json:"name,omitempty"` + URL string `json:"url,omitempty"` + Type string `json:"type,omitempty"` + ContentType string `json:"content_type,omitempty"` + Bytes int64 `json:"bytes,omitempty"` + CreatedAt int64 `json:"created_at,omitempty"` + FileID string `json:"file_id,omitempty"` + ChatID string `json:"chat_id,omitempty"` + AssistantID string `json:"assistant_id,omitempty"` +} + +// Action the action +type Action struct { + Name string `json:"name,omitempty"` + Type string `json:"type"` + Payload interface{} `json:"payload,omitempty"` +} + +// New create a new message +func New() *Message { return &Message{Actions: []Action{}} } + +// NewString create a new message from string +func NewString(content string) (*Message, error) { + if strings.HasPrefix(content, "{") && strings.HasSuffix(content, "}") { + var msg Message + if err := jsoniter.UnmarshalFromString(content, &msg); err != nil { + return nil, err + } + return &msg, nil + } + return &Message{Text: content}, nil +} + +// NewOpenAI create a new message from OpenAI response +func NewOpenAI(data []byte) *Message { + if data == nil || len(data) == 0 { + return nil + } + + msg := New() + text := string(data) + data = []byte(strings.TrimPrefix(text, "data: ")) + + switch { + case strings.Contains(text, `"delta":{`) && strings.Contains(text, `"content":`): + var message openai.Message + if err := jsoniter.Unmarshal(data, &message); err != nil { + msg.Text = err.Error() + "\n" + string(data) + return msg + } + + if len(message.Choices) > 0 { + msg.Text = message.Choices[0].Delta.Content + } + + case strings.Contains(text, `[DONE]`): + msg.IsDone = true + + case strings.Contains(text, `"finish_reason":"stop"`): + msg.IsDone = true + + default: + str := strings.TrimPrefix(strings.Trim(string(data), "\""), "data: ") + msg.Type = "error" + msg.Text = str + } + + return msg +} + +// String returns the string representation +func (m *Message) String() string { + if m.Text != "" { + return m.Text + } + return "" +} + +// SetText set the text +func (m *Message) SetText(text string) *Message { + m.Text = text + if m.Data != nil { + if replaced := helper.Bind(text, m.Data); replaced != nil { + if replacedText, ok := replaced.(string); ok { + m.Text = replacedText + } + } + } + return m +} + +// Error set the error +func (m *Message) Error(message interface{}) *Message { + m.Type = "error" + switch v := message.(type) { + case error: + m.Text = v.Error() + case string: + m.Text = v + default: + m.Text = fmt.Sprintf("%v", message) + } + return m +} + +// Map set from map +func (m *Message) Map(msg map[string]interface{}) *Message { + if msg == nil { + return m + } + + if text, ok := msg["text"].(string); ok { + m.Text = text + } + if typ, ok := msg["type"].(string); ok { + m.Type = typ + } + if done, ok := msg["done"].(bool); ok { + m.IsDone = done + } + if actions, ok := msg["actions"].([]interface{}); ok { + for _, action := range actions { + if v, ok := action.(map[string]interface{}); ok { + action := Action{} + if name, ok := v["name"].(string); ok { + action.Name = name + } + if t, ok := v["type"].(string); ok { + action.Type = t + } + if payload, ok := v["payload"].(map[string]interface{}); ok { + action.Payload = payload + } + m.Actions = append(m.Actions, action) + } + } + } + if data, ok := msg["data"].(map[string]interface{}); ok { + m.Data = data + } + return m +} + +// Done set the done flag +func (m *Message) Done() *Message { + m.IsDone = true + return m +} + +// Action add an action +func (m *Message) Action(name string, t string, payload interface{}, next string) *Message { + if m.Data != nil { + payload = helper.Bind(payload, m.Data) + } + m.Actions = append(m.Actions, Action{ + Name: name, + Type: t, + Payload: payload, + }) + return m +} + +// Bind replace with data +func (m *Message) Bind(data map[string]interface{}) *Message { + if data == nil { + return m + } + m.Data = maps.Of(data).Dot() + return m +} + +// Write writes the message to response writer +func (m *Message) Write(w gin.ResponseWriter) bool { + defer func() { + if r := recover(); r != nil { + message := "Write Response Exception: (if client close the connection, it's normal) \n %s\n\n" + color.Red(message, r) + } + }() + + data, err := jsoniter.Marshal(m) + if err != nil { + log.Error("%s", err.Error()) + return false + } + + data = append([]byte("data: "), data...) + data = append(data, []byte("\n\n")...) + + if _, err := w.Write(data); err != nil { + color.Red("Write JSON Message Error: %s", err.Error()) + return false + } + w.Flush() + return true +} + +// Append appends content to the byte slice +func (m *Message) Append(content []byte) []byte { + return append(content, []byte(m.Text)...) +} + +// WriteError writes an error message to response writer +func (m *Message) WriteError(w gin.ResponseWriter, message string) { + errMsg := strings.Trim(exception.New(message, 500).Message, "\"") + data := []byte(fmt.Sprintf(`{"text":"%s","type":"error"`, errMsg)) + if m.IsDone { + data = []byte(fmt.Sprintf(`{"text":"%s","type":"error","done":true`, errMsg)) + } + data = append([]byte("data: "), data...) + data = append(data, []byte("}\n\n")...) + + if _, err := w.Write(data); err != nil { + color.Red("Write JSON Message Error: %s", message) + } + w.Flush() +} + +// MarshalJSON implements json.Marshaler interface +func (m *Message) MarshalJSON() ([]byte, error) { + type Alias Message + return jsoniter.Marshal(&struct { + *Alias + }{ + Alias: (*Alias)(m), + }) +} + +// UnmarshalJSON implements json.Unmarshaler interface +func (m *Message) UnmarshalJSON(data []byte) error { + type Alias Message + aux := &struct { + *Alias + }{ + Alias: (*Alias)(m), + } + if err := jsoniter.Unmarshal(data, &aux); err != nil { + return err + } + return nil +} + +// MarshalJSON implements json.Marshaler interface +func (a *Action) MarshalJSON() ([]byte, error) { + type Alias Action + return jsoniter.Marshal(&struct { + *Alias + }{ + Alias: (*Alias)(a), + }) +} + +// UnmarshalJSON implements json.Unmarshaler interface +func (a *Action) UnmarshalJSON(data []byte) error { + type Alias Action + aux := &struct { + *Alias + }{ + Alias: (*Alias)(a), + } + if err := jsoniter.Unmarshal(data, &aux); err != nil { + return err + } + return nil +} diff --git a/neo/message/types.go b/neo/message/types.go deleted file mode 100644 index ed54cf31..00000000 --- a/neo/message/types.go +++ /dev/null @@ -1,27 +0,0 @@ -package message - -// Message the message -type Message struct { - Text string `json:"text,omitempty"` - Type string `json:"type,omitempty"` - Done bool `json:"done,omitempty"` - Confirm bool `json:"confirm,omitempty"` - Command *Command `json:"command,omitempty"` - Actions []Action `json:"actions,omitempty"` - Data map[string]interface{} `json:"-,omitempty"` -} - -// Action the action -type Action struct { - Name string `json:"name,omitempty"` - Type string `json:"type"` - Payload interface{} `json:"payload,omitempty"` - Next string `json:"next,omitempty"` -} - -// Command the command -type Command struct { - ID string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Reqeust string `json:"request,omitempty"` -} diff --git a/neo/neo.go b/neo/neo.go index 856082e6..0a034a67 100644 --- a/neo/neo.go +++ b/neo/neo.go @@ -124,7 +124,6 @@ func (neo *DSL) GenerateWithAI(ctx Context, input string, messageType string, sy clientBreak := make(chan bool, 1) done := make(chan bool, 1) fail := make(chan error, 1) - content := []byte{} // Chat with AI in background @@ -142,26 +141,28 @@ func (neo *DSL) GenerateWithAI(ctx Context, input string, messageType string, sy // Handle error if msg.Type == "error" { - fail <- fmt.Errorf("%s", msg.Message.Text) + fail <- fmt.Errorf("%s", msg.Text) return 0 // break } // Append content and send message content = msg.Append(content) - - // Only send real-time messages if not in silent mode - if !silent && msg.Message != nil && msg.Message.Text != "" { - message.New(). - Map(map[string]interface{}{ - "text": msg.Message.Text, - "done": msg.Message.Done, - }). - Write(c.Writer) + if !silent { + value := msg.String() + if value != "" { + message.New(). + Map(map[string]interface{}{ + "text": value, + "done": msg.IsDone, + }). + Write(c.Writer) + } } // Complete the stream - if msg.Message.Done { - if !silent && msg.Message.Text == "" { + if msg.IsDone { + value := msg.String() + if value == "" { msg.Write(c.Writer) } done <- true @@ -267,7 +268,6 @@ func (neo *DSL) Download(ctx Context, c *gin.Context) (*assistant.FileResponse, // chat chat with AI func (neo *DSL) chat(ast assistant.API, ctx Context, messages []map[string]interface{}, c *gin.Context) error { - if ast == nil { msg := message.New().Error("assistant is not initialized").Done() msg.Write(c.Writer) @@ -293,24 +293,26 @@ func (neo *DSL) chat(ast assistant.API, ctx Context, messages []map[string]inter // Handle error if msg.Type == "error" { - message.New().Error(msg.Message.Text).Done().Write(c.Writer) + value := msg.String() + message.New().Error(value).Done().Write(c.Writer) return 0 // break } // Append content and send message content = msg.Append(content) - if msg.Message != nil && msg.Message.Text != "" { + value := msg.String() + if value != "" { message.New(). Map(map[string]interface{}{ - "text": msg.Message.Text, - "done": msg.Message.Done, + "text": value, + "done": msg.IsDone, }). Write(c.Writer) } // Complete the stream - if msg.Message != nil && msg.Message.Done { - if msg.Message.Text == "" { + if msg.IsDone { + if value == "" { msg.Write(c.Writer) } done <- true @@ -544,9 +546,11 @@ func (neo *DSL) createConversation() error { // sendMessage sends a message to the client func (neo *DSL) sendMessage(w gin.ResponseWriter, data interface{}) error { - msg := message.New().Map(data.(map[string]interface{})) - if !msg.Write(w) { - return fmt.Errorf("failed to write message to stream") + if msg, ok := data.(map[string]interface{}); ok { + if !message.New().Map(msg).Write(w) { + return fmt.Errorf("failed to write message to stream") + } + return nil } - return nil + return fmt.Errorf("invalid message data type") } From e530ccc1cd33f8c6b038b33655445b92276d9a31 Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 1 Jan 2025 10:43:05 +0800 Subject: [PATCH 2/8] Refactor conversation management and migrate to store-based architecture in Neo API - Replaced the conversation handling logic with a new store-based approach, enhancing data management and retrieval capabilities. - Updated all relevant methods to utilize the new store interface, including GetChats, GetChat, GetHistory, and SaveAssistant, ensuring consistent functionality across the API. - Removed deprecated conversation-related files and structures, streamlining the codebase and improving maintainability. - Enhanced the AssistantFilter and AssistantResponse types to support the new store architecture, improving filtering and pagination capabilities. - Updated tests to cover the new store-based methods and ensure robust functionality across different storage backends. --- neo/api.go | 32 ++-- neo/assistant/assistant.go | 26 +++ neo/load.go | 12 +- neo/neo.go | 26 +-- neo/process.go | 37 ++-- neo/{conversation => store}/mongo.go | 6 +- neo/{conversation => store}/redis.go | 6 +- neo/{conversation => store}/types.go | 6 +- neo/{conversation => store}/weaviate.go | 6 +- neo/{conversation => store}/xun.go | 6 +- neo/{conversation => store}/xun_test.go | 233 +++++++++++------------- neo/types.go | 40 ++-- 12 files changed, 222 insertions(+), 214 deletions(-) create mode 100644 neo/assistant/assistant.go rename neo/{conversation => store}/mongo.go (93%) rename neo/{conversation => store}/redis.go (93%) rename neo/{conversation => store}/types.go (98%) rename neo/{conversation => store}/weaviate.go (93%) rename neo/{conversation => store}/xun.go (99%) rename neo/{conversation => store}/xun_test.go (79%) diff --git a/neo/api.go b/neo/api.go index be5640a4..1ecc7dd3 100644 --- a/neo/api.go +++ b/neo/api.go @@ -15,8 +15,8 @@ import ( "github.com/yaoapp/gou/connector" "github.com/yaoapp/gou/process" "github.com/yaoapp/yao/helper" - "github.com/yaoapp/yao/neo/conversation" "github.com/yaoapp/yao/neo/message" + "github.com/yaoapp/yao/neo/store" ) // API registers the Neo API endpoints @@ -227,7 +227,7 @@ func (neo *DSL) handleChatList(c *gin.Context) { } // Create filter from query parameters - filter := conversation.ChatFilter{ + filter := store.ChatFilter{ Keywords: c.Query("keywords"), Order: c.Query("order"), } @@ -245,7 +245,7 @@ func (neo *DSL) handleChatList(c *gin.Context) { } } - response, err := neo.Conversation.GetChats(sid, filter) + response, err := neo.Store.GetChats(sid, filter) if err != nil { c.JSON(500, gin.H{"message": err.Error(), "code": 500}) c.Done() @@ -266,7 +266,7 @@ func (neo *DSL) handleChatHistory(c *gin.Context) { } cid := c.Query("chat_id") - history, err := neo.Conversation.GetHistory(sid, cid) + history, err := neo.Store.GetHistory(sid, cid) if err != nil { c.JSON(500, gin.H{"message": err.Error(), "code": 500}) c.Done() @@ -450,7 +450,7 @@ func (neo *DSL) handleChatDetail(c *gin.Context) { return } - chat, err := neo.Conversation.GetChat(sid, chatID) + chat, err := neo.Store.GetChat(sid, chatID) if err != nil { c.JSON(500, gin.H{"message": err.Error(), "code": 500}) c.Done() @@ -475,14 +475,14 @@ func (neo *DSL) handleMentions(c *gin.Context) { mentionable := true // Query mentionable assistants - filter := conversation.AssistantFilter{ + filter := store.AssistantFilter{ Keywords: keywords, Mentionable: &mentionable, Page: 1, PageSize: 20, } - response, err := neo.Conversation.GetAssistants(filter) + response, err := neo.Store.GetAssistants(filter) if err != nil { c.JSON(500, gin.H{"message": err.Error(), "code": 500}) c.Done() @@ -552,7 +552,7 @@ func (neo *DSL) handleChatUpdate(c *gin.Context) { return } - err := neo.Conversation.UpdateChatTitle(sid, chatID, body.Title) + err := neo.Store.UpdateChatTitle(sid, chatID, body.Title) if err != nil { c.JSON(500, gin.H{"message": err.Error(), "code": 500}) c.Done() @@ -579,7 +579,7 @@ func (neo *DSL) handleChatDelete(c *gin.Context) { return } - err := neo.Conversation.DeleteChat(sid, chatID) + err := neo.Store.DeleteChat(sid, chatID) if err != nil { c.JSON(500, gin.H{"message": err.Error(), "code": 500}) c.Done() @@ -599,7 +599,7 @@ func (neo *DSL) handleChatsDeleteAll(c *gin.Context) { return } - err := neo.Conversation.DeleteAllChats(sid) + err := neo.Store.DeleteAllChats(sid) if err != nil { c.JSON(500, gin.H{"message": err.Error(), "code": 500}) c.Done() @@ -840,7 +840,7 @@ func (neo *DSL) handleGenerateCustom(c *gin.Context) { // handleAssistantList handles listing assistants func (neo *DSL) handleAssistantList(c *gin.Context) { // Parse filter parameters - filter := conversation.AssistantFilter{ + filter := store.AssistantFilter{ Page: 1, PageSize: 20, } @@ -894,7 +894,7 @@ func (neo *DSL) handleAssistantList(c *gin.Context) { } } - response, err := neo.Conversation.GetAssistants(filter) + response, err := neo.Store.GetAssistants(filter) if err != nil { c.JSON(500, gin.H{"message": err.Error(), "code": 500}) c.Done() @@ -930,13 +930,13 @@ func (neo *DSL) handleAssistantDetail(c *gin.Context) { return } - filter := conversation.AssistantFilter{ + filter := store.AssistantFilter{ AssistantID: assistantID, Page: 1, PageSize: 1, } - response, err := neo.Conversation.GetAssistants(filter) + response, err := neo.Store.GetAssistants(filter) if err != nil { c.JSON(500, gin.H{"message": err.Error(), "code": 500}) c.Done() @@ -962,7 +962,7 @@ func (neo *DSL) handleAssistantSave(c *gin.Context) { return } - id, err := neo.Conversation.SaveAssistant(assistant) + id, err := neo.Store.SaveAssistant(assistant) if err != nil { c.JSON(500, gin.H{"message": err.Error(), "code": 500}) c.Done() @@ -987,7 +987,7 @@ func (neo *DSL) handleAssistantDelete(c *gin.Context) { return } - err := neo.Conversation.DeleteAssistant(assistantID) + err := neo.Store.DeleteAssistant(assistantID) if err != nil { c.JSON(500, gin.H{"message": err.Error(), "code": 500}) c.Done() diff --git a/neo/assistant/assistant.go b/neo/assistant/assistant.go new file mode 100644 index 00000000..0de36580 --- /dev/null +++ b/neo/assistant/assistant.go @@ -0,0 +1,26 @@ +package assistant + +import "github.com/yaoapp/yao/neo/store" + +// loadedAssistant the loaded assistant +var loadedAssistant = map[string]*Assistant{} + +// LoadLocal create a new assistant from local +func LoadLocal(path string) *Assistant { + return nil +} + +// LoadZip create a new assistant from zip +func LoadZip(zip string) *Assistant { + return nil +} + +// LoadRemote create a new assistant from remote +func LoadRemote(url string) *Assistant { + return nil +} + +// LoadStore create a new assistant from store +func LoadStore(store store.Store) *Assistant { + return nil +} diff --git a/neo/load.go b/neo/load.go index dec5fb2f..0a918d81 100644 --- a/neo/load.go +++ b/neo/load.go @@ -9,7 +9,7 @@ import ( "github.com/yaoapp/gou/application" "github.com/yaoapp/yao/config" "github.com/yaoapp/yao/neo/assistant" - "github.com/yaoapp/yao/neo/conversation" + "github.com/yaoapp/yao/neo/store" ) // Neo the neo AI assistant @@ -23,7 +23,7 @@ func Load(cfg config.Config) error { Prompts: []assistant.Prompt{}, Option: map[string]interface{}{}, Allows: []string{}, - ConversationSetting: conversation.Setting{ + StoreSetting: store.Setting{ Table: "yao_neo_conversation", Connector: "default", }, @@ -39,14 +39,14 @@ func Load(cfg config.Config) error { return err } - if setting.ConversationSetting.MaxSize == 0 { - setting.ConversationSetting.MaxSize = 100 + if setting.StoreSetting.MaxSize == 0 { + setting.StoreSetting.MaxSize = 100 } Neo = &setting - // Conversation Setting - err = Neo.createConversation() + // Store Setting + err = Neo.createStore() if err != nil { return err } diff --git a/neo/neo.go b/neo/neo.go index 0a034a67..97d16767 100644 --- a/neo/neo.go +++ b/neo/neo.go @@ -13,8 +13,8 @@ import ( "github.com/yaoapp/yao/neo/assistant" "github.com/yaoapp/yao/neo/assistant/local" "github.com/yaoapp/yao/neo/assistant/openai" - "github.com/yaoapp/yao/neo/conversation" "github.com/yaoapp/yao/neo/message" + "github.com/yaoapp/yao/neo/store" "github.com/yaoapp/yao/share" ) @@ -473,7 +473,7 @@ func (neo *DSL) createDefaultAssistant() (assistant.API, error) { // chatMessages get the chat messages func (neo *DSL) chatMessages(ctx Context, content ...string) ([]map[string]interface{}, error) { - history, err := neo.Conversation.GetHistory(ctx.Sid, ctx.ChatID) + history, err := neo.Store.GetHistory(ctx.Sid, ctx.ChatID) if err != nil { return nil, err } @@ -493,7 +493,7 @@ func (neo *DSL) chatMessages(ctx Context, content ...string) ([]map[string]inter func (neo *DSL) saveHistory(sid string, chatID string, content []byte, messages []map[string]interface{}) { if len(content) > 0 && sid != "" && len(messages) > 0 { - err := neo.Conversation.SaveHistory( + err := neo.Store.SaveHistory( sid, []map[string]interface{}{ {"role": "user", "content": messages[len(messages)-1]["content"], "name": sid}, @@ -509,39 +509,39 @@ func (neo *DSL) saveHistory(sid string, chatID string, content []byte, messages } } -// createConversation create a new conversation -func (neo *DSL) createConversation() error { +// createStore create a new store +func (neo *DSL) createStore() error { var err error - if neo.ConversationSetting.Connector == "default" || neo.ConversationSetting.Connector == "" { - neo.Conversation, err = conversation.NewXun(neo.ConversationSetting) + if neo.StoreSetting.Connector == "default" || neo.StoreSetting.Connector == "" { + neo.Store, err = store.NewXun(neo.StoreSetting) return err } // other connector - conn, err := connector.Select(neo.ConversationSetting.Connector) + conn, err := connector.Select(neo.StoreSetting.Connector) if err != nil { return err } if conn.Is(connector.DATABASE) { - neo.Conversation, err = conversation.NewXun(neo.ConversationSetting) + neo.Store, err = store.NewXun(neo.StoreSetting) return err } else if conn.Is(connector.REDIS) { - neo.Conversation = conversation.NewRedis() + neo.Store = store.NewRedis() return nil } else if conn.Is(connector.MONGO) { - neo.Conversation = conversation.NewMongo() + neo.Store = store.NewMongo() return nil } else if conn.Is(connector.WEAVIATE) { - neo.Conversation = conversation.NewWeaviate() + neo.Store = store.NewWeaviate() return nil } - return fmt.Errorf("%s conversation connector %s not support", neo.ID, neo.ConversationSetting.Connector) + return fmt.Errorf("%s store connector %s not support", neo.ID, neo.StoreSetting.Connector) } // sendMessage sends a message to the client diff --git a/neo/process.go b/neo/process.go index 7a27cbac..6ccd0a5d 100644 --- a/neo/process.go +++ b/neo/process.go @@ -7,8 +7,8 @@ import ( "github.com/gin-gonic/gin" "github.com/yaoapp/gou/process" "github.com/yaoapp/kun/exception" - "github.com/yaoapp/yao/neo/conversation" "github.com/yaoapp/yao/neo/message" + "github.com/yaoapp/yao/neo/store" ) // GetNeo returns the Neo instance @@ -32,7 +32,6 @@ func init() { // ProcessWrite process the write request func ProcessWrite(process *process.Process) interface{} { - process.ValidateArgNums(2) w, ok := process.Args[0].(gin.ResponseWriter) @@ -63,11 +62,11 @@ func processAssistantCreate(process *process.Process) interface{} { data := process.ArgsMap(0) neo := GetNeo() - if neo.Conversation == nil { - exception.New("Neo conversation is not initialized", 500).Throw() + if neo.Store == nil { + exception.New("Neo store is not initialized", 500).Throw() } - id, err := neo.Conversation.SaveAssistant(data) + id, err := neo.Store.SaveAssistant(data) if err != nil { exception.New("Failed to create assistant: %s", 500, err.Error()).Throw() } @@ -81,11 +80,11 @@ func processAssistantSave(process *process.Process) interface{} { data := process.ArgsMap(0) neo := GetNeo() - if neo.Conversation == nil { - exception.New("Neo conversation is not initialized", 500).Throw() + if neo.Store == nil { + exception.New("Neo store is not initialized", 500).Throw() } - id, err := neo.Conversation.SaveAssistant(data) + id, err := neo.Store.SaveAssistant(data) if err != nil { exception.New("Failed to save assistant: %s", 500, err.Error()).Throw() } @@ -99,11 +98,11 @@ func processAssistantDelete(process *process.Process) interface{} { assistantID := process.ArgsString(0) neo := GetNeo() - if neo.Conversation == nil { - exception.New("Neo conversation is not initialized", 500).Throw() + if neo.Store == nil { + exception.New("Neo store is not initialized", 500).Throw() } - err := neo.Conversation.DeleteAssistant(assistantID) + err := neo.Store.DeleteAssistant(assistantID) if err != nil { exception.New("Failed to delete assistant: %s", 500, err.Error()).Throw() } @@ -114,7 +113,7 @@ func processAssistantDelete(process *process.Process) interface{} { // processAssistantSearch process the assistant search request func processAssistantSearch(process *process.Process) interface{} { params := process.ArgsMap(0) - filter := conversation.AssistantFilter{} + filter := store.AssistantFilter{} // Parse page and pagesize if page, ok := params["page"]; ok { @@ -165,11 +164,11 @@ func processAssistantSearch(process *process.Process) interface{} { // Get assistants neo := GetNeo() - if neo.Conversation == nil { - exception.New("Neo conversation is not initialized", 500).Throw() + if neo.Store == nil { + exception.New("Neo store is not initialized", 500).Throw() } - res, err := neo.Conversation.GetAssistants(filter) + res, err := neo.Store.GetAssistants(filter) if err != nil { exception.New("get assistants error: %s", 500, err).Throw() } @@ -183,17 +182,17 @@ func processAssistantFind(process *process.Process) interface{} { assistantID := process.ArgsString(0) neo := GetNeo() - if neo.Conversation == nil { - exception.New("Neo conversation is not initialized", 500).Throw() + if neo.Store == nil { + exception.New("Neo store is not initialized", 500).Throw() } - filter := conversation.AssistantFilter{ + filter := store.AssistantFilter{ AssistantID: assistantID, Page: 1, PageSize: 1, } - res, err := neo.Conversation.GetAssistants(filter) + res, err := neo.Store.GetAssistants(filter) if err != nil { exception.New("Failed to find assistant: %s", 500, err.Error()).Throw() } diff --git a/neo/conversation/mongo.go b/neo/store/mongo.go similarity index 93% rename from neo/conversation/mongo.go rename to neo/store/mongo.go index d4b20a16..88336b1f 100644 --- a/neo/conversation/mongo.go +++ b/neo/store/mongo.go @@ -1,10 +1,10 @@ -package conversation +package store // Mongo represents a MongoDB-based conversation storage type Mongo struct{} -// NewMongo creates a new MongoDB conversation storage -func NewMongo() *Mongo { +// NewMongo create a new mongo store +func NewMongo() Store { return &Mongo{} } diff --git a/neo/conversation/redis.go b/neo/store/redis.go similarity index 93% rename from neo/conversation/redis.go rename to neo/store/redis.go index fb150570..ad249543 100644 --- a/neo/conversation/redis.go +++ b/neo/store/redis.go @@ -1,10 +1,10 @@ -package conversation +package store // Redis represents a Redis-based conversation storage type Redis struct{} -// NewRedis creates a new Redis conversation storage -func NewRedis() *Redis { +// NewRedis create a new redis store +func NewRedis() Store { return &Redis{} } diff --git a/neo/conversation/types.go b/neo/store/types.go similarity index 98% rename from neo/conversation/types.go rename to neo/store/types.go index 521fc176..ccfa2039 100644 --- a/neo/conversation/types.go +++ b/neo/store/types.go @@ -1,4 +1,4 @@ -package conversation +package store // Setting represents the conversation configuration structure // Used to configure basic conversation parameters including connector, user field, table name, etc. @@ -69,9 +69,9 @@ type AssistantResponse struct { Total int64 `json:"total"` // Total number of items } -// Conversation defines the conversation storage interface +// Store defines the conversation storage interface // Provides basic operations required for conversation management -type Conversation interface { +type Store interface { // GetChats retrieves a list of chats // sid: Session ID // filter: Filter conditions diff --git a/neo/conversation/weaviate.go b/neo/store/weaviate.go similarity index 93% rename from neo/conversation/weaviate.go rename to neo/store/weaviate.go index 8462e843..4b2dda07 100644 --- a/neo/conversation/weaviate.go +++ b/neo/store/weaviate.go @@ -1,10 +1,10 @@ -package conversation +package store // Weaviate represents a Weaviate-based conversation storage type Weaviate struct{} -// NewWeaviate creates a new Weaviate conversation storage -func NewWeaviate() *Weaviate { +// NewWeaviate create a new weaviate store +func NewWeaviate() Store { return &Weaviate{} } diff --git a/neo/conversation/xun.go b/neo/store/xun.go similarity index 99% rename from neo/conversation/xun.go rename to neo/store/xun.go index d6ee4efa..2407bdc2 100644 --- a/neo/conversation/xun.go +++ b/neo/store/xun.go @@ -1,4 +1,4 @@ -package conversation +package store import ( "fmt" @@ -45,8 +45,8 @@ type Xun struct { // DeleteAssistant deletes an assistant by assistant_id // GetAssistants retrieves a paginated list of assistants with filtering -// NewXun create a new conversation -func NewXun(setting Setting) (*Xun, error) { +// NewXun create a new xun store +func NewXun(setting Setting) (Store, error) { conv := &Xun{setting: setting} if setting.Connector == "default" { conv.query = capsule.Global.Query() diff --git a/neo/conversation/xun_test.go b/neo/store/xun_test.go similarity index 79% rename from neo/conversation/xun_test.go rename to neo/store/xun_test.go index 5b74c8a5..a8613c3a 100644 --- a/neo/conversation/xun_test.go +++ b/neo/store/xun_test.go @@ -1,4 +1,4 @@ -package conversation +package store import ( "fmt" @@ -38,7 +38,7 @@ func TestNewXunDefault(t *testing.T) { // Add a small delay to ensure table is created time.Sleep(100 * time.Millisecond) - conv, err := NewXun(Setting{ + store, err := NewXun(Setting{ Connector: "default", Table: "__unit_test_conversation", }) @@ -69,38 +69,47 @@ func TestNewXunDefault(t *testing.T) { } assert.Equal(t, true, has) - // validate the history table - tab, err := conv.schema.GetTable(conv.getHistoryTable()) - if err != nil { - t.Fatal(err) + // Validate table structure by attempting operations + // Test history operations + messages := []map[string]interface{}{ + {"role": "user", "content": "test message"}, + } + err = store.SaveHistory("test_user", messages, "test_chat", nil) + assert.Nil(t, err) + + history, err := store.GetHistory("test_user", "test_chat") + assert.Nil(t, err) + assert.NotEmpty(t, history) + + // Test chat operations + err = store.UpdateChatTitle("test_user", "test_chat", "Test Chat") + assert.Nil(t, err) + + chat, err := store.GetChat("test_user", "test_chat") + assert.Nil(t, err) + assert.NotNil(t, chat) + + // Test assistant operations + assistant := map[string]interface{}{ + "name": "Test Assistant", + "type": "assistant", + "connector": "test", + "description": "Test Description", + "tags": []string{"test"}, + "mentionable": true, + "automated": true, } - fields := []string{"id", "sid", "cid", "uid", "role", "name", "content", "context", "created_at", "updated_at", "expired_at"} - for _, field := range fields { - assert.Equal(t, true, tab.HasColumn(field)) - } + id, err := store.SaveAssistant(assistant) + assert.Nil(t, err) + assert.NotNil(t, id) - // validate the chat table - tab, err = conv.schema.GetTable(conv.getChatTable()) - if err != nil { - t.Fatal(err) - } + // Clean up test data + err = store.DeleteChat("test_user", "test_chat") + assert.Nil(t, err) - chatFields := []string{"id", "chat_id", "title", "sid", "created_at", "updated_at"} - for _, field := range chatFields { - assert.Equal(t, true, tab.HasColumn(field)) - } - - // validate the assistant table - tab, err = conv.schema.GetTable(conv.getAssistantTable()) - if err != nil { - t.Fatal(err) - } - - assistantFields := []string{"id", "assistant_id", "type", "name", "avatar", "connector", "description", "options", "prompts", "flows", "files", "functions", "tags", "readonly", "permissions", "automated", "mentionable", "created_at", "updated_at"} - for _, field := range assistantFields { - assert.Equal(t, true, tab.HasColumn(field)) - } + err = store.DeleteAssistant(id.(string)) + assert.Nil(t, err) } func TestNewXunConnector(t *testing.T) { @@ -128,7 +137,7 @@ func TestNewXunConnector(t *testing.T) { // Add a small delay to ensure table is created time.Sleep(100 * time.Millisecond) - conv, err := NewXun(Setting{ + store, err := NewXun(Setting{ Connector: "mysql", Table: "__unit_test_conversation", }) @@ -159,38 +168,19 @@ func TestNewXunConnector(t *testing.T) { } assert.Equal(t, true, has) - // validate the history table - tab, err := conv.schema.GetTable(conv.getHistoryTable()) - if err != nil { - t.Fatal(err) + // Test basic operations + messages := []map[string]interface{}{ + {"role": "user", "content": "test message"}, } + err = store.SaveHistory("test_user", messages, "test_chat", nil) + assert.Nil(t, err) - fields := []string{"id", "sid", "cid", "uid", "role", "name", "content", "context", "created_at", "updated_at", "expired_at"} - for _, field := range fields { - assert.Equal(t, true, tab.HasColumn(field)) - } + history, err := store.GetHistory("test_user", "test_chat") + assert.Nil(t, err) + assert.NotEmpty(t, history) - // validate the chat table - tab, err = conv.schema.GetTable(conv.getChatTable()) - if err != nil { - t.Fatal(err) - } - - chatFields := []string{"id", "chat_id", "title", "sid", "created_at", "updated_at"} - for _, field := range chatFields { - assert.Equal(t, true, tab.HasColumn(field)) - } - - // validate the assistant table - tab, err = conv.schema.GetTable(conv.getAssistantTable()) - if err != nil { - t.Fatal(err) - } - - assistantFields := []string{"id", "assistant_id", "type", "name", "avatar", "connector", "description", "options", "prompts", "flows", "files", "functions", "tags", "readonly", "permissions", "automated", "mentionable", "created_at", "updated_at"} - for _, field := range assistantFields { - assert.Equal(t, true, tab.HasColumn(field)) - } + err = store.DeleteChat("test_user", "test_chat") + assert.Nil(t, err) } func TestXunSaveAndGetHistory(t *testing.T) { @@ -209,7 +199,7 @@ func TestXunSaveAndGetHistory(t *testing.T) { t.Fatal(err) } - conv, err := NewXun(Setting{ + store, err := NewXun(Setting{ Connector: "default", Table: "__unit_test_conversation", TTL: 3600, @@ -217,14 +207,14 @@ func TestXunSaveAndGetHistory(t *testing.T) { // save the history cid := "123456" - err = conv.SaveHistory("123456", []map[string]interface{}{ + err = store.SaveHistory("123456", []map[string]interface{}{ {"role": "user", "name": "user1", "content": "hello"}, {"role": "assistant", "name": "user1", "content": "Hello there, how"}, }, cid, nil) assert.Nil(t, err) // get the history - data, err := conv.GetHistory("123456", cid) + data, err := store.GetHistory("123456", cid) if err != nil { t.Fatal(err) } @@ -247,7 +237,7 @@ func TestXunSaveAndGetHistoryWithCID(t *testing.T) { t.Fatal(err) } - conv, err := NewXun(Setting{ + store, err := NewXun(Setting{ Connector: "default", Table: "__unit_test_conversation", TTL: 3600, @@ -260,11 +250,11 @@ func TestXunSaveAndGetHistoryWithCID(t *testing.T) { {"role": "user", "name": "user1", "content": "hello"}, {"role": "assistant", "name": "assistant1", "content": "Hi! How can I help you?"}, } - err = conv.SaveHistory(sid, messages, cid, nil) + err = store.SaveHistory(sid, messages, cid, nil) assert.Nil(t, err) // get the history for specific cid - data, err := conv.GetHistory(sid, cid) + data, err := store.GetHistory(sid, cid) if err != nil { t.Fatal(err) } @@ -275,25 +265,25 @@ func TestXunSaveAndGetHistoryWithCID(t *testing.T) { moreMessages := []map[string]interface{}{ {"role": "user", "name": "user1", "content": "another message"}, } - err = conv.SaveHistory(sid, moreMessages, anotherCID, nil) + err = store.SaveHistory(sid, moreMessages, anotherCID, nil) assert.Nil(t, err) // get history for the first cid - should still be 2 messages - data, err = conv.GetHistory(sid, cid) + data, err = store.GetHistory(sid, cid) if err != nil { t.Fatal(err) } assert.Equal(t, 2, len(data)) // get history for the second cid - should be 1 message - data, err = conv.GetHistory(sid, anotherCID) + data, err = store.GetHistory(sid, anotherCID) if err != nil { t.Fatal(err) } assert.Equal(t, 1, len(data)) // get all history for the sid without specifying cid - allData, err := conv.GetHistory(sid, cid) + allData, err := store.GetHistory(sid, cid) if err != nil { t.Fatal(err) } @@ -316,7 +306,7 @@ func TestXunGetChats(t *testing.T) { t.Fatal(err) } - conv, err := NewXun(Setting{ + store, err := NewXun(Setting{ Connector: "default", Table: "__unit_test_conversation", }) @@ -333,22 +323,15 @@ func TestXunGetChats(t *testing.T) { // Create chats with different dates for i := 0; i < 5; i++ { chatID := fmt.Sprintf("chat_%d", i) - // First create the chat with a title - err = conv.newQueryChat().Insert(map[string]interface{}{ - "chat_id": chatID, - "title": fmt.Sprintf("Test Chat %d", i), - "sid": sid, - "created_at": time.Now(), - }) - if err != nil { - t.Fatal(err) - } + title := fmt.Sprintf("Test Chat %d", i) - // Then save the history - err = conv.SaveHistory(sid, messages, chatID, nil) - if err != nil { - t.Fatal(err) - } + // Save history first to create the chat + err = store.SaveHistory(sid, messages, chatID, nil) + assert.Nil(t, err) + + // Update the chat title + err = store.UpdateChatTitle(sid, chatID, title) + assert.Nil(t, err) } // Test getting chats with default filter @@ -356,7 +339,7 @@ func TestXunGetChats(t *testing.T) { PageSize: 10, Order: "desc", } - groups, err := conv.GetChats(sid, filter) + groups, err := store.GetChats(sid, filter) if err != nil { t.Fatal(err) } @@ -365,7 +348,7 @@ func TestXunGetChats(t *testing.T) { // Test with keywords filter.Keywords = "test" - groups, err = conv.GetChats(sid, filter) + groups, err = store.GetChats(sid, filter) if err != nil { t.Fatal(err) } @@ -379,7 +362,7 @@ func TestXunDeleteChat(t *testing.T) { defer capsule.Schema().DropTableIfExists("__unit_test_conversation_history") defer capsule.Schema().DropTableIfExists("__unit_test_conversation_chat") - conv, err := NewXun(Setting{ + store, err := NewXun(Setting{ Connector: "default", Table: "__unit_test_conversation", }) @@ -395,20 +378,20 @@ func TestXunDeleteChat(t *testing.T) { } // Save the chat and history - err = conv.SaveHistory(sid, messages, cid, nil) + err = store.SaveHistory(sid, messages, cid, nil) assert.Nil(t, err) // Verify chat exists - chat, err := conv.GetChat(sid, cid) + chat, err := store.GetChat(sid, cid) assert.Nil(t, err) assert.NotNil(t, chat) // Delete the chat - err = conv.DeleteChat(sid, cid) + err = store.DeleteChat(sid, cid) assert.Nil(t, err) // Verify chat is deleted - chat, err = conv.GetChat(sid, cid) + chat, err = store.GetChat(sid, cid) assert.Nil(t, err) assert.Equal(t, (*ChatInfo)(nil), chat) } @@ -419,7 +402,7 @@ func TestXunDeleteAllChats(t *testing.T) { defer capsule.Schema().DropTableIfExists("__unit_test_conversation_history") defer capsule.Schema().DropTableIfExists("__unit_test_conversation_chat") - conv, err := NewXun(Setting{ + store, err := NewXun(Setting{ Connector: "default", Table: "__unit_test_conversation", }) @@ -436,21 +419,21 @@ func TestXunDeleteAllChats(t *testing.T) { // Save multiple chats for i := 0; i < 3; i++ { cid := fmt.Sprintf("test_chat_%d", i) - err = conv.SaveHistory(sid, messages, cid, nil) + err = store.SaveHistory(sid, messages, cid, nil) assert.Nil(t, err) } // Verify chats exist - response, err := conv.GetChats(sid, ChatFilter{}) + response, err := store.GetChats(sid, ChatFilter{}) assert.Nil(t, err) assert.Greater(t, response.Total, int64(0)) // Delete all chats - err = conv.DeleteAllChats(sid) + err = store.DeleteAllChats(sid) assert.Nil(t, err) // Verify all chats are deleted - response, err = conv.GetChats(sid, ChatFilter{}) + response, err = store.GetChats(sid, ChatFilter{}) assert.Nil(t, err) assert.Equal(t, int64(0), response.Total) } @@ -470,7 +453,7 @@ func TestXunAssistantCRUD(t *testing.T) { // Add a small delay to ensure table is created time.Sleep(100 * time.Millisecond) - conv, err := NewXun(Setting{ + store, err := NewXun(Setting{ Connector: "default", Table: "__unit_test_conversation", }) @@ -495,7 +478,7 @@ func TestXunAssistantCRUD(t *testing.T) { } // Test SaveAssistant (Create) with string JSON - v, err := conv.SaveAssistant(assistant) + v, err := store.SaveAssistant(assistant) assert.Nil(t, err) assistantID := v.(string) assert.NotEmpty(t, assistantID) @@ -519,7 +502,7 @@ func TestXunAssistantCRUD(t *testing.T) { } // Test SaveAssistant (Create) with native types - v, err = conv.SaveAssistant(assistant2) + v, err = store.SaveAssistant(assistant2) assert.Nil(t, err) assistant2ID := v.(string) assert.NotEmpty(t, assistant2ID) @@ -542,13 +525,13 @@ func TestXunAssistantCRUD(t *testing.T) { } // Test SaveAssistant (Create) with nil fields - v, err = conv.SaveAssistant(assistant3) + v, err = store.SaveAssistant(assistant3) assert.Nil(t, err) assistant3ID := v.(string) assert.NotEmpty(t, assistant3ID) // Test GetAssistants to verify JSON fields are properly stored - resp, err := conv.GetAssistants(AssistantFilter{}) + resp, err := store.GetAssistants(AssistantFilter{}) assert.Nil(t, err) assert.Equal(t, 3, len(resp.Data)) @@ -614,11 +597,11 @@ func TestXunAssistantCRUD(t *testing.T) { // Test updating with mixed JSON formats assistant2["assistant_id"] = assistant2ID - _, err = conv.SaveAssistant(assistant2) + _, err = store.SaveAssistant(assistant2) assert.Nil(t, err) // Verify update - resp, err = conv.GetAssistants(AssistantFilter{}) + resp, err = store.GetAssistants(AssistantFilter{}) assert.Nil(t, err) for _, item := range resp.Data { if item["assistant_id"].(string) == assistant2ID { @@ -630,14 +613,14 @@ func TestXunAssistantCRUD(t *testing.T) { } // Test DeleteAssistant - err = conv.DeleteAssistant(assistantID) + err = store.DeleteAssistant(assistantID) assert.Nil(t, err) - err = conv.DeleteAssistant(assistant2ID) + err = store.DeleteAssistant(assistant2ID) assert.Nil(t, err) - err = conv.DeleteAssistant(assistant3ID) + err = store.DeleteAssistant(assistant3ID) assert.Nil(t, err) - resp, err = conv.GetAssistants(AssistantFilter{}) + resp, err = store.GetAssistants(AssistantFilter{}) assert.Nil(t, err) assert.Equal(t, 0, len(resp.Data)) } @@ -658,7 +641,7 @@ func TestXunAssistantPagination(t *testing.T) { // Add a small delay to ensure table is created time.Sleep(100 * time.Millisecond) - conv, err := NewXun(Setting{ + store, err := NewXun(Setting{ Connector: "default", Table: "__unit_test_conversation", }) @@ -692,12 +675,12 @@ func TestXunAssistantPagination(t *testing.T) { "mentionable": mentionable, "automated": automated, } - _, err = conv.SaveAssistant(assistant) + _, err = store.SaveAssistant(assistant) assert.Nil(t, err) } // Test first page - resp, err := conv.GetAssistants(AssistantFilter{ + resp, err := store.GetAssistants(AssistantFilter{ Page: 1, PageSize: 10, }) @@ -709,7 +692,7 @@ func TestXunAssistantPagination(t *testing.T) { assert.Equal(t, 0, resp.Prev) // Test second page - resp, err = conv.GetAssistants(AssistantFilter{ + resp, err = store.GetAssistants(AssistantFilter{ Page: 2, PageSize: 10, }) @@ -719,7 +702,7 @@ func TestXunAssistantPagination(t *testing.T) { assert.Equal(t, 1, resp.Prev) // Test last page - resp, err = conv.GetAssistants(AssistantFilter{ + resp, err = store.GetAssistants(AssistantFilter{ Page: 3, PageSize: 10, }) @@ -729,7 +712,7 @@ func TestXunAssistantPagination(t *testing.T) { assert.Equal(t, 2, resp.Prev) // Test filtering with tags - resp, err = conv.GetAssistants(AssistantFilter{ + resp, err = store.GetAssistants(AssistantFilter{ Tags: []string{"tag0"}, Page: 1, PageSize: 10, @@ -738,7 +721,7 @@ func TestXunAssistantPagination(t *testing.T) { assert.Equal(t, 5, len(resp.Data)) // Test filtering with keywords - resp, err = conv.GetAssistants(AssistantFilter{ + resp, err = store.GetAssistants(AssistantFilter{ Keywords: "Assistant 1", Page: 1, PageSize: 10, @@ -747,7 +730,7 @@ func TestXunAssistantPagination(t *testing.T) { assert.Greater(t, len(resp.Data), 0) // Test filtering with connector - resp, err = conv.GetAssistants(AssistantFilter{ + resp, err = store.GetAssistants(AssistantFilter{ Connector: "connector0", Page: 1, PageSize: 10, @@ -757,7 +740,7 @@ func TestXunAssistantPagination(t *testing.T) { // Test filtering with mentionable mentionableTrue := true - resp, err = conv.GetAssistants(AssistantFilter{ + resp, err = store.GetAssistants(AssistantFilter{ Mentionable: &mentionableTrue, Page: 1, PageSize: 10, @@ -767,7 +750,7 @@ func TestXunAssistantPagination(t *testing.T) { // Test filtering with automated automatedTrue := true - resp, err = conv.GetAssistants(AssistantFilter{ + resp, err = store.GetAssistants(AssistantFilter{ Automated: &automatedTrue, Page: 1, PageSize: 10, @@ -780,7 +763,7 @@ func TestXunAssistantPagination(t *testing.T) { firstAssistantID := resp.Data[0]["assistant_id"].(string) // Test exact match with assistant_id - resp, err = conv.GetAssistants(AssistantFilter{ + resp, err = store.GetAssistants(AssistantFilter{ AssistantID: firstAssistantID, Page: 1, PageSize: 10, @@ -790,7 +773,7 @@ func TestXunAssistantPagination(t *testing.T) { assert.Equal(t, firstAssistantID, resp.Data[0]["assistant_id"]) // Test assistant_id with other filters - resp, err = conv.GetAssistants(AssistantFilter{ + resp, err = store.GetAssistants(AssistantFilter{ AssistantID: firstAssistantID, Select: []string{"name", "assistant_id", "description"}, Page: 1, @@ -807,7 +790,7 @@ func TestXunAssistantPagination(t *testing.T) { assert.NotContains(t, resp.Data[0], "options") // Test non-existent assistant_id - resp, err = conv.GetAssistants(AssistantFilter{ + resp, err = store.GetAssistants(AssistantFilter{ AssistantID: "non-existent-id", Page: 1, PageSize: 10, @@ -816,7 +799,7 @@ func TestXunAssistantPagination(t *testing.T) { assert.Equal(t, 0, len(resp.Data)) // Test combined filters - resp, err = conv.GetAssistants(AssistantFilter{ + resp, err = store.GetAssistants(AssistantFilter{ Tags: []string{"tag0"}, Keywords: "Assistant", Connector: "connector0", @@ -828,7 +811,7 @@ func TestXunAssistantPagination(t *testing.T) { assert.Nil(t, err) // Test filtering with select fields - resp, err = conv.GetAssistants(AssistantFilter{ + resp, err = store.GetAssistants(AssistantFilter{ Select: []string{"name", "description", "tags"}, Page: 1, PageSize: 10, @@ -851,7 +834,7 @@ func TestXunAssistantPagination(t *testing.T) { } // Test filtering with select fields and other filters combined - resp, err = conv.GetAssistants(AssistantFilter{ + resp, err = store.GetAssistants(AssistantFilter{ Tags: []string{"tag0"}, Keywords: "Assistant", Select: []string{"name", "tags"}, diff --git a/neo/types.go b/neo/types.go index 0582cba3..bba705d0 100644 --- a/neo/types.go +++ b/neo/types.go @@ -5,30 +5,30 @@ import ( "github.com/gin-gonic/gin" "github.com/yaoapp/yao/neo/assistant" - "github.com/yaoapp/yao/neo/conversation" + "github.com/yaoapp/yao/neo/store" ) // DSL AI assistant type DSL struct { - ID string `json:"-" yaml:"-"` - Name string `json:"name,omitempty" yaml:"name,omitempty"` - Use string `json:"use,omitempty" yaml:"use,omitempty"` // Which assistant to use default - Guard string `json:"guard,omitempty" yaml:"guard,omitempty"` - Connector string `json:"connector" yaml:"connector"` - ConversationSetting conversation.Setting `json:"conversation" yaml:"conversation"` - Option map[string]interface{} `json:"option" yaml:"option"` - Prepare string `json:"prepare,omitempty" yaml:"prepare,omitempty"` - Create string `json:"create,omitempty" yaml:"create,omitempty"` - Write string `json:"write,omitempty" yaml:"write,omitempty"` - AssistantListHook string `json:"assistants,omitempty" yaml:"assistants,omitempty"` // Get the assistant list from the hook - MentionHook string `json:"mentions,omitempty"` // Get the mention list from the hook - Prompts []assistant.Prompt `json:"prompts,omitempty" yaml:"prompts,omitempty"` - Allows []string `json:"allows,omitempty" yaml:"allows,omitempty"` - Assistant assistant.API `json:"-" yaml:"-"` // The default assistant - Conversation conversation.Conversation `json:"-" yaml:"-"` - GuardHandlers []gin.HandlerFunc `json:"-" yaml:"-"` - AssistantList []assistant.Assistant `json:"-" yaml:"-"` - AssistantMaps map[string]assistant.Assistant `json:"-" yaml:"-"` + ID string `json:"-" yaml:"-"` + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Use string `json:"use,omitempty" yaml:"use,omitempty"` // Which assistant to use default + Guard string `json:"guard,omitempty" yaml:"guard,omitempty"` + Connector string `json:"connector" yaml:"connector"` + StoreSetting store.Setting `json:"store" yaml:"store"` + Option map[string]interface{} `json:"option" yaml:"option"` + Prepare string `json:"prepare,omitempty" yaml:"prepare,omitempty"` + Create string `json:"create,omitempty" yaml:"create,omitempty"` + Write string `json:"write,omitempty" yaml:"write,omitempty"` + AssistantListHook string `json:"assistants,omitempty" yaml:"assistants,omitempty"` // Get the assistant list from the hook + MentionHook string `json:"mentions,omitempty"` // Get the mention list from the hook + Prompts []assistant.Prompt `json:"prompts,omitempty" yaml:"prompts,omitempty"` + Allows []string `json:"allows,omitempty" yaml:"allows,omitempty"` + Assistant assistant.API `json:"-" yaml:"-"` // The default assistant + Store store.Store `json:"-" yaml:"-"` + GuardHandlers []gin.HandlerFunc `json:"-" yaml:"-"` + AssistantList []assistant.Assistant `json:"-" yaml:"-"` + AssistantMaps map[string]assistant.Assistant `json:"-" yaml:"-"` } // Mention list From d0d110b0eccd93bee6c5cdbf0fbd9151eba83baf Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 1 Jan 2025 12:31:47 +0800 Subject: [PATCH 3/8] Refactor assistant management and enhance storage retrieval in Neo API - Removed Weaviate store implementation, streamlining the codebase and focusing on Mongo and Redis backends. - Introduced GetAssistant method in both Mongo and Redis stores to retrieve a single assistant by ID, improving data access capabilities. - Updated LoadStore function to utilize the new storage retrieval logic, enhancing the assistant loading process. - Enhanced the Assistant struct to include a Script field for better management of assistant scripts. - Improved tests to cover the new GetAssistant functionality, ensuring robust error handling and data retrieval across different scenarios. --- neo/assistant/assistant.go | 220 ++++++++++++++++++++++++++++++-- neo/assistant/assistant_test.go | 196 ++++++++++++++++++++++++++++ neo/assistant/cache.go | 105 +++++++++++++++ neo/assistant/cache_test.go | 151 ++++++++++++++++++++++ neo/assistant/types.go | 3 + neo/neo.go | 4 - neo/store/mongo.go | 5 + neo/store/redis.go | 5 + neo/store/types.go | 5 + neo/store/weaviate.go | 59 --------- neo/store/xun.go | 27 ++++ neo/store/xun_test.go | 53 ++++++++ 12 files changed, 756 insertions(+), 77 deletions(-) create mode 100644 neo/assistant/assistant_test.go create mode 100644 neo/assistant/cache.go create mode 100644 neo/assistant/cache_test.go delete mode 100644 neo/store/weaviate.go diff --git a/neo/assistant/assistant.go b/neo/assistant/assistant.go index 0de36580..f3ad5640 100644 --- a/neo/assistant/assistant.go +++ b/neo/assistant/assistant.go @@ -1,26 +1,218 @@ package assistant -import "github.com/yaoapp/yao/neo/store" +import ( + "fmt" + "path/filepath" + "regexp" + "strings" + "time" -// loadedAssistant the loaded assistant -var loadedAssistant = map[string]*Assistant{} + jsoniter "github.com/json-iterator/go" + "github.com/yaoapp/gou/fs" + v8 "github.com/yaoapp/gou/runtime/v8" + "github.com/yaoapp/yao/neo/store" + "github.com/yaoapp/yao/share" + "gopkg.in/yaml.v3" +) -// LoadLocal create a new assistant from local -func LoadLocal(path string) *Assistant { - return nil +// loaded the loaded assistant +var loaded = NewCache(200) // 200 is the default capacity +var storage store.Store = nil + +// SetStorage set the storage +func SetStorage(s store.Store) { + storage = s } -// LoadZip create a new assistant from zip -func LoadZip(zip string) *Assistant { - return nil +// SetCache set the cache +func SetCache(capacity int) { + ClearCache() + loaded = NewCache(capacity) } -// LoadRemote create a new assistant from remote -func LoadRemote(url string) *Assistant { - return nil +// ClearCache clear the cache +func ClearCache() { + if loaded != nil { + loaded.Clear() + loaded = nil + } } // LoadStore create a new assistant from store -func LoadStore(store store.Store) *Assistant { - return nil +func LoadStore(id string) (*Assistant, error) { + assistant, exists := loaded.Get(id) + if exists { + return assistant, nil + } + + if storage == nil { + return nil, fmt.Errorf("storage is not set") + } + + data, err := storage.GetAssistant(id) + if err != nil { + return nil, err + } + + assistant, err = loadMap(data) + if err != nil { + return nil, err + } + + loaded.Put(assistant) + return assistant, nil +} + +// LoadPath load assistant from path +func LoadPath(path string) (*Assistant, error) { + app, err := fs.Get("app") + if err != nil { + return nil, err + } + + pkgfile := filepath.Join(path, "package.yao") + if has, _ := app.Exists(pkgfile); !has { + return nil, fmt.Errorf("package.yao not found in %s", path) + } + + pkg, err := app.ReadFile(pkgfile) + if err != nil { + return nil, err + } + + id := strings.ReplaceAll(strings.TrimPrefix(path, "/assistants/"), "/", ".") + var data map[string]interface{} + err = jsoniter.Unmarshal(pkg, &data) + if err != nil { + return nil, err + } + + // assistant_id + data["assistant_id"] = id + + // prompts + promptsfile := filepath.Join(path, "prompts.yml") + if has, _ := app.Exists(promptsfile); has { + prompts, err := loadPrompts(promptsfile, path) + if err != nil { + return nil, err + } + data["prompts"] = prompts + } + + // load script + scriptfile := filepath.Join(path, "src", "index.ts") + if has, _ := app.Exists(scriptfile); has { + script, err := loadScript(scriptfile, path) + if err != nil { + return nil, err + } + data["script"] = script + } + + // load functions + + // load flow + + return loadMap(data) +} + +func loadMap(data map[string]interface{}) (*Assistant, error) { + + assistant := &Assistant{} + + // assistant_id is required + id, ok := data["assistant_id"].(string) + if !ok { + return nil, fmt.Errorf("assistant_id is required") + } + assistant.ID = id + + // name is required + name, ok := data["name"].(string) + if !ok { + return nil, fmt.Errorf("name is required") + } + assistant.Name = name + + // avatar + if avatar, ok := data["avatar"].(string); ok { + assistant.Avatar = avatar + } + + // connector + if connector, ok := data["connector"].(string); ok { + assistant.Connector = connector + } + + // prompts + if v, ok := data["prompts"].(string); ok { + var prompts []Prompt + err := yaml.Unmarshal([]byte(v), &prompts) + if err != nil { + return nil, err + } + assistant.Prompts = prompts + } + + // script + if data["script"] != nil { + switch v := data["script"].(type) { + case string: + file := fmt.Sprintf("assistants/%s/src/index.ts", assistant.ID) + script, err := loadScriptSource(v, file) + if err != nil { + return nil, err + } + assistant.Script = script + case *v8.Script: + assistant.Script = v + } + } + + return assistant, nil +} + +func loadPrompts(file string, root string) (string, error) { + + app, err := fs.Get("app") + if err != nil { + return "", err + } + + prompts, err := app.ReadFile(file) + if err != nil { + return "", err + } + + re := regexp.MustCompile(`@assets/([^\s]+\.(md|yml|yaml|json|txt))`) + prompts = re.ReplaceAllFunc(prompts, func(s []byte) []byte { + asset := re.FindStringSubmatch(string(s))[1] + assetFile := filepath.Join(root, "assets", asset) + assetContent, err := app.ReadFile(assetFile) + if err != nil { + return []byte("") + } + // Add proper YAML formatting for content + lines := strings.Split(string(assetContent), "\n") + formattedContent := "|\n" + for _, line := range lines { + formattedContent += " " + line + "\n" + } + return []byte(formattedContent) + }) + + return string(prompts), nil +} + +func loadScript(file string, root string) (*v8.Script, error) { + return v8.Load(file, share.ID(root, file)) +} + +func loadScriptSource(source string, file string) (*v8.Script, error) { + script, err := v8.MakeScript([]byte(source), file, 5*time.Second, true) + if err != nil { + return nil, err + } + return script, nil } diff --git a/neo/assistant/assistant_test.go b/neo/assistant/assistant_test.go new file mode 100644 index 00000000..f3319c1c --- /dev/null +++ b/neo/assistant/assistant_test.go @@ -0,0 +1,196 @@ +package assistant + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/neo/store" + "github.com/yaoapp/yao/test" +) + +func prepare(t *testing.T) { + test.Prepare(t, config.Conf) +} + +func TestAssistant_LoadPath(t *testing.T) { + prepare(t) + defer test.Clean() + + assistant, err := LoadPath("/assistants/modi") + if err != nil { + t.Fatal(err) + } + + // Validate basic properties + assert.NotNil(t, assistant) + assert.Equal(t, "modi", assistant.ID) + assert.Equal(t, "Modi", assistant.Name) + assert.Equal(t, "https://api.dicebear.com/7.x/bottts/svg?seed=Modi", assistant.Avatar) + assert.Equal(t, "deepseek", assistant.Connector) + assert.NotNil(t, assistant.Prompts) + assert.NotNil(t, assistant.Script) + + // Test non-existent assistant + _, err = LoadPath("/assistants/non-existent") + assert.Error(t, err) +} + +func TestAssistant_LoadStore(t *testing.T) { + prepare(t) + defer test.Clean() + + // Test with nil storage + _, err := LoadStore("test-id") + assert.Error(t, err) + assert.Contains(t, err.Error(), "storage is not set") + + // Setup mock storage + mockStore := &mockStore{ + data: map[string]map[string]interface{}{ + "test-id": { + "assistant_id": "test-id", + "name": "Test Assistant", + "avatar": "test-avatar", + "connector": "test-connector", + }, + }, + } + SetStorage(mockStore) + defer SetStorage(nil) + + // Test loading from store + assistant, err := LoadStore("test-id") + assert.NoError(t, err) + assert.NotNil(t, assistant) + assert.Equal(t, "test-id", assistant.ID) + assert.Equal(t, "Test Assistant", assistant.Name) + assert.Equal(t, "test-avatar", assistant.Avatar) + assert.Equal(t, "test-connector", assistant.Connector) + + // Test cache functionality + assistant2, err := LoadStore("test-id") + assert.NoError(t, err) + assert.Equal(t, assistant, assistant2) // Should be the same instance from cache + + // Test non-existent assistant + _, err = LoadStore("non-existent") + assert.Error(t, err) +} + +func TestAssistant_Cache(t *testing.T) { + prepare(t) + defer test.Clean() + + // Clear any existing cache first + ClearCache() + + // Test cache operations + SetCache(2) // Set small cache size for testing + assert.Equal(t, 2, loaded.capacity, "Cache capacity should be 2") + + // Create test assistants + assistant1 := &Assistant{ID: "id1", Name: "Assistant 1"} + assistant2 := &Assistant{ID: "id2", Name: "Assistant 2"} + assistant3 := &Assistant{ID: "id3", Name: "Assistant 3"} + + // Test Put and Get + loaded.Put(assistant1) + assert.Equal(t, 1, loaded.Len(), "Cache should have 1 item") + + loaded.Put(assistant2) + assert.Equal(t, 2, loaded.Len(), "Cache should have 2 items") + + // Test cache hit + cached, exists := loaded.Get("id1") + assert.True(t, exists) + assert.Equal(t, assistant1, cached) + + // Test cache eviction (LRU) + // At this point: assistant1 is most recently used (due to Get), then assistant2 + loaded.Put(assistant3) // This should evict assistant2 since it's least recently used + assert.Equal(t, 2, loaded.Len(), "Cache should still have 2 items") + _, exists = loaded.Get("id2") + assert.False(t, exists, "assistant2 should have been evicted (least recently used)") + _, exists = loaded.Get("id1") + assert.True(t, exists, "assistant1 should still be in cache (was accessed recently)") + _, exists = loaded.Get("id3") + assert.True(t, exists, "assistant3 should be in cache (most recently added)") + + // Test clear cache + ClearCache() + assert.Nil(t, loaded) + + // Test setting new cache capacity + SetCache(100) + assert.NotNil(t, loaded) +} + +// mockStore implements store.Store interface for testing +type mockStore struct { + data map[string]map[string]interface{} +} + +func (m *mockStore) GetAssistant(id string) (map[string]interface{}, error) { + if data, ok := m.data[id]; ok { + return data, nil + } + return nil, fmt.Errorf("assistant not found: %s", id) +} + +// Add other required interface methods with empty implementations +func (m *mockStore) GetThread(id string) (map[string]interface{}, error) { return nil, nil } +func (m *mockStore) GetMessage(id string) (map[string]interface{}, error) { return nil, nil } +func (m *mockStore) GetFile(id string) (map[string]interface{}, error) { return nil, nil } +func (m *mockStore) CreateAssistant(data map[string]interface{}) (map[string]interface{}, error) { + return nil, nil +} +func (m *mockStore) CreateThread(data map[string]interface{}) (map[string]interface{}, error) { + return nil, nil +} +func (m *mockStore) CreateMessage(data map[string]interface{}) (map[string]interface{}, error) { + return nil, nil +} +func (m *mockStore) CreateFile(data map[string]interface{}) (map[string]interface{}, error) { + return nil, nil +} +func (m *mockStore) UpdateAssistant(id string, data map[string]interface{}) error { return nil } +func (m *mockStore) UpdateThread(id string, data map[string]interface{}) error { return nil } +func (m *mockStore) UpdateMessage(id string, data map[string]interface{}) error { return nil } +func (m *mockStore) UpdateFile(id string, data map[string]interface{}) error { return nil } +func (m *mockStore) DeleteAssistant(id string) error { return nil } +func (m *mockStore) DeleteThread(id string) error { return nil } +func (m *mockStore) DeleteMessage(id string) error { return nil } +func (m *mockStore) DeleteFile(id string) error { return nil } +func (m *mockStore) ListAssistants(query map[string]interface{}) ([]map[string]interface{}, error) { + return nil, nil +} +func (m *mockStore) ListThreads(query map[string]interface{}) ([]map[string]interface{}, error) { + return nil, nil +} +func (m *mockStore) ListMessages(query map[string]interface{}) ([]map[string]interface{}, error) { + return nil, nil +} +func (m *mockStore) ListFiles(query map[string]interface{}) ([]map[string]interface{}, error) { + return nil, nil +} +func (m *mockStore) DeleteAllChats(id string) error { return nil } +func (m *mockStore) DeleteChat(id string, chatID string) error { return nil } +func (m *mockStore) GetAssistants(filter store.AssistantFilter) (*store.AssistantResponse, error) { + return nil, nil +} +func (m *mockStore) GetChat(id string, chatID string) (*store.ChatInfo, error) { return nil, nil } +func (m *mockStore) GetChats(id string, filter store.ChatFilter) (*store.ChatGroupResponse, error) { + return nil, nil +} +func (m *mockStore) GetHistory(id string, chatID string) ([]map[string]interface{}, error) { + return nil, nil +} +func (m *mockStore) SaveAssistant(assistant map[string]interface{}) (interface{}, error) { + return nil, nil +} +func (m *mockStore) SaveHistory(sid string, messages []map[string]interface{}, cid string, context map[string]interface{}) error { + return nil +} +func (m *mockStore) UpdateChatTitle(sid string, cid string, title string) error { return nil } diff --git a/neo/assistant/cache.go b/neo/assistant/cache.go new file mode 100644 index 00000000..47383af9 --- /dev/null +++ b/neo/assistant/cache.go @@ -0,0 +1,105 @@ +package assistant + +import ( + "container/list" + "sync" +) + +// Cache represents a thread-safe LRU cache for Assistant objects +type Cache struct { + capacity int + mu sync.RWMutex + list *list.List + items map[string]*list.Element +} + +// cacheItem represents an item in the cache +type cacheItem struct { + key string + value *Assistant +} + +// NewCache creates a new LRU cache with the given capacity +func NewCache(capacity int) *Cache { + return &Cache{ + capacity: capacity, + list: list.New(), + items: make(map[string]*list.Element), + } +} + +// Get retrieves an Assistant from the cache by its ID +func (c *Cache) Get(id string) (*Assistant, bool) { + c.mu.Lock() + defer c.mu.Unlock() + + if element, exists := c.items[id]; exists { + c.list.MoveToFront(element) + return element.Value.(*cacheItem).value, true + } + return nil, false +} + +// Put adds or updates an Assistant in the cache +func (c *Cache) Put(assistant *Assistant) { + if assistant == nil || assistant.ID == "" { + return + } + + c.mu.Lock() + defer c.mu.Unlock() + + // If item exists, update it and move to front + if element, exists := c.items[assistant.ID]; exists { + c.list.MoveToFront(element) + element.Value.(*cacheItem).value = assistant + return + } + + // If cache is at capacity, remove oldest item before adding new one + if c.list.Len() >= c.capacity { + c.removeOldest() + } + + // Add new item + element := c.list.PushFront(&cacheItem{ + key: assistant.ID, + value: assistant, + }) + c.items[assistant.ID] = element +} + +// Remove removes an Assistant from the cache +func (c *Cache) Remove(id string) { + c.mu.Lock() + defer c.mu.Unlock() + + if element, exists := c.items[id]; exists { + c.list.Remove(element) + delete(c.items, id) + } +} + +// Len returns the current number of items in the cache +func (c *Cache) Len() int { + c.mu.RLock() + defer c.mu.RUnlock() + return c.list.Len() +} + +// Clear removes all items from the cache +func (c *Cache) Clear() { + c.mu.Lock() + defer c.mu.Unlock() + + c.list.Init() + c.items = make(map[string]*list.Element) +} + +// removeOldest removes the least recently used item from the cache +func (c *Cache) removeOldest() { + if element := c.list.Back(); element != nil { + c.list.Remove(element) + delete(c.items, element.Value.(*cacheItem).key) + } +} diff --git a/neo/assistant/cache_test.go b/neo/assistant/cache_test.go new file mode 100644 index 00000000..9ae260ce --- /dev/null +++ b/neo/assistant/cache_test.go @@ -0,0 +1,151 @@ +package assistant + +import ( + "sync" + "testing" +) + +func TestCache_Basic(t *testing.T) { + cache := NewCache(2) + + // Test empty cache + if cache.Len() != 0 { + t.Errorf("Expected empty cache, got length %d", cache.Len()) + } + + // Test adding items + assistant1 := &Assistant{ID: "1", Name: "Test1"} + assistant2 := &Assistant{ID: "2", Name: "Test2"} + + cache.Put(assistant1) + cache.Put(assistant2) + + if cache.Len() != 2 { + t.Errorf("Expected cache length 2, got %d", cache.Len()) + } + + // Test getting items + if a, exists := cache.Get("1"); !exists || a.ID != "1" { + t.Error("Failed to get assistant1") + } + + if a, exists := cache.Get("2"); !exists || a.ID != "2" { + t.Error("Failed to get assistant2") + } +} + +func TestCache_LRU(t *testing.T) { + cache := NewCache(2) + + assistant1 := &Assistant{ID: "1", Name: "Test1"} + assistant2 := &Assistant{ID: "2", Name: "Test2"} + assistant3 := &Assistant{ID: "3", Name: "Test3"} + + // Add first two items + cache.Put(assistant1) + cache.Put(assistant2) + + // Access assistant1 to make it most recently used + cache.Get("1") + + // Add third item, should evict assistant2 + cache.Put(assistant3) + + // Check assistant2 was evicted + if _, exists := cache.Get("2"); exists { + t.Error("Assistant2 should have been evicted") + } + + // Check assistant1 and assistant3 are still present + if _, exists := cache.Get("1"); !exists { + t.Error("Assistant1 should still be in cache") + } + if _, exists := cache.Get("3"); !exists { + t.Error("Assistant3 should be in cache") + } +} + +func TestCache_Remove(t *testing.T) { + cache := NewCache(2) + + assistant1 := &Assistant{ID: "1", Name: "Test1"} + cache.Put(assistant1) + + // Test remove existing item + cache.Remove("1") + if cache.Len() != 0 { + t.Error("Cache should be empty after removing item") + } + + // Test remove non-existing item + cache.Remove("nonexistent") + if cache.Len() != 0 { + t.Error("Cache length should not change when removing non-existent item") + } +} + +func TestCache_Clear(t *testing.T) { + cache := NewCache(2) + + assistant1 := &Assistant{ID: "1", Name: "Test1"} + assistant2 := &Assistant{ID: "2", Name: "Test2"} + + cache.Put(assistant1) + cache.Put(assistant2) + + cache.Clear() + if cache.Len() != 0 { + t.Error("Cache should be empty after clear") + } +} + +func TestCache_Concurrent(t *testing.T) { + cache := NewCache(100) + var wg sync.WaitGroup + workers := 10 + iterations := 100 + + // Concurrent writes + for i := 0; i < workers; i++ { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + for j := 0; j < iterations; j++ { + assistant := &Assistant{ + ID: string(rune('A' + workerID)), + Name: "Test", + } + cache.Put(assistant) + } + }(i) + } + + // Concurrent reads + for i := 0; i < workers; i++ { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + for j := 0; j < iterations; j++ { + cache.Get(string(rune('A' + workerID))) + } + }(i) + } + + wg.Wait() +} + +func TestCache_NilInput(t *testing.T) { + cache := NewCache(2) + + // Test putting nil assistant + cache.Put(nil) + if cache.Len() != 0 { + t.Error("Cache should not store nil assistant") + } + + // Test putting assistant with empty ID + cache.Put(&Assistant{ID: "", Name: "Test"}) + if cache.Len() != 0 { + t.Error("Cache should not store assistant with empty ID") + } +} diff --git a/neo/assistant/types.go b/neo/assistant/types.go index a122ed06..30b8c801 100644 --- a/neo/assistant/types.go +++ b/neo/assistant/types.go @@ -4,6 +4,8 @@ import ( "context" "io" "mime/multipart" + + v8 "github.com/yaoapp/gou/runtime/v8" ) // API the assistant API interface @@ -40,6 +42,7 @@ type Assistant struct { Option map[string]interface{} `json:"option,omitempty"` // AI Option Prompts []Prompt `json:"prompts,omitempty"` // AI Prompts Flows []map[string]interface{} `json:"flows,omitempty"` // Assistant Flows + Script *v8.Script `json:"-" yaml:"-"` // Assistant Script API API `json:"-" yaml:"-"` // Assistant API } diff --git a/neo/neo.go b/neo/neo.go index 97d16767..f072dc1c 100644 --- a/neo/neo.go +++ b/neo/neo.go @@ -535,10 +535,6 @@ func (neo *DSL) createStore() error { } else if conn.Is(connector.MONGO) { neo.Store = store.NewMongo() return nil - - } else if conn.Is(connector.WEAVIATE) { - neo.Store = store.NewWeaviate() - return nil } return fmt.Errorf("%s store connector %s not support", neo.ID, neo.StoreSetting.Connector) diff --git a/neo/store/mongo.go b/neo/store/mongo.go index 88336b1f..9c7ec275 100644 --- a/neo/store/mongo.go +++ b/neo/store/mongo.go @@ -57,3 +57,8 @@ func (m *Mongo) DeleteAssistant(assistantID string) error { func (m *Mongo) GetAssistants(filter AssistantFilter) (*AssistantResponse, error) { return &AssistantResponse{}, nil } + +// GetAssistant retrieves a single assistant by ID +func (m *Mongo) GetAssistant(assistantID string) (map[string]interface{}, error) { + return map[string]interface{}{}, nil +} diff --git a/neo/store/redis.go b/neo/store/redis.go index ad249543..94ab30db 100644 --- a/neo/store/redis.go +++ b/neo/store/redis.go @@ -57,3 +57,8 @@ func (r *Redis) DeleteAssistant(assistantID string) error { func (r *Redis) GetAssistants(filter AssistantFilter) (*AssistantResponse, error) { return &AssistantResponse{}, nil } + +// GetAssistant retrieves a single assistant by ID +func (r *Redis) GetAssistant(assistantID string) (map[string]interface{}, error) { + return map[string]interface{}{}, nil +} diff --git a/neo/store/types.go b/neo/store/types.go index ccfa2039..628a7627 100644 --- a/neo/store/types.go +++ b/neo/store/types.go @@ -130,4 +130,9 @@ type Store interface { // filter: Filter conditions // Returns: Paginated assistant list and potential error GetAssistants(filter AssistantFilter) (*AssistantResponse, error) + + // GetAssistant retrieves a single assistant by ID + // assistantID: Assistant ID + // Returns: Assistant information and potential error + GetAssistant(assistantID string) (map[string]interface{}, error) } diff --git a/neo/store/weaviate.go b/neo/store/weaviate.go deleted file mode 100644 index 4b2dda07..00000000 --- a/neo/store/weaviate.go +++ /dev/null @@ -1,59 +0,0 @@ -package store - -// Weaviate represents a Weaviate-based conversation storage -type Weaviate struct{} - -// NewWeaviate create a new weaviate store -func NewWeaviate() Store { - return &Weaviate{} -} - -// GetChats retrieves a list of chats -func (w *Weaviate) GetChats(sid string, filter ChatFilter) (*ChatGroupResponse, error) { - return &ChatGroupResponse{}, nil -} - -// GetChat retrieves a single chat's information -func (w *Weaviate) GetChat(sid string, cid string) (*ChatInfo, error) { - return &ChatInfo{}, nil -} - -// GetHistory retrieves chat history -func (w *Weaviate) GetHistory(sid string, cid string) ([]map[string]interface{}, error) { - return []map[string]interface{}{}, nil -} - -// SaveHistory saves chat history -func (w *Weaviate) SaveHistory(sid string, messages []map[string]interface{}, cid string, context map[string]interface{}) error { - return nil -} - -// DeleteChat deletes a single chat -func (w *Weaviate) DeleteChat(sid string, cid string) error { - return nil -} - -// DeleteAllChats deletes all chats -func (w *Weaviate) DeleteAllChats(sid string) error { - return nil -} - -// UpdateChatTitle updates chat title -func (w *Weaviate) UpdateChatTitle(sid string, cid string, title string) error { - return nil -} - -// SaveAssistant saves assistant information -func (w *Weaviate) SaveAssistant(assistant map[string]interface{}) (interface{}, error) { - return assistant["assistant_id"], nil -} - -// DeleteAssistant deletes an assistant -func (w *Weaviate) DeleteAssistant(assistantID string) error { - return nil -} - -// GetAssistants retrieves a list of assistants -func (w *Weaviate) GetAssistants(filter AssistantFilter) (*AssistantResponse, error) { - return &AssistantResponse{}, nil -} diff --git a/neo/store/xun.go b/neo/store/xun.go index 2407bdc2..41ee1ade 100644 --- a/neo/store/xun.go +++ b/neo/store/xun.go @@ -44,6 +44,7 @@ type Xun struct { // SaveAssistant creates or updates an assistant // DeleteAssistant deletes an assistant by assistant_id // GetAssistants retrieves a paginated list of assistants with filtering +// GetAssistant retrieves a single assistant by assistant_id // NewXun create a new xun store func NewXun(setting Setting) (Store, error) { @@ -923,3 +924,29 @@ func (conv *Xun) GetAssistants(filter AssistantFilter) (*AssistantResponse, erro Total: total, }, nil } + +// GetAssistant retrieves a single assistant by ID +func (conv *Xun) GetAssistant(assistantID string) (map[string]interface{}, error) { + row, err := conv.query.New(). + Table(conv.getAssistantTable()). + Where("assistant_id", assistantID). + First() + if err != nil { + return nil, err + } + + if row == nil { + return nil, fmt.Errorf("assistant %s not found", assistantID) + } + + data := row.ToMap() + if data == nil || len(data) == 0 { + return nil, fmt.Errorf("assistant %s not found", assistantID) + } + + // Parse JSON fields + jsonFields := []string{"tags", "options", "prompts", "flows", "files", "functions", "permissions"} + conv.parseJSONFields(data, jsonFields) + + return data, nil +} diff --git a/neo/store/xun_test.go b/neo/store/xun_test.go index a8613c3a..a41ae3fa 100644 --- a/neo/store/xun_test.go +++ b/neo/store/xun_test.go @@ -483,6 +483,20 @@ func TestXunAssistantCRUD(t *testing.T) { assistantID := v.(string) assert.NotEmpty(t, assistantID) + // Test GetAssistant for the first assistant + assistantData, err := store.GetAssistant(assistantID) + assert.Nil(t, err) + assert.NotNil(t, assistantData) + assert.Equal(t, "Test Assistant", assistantData["name"]) + assert.Equal(t, "assistant", assistantData["type"]) + assert.Equal(t, "https://example.com/avatar.png", assistantData["avatar"]) + assert.Equal(t, "openai", assistantData["connector"]) + assert.Equal(t, "Test Description", assistantData["description"]) + assert.Equal(t, []interface{}{"tag1", "tag2", "tag3"}, assistantData["tags"]) + assert.Equal(t, map[string]interface{}{"model": "gpt-4"}, assistantData["options"]) + assert.Equal(t, int64(1), assistantData["mentionable"]) + assert.Equal(t, int64(1), assistantData["automated"]) + // Test case 2: JSON fields as native types assistant2 := map[string]interface{}{ "name": "Test Assistant 2", @@ -507,6 +521,24 @@ func TestXunAssistantCRUD(t *testing.T) { assistant2ID := v.(string) assert.NotEmpty(t, assistant2ID) + // Test GetAssistant for the second assistant + assistant2Data, err := store.GetAssistant(assistant2ID) + assert.Nil(t, err) + assert.NotNil(t, assistant2Data) + assert.Equal(t, "Test Assistant 2", assistant2Data["name"]) + assert.Equal(t, []interface{}{"tag1", "tag2", "tag3"}, assistant2Data["tags"]) + assert.Equal(t, map[string]interface{}{"model": "gpt-4"}, assistant2Data["options"]) + assert.Equal(t, []interface{}{"prompt1", "prompt2"}, assistant2Data["prompts"]) + assert.Equal(t, []interface{}{"flow1", "flow2"}, assistant2Data["flows"]) + assert.Equal(t, []interface{}{"file1", "file2"}, assistant2Data["files"]) + assert.Equal(t, []interface{}{ + map[string]interface{}{"name": "func1"}, + map[string]interface{}{"name": "func2"}, + }, assistant2Data["functions"]) + assert.Equal(t, map[string]interface{}{"read": true, "write": true}, assistant2Data["permissions"]) + assert.Equal(t, int64(1), assistant2Data["mentionable"]) + assert.Equal(t, int64(1), assistant2Data["automated"]) + // Test case 3: Test with nil JSON fields assistant3 := map[string]interface{}{ "name": "Test Assistant 3", @@ -530,6 +562,27 @@ func TestXunAssistantCRUD(t *testing.T) { assistant3ID := v.(string) assert.NotEmpty(t, assistant3ID) + // Test GetAssistant for the third assistant + assistant3Data, err := store.GetAssistant(assistant3ID) + assert.Nil(t, err) + assert.NotNil(t, assistant3Data) + assert.Equal(t, "Test Assistant 3", assistant3Data["name"]) + assert.Nil(t, assistant3Data["tags"]) + assert.Nil(t, assistant3Data["options"]) + assert.Nil(t, assistant3Data["prompts"]) + assert.Nil(t, assistant3Data["flows"]) + assert.Nil(t, assistant3Data["files"]) + assert.Nil(t, assistant3Data["functions"]) + assert.Nil(t, assistant3Data["permissions"]) + assert.Equal(t, int64(1), assistant3Data["mentionable"]) + assert.Equal(t, int64(1), assistant3Data["automated"]) + + // Test GetAssistant with non-existent ID + nonExistentData, err := store.GetAssistant("non-existent-id") + assert.Error(t, err) + assert.Nil(t, nonExistentData) + assert.Contains(t, err.Error(), "not found") + // Test GetAssistants to verify JSON fields are properly stored resp, err := store.GetAssistants(AssistantFilter{}) assert.Nil(t, err) From f8ce8fc1932ab02a147aa69f24dd05d0f42e1b38 Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 1 Jan 2025 16:48:06 +0800 Subject: [PATCH 4/8] Enhance assistant management and API functionality in Neo - Added a new endpoint to retrieve all assistant tags, improving data accessibility for clients. - Updated the assistant list handling to support filtering by built-in status and assistant ID, enhancing the filtering capabilities. - Introduced a method to load built-in assistants, streamlining the assistant initialization process. - Enhanced the Assistant struct with new fields for path, built-in status, and sorting, improving data organization. - Implemented validation and cloning methods for the Assistant struct, ensuring data integrity and ease of use. - Updated tests to cover new functionalities, including validation, cloning, and tag retrieval, ensuring robust functionality across the assistant management operations. --- neo/api.go | 36 +++ neo/assistant/assistant.go | 243 ++++++++++++++++- neo/assistant/assistant_test.go | 194 +++++++++++++- neo/assistant/types.go | 9 +- neo/load.go | 7 + neo/store/mongo.go | 10 + neo/store/redis.go | 10 + neo/store/types.go | 10 + neo/store/xun.go | 97 ++++++- neo/store/xun_test.go | 445 ++++++++++++++++++++++++++++++-- 10 files changed, 1039 insertions(+), 22 deletions(-) diff --git a/neo/api.go b/neo/api.go index 1ecc7dd3..aebe7bad 100644 --- a/neo/api.go +++ b/neo/api.go @@ -62,6 +62,9 @@ func (neo *DSL) API(router *gin.Engine, path string) error { // List assistants example: // curl -X GET 'http://localhost:5099/api/__yao/neo/assistants?page=1&pagesize=20&tags=tag1,tag2&token=xxx' router.GET(path+"/assistants", append(middlewares, neo.handleAssistantList)...) + // Get all assistant tags example: + // curl -X GET 'http://localhost:5099/api/__yao/neo/assistants/tags?token=xxx' + router.GET(path+"/assistants/tags", append(middlewares, neo.handleAssistantTags)...) // Get assistant details example: // curl -X GET 'http://localhost:5099/api/__yao/neo/assistants/assistant_123?token=xxx' @@ -878,6 +881,14 @@ func (neo *DSL) handleAssistantList(c *gin.Context) { filter.Select = strings.Split(selectFields, ",") } + // Parse built_in (support various boolean formats) + if builtIn := c.Query("built_in"); builtIn != "" { + val := parseBoolValue(builtIn) + if val != nil { + filter.BuiltIn = val + } + } + // Parse mentionable (support various boolean formats) if mentionable := c.Query("mentionable"); mentionable != "" { val := parseBoolValue(mentionable) @@ -894,6 +905,11 @@ func (neo *DSL) handleAssistantList(c *gin.Context) { } } + // Parse assistant_id + if assistantID := c.Query("assistant_id"); assistantID != "" { + filter.AssistantID = assistantID + } + response, err := neo.Store.GetAssistants(filter) if err != nil { c.JSON(500, gin.H{"message": err.Error(), "code": 500}) @@ -1023,3 +1039,23 @@ func (neo *DSL) handleConnectors(c *gin.Context) { c.JSON(200, gin.H{"data": options}) c.Done() } + +// handleAssistantTags handles getting all assistant tags +func (neo *DSL) handleAssistantTags(c *gin.Context) { + sid := c.GetString("__sid") + if sid == "" { + c.JSON(400, gin.H{"message": "sid is required", "code": 400}) + c.Done() + return + } + + tags, err := neo.Store.GetAssistantTags() + if err != nil { + c.JSON(500, gin.H{"message": err.Error(), "code": 500}) + c.Done() + return + } + + c.JSON(200, gin.H{"data": tags}) + c.Done() +} diff --git a/neo/assistant/assistant.go b/neo/assistant/assistant.go index f3ad5640..1a85867a 100644 --- a/neo/assistant/assistant.go +++ b/neo/assistant/assistant.go @@ -19,6 +19,63 @@ import ( var loaded = NewCache(200) // 200 is the default capacity var storage store.Store = nil +// LoadBuiltIn load the built-in assistants +func LoadBuiltIn() error { + root := `/assistants` + app, err := fs.Get("app") + if err != nil { + return err + } + + // Remove the built-in assistants + if storage != nil { + builtIn := true + _, err := storage.DeleteAssistants(store.AssistantFilter{BuiltIn: &builtIn}) + if err != nil { + return err + } + } + + // Check if the assistant is built-in + if exists, _ := app.Exists(root); !exists { + return nil + } + + paths, err := app.ReadDir(root, true) + if err != nil { + return err + } + + sort := 1 + for _, path := range paths { + pkgfile := filepath.Join(path, "package.yao") + if has, _ := app.Exists(pkgfile); !has { + continue + } + + assistant, err := LoadPath(path) + if err != nil { + return err + } + + assistant.Readonly = true + assistant.BuiltIn = true + assistant.Sort = sort + sort++ + loaded.Put(assistant) + + // Save the assistant + if storage != nil { + _, err := storage.SaveAssistant(assistant.Map()) + if err != nil { + return err + } + } + } + + return nil +} + // SetStorage set the storage func SetStorage(s store.Store) { storage = s @@ -89,7 +146,8 @@ func LoadPath(path string) (*Assistant, error) { // assistant_id data["assistant_id"] = id - + data["type"] = "assistant" + data["path"] = path // prompts promptsfile := filepath.Join(path, "prompts.yml") if has, _ := app.Exists(promptsfile); has { @@ -140,6 +198,41 @@ func loadMap(data map[string]interface{}) (*Assistant, error) { assistant.Avatar = avatar } + // Type + if v, ok := data["type"].(string); ok { + assistant.Type = v + } + + // Mentionable + if v, ok := data["mentionable"].(bool); ok { + assistant.Mentionable = v + } + + // Automated + if v, ok := data["automated"].(bool); ok { + assistant.Automated = v + } + + // Readonly + if v, ok := data["readonly"].(bool); ok { + assistant.Readonly = v + } + + // built_in + if v, ok := data["built_in"].(bool); ok { + assistant.BuiltIn = v + } + + // sort + if v, ok := data["sort"].(int); ok { + assistant.Sort = v + } + + // path + if v, ok := data["path"].(string); ok { + assistant.Path = v + } + // connector if connector, ok := data["connector"].(string); ok { assistant.Connector = connector @@ -216,3 +309,151 @@ func loadScriptSource(source string, file string) (*v8.Script, error) { } return script, nil } + +// Save save the assistant +func (ast *Assistant) Save() error { + if storage == nil { + return fmt.Errorf("storage is not set") + } + + _, err := storage.SaveAssistant(ast.Map()) + return err +} + +// Map convert the assistant to a map +func (ast *Assistant) Map() map[string]interface{} { + + if ast == nil { + return nil + } + + return map[string]interface{}{ + "assistant_id": ast.ID, + "type": ast.Type, + "name": ast.Name, + "readonly": ast.Readonly, + "avatar": ast.Avatar, + "connector": ast.Connector, + "path": ast.Path, + "built_in": ast.BuiltIn, + "sort": ast.Sort, + "description": ast.Description, + "options": ast.Options, + "prompts": ast.Prompts, + "tags": ast.Tags, + "mentionable": ast.Mentionable, + "automated": ast.Automated, + } +} + +// Validate validates the assistant configuration +func (ast *Assistant) Validate() error { + if ast.ID == "" { + return fmt.Errorf("assistant_id is required") + } + if ast.Name == "" { + return fmt.Errorf("name is required") + } + if ast.Connector == "" { + return fmt.Errorf("connector is required") + } + return nil +} + +// Clone creates a deep copy of the assistant +func (ast *Assistant) Clone() *Assistant { + if ast == nil { + return nil + } + + clone := &Assistant{ + ID: ast.ID, + Type: ast.Type, + Name: ast.Name, + Avatar: ast.Avatar, + Connector: ast.Connector, + Path: ast.Path, + BuiltIn: ast.BuiltIn, + Sort: ast.Sort, + Description: ast.Description, + Readonly: ast.Readonly, + Mentionable: ast.Mentionable, + Automated: ast.Automated, + Script: ast.Script, + API: ast.API, + } + + // Deep copy tags + if ast.Tags != nil { + clone.Tags = make([]string, len(ast.Tags)) + copy(clone.Tags, ast.Tags) + } + + // Deep copy options + if ast.Options != nil { + clone.Options = make(map[string]interface{}) + for k, v := range ast.Options { + clone.Options[k] = v + } + } + + // Deep copy prompts + if ast.Prompts != nil { + clone.Prompts = make([]Prompt, len(ast.Prompts)) + copy(clone.Prompts, ast.Prompts) + } + + // Deep copy flows + if ast.Flows != nil { + clone.Flows = make([]map[string]interface{}, len(ast.Flows)) + for i, flow := range ast.Flows { + cloneFlow := make(map[string]interface{}) + for k, v := range flow { + cloneFlow[k] = v + } + clone.Flows[i] = cloneFlow + } + } + + return clone +} + +// Update updates the assistant properties +func (ast *Assistant) Update(data map[string]interface{}) error { + if ast == nil { + return fmt.Errorf("assistant is nil") + } + + if v, ok := data["name"].(string); ok { + ast.Name = v + } + if v, ok := data["avatar"].(string); ok { + ast.Avatar = v + } + if v, ok := data["description"].(string); ok { + ast.Description = v + } + if v, ok := data["connector"].(string); ok { + ast.Connector = v + } + if v, ok := data["type"].(string); ok { + ast.Type = v + } + if v, ok := data["sort"].(int); ok { + ast.Sort = v + } + if v, ok := data["mentionable"].(bool); ok { + ast.Mentionable = v + } + if v, ok := data["automated"].(bool); ok { + ast.Automated = v + } + if v, ok := data["tags"].([]string); ok { + ast.Tags = v + } + if v, ok := data["options"].(map[string]interface{}); ok { + ast.Options = v + } + + return ast.Validate() +} diff --git a/neo/assistant/assistant_test.go b/neo/assistant/assistant_test.go index f3319c1c..fb4f20e1 100644 --- a/neo/assistant/assistant_test.go +++ b/neo/assistant/assistant_test.go @@ -127,6 +127,196 @@ func TestAssistant_Cache(t *testing.T) { assert.NotNil(t, loaded) } +func TestAssistant_Validate(t *testing.T) { + tests := []struct { + name string + ast *Assistant + wantErr bool + }{ + { + name: "valid assistant", + ast: &Assistant{ + ID: "test-id", + Name: "Test Assistant", + Connector: "test-connector", + }, + wantErr: false, + }, + { + name: "missing id", + ast: &Assistant{ + Name: "Test Assistant", + Connector: "test-connector", + }, + wantErr: true, + }, + { + name: "missing name", + ast: &Assistant{ + ID: "test-id", + Connector: "test-connector", + }, + wantErr: true, + }, + { + name: "missing connector", + ast: &Assistant{ + ID: "test-id", + Name: "Test Assistant", + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.ast.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Assistant.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestAssistant_Clone(t *testing.T) { + // Create a test assistant with all fields populated + original := &Assistant{ + ID: "test-id", + Type: "test-type", + Name: "Test Assistant", + Avatar: "test-avatar", + Connector: "test-connector", + Path: "test-path", + BuiltIn: true, + Sort: 1, + Description: "test description", + Tags: []string{"tag1", "tag2"}, + Readonly: true, + Mentionable: true, + Automated: true, + Options: map[string]interface{}{"key": "value"}, + Prompts: []Prompt{{Role: "system", Content: "test"}}, + Flows: []map[string]interface{}{{"step": "test"}}, + } + + // Clone the assistant + clone := original.Clone() + + // Verify all fields are correctly cloned + assert.Equal(t, original.ID, clone.ID) + assert.Equal(t, original.Type, clone.Type) + assert.Equal(t, original.Name, clone.Name) + assert.Equal(t, original.Avatar, clone.Avatar) + assert.Equal(t, original.Connector, clone.Connector) + assert.Equal(t, original.Path, clone.Path) + assert.Equal(t, original.BuiltIn, clone.BuiltIn) + assert.Equal(t, original.Sort, clone.Sort) + assert.Equal(t, original.Description, clone.Description) + assert.Equal(t, original.Tags, clone.Tags) + assert.Equal(t, original.Readonly, clone.Readonly) + assert.Equal(t, original.Mentionable, clone.Mentionable) + assert.Equal(t, original.Automated, clone.Automated) + assert.Equal(t, original.Options, clone.Options) + assert.Equal(t, original.Prompts, clone.Prompts) + assert.Equal(t, original.Flows, clone.Flows) + + // Verify deep copy by modifying original + original.Tags[0] = "modified" + original.Options["key"] = "modified" + original.Flows[0]["step"] = "modified" + assert.NotEqual(t, original.Tags[0], clone.Tags[0]) + assert.NotEqual(t, original.Options["key"], clone.Options["key"]) + assert.NotEqual(t, original.Flows[0]["step"], clone.Flows[0]["step"]) + + // Test nil case + var nilAssistant *Assistant + assert.Nil(t, nilAssistant.Clone()) +} + +func TestAssistant_Update(t *testing.T) { + // Create a test assistant + ast := &Assistant{ + ID: "test-id", + Name: "Original Name", + Connector: "original-connector", + } + + // Test updating various fields + updates := map[string]interface{}{ + "name": "Updated Name", + "avatar": "updated-avatar", + "description": "Updated description", + "connector": "updated-connector", + "type": "updated-type", + "sort": 2, + "mentionable": true, + "automated": true, + "tags": []string{"new-tag"}, + "options": map[string]interface{}{"new": "value"}, + } + + err := ast.Update(updates) + assert.NoError(t, err) + + // Verify updates + assert.Equal(t, "Updated Name", ast.Name) + assert.Equal(t, "updated-avatar", ast.Avatar) + assert.Equal(t, "Updated description", ast.Description) + assert.Equal(t, "updated-connector", ast.Connector) + assert.Equal(t, "updated-type", ast.Type) + assert.Equal(t, 2, ast.Sort) + assert.True(t, ast.Mentionable) + assert.True(t, ast.Automated) + assert.Equal(t, []string{"new-tag"}, ast.Tags) + assert.Equal(t, map[string]interface{}{"new": "value"}, ast.Options) + + // Test nil assistant + var nilAssistant *Assistant + err = nilAssistant.Update(updates) + assert.Error(t, err) + + // Test invalid update that would make the assistant invalid + invalidUpdates := map[string]interface{}{ + "name": "", + } + err = ast.Update(invalidUpdates) + assert.Error(t, err) +} + +func TestLoadBuiltIn(t *testing.T) { + prepare(t) + defer test.Clean() + + // Clear any existing cache and storage + ClearCache() + SetStorage(nil) + + // Create a mock store to verify built-in assistants are saved + mockStore := &mockStore{ + data: make(map[string]map[string]interface{}), + } + SetStorage(mockStore) + SetCache(100) + + // Test loading built-in assistants + err := LoadBuiltIn() + assert.NoError(t, err) + + // Verify Modi assistant was loaded + assistant, exists := loaded.Get("modi") + assert.True(t, exists, "Modi assistant should be loaded in cache") + if exists { + assert.Equal(t, "modi", assistant.ID) + assert.Equal(t, "Modi", assistant.Name) + assert.Equal(t, "deepseek", assistant.Connector) + assert.True(t, assistant.BuiltIn) + assert.True(t, assistant.Readonly) + assert.NotNil(t, assistant.Prompts) + assert.NotNil(t, assistant.Script) + } + +} + // mockStore implements store.Store interface for testing type mockStore struct { data map[string]map[string]interface{} @@ -193,4 +383,6 @@ func (m *mockStore) SaveAssistant(assistant map[string]interface{}) (interface{} func (m *mockStore) SaveHistory(sid string, messages []map[string]interface{}, cid string, context map[string]interface{}) error { return nil } -func (m *mockStore) UpdateChatTitle(sid string, cid string, title string) error { return nil } +func (m *mockStore) UpdateChatTitle(sid string, cid string, title string) error { return nil } +func (m *mockStore) DeleteAssistants(filter store.AssistantFilter) (int64, error) { return 0, nil } +func (m *mockStore) GetAssistantTags() ([]string, error) { return []string{}, nil } diff --git a/neo/assistant/types.go b/neo/assistant/types.go index 30b8c801..977cf5e9 100644 --- a/neo/assistant/types.go +++ b/neo/assistant/types.go @@ -38,8 +38,15 @@ type Assistant struct { Name string `json:"name,omitempty"` // Assistant Name Avatar string `json:"avatar,omitempty"` // Assistant Avatar Connector string `json:"connector"` // AI Connector + Path string `json:"path,omitempty"` // Assistant Path + BuiltIn bool `json:"built_in,omitempty"` // Whether this is a built-in assistant + Sort int `json:"sort,omitempty"` // Assistant Sort Description string `json:"description,omitempty"` // Assistant Description - Option map[string]interface{} `json:"option,omitempty"` // AI Option + Tags []string `json:"tags,omitempty"` // Assistant Tags + Readonly bool `json:"readonly,omitempty"` // Whether this assistant is readonly + Mentionable bool `json:"mentionable,omitempty"` // Whether this assistant is mentionable + Automated bool `json:"automated,omitempty"` // Whether this assistant is automated + Options map[string]interface{} `json:"options,omitempty"` // AI Options Prompts []Prompt `json:"prompts,omitempty"` // AI Prompts Flows []map[string]interface{} `json:"flows,omitempty"` // Assistant Flows Script *v8.Script `json:"-" yaml:"-"` // Assistant Script diff --git a/neo/load.go b/neo/load.go index 0a918d81..6b87957f 100644 --- a/neo/load.go +++ b/neo/load.go @@ -51,6 +51,13 @@ func Load(cfg config.Config) error { return err } + // Load Built-in Assistants + assistant.SetStorage(Neo.Store) + err = assistant.LoadBuiltIn() + if err != nil { + return err + } + // Query Assistant List ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() diff --git a/neo/store/mongo.go b/neo/store/mongo.go index 9c7ec275..e94b4278 100644 --- a/neo/store/mongo.go +++ b/neo/store/mongo.go @@ -62,3 +62,13 @@ func (m *Mongo) GetAssistants(filter AssistantFilter) (*AssistantResponse, error func (m *Mongo) GetAssistant(assistantID string) (map[string]interface{}, error) { return map[string]interface{}{}, nil } + +// DeleteAssistants deletes assistants based on filter conditions (not implemented) +func (mongo *Mongo) DeleteAssistants(filter AssistantFilter) (int64, error) { + return 0, nil +} + +// GetAssistantTags retrieves all unique tags from assistants +func (conv *Mongo) GetAssistantTags() ([]string, error) { + return []string{}, nil +} diff --git a/neo/store/redis.go b/neo/store/redis.go index 94ab30db..50206e1a 100644 --- a/neo/store/redis.go +++ b/neo/store/redis.go @@ -62,3 +62,13 @@ func (r *Redis) GetAssistants(filter AssistantFilter) (*AssistantResponse, error func (r *Redis) GetAssistant(assistantID string) (map[string]interface{}, error) { return map[string]interface{}{}, nil } + +// DeleteAssistants deletes assistants based on filter conditions (not implemented) +func (redis *Redis) DeleteAssistants(filter AssistantFilter) (int64, error) { + return 0, nil +} + +// GetAssistantTags retrieves all unique tags from assistants +func (conv *Redis) GetAssistantTags() ([]string, error) { + return []string{}, nil +} diff --git a/neo/store/types.go b/neo/store/types.go index 628a7627..925c5ecd 100644 --- a/neo/store/types.go +++ b/neo/store/types.go @@ -52,6 +52,7 @@ type AssistantFilter struct { AssistantID string `json:"assistant_id,omitempty"` // Filter by assistant ID Mentionable *bool `json:"mentionable,omitempty"` // Filter by mentionable status Automated *bool `json:"automated,omitempty"` // Filter by automation status + BuiltIn *bool `json:"built_in,omitempty"` // Filter by built-in status Page int `json:"page,omitempty"` // Page number, starting from 1 PageSize int `json:"pagesize,omitempty"` // Items per page Select []string `json:"select,omitempty"` // Fields to return, returns all fields if empty @@ -135,4 +136,13 @@ type Store interface { // assistantID: Assistant ID // Returns: Assistant information and potential error GetAssistant(assistantID string) (map[string]interface{}, error) + + // DeleteAssistants deletes assistants based on filter conditions + // filter: Filter conditions + // Returns: Number of deleted records and potential error + DeleteAssistants(filter AssistantFilter) (int64, error) + + // GetAssistantTags retrieves all unique tags from assistants + // Returns: List of tags and potential error + GetAssistantTags() ([]string, error) } diff --git a/neo/store/xun.go b/neo/store/xun.go index 41ee1ade..05c9c068 100644 --- a/neo/store/xun.go +++ b/neo/store/xun.go @@ -225,6 +225,9 @@ func (conv *Xun) initAssistantTable() error { table.String("avatar", 200).Null() // assistant avatar table.String("connector", 200).NotNull() // assistant connector table.Text("description").Null() // assistant description + table.String("path", 200).Null() // assistant storage path + table.Integer("sort").SetDefault(9999).Index() // assistant sort order + table.Boolean("built_in").SetDefault(false).Index() // whether this is a built-in assistant table.JSON("options").Null() // assistant options table.JSON("prompts").Null() // assistant prompts table.JSON("flows").Null() // assistant flows @@ -251,7 +254,7 @@ func (conv *Xun) initAssistantTable() error { return err } - fields := []string{"id", "assistant_id", "type", "name", "avatar", "connector", "description", "options", "prompts", "flows", "files", "functions", "tags", "mentionable", "created_at", "updated_at"} + fields := []string{"id", "assistant_id", "type", "name", "avatar", "connector", "description", "path", "sort", "built_in", "options", "prompts", "flows", "files", "functions", "tags", "mentionable", "created_at", "updated_at"} for _, field := range fields { if !tab.HasColumn(field) { return fmt.Errorf("%s is required", field) @@ -845,6 +848,11 @@ func (conv *Xun) GetAssistants(filter AssistantFilter) (*AssistantResponse, erro qb.Where("automated", *filter.Automated) } + // Apply built_in filter if provided + if filter.BuiltIn != nil { + qb.Where("built_in", *filter.BuiltIn) + } + // Set defaults for pagination if filter.PageSize <= 0 { filter.PageSize = 20 @@ -881,7 +889,8 @@ func (conv *Xun) GetAssistants(filter AssistantFilter) (*AssistantResponse, erro } // Get paginated results - rows, err := qb.OrderBy("created_at", "desc"). + rows, err := qb.OrderBy("sort", "asc"). + OrderBy("updated_at", "desc"). Offset(offset). Limit(filter.PageSize). Get() @@ -950,3 +959,87 @@ func (conv *Xun) GetAssistant(assistantID string) (map[string]interface{}, error return data, nil } + +// DeleteAssistants deletes assistants based on filter conditions +func (conv *Xun) DeleteAssistants(filter AssistantFilter) (int64, error) { + qb := conv.query.New(). + Table(conv.getAssistantTable()) + + // Apply tag filter if provided + if filter.Tags != nil && len(filter.Tags) > 0 { + qb.Where(func(qb query.Query) { + for i, tag := range filter.Tags { + pattern := fmt.Sprintf("%%\"%s\"%%", tag) + if i == 0 { + qb.Where("tags", "like", pattern) + } else { + qb.OrWhere("tags", "like", pattern) + } + } + }) + } + + // Apply keyword filter if provided + if filter.Keywords != "" { + qb.Where(func(qb query.Query) { + qb.Where("name", "like", fmt.Sprintf("%%%s%%", filter.Keywords)). + OrWhere("description", "like", fmt.Sprintf("%%%s%%", filter.Keywords)) + }) + } + + // Apply connector filter if provided + if filter.Connector != "" { + qb.Where("connector", filter.Connector) + } + + // Apply assistant_id filter if provided + if filter.AssistantID != "" { + qb.Where("assistant_id", filter.AssistantID) + } + + // Apply mentionable filter if provided + if filter.Mentionable != nil { + qb.Where("mentionable", *filter.Mentionable) + } + + // Apply automated filter if provided + if filter.Automated != nil { + qb.Where("automated", *filter.Automated) + } + + // Apply built_in filter if provided + if filter.BuiltIn != nil { + qb.Where("built_in", *filter.BuiltIn) + } + + // Execute delete and return number of deleted records + return qb.Delete() +} + +// GetAssistantTags retrieves all unique tags from assistants +func (conv *Xun) GetAssistantTags() ([]string, error) { + q := conv.newQuery().Table(conv.getAssistantTable()) + rows, err := q.Select("tags").GroupBy("tags").Get() + if err != nil { + return nil, err + } + + tagSet := map[string]bool{} + for _, row := range rows { + if tags, ok := row["tags"].(string); ok && tags != "" { + var tagList []string + if err := jsoniter.UnmarshalFromString(tags, &tagList); err == nil { + for _, tag := range tagList { + tagSet[tag] = true + } + } + } + } + + // Convert map keys to slice + tags := make([]string, 0, len(tagSet)) + for tag := range tagSet { + tags = append(tags, tag) + } + return tags, nil +} diff --git a/neo/store/xun_test.go b/neo/store/xun_test.go index a41ae3fa..dc42de04 100644 --- a/neo/store/xun_test.go +++ b/neo/store/xun_test.go @@ -471,6 +471,9 @@ func TestXunAssistantCRUD(t *testing.T) { "avatar": "https://example.com/avatar.png", "connector": "openai", "description": "Test Description", + "path": "/assistants/test", + "sort": 100, + "built_in": true, "tags": tagsJSON, "options": optionsJSON, "mentionable": true, @@ -492,6 +495,9 @@ func TestXunAssistantCRUD(t *testing.T) { assert.Equal(t, "https://example.com/avatar.png", assistantData["avatar"]) assert.Equal(t, "openai", assistantData["connector"]) assert.Equal(t, "Test Description", assistantData["description"]) + assert.Equal(t, "/assistants/test", assistantData["path"]) + assert.Equal(t, int64(100), assistantData["sort"]) + assert.Equal(t, int64(1), assistantData["built_in"]) assert.Equal(t, []interface{}{"tag1", "tag2", "tag3"}, assistantData["tags"]) assert.Equal(t, map[string]interface{}{"model": "gpt-4"}, assistantData["options"]) assert.Equal(t, int64(1), assistantData["mentionable"]) @@ -504,6 +510,9 @@ func TestXunAssistantCRUD(t *testing.T) { "avatar": "https://example.com/avatar2.png", "connector": "openai", "description": "Test Description 2", + "path": "/assistants/test2", + "sort": 200, + "built_in": false, "tags": []string{"tag1", "tag2", "tag3"}, "options": map[string]interface{}{"model": "gpt-4"}, "prompts": []string{"prompt1", "prompt2"}, @@ -545,6 +554,9 @@ func TestXunAssistantCRUD(t *testing.T) { "type": "assistant", "connector": "openai", "description": "Test Description 3", + "path": nil, + "sort": 9999, + "built_in": false, "tags": nil, "options": nil, "prompts": nil, @@ -665,14 +677,232 @@ func TestXunAssistantCRUD(t *testing.T) { } } - // Test DeleteAssistant - err = store.DeleteAssistant(assistantID) + // Test non-existent assistant_id + resp, err = store.GetAssistants(AssistantFilter{ + AssistantID: "non-existent-id", + Page: 1, + PageSize: 10, + }) assert.Nil(t, err) - err = store.DeleteAssistant(assistant2ID) + assert.Equal(t, 0, len(resp.Data)) + + // Test filtering with select fields + resp, err = store.GetAssistants(AssistantFilter{ + Select: []string{"name", "description", "tags"}, + Page: 1, + PageSize: 10, + }) assert.Nil(t, err) - err = store.DeleteAssistant(assistant3ID) + // Verify only selected fields are returned + for _, item := range resp.Data { + // These fields should exist + assert.Contains(t, item, "name") + assert.Contains(t, item, "description") + assert.Contains(t, item, "tags") + // These fields should not exist + assert.NotContains(t, item, "options") + assert.NotContains(t, item, "prompts") + assert.NotContains(t, item, "flows") + assert.NotContains(t, item, "files") + assert.NotContains(t, item, "functions") + assert.NotContains(t, item, "permissions") + } + + // Test filtering with select fields and other filters combined + resp, err = store.GetAssistants(AssistantFilter{ + Tags: []string{"tag1"}, + Keywords: "Assistant", + Select: []string{"name", "tags"}, + Page: 1, + PageSize: 10, + }) + assert.Nil(t, err) + // Verify only selected fields are returned + for _, item := range resp.Data { + // These fields should exist + assert.Contains(t, item, "name") + assert.Contains(t, item, "tags") + // These fields should not exist + assert.NotContains(t, item, "description") + assert.NotContains(t, item, "options") + assert.NotContains(t, item, "prompts") + assert.NotContains(t, item, "flows") + assert.NotContains(t, item, "files") + assert.NotContains(t, item, "functions") + assert.NotContains(t, item, "permissions") + } + + // Test filtering with automated + automatedTrue := true + resp, err = store.GetAssistants(AssistantFilter{ + Automated: &automatedTrue, + Page: 1, + PageSize: 10, + }) + assert.Nil(t, err) + assert.Greater(t, len(resp.Data), 0) + + // Test filtering with mentionable + mentionableTrue := true + resp, err = store.GetAssistants(AssistantFilter{ + Mentionable: &mentionableTrue, + Page: 1, + PageSize: 10, + }) + assert.Nil(t, err) + assert.Greater(t, len(resp.Data), 0) + + // Test combined filters + resp, err = store.GetAssistants(AssistantFilter{ + Tags: []string{"tag1"}, + Keywords: "Assistant", + Connector: "openai", + Mentionable: &mentionableTrue, + Automated: &automatedTrue, + Page: 1, + PageSize: 10, + }) assert.Nil(t, err) + // Test filtering with built_in + builtInTrue := true + resp, err = store.GetAssistants(AssistantFilter{ + BuiltIn: &builtInTrue, + Page: 1, + PageSize: 10, + }) + assert.Nil(t, err) + for _, assistant := range resp.Data { + assert.Equal(t, int64(1), assistant["built_in"], "All assistants should be built-in") + } + + builtInFalse := false + resp, err = store.GetAssistants(AssistantFilter{ + BuiltIn: &builtInFalse, + Page: 1, + PageSize: 10, + }) + assert.Nil(t, err) + for _, assistant := range resp.Data { + assert.Equal(t, int64(0), assistant["built_in"], "All assistants should not be built-in") + } + + // Now test the delete operations + // First create some test data for delete operations + for i := 0; i < 5; i++ { + assistant := map[string]interface{}{ + "name": fmt.Sprintf("Delete Test Assistant %d", i), + "type": "assistant", + "connector": "openai", + "description": fmt.Sprintf("Delete Test Description %d", i), + "tags": []string{"delete-tag1", "delete-tag2"}, + "built_in": i%2 == 0, + "mentionable": true, + "automated": true, + } + _, err = store.SaveAssistant(assistant) + assert.Nil(t, err) + } + + // Test delete by connector + count, err := store.DeleteAssistants(AssistantFilter{ + Connector: "openai", + }) + assert.Nil(t, err) + assert.Greater(t, count, int64(0)) + + // Verify deletion + resp, err = store.GetAssistants(AssistantFilter{ + Connector: "openai", + }) + assert.Nil(t, err) + assert.Equal(t, 0, len(resp.Data)) + + // Create more test data for built_in test + for i := 0; i < 5; i++ { + assistant := map[string]interface{}{ + "name": fmt.Sprintf("Built-in Test Assistant %d", i), + "type": "assistant", + "connector": "openai", + "description": fmt.Sprintf("Built-in Test Description %d", i), + "tags": []string{"builtin-tag1", "builtin-tag2"}, + "built_in": true, + "mentionable": true, + "automated": true, + } + _, err = store.SaveAssistant(assistant) + assert.Nil(t, err) + } + + // Test delete by built_in status + builtInTrue = true + count, err = store.DeleteAssistants(AssistantFilter{ + BuiltIn: &builtInTrue, + }) + assert.Nil(t, err) + assert.Greater(t, count, int64(0)) + + // Verify deletion + resp, err = store.GetAssistants(AssistantFilter{ + BuiltIn: &builtInTrue, + }) + assert.Nil(t, err) + assert.Equal(t, 0, len(resp.Data)) + + // Create more test data for tags test + for i := 0; i < 5; i++ { + assistant := map[string]interface{}{ + "name": fmt.Sprintf("Tags Test Assistant %d", i), + "type": "assistant", + "connector": "openai", + "description": fmt.Sprintf("Tags Test Description %d", i), + "tags": []string{"tag1", "tag2"}, + "built_in": false, + "mentionable": true, + "automated": true, + } + _, err = store.SaveAssistant(assistant) + assert.Nil(t, err) + } + + // Test delete by tags + count, err = store.DeleteAssistants(AssistantFilter{ + Tags: []string{"tag1"}, + }) + assert.Nil(t, err) + assert.Greater(t, count, int64(0)) + + // Verify deletion + resp, err = store.GetAssistants(AssistantFilter{ + Tags: []string{"tag1"}, + }) + assert.Nil(t, err) + assert.Equal(t, 0, len(resp.Data)) + + // Create more test data for keywords test + for i := 0; i < 5; i++ { + assistant := map[string]interface{}{ + "name": fmt.Sprintf("Keywords Test Assistant %d", i), + "type": "assistant", + "connector": "openai", + "description": fmt.Sprintf("Keywords Test Description %d", i), + "tags": []string{"keyword-tag1", "keyword-tag2"}, + "built_in": false, + "mentionable": true, + "automated": true, + } + _, err = store.SaveAssistant(assistant) + assert.Nil(t, err) + } + + // Test delete by keywords + count, err = store.DeleteAssistants(AssistantFilter{ + Keywords: "Keywords Test", + }) + assert.Nil(t, err) + assert.Greater(t, count, int64(0)) + + // Verify all assistants are deleted resp, err = store.GetAssistants(AssistantFilter{}) assert.Nil(t, err) assert.Equal(t, 0, len(resp.Data)) @@ -725,6 +955,9 @@ func TestXunAssistantPagination(t *testing.T) { "connector": fmt.Sprintf("connector%d", i%3), "description": fmt.Sprintf("Description %d", i), "tags": tagsJSON, + "sort": 9999 - i, + "updated_at": time.Now().Add(time.Duration(-i) * time.Hour), + "built_in": i%2 == 0, "mentionable": mentionable, "automated": automated, } @@ -744,6 +977,25 @@ func TestXunAssistantPagination(t *testing.T) { assert.Equal(t, 2, resp.Next) assert.Equal(t, 0, resp.Prev) + // Verify sorting order (sort ASC, updated_at DESC) + for i := 1; i < len(resp.Data); i++ { + curr := resp.Data[i]["sort"].(int64) + prev := resp.Data[i-1]["sort"].(int64) + assert.True(t, curr >= prev, "Results should be sorted by sort ASC") + + // When sort values are equal, check updated_at if both values exist + if curr == prev { + currTime, currOk := resp.Data[i]["updated_at"].(time.Time) + prevTime, prevOk := resp.Data[i-1]["updated_at"].(time.Time) + + // Only compare times if both values exist + if currOk && prevOk { + assert.True(t, currTime.Before(prevTime) || currTime.Equal(prevTime), + "Results with same sort should be ordered by updated_at DESC") + } + } + } + // Test second page resp, err = store.GetAssistants(AssistantFilter{ Page: 2, @@ -811,7 +1063,30 @@ func TestXunAssistantPagination(t *testing.T) { assert.Nil(t, err) assert.Greater(t, len(resp.Data), 0) - // Test filtering by assistant_id + // Test filtering with built_in + builtInTrue := true + resp, err = store.GetAssistants(AssistantFilter{ + BuiltIn: &builtInTrue, + Page: 1, + PageSize: 10, + }) + assert.Nil(t, err) + for _, assistant := range resp.Data { + assert.Equal(t, int64(1), assistant["built_in"], "All assistants should be built-in") + } + + builtInFalse := false + resp, err = store.GetAssistants(AssistantFilter{ + BuiltIn: &builtInFalse, + Page: 1, + PageSize: 10, + }) + assert.Nil(t, err) + for _, assistant := range resp.Data { + assert.Equal(t, int64(0), assistant["built_in"], "All assistants should not be built-in") + } + + // Test assistant_id with other filters // First get an assistant_id from previous results firstAssistantID := resp.Data[0]["assistant_id"].(string) @@ -851,18 +1126,6 @@ func TestXunAssistantPagination(t *testing.T) { assert.Nil(t, err) assert.Equal(t, 0, len(resp.Data)) - // Test combined filters - resp, err = store.GetAssistants(AssistantFilter{ - Tags: []string{"tag0"}, - Keywords: "Assistant", - Connector: "connector0", - Mentionable: &mentionableTrue, - Automated: &automatedTrue, - Page: 1, - PageSize: 10, - }) - assert.Nil(t, err) - // Test filtering with select fields resp, err = store.GetAssistants(AssistantFilter{ Select: []string{"name", "description", "tags"}, @@ -909,4 +1172,152 @@ func TestXunAssistantPagination(t *testing.T) { assert.NotContains(t, item, "functions") assert.NotContains(t, item, "permissions") } + + // Test combined filters + resp, err = store.GetAssistants(AssistantFilter{ + Tags: []string{"tag0"}, + Keywords: "Assistant", + Connector: "connector0", + Mentionable: &mentionableTrue, + Automated: &automatedTrue, + Page: 1, + PageSize: 10, + }) + assert.Nil(t, err) + + // Now test the delete operations + // Test delete by connector + count, err := store.DeleteAssistants(AssistantFilter{ + Connector: "connector0", + }) + assert.Nil(t, err) + assert.Greater(t, count, int64(0)) + + // Verify deletion + resp, err = store.GetAssistants(AssistantFilter{ + Connector: "connector0", + }) + assert.Nil(t, err) + assert.Equal(t, 0, len(resp.Data)) + + // Test delete by built_in status + builtInTrue = true + count, err = store.DeleteAssistants(AssistantFilter{ + BuiltIn: &builtInTrue, + }) + assert.Nil(t, err) + assert.Greater(t, count, int64(0)) + + // Verify deletion + resp, err = store.GetAssistants(AssistantFilter{ + BuiltIn: &builtInTrue, + }) + assert.Nil(t, err) + assert.Equal(t, 0, len(resp.Data)) + + // Test delete by tags + count, err = store.DeleteAssistants(AssistantFilter{ + Tags: []string{"tag1"}, + }) + assert.Nil(t, err) + assert.Greater(t, count, int64(0)) + + // Verify deletion + resp, err = store.GetAssistants(AssistantFilter{ + Tags: []string{"tag1"}, + }) + assert.Nil(t, err) + assert.Equal(t, 0, len(resp.Data)) + + // Test delete by keywords + count, err = store.DeleteAssistants(AssistantFilter{ + Keywords: "Assistant", + }) + assert.Nil(t, err) + assert.Greater(t, count, int64(0)) + + // Verify all assistants are deleted + resp, err = store.GetAssistants(AssistantFilter{}) + assert.Nil(t, err) + assert.Equal(t, 0, len(resp.Data)) +} + +func TestGetAssistantTags(t *testing.T) { + + test.Prepare(t, config.Conf) + defer test.Clean() + defer capsule.Schema().DropTableIfExists("__unit_test_conversation_assistant") + + store, err := NewXun(Setting{ + Connector: "default", + Table: "__unit_test_conversation", + }) + if err != nil { + t.Fatal(err) + } + + // Create test assistants with tags + assistants := []map[string]interface{}{ + { + "assistant_id": "test-assistant-1", + "type": "assistant", + "connector": "test", + "tags": []string{"tag1", "tag2"}, + "name": "Test Assistant 1", + }, + { + "assistant_id": "test-assistant-2", + "type": "assistant", + "connector": "test", + "tags": []string{"tag2", "tag3"}, + "name": "Test Assistant 2", + }, + { + "assistant_id": "test-assistant-3", + "type": "assistant", + "connector": "test", + "tags": []string{"tag1", "tag3", "tag4"}, + "name": "Test Assistant 3", + }, + } + + // Save test assistants + for _, assistant := range assistants { + _, err := store.SaveAssistant(assistant) + if err != nil { + t.Fatal(err) + } + } + + // Get tags + tags, err := store.GetAssistantTags() + if err != nil { + t.Fatal(err) + } + + // Verify results + expectedTags := map[string]bool{ + "tag1": true, + "tag2": true, + "tag3": true, + "tag4": true, + } + + if len(tags) != len(expectedTags) { + t.Errorf("Expected %d tags, got %d", len(expectedTags), len(tags)) + } + + for _, tag := range tags { + if !expectedTags[tag] { + t.Errorf("Unexpected tag found: %s", tag) + } + } + + // Cleanup + for _, assistant := range assistants { + err := store.DeleteAssistant(assistant["assistant_id"].(string)) + if err != nil { + t.Fatal(err) + } + } } From 8d320f0e410f95d9f84539e8b6e8a8a41768fda1 Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 1 Jan 2025 16:55:11 +0800 Subject: [PATCH 5/8] Enhance assistant struct with tags and options handling in Neo API - Added support for initializing assistant tags with a default value of "Built-in" if none are provided, improving data consistency. - Implemented loading of tags, options, and description from the input data map, enhancing the flexibility of assistant configuration. - Updated the LoadBuiltIn and loadMap functions to accommodate new fields, streamlining the assistant loading process. --- neo/assistant/assistant.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/neo/assistant/assistant.go b/neo/assistant/assistant.go index 1a85867a..9f2f92ad 100644 --- a/neo/assistant/assistant.go +++ b/neo/assistant/assistant.go @@ -61,6 +61,10 @@ func LoadBuiltIn() error { assistant.Readonly = true assistant.BuiltIn = true assistant.Sort = sort + if assistant.Tags == nil { + assistant.Tags = []string{"Built-in"} + } + sort++ loaded.Put(assistant) @@ -238,6 +242,21 @@ func loadMap(data map[string]interface{}) (*Assistant, error) { assistant.Connector = connector } + // tags + if v, ok := data["tags"].([]string); ok { + assistant.Tags = v + } + + // options + if v, ok := data["options"].(map[string]interface{}); ok { + assistant.Options = v + } + + // description + if v, ok := data["description"].(string); ok { + assistant.Description = v + } + // prompts if v, ok := data["prompts"].(string); ok { var prompts []Prompt From fd5d701a23221f08ce103a400321aef4dd652426 Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 1 Jan 2025 17:34:17 +0800 Subject: [PATCH 6/8] Refactor assistant loading and initialization in Neo API - Simplified the assistant loading process by removing the asynchronous query for the assistant list and replacing it with a direct call to retrieve the default assistant. - Introduced a new method, defaultAssistant, to streamline the retrieval of the default assistant based on the current configuration. - Enhanced the LoadStore function to support loading assistants from a specified path, improving flexibility in assistant management. - Updated the overall structure for better readability and maintainability, ensuring a more efficient assistant initialization process. --- neo/assistant/api.go | 38 ++++++++++++++++++++++++++++++++++++++ neo/assistant/assistant.go | 11 +++++++++++ neo/load.go | 34 +++++----------------------------- neo/neo.go | 14 ++++++++++++++ 4 files changed, 68 insertions(+), 29 deletions(-) create mode 100644 neo/assistant/api.go diff --git a/neo/assistant/api.go b/neo/assistant/api.go new file mode 100644 index 00000000..69c22a00 --- /dev/null +++ b/neo/assistant/api.go @@ -0,0 +1,38 @@ +package assistant + +// Get get the assistant by id +func Get(id string) (*Assistant, error) { + return LoadStore(id) +} + +// GetByConnector get the assistant by connector +func GetByConnector(connector string, name string) (*Assistant, error) { + id := "connector:" + connector + + assistant, exists := loaded.Get(id) + if exists { + return assistant, nil + } + + data := map[string]interface{}{ + "assistant_id": id, + "connector": connector, + "description": "Default assistant for " + connector, + "name": name, + "type": "assistant", + } + + assistant, err := loadMap(data) + if err != nil { + return nil, err + + } + loaded.Put(assistant) + return assistant, nil +} + +// Init init the assistant +// Choose the connector and initialize the assistant +func (ast *Assistant) initialize() error { + return nil +} diff --git a/neo/assistant/assistant.go b/neo/assistant/assistant.go index 9f2f92ad..ee57d890 100644 --- a/neo/assistant/assistant.go +++ b/neo/assistant/assistant.go @@ -115,6 +115,17 @@ func LoadStore(id string) (*Assistant, error) { return nil, err } + // Load from path + if data["path"] != nil { + assistant, err = LoadPath(data["path"].(string)) + if err != nil { + return nil, err + } + loaded.Put(assistant) + return assistant, nil + } + + // Load from store assistant, err = loadMap(data) if err != nil { return nil, err diff --git a/neo/load.go b/neo/load.go index 6b87957f..53dfe9a6 100644 --- a/neo/load.go +++ b/neo/load.go @@ -1,10 +1,7 @@ package neo import ( - "context" - "fmt" "path/filepath" - "time" "github.com/yaoapp/gou/application" "github.com/yaoapp/yao/config" @@ -58,32 +55,11 @@ func Load(cfg config.Config) error { return err } - // Query Assistant List - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - listDone := make(chan error, 1) - go func() { - list, err := Neo.HookAssistants(ctx, assistant.QueryParam{Limit: 100}) - Neo.updateAssistantList(list) - listDone <- err - }() - - select { - case err := <-listDone: - if err != nil { - return fmt.Errorf("Neo assistant list failed: %w", err) - } - - // Create Default Assistant - Neo.Assistant, err = Neo.createDefaultAssistant() - if err != nil { - return err - } - - return nil - case <-ctx.Done(): - return fmt.Errorf("Neo assistant list timeout: %w", ctx.Err()) + defaultAssistant, err := Neo.defaultAssistant() + if err != nil { + return err } + Neo.Assistant = defaultAssistant.API + return nil } diff --git a/neo/neo.go b/neo/neo.go index f072dc1c..e5cf0399 100644 --- a/neo/neo.go +++ b/neo/neo.go @@ -346,6 +346,20 @@ func (neo *DSL) chat(ast assistant.API, ctx Context, messages []map[string]inter } } +// defaultAssistant get the default assistant +func (neo *DSL) defaultAssistant() (*assistant.Assistant, error) { + if neo.Use != "" { + return assistant.Get(neo.Use) + } + + name := neo.Name + if name == "" { + name = "Neo" + } + + return assistant.GetByConnector(neo.Connector, name) +} + // updateAssistantList update the assistant list func (neo *DSL) updateAssistantList(list []assistant.Assistant) { lock.Lock() From 0d05842c28dd9af283cde6bd3fe467670c16000e Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 2 Jan 2025 11:07:04 +0800 Subject: [PATCH 7/8] Update Go module dependencies and version in go.mod and go.sum - Bump Go version from 1.20 to 1.22.2 for improved performance and features. - Update indirect dependencies to their latest versions, including: - github.com/cespare/xxhash/v2 from v2.2.0 to v2.3.0 - github.com/golang/protobuf from v1.5.3 to v1.5.4 - github.com/klauspost/compress from v1.17.3 to v1.17.4 - golang.org/x/oauth2 from v0.14.0 to v0.23.0 - google.golang.org/grpc from v1.60.1 to v1.69.2 - google.golang.org/protobuf from v1.34.2 to v1.36.1 - Add new dependencies for OpenTelemetry packages, enhancing observability capabilities. --- go.mod | 17 ++++++++--------- go.sum | 60 ++++++++++++++++++++++++++++++++++++++-------------------- 2 files changed, 47 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index 96395538..2bce29e0 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/yaoapp/yao -go 1.20 +go 1.22.2 require ( github.com/PuerkitoBio/goquery v1.9.2 @@ -40,7 +40,7 @@ require ( github.com/blang/semver/v4 v4.0.0 // indirect github.com/bytedance/sonic v1.11.9 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -56,7 +56,7 @@ require ( github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect github.com/go-sql-driver/mysql v1.7.1 // indirect github.com/goccy/go-json v0.10.3 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -69,7 +69,7 @@ require ( github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmoiron/sqlx v1.3.5 // indirect - github.com/klauspost/compress v1.17.3 // indirect + github.com/klauspost/compress v1.17.4 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lib/pq v1.10.9 // indirect @@ -112,14 +112,13 @@ require ( golang.org/x/arch v0.8.0 // indirect golang.org/x/image v0.18.0 // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/oauth2 v0.14.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 // indirect - google.golang.org/grpc v1.60.1 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241230172942-26aa7a208def // indirect + google.golang.org/grpc v1.69.2 // indirect + google.golang.org/protobuf v1.36.1 // indirect ) // go env -w GOPRIVATE=github.com/yaoapp/* diff --git a/go.sum b/go.sum index fc31d260..69ac784d 100644 --- a/go.sum +++ b/go.sum @@ -9,14 +9,15 @@ github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= +github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/bytedance/sonic v1.11.9 h1:LFHENlIY/SLzDWverzdOvgMztTxcfcF+cqNsz9pK5zg= github.com/bytedance/sonic v1.11.9/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/caarlos0/env/v6 v6.10.1 h1:t1mPSxNpei6M5yAeu1qtRdPAK29Nbcf/n3G7x+b3/II= github.com/caarlos0/env/v6 v6.10.1/go.mod h1:hvp/ryKXKipEkcuYjs9mI4bBCg+UI0Yhgm5Zu0ddvwc= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= @@ -51,7 +52,12 @@ github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= @@ -71,16 +77,14 @@ github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keL github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -105,12 +109,14 @@ github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519 h1:nqAlWFEdqI0ClbTDrhDvE/8LeQ4pftrqKUX9w5k0j3s= +github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8= github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= +github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= @@ -118,8 +124,8 @@ github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwA github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA= -github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -161,12 +167,15 @@ github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJ github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkoukk/tiktoken-go v0.1.7 h1:qOBHXX4PHtvIvmOtyg1EeKlwFRiMKAcoMp4Q+bLQDmw= @@ -205,6 +214,7 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8 github.com/tcnksm/go-gitconfig v0.1.2 h1:iiDhRitByXAEyjgBqsKi9QU4o2TNtv9kPP3RgPgXBPw= github.com/tcnksm/go-gitconfig v0.1.2/go.mod h1:/8EhP4H7oJZdIPyT+/UIsG87kTzrzM4UsLGSItWYCpE= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= +github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= github.com/tidwall/buntdb v1.3.0 h1:gdhWO+/YwoB2qZMeAU9JcWWsHSYU3OvcieYgFRS0zwA= @@ -215,6 +225,7 @@ github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vl github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= +github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -251,6 +262,16 @@ github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mongodb.org/mongo-driver v1.13.0 h1:67DgFFjYOCMWdtTEmKFpV3ffWlFnh+CYZ8ZS/tXWUfY= go.mongodb.org/mongo-driver v1.13.0/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= +go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= @@ -285,8 +306,8 @@ golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.14.0 h1:P0Vrf/2538nmC0H+pEQ3MNFRRnVR7RlqyVw+bvm26z0= -golang.org/x/oauth2 v0.14.0/go.mod h1:lAtNWgaWfL4cm7j2OV8TxGi9Qb7ECORx8DktCY74OwM= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -344,16 +365,12 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 h1:/jFB8jK5R3Sq3i/lmeZO0cATSzFfZaJq1J2Euan3XKU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0/go.mod h1:FUoWkonphQm3RhTS+kOEhF8h0iDpm4tdXolVCeZ9KKA= -google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= -google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241230172942-26aa7a208def h1:4P81qv5JXI/sDNae2ClVx88cgDDA6DPilADkG9tYKz8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241230172942-26aa7a208def/go.mod h1:bdAgzvd4kFrpykc5/AC2eLUiegK9T/qxZHD4hXYf/ho= +google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= +google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= +google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -364,6 +381,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From fdf71f4cfdb91bbb3f8fe244b71e0b2e04d3fe3d Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 2 Jan 2025 13:06:36 +0800 Subject: [PATCH 8/8] Update Go version and dependencies; enhance Neo API functionality - Bump Go version from 1.22.2 to 1.23 for improved performance. - Update dependencies in go.mod and go.sum, including: - github.com/PuerkitoBio/goquery to v1.10.1 - github.com/dchest/captcha to v1.1.0 - github.com/evanw/esbuild to v0.24.2 - github.com/fatih/color to v1.18.0 - github.com/fsnotify/fsnotify to v1.8.0 - github.com/spf13/cobra to v1.8.1 - github.com/stretchr/testify to v1.10.0 - github.com/xuri/excelize/v2 to v2.9.0 - Introduce RAG (Retrieval-Augmented Generation) initialization in the Neo API, enhancing assistant capabilities. - Refactor workflows to use Go 1.23 for testing and CI processes, ensuring compatibility with the latest features. --- .github/workflows/pr-test.yml | 4 +- .github/workflows/unit-test.yml | 3 +- go.mod | 83 ++++++------ go.sum | 216 ++++++++++++++++---------------- neo/load.go | 20 +++ neo/rag/rag.go | 120 ++++++++++++++++++ neo/rag/types.go | 29 +++++ neo/types.go | 3 + 8 files changed, 328 insertions(+), 150 deletions(-) create mode 100644 neo/rag/rag.go create mode 100644 neo/rag/types.go diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 7abd9ae9..175cf274 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -69,7 +69,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - go: [1.20.0, 1.21.1] + go: [1.23] db: [MySQL8.0, MySQL5.7, SQLite3] redis: [4, 5, 6] mongo: ["6.0"] @@ -201,7 +201,7 @@ jobs: uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - + - name: Start MongoDB uses: supercharge/mongodb-github-action@1.8.0 with: diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 56f54ae0..1b35bcc1 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -73,12 +73,11 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - go: [1.20.0, 1.21.1] + go: [1.23] db: [MySQL8.0, MySQL5.7, SQLite3] redis: [4, 5, 6] mongo: ["6.0"] steps: - - name: Checkout Kun uses: actions/checkout@v4 with: diff --git a/go.mod b/go.mod index 2bce29e0..e99535e3 100644 --- a/go.mod +++ b/go.mod @@ -1,17 +1,17 @@ module github.com/yaoapp/yao -go 1.22.2 +go 1.23 require ( - github.com/PuerkitoBio/goquery v1.9.2 + github.com/PuerkitoBio/goquery v1.10.1 github.com/blang/semver v3.5.1+incompatible github.com/caarlos0/env/v6 v6.10.1 - github.com/dchest/captcha v1.0.0 + github.com/dchest/captcha v1.1.0 github.com/elazarl/go-bindata-assetfs v1.0.1 - github.com/evanw/esbuild v0.19.5 + github.com/evanw/esbuild v0.24.2 github.com/expr-lang/expr v1.16.9 - github.com/fatih/color v1.16.0 - github.com/fsnotify/fsnotify v1.7.0 + github.com/fatih/color v1.18.0 + github.com/fsnotify/fsnotify v1.8.0 github.com/gin-gonic/gin v1.10.0 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/google/uuid v1.6.0 @@ -20,9 +20,9 @@ require ( github.com/json-iterator/go v1.1.12 github.com/pkoukk/tiktoken-go v0.1.7 github.com/rhysd/go-github-selfupdate v1.2.3 - github.com/spf13/cobra v1.8.0 - github.com/stretchr/testify v1.9.0 - github.com/xuri/excelize/v2 v2.8.0 + github.com/spf13/cobra v1.8.1 + github.com/stretchr/testify v1.10.0 + github.com/xuri/excelize/v2 v2.9.0 github.com/yaoapp/gou v0.10.3 github.com/yaoapp/kun v0.9.0 github.com/yaoapp/xun v0.9.0 @@ -35,65 +35,66 @@ require ( ) require ( + filippo.io/edwards25519 v1.1.0 // indirect github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 // indirect - github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/andybalholm/cascadia v1.3.3 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/bytedance/sonic v1.11.9 // indirect - github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/bytedance/sonic v1.12.6 // indirect + github.com/bytedance/sonic/loader v0.2.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dlclark/regexp2 v1.11.4 // indirect - github.com/gabriel-vasile/mimetype v1.4.4 // indirect - github.com/gin-contrib/sse v0.1.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.7 // indirect + github.com/gin-contrib/sse v1.0.0 // indirect github.com/go-errors/errors v1.5.1 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.22.0 // indirect + github.com/go-playground/validator/v10 v10.23.0 // indirect github.com/go-redis/redis/v8 v8.11.5 // indirect github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect - github.com/go-sql-driver/mysql v1.7.1 // indirect - github.com/goccy/go-json v0.10.3 // indirect + github.com/go-sql-driver/mysql v1.8.1 // indirect + github.com/goccy/go-json v0.10.4 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/gorilla/websocket v1.5.1 // indirect + github.com/gorilla/websocket v1.5.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-hclog v1.5.0 // indirect - github.com/hashicorp/go-plugin v1.6.0 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect + github.com/hashicorp/go-plugin v1.6.2 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect - github.com/hashicorp/yamux v0.1.1 // indirect + github.com/hashicorp/yamux v0.1.2 // indirect github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/jmoiron/sqlx v1.3.5 // indirect - github.com/klauspost/compress v1.17.4 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/jmoiron/sqlx v1.4.0 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lib/pq v1.10.9 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-sqlite3 v1.14.18 // indirect - github.com/miekg/dns v1.1.57 // indirect - github.com/mitchellh/go-testing-interface v1.14.1 // indirect + github.com/mattn/go-sqlite3 v1.14.24 // indirect + github.com/miekg/dns v1.1.62 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/montanaflynn/stats v0.7.1 // indirect github.com/oklog/run v1.1.0 // indirect - github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/qdrant/go-client v1.12.0 // indirect github.com/richardlehane/mscfb v1.0.4 // indirect - github.com/richardlehane/msoleps v1.0.3 // indirect + github.com/richardlehane/msoleps v1.0.4 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tcnksm/go-gitconfig v0.1.2 // indirect github.com/tidwall/btree v1.7.0 // indirect - github.com/tidwall/buntdb v1.3.0 // indirect - github.com/tidwall/gjson v1.17.0 // indirect + github.com/tidwall/buntdb v1.3.2 // indirect + github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/grect v0.1.4 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect @@ -101,21 +102,21 @@ require ( github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect - github.com/ulikunitz/xz v0.5.11 // indirect + github.com/ulikunitz/xz v0.5.12 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53 // indirect - github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 // indirect - github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect - go.mongodb.org/mongo-driver v1.13.0 // indirect - golang.org/x/arch v0.8.0 // indirect - golang.org/x/image v0.18.0 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/oauth2 v0.23.0 // indirect + github.com/xuri/efp v0.0.0-20241211021726-c4e992084aa6 // indirect + github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + go.mongodb.org/mongo-driver v1.17.1 // indirect + golang.org/x/arch v0.12.0 // indirect + golang.org/x/image v0.23.0 // indirect + golang.org/x/mod v0.22.0 // indirect + golang.org/x/oauth2 v0.24.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/tools v0.28.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241230172942-26aa7a208def // indirect google.golang.org/grpc v1.69.2 // indirect google.golang.org/protobuf v1.36.1 // indirect diff --git a/go.sum b/go.sum index 69ac784d..a9b45ed1 100644 --- a/go.sum +++ b/go.sum @@ -1,19 +1,22 @@ -github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= -github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/PuerkitoBio/goquery v1.10.1 h1:Y8JGYUkXWTGRB6Ars3+j3kN0xg1YqqlwvdTV8WTFQcU= +github.com/PuerkitoBio/goquery v1.10.1/go.mod h1:IYiHrOMps66ag56LEH7QYDDupKXyo5A8qrjIx3ZtujY= github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 h1:ZBbLwSJqkHBuFDA6DUhhse0IGJ7T5bemHyNILUjvOq4= github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2/go.mod h1:VSw57q4QFiWDbRnjdX8Cb3Ow0SFncRw+bA/ofY6Q83w= -github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= -github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= -github.com/bytedance/sonic v1.11.9 h1:LFHENlIY/SLzDWverzdOvgMztTxcfcF+cqNsz9pK5zg= -github.com/bytedance/sonic v1.11.9/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= -github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic v1.12.6 h1:/isNmCUF2x3Sh8RAp/4mh4ZGkcFAX/hLrzrK3AvpRzk= +github.com/bytedance/sonic v1.12.6/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.1 h1:1GgorWTqf12TA8mma4DDSbaQigE2wOgQo7iCjjJv3+E= +github.com/bytedance/sonic/loader v0.2.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/caarlos0/env/v6 v6.10.1 h1:t1mPSxNpei6M5yAeu1qtRdPAK29Nbcf/n3G7x+b3/II= github.com/caarlos0/env/v6 v6.10.1/go.mod h1:hvp/ryKXKipEkcuYjs9mI4bBCg+UI0Yhgm5Zu0ddvwc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -22,32 +25,32 @@ github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/ github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o= -github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo= +github.com/dchest/captcha v1.1.0 h1:2kt47EoYUUkaISobUdTbqwx55xvKOJxyScVfw25xzhQ= +github.com/dchest/captcha v1.1.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/elazarl/go-bindata-assetfs v1.0.1 h1:m0kkaHRKEu7tUIUFVwhGGGYClXvyl4RE03qmvRTNfbw= github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= -github.com/evanw/esbuild v0.19.5 h1:9ildZqajUJzDAwNf9MyQsLh2RdDRKTq3kcyyzhE39us= -github.com/evanw/esbuild v0.19.5/go.mod h1:D2vIQZqV/vIf/VRHtViaUtViZmG7o+kKmlBfVQuRi48= +github.com/evanw/esbuild v0.24.2 h1:PQExybVBrjHjN6/JJiShRGIXh1hWVm6NepVnhZhrt0A= +github.com/evanw/esbuild v0.24.2/go.mod h1:D2vIQZqV/vIf/VRHtViaUtViZmG7o+kKmlBfVQuRi48= github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI= github.com/expr-lang/expr v1.16.9/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I= -github.com/gabriel-vasile/mimetype v1.4.4/go.mod h1:JwLei5XPtWdGiMFB5Pjle1oEeoSeEuJfJE+TtfvdB/s= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= +github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU= +github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E= +github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= @@ -62,24 +65,22 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.22.0 h1:k6HsTZ0sTnROkhS//R0O+55JgM8C4Bx7ia+JlgcnOao= -github.com/go-playground/validator/v10 v10.22.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o= +github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q= github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= -github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM= +github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -93,21 +94,21 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= -github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-plugin v1.6.0 h1:wgd4KxHJTVGGqWBq4QPB1i5BZNEx9BR8+OFmHDmTk8A= -github.com/hashicorp/go-plugin v1.6.0/go.mod h1:lBS5MtSSBZk0SHc66KACcjjlU6WzEVP/8pwz68aMkCI= +github.com/hashicorp/go-plugin v1.6.2 h1:zdGAEd0V1lCaU0u+MxWQhtSDQmahpkwOun8U8EiRVog= +github.com/hashicorp/go-plugin v1.6.2/go.mod h1:CkgLQ5CZqNmdL9U9JzM532t8ZiYQ35+pj3b1FD37R0Q= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= -github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= +github.com/hashicorp/yamux v0.1.2 h1:XtB8kyFOyHXYVFnwT5C3+Bdo8gArse7j2AQ0DA0Uey8= +github.com/hashicorp/yamux v0.1.2/go.mod h1:C+zze2n6e/7wshOZep2A70/aQU6QBRWJO/G6FT1wIns= github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519 h1:nqAlWFEdqI0ClbTDrhDvE/8LeQ4pftrqKUX9w5k0j3s= github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -117,27 +118,26 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= -github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= -github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -149,13 +149,11 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI= -github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= -github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= -github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= -github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= +github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -163,7 +161,6 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -176,26 +173,30 @@ github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= -github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= -github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pkoukk/tiktoken-go v0.1.7 h1:qOBHXX4PHtvIvmOtyg1EeKlwFRiMKAcoMp4Q+bLQDmw= github.com/pkoukk/tiktoken-go v0.1.7/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/qdrant/go-client v1.12.0 h1:KqsIKDAw5iQmxDzRjbzRjhvQ+Igyr7Y84vDCinf1T4M= +github.com/qdrant/go-client v1.12.0/go.mod h1:zFa6t5Y3Oqecoa0aSsGWhMqQWq3x3kTPvm0sMf5qplw= github.com/rhysd/go-github-selfupdate v1.2.3 h1:iaa+J202f+Nc+A8zi75uccC8Wg3omaM7HDeimXA22Ag= github.com/rhysd/go-github-selfupdate v1.2.3/go.mod h1:mp/N8zj6jFfBQy/XMYoWsmfzxazpPAODuqarmPDe2Rg= github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= -github.com/richardlehane/msoleps v1.0.3 h1:aznSZzrwYRl3rLKRT3gUk9am7T/mLNSnJINvN0AQoVM= -github.com/richardlehane/msoleps v1.0.3/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00= +github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -209,19 +210,19 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tcnksm/go-gitconfig v0.1.2 h1:iiDhRitByXAEyjgBqsKi9QU4o2TNtv9kPP3RgPgXBPw= github.com/tcnksm/go-gitconfig v0.1.2/go.mod h1:/8EhP4H7oJZdIPyT+/UIsG87kTzrzM4UsLGSItWYCpE= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= -github.com/tidwall/buntdb v1.3.0 h1:gdhWO+/YwoB2qZMeAU9JcWWsHSYU3OvcieYgFRS0zwA= -github.com/tidwall/buntdb v1.3.0/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= +github.com/tidwall/buntdb v1.3.2 h1:qd+IpdEGs0pZci37G4jF51+fSKlkuUTMXuHhXL1AkKg= +github.com/tidwall/buntdb v1.3.2/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= -github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= @@ -240,28 +241,25 @@ github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2 github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= -github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= -github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53 h1:Chd9DkqERQQuHpXjR/HSV1jLZA6uaoiwwH3vSuF3IW0= -github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= -github.com/xuri/excelize/v2 v2.8.0 h1:Vd4Qy809fupgp1v7X+nCS/MioeQmYVVzi495UCTqB7U= -github.com/xuri/excelize/v2 v2.8.0/go.mod h1:6iA2edBTKxKbZAa7X5bDhcCg51xdOn1Ar5sfoXRGrQg= -github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= -github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 h1:qhbILQo1K3mphbwKh1vNm4oGezE1eF9fQWmNiIpSfI4= -github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= -github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= +github.com/xuri/efp v0.0.0-20241211021726-c4e992084aa6 h1:8m6DWBG+dlFNbx5ynvrE7NgI+Y7OlZVMVTpayoW+rCc= +github.com/xuri/efp v0.0.0-20241211021726-c4e992084aa6/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.9.0 h1:1tgOaEq92IOEumR1/JfYS/eR0KHOCsRv/rYXXh6YJQE= +github.com/xuri/excelize/v2 v2.9.0/go.mod h1:uqey4QBZ9gdMeWApPLdhm9x+9o2lq4iVmjiLfBS5hdE= +github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 h1:hPVCafDV85blFTabnqKgNhDCkJX25eik94Si9cTER4A= +github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.mongodb.org/mongo-driver v1.13.0 h1:67DgFFjYOCMWdtTEmKFpV3ffWlFnh+CYZ8ZS/tXWUfY= -go.mongodb.org/mongo-driver v1.13.0/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ= +go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM= +go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4= go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= @@ -272,56 +270,58 @@ go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4Jjx go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= -golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= -golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/arch v0.12.0 h1:UsYJhbzPYGsT0HbEdmYcqtCv8UNGvnaL561NnIUvaKg= +golang.org/x/arch v0.12.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8= -golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= -golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= +golang.org/x/image v0.23.0 h1:HseQ7c2OpPKTPVzNjG5fwJsOTCiiwS4QdsYi5XU6H68= +golang.org/x/image v0.23.0/go.mod h1:wJJBTdLfCCf3tiHa1fNxpZmUI4mmoZvwMCPP0ddoNKY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= -golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= +golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -332,35 +332,42 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= +golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -386,4 +393,3 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/neo/load.go b/neo/load.go index 53dfe9a6..ef45f86a 100644 --- a/neo/load.go +++ b/neo/load.go @@ -3,15 +3,32 @@ package neo import ( "path/filepath" + "github.com/fatih/color" "github.com/yaoapp/gou/application" + "github.com/yaoapp/kun/log" "github.com/yaoapp/yao/config" "github.com/yaoapp/yao/neo/assistant" + "github.com/yaoapp/yao/neo/rag" "github.com/yaoapp/yao/neo/store" ) // Neo the neo AI assistant var Neo *DSL +// initRAG initialize the RAG instance +func (neo *DSL) initRAG() { + if neo.RAGSetting.Engine.Driver == "" { + return + } + instance, err := rag.New(neo.RAGSetting) + if err != nil { + color.Red("[Neo] Failed to initialize RAG: %v", err) + log.Error("[Neo] Failed to initialize RAG: %v", err) + return + } + neo.RAG = instance +} + // Load load AIGC func Load(cfg config.Config) error { @@ -48,6 +65,9 @@ func Load(cfg config.Config) error { return err } + // Initialize RAG + Neo.initRAG() + // Load Built-in Assistants assistant.SetStorage(Neo.Store) err = assistant.LoadBuiltIn() diff --git a/neo/rag/rag.go b/neo/rag/rag.go new file mode 100644 index 00000000..ed14fae3 --- /dev/null +++ b/neo/rag/rag.go @@ -0,0 +1,120 @@ +package rag + +import ( + "fmt" + "os" + "strings" + + "github.com/yaoapp/gou/rag" + "github.com/yaoapp/gou/rag/driver" +) + +// RAG the RAG instance +type RAG struct { + setting Setting + engine driver.Engine + vectorizer driver.Vectorizer + fileUpload driver.FileUpload +} + +// parseEnvValue parse environment variable if the value starts with $ENV. +func parseEnvValue(value string) string { + if strings.HasPrefix(value, "$ENV.") { + envKey := strings.TrimPrefix(value, "$ENV.") + if envVal := os.Getenv(envKey); envVal != "" { + return envVal + } + } + return value +} + +// convertOptions convert interface{} options map to string map and parse environment variables +func convertOptions(options map[string]interface{}) map[string]string { + converted := make(map[string]string) + for k, v := range options { + if str, ok := v.(string); ok { + converted[k] = parseEnvValue(str) + } + } + return converted +} + +// New create a new RAG instance +func New(setting Setting) (*RAG, error) { + if setting.Engine.Driver == "" { + return nil, fmt.Errorf("engine driver is required") + } + + if setting.Vectorizer.Driver == "" { + return nil, fmt.Errorf("vectorizer driver is required") + } + + // Set default values + if setting.Upload.ChunkSize == 0 { + setting.Upload.ChunkSize = 1024 + } + + if setting.Upload.ChunkOverlap == 0 { + setting.Upload.ChunkOverlap = 256 + } + + if setting.IndexPrefix == "" { + setting.IndexPrefix = "yao_neo_" + } + + // Convert options map for vectorizer and handle environment variables + vectorizerOpts := convertOptions(setting.Vectorizer.Options) + + // Create vectorizer + vectorizer, err := rag.NewVectorizer(setting.Vectorizer.Driver, driver.VectorizeConfig{ + Model: vectorizerOpts["model"], + Options: vectorizerOpts, + }) + if err != nil { + return nil, fmt.Errorf("create vectorizer: %v", err) + } + + // Convert options map for engine and handle environment variables + engineOpts := convertOptions(setting.Engine.Options) + + // Create engine + engine, err := rag.NewEngine(setting.Engine.Driver, driver.IndexConfig{ + Options: engineOpts, + }, vectorizer) + if err != nil { + return nil, fmt.Errorf("create engine: %v", err) + } + + // Create file upload + fileUpload, err := rag.NewFileUpload(setting.Engine.Driver, engine, vectorizer) + if err != nil { + return nil, fmt.Errorf("create file upload: %v", err) + } + + return &RAG{ + setting: setting, + engine: engine, + vectorizer: vectorizer, + fileUpload: fileUpload, + }, nil +} + +// Setting get the RAG settings +func (rag *RAG) Setting() Setting { + return rag.setting +} + +// Engine get the vector database engine +func (rag *RAG) Engine() driver.Engine { + return rag.engine +} + +// Vectorizer get the text vectorizer +func (rag *RAG) Vectorizer() driver.Vectorizer { + return rag.vectorizer +} + +// FileUpload get the file upload handler +func (rag *RAG) FileUpload() driver.FileUpload { + return rag.fileUpload +} diff --git a/neo/rag/types.go b/neo/rag/types.go new file mode 100644 index 00000000..2b4d6750 --- /dev/null +++ b/neo/rag/types.go @@ -0,0 +1,29 @@ +package rag + +// Setting RAG settings +type Setting struct { + Engine Engine `json:"engine" yaml:"engine"` + Vectorizer Vectorizer `json:"vectorizer" yaml:"vectorizer"` + Upload Upload `json:"upload" yaml:"upload"` + IndexPrefix string `json:"index_prefix" yaml:"index_prefix"` +} + +// Engine the vector database engine settings +type Engine struct { + Driver string `json:"driver" yaml:"driver"` + Options map[string]interface{} `json:"options" yaml:"options"` +} + +// Vectorizer the text vectorizer settings +type Vectorizer struct { + Driver string `json:"driver" yaml:"driver"` + Options map[string]interface{} `json:"options" yaml:"options"` +} + +// Upload the file upload settings +type Upload struct { + Async bool `json:"async" yaml:"async"` + AllowedTypes []string `json:"allowed_types" yaml:"allowed_types"` + ChunkSize int `json:"chunk_size" yaml:"chunk_size"` + ChunkOverlap int `json:"chunk_overlap" yaml:"chunk_overlap"` +} diff --git a/neo/types.go b/neo/types.go index bba705d0..a692fb98 100644 --- a/neo/types.go +++ b/neo/types.go @@ -5,6 +5,7 @@ import ( "github.com/gin-gonic/gin" "github.com/yaoapp/yao/neo/assistant" + "github.com/yaoapp/yao/neo/rag" "github.com/yaoapp/yao/neo/store" ) @@ -16,6 +17,7 @@ type DSL struct { Guard string `json:"guard,omitempty" yaml:"guard,omitempty"` Connector string `json:"connector" yaml:"connector"` StoreSetting store.Setting `json:"store" yaml:"store"` + RAGSetting rag.Setting `json:"rag" yaml:"rag"` Option map[string]interface{} `json:"option" yaml:"option"` Prepare string `json:"prepare,omitempty" yaml:"prepare,omitempty"` Create string `json:"create,omitempty" yaml:"create,omitempty"` @@ -26,6 +28,7 @@ type DSL struct { Allows []string `json:"allows,omitempty" yaml:"allows,omitempty"` Assistant assistant.API `json:"-" yaml:"-"` // The default assistant Store store.Store `json:"-" yaml:"-"` + RAG *rag.RAG `json:"-" yaml:"-"` GuardHandlers []gin.HandlerFunc `json:"-" yaml:"-"` AssistantList []assistant.Assistant `json:"-" yaml:"-"` AssistantMaps map[string]assistant.Assistant `json:"-" yaml:"-"`