From 55c0580fc38e64e2fe703cde72fb4d16da600e6e Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 3 Jan 2025 16:37:43 +0800 Subject: [PATCH] Refactor Neo API assistant management and remove obsolete functions - Removed unused assistant-related functions and types from the Neo API, including local chat and file handling, to streamline the codebase. - Refactored the assistant initialization process, consolidating the loading and configuration of assistants, including Vision and Connector settings. - Updated the DSL struct to enhance clarity and maintainability, ensuring better configuration management for assistants. - Improved error handling and modularity in the assistant loading process, paving the way for future enhancements in AI functionalities. --- neo/assistant/api.go | 238 ++++++++++++++++++++++++++++++++- neo/assistant/assistant.go | 2 +- neo/assistant/load.go | 49 +++++++ neo/assistant/local/chat.go | 21 --- neo/assistant/local/file.go | 162 ---------------------- neo/assistant/local/local.go | 34 ----- neo/assistant/openai/chat.go | 118 ---------------- neo/assistant/openai/file.go | 167 ----------------------- neo/assistant/openai/openai.go | 51 ------- neo/assistant/openai/thread.go | 21 --- neo/assistant/types.go | 3 +- neo/hooks.go | 103 -------------- neo/load.go | 10 +- neo/neo.go | 184 ++----------------------- neo/types.go | 42 +++--- 15 files changed, 324 insertions(+), 881 deletions(-) delete mode 100644 neo/assistant/local/chat.go delete mode 100644 neo/assistant/local/file.go delete mode 100644 neo/assistant/local/local.go delete mode 100644 neo/assistant/openai/chat.go delete mode 100644 neo/assistant/openai/file.go delete mode 100644 neo/assistant/openai/openai.go delete mode 100644 neo/assistant/openai/thread.go diff --git a/neo/assistant/api.go b/neo/assistant/api.go index 69c22a00..260ce88f 100644 --- a/neo/assistant/api.go +++ b/neo/assistant/api.go @@ -1,5 +1,20 @@ package assistant +import ( + "context" + "crypto/sha256" + "encoding/base64" + "fmt" + "io" + "mime/multipart" + "path/filepath" + "strings" + "time" + + "github.com/yaoapp/gou/fs" + chatMessage "github.com/yaoapp/yao/neo/message" +) + // Get get the assistant by id func Get(id string) (*Assistant, error) { return LoadStore(id) @@ -25,14 +40,229 @@ func GetByConnector(connector string, name string) (*Assistant, error) { assistant, err := loadMap(data) if err != nil { return nil, err - } loaded.Put(assistant) return assistant, nil } -// Init init the assistant -// Choose the connector and initialize the assistant -func (ast *Assistant) initialize() error { +// AllowedFileTypes the allowed file types +var AllowedFileTypes = map[string]string{ + "application/json": "json", + "application/pdf": "pdf", + "application/msword": "doc", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx", + "application/vnd.oasis.opendocument.text": "odt", + "application/vnd.ms-excel": "xls", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx", + "application/vnd.ms-powerpoint": "ppt", + "application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx", +} + +// MaxSize 20M max file size +var MaxSize int64 = 20 * 1024 * 1024 + +// Chat implements the chat functionality +func (ast *Assistant) Chat(ctx context.Context, messages []map[string]interface{}, option map[string]interface{}, cb func(data []byte) int) error { + if ast.openai == nil { + return fmt.Errorf("openai is not initialized") + } + + requestMessages, err := ast.requestMessages(ctx, messages) + if err != nil { + return fmt.Errorf("request messages error: %s", err.Error()) + } + + _, ext := ast.openai.ChatCompletionsWith(ctx, requestMessages, option, cb) + if ext != nil { + return fmt.Errorf("openai chat completions with error: %s", ext.Message) + } + return nil } + +func (ast *Assistant) requestMessages(ctx context.Context, messages []map[string]interface{}) ([]map[string]interface{}, error) { + newMessages := []map[string]interface{}{} + length := len(messages) + for index, message := range messages { + role, ok := message["role"].(string) + if !ok { + return nil, fmt.Errorf("role must be string") + } + + content, ok := message["content"].(string) + if !ok { + return nil, fmt.Errorf("content must be string") + } + + newMessage := map[string]interface{}{ + "role": role, + "content": content, + } + + if name, ok := message["name"].(string); ok { + newMessage["name"] = name + } + + // Special handling for user messages with JSON content last message + if role == "user" && index == length-1 { + content = strings.TrimSpace(content) + msg, err := chatMessage.NewString(content) + if err != nil { + return nil, fmt.Errorf("new string error: %s", err.Error()) + } + + newMessage["content"] = msg.Text + if msg.Attachments != nil { + content, err := ast.withAttachments(ctx, msg) + if err != nil { + return nil, fmt.Errorf("with attachments error: %s", err.Error()) + } + newMessage["content"] = content + } + } + + newMessages = append(newMessages, newMessage) + } + return newMessages, nil +} + +func (ast *Assistant) withAttachments(ctx context.Context, msg *chatMessage.Message) ([]map[string]interface{}, error) { + contents := []map[string]interface{}{{"type": "text", "text": msg.Text}} + images := []string{} + for _, attachment := range msg.Attachments { + if strings.HasPrefix(attachment.ContentType, "image/") { + images = append(images, attachment.FileID) + } + } + + if len(images) == 0 { + return contents, nil + } + + for _, image := range images { + bytes64, err := ast.ReadBase64(ctx, image) + if err != nil { + return nil, fmt.Errorf("read base64 error: %s", err.Error()) + } + + contents = append(contents, map[string]interface{}{ + "type": "image_url", + "image_url": map[string]string{ + "url": fmt.Sprintf("data:image/jpeg;base64,%s", bytes64), + }, + }) + } + + return contents, nil +} + +// Upload implements file upload functionality +func (ast *Assistant) Upload(ctx context.Context, file *multipart.FileHeader, reader io.Reader, option map[string]interface{}) (*File, error) { + // check file size + if file.Size > MaxSize { + return nil, fmt.Errorf("file size %d exceeds the maximum size of %d", file.Size, MaxSize) + } + + contentType := file.Header.Get("Content-Type") + if !ast.allowed(contentType) { + return nil, fmt.Errorf("file type %s not allowed", contentType) + } + + data, err := fs.Get("data") + if err != nil { + return nil, err + } + + ext := filepath.Ext(file.Filename) + id, err := ast.id(file.Filename, ext) + if err != nil { + return nil, err + } + + filename := id + _, err = data.Write(filename, reader, 0644) + if err != nil { + return nil, err + } + + return &File{ + ID: filename, + Filename: filename, + ContentType: contentType, + Bytes: int(file.Size), + CreatedAt: int(time.Now().Unix()), + }, nil +} + +func (ast *Assistant) allowed(contentType string) bool { + if _, ok := AllowedFileTypes[contentType]; ok { + return true + } + if strings.HasPrefix(contentType, "text/") || strings.HasPrefix(contentType, "image/") || + strings.HasPrefix(contentType, "audio/") || strings.HasPrefix(contentType, "video/") { + return true + } + return false +} + +func (ast *Assistant) id(temp string, ext string) (string, error) { + date := time.Now().Format("20060102") + hash := fmt.Sprintf("%x", sha256.Sum256([]byte(temp)))[:8] + return fmt.Sprintf("/__assistants/%s/%s/%s%s", ast.ID, date, hash, ext), nil +} + +// Download implements file download functionality +func (ast *Assistant) Download(ctx context.Context, fileID string) (*FileResponse, error) { + data, err := fs.Get("data") + if err != nil { + return nil, fmt.Errorf("get filesystem error: %s", err.Error()) + } + + exists, err := data.Exists(fileID) + if err != nil { + return nil, fmt.Errorf("check file error: %s", err.Error()) + } + if !exists { + return nil, fmt.Errorf("file %s not found", fileID) + } + + reader, err := data.ReadCloser(fileID) + if err != nil { + return nil, err + } + + ext := filepath.Ext(fileID) + contentType := "application/octet-stream" + if v, err := data.MimeType(fileID); err == nil { + contentType = v + } + + return &FileResponse{ + Reader: reader, + ContentType: contentType, + Extension: ext, + }, nil +} + +// ReadBase64 implements base64 file reading functionality +func (ast *Assistant) ReadBase64(ctx context.Context, fileID string) (string, error) { + data, err := fs.Get("data") + if err != nil { + return "", fmt.Errorf("get filesystem error: %s", err.Error()) + } + + exists, err := data.Exists(fileID) + if err != nil { + return "", fmt.Errorf("check file error: %s", err.Error()) + } + if !exists { + return "", fmt.Errorf("file %s not found", fileID) + } + + content, err := data.ReadFile(fileID) + if err != nil { + return "", fmt.Errorf("read file error: %s", err.Error()) + } + + return base64.StdEncoding.EncodeToString(content), nil +} diff --git a/neo/assistant/assistant.go b/neo/assistant/assistant.go index fd24eb9f..f4bc9fd0 100644 --- a/neo/assistant/assistant.go +++ b/neo/assistant/assistant.go @@ -174,7 +174,7 @@ func (ast *Assistant) Clone() *Assistant { Mentionable: ast.Mentionable, Automated: ast.Automated, Script: ast.Script, - API: ast.API, + openai: ast.openai, } // Deep copy tags diff --git a/neo/assistant/load.go b/neo/assistant/load.go index 734d612e..b17e8b02 100644 --- a/neo/assistant/load.go +++ b/neo/assistant/load.go @@ -12,6 +12,8 @@ import ( "github.com/yaoapp/gou/rag/driver" v8 "github.com/yaoapp/gou/runtime/v8" "github.com/yaoapp/yao/neo/store" + neovision "github.com/yaoapp/yao/neo/vision" + "github.com/yaoapp/yao/openai" "github.com/yaoapp/yao/share" "gopkg.in/yaml.v3" ) @@ -20,6 +22,8 @@ import ( var loaded = NewCache(200) // 200 is the default capacity var storage store.Store = nil var rag *RAG = nil +var vision *neovision.Vision = nil +var defaultConnector string = "" // default connector // LoadBuiltIn load the built-in assistants func LoadBuiltIn() error { @@ -73,6 +77,12 @@ func LoadBuiltIn() error { return err } + // Initialize the assistant + err = assistant.initialize() + if err != nil { + return err + } + sort++ loaded.Put(assistant) @@ -86,6 +96,16 @@ func SetStorage(s store.Store) { storage = s } +// SetVision set the vision +func SetVision(v *neovision.Vision) { + vision = v +} + +// SetConnector set the connector +func SetConnector(c string) { + defaultConnector = c +} + // SetRAG set the RAG engine // e: the RAG engine // u: the RAG file uploader @@ -115,6 +135,11 @@ func ClearCache() { // LoadStore create a new assistant from store func LoadStore(id string) (*Assistant, error) { + + if id == "" { + return nil, fmt.Errorf("assistant_id is required") + } + assistant, exists := loaded.Get(id) if exists { return assistant, nil @@ -331,6 +356,12 @@ func loadMap(data map[string]interface{}) (*Assistant, error) { assistant.UpdatedAt = ts } + // Initialize the assistant + err := assistant.initialize() + if err != nil { + return nil, err + } + return assistant, nil } @@ -398,3 +429,21 @@ func loadScriptSource(source string, file string) (*v8.Script, error) { } return script, nil } + +// Init init the assistant +// Choose the connector and initialize the assistant +func (ast *Assistant) initialize() error { + + conn := defaultConnector + if ast.Connector != "" { + conn = ast.Connector + } + ast.Connector = conn + + api, err := openai.New(conn) + if err != nil { + return err + } + ast.openai = api + return nil +} diff --git a/neo/assistant/local/chat.go b/neo/assistant/local/chat.go deleted file mode 100644 index 8c2518cb..00000000 --- a/neo/assistant/local/chat.go +++ /dev/null @@ -1,21 +0,0 @@ -package local - -import ( - "context" - "fmt" -) - -// Chat the chat -func (ast *Local) Chat(ctx context.Context, messages []map[string]interface{}, option map[string]interface{}, cb func(data []byte) int) error { - - if ast.openai == nil { - return fmt.Errorf("api is not initialized") - } - - _, ext := ast.openai.ChatCompletionsWith(ctx, messages, option, cb) - if ext != nil { - return fmt.Errorf("openai chat completions with error: %s", ext.Message) - } - - return nil -} diff --git a/neo/assistant/local/file.go b/neo/assistant/local/file.go deleted file mode 100644 index cb7a692a..00000000 --- a/neo/assistant/local/file.go +++ /dev/null @@ -1,162 +0,0 @@ -package local - -import ( - "context" - "crypto/sha256" - "encoding/base64" - "fmt" - "io" - "mime/multipart" - "path/filepath" - "strings" - "time" - - "github.com/yaoapp/gou/fs" - "github.com/yaoapp/yao/neo/assistant" -) - -// AllowedFileTypes the allowed file types -var AllowedFileTypes = map[string]string{ - "application/json": "json", - "application/pdf": "pdf", - "application/msword": "doc", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx", - "application/vnd.oasis.opendocument.text": "odt", - "application/vnd.ms-excel": "xls", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx", - "application/vnd.ms-powerpoint": "ppt", - "application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx", -} - -// MaxSize 20M max file size -var MaxSize int64 = 20 * 1024 * 1024 - -// Upload the file -func (ast *Local) Upload(ctx context.Context, file *multipart.FileHeader, reader io.Reader, option map[string]interface{}) (*assistant.File, error) { - - // check file size - if file.Size > MaxSize { - return nil, fmt.Errorf("file size %d exceeds the maximum size of %d", file.Size, MaxSize) - } - - contentType := file.Header.Get("Content-Type") - if !ast.allowed(contentType) { - return nil, fmt.Errorf("file type %s not allowed", contentType) - } - - data, err := fs.Get("data") - if err != nil { - return nil, err - } - - ext := filepath.Ext(file.Filename) - id, err := ast.id(file.Filename, ext) - if err != nil { - return nil, err - } - - filename := fmt.Sprintf("%s%s", id, ext) - _, err = data.Write(filename, reader, 0644) - if err != nil { - return nil, err - } - - return &assistant.File{ - ID: filename, - Filename: filename, - ContentType: contentType, - Bytes: int(file.Size), - CreatedAt: int(time.Now().Unix()), - }, nil -} - -func (ast *Local) id(temp string, ext string) (string, error) { - date := time.Now().Format("20060102") - hash := fmt.Sprintf("%x", sha256.Sum256([]byte(temp)))[:8] - return fmt.Sprintf("/__assistants/%s/%s/%s%s", ast.ID, date, hash, ext), nil -} - -func (ast *Local) allowed(contentType string) bool { - if _, ok := AllowedFileTypes[contentType]; ok { - return true - } - // text/* // image/* // audio/* // video/* - if strings.HasPrefix(contentType, "text/") || strings.HasPrefix(contentType, "image/") || strings.HasPrefix(contentType, "audio/") || strings.HasPrefix(contentType, "video/") { - return true - } - return false -} - -// Download downloads a file -func (ast *Local) Download(ctx context.Context, fileID string) (*assistant.FileResponse, error) { - - // Get the data filesystem - data, err := fs.Get("data") - if err != nil { - return nil, fmt.Errorf("get filesystem error: %s", err.Error()) - } - - // Check if file exists - exists, err := data.Exists(fileID) - if err != nil { - return nil, fmt.Errorf("check file error: %s", err.Error()) - } - if !exists { - return nil, fmt.Errorf("file %s not found", fileID) - } - - // Open the file - reader, err := data.ReadCloser(fileID) - if err != nil { - return nil, err - } - - // Get content type and extension - ext := filepath.Ext(fileID) - - // Get content type from mime type - contentType := "application/octet-stream" - if v, err := data.MimeType(fileID); err == nil { - contentType = v - } - - for mimeType, extension := range AllowedFileTypes { - if "."+extension == ext { - contentType = mimeType - break - } - } - - return &assistant.FileResponse{ - Reader: reader, - ContentType: contentType, - Extension: ext, - }, nil -} - -// ReadBase64 reads a file and returns its base64 encoded content -func (ast *Local) ReadBase64(ctx context.Context, fileID string) (string, error) { - // Get the data filesystem - data, err := fs.Get("data") - if err != nil { - return "", fmt.Errorf("get filesystem error: %s", err.Error()) - } - - // Check if file exists - exists, err := data.Exists(fileID) - if err != nil { - return "", fmt.Errorf("check file error: %s", err.Error()) - } - if !exists { - return "", fmt.Errorf("file %s not found", fileID) - } - - // Read file content - content, err := data.ReadFile(fileID) - if err != nil { - return "", fmt.Errorf("read file error: %s", err.Error()) - } - - // Encode to base64 - return base64.StdEncoding.EncodeToString(content), nil -} diff --git a/neo/assistant/local/local.go b/neo/assistant/local/local.go deleted file mode 100644 index 7a1386ff..00000000 --- a/neo/assistant/local/local.go +++ /dev/null @@ -1,34 +0,0 @@ -package local - -import ( - "context" - - "github.com/yaoapp/gou/connector" - "github.com/yaoapp/yao/neo/assistant" - "github.com/yaoapp/yao/openai" -) - -// Local the local assistant -type Local struct { - ID string `json:"assistant_id"` - Prompts []assistant.Prompt `json:"prompts,omitempty"` - Connector connector.Connector `json:"-" yaml:"-"` - openai *openai.OpenAI -} - -// New create a new local assistant -func New(connector connector.Connector, prompts []assistant.Prompt, id string) (*Local, error) { - - setting := connector.Setting() - api, err := openai.NewOpenAI(setting) - if err != nil { - return nil, err - } - - return &Local{Connector: connector, ID: id, Prompts: prompts, openai: api}, nil -} - -// List list all assistants -func (ast *Local) List(ctx context.Context, param assistant.QueryParam) ([]assistant.Assistant, error) { - return nil, nil -} diff --git a/neo/assistant/openai/chat.go b/neo/assistant/openai/chat.go deleted file mode 100644 index 1bf80a8c..00000000 --- a/neo/assistant/openai/chat.go +++ /dev/null @@ -1,118 +0,0 @@ -package openai - -import ( - "context" - "fmt" - "strings" - - chatMessage "github.com/yaoapp/yao/neo/message" -) - -// Chat the chat struct -type Chat struct { - ID string `json:"chat_id"` - ThreadID string `json:"thread_id"` -} - -// NewChat create a new chat -func (ast *OpenAI) NewChat() {} - -// Chat the chat -func (ast *OpenAI) Chat(ctx context.Context, messages []map[string]interface{}, option map[string]interface{}, cb func(data []byte) int) error { - - if ast.openai == nil { - return fmt.Errorf("openai is not initialized") - } - - requestMessages, err := ast.requestMessages(ctx, messages) - if err != nil { - return fmt.Errorf("request messages error: %s", err.Error()) - } - - _, ext := ast.openai.ChatCompletionsWith(ctx, requestMessages, option, cb) - if ext != nil { - return fmt.Errorf("openai chat completions with error: %s", ext.Message) - } - - return nil -} - -func (ast *OpenAI) requestMessages(ctx context.Context, messages []map[string]interface{}) ([]map[string]interface{}, error) { - newMessages := []map[string]interface{}{} - length := len(messages) - for index, message := range messages { - role, ok := message["role"].(string) - if !ok { - return nil, fmt.Errorf("role must be string") - } - - content, ok := message["content"].(string) - if !ok { - return nil, fmt.Errorf("content must be string") - } - - newMessage := map[string]interface{}{ - "role": role, - "content": content, - } - - // Handle name if present - if name, ok := message["name"].(string); ok { - newMessage["name"] = name - } - - newMessage["content"] = content - - // Special handling for user messages with JSON content last message - if role == "user" && index == length-1 { - content = strings.TrimSpace(content) - msg, err := chatMessage.NewString(content) - if err != nil { - return nil, fmt.Errorf("new string error: %s", err.Error()) - } - - newMessage["content"] = msg.Text - if msg.Attachments != nil { - content, err := ast.withAttachments(ctx, msg) - if err != nil { - return nil, fmt.Errorf("with attachments error: %s", err.Error()) - } - newMessage["content"] = content - } - } - - newMessages = append(newMessages, newMessage) - } - return newMessages, nil -} - -func (ast *OpenAI) withAttachments(ctx context.Context, msg *chatMessage.Message) ([]map[string]interface{}, error) { - contents := []map[string]interface{}{{"type": "text", "text": msg.Text}} - images := []string{} - for _, attachment := range msg.Attachments { - if strings.HasPrefix(attachment.ContentType, "image/") { - images = append(images, attachment.FileID) - } - } - - if len(images) == 0 { - return contents, nil - } - - for _, image := range images { - bytes64, err := ast.ReadBase64(ctx, image) - if err != nil { - return nil, fmt.Errorf("read base64 error: %s", err.Error()) - } - - contents = append(contents, map[string]interface{}{ - "type": "image_url", - "image_url": map[string]string{ - "url": fmt.Sprintf("data:image/jpeg;base64,%s", bytes64), - }, - }, - ) - } - - return contents, nil -} diff --git a/neo/assistant/openai/file.go b/neo/assistant/openai/file.go deleted file mode 100644 index 062a688b..00000000 --- a/neo/assistant/openai/file.go +++ /dev/null @@ -1,167 +0,0 @@ -package openai - -import ( - "context" - "crypto/sha256" - "encoding/base64" - "fmt" - "io" - "mime/multipart" - "path/filepath" - "strings" - "time" - - "github.com/yaoapp/gou/fs" - "github.com/yaoapp/yao/neo/assistant" -) - -// AllowedFileTypes the allowed file types -var AllowedFileTypes = map[string]string{ - "application/json": "json", - "application/pdf": "pdf", - "application/msword": "doc", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx", - "application/vnd.oasis.opendocument.text": "odt", - "application/vnd.ms-excel": "xls", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx", - "application/vnd.ms-powerpoint": "ppt", - "application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx", -} - -// MaxSize 20M max file size -var MaxSize int64 = 20 * 1024 * 1024 - -// Upload the file -func (ast *OpenAI) Upload(ctx context.Context, file *multipart.FileHeader, reader io.Reader, option map[string]interface{}) (*assistant.File, error) { - - // check file size - if file.Size > MaxSize { - return nil, fmt.Errorf("file size %d exceeds the maximum size of %d", file.Size, MaxSize) - } - - contentType := file.Header.Get("Content-Type") - if !ast.allowed(contentType) { - return nil, fmt.Errorf("file type %s not allowed", contentType) - } - - data, err := fs.Get("data") - if err != nil { - return nil, err - } - - ext := filepath.Ext(file.Filename) - id, err := ast.id(file.Filename, ext) - if err != nil { - return nil, err - } - - filename := id - _, err = data.Write(filename, reader, 0644) - if err != nil { - return nil, err - } - - return &assistant.File{ - ID: filename, - Filename: filename, - ContentType: contentType, - Bytes: int(file.Size), - CreatedAt: int(time.Now().Unix()), - }, nil -} - -func (ast *OpenAI) id(temp string, ext string) (string, error) { - date := time.Now().Format("20060102") - hash := fmt.Sprintf("%x", sha256.Sum256([]byte(temp)))[:8] - return fmt.Sprintf("/__assistants/%s/%s/%s%s", ast.ID, date, hash, ext), nil -} - -func (ast *OpenAI) allowed(contentType string) bool { - if _, ok := AllowedFileTypes[contentType]; ok { - return true - } - // text/* // image/* // audio/* // video/* - if strings.HasPrefix(contentType, "text/") || strings.HasPrefix(contentType, "image/") || strings.HasPrefix(contentType, "audio/") || strings.HasPrefix(contentType, "video/") { - return true - } - return false -} - -// FileLists list all files -func (ast *OpenAI) FileLists() {} - -// FileDelete delete a file -func (ast *OpenAI) FileDelete() {} - -// FileContent get the content of a file -func (ast *OpenAI) FileContent() {} - -// FileInfo get the information of a file -func (ast *OpenAI) FileInfo() {} - -// Download downloads a file -func (ast *OpenAI) Download(ctx context.Context, fileID string) (*assistant.FileResponse, error) { - - // Get the data filesystem - data, err := fs.Get("data") - if err != nil { - return nil, fmt.Errorf("get filesystem error: %s", err.Error()) - } - - // Check if file exists - exists, err := data.Exists(fileID) - if err != nil { - return nil, fmt.Errorf("check file error: %s", err.Error()) - } - if !exists { - return nil, fmt.Errorf("file %s not found", fileID) - } - - // Open the file - reader, err := data.ReadCloser(fileID) - if err != nil { - return nil, err - } - - // Get content type and extension - ext := filepath.Ext(fileID) - - // Get content type from mime type - contentType := "application/octet-stream" - if v, err := data.MimeType(fileID); err == nil { - contentType = v - } - - return &assistant.FileResponse{ - Reader: reader, - ContentType: contentType, - Extension: ext, - }, nil -} - -// ReadBase64 reads a file and returns its base64 encoded content -func (ast *OpenAI) ReadBase64(ctx context.Context, fileID string) (string, error) { - // Get the data filesystem - data, err := fs.Get("data") - if err != nil { - return "", fmt.Errorf("get filesystem error: %s", err.Error()) - } - - // Check if file exists - exists, err := data.Exists(fileID) - if err != nil { - return "", fmt.Errorf("check file error: %s", err.Error()) - } - if !exists { - return "", fmt.Errorf("file %s not found", fileID) - } - - // Read file content - content, err := data.ReadFile(fileID) - if err != nil { - return "", fmt.Errorf("read file error: %s", err.Error()) - } - - // Encode to base64 - return base64.StdEncoding.EncodeToString(content), nil -} diff --git a/neo/assistant/openai/openai.go b/neo/assistant/openai/openai.go deleted file mode 100644 index 38f5135c..00000000 --- a/neo/assistant/openai/openai.go +++ /dev/null @@ -1,51 +0,0 @@ -package openai - -import ( - "context" - - "github.com/yaoapp/gou/connector" - "github.com/yaoapp/yao/neo/assistant" - api "github.com/yaoapp/yao/openai" -) - -// OpenAI the openai assistant -type OpenAI struct { - ID string `json:"assistant_id"` // the assistant id - Connector connector.Connector `json:"-" yaml:"-"` - openai *api.OpenAI -} - -// New create a new openai assistant -func New(connector connector.Connector, id string) (*OpenAI, error) { - - setting := connector.Setting() - openai, err := api.NewOpenAI(setting) - if err != nil { - return nil, err - } - - return &OpenAI{ID: id, Connector: connector, openai: openai}, nil -} - -// Current set the current assistant -func (ast *OpenAI) Current(id string) *OpenAI { - ast.ID = id - return ast -} - -// List list all assistants -func (ast *OpenAI) List(ctx context.Context, param assistant.QueryParam) ([]assistant.Assistant, error) { - return nil, nil -} - -// Create create a new assistant -func (ast *OpenAI) Create() {} - -// Delete delete an assistant -func (ast *OpenAI) Delete() {} - -// Update update an assistant -func (ast *OpenAI) Update() {} - -// Get get an assistant -func (ast *OpenAI) Get() {} diff --git a/neo/assistant/openai/thread.go b/neo/assistant/openai/thread.go deleted file mode 100644 index 8f5975e8..00000000 --- a/neo/assistant/openai/thread.go +++ /dev/null @@ -1,21 +0,0 @@ -package openai - -// Thread the thread struct -type Thread struct { - ID string `json:"thread_id"` -} - -// ThreadList list all threads -func (ast *OpenAI) ThreadList() {} - -// ThreadCreate create a new thread -func (ast *OpenAI) ThreadCreate() {} - -// ThreadGet get a thread -func (ast *OpenAI) ThreadGet(id string) {} - -// ThreadDelete delete a thread -func (ast *OpenAI) ThreadDelete() {} - -// ThreadUpdate update a thread -func (ast *OpenAI) ThreadUpdate() {} diff --git a/neo/assistant/types.go b/neo/assistant/types.go index 5cec9ba6..480f8146 100644 --- a/neo/assistant/types.go +++ b/neo/assistant/types.go @@ -7,6 +7,7 @@ import ( "github.com/yaoapp/gou/rag/driver" v8 "github.com/yaoapp/gou/runtime/v8" + api "github.com/yaoapp/yao/openai" ) // API the assistant API interface @@ -64,9 +65,9 @@ type Assistant struct { Prompts []Prompt `json:"prompts,omitempty"` // AI Prompts Flows []map[string]interface{} `json:"flows,omitempty"` // Assistant Flows Script *v8.Script `json:"-" yaml:"-"` // Assistant Script - API API `json:"-" yaml:"-"` // Assistant API CreatedAt int64 `json:"created_at"` // Creation timestamp UpdatedAt int64 `json:"updated_at"` // Last update timestamp + openai *api.OpenAI // OpenAI API } // File the file diff --git a/neo/hooks.go b/neo/hooks.go index 775f492f..4ec7dabb 100644 --- a/neo/hooks.go +++ b/neo/hooks.go @@ -7,7 +7,6 @@ import ( "github.com/gin-gonic/gin" jsoniter "github.com/json-iterator/go" "github.com/yaoapp/gou/process" - "github.com/yaoapp/yao/neo/assistant" ) // HookCreate create the assistant @@ -69,51 +68,6 @@ func (neo *DSL) HookCreate(ctx Context, messages []map[string]interface{}, c *gi return CreateResponse{AssistantID: assistantID, ChatID: ctx.ChatID}, nil } -// HookAssistants query the assistant list from the assistant list hook -func (neo *DSL) HookAssistants(ctx context.Context, param assistant.QueryParam) ([]assistant.Assistant, error) { - if neo.AssistantListHook == "" { - return nil, nil - } - - // Create a context with 10 second timeout - timeoutCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - p, err := process.Of(neo.AssistantListHook, param) - if err != nil { - return nil, err - } - - err = p.WithContext(timeoutCtx).Execute() - if err != nil { - return nil, err - } - defer p.Release() - - // Check if context was canceled - if timeoutCtx.Err() != nil { - return nil, timeoutCtx.Err() - } - - value := p.Value() - if value == nil { - return nil, nil - } - - var list []assistant.Assistant - bytes, err := jsoniter.Marshal(value) - if err != nil { - return nil, err - } - - err = jsoniter.Unmarshal(bytes, &list) - if err != nil { - return nil, err - } - - return list, nil -} - // HookPrepare executes the prepare hook before AI is called func (neo *DSL) HookPrepare(ctx Context, messages []map[string]interface{}) ([]map[string]interface{}, error) { if neo.Prepare == "" { @@ -194,60 +148,3 @@ func (neo *DSL) HookWrite(ctx Context, messages []map[string]interface{}, respon return result, nil } - -// HookMention query the mention list -func (neo *DSL) HookMention(ctx context.Context, keywords string) ([]Mention, error) { - - // Default Get the assistant list - if neo.MentionHook == "" { - var mentions []Mention - assistants := neo.GetAssistants() - for _, assistant := range assistants { - mentions = append(mentions, Mention{ - ID: assistant.ID, - Name: assistant.Name, - Type: "assistant", - }) - } - - return mentions, nil - } - - // Create a context with 10 second timeout - timeoutCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - p, err := process.Of(neo.MentionHook, keywords) - if err != nil { - return nil, err - } - - err = p.WithContext(timeoutCtx).Execute() - if err != nil { - return nil, err - } - defer p.Release() - - // Check if context was canceled - if timeoutCtx.Err() != nil { - return nil, timeoutCtx.Err() - } - - value := p.Value() - if value == nil { - return nil, nil - } - - var list []Mention - bytes, err := jsoniter.Marshal(value) - if err != nil { - return nil, err - } - - err = jsoniter.Unmarshal(bytes, &list) - if err != nil { - return nil, err - } - - return list, nil -} diff --git a/neo/load.go b/neo/load.go index 3a185730..4ee561e4 100644 --- a/neo/load.go +++ b/neo/load.go @@ -155,6 +155,14 @@ func (neo *DSL) initAssistant() error { ) } + // Assistant Vision + if Neo.Vision != nil { + assistant.SetVision(Neo.Vision) + } + + // Default Connector + assistant.SetConnector(Neo.Connector) + // Load Built-in Assistants err := assistant.LoadBuiltIn() if err != nil { @@ -167,7 +175,7 @@ func (neo *DSL) initAssistant() error { return err } - Neo.Assistant = defaultAssistant.API + Neo.Assistant = defaultAssistant return nil } diff --git a/neo/neo.go b/neo/neo.go index f072dc1c..b114b0d8 100644 --- a/neo/neo.go +++ b/neo/neo.go @@ -1,21 +1,15 @@ package neo import ( - "context" "fmt" "os" "strings" "sync" "github.com/gin-gonic/gin" - "github.com/yaoapp/gou/connector" "github.com/yaoapp/kun/log" "github.com/yaoapp/yao/neo/assistant" - "github.com/yaoapp/yao/neo/assistant/local" - "github.com/yaoapp/yao/neo/assistant/openai" "github.com/yaoapp/yao/neo/message" - "github.com/yaoapp/yao/neo/store" - "github.com/yaoapp/yao/share" ) // Lock the assistant list @@ -39,23 +33,20 @@ func (neo *DSL) Answer(ctx Context, question string, c *gin.Context) error { } // Select Assistant - ast, err := neo.selectAssistant(res.AssistantID) + ast, err := neo.Select(res.AssistantID) if err != nil { return err } - // Chat with AI return neo.chat(ast, ctx, messages, c) } -// GetAssistants returns the list of assistants -func (neo *DSL) GetAssistants() []assistant.Assistant { - return neo.AssistantList -} - -// GetMentions returns the mention list -func (neo *DSL) GetMentions(keywords string) ([]Mention, error) { - return neo.HookMention(context.Background(), keywords) +// Select select an assistant +func (neo *DSL) Select(id string) (assistant.API, error) { + if id == "" { + return neo.Assistant, nil + } + return assistant.Get(id) } // GeneratePrompts generate prompts for the AI assistant @@ -110,7 +101,7 @@ func (neo *DSL) GenerateWithAI(ctx Context, input string, messageType string, sy } // Select Assistant - ast, err := neo.selectAssistant(res.AssistantID) + ast, err := neo.Select(res.AssistantID) if err != nil { return "", err } @@ -234,7 +225,7 @@ func (neo *DSL) Upload(ctx Context, c *gin.Context) (*assistant.File, error) { } // Select Assistant - ast, err := neo.selectAssistant(res.AssistantID) + ast, err := neo.Select(res.AssistantID) if err != nil { return nil, err } @@ -257,7 +248,7 @@ func (neo *DSL) Download(ctx Context, c *gin.Context) (*assistant.FileResponse, } // Select Assistant - ast, err := neo.selectAssistant(res.AssistantID) + ast, err := neo.Select(res.AssistantID) if err != nil { return nil, err } @@ -346,130 +337,6 @@ func (neo *DSL) chat(ast assistant.API, ctx Context, messages []map[string]inter } } -// updateAssistantList update the assistant list -func (neo *DSL) updateAssistantList(list []assistant.Assistant) { - lock.Lock() - defer lock.Unlock() - neo.AssistantList = list - neo.AssistantMaps = make(map[string]assistant.Assistant) - if list != nil { - for _, assistant := range list { - neo.AssistantMaps[assistant.ID] = assistant - } - } -} - -// selectAssistant select the assistant -func (neo *DSL) selectAssistant(assistantID string) (assistant.API, error) { - ast := neo.Assistant - if assistantID != "" { - ast, err := neo.newAssistant(assistantID) - if err != nil { - return nil, err - } - return ast, nil - } - return ast, nil -} - -// newAssistant create a new assistant -func (neo *DSL) newAssistant(id string) (assistant.API, error) { - // Try to find assistant in AssistantList first - if id != "" && neo.AssistantMaps != nil { - if ast, ok := neo.AssistantMaps[id]; ok { - - if ast.API != nil { - return ast.API, nil - } - api, err := neo.newAssistantByConfig(&ast) - if err != nil { - return nil, err - } - ast.API = api - return api, nil - } - } - return neo.newAssistantByConnector(id) -} - -// newAssistantByConfig create a new assistant from assistant configuration -func (neo *DSL) newAssistantByConfig(ast *assistant.Assistant) (assistant.API, error) { - return neo.newAssistantByConnector(ast.Connector) -} - -// newAssistantByConnector create a new assistant from connector id -func (neo *DSL) newAssistantByConnector(id string) (assistant.API, error) { - // Moapi connector - if id == "" || strings.HasPrefix(id, "moapi") { - return neo.newMoapiAssistant(id) - } - - // Other connector - conn, err := connector.Select(id) - if err != nil { - return nil, fmt.Errorf("Neo assistant connector %s not support", id) - } - - if conn.Is(connector.OPENAI) { - api, err := openai.New(conn, id) - if err != nil { - return nil, fmt.Errorf("Create openai assistant error: %s", err.Error()) - } - return api, nil - } - - // Base on the assistant list hook - api, err := local.New(conn, neo.Prompts, id) - if err != nil { - return nil, fmt.Errorf("Create local assistant error: %s", err.Error()) - } - return api, nil -} - -// newMoapiAssistant creates a new moapi assistant -func (neo *DSL) newMoapiAssistant(id string) (assistant.API, error) { - model := "gpt-3.5-turbo" - if strings.HasPrefix(id, "moapi:") { - model = strings.TrimPrefix(id, "moapi:") - } - - // Get the moapi setting - url := share.MoapiHosts[0] - if share.App.Moapi.Mirrors != nil { - url = share.App.Moapi.Mirrors[0] - } - key := share.App.Moapi.Secret - organization := share.App.Moapi.Organization - - if !strings.HasPrefix(url, "http") { - url = "https://" + url - } - - // Check the moapi secret - if key == "" { - return nil, fmt.Errorf("The moapi secret is empty") - } - - conn, err := connector.New(`moapi`, `__yao.moapi`, []byte(`{"name":"Moapi", "options":{"model": "`+model+`", "key": "`+key+`", "organization": "`+organization+`", "host": "`+url+`"}}`)) - if err != nil { - return nil, fmt.Errorf("Create moapi assistant error: %s", err.Error()) - } - - api, err := openai.New(conn, strings.ReplaceAll(id, ":", "_")) - if err != nil { - return nil, fmt.Errorf("Create openai assistant error: %s", err.Error()) - } - return api, nil -} - -// createDefaultAssistant create a default assistant -func (neo *DSL) createDefaultAssistant() (assistant.API, error) { - if neo.Use != "" { - return neo.newAssistant(neo.Use) - } - return neo.newAssistant(neo.Connector) -} - // chatMessages get the chat messages func (neo *DSL) chatMessages(ctx Context, content ...string) ([]map[string]interface{}, error) { @@ -509,37 +376,6 @@ func (neo *DSL) saveHistory(sid string, chatID string, content []byte, messages } } -// createStore create a new store -func (neo *DSL) createStore() error { - - var err error - if neo.StoreSetting.Connector == "default" || neo.StoreSetting.Connector == "" { - neo.Store, err = store.NewXun(neo.StoreSetting) - return err - } - - // other connector - conn, err := connector.Select(neo.StoreSetting.Connector) - if err != nil { - return err - } - - if conn.Is(connector.DATABASE) { - neo.Store, err = store.NewXun(neo.StoreSetting) - return err - - } else if conn.Is(connector.REDIS) { - neo.Store = store.NewRedis() - return nil - - } else if conn.Is(connector.MONGO) { - neo.Store = store.NewMongo() - return nil - } - - return fmt.Errorf("%s store connector %s not support", neo.ID, neo.StoreSetting.Connector) -} - // sendMessage sends a message to the client func (neo *DSL) sendMessage(w gin.ResponseWriter, data interface{}) error { if msg, ok := data.(map[string]interface{}); ok { diff --git a/neo/types.go b/neo/types.go index 53c9ac24..e3511168 100644 --- a/neo/types.go +++ b/neo/types.go @@ -13,29 +13,25 @@ import ( // DSL AI assistant type DSL struct { - ID string `json:"-" yaml:"-"` - Name string `json:"name,omitempty" yaml:"name,omitempty"` - Use string `json:"use,omitempty" yaml:"use,omitempty"` // Which assistant to use default - Guard string `json:"guard,omitempty" yaml:"guard,omitempty"` - Connector string `json:"connector" yaml:"connector"` - StoreSetting store.Setting `json:"store" yaml:"store"` - RAGSetting rag.Setting `json:"rag" yaml:"rag"` - VisionSetting VisionSetting `json:"vision" yaml:"vision"` - Option map[string]interface{} `json:"option" yaml:"option"` - Prepare string `json:"prepare,omitempty" yaml:"prepare,omitempty"` - Create string `json:"create,omitempty" yaml:"create,omitempty"` - Write string `json:"write,omitempty" yaml:"write,omitempty"` - AssistantListHook string `json:"assistants,omitempty" yaml:"assistants,omitempty"` // Get the assistant list from the hook - MentionHook string `json:"mentions,omitempty"` // Get the mention list from the hook - Prompts []assistant.Prompt `json:"prompts,omitempty" yaml:"prompts,omitempty"` - Allows []string `json:"allows,omitempty" yaml:"allows,omitempty"` - Assistant assistant.API `json:"-" yaml:"-"` // The default assistant - Store store.Store `json:"-" yaml:"-"` - RAG *rag.RAG `json:"-" yaml:"-"` - Vision *vision.Vision `json:"-" yaml:"-"` - GuardHandlers []gin.HandlerFunc `json:"-" yaml:"-"` - AssistantList []assistant.Assistant `json:"-" yaml:"-"` - AssistantMaps map[string]assistant.Assistant `json:"-" yaml:"-"` + ID string `json:"-" yaml:"-"` + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Use string `json:"use,omitempty" yaml:"use,omitempty"` // Which assistant to use default + Guard string `json:"guard,omitempty" yaml:"guard,omitempty"` + Connector string `json:"connector" yaml:"connector"` + StoreSetting store.Setting `json:"store" yaml:"store"` + RAGSetting rag.Setting `json:"rag" yaml:"rag"` + VisionSetting VisionSetting `json:"vision" yaml:"vision"` + Option map[string]interface{} `json:"option" yaml:"option"` + Prepare string `json:"prepare,omitempty" yaml:"prepare,omitempty"` + Create string `json:"create,omitempty" yaml:"create,omitempty"` + Write string `json:"write,omitempty" yaml:"write,omitempty"` + Prompts []assistant.Prompt `json:"prompts,omitempty" yaml:"prompts,omitempty"` + Allows []string `json:"allows,omitempty" yaml:"allows,omitempty"` + Assistant assistant.API `json:"-" yaml:"-"` // The default assistant + Store store.Store `json:"-" yaml:"-"` + RAG *rag.RAG `json:"-" yaml:"-"` + Vision *vision.Vision `json:"-" yaml:"-"` + GuardHandlers []gin.HandlerFunc `json:"-" yaml:"-"` } // VisionSetting the vision setting