diff --git a/neo/attachment/README.md b/attachment/README.md similarity index 92% rename from neo/attachment/README.md rename to attachment/README.md index 780fc954..661dd3ab 100644 --- a/neo/attachment/README.md +++ b/attachment/README.md @@ -14,7 +14,7 @@ A comprehensive file upload package for Go that supports chunked uploads, file f - File size limits - MIME type and extension validation - Wildcard pattern support (e.g., `image/*`, `text/*`) -- **Flexible File Organization**: Hierarchical storage with user/chat/assistant organization +- **Flexible File Organization**: Hierarchical storage with multi-level group organization - **Multiple Read Methods**: Stream, bytes, and base64 encoding - **Global Manager Registry**: Support for registering and accessing managers globally - **Upload Status Tracking**: Track upload progress with status field @@ -73,8 +73,7 @@ func main() { fileHeader.Header.Set("Content-Type", "text/plain") option := attachment.UploadOption{ - UserID: "user123", - ChatID: "chat456", + Groups: []string{"user123", "chat456"}, // Multi-level groups (e.g., user, chat, knowledge, etc.) OriginalFilename: "my_document.txt", // Preserve original filename } @@ -196,6 +195,29 @@ option := attachment.UploadOption{ file, err := manager.Upload(ctx, imageHeader, imageReader, option) ``` +### Multi-level Groups + +The `Groups` field supports hierarchical file organization: + +```go +// Single level grouping +option := attachment.UploadOption{ + Groups: []string{"users"}, +} + +// Multi-level grouping +option := attachment.UploadOption{ + Groups: []string{"users", "user123", "chats", "chat456"}, +} + +// Knowledge base organization +option := attachment.UploadOption{ + Groups: []string{"knowledge", "documents", "technical"}, +} +``` + +This creates nested directory structures for better organization and access control. + ### File Validation #### Size Limits @@ -295,18 +317,18 @@ Files are organized in a hierarchical structure: ``` attachments/ ├── 20240101/ # Date (YYYYMMDD) -│ └── user123/ # User ID (optional) -│ └── chat456/ # Chat ID (optional) -│ └── assistant789/ # Assistant ID (optional) -│ └── ab/ # First 2 chars of hash -│ └── cd/ # Next 2 chars of hash +│ └── user123/ # First level group (optional) +│ └── chat456/ # Second level group (optional) +│ └── knowledge/ # Additional group levels (optional) +│ └── ab/ # First 2 chars of hash +│ └── cd/ # Next 2 chars of hash │ └── abcdef12.txt # Hash + extension ``` The file ID generation includes: - Date prefix for organization -- User/Chat/Assistant IDs for access control +- Multi-level groups for access control and organization - Content hash for deduplication - Original file extension @@ -374,8 +396,7 @@ Options for file upload: - `CompressImage`: Enable image compression - `CompressSize`: Maximum image dimension (default: 1920) - `Gzip`: Enable gzip compression -- `Knowledge`: Push to knowledge base -- `UserID`, `ChatID`, `AssistantID`: Organization IDs +- `Groups`: Multi-level group identifiers for hierarchical file organization (e.g., []string{"user123", "chat456", "knowledge"}) - `OriginalFilename`: Original filename to preserve (avoids encoding issues) #### `File` @@ -465,7 +486,7 @@ The package includes comprehensive tests for: - **File Type Validation**: Prevents upload of unauthorized file types - **Size Limits**: Configurable file size restrictions - **Path Sanitization**: Secure file path generation -- **Access Control**: User/Chat/Assistant-based file organization +- **Access Control**: Multi-level hierarchical file organization ## License @@ -536,8 +557,7 @@ for start := int64(0); start < totalSize; start += chunkSize { chunkHeader.Header.Set("Content-Sync", "true") // Enable synchronization option := attachment.UploadOption{ - UserID: "user123", - ChatID: "chat456", + Groups: []string{"user123", "chat456"}, // Multi-level groups OriginalFilename: "my_large_file.zip", // Preserve original name } diff --git a/neo/attachment/compresses.go b/attachment/compresses.go similarity index 100% rename from neo/attachment/compresses.go rename to attachment/compresses.go diff --git a/neo/attachment/example_usage.go b/attachment/example_usage.go similarity index 81% rename from neo/attachment/example_usage.go rename to attachment/example_usage.go index 4236a4f9..18c35d2c 100644 --- a/neo/attachment/example_usage.go +++ b/attachment/example_usage.go @@ -8,7 +8,7 @@ import ( "mime/multipart" "strings" - "github.com/yaoapp/yao/neo/attachment/s3" + "github.com/yaoapp/yao/attachment/s3" ) // ExampleUsage demonstrates how to use the attachment package @@ -95,10 +95,8 @@ func ExampleUsage() { fileHeader.Header.Set("Content-Type", "text/plain") uploadOption := UploadOption{ - UserID: "user123", - ChatID: "chat456", - AssistantID: "assistant789", - Gzip: false, // No compression for small text files + Groups: []string{"user123"}, + Gzip: false, // No compression for small text files } file, err := localManager.Upload(ctx, fileHeader, strings.NewReader(content), uploadOption) @@ -120,7 +118,7 @@ func ExampleUsage() { gzipFileHeader.Header.Set("Content-Type", "text/plain") gzipOption := UploadOption{ - UserID: "user123", + Groups: []string{"user123"}, Gzip: true, // Enable compression } @@ -133,7 +131,7 @@ func ExampleUsage() { // 6. Example: Image upload with compression imageUploadOption := UploadOption{ - UserID: "user123", + Groups: []string{"user123"}, CompressImage: true, CompressSize: 1920, // Resize to max 1920px Gzip: false, @@ -152,6 +150,75 @@ func ExampleUsage() { fmt.Printf("Image upload option configured: compress=%v, size=%d\n", imageUploadOption.CompressImage, imageUploadOption.CompressSize) + // 6.5. Example: Multi-level groups + fmt.Println("\n--- Multi-level Groups Examples ---") + + // Single level grouping + singleGroupOption := UploadOption{ + Groups: []string{"knowledge"}, + } + + singleGroupHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "knowledge_doc.txt", + Size: int64(len("Knowledge base document")), + Header: make(map[string][]string), + }, + } + singleGroupHeader.Header.Set("Content-Type", "text/plain") + + singleFile, err := localManager.Upload(ctx, singleGroupHeader, strings.NewReader("Knowledge base document"), singleGroupOption) + if err != nil { + log.Printf("Failed to upload single group file: %v", err) + } else { + fmt.Printf("Single group file uploaded: %s (ID: %s)\n", singleFile.Filename, singleFile.ID) + } + + // Multi-level grouping + multiGroupOption := UploadOption{ + Groups: []string{"users", "user123", "chats", "chat456", "documents"}, + } + + multiGroupHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "chat_document.txt", + Size: int64(len("Document in user chat")), + Header: make(map[string][]string), + }, + } + multiGroupHeader.Header.Set("Content-Type", "text/plain") + + multiFile, err := localManager.Upload(ctx, multiGroupHeader, strings.NewReader("Document in user chat"), multiGroupOption) + if err != nil { + log.Printf("Failed to upload multi-group file: %v", err) + } else { + fmt.Printf("Multi-level group file uploaded: %s (ID: %s)\n", multiFile.Filename, multiFile.ID) + fmt.Printf("File path includes hierarchy: users/user123/chats/chat456/documents\n") + } + + // Knowledge base organization + knowledgeOption := UploadOption{ + Groups: []string{"knowledge", "technical", "api", "documentation"}, + } + + knowledgeHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "api_guide.md", + Size: int64(len("# API Documentation\n\nThis is technical documentation.")), + Header: make(map[string][]string), + }, + } + knowledgeHeader.Header.Set("Content-Type", "text/markdown") + + knowledgeFile, err := localManager.Upload(ctx, knowledgeHeader, + strings.NewReader("# API Documentation\n\nThis is technical documentation."), knowledgeOption) + if err != nil { + log.Printf("Failed to upload knowledge file: %v", err) + } else { + fmt.Printf("Knowledge base file uploaded: %s (ID: %s)\n", knowledgeFile.Filename, knowledgeFile.ID) + fmt.Printf("Organized in: knowledge/technical/api/documentation\n") + } + // 7. Example: Chunked upload largeContent = strings.Repeat("This is a large file content that will be uploaded in chunks. ", 1000) chunkSize := 1024 @@ -183,7 +250,7 @@ func ExampleUsage() { chunkHeader.Header.Set("Content-Uid", uid) chunkOption := UploadOption{ - UserID: "user123", + Groups: []string{"user123"}, Gzip: true, // Compress chunks } @@ -273,7 +340,7 @@ func ExampleUsage() { testHeader.Header.Set("Content-Type", "text/plain") testFile, err := globalManager.Upload(ctx, testHeader, strings.NewReader(testContent), UploadOption{ - UserID: "global_user", + Groups: []string{"global_user"}, }) if err != nil { log.Printf("Failed to upload with global manager: %v", err) @@ -356,7 +423,7 @@ func ExampleChunkedUpload(manager *Manager, filename string, totalSize int64, co } option := UploadOption{ - UserID: "user123", + Groups: []string{"user123"}, Gzip: false, // Disable compression for this example } @@ -422,7 +489,7 @@ func ExampleS3Upload() { fileHeader.Header.Set("Content-Type", "text/plain") file, err := s3Manager.Upload(ctx, fileHeader, strings.NewReader(content), UploadOption{ - UserID: "s3_user", + Groups: []string{"s3_user"}, }) if err != nil { log.Printf("S3 upload failed: %v", err) diff --git a/neo/attachment/fileheader.go b/attachment/fileheader.go similarity index 100% rename from neo/attachment/fileheader.go rename to attachment/fileheader.go diff --git a/neo/attachment/gzip.go b/attachment/gzip.go similarity index 100% rename from neo/attachment/gzip.go rename to attachment/gzip.go diff --git a/neo/attachment/local/storage.go b/attachment/local/storage.go similarity index 100% rename from neo/attachment/local/storage.go rename to attachment/local/storage.go diff --git a/neo/attachment/local/storage_test.go b/attachment/local/storage_test.go similarity index 100% rename from neo/attachment/local/storage_test.go rename to attachment/local/storage_test.go diff --git a/neo/attachment/manager.go b/attachment/manager.go similarity index 97% rename from neo/attachment/manager.go rename to attachment/manager.go index a652555f..cbe5503f 100644 --- a/neo/attachment/manager.go +++ b/attachment/manager.go @@ -18,9 +18,9 @@ import ( "sync" "time" + "github.com/yaoapp/yao/attachment/local" + "github.com/yaoapp/yao/attachment/s3" "github.com/yaoapp/yao/config" - "github.com/yaoapp/yao/neo/attachment/local" - "github.com/yaoapp/yao/neo/attachment/s3" ) // Managers the managers @@ -161,7 +161,6 @@ func New(option ManagerOption) (*Manager, error) { return nil, err } manager.storage = storage - break case "s3": storage, err := s3.New(option.Options) @@ -169,7 +168,6 @@ func New(option ManagerOption) (*Manager, error) { return nil, err } manager.storage = storage - break default: return nil, fmt.Errorf("driver %s does not support", option.Driver) @@ -194,7 +192,7 @@ func New(option ManagerOption) (*Manager, error) { } // init allowedTypes - if option.AllowedTypes != nil && len(option.AllowedTypes) > 0 { + if len(option.AllowedTypes) > 0 { for _, t := range option.AllowedTypes { t = strings.TrimSpace(t) if strings.HasSuffix(t, "*") { @@ -546,16 +544,12 @@ func (manager Manager) generateFileID(file *FileHeader, extension string, option hash := fmt.Sprintf("%x", sha256.Sum256([]byte(filename)))[:8] date := time.Now().Format("20060102") path := filepath.Join("attachments", date) - if option.UserID != "" { - path = filepath.Join(path, option.UserID) - } - if option.ChatID != "" { - path = filepath.Join(path, option.ChatID) - } - - if option.AssistantID != "" { - path = filepath.Join(path, option.AssistantID) + // Build multi-level group path + for _, group := range option.Groups { + if group != "" { + path = filepath.Join(path, group) + } } id := filepath.Join(path, hash[:2], hash[2:4], hash) + extension diff --git a/neo/attachment/manager_test.go b/attachment/manager_test.go similarity index 65% rename from neo/attachment/manager_test.go rename to attachment/manager_test.go index 089180fe..13f14151 100644 --- a/neo/attachment/manager_test.go +++ b/attachment/manager_test.go @@ -40,11 +40,7 @@ func TestManagerUpload(t *testing.T) { } fileHeader.Header.Set("Content-Type", "text/plain") - option := UploadOption{ - UserID: "user123", - ChatID: "chat456", - AssistantID: "assistant789", - } + option := UploadOption{Groups: []string{"user123"}} file, err := manager.Upload(context.Background(), fileHeader, reader, option) if err != nil { @@ -103,7 +99,7 @@ func TestManagerUpload(t *testing.T) { option := UploadOption{ Gzip: true, - UserID: "user123", + Groups: []string{"user123"}, } file, err := manager.Upload(context.Background(), fileHeader, reader, option) @@ -152,7 +148,7 @@ func TestManagerUpload(t *testing.T) { fmt.Sprintf("bytes %d-%d/%d", start, end, totalSize)) fileHeader.Header.Set("Content-Uid", "unique-file-id-123") - option := UploadOption{UserID: "user123"} + option := UploadOption{Groups: []string{"user123"}} file, err := manager.Upload(context.Background(), fileHeader, bytes.NewReader(chunk), option) if err != nil { t.Fatalf("Failed to upload chunk starting at %d: %v", start, err) @@ -176,6 +172,122 @@ func TestManagerUpload(t *testing.T) { }) } +func TestManagerMultiLevelGroups(t *testing.T) { + // Create a local storage manager + manager, err := New(ManagerOption{ + Driver: "local", + MaxSize: "10M", + AllowedTypes: []string{"text/*", "image/*"}, + Options: map[string]interface{}{ + "path": "/tmp/test_attachments", + }, + }) + if err != nil { + t.Fatalf("Failed to create manager: %v", err) + } + + // Test multi-level groups + t.Run("MultiLevelGroups", func(t *testing.T) { + content := "Test content for multi-level groups" + reader := strings.NewReader(content) + + fileHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "multilevel.txt", + Size: int64(len(content)), + Header: make(map[string][]string), + }, + } + fileHeader.Header.Set("Content-Type", "text/plain") + + // Test with multi-level groups + option := UploadOption{ + Groups: []string{"users", "user123", "chats", "chat456", "documents"}, + } + + file, err := manager.Upload(context.Background(), fileHeader, reader, option) + if err != nil { + t.Fatalf("Failed to upload file with multi-level groups: %v", err) + } + + // Verify the file ID contains the nested structure + if !strings.Contains(file.ID, "users") || + !strings.Contains(file.ID, "user123") || + !strings.Contains(file.ID, "chats") || + !strings.Contains(file.ID, "chat456") || + !strings.Contains(file.ID, "documents") { + t.Errorf("File ID should contain all group levels: %s", file.ID) + } + + // Test download + downloadedContent, err := manager.Read(context.Background(), file.ID) + if err != nil { + t.Fatalf("Failed to read file with multi-level groups: %v", err) + } + + if string(downloadedContent) != content { + t.Errorf("Content mismatch for multi-level groups file") + } + }) + + // Test single group (backward compatibility) + t.Run("SingleGroup", func(t *testing.T) { + content := "Test content for single group" + reader := strings.NewReader(content) + + fileHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "single.txt", + Size: int64(len(content)), + Header: make(map[string][]string), + }, + } + fileHeader.Header.Set("Content-Type", "text/plain") + + option := UploadOption{ + Groups: []string{"knowledge"}, + } + + file, err := manager.Upload(context.Background(), fileHeader, reader, option) + if err != nil { + t.Fatalf("Failed to upload file with single group: %v", err) + } + + if !strings.Contains(file.ID, "knowledge") { + t.Errorf("File ID should contain group: %s", file.ID) + } + }) + + // Test empty groups (no grouping) + t.Run("EmptyGroups", func(t *testing.T) { + content := "Test content without groups" + reader := strings.NewReader(content) + + fileHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "nogroup.txt", + Size: int64(len(content)), + Header: make(map[string][]string), + }, + } + fileHeader.Header.Set("Content-Type", "text/plain") + + option := UploadOption{ + Groups: []string{}, // Empty groups + } + + file, err := manager.Upload(context.Background(), fileHeader, reader, option) + if err != nil { + t.Fatalf("Failed to upload file without groups: %v", err) + } + + // Should still work and create valid file ID + if file.ID == "" { + t.Error("File ID should not be empty") + } + }) +} + func TestManagerValidation(t *testing.T) { manager, err := New(ManagerOption{ Driver: "local", diff --git a/neo/attachment/s3/storage.go b/attachment/s3/storage.go similarity index 100% rename from neo/attachment/s3/storage.go rename to attachment/s3/storage.go diff --git a/neo/attachment/s3/storage_test.go b/attachment/s3/storage_test.go similarity index 100% rename from neo/attachment/s3/storage_test.go rename to attachment/s3/storage_test.go diff --git a/neo/attachment/types.go b/attachment/types.go similarity index 74% rename from neo/attachment/types.go rename to attachment/types.go index a1a03eb9..6dbd4fe3 100644 --- a/neo/attachment/types.go +++ b/attachment/types.go @@ -76,14 +76,11 @@ type allowedType struct { // UploadOption the upload option type UploadOption struct { - CompressImage bool `json:"compress_image,omitempty" form:"compress_image"` // Compress the file, Optional, default is true - CompressSize int `json:"compress_size,omitempty" form:"compress_size"` // Compress the file size, Optional, default is 1920, if compress_image is true, the file size will be compressed to the compress_size - Gzip bool `json:"gzip,omitempty" form:"gzip"` // Gzip the file, Optional, default is false - Knowledge bool `json:"knowledge,omitempty" form:"knowledge"` // Push to knowledge base, Optional, default is false - ChatID string `json:"chat_id,omitempty" form:"chat_id"` // Chat ID, Optional - AssistantID string `json:"assistant_id,omitempty" form:"assistant_id"` // Assistant ID, Optional - UserID string `json:"user_id,omitempty"` // User ID, Optional - OriginalFilename string `json:"original_filename,omitempty" form:"original_filename"` // Original filename sent separately to avoid encoding issues + CompressImage bool `json:"compress_image,omitempty" form:"compress_image"` // Compress the file, Optional, default is true + CompressSize int `json:"compress_size,omitempty" form:"compress_size"` // Compress the file size, Optional, default is 1920, if compress_image is true, the file size will be compressed to the compress_size + Gzip bool `json:"gzip,omitempty" form:"gzip"` // Gzip the file, Optional, default is false + OriginalFilename string `json:"original_filename,omitempty" form:"original_filename"` // Original filename sent separately to avoid encoding issues + Groups []string `json:"groups,omitempty" form:"groups"` // Groups, Optional, default is empty, Multi-level groups like ["user", "user123", "chat", "chat456"] } // FileHeader the file header diff --git a/neo/api.go b/neo/api.go index 8e906eb1..f9ff8470 100644 --- a/neo/api.go +++ b/neo/api.go @@ -14,9 +14,9 @@ import ( "github.com/yaoapp/gou/api" "github.com/yaoapp/gou/connector" "github.com/yaoapp/gou/process" + "github.com/yaoapp/yao/attachment" "github.com/yaoapp/yao/helper" "github.com/yaoapp/yao/neo/assistant" - "github.com/yaoapp/yao/neo/attachment" chatctx "github.com/yaoapp/yao/neo/context" "github.com/yaoapp/yao/neo/message" "github.com/yaoapp/yao/neo/store" @@ -219,21 +219,37 @@ func (neo *DSL) handleUpload(c *gin.Context) { // Validate the option with the storage option.UserID = fmt.Sprintf("%v", uid) - if storage == "chat" { + // Build multi-level groups based on storage type and IDs + var groups []string + switch storage { + case "chat": if option.ChatID == "" { c.JSON(400, gin.H{"message": "chat_id is required", "code": 400}) c.Done() return } - - } else if storage == "knowledge" { + // Build groups: ["users", "user123", "chats", "chat456"] + groups = []string{"users", option.UserID, "chats", option.ChatID} + if option.AssistantID != "" { + // Add assistant level: ["users", "user123", "chats", "chat456", "assistants", "assistant789"] + groups = append(groups, "assistants", option.AssistantID) + } + case "knowledge": if option.CollectionID == "" { c.JSON(400, gin.H{"message": "collection_id is required", "code": 400}) c.Done() return } + // Build groups: ["knowledge", "collection123", "users", "user456"] + groups = []string{"knowledge", option.CollectionID, "users", option.UserID} + case "assets": + // Build groups: ["assets", "users", "user123"] + groups = []string{"assets", "users", option.UserID} } + // Set the groups in the attachment upload option + option.UploadOption.Groups = groups + // Get the file file, err := c.FormFile("file") if err != nil { @@ -396,7 +412,6 @@ func (neo *DSL) handleDownload(c *gin.Context) { return } c.Done() - return } diff --git a/neo/load.go b/neo/load.go index 6d6d5394..8c14c682 100644 --- a/neo/load.go +++ b/neo/load.go @@ -7,9 +7,9 @@ import ( "github.com/yaoapp/gou/application" "github.com/yaoapp/gou/connector" "github.com/yaoapp/gou/model" + "github.com/yaoapp/yao/attachment" "github.com/yaoapp/yao/config" "github.com/yaoapp/yao/neo/assistant" - "github.com/yaoapp/yao/neo/attachment" "github.com/yaoapp/yao/neo/i18n" "github.com/yaoapp/yao/neo/store" ) diff --git a/neo/types.go b/neo/types.go index 557cccf8..9cbbf259 100644 --- a/neo/types.go +++ b/neo/types.go @@ -2,8 +2,8 @@ package neo import ( "github.com/gin-gonic/gin" + "github.com/yaoapp/yao/attachment" "github.com/yaoapp/yao/neo/assistant" - "github.com/yaoapp/yao/neo/attachment" "github.com/yaoapp/yao/neo/rag" "github.com/yaoapp/yao/neo/store" "github.com/yaoapp/yao/neo/vision" @@ -92,6 +92,10 @@ type UploadOption struct { Public bool `json:"public,omitempty" yaml:"public,omitempty, form:public"` // The public of the file, default is false Scope interface{} `json:"scope,omitempty" yaml:"scope,omitempty, form:scope"` // The scope of the file, default is private CollectionID string `json:"collection_id,omitempty" yaml:"collection_id,omitempty, form:collection_id"` // The collection id of the file, default is empty + Knowledge bool `json:"knowledge,omitempty" form:"knowledge"` // Push to knowledge base, Optional, default is false + ChatID string `json:"chat_id,omitempty" form:"chat_id"` // Chat ID, Optional + AssistantID string `json:"assistant_id,omitempty" form:"assistant_id"` // Assistant ID, Optional + UserID string `json:"user_id,omitempty"` // User ID, Optional (used to build Groups) } // Knowledge base Settings