Merge pull request #1041 from trheyi/main

Refactor attachment handling and enhance upload options

attachment/README.md

@@ -14,7 +14,7 @@ A comprehensive file upload package for Go that supports chunked uploads, file f
   - File size limits
   - MIME type and extension validation
   - Wildcard pattern support (e.g., `image/*`, `text/*`)
-- **Flexible File Organization**: Hierarchical storage with user/chat/assistant organization
+- **Flexible File Organization**: Hierarchical storage with multi-level group organization
 - **Multiple Read Methods**: Stream, bytes, and base64 encoding
 - **Global Manager Registry**: Support for registering and accessing managers globally
 - **Upload Status Tracking**: Track upload progress with status field
@@ -73,8 +73,7 @@ func main() {
     fileHeader.Header.Set("Content-Type", "text/plain")
 
     option := attachment.UploadOption{
-        UserID:           "user123",
+        Groups:           []string{"user123", "chat456"}, // Multi-level groups (e.g., user, chat, knowledge, etc.)
-        ChatID:           "chat456",
         OriginalFilename: "my_document.txt", // Preserve original filename
     }
 
@@ -196,6 +195,29 @@ option := attachment.UploadOption{
 file, err := manager.Upload(ctx, imageHeader, imageReader, option)
 ```
 
+### Multi-level Groups
+
+The `Groups` field supports hierarchical file organization:
+
+```go
+// Single level grouping
+option := attachment.UploadOption{
+    Groups: []string{"users"},
+}
+
+// Multi-level grouping
+option := attachment.UploadOption{
+    Groups: []string{"users", "user123", "chats", "chat456"},
+}
+
+// Knowledge base organization
+option := attachment.UploadOption{
+    Groups: []string{"knowledge", "documents", "technical"},
+}
+```
+
+This creates nested directory structures for better organization and access control.
+
 ### File Validation
 
 #### Size Limits
@@ -295,9 +317,9 @@ Files are organized in a hierarchical structure:
 ```
 attachments/
 ├── 20240101/           # Date (YYYYMMDD)
-│   └── user123/        # User ID (optional)
+│   └── user123/        # First level group (optional)
-│       └── chat456/    # Chat ID (optional)
+│       └── chat456/    # Second level group (optional)
-│           └── assistant789/  # Assistant ID (optional)
+│           └── knowledge/  # Additional group levels (optional)
 │               └── ab/     # First 2 chars of hash
 │                   └── cd/ # Next 2 chars of hash
 │                       └── abcdef12.txt  # Hash + extension
@@ -306,7 +328,7 @@ attachments/
 The file ID generation includes:
 
 - Date prefix for organization
-- User/Chat/Assistant IDs for access control
+- Multi-level groups for access control and organization
 - Content hash for deduplication
 - Original file extension
 
@@ -374,8 +396,7 @@ Options for file upload:
 - `CompressImage`: Enable image compression
 - `CompressSize`: Maximum image dimension (default: 1920)
 - `Gzip`: Enable gzip compression
-- `Knowledge`: Push to knowledge base
+- `Groups`: Multi-level group identifiers for hierarchical file organization (e.g., []string{"user123", "chat456", "knowledge"})
-- `UserID`, `ChatID`, `AssistantID`: Organization IDs
 - `OriginalFilename`: Original filename to preserve (avoids encoding issues)
 
 #### `File`
@@ -465,7 +486,7 @@ The package includes comprehensive tests for:
 - **File Type Validation**: Prevents upload of unauthorized file types
 - **Size Limits**: Configurable file size restrictions
 - **Path Sanitization**: Secure file path generation
-- **Access Control**: User/Chat/Assistant-based file organization
+- **Access Control**: Multi-level hierarchical file organization
 
 ## License
 
@@ -536,8 +557,7 @@ for start := int64(0); start < totalSize; start += chunkSize {
     chunkHeader.Header.Set("Content-Sync", "true") // Enable synchronization
 
     option := attachment.UploadOption{
-        UserID:           "user123",
+        Groups:           []string{"user123", "chat456"}, // Multi-level groups
-        ChatID:           "chat456",
         OriginalFilename: "my_large_file.zip", // Preserve original name
     }
 

attachment/example_usage.go

@@ -8,7 +8,7 @@ import (
 	"mime/multipart"
 	"strings"
 
-	"github.com/yaoapp/yao/neo/attachment/s3"
+	"github.com/yaoapp/yao/attachment/s3"
 )
 
 // ExampleUsage demonstrates how to use the attachment package
@@ -95,9 +95,7 @@ func ExampleUsage() {
 	fileHeader.Header.Set("Content-Type", "text/plain")
 
 	uploadOption := UploadOption{
-		UserID:      "user123",
+		Groups: []string{"user123"},
-		ChatID:      "chat456",
-		AssistantID: "assistant789",
 		Gzip:   false, // No compression for small text files
 	}
 
@@ -120,7 +118,7 @@ func ExampleUsage() {
 	gzipFileHeader.Header.Set("Content-Type", "text/plain")
 
 	gzipOption := UploadOption{
-		UserID: "user123",
+		Groups: []string{"user123"},
 		Gzip:   true, // Enable compression
 	}
 
@@ -133,7 +131,7 @@ func ExampleUsage() {
 
 	// 6. Example: Image upload with compression
 	imageUploadOption := UploadOption{
-		UserID:        "user123",
+		Groups:        []string{"user123"},
 		CompressImage: true,
 		CompressSize:  1920, // Resize to max 1920px
 		Gzip:          false,
@@ -152,6 +150,75 @@ func ExampleUsage() {
 	fmt.Printf("Image upload option configured: compress=%v, size=%d\n",
 		imageUploadOption.CompressImage, imageUploadOption.CompressSize)
 
+	// 6.5. Example: Multi-level groups
+	fmt.Println("\n--- Multi-level Groups Examples ---")
+
+	// Single level grouping
+	singleGroupOption := UploadOption{
+		Groups: []string{"knowledge"},
+	}
+
+	singleGroupHeader := &FileHeader{
+		FileHeader: &multipart.FileHeader{
+			Filename: "knowledge_doc.txt",
+			Size:     int64(len("Knowledge base document")),
+			Header:   make(map[string][]string),
+		},
+	}
+	singleGroupHeader.Header.Set("Content-Type", "text/plain")
+
+	singleFile, err := localManager.Upload(ctx, singleGroupHeader, strings.NewReader("Knowledge base document"), singleGroupOption)
+	if err != nil {
+		log.Printf("Failed to upload single group file: %v", err)
+	} else {
+		fmt.Printf("Single group file uploaded: %s (ID: %s)\n", singleFile.Filename, singleFile.ID)
+	}
+
+	// Multi-level grouping
+	multiGroupOption := UploadOption{
+		Groups: []string{"users", "user123", "chats", "chat456", "documents"},
+	}
+
+	multiGroupHeader := &FileHeader{
+		FileHeader: &multipart.FileHeader{
+			Filename: "chat_document.txt",
+			Size:     int64(len("Document in user chat")),
+			Header:   make(map[string][]string),
+		},
+	}
+	multiGroupHeader.Header.Set("Content-Type", "text/plain")
+
+	multiFile, err := localManager.Upload(ctx, multiGroupHeader, strings.NewReader("Document in user chat"), multiGroupOption)
+	if err != nil {
+		log.Printf("Failed to upload multi-group file: %v", err)
+	} else {
+		fmt.Printf("Multi-level group file uploaded: %s (ID: %s)\n", multiFile.Filename, multiFile.ID)
+		fmt.Printf("File path includes hierarchy: users/user123/chats/chat456/documents\n")
+	}
+
+	// Knowledge base organization
+	knowledgeOption := UploadOption{
+		Groups: []string{"knowledge", "technical", "api", "documentation"},
+	}
+
+	knowledgeHeader := &FileHeader{
+		FileHeader: &multipart.FileHeader{
+			Filename: "api_guide.md",
+			Size:     int64(len("# API Documentation\n\nThis is technical documentation.")),
+			Header:   make(map[string][]string),
+		},
+	}
+	knowledgeHeader.Header.Set("Content-Type", "text/markdown")
+
+	knowledgeFile, err := localManager.Upload(ctx, knowledgeHeader,
+		strings.NewReader("# API Documentation\n\nThis is technical documentation."), knowledgeOption)
+	if err != nil {
+		log.Printf("Failed to upload knowledge file: %v", err)
+	} else {
+		fmt.Printf("Knowledge base file uploaded: %s (ID: %s)\n", knowledgeFile.Filename, knowledgeFile.ID)
+		fmt.Printf("Organized in: knowledge/technical/api/documentation\n")
+	}
+
 	// 7. Example: Chunked upload
 	largeContent = strings.Repeat("This is a large file content that will be uploaded in chunks. ", 1000)
 	chunkSize := 1024
@@ -183,7 +250,7 @@ func ExampleUsage() {
 		chunkHeader.Header.Set("Content-Uid", uid)
 
 		chunkOption := UploadOption{
-			UserID: "user123",
+			Groups: []string{"user123"},
 			Gzip:   true, // Compress chunks
 		}
 
@@ -273,7 +340,7 @@ func ExampleUsage() {
 		testHeader.Header.Set("Content-Type", "text/plain")
 
 		testFile, err := globalManager.Upload(ctx, testHeader, strings.NewReader(testContent), UploadOption{
-			UserID: "global_user",
+			Groups: []string{"global_user"},
 		})
 		if err != nil {
 			log.Printf("Failed to upload with global manager: %v", err)
@@ -356,7 +423,7 @@ func ExampleChunkedUpload(manager *Manager, filename string, totalSize int64, co
 		}
 
 		option := UploadOption{
-			UserID: "user123",
+			Groups: []string{"user123"},
 			Gzip:   false, // Disable compression for this example
 		}
 
@@ -422,7 +489,7 @@ func ExampleS3Upload() {
 	fileHeader.Header.Set("Content-Type", "text/plain")
 
 	file, err := s3Manager.Upload(ctx, fileHeader, strings.NewReader(content), UploadOption{
-		UserID: "s3_user",
+		Groups: []string{"s3_user"},
 	})
 	if err != nil {
 		log.Printf("S3 upload failed: %v", err)

attachment/manager.go

@@ -18,9 +18,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/yaoapp/yao/attachment/local"
+	"github.com/yaoapp/yao/attachment/s3"
 	"github.com/yaoapp/yao/config"
-	"github.com/yaoapp/yao/neo/attachment/local"
-	"github.com/yaoapp/yao/neo/attachment/s3"
 )
 
 // Managers the managers
@@ -161,7 +161,6 @@ func New(option ManagerOption) (*Manager, error) {
 			return nil, err
 		}
 		manager.storage = storage
-		break
 
 	case "s3":
 		storage, err := s3.New(option.Options)
@@ -169,7 +168,6 @@ func New(option ManagerOption) (*Manager, error) {
 			return nil, err
 		}
 		manager.storage = storage
-		break
 
 	default:
 		return nil, fmt.Errorf("driver %s does not support", option.Driver)
@@ -194,7 +192,7 @@ func New(option ManagerOption) (*Manager, error) {
 	}
 
 	// init allowedTypes
-	if option.AllowedTypes != nil && len(option.AllowedTypes) > 0 {
+	if len(option.AllowedTypes) > 0 {
 		for _, t := range option.AllowedTypes {
 			t = strings.TrimSpace(t)
 			if strings.HasSuffix(t, "*") {
@@ -546,16 +544,12 @@ func (manager Manager) generateFileID(file *FileHeader, extension string, option
 	hash := fmt.Sprintf("%x", sha256.Sum256([]byte(filename)))[:8]
 	date := time.Now().Format("20060102")
 	path := filepath.Join("attachments", date)
-	if option.UserID != "" {
-		path = filepath.Join(path, option.UserID)
-	}
 
-	if option.ChatID != "" {
+	// Build multi-level group path
-		path = filepath.Join(path, option.ChatID)
+	for _, group := range option.Groups {
+		if group != "" {
+			path = filepath.Join(path, group)
 		}
-
-	if option.AssistantID != "" {
-		path = filepath.Join(path, option.AssistantID)
 	}
 
 	id := filepath.Join(path, hash[:2], hash[2:4], hash) + extension

attachment/manager_test.go

@@ -40,11 +40,7 @@ func TestManagerUpload(t *testing.T) {
 		}
 		fileHeader.Header.Set("Content-Type", "text/plain")
 
-		option := UploadOption{
+		option := UploadOption{Groups: []string{"user123"}}
-			UserID:      "user123",
-			ChatID:      "chat456",
-			AssistantID: "assistant789",
-		}
 
 		file, err := manager.Upload(context.Background(), fileHeader, reader, option)
 		if err != nil {
@@ -103,7 +99,7 @@ func TestManagerUpload(t *testing.T) {
 
 		option := UploadOption{
 			Gzip:   true,
-			UserID: "user123",
+			Groups: []string{"user123"},
 		}
 
 		file, err := manager.Upload(context.Background(), fileHeader, reader, option)
@@ -152,7 +148,7 @@ func TestManagerUpload(t *testing.T) {
 				fmt.Sprintf("bytes %d-%d/%d", start, end, totalSize))
 			fileHeader.Header.Set("Content-Uid", "unique-file-id-123")
 
-			option := UploadOption{UserID: "user123"}
+			option := UploadOption{Groups: []string{"user123"}}
 			file, err := manager.Upload(context.Background(), fileHeader, bytes.NewReader(chunk), option)
 			if err != nil {
 				t.Fatalf("Failed to upload chunk starting at %d: %v", start, err)
@@ -176,6 +172,122 @@ func TestManagerUpload(t *testing.T) {
 	})
 }
 
+func TestManagerMultiLevelGroups(t *testing.T) {
+	// Create a local storage manager
+	manager, err := New(ManagerOption{
+		Driver:       "local",
+		MaxSize:      "10M",
+		AllowedTypes: []string{"text/*", "image/*"},
+		Options: map[string]interface{}{
+			"path": "/tmp/test_attachments",
+		},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create manager: %v", err)
+	}
+
+	// Test multi-level groups
+	t.Run("MultiLevelGroups", func(t *testing.T) {
+		content := "Test content for multi-level groups"
+		reader := strings.NewReader(content)
+
+		fileHeader := &FileHeader{
+			FileHeader: &multipart.FileHeader{
+				Filename: "multilevel.txt",
+				Size:     int64(len(content)),
+				Header:   make(map[string][]string),
+			},
+		}
+		fileHeader.Header.Set("Content-Type", "text/plain")
+
+		// Test with multi-level groups
+		option := UploadOption{
+			Groups: []string{"users", "user123", "chats", "chat456", "documents"},
+		}
+
+		file, err := manager.Upload(context.Background(), fileHeader, reader, option)
+		if err != nil {
+			t.Fatalf("Failed to upload file with multi-level groups: %v", err)
+		}
+
+		// Verify the file ID contains the nested structure
+		if !strings.Contains(file.ID, "users") ||
+			!strings.Contains(file.ID, "user123") ||
+			!strings.Contains(file.ID, "chats") ||
+			!strings.Contains(file.ID, "chat456") ||
+			!strings.Contains(file.ID, "documents") {
+			t.Errorf("File ID should contain all group levels: %s", file.ID)
+		}
+
+		// Test download
+		downloadedContent, err := manager.Read(context.Background(), file.ID)
+		if err != nil {
+			t.Fatalf("Failed to read file with multi-level groups: %v", err)
+		}
+
+		if string(downloadedContent) != content {
+			t.Errorf("Content mismatch for multi-level groups file")
+		}
+	})
+
+	// Test single group (backward compatibility)
+	t.Run("SingleGroup", func(t *testing.T) {
+		content := "Test content for single group"
+		reader := strings.NewReader(content)
+
+		fileHeader := &FileHeader{
+			FileHeader: &multipart.FileHeader{
+				Filename: "single.txt",
+				Size:     int64(len(content)),
+				Header:   make(map[string][]string),
+			},
+		}
+		fileHeader.Header.Set("Content-Type", "text/plain")
+
+		option := UploadOption{
+			Groups: []string{"knowledge"},
+		}
+
+		file, err := manager.Upload(context.Background(), fileHeader, reader, option)
+		if err != nil {
+			t.Fatalf("Failed to upload file with single group: %v", err)
+		}
+
+		if !strings.Contains(file.ID, "knowledge") {
+			t.Errorf("File ID should contain group: %s", file.ID)
+		}
+	})
+
+	// Test empty groups (no grouping)
+	t.Run("EmptyGroups", func(t *testing.T) {
+		content := "Test content without groups"
+		reader := strings.NewReader(content)
+
+		fileHeader := &FileHeader{
+			FileHeader: &multipart.FileHeader{
+				Filename: "nogroup.txt",
+				Size:     int64(len(content)),
+				Header:   make(map[string][]string),
+			},
+		}
+		fileHeader.Header.Set("Content-Type", "text/plain")
+
+		option := UploadOption{
+			Groups: []string{}, // Empty groups
+		}
+
+		file, err := manager.Upload(context.Background(), fileHeader, reader, option)
+		if err != nil {
+			t.Fatalf("Failed to upload file without groups: %v", err)
+		}
+
+		// Should still work and create valid file ID
+		if file.ID == "" {
+			t.Error("File ID should not be empty")
+		}
+	})
+}
+
 func TestManagerValidation(t *testing.T) {
 	manager, err := New(ManagerOption{
 		Driver:       "local",

attachment/types.go

@@ -79,11 +79,8 @@ type UploadOption struct {
 	CompressImage    bool     `json:"compress_image,omitempty" form:"compress_image"`       // Compress the file, Optional, default is true
 	CompressSize     int      `json:"compress_size,omitempty" form:"compress_size"`         // Compress the file size, Optional, default is 1920, if compress_image is true, the file size will be compressed to the compress_size
 	Gzip             bool     `json:"gzip,omitempty" form:"gzip"`                           // Gzip the file, Optional, default is false
-	Knowledge        bool   `json:"knowledge,omitempty" form:"knowledge"`                 // Push to knowledge base, Optional, default is false
-	ChatID           string `json:"chat_id,omitempty" form:"chat_id"`                     // Chat ID, Optional
-	AssistantID      string `json:"assistant_id,omitempty" form:"assistant_id"`           // Assistant ID, Optional
-	UserID           string `json:"user_id,omitempty"`                                    // User ID, Optional
 	OriginalFilename string   `json:"original_filename,omitempty" form:"original_filename"` // Original filename sent separately to avoid encoding issues
+	Groups           []string `json:"groups,omitempty" form:"groups"`                       // Groups, Optional, default is empty, Multi-level groups like ["user", "user123", "chat", "chat456"]
 }
 
 // FileHeader the file header

neo/api.go

@@ -14,9 +14,9 @@ import (
 	"github.com/yaoapp/gou/api"
 	"github.com/yaoapp/gou/connector"
 	"github.com/yaoapp/gou/process"
+	"github.com/yaoapp/yao/attachment"
 	"github.com/yaoapp/yao/helper"
 	"github.com/yaoapp/yao/neo/assistant"
-	"github.com/yaoapp/yao/neo/attachment"
 	chatctx "github.com/yaoapp/yao/neo/context"
 	"github.com/yaoapp/yao/neo/message"
 	"github.com/yaoapp/yao/neo/store"
@@ -219,21 +219,37 @@ func (neo *DSL) handleUpload(c *gin.Context) {
 	// Validate the option with the storage
 	option.UserID = fmt.Sprintf("%v", uid)
 
-	if storage == "chat" {
+	// Build multi-level groups based on storage type and IDs
+	var groups []string
+	switch storage {
+	case "chat":
 		if option.ChatID == "" {
 			c.JSON(400, gin.H{"message": "chat_id is required", "code": 400})
 			c.Done()
 			return
 		}
-
+		// Build groups: ["users", "user123", "chats", "chat456"]
-	} else if storage == "knowledge" {
+		groups = []string{"users", option.UserID, "chats", option.ChatID}
+		if option.AssistantID != "" {
+			// Add assistant level: ["users", "user123", "chats", "chat456", "assistants", "assistant789"]
+			groups = append(groups, "assistants", option.AssistantID)
+		}
+	case "knowledge":
 		if option.CollectionID == "" {
 			c.JSON(400, gin.H{"message": "collection_id is required", "code": 400})
 			c.Done()
 			return
 		}
+		// Build groups: ["knowledge", "collection123", "users", "user456"]
+		groups = []string{"knowledge", option.CollectionID, "users", option.UserID}
+	case "assets":
+		// Build groups: ["assets", "users", "user123"]
+		groups = []string{"assets", "users", option.UserID}
 	}
 
+	// Set the groups in the attachment upload option
+	option.UploadOption.Groups = groups
+
 	// Get the file
 	file, err := c.FormFile("file")
 	if err != nil {
@@ -396,7 +412,6 @@ func (neo *DSL) handleDownload(c *gin.Context) {
 		return
 	}
 	c.Done()
-	return
 
 }
 

neo/load.go

@@ -7,9 +7,9 @@ import (
 	"github.com/yaoapp/gou/application"
 	"github.com/yaoapp/gou/connector"
 	"github.com/yaoapp/gou/model"
+	"github.com/yaoapp/yao/attachment"
 	"github.com/yaoapp/yao/config"
 	"github.com/yaoapp/yao/neo/assistant"
-	"github.com/yaoapp/yao/neo/attachment"
 	"github.com/yaoapp/yao/neo/i18n"
 	"github.com/yaoapp/yao/neo/store"
 )

neo/types.go

@@ -2,8 +2,8 @@ package neo
 
 import (
 	"github.com/gin-gonic/gin"
+	"github.com/yaoapp/yao/attachment"
 	"github.com/yaoapp/yao/neo/assistant"
-	"github.com/yaoapp/yao/neo/attachment"
 	"github.com/yaoapp/yao/neo/rag"
 	"github.com/yaoapp/yao/neo/store"
 	"github.com/yaoapp/yao/neo/vision"
@@ -92,6 +92,10 @@ type UploadOption struct {
 	Public       bool        `json:"public,omitempty" yaml:"public,omitempty, form:public"`                      // The public of the file, default is false
 	Scope        interface{} `json:"scope,omitempty" yaml:"scope,omitempty, form:scope"`                         // The scope of the file, default is private
 	CollectionID string      `json:"collection_id,omitempty" yaml:"collection_id,omitempty, form:collection_id"` // The collection id of the file, default is empty
+	Knowledge    bool        `json:"knowledge,omitempty" form:"knowledge"`                                       // Push to knowledge base, Optional, default is false
+	ChatID       string      `json:"chat_id,omitempty" form:"chat_id"`                                           // Chat ID, Optional
+	AssistantID  string      `json:"assistant_id,omitempty" form:"assistant_id"`                                 // Assistant ID, Optional
+	UserID       string      `json:"user_id,omitempty"`                                                          // User ID, Optional (used to build Groups)
 }
 
 // Knowledge base Settings