diff --git a/attachment/local/storage.go b/attachment/local/storage.go
index 0ba0ce1c..65db9196 100644
--- a/attachment/local/storage.go
+++ b/attachment/local/storage.go
@@ -6,6 +6,8 @@ import (
"crypto/sha256"
"fmt"
"io"
+ "mime"
+ "net/http"
"os"
"path/filepath"
"strings"
@@ -259,3 +261,250 @@ func (storage *Storage) makeID(filename string, ext string) string {
name := strings.TrimSuffix(filepath.Base(filename), ext)
return fmt.Sprintf("%s/%s-%s%s", date, name, hash, ext)
}
+
+// LocalPath returns the absolute path of the file and its content type
+func (storage *Storage) LocalPath(ctx context.Context, path string) (string, string, error) {
+ fullPath := filepath.Join(storage.Path, path)
+
+ // Check if file exists
+ if _, err := os.Stat(fullPath); os.IsNotExist(err) {
+ return "", "", fmt.Errorf("file not found: %s", path)
+ }
+
+ // For gzipped files, we need to detect the original content type, not the gzip wrapper
+ var contentType string
+ var err error
+
+ if strings.HasSuffix(path, ".gz") {
+ // For gzipped files, detect content type of the decompressed content
+ originalPath := strings.TrimSuffix(path, ".gz")
+ ext := filepath.Ext(originalPath)
+
+ // First try to detect by original file extension
+ contentType, err = detectContentTypeFromExtension(ext)
+ if err != nil || contentType == "application/octet-stream" {
+ // Fallback: decompress and detect from content
+ contentType, err = detectContentTypeFromGzippedFile(fullPath)
+ if err != nil {
+ return "", "", fmt.Errorf("failed to detect content type from gzipped file: %w", err)
+ }
+ }
+ } else {
+ // Regular file content type detection
+ contentType, err = detectContentType(fullPath)
+ if err != nil {
+ return "", "", fmt.Errorf("failed to detect content type: %w", err)
+ }
+ }
+
+ // Return absolute path
+ absPath, err := filepath.Abs(fullPath)
+ if err != nil {
+ return "", "", fmt.Errorf("failed to get absolute path: %w", err)
+ }
+
+ return absPath, contentType, nil
+}
+
+// detectContentType detects content type based on file extension and content
+func detectContentType(filePath string) (string, error) {
+ // First try to detect by file extension
+ ext := strings.ToLower(filepath.Ext(filePath))
+
+ // Common file extensions mapping
+ switch ext {
+ case ".txt":
+ return "text/plain", nil
+ case ".html", ".htm":
+ return "text/html", nil
+ case ".css":
+ return "text/css", nil
+ case ".js":
+ return "application/javascript", nil
+ case ".json":
+ return "application/json", nil
+ case ".xml":
+ return "application/xml", nil
+ case ".jpg", ".jpeg":
+ return "image/jpeg", nil
+ case ".png":
+ return "image/png", nil
+ case ".gif":
+ return "image/gif", nil
+ case ".webp":
+ return "image/webp", nil
+ case ".svg":
+ return "image/svg+xml", nil
+ case ".pdf":
+ return "application/pdf", nil
+ case ".doc":
+ return "application/msword", nil
+ case ".docx":
+ return "application/vnd.openxmlformats-officedocument.wordprocessingml.document", nil
+ case ".xls":
+ return "application/vnd.ms-excel", nil
+ case ".xlsx":
+ return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", nil
+ case ".ppt":
+ return "application/vnd.ms-powerpoint", nil
+ case ".pptx":
+ return "application/vnd.openxmlformats-officedocument.presentationml.presentation", nil
+ case ".zip":
+ return "application/zip", nil
+ case ".tar":
+ return "application/x-tar", nil
+ case ".gz":
+ return "application/gzip", nil
+ case ".mp3":
+ return "audio/mpeg", nil
+ case ".wav":
+ return "audio/wav", nil
+ case ".m4a":
+ return "audio/mp4", nil
+ case ".ogg":
+ return "audio/ogg", nil
+ case ".mp4":
+ return "video/mp4", nil
+ case ".avi":
+ return "video/x-msvideo", nil
+ case ".mov":
+ return "video/quicktime", nil
+ case ".webm":
+ return "video/webm", nil
+ case ".md", ".mdx":
+ return "text/markdown", nil
+ case ".yao":
+ return "application/yao", nil
+ case ".csv":
+ return "text/csv", nil
+ }
+
+ // Try to detect by MIME package
+ if contentType := mime.TypeByExtension(ext); contentType != "" {
+ return contentType, nil
+ }
+
+ // Fallback: detect by reading file content
+ file, err := os.Open(filePath)
+ if err != nil {
+ return "application/octet-stream", nil // Default fallback
+ }
+ defer file.Close()
+
+ // Read first 512 bytes for content detection
+ buffer := make([]byte, 512)
+ n, err := file.Read(buffer)
+ if err != nil && err != io.EOF {
+ return "application/octet-stream", nil
+ }
+
+ // Use http.DetectContentType to detect based on content
+ contentType := http.DetectContentType(buffer[:n])
+ return contentType, nil
+}
+
+// detectContentTypeFromExtension detects content type based only on file extension
+func detectContentTypeFromExtension(ext string) (string, error) {
+ ext = strings.ToLower(ext)
+
+ // Common file extensions mapping
+ switch ext {
+ case ".txt":
+ return "text/plain", nil
+ case ".html", ".htm":
+ return "text/html", nil
+ case ".css":
+ return "text/css", nil
+ case ".js":
+ return "application/javascript", nil
+ case ".json":
+ return "application/json", nil
+ case ".xml":
+ return "application/xml", nil
+ case ".jpg", ".jpeg":
+ return "image/jpeg", nil
+ case ".png":
+ return "image/png", nil
+ case ".gif":
+ return "image/gif", nil
+ case ".webp":
+ return "image/webp", nil
+ case ".svg":
+ return "image/svg+xml", nil
+ case ".pdf":
+ return "application/pdf", nil
+ case ".doc":
+ return "application/msword", nil
+ case ".docx":
+ return "application/vnd.openxmlformats-officedocument.wordprocessingml.document", nil
+ case ".xls":
+ return "application/vnd.ms-excel", nil
+ case ".xlsx":
+ return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", nil
+ case ".ppt":
+ return "application/vnd.ms-powerpoint", nil
+ case ".pptx":
+ return "application/vnd.openxmlformats-officedocument.presentationml.presentation", nil
+ case ".zip":
+ return "application/zip", nil
+ case ".tar":
+ return "application/x-tar", nil
+ case ".mp3":
+ return "audio/mpeg", nil
+ case ".wav":
+ return "audio/wav", nil
+ case ".m4a":
+ return "audio/mp4", nil
+ case ".ogg":
+ return "audio/ogg", nil
+ case ".mp4":
+ return "video/mp4", nil
+ case ".avi":
+ return "video/x-msvideo", nil
+ case ".mov":
+ return "video/quicktime", nil
+ case ".webm":
+ return "video/webm", nil
+ case ".md", ".mdx":
+ return "text/markdown", nil
+ case ".yao":
+ return "application/yao", nil
+ case ".csv":
+ return "text/csv", nil
+ }
+
+ // Try to detect by MIME package
+ if contentType := mime.TypeByExtension(ext); contentType != "" {
+ return contentType, nil
+ }
+
+ // Return default if not found
+ return "application/octet-stream", nil
+}
+
+// detectContentTypeFromGzippedFile detects content type by decompressing and reading gzipped file
+func detectContentTypeFromGzippedFile(gzippedFilePath string) (string, error) {
+ file, err := os.Open(gzippedFilePath)
+ if err != nil {
+ return "", err
+ }
+ defer file.Close()
+
+ // Create gzip reader
+ gzipReader, err := gzip.NewReader(file)
+ if err != nil {
+ return "", err
+ }
+ defer gzipReader.Close()
+
+ // Read first 512 bytes of decompressed content
+ buffer := make([]byte, 512)
+ n, err := gzipReader.Read(buffer)
+ if err != nil && err != io.EOF {
+ return "", err
+ }
+
+ // Use http.DetectContentType to detect based on decompressed content
+ contentType := http.DetectContentType(buffer[:n])
+ return contentType, nil
+}
diff --git a/attachment/local/storage_test.go b/attachment/local/storage_test.go
index 793164ee..d1e41d82 100644
--- a/attachment/local/storage_test.go
+++ b/attachment/local/storage_test.go
@@ -223,4 +223,81 @@ func TestLocalStorage(t *testing.T) {
exists = storage.Exists(context.Background(), fileID)
assert.False(t, exists)
})
+
+ t.Run("LocalPath", func(t *testing.T) {
+ storage, err := New(map[string]interface{}{
+ "path": testPath,
+ })
+ assert.NoError(t, err)
+
+ // Test different file types to verify content type detection
+ testFiles := []struct {
+ name string
+ content []byte
+ contentType string
+ expectedCT string
+ }{
+ {"test.txt", []byte("Hello World"), "text/plain", "text/plain"},
+ {"test.json", []byte(`{"key": "value"}`), "application/json", "application/json"},
+ {"test.html", []byte("
Test"), "text/html", "text/html"},
+ {"test.csv", []byte("col1,col2\nval1,val2"), "text/csv", "text/csv"},
+ {"test.md", []byte("# Markdown Content"), "text/markdown", "text/markdown"},
+ {"test.yao", []byte("yao file content"), "application/yao", "application/yao"},
+ }
+
+ for _, tf := range testFiles {
+ // Upload file
+ _, err = storage.Upload(context.Background(), tf.name, bytes.NewReader(tf.content), tf.contentType)
+ assert.NoError(t, err, "Failed to upload %s", tf.name)
+
+ // Get local path and content type
+ localPath, detectedCT, err := storage.LocalPath(context.Background(), tf.name)
+ assert.NoError(t, err, "Failed to get local path for %s", tf.name)
+ assert.NotEmpty(t, localPath, "Local path should not be empty for %s", tf.name)
+ assert.Equal(t, tf.expectedCT, detectedCT, "Content type mismatch for %s", tf.name)
+
+ // Verify the path is absolute
+ assert.True(t, filepath.IsAbs(localPath), "Path should be absolute for %s", tf.name)
+
+ // Verify the file exists at the returned path
+ _, err = os.Stat(localPath)
+ assert.NoError(t, err, "File should exist at local path for %s", tf.name)
+
+ // Verify file content
+ fileContent, err := os.ReadFile(localPath)
+ assert.NoError(t, err, "Failed to read file at local path for %s", tf.name)
+ assert.Equal(t, tf.content, fileContent, "File content mismatch for %s", tf.name)
+ }
+ })
+
+ t.Run("LocalPath_NonExistentFile", func(t *testing.T) {
+ storage, err := New(map[string]interface{}{
+ "path": testPath,
+ })
+ assert.NoError(t, err)
+
+ // Test with non-existent file
+ _, _, err = storage.LocalPath(context.Background(), "non-existent.txt")
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "file not found")
+ })
+
+ t.Run("LocalPath_ContentDetection", func(t *testing.T) {
+ storage, err := New(map[string]interface{}{
+ "path": testPath,
+ })
+ assert.NoError(t, err)
+
+ // Upload a file without extension but with recognizable content
+ htmlContent := []byte("TestHello
")
+ _, err = storage.Upload(context.Background(), "noext", bytes.NewReader(htmlContent), "application/octet-stream")
+ assert.NoError(t, err)
+
+ // Get local path - should detect HTML content type
+ localPath, contentType, err := storage.LocalPath(context.Background(), "noext")
+ assert.NoError(t, err)
+ assert.NotEmpty(t, localPath)
+ // Content detection should identify this as HTML
+ assert.Equal(t, "text/html; charset=utf-8", contentType)
+ })
}
diff --git a/attachment/manager.go b/attachment/manager.go
index fbe2585d..61303cd4 100644
--- a/attachment/manager.go
+++ b/attachment/manager.go
@@ -215,6 +215,18 @@ func New(option ManagerOption) (*Manager, error) {
return manager, nil
}
+// LocalPath gets the local path of the file
+func (manager Manager) LocalPath(ctx context.Context, fileID string) (string, string, error) {
+ // Get the real storage path from database
+ storagePath, err := manager.getStoragePathFromDatabase(ctx, fileID)
+ if err != nil {
+ return "", "", err
+ }
+
+ // Call the storage implementation
+ return manager.storage.LocalPath(ctx, storagePath)
+}
+
// Upload uploads a file, Content-Sync must be true for chunked upload
func (manager Manager) Upload(ctx context.Context, fileheader *FileHeader, reader io.Reader, option UploadOption) (*File, error) {
diff --git a/attachment/manager_test.go b/attachment/manager_test.go
index 21ebc129..29e7e2d5 100644
--- a/attachment/manager_test.go
+++ b/attachment/manager_test.go
@@ -7,6 +7,7 @@ import (
"fmt"
"mime/multipart"
"os"
+ "path/filepath"
"strings"
"testing"
"time"
@@ -828,3 +829,282 @@ func TestList(t *testing.T) {
t.Logf("Successfully tested list functionality with %d files", len(uploadedFiles))
}
+
+func TestManagerLocalPath(t *testing.T) {
+ test.Prepare(t, config.Conf)
+ defer test.Clean()
+
+ // Test with local storage
+ t.Run("LocalStorage", func(t *testing.T) {
+ // Create a local storage manager
+ manager, err := New(ManagerOption{
+ Driver: "local",
+ MaxSize: "10M",
+ AllowedTypes: []string{"text/*", "image/*", "application/*", ".txt", ".json", ".html", ".csv", ".yao"},
+ Options: map[string]interface{}{
+ "path": "/tmp/test_localpath_attachments",
+ },
+ })
+ if err != nil {
+ t.Fatalf("Failed to create local manager: %v", err)
+ }
+ manager.Name = "localpath-test"
+
+ // Test different file types
+ testFiles := []struct {
+ filename string
+ content string
+ contentType string
+ expectedCT string
+ }{
+ {"test.txt", "Hello LocalPath", "text/plain", "text/plain"},
+ {"test.json", `{"localpath": "test"}`, "application/json", "application/json"},
+ {"test.html", "LocalPath Test", "text/html", "text/html"},
+ {"test.csv", "col1,col2\nlocalpath,test", "text/csv", "text/csv"},
+ {"test.yao", "localpath yao content", "application/yao", "application/yao"},
+ }
+
+ for _, tf := range testFiles {
+ // Upload file
+ reader := strings.NewReader(tf.content)
+ fileHeader := &FileHeader{
+ FileHeader: &multipart.FileHeader{
+ Filename: tf.filename,
+ Size: int64(len(tf.content)),
+ Header: make(map[string][]string),
+ },
+ }
+ fileHeader.Header.Set("Content-Type", tf.contentType)
+
+ option := UploadOption{
+ Groups: []string{"localpath", "test"},
+ OriginalFilename: tf.filename,
+ }
+
+ file, err := manager.Upload(context.Background(), fileHeader, reader, option)
+ if err != nil {
+ t.Fatalf("Failed to upload file %s: %v", tf.filename, err)
+ }
+
+ // Test LocalPath
+ localPath, detectedCT, err := manager.LocalPath(context.Background(), file.ID)
+ if err != nil {
+ t.Fatalf("Failed to get local path for %s: %v", tf.filename, err)
+ }
+
+ // Verify path is absolute
+ if !filepath.IsAbs(localPath) {
+ t.Errorf("Expected absolute path for %s, got: %s", tf.filename, localPath)
+ }
+
+ // Verify content type
+ if detectedCT != tf.expectedCT {
+ t.Errorf("Expected content type %s for %s, got: %s", tf.expectedCT, tf.filename, detectedCT)
+ }
+
+ // Verify file exists
+ if _, err := os.Stat(localPath); os.IsNotExist(err) {
+ t.Errorf("File should exist at local path %s for %s", localPath, tf.filename)
+ }
+
+ // Verify file content
+ fileContent, err := os.ReadFile(localPath)
+ if err != nil {
+ t.Fatalf("Failed to read file at local path for %s: %v", tf.filename, err)
+ }
+
+ if string(fileContent) != tf.content {
+ t.Errorf("File content mismatch for %s. Expected: %s, Got: %s", tf.filename, tf.content, string(fileContent))
+ }
+
+ t.Logf("File %s - ID: %s, LocalPath: %s, ContentType: %s", tf.filename, file.ID, localPath, detectedCT)
+ }
+ })
+
+ // Test with gzipped files in local storage
+ t.Run("LocalStorage_Gzipped", func(t *testing.T) {
+ manager, err := New(ManagerOption{
+ Driver: "local",
+ MaxSize: "10M",
+ AllowedTypes: []string{"text/*"},
+ Options: map[string]interface{}{
+ "path": "/tmp/test_localpath_gzip_attachments",
+ },
+ })
+ if err != nil {
+ t.Fatalf("Failed to create local manager: %v", err)
+ }
+ manager.Name = "localpath-gzip-test"
+
+ content := "This content will be gzipped"
+ reader := strings.NewReader(content)
+
+ fileHeader := &FileHeader{
+ FileHeader: &multipart.FileHeader{
+ Filename: "gzipped.txt",
+ Size: int64(len(content)),
+ Header: make(map[string][]string),
+ },
+ }
+ fileHeader.Header.Set("Content-Type", "text/plain")
+
+ option := UploadOption{
+ Groups: []string{"gzip", "test"},
+ OriginalFilename: "gzipped.txt",
+ Gzip: true, // Enable gzip compression
+ }
+
+ file, err := manager.Upload(context.Background(), fileHeader, reader, option)
+ if err != nil {
+ t.Fatalf("Failed to upload gzipped file: %v", err)
+ }
+
+ // Test LocalPath - should get decompressed content
+ localPath, contentType, err := manager.LocalPath(context.Background(), file.ID)
+ if err != nil {
+ t.Fatalf("Failed to get local path for gzipped file: %v", err)
+ }
+
+ // Verify content type
+ if contentType != "text/plain" {
+ t.Errorf("Expected content type text/plain, got: %s", contentType)
+ }
+
+ // For gzipped files in local storage, the storage path ends with .gz
+ // but the content should be accessible normally through Read methods
+ fileContent, err := manager.Read(context.Background(), file.ID)
+ if err != nil {
+ t.Fatalf("Failed to read gzipped file: %v", err)
+ }
+
+ if string(fileContent) != content {
+ t.Errorf("Gzipped file content mismatch. Expected: %s, Got: %s", content, string(fileContent))
+ }
+
+ t.Logf("Gzipped file - ID: %s, LocalPath: %s, ContentType: %s", file.ID, localPath, contentType)
+ })
+}
+
+func TestManagerLocalPath_NonExistentFile(t *testing.T) {
+ test.Prepare(t, config.Conf)
+ defer test.Clean()
+
+ manager, err := New(ManagerOption{
+ Driver: "local",
+ AllowedTypes: []string{"text/*"},
+ Options: map[string]interface{}{
+ "path": "/tmp/test_localpath_nonexistent",
+ },
+ })
+ if err != nil {
+ t.Fatalf("Failed to create manager: %v", err)
+ }
+ manager.Name = "nonexistent-test"
+
+ // Test with non-existent file ID
+ _, _, err = manager.LocalPath(context.Background(), "non-existent-file-id")
+ if err == nil {
+ t.Error("Expected error for non-existent file ID")
+ }
+
+ // Should contain "file not found" in the error chain
+ if !strings.Contains(err.Error(), "file not found") {
+ t.Errorf("Expected 'file not found' in error message, got: %s", err.Error())
+ }
+}
+
+func TestManagerLocalPath_ValidationFlow(t *testing.T) {
+ test.Prepare(t, config.Conf)
+ defer test.Clean()
+
+ manager, err := New(ManagerOption{
+ Driver: "local",
+ AllowedTypes: []string{"text/*"},
+ Options: map[string]interface{}{
+ "path": "/tmp/test_localpath_validation",
+ },
+ })
+ if err != nil {
+ t.Fatalf("Failed to create manager: %v", err)
+ }
+ manager.Name = "validation-test"
+
+ // Upload a file
+ content := "Validation flow test content"
+ reader := strings.NewReader(content)
+
+ fileHeader := &FileHeader{
+ FileHeader: &multipart.FileHeader{
+ Filename: "validation.txt",
+ Size: int64(len(content)),
+ Header: make(map[string][]string),
+ },
+ }
+ fileHeader.Header.Set("Content-Type", "text/plain")
+
+ option := UploadOption{
+ Groups: []string{"validation"},
+ OriginalFilename: "original-validation.txt",
+ }
+
+ file, err := manager.Upload(context.Background(), fileHeader, reader, option)
+ if err != nil {
+ t.Fatalf("Failed to upload file: %v", err)
+ }
+
+ // Test complete flow: Upload -> LocalPath -> Verify -> Delete
+ t.Run("CompleteFlow", func(t *testing.T) {
+ // Get local path
+ localPath, contentType, err := manager.LocalPath(context.Background(), file.ID)
+ if err != nil {
+ t.Fatalf("Failed to get local path: %v", err)
+ }
+
+ // Verify all properties
+ if !filepath.IsAbs(localPath) {
+ t.Error("Path should be absolute")
+ }
+
+ if contentType != "text/plain" {
+ t.Errorf("Expected content type text/plain, got: %s", contentType)
+ }
+
+ // Verify file exists
+ stat, err := os.Stat(localPath)
+ if err != nil {
+ t.Fatalf("File should exist at local path: %v", err)
+ }
+
+ if stat.Size() != int64(len(content)) {
+ t.Errorf("File size mismatch. Expected: %d, Got: %d", len(content), stat.Size())
+ }
+
+ // Verify file content matches
+ fileContent, err := os.ReadFile(localPath)
+ if err != nil {
+ t.Fatalf("Failed to read file: %v", err)
+ }
+
+ if string(fileContent) != content {
+ t.Errorf("Content mismatch. Expected: %s, Got: %s", content, string(fileContent))
+ }
+
+ // Verify through manager's Read method as well
+ managerContent, err := manager.Read(context.Background(), file.ID)
+ if err != nil {
+ t.Fatalf("Failed to read through manager: %v", err)
+ }
+
+ if string(managerContent) != content {
+ t.Errorf("Manager read content mismatch. Expected: %s, Got: %s", content, string(managerContent))
+ }
+
+ t.Logf("Validation complete - LocalPath: %s, Size: %d bytes, ContentType: %s", localPath, stat.Size(), contentType)
+ })
+
+ // Clean up
+ err = manager.Delete(context.Background(), file.ID)
+ if err != nil {
+ t.Logf("Warning: Failed to delete test file: %v", err)
+ }
+}
diff --git a/attachment/s3/storage.go b/attachment/s3/storage.go
index 5209a481..7daa7c02 100644
--- a/attachment/s3/storage.go
+++ b/attachment/s3/storage.go
@@ -9,6 +9,9 @@ import (
"image/jpeg"
"image/png"
"io"
+ "mime"
+ "net/http"
+ "os"
"path/filepath"
"strings"
"time"
@@ -32,6 +35,7 @@ type Storage struct {
Secret string `json:"secret" yaml:"secret"`
Bucket string `json:"bucket" yaml:"bucket"`
Expiration time.Duration `json:"expiration" yaml:"expiration"`
+ CacheDir string `json:"cache_dir" yaml:"cache_dir"`
client *s3.Client
prefix string
compression bool
@@ -69,6 +73,13 @@ func New(options map[string]interface{}) (*Storage, error) {
storage.prefix = prefix
}
+ if cacheDir, ok := options["cache_dir"].(string); ok {
+ storage.CacheDir = cacheDir
+ } else {
+ // Use system temp directory as default
+ storage.CacheDir = os.TempDir()
+ }
+
if exp, ok := options["expiration"].(time.Duration); ok {
storage.Expiration = exp
}
@@ -103,6 +114,12 @@ func New(options map[string]interface{}) (*Storage, error) {
}
storage.client = s3.New(opts)
+
+ // Ensure cache directory exists
+ if err := os.MkdirAll(storage.CacheDir, 0755); err != nil {
+ return nil, fmt.Errorf("failed to create cache directory %s: %w", storage.CacheDir, err)
+ }
+
return storage, nil
}
@@ -445,3 +462,278 @@ func compressImage(data []byte, contentType string) ([]byte, error) {
return buf.Bytes(), nil
}
+
+// LocalPath downloads the file to cache directory and returns absolute path with content type
+func (storage *Storage) LocalPath(ctx context.Context, path string) (string, string, error) {
+ if storage.client == nil {
+ return "", "", fmt.Errorf("s3 client not initialized")
+ }
+
+ // Create cache file path using the same structure as storage path
+ cacheFilePath := filepath.Join(storage.CacheDir, "s3_cache", path)
+
+ // Create directory for cache file
+ dir := filepath.Dir(cacheFilePath)
+ if err := os.MkdirAll(dir, 0755); err != nil {
+ return "", "", fmt.Errorf("failed to create cache directory: %w", err)
+ }
+
+ // Check if file already exists in cache and is not outdated
+ if _, err := os.Stat(cacheFilePath); err == nil {
+ // File exists in cache, detect content type and return
+ contentType, err := detectContentType(cacheFilePath)
+ if err != nil {
+ return "", "", fmt.Errorf("failed to detect content type: %w", err)
+ }
+ return cacheFilePath, contentType, nil
+ }
+
+ // Download file from S3 to cache
+ key := filepath.Join(storage.prefix, path)
+ result, err := storage.client.GetObject(ctx, &s3.GetObjectInput{
+ Bucket: aws.String(storage.Bucket),
+ Key: aws.String(key),
+ })
+ if err != nil {
+ return "", "", fmt.Errorf("failed to download file %s: %w", path, err)
+ }
+ defer result.Body.Close()
+
+ // Create cache file
+ cacheFile, err := os.Create(cacheFilePath)
+ if err != nil {
+ return "", "", fmt.Errorf("failed to create cache file: %w", err)
+ }
+ defer cacheFile.Close()
+
+ // Handle gzipped files - decompress during download
+ var reader io.Reader = result.Body
+ if strings.HasSuffix(path, ".gz") {
+ gzipReader, err := gzip.NewReader(result.Body)
+ if err != nil {
+ return "", "", fmt.Errorf("failed to create gzip reader: %w", err)
+ }
+ defer gzipReader.Close()
+ reader = gzipReader
+
+ // Remove .gz extension from cache file path since we're decompressing
+ newCacheFilePath := strings.TrimSuffix(cacheFilePath, ".gz")
+ cacheFile.Close()
+ os.Remove(cacheFilePath)
+
+ cacheFile, err = os.Create(newCacheFilePath)
+ if err != nil {
+ return "", "", fmt.Errorf("failed to create decompressed cache file: %w", err)
+ }
+ defer cacheFile.Close()
+ cacheFilePath = newCacheFilePath
+ }
+
+ // Copy file content to cache
+ _, err = io.Copy(cacheFile, reader)
+ if err != nil {
+ return "", "", fmt.Errorf("failed to copy file to cache: %w", err)
+ }
+
+ // For files that were decompressed from .gz, we need to detect the original content type
+ var contentType string
+ if strings.HasSuffix(path, ".gz") {
+ // Original path was gzipped, detect content type of decompressed content
+ originalPath := strings.TrimSuffix(path, ".gz")
+ ext := filepath.Ext(originalPath)
+
+ // First try to detect by original file extension
+ contentType, err = detectContentTypeFromExtension(ext)
+ if err != nil || contentType == "application/octet-stream" {
+ // Fallback: detect from decompressed content
+ contentType, err = detectContentType(cacheFilePath)
+ if err != nil {
+ return "", "", fmt.Errorf("failed to detect content type: %w", err)
+ }
+ }
+ } else {
+ // Regular file content type detection
+ contentType, err = detectContentType(cacheFilePath)
+ if err != nil {
+ return "", "", fmt.Errorf("failed to detect content type: %w", err)
+ }
+ }
+
+ return cacheFilePath, contentType, nil
+}
+
+// detectContentType detects content type based on file extension and content
+func detectContentType(filePath string) (string, error) {
+ // First try to detect by file extension
+ ext := strings.ToLower(filepath.Ext(filePath))
+
+ // Common file extensions mapping
+ switch ext {
+ case ".txt":
+ return "text/plain", nil
+ case ".html", ".htm":
+ return "text/html", nil
+ case ".css":
+ return "text/css", nil
+ case ".js":
+ return "application/javascript", nil
+ case ".json":
+ return "application/json", nil
+ case ".xml":
+ return "application/xml", nil
+ case ".jpg", ".jpeg":
+ return "image/jpeg", nil
+ case ".png":
+ return "image/png", nil
+ case ".gif":
+ return "image/gif", nil
+ case ".webp":
+ return "image/webp", nil
+ case ".svg":
+ return "image/svg+xml", nil
+ case ".pdf":
+ return "application/pdf", nil
+ case ".doc":
+ return "application/msword", nil
+ case ".docx":
+ return "application/vnd.openxmlformats-officedocument.wordprocessingml.document", nil
+ case ".xls":
+ return "application/vnd.ms-excel", nil
+ case ".xlsx":
+ return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", nil
+ case ".ppt":
+ return "application/vnd.ms-powerpoint", nil
+ case ".pptx":
+ return "application/vnd.openxmlformats-officedocument.presentationml.presentation", nil
+ case ".zip":
+ return "application/zip", nil
+ case ".tar":
+ return "application/x-tar", nil
+ case ".gz":
+ return "application/gzip", nil
+ case ".mp3":
+ return "audio/mpeg", nil
+ case ".wav":
+ return "audio/wav", nil
+ case ".m4a":
+ return "audio/mp4", nil
+ case ".ogg":
+ return "audio/ogg", nil
+ case ".mp4":
+ return "video/mp4", nil
+ case ".avi":
+ return "video/x-msvideo", nil
+ case ".mov":
+ return "video/quicktime", nil
+ case ".webm":
+ return "video/webm", nil
+ case ".md", ".mdx":
+ return "text/markdown", nil
+ case ".yao":
+ return "application/yao", nil
+ case ".csv":
+ return "text/csv", nil
+ }
+
+ // Try to detect by MIME package
+ if contentType := mime.TypeByExtension(ext); contentType != "" {
+ return contentType, nil
+ }
+
+ // Fallback: detect by reading file content
+ file, err := os.Open(filePath)
+ if err != nil {
+ return "application/octet-stream", nil // Default fallback
+ }
+ defer file.Close()
+
+ // Read first 512 bytes for content detection
+ buffer := make([]byte, 512)
+ n, err := file.Read(buffer)
+ if err != nil && err != io.EOF {
+ return "application/octet-stream", nil
+ }
+
+ // Use http.DetectContentType to detect based on content
+ contentType := http.DetectContentType(buffer[:n])
+ return contentType, nil
+}
+
+// detectContentTypeFromExtension detects content type based only on file extension
+func detectContentTypeFromExtension(ext string) (string, error) {
+ ext = strings.ToLower(ext)
+
+ // Common file extensions mapping
+ switch ext {
+ case ".txt":
+ return "text/plain", nil
+ case ".html", ".htm":
+ return "text/html", nil
+ case ".css":
+ return "text/css", nil
+ case ".js":
+ return "application/javascript", nil
+ case ".json":
+ return "application/json", nil
+ case ".xml":
+ return "application/xml", nil
+ case ".jpg", ".jpeg":
+ return "image/jpeg", nil
+ case ".png":
+ return "image/png", nil
+ case ".gif":
+ return "image/gif", nil
+ case ".webp":
+ return "image/webp", nil
+ case ".svg":
+ return "image/svg+xml", nil
+ case ".pdf":
+ return "application/pdf", nil
+ case ".doc":
+ return "application/msword", nil
+ case ".docx":
+ return "application/vnd.openxmlformats-officedocument.wordprocessingml.document", nil
+ case ".xls":
+ return "application/vnd.ms-excel", nil
+ case ".xlsx":
+ return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", nil
+ case ".ppt":
+ return "application/vnd.ms-powerpoint", nil
+ case ".pptx":
+ return "application/vnd.openxmlformats-officedocument.presentationml.presentation", nil
+ case ".zip":
+ return "application/zip", nil
+ case ".tar":
+ return "application/x-tar", nil
+ case ".mp3":
+ return "audio/mpeg", nil
+ case ".wav":
+ return "audio/wav", nil
+ case ".m4a":
+ return "audio/mp4", nil
+ case ".ogg":
+ return "audio/ogg", nil
+ case ".mp4":
+ return "video/mp4", nil
+ case ".avi":
+ return "video/x-msvideo", nil
+ case ".mov":
+ return "video/quicktime", nil
+ case ".webm":
+ return "video/webm", nil
+ case ".md", ".mdx":
+ return "text/markdown", nil
+ case ".yao":
+ return "application/yao", nil
+ case ".csv":
+ return "text/csv", nil
+ }
+
+ // Try to detect by MIME package
+ if contentType := mime.TypeByExtension(ext); contentType != "" {
+ return contentType, nil
+ }
+
+ // Return default if not found
+ return "application/octet-stream", nil
+}
diff --git a/attachment/s3/storage_test.go b/attachment/s3/storage_test.go
index b8750d59..06179fcc 100644
--- a/attachment/s3/storage_test.go
+++ b/attachment/s3/storage_test.go
@@ -5,9 +5,13 @@ import (
"context"
"io"
"os"
+ "path/filepath"
+ "strings"
"testing"
"time"
+ "compress/gzip"
+
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
)
@@ -196,4 +200,164 @@ func TestS3Storage(t *testing.T) {
assert.Error(t, err)
assert.Contains(t, err.Error(), "bucket is required")
})
+
+ t.Run("LocalPath", func(t *testing.T) {
+ skipIfNoS3Config(t)
+
+ // Create storage with custom cache directory
+ tempCacheDir, err := os.MkdirTemp("", "s3_cache_test")
+ assert.NoError(t, err)
+ defer os.RemoveAll(tempCacheDir)
+
+ config := getS3Config()
+ config["cache_dir"] = tempCacheDir
+
+ storage, err := New(config)
+ assert.NoError(t, err)
+
+ // Test different file types
+ testFiles := []struct {
+ name string
+ content []byte
+ contentType string
+ expectedCT string
+ }{
+ {"test.txt", []byte("Hello S3 World"), "text/plain", "text/plain"},
+ {"test.json", []byte(`{"s3": "test"}`), "application/json", "application/json"},
+ {"test.html", []byte("S3 Test"), "text/html", "text/html"},
+ {"test.csv", []byte("s3,test\nval1,val2"), "text/csv", "text/csv"},
+ {"test.md", []byte("# S3 Markdown"), "text/markdown", "text/markdown"},
+ {"test.yao", []byte("s3 yao content"), "application/yao", "application/yao"},
+ }
+
+ for _, tf := range testFiles {
+ // Upload file to S3
+ fileID := "s3-localpath-" + uuid.New().String() + "-" + tf.name
+ _, err = storage.Upload(context.Background(), fileID, bytes.NewReader(tf.content), tf.contentType)
+ assert.NoError(t, err, "Failed to upload %s", tf.name)
+
+ // Get local path - first call should download to cache
+ localPath1, detectedCT1, err := storage.LocalPath(context.Background(), fileID)
+ assert.NoError(t, err, "Failed to get local path for %s", tf.name)
+ assert.NotEmpty(t, localPath1, "Local path should not be empty for %s", tf.name)
+ assert.Equal(t, tf.expectedCT, detectedCT1, "Content type mismatch for %s", tf.name)
+
+ // Verify the path is absolute
+ assert.True(t, filepath.IsAbs(localPath1), "Path should be absolute for %s", tf.name)
+
+ // Verify the file exists at the returned path
+ _, err = os.Stat(localPath1)
+ assert.NoError(t, err, "File should exist at local path for %s", tf.name)
+
+ // Verify file content
+ fileContent, err := os.ReadFile(localPath1)
+ assert.NoError(t, err, "Failed to read file at local path for %s", tf.name)
+ assert.Equal(t, tf.content, fileContent, "File content mismatch for %s", tf.name)
+
+ // Get local path again - should use cached version
+ localPath2, detectedCT2, err := storage.LocalPath(context.Background(), fileID)
+ assert.NoError(t, err, "Failed to get cached local path for %s", tf.name)
+ assert.Equal(t, localPath1, localPath2, "Cached path should be same as first call for %s", tf.name)
+ assert.Equal(t, detectedCT1, detectedCT2, "Cached content type should be same as first call for %s", tf.name)
+
+ // Clean up from S3
+ storage.Delete(context.Background(), fileID)
+ }
+ })
+
+ t.Run("LocalPath_GzippedFile", func(t *testing.T) {
+ skipIfNoS3Config(t)
+
+ // Create storage with custom cache directory
+ tempCacheDir, err := os.MkdirTemp("", "s3_cache_gzip_test")
+ assert.NoError(t, err)
+ defer os.RemoveAll(tempCacheDir)
+
+ config := getS3Config()
+ config["cache_dir"] = tempCacheDir
+
+ storage, err := New(config)
+ assert.NoError(t, err)
+
+ // Create gzipped content
+ originalContent := []byte("This content will be gzipped and stored in S3")
+ var gzipBuf bytes.Buffer
+ gzipWriter := gzip.NewWriter(&gzipBuf)
+ _, err = gzipWriter.Write(originalContent)
+ assert.NoError(t, err)
+ gzipWriter.Close()
+
+ // Upload gzipped file
+ fileID := "gzipped-" + uuid.New().String() + ".txt.gz"
+ _, err = storage.Upload(context.Background(), fileID, bytes.NewReader(gzipBuf.Bytes()), "text/plain")
+ assert.NoError(t, err)
+
+ // Get local path - should decompress during download
+ localPath, contentType, err := storage.LocalPath(context.Background(), fileID)
+ assert.NoError(t, err)
+ assert.NotEmpty(t, localPath)
+
+ // Verify the file is decompressed in cache (path should not end with .gz)
+ assert.False(t, strings.HasSuffix(localPath, ".gz"), "Cached file should be decompressed")
+
+ // Verify content is decompressed
+ cachedContent, err := os.ReadFile(localPath)
+ assert.NoError(t, err)
+ assert.Equal(t, originalContent, cachedContent, "Cached file should contain decompressed content")
+
+ // Verify content type
+ assert.Equal(t, "text/plain", contentType)
+
+ // Clean up
+ storage.Delete(context.Background(), fileID)
+ })
+
+ t.Run("LocalPath_NonExistentFile", func(t *testing.T) {
+ skipIfNoS3Config(t)
+
+ storage, err := New(getS3Config())
+ assert.NoError(t, err)
+
+ // Test with non-existent file
+ nonExistentFileID := "non-existent-" + uuid.New().String() + ".txt"
+ _, _, err = storage.LocalPath(context.Background(), nonExistentFileID)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "failed to download file")
+ })
+
+ t.Run("LocalPath_CustomCacheDir", func(t *testing.T) {
+ skipIfNoS3Config(t)
+
+ // Create custom cache directory
+ customCacheDir, err := os.MkdirTemp("", "custom_s3_cache")
+ assert.NoError(t, err)
+ defer os.RemoveAll(customCacheDir)
+
+ config := getS3Config()
+ config["cache_dir"] = customCacheDir
+
+ storage, err := New(config)
+ assert.NoError(t, err)
+
+ // Verify cache directory is set correctly
+ assert.Equal(t, customCacheDir, storage.CacheDir)
+
+ // Upload a test file
+ content := []byte("Custom cache directory test")
+ fileID := "custom-cache-" + uuid.New().String() + ".txt"
+ _, err = storage.Upload(context.Background(), fileID, bytes.NewReader(content), "text/plain")
+ assert.NoError(t, err)
+
+ // Get local path
+ localPath, contentType, err := storage.LocalPath(context.Background(), fileID)
+ assert.NoError(t, err)
+ assert.NotEmpty(t, localPath)
+ assert.Equal(t, "text/plain", contentType)
+
+ // Verify the file is cached in the custom directory
+ assert.True(t, strings.HasPrefix(localPath, customCacheDir), "File should be cached in custom directory")
+
+ // Clean up
+ storage.Delete(context.Background(), fileID)
+ })
}
diff --git a/attachment/types.go b/attachment/types.go
index f5526729..99c9232c 100644
--- a/attachment/types.go
+++ b/attachment/types.go
@@ -43,6 +43,9 @@ type FileManager interface {
// Delete deletes a file
Delete(ctx context.Context, fileID string) error
+
+ // LocalPath gets the local path of the file
+ LocalPath(ctx context.Context, fileID string) (string, string, error)
}
// File the file
@@ -103,6 +106,7 @@ type Storage interface {
URL(ctx context.Context, path string) string
Exists(ctx context.Context, path string) bool
Delete(ctx context.Context, path string) error
+ LocalPath(ctx context.Context, path string) (string, string, error) // Returns absolute path and content type
}
// ManagerOption the manager option
diff --git a/openapi/kb/document.go b/openapi/kb/document.go
index 533408fb..87ecd240 100644
--- a/openapi/kb/document.go
+++ b/openapi/kb/document.go
@@ -4,6 +4,7 @@ import (
"net/http"
"github.com/gin-gonic/gin"
+ "github.com/yaoapp/yao/attachment"
"github.com/yaoapp/yao/kb"
"github.com/yaoapp/yao/openapi/response"
)
@@ -12,8 +13,19 @@ import (
// AddFile adds a file to a collection
func AddFile(c *gin.Context) {
+
var req AddFileRequest
+ // Check if kb.Instance is available
+ if kb.Instance == nil {
+ errorResp := &response.ErrorResponse{
+ Code: response.ErrServerError.Code,
+ ErrorDescription: "Knowledge base not initialized",
+ }
+ response.RespondWithError(c, response.StatusInternalServerError, errorResp)
+ return
+ }
+
// Parse and bind JSON request
if err := c.ShouldBindJSON(&req); err != nil {
errorResp := &response.ErrorResponse{
@@ -34,24 +46,41 @@ func AddFile(c *gin.Context) {
return
}
- // Check if kb.Instance is available
- if kb.Instance == nil {
+ // Get file manager
+ m, ok := attachment.Managers[req.Uploader]
+ if !ok {
+ errorResp := &response.ErrorResponse{
+ Code: response.ErrInvalidRequest.Code,
+ ErrorDescription: "Invalid uploader: " + req.Uploader + " not found",
+ }
+ response.RespondWithError(c, response.StatusNotFound, errorResp)
+ return
+ }
+
+ // Check if the file exists
+ exists := m.Exists(c.Request.Context(), req.FileID)
+ if !exists {
+ errorResp := &response.ErrorResponse{
+ Code: response.ErrInvalidRequest.Code,
+ ErrorDescription: "File not found: " + req.FileID,
+ }
+ response.RespondWithError(c, response.StatusNotFound, errorResp)
+ return
+ }
+
+ // Get the options of the manager
+ path, contentType, err := m.LocalPath(c.Request.Context(), req.FileID)
+ if err != nil {
errorResp := &response.ErrorResponse{
Code: response.ErrServerError.Code,
- ErrorDescription: "Knowledge base not initialized",
+ ErrorDescription: "Failed to get local path: " + err.Error(),
}
response.RespondWithError(c, response.StatusInternalServerError, errorResp)
return
}
- // TODO: Call external function to get file info
- // filename, contentType, err := GetFileInfo(req.FileID)
- // For now, use hardcoded values
- filename := "document.pdf"
- contentType := "application/pdf"
-
// Convert request to UpsertOptions
- upsertOptions, err := req.BaseUpsertRequest.ToUpsertOptions(filename, contentType)
+ upsertOptions, err := req.BaseUpsertRequest.ToUpsertOptions(path, contentType)
if err != nil {
errorResp := &response.ErrorResponse{
Code: response.ErrInvalidRequest.Code,
diff --git a/openapi/kb/utils.go b/openapi/kb/utils.go
index 37fd1ef2..3321b421 100644
--- a/openapi/kb/utils.go
+++ b/openapi/kb/utils.go
@@ -91,7 +91,8 @@ type BaseUpsertRequest struct {
// AddFileRequest represents the request for AddFile API
type AddFileRequest struct {
BaseUpsertRequest
- FileID string `json:"file_id" binding:"required"`
+ FileID string `json:"file_id" binding:"required"`
+ Uploader string `json:"uploader,omitempty"` // The name of the uploader, e.g. "s3", "local", "webdav", etc.
}
// AddTextRequest represents the request for AddText API