From 96d2f22e77625cabc988f622c0af10e6f8af5020 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 31 May 2025 10:07:19 +0800 Subject: [PATCH] Update dependencies in go.mod and go.sum; refactor file handling in neo package - Added the jsonrepair package to go.mod for enhanced JSON handling capabilities. - Removed unused dependencies from go.mod and updated existing ones for better performance. - Refactored file upload and download methods in the neo package to return nil values, indicating a temporary placeholder for future implementation. - Updated the Message struct to use the new attachment package for better organization of file attachments. --- go.mod | 4 +- go.sum | 132 +------- neo/assistant/types.go | 7 +- neo/attachment/README.md | 450 ++++++++++++++++++++++++++ neo/attachment/compresses.go | 76 +++++ neo/attachment/example_usage.go | 445 ++++++++++++++++++++++++++ neo/attachment/fileheader.go | 134 ++++++++ neo/attachment/gzip.go | 285 +++++++++++++++++ neo/attachment/local/storage.go | 216 +++++++++++++ neo/attachment/local/storage_test.go | 221 +++++++++++++ neo/attachment/manager.go | 452 +++++++++++++++++++++++++++ neo/attachment/manager_test.go | 244 +++++++++++++++ neo/attachment/s3/storage.go | 381 ++++++++++++++++++++++ neo/attachment/s3/storage_test.go | 191 +++++++++++ neo/attachment/types.go | 93 ++++++ neo/message/message.go | 71 ++--- neo/neo.go | 9 +- 17 files changed, 3235 insertions(+), 176 deletions(-) create mode 100644 neo/attachment/README.md create mode 100644 neo/attachment/compresses.go create mode 100644 neo/attachment/example_usage.go create mode 100644 neo/attachment/fileheader.go create mode 100644 neo/attachment/gzip.go create mode 100644 neo/attachment/local/storage.go create mode 100644 neo/attachment/local/storage_test.go create mode 100644 neo/attachment/manager.go create mode 100644 neo/attachment/manager_test.go create mode 100644 neo/attachment/s3/storage.go create mode 100644 neo/attachment/s3/storage_test.go create mode 100644 neo/attachment/types.go diff --git a/go.mod b/go.mod index 74b3cfbf..f306d810 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/joho/godotenv v1.5.1 github.com/json-iterator/go v1.1.12 + github.com/kaptinlin/jsonrepair v0.1.1 github.com/mozillazg/go-pinyin v0.20.0 github.com/pkoukk/tiktoken-go v0.1.7 github.com/rhysd/go-github-selfupdate v1.2.3 @@ -59,7 +60,6 @@ require ( github.com/bytedance/sonic/loader v0.2.4 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.5 // indirect - github.com/cloudwego/iasm v0.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dlclark/regexp2 v1.11.5 // indirect @@ -86,7 +86,6 @@ require ( github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmoiron/sqlx v1.4.0 // indirect - github.com/kaptinlin/jsonrepair v0.1.1 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/leodido/go-urn v1.4.0 // indirect @@ -97,7 +96,6 @@ require ( github.com/miekg/dns v1.1.66 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/montanaflynn/stats v0.7.1 // indirect github.com/oklog/run v1.1.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect diff --git a/go.sum b/go.sum index fc0da206..22ed0dd8 100644 --- a/go.sum +++ b/go.sum @@ -1,59 +1,33 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/PuerkitoBio/goquery v1.10.1 h1:Y8JGYUkXWTGRB6Ars3+j3kN0xg1YqqlwvdTV8WTFQcU= -github.com/PuerkitoBio/goquery v1.10.1/go.mod h1:IYiHrOMps66ag56LEH7QYDDupKXyo5A8qrjIx3ZtujY= github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo= github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 h1:ZBbLwSJqkHBuFDA6DUhhse0IGJ7T5bemHyNILUjvOq4= github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2/go.mod h1:VSw57q4QFiWDbRnjdX8Cb3Ow0SFncRw+bA/ofY6Q83w= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= -github.com/aws/aws-sdk-go-v2 v1.32.7 h1:ky5o35oENWi0JYWUZkB7WYvVPP+bcRF5/Iq7JWSb5Rw= -github.com/aws/aws-sdk-go-v2 v1.32.7/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= -github.com/aws/aws-sdk-go-v2/credentials v1.17.48 h1:IYdLD1qTJ0zanRavulofmqut4afs45mOWEI+MzZtTfQ= -github.com/aws/aws-sdk-go-v2/credentials v1.17.48/go.mod h1:tOscxHN3CGmuX9idQ3+qbkzrjVIx32lqDSU1/0d/qXs= github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM= github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 h1:I/5wmGMffY4happ8NOCuIUEWGUvvFp5NSeQcXl9RHcI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26/go.mod h1:FR8f4turZtNy6baO0KJ5FJUmXH/cSkI9fOngs0yl6mA= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 h1:zXFLuEuMMUOvEARXFUVJdfqZ4bvvSgdGRq/ATcrQxzM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26/go.mod h1:3o2Wpy0bogG1kyOPrgkXA8pgIfEEv0+m19O9D5+W8y8= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 h1:GeNJsIFHB+WW5ap2Tec4K6dzcVTsRbsT1Lra46Hv9ME= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26/go.mod h1:zfgMpwHDXX2WGoG84xG2H+ZlPTkJUU4YUvx2svLQYWo= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 h1:tB4tNw83KcajNAzaIMhkhVI2Nt8fAZd5A5ro113FEMY= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7/go.mod h1:lvpyBGkZ3tZ9iSsUIcC2EWp+0ywa7aK3BLT+FwZi+mQ= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 h1:4nm2G6A4pV9rdlWzGMPv4BNtQp22v1hg3yrtkYpeLl8= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 h1:8eUsivBQzZHqe/3FE+cqwfH+0p5Jo8PFM/QYQSmeZ+M= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7/go.mod h1:kLPQvGUmxn/fqiCrDeohwG33bq2pQpGeY62yRO6Nrh0= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 h1:Hi0KGbrnr57bEHWM0bJ1QcBzxLrL/k2DHvGYhb8+W1w= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7/go.mod h1:wKNgWgExdjjrm4qvfbTorkvocEstaoDl4WCvGfeCy9c= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 h1:aOVVZJgWbaH+EJYPvEgkNhCEbXXvH7+oML36oaPK3zE= -github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1/go.mod h1:r+xl5yzMk9083rMR+sJ5TYj9Tihvf/l1oxzZXDgGj2Q= github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 h1:BRXS0U76Z8wfF+bnkilA2QwpIch6URlm++yPUt9QPmQ= github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3/go.mod h1:bNXKFFyaiVvWuR6O16h/I1724+aXe/tAkA9/QS01t5k= -github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= -github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= @@ -62,26 +36,18 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= -github.com/bytedance/sonic v1.12.6 h1:/isNmCUF2x3Sh8RAp/4mh4ZGkcFAX/hLrzrK3AvpRzk= -github.com/bytedance/sonic v1.12.6/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.1 h1:1GgorWTqf12TA8mma4DDSbaQigE2wOgQo7iCjjJv3+E= -github.com/bytedance/sonic/loader v0.2.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/caarlos0/env/v6 v6.10.1 h1:t1mPSxNpei6M5yAeu1qtRdPAK29Nbcf/n3G7x+b3/II= github.com/caarlos0/env/v6 v6.10.1/go.mod h1:hvp/ryKXKipEkcuYjs9mI4bBCg+UI0Yhgm5Zu0ddvwc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= -github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -90,18 +56,12 @@ github.com/dchest/captcha v1.1.0 h1:2kt47EoYUUkaISobUdTbqwx55xvKOJxyScVfw25xzhQ= github.com/dchest/captcha v1.1.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= -github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/elazarl/go-bindata-assetfs v1.0.1 h1:m0kkaHRKEu7tUIUFVwhGGGYClXvyl4RE03qmvRTNfbw= github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= -github.com/evanw/esbuild v0.24.2 h1:PQExybVBrjHjN6/JJiShRGIXh1hWVm6NepVnhZhrt0A= -github.com/evanw/esbuild v0.24.2/go.mod h1:D2vIQZqV/vIf/VRHtViaUtViZmG7o+kKmlBfVQuRi48= github.com/evanw/esbuild v0.25.4 h1:k1bTSim+usBG27w7BfOCorhgx3tO+6bAfMj5pR+6SKg= github.com/evanw/esbuild v0.25.4/go.mod h1:D2vIQZqV/vIf/VRHtViaUtViZmG7o+kKmlBfVQuRi48= -github.com/expr-lang/expr v1.17.2 h1:o0A99O/Px+/DTjEnQiodAgOIK9PPxL8DtXhBRKC+Iso= -github.com/expr-lang/expr v1.17.2/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/expr-lang/expr v1.17.3 h1:myeTTuDFz7k6eFe/JPlep/UsiIjVhG61FMHFu63U7j0= github.com/expr-lang/expr v1.17.3/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= @@ -110,20 +70,12 @@ github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= -github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= -github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU= github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= -github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E= -github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0= github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= -github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= -github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= @@ -138,20 +90,15 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o= -github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q= github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= -github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-sql-driver/mysql v1.9.2 h1:4cNKDYQ1I84SXslGddlsrMhc8k4LeDVj6Ad6WRjiHuU= github.com/go-sql-driver/mysql v1.9.2/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= -github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM= -github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= @@ -160,8 +107,6 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -184,8 +129,6 @@ github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB1 github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-plugin v1.6.2 h1:zdGAEd0V1lCaU0u+MxWQhtSDQmahpkwOun8U8EiRVog= -github.com/hashicorp/go-plugin v1.6.2/go.mod h1:CkgLQ5CZqNmdL9U9JzM532t8ZiYQ35+pj3b1FD37R0Q= github.com/hashicorp/go-plugin v1.6.3 h1:xgHB+ZUSYeuJi96WtxEjzi23uh7YQpznjGh0U0UUrwg= github.com/hashicorp/go-plugin v1.6.3/go.mod h1:MRobyh+Wc/nYy1V4KAXUiYfzxoYhs7V1mlH1Z7iY2h0= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= @@ -209,13 +152,9 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kaptinlin/jsonrepair v0.1.1 h1:Ddn1sN1cZXuXeKA9vpaHAtBETnGSFBZFaaYfoN2Uo8c= github.com/kaptinlin/jsonrepair v0.1.1/go.mod h1:SivjE7np/GsSrk7UX/9mibH6VF8cVpD2aUmg7vceg2k= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= -github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= @@ -239,12 +178,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= -github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= -github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= github.com/miekg/dns v1.1.66 h1:FeZXOS3VCVsKnEAd+wBkjMC3D2K+ww66Cq3VnCINuJE= github.com/miekg/dns v1.1.66/go.mod h1:jGFzBsSNbJw6z1HYut1RKBKHA9PBdxeHrZG8J+gC2WE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -252,8 +187,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/mozillazg/go-pinyin v0.20.0 h1:BtR3DsxpApHfKReaPO1fCqF4pThRwH9uwvXzm+GnMFQ= @@ -268,16 +201,12 @@ github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkoukk/tiktoken-go v0.1.7 h1:qOBHXX4PHtvIvmOtyg1EeKlwFRiMKAcoMp4Q+bLQDmw= github.com/pkoukk/tiktoken-go v0.1.7/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/qdrant/go-client v1.12.0 h1:KqsIKDAw5iQmxDzRjbzRjhvQ+Igyr7Y84vDCinf1T4M= -github.com/qdrant/go-client v1.12.0/go.mod h1:zFa6t5Y3Oqecoa0aSsGWhMqQWq3x3kTPvm0sMf5qplw= github.com/qdrant/go-client v1.14.0 h1:cyz9OOooAexudw5w69LRe9vKCQFYJvaFvt9icOciI1U= github.com/qdrant/go-client v1.14.0/go.mod h1:iO8ts78jL4x6LDHFOViyYWELVtIBDTjOykBmiOTHLnQ= github.com/rhysd/go-github-selfupdate v1.2.3 h1:iaa+J202f+Nc+A8zi75uccC8Wg3omaM7HDeimXA22Ag= @@ -296,22 +225,15 @@ github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= -github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cast v1.8.0 h1:gEN9K4b8Xws4EX0+a0reLmhq8moKn7ntRlQYgjPeCDk= github.com/spf13/cast v1.8.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -319,7 +241,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tcnksm/go-gitconfig v0.1.2 h1:iiDhRitByXAEyjgBqsKi9QU4o2TNtv9kPP3RgPgXBPw= @@ -355,50 +276,35 @@ github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZ github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/watchfultele/jsonrepair v0.0.0-20250207052432-e4397ed42611 h1:CWCIUJ4cqhc3ct+HbV4EJtBdRFjUVOBne8mmIHXxU6A= -github.com/watchfultele/jsonrepair v0.0.0-20250207052432-e4397ed42611/go.mod h1:63urp6bG6c9cw3DuFtNEr5g9tbjFXaPTn0sBiBDVB7g= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/xuri/efp v0.0.0-20241211021726-c4e992084aa6 h1:8m6DWBG+dlFNbx5ynvrE7NgI+Y7OlZVMVTpayoW+rCc= -github.com/xuri/efp v0.0.0-20241211021726-c4e992084aa6/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8= github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= -github.com/xuri/excelize/v2 v2.9.0 h1:1tgOaEq92IOEumR1/JfYS/eR0KHOCsRv/rYXXh6YJQE= -github.com/xuri/excelize/v2 v2.9.0/go.mod h1:uqey4QBZ9gdMeWApPLdhm9x+9o2lq4iVmjiLfBS5hdE= github.com/xuri/excelize/v2 v2.9.1 h1:VdSGk+rraGmgLHGFaGG9/9IWu1nj4ufjJ7uwMDtj8Qw= github.com/xuri/excelize/v2 v2.9.1/go.mod h1:x7L6pKz2dvo9ejrRuD8Lnl98z4JLt0TGAwjhW+EiP8s= -github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 h1:hPVCafDV85blFTabnqKgNhDCkJX25eik94Si9cTER4A= -github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= github.com/xuri/nfp v0.0.1 h1:MDamSGatIvp8uOmDP8FnmjuQpu90NzdJxo7242ANR9Q= github.com/xuri/nfp v0.0.1/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM= -go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4= go.mongodb.org/mongo-driver v1.17.3 h1:TQyXhnsWfWtgAhMtOgtYHMTkZIfBTpMTsMnd9ZBeHxQ= go.mongodb.org/mongo-driver v1.17.3/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= -go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= -go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= -go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= -go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= -go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= -go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= -go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= -go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= -go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= -golang.org/x/arch v0.12.0 h1:UsYJhbzPYGsT0HbEdmYcqtCv8UNGvnaL561NnIUvaKg= -golang.org/x/arch v0.12.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= golang.org/x/arch v0.17.0 h1:4O3dfLzd+lQewptAHqjewQZQDyEdejz3VwgeYwkZneU= golang.org/x/arch v0.17.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -408,12 +314,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= -golang.org/x/image v0.23.0 h1:HseQ7c2OpPKTPVzNjG5fwJsOTCiiwS4QdsYi5XU6H68= -golang.org/x/image v0.23.0/go.mod h1:wJJBTdLfCCf3tiHa1fNxpZmUI4mmoZvwMCPP0ddoNKY= golang.org/x/image v0.27.0 h1:C8gA4oWU/tKkdCfYT6T2u4faJu3MeNS5O8UPWlPF61w= golang.org/x/image v0.27.0/go.mod h1:xbdrClrAUway1MUTEZDq9mz/UpRwYAkFFNUslZtcB+g= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -421,8 +323,6 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -438,14 +338,10 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= -golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -456,8 +352,6 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -480,8 +374,6 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= @@ -505,8 +397,6 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -515,24 +405,16 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= -golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241230172942-26aa7a208def h1:4P81qv5JXI/sDNae2ClVx88cgDDA6DPilADkG9tYKz8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241230172942-26aa7a208def/go.mod h1:bdAgzvd4kFrpykc5/AC2eLUiegK9T/qxZHD4hXYf/ho= google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 h1:cJfm9zPbe1e873mHJzmQ1nwVEeRDU/T1wXDK2kUSU34= google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= -google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= -google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= -google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/neo/assistant/types.go b/neo/assistant/types.go index 6d261a32..bded94e3 100644 --- a/neo/assistant/types.go +++ b/neo/assistant/types.go @@ -3,7 +3,6 @@ package assistant import ( "context" "io" - "mime/multipart" "github.com/gin-gonic/gin" "github.com/yaoapp/gou/rag/driver" @@ -22,9 +21,9 @@ const ( // API the assistant API interface type API interface { Chat(ctx context.Context, messages []message.Message, option map[string]interface{}, cb func(data []byte) int) error - Upload(ctx context.Context, file *multipart.FileHeader, reader io.Reader, option map[string]interface{}) (*File, error) - Download(ctx context.Context, fileID string) (*FileResponse, error) - ReadBase64(ctx context.Context, fileID string) (string, error) + // Upload(ctx context.Context, file *multipart.FileHeader, reader io.Reader, option map[string]interface{}) (*File, error) + // Download(ctx context.Context, fileID string) (*FileResponse, error) + // ReadBase64(ctx context.Context, fileID string) (string, error) GetPlaceholder(locale string) *Placeholder Execute(c *gin.Context, ctx chatctx.Context, input interface{}, options map[string]interface{}, callback ...interface{}) (interface{}, error) diff --git a/neo/attachment/README.md b/neo/attachment/README.md new file mode 100644 index 00000000..703f5961 --- /dev/null +++ b/neo/attachment/README.md @@ -0,0 +1,450 @@ +# Attachment Package + +A comprehensive file upload package for Go that supports chunked uploads, file format validation, compression, and multiple storage backends. + +## Features + +- **Multiple Storage Backends**: Local filesystem and S3-compatible storage +- **Chunked Upload Support**: Handle large files with standard HTTP Content-Range headers +- **File Compression**: + - Gzip compression for any file type + - Image compression with configurable size limits +- **File Validation**: + - File size limits + - MIME type and extension validation + - Wildcard pattern support (e.g., `image/*`, `text/*`) +- **Flexible File Organization**: Hierarchical storage with user/chat/assistant organization +- **Multiple Read Methods**: Stream, bytes, and base64 encoding +- **Global Manager Registry**: Support for registering and accessing managers globally + +## Installation + +```bash +go get github.com/yaoapp/yao/neo/attachment +``` + +## Quick Start + +### Basic Usage + +```go +package main + +import ( + "context" + "strings" + "mime/multipart" + "github.com/yaoapp/yao/neo/attachment" +) + +func main() { + // Create a manager + manager, err := attachment.New(attachment.ManagerOption{ + Driver: "local", + MaxSize: "20M", + ChunkSize: "2M", + AllowedTypes: []string{"text/*", "image/*", ".pdf"}, + Options: map[string]interface{}{ + "path": "/var/uploads", + }, + }) + if err != nil { + panic(err) + } + + // Upload a file + content := "Hello, World!" + fileHeader := &attachment.FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "hello.txt", + Size: int64(len(content)), + Header: make(map[string][]string), + }, + } + fileHeader.Header.Set("Content-Type", "text/plain") + + option := attachment.UploadOption{ + UserID: "user123", + ChatID: "chat456", + } + + file, err := manager.Upload(context.Background(), fileHeader, strings.NewReader(content), option) + if err != nil { + panic(err) + } + + // Read the file back + data, err := manager.Read(context.Background(), file.ID) + if err != nil { + panic(err) + } + + println(string(data)) // Output: Hello, World! +} +``` + +### Storage Backends + +#### Local Storage + +```go +manager, err := attachment.New(attachment.ManagerOption{ + Driver: "local", + MaxSize: "20M", + Options: map[string]interface{}{ + "path": "/var/uploads", + "base_url": "https://example.com/files", + }, +}) +``` + +#### S3 Storage + +```go +manager, err := attachment.New(attachment.ManagerOption{ + Driver: "s3", + MaxSize: "100M", + Options: map[string]interface{}{ + "endpoint": "https://s3.amazonaws.com", + "region": "us-east-1", + "key": "your-access-key", + "secret": "your-secret-key", + "bucket": "your-bucket-name", + "prefix": "attachments/", + }, +}) +``` + +### Chunked Upload + +For large files, you can upload in chunks using standard HTTP Content-Range headers: + +```go +// Upload chunks +totalSize := int64(1024000) // 1MB file +chunkSize := int64(1024) // 1KB chunks +uid := "unique-file-id-123" + +for start := int64(0); start < totalSize; start += chunkSize { + end := start + chunkSize - 1 + if end >= totalSize { + end = totalSize - 1 + } + + chunkData := make([]byte, end-start+1) + // ... fill chunkData with actual data ... + + chunkHeader := &attachment.FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "large_file.zip", + Size: end - start + 1, + Header: make(map[string][]string), + }, + } + chunkHeader.Header.Set("Content-Type", "application/zip") + chunkHeader.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, totalSize)) + chunkHeader.Header.Set("Content-Uid", uid) + + file, err := manager.Upload(ctx, chunkHeader, bytes.NewReader(chunkData), option) + if err != nil { + return err + } + + // File is complete when the last chunk is uploaded + if chunkHeader.Complete() { + fmt.Printf("Upload complete: %s\n", file.ID) + break + } +} +``` + +### Compression + +#### Gzip Compression + +```go +option := attachment.UploadOption{ + Gzip: true, // Enable gzip compression +} + +file, err := manager.Upload(ctx, fileHeader, reader, option) +``` + +#### Image Compression + +```go +option := attachment.UploadOption{ + CompressImage: true, + CompressSize: 1920, // Max dimension in pixels (default: 1920) +} + +file, err := manager.Upload(ctx, imageHeader, imageReader, option) +``` + +### File Validation + +#### Size Limits + +```go +manager, err := attachment.New(attachment.ManagerOption{ + MaxSize: "20M", // Maximum file size + // Supports: B, K, M, G (e.g., "1024B", "2K", "10M", "1G") +}) +``` + +#### Type Validation + +```go +manager, err := attachment.New(attachment.ManagerOption{ + AllowedTypes: []string{ + "text/*", // All text types + "image/*", // All image types + "application/pdf", // Specific MIME type + ".txt", // File extension + ".jpg", // File extension + }, +}) +``` + +### Reading Files + +#### Stream Reading + +```go +response, err := manager.Download(ctx, fileID) +if err != nil { + return err +} +defer response.Reader.Close() + +// Use response.Reader as io.ReadCloser +// response.ContentType contains the MIME type +// response.Extension contains the file extension +``` + +#### Read as Bytes + +```go +data, err := manager.Read(ctx, fileID) +if err != nil { + return err +} +// data is []byte +``` + +#### Read as Base64 + +```go +base64Data, err := manager.ReadBase64(ctx, fileID) +if err != nil { + return err +} +// base64Data is string +``` + +### Global Managers + +You can register managers globally for easy access: + +```go +// Register managers +attachment.Register("local", "local", attachment.ManagerOption{ + Driver: "local", + Options: map[string]interface{}{ + "path": "/var/uploads", + }, +}) + +attachment.Register("s3", "s3", attachment.ManagerOption{ + Driver: "s3", + Options: map[string]interface{}{ + "bucket": "my-bucket", + "key": "access-key", + "secret": "secret-key", + }, +}) + +// Use global managers +localManager := attachment.Managers["local"] +s3Manager := attachment.Managers["s3"] +``` + +## File Organization + +Files are organized in a hierarchical structure: + +``` +attachments/ +├── 20240101/ # Date (YYYYMMDD) +│ └── user123/ # User ID (optional) +│ └── chat456/ # Chat ID (optional) +│ └── assistant789/ # Assistant ID (optional) +│ └── ab/ # First 2 chars of hash +│ └── cd/ # Next 2 chars of hash +│ └── abcdef12.txt # Hash + extension +``` + +The file ID generation includes: + +- Date prefix for organization +- User/Chat/Assistant IDs for access control +- Content hash for deduplication +- Original file extension + +## API Reference + +### Manager + +#### `New(option ManagerOption) (*Manager, error)` + +Creates a new attachment manager. + +#### `Register(name string, driver string, option ManagerOption) (*Manager, error)` + +Registers a global attachment manager. + +#### `Upload(ctx context.Context, fileheader *FileHeader, reader io.Reader, option UploadOption) (*File, error)` + +Uploads a file (supports chunked upload). + +#### `Download(ctx context.Context, fileID string) (*FileResponse, error)` + +Downloads a file as a stream. + +#### `Read(ctx context.Context, fileID string) ([]byte, error)` + +Reads a file as bytes. + +#### `ReadBase64(ctx context.Context, fileID string) (string, error)` + +Reads a file as base64 encoded string. + +### Storage Interface + +All storage backends implement the following interface: + +```go +type Storage interface { + Upload(ctx context.Context, fileID string, reader io.Reader, contentType string) (string, error) + UploadChunk(ctx context.Context, fileID string, chunkIndex int, reader io.Reader, contentType string) error + MergeChunks(ctx context.Context, fileID string, totalChunks int) error + Download(ctx context.Context, fileID string) (io.ReadCloser, string, error) + Reader(ctx context.Context, fileID string) (io.ReadCloser, error) + URL(ctx context.Context, fileID string) string + Exists(ctx context.Context, fileID string) bool + Delete(ctx context.Context, fileID string) error +} +``` + +### Types + +#### `ManagerOption` + +Configuration for creating a manager: + +- `Driver`: "local" or "s3" +- `MaxSize`: Maximum file size (e.g., "20M") +- `ChunkSize`: Chunk size for uploads (e.g., "2M") +- `AllowedTypes`: Array of allowed MIME types/extensions +- `Options`: Driver-specific options + +#### `UploadOption` + +Options for file upload: + +- `CompressImage`: Enable image compression +- `CompressSize`: Maximum image dimension (default: 1920) +- `Gzip`: Enable gzip compression +- `UserID`, `ChatID`, `AssistantID`: Organization IDs + +#### `File` + +Uploaded file information: + +- `ID`: Unique file identifier +- `Filename`: Original filename +- `ContentType`: MIME type +- `Bytes`: File size +- `CreatedAt`: Upload timestamp + +#### `FileResponse` + +Download response: + +- `Reader`: io.ReadCloser for file content +- `ContentType`: MIME type +- `Extension`: File extension + +## Chunked Upload Details + +The package supports chunked uploads using standard HTTP headers: + +- `Content-Range`: Specifies byte range (e.g., "bytes 0-1023/2048") +- `Content-Uid`: Unique identifier for the file being uploaded + +### Chunk Index Calculation + +The package uses a standard chunk size (1024 bytes by default) to calculate chunk indices consistently. This ensures proper chunk ordering during merge operations. + +### Content Type Preservation + +For chunked uploads, the content type is preserved from the first chunk and applied to the final merged file, ensuring proper MIME type handling across all storage backends. + +## Error Handling + +The package returns descriptive errors for common issues: + +- File size exceeds limit +- Unsupported file type +- Storage backend errors +- Invalid chunk information +- Missing required configuration + +## Testing + +Run the tests: + +```bash +# Run all tests +go test ./... + +# Run with S3 credentials (optional) +export S3_ACCESS_KEY="your-key" +export S3_SECRET_KEY="your-secret" +export S3_BUCKET="your-bucket" +export S3_API="https://your-s3-endpoint" +go test ./... +``` + +The package includes comprehensive tests for: + +- Basic file upload/download +- Chunked uploads with content type preservation +- Compression (gzip and image) +- File validation (size, type, wildcards) +- Multiple storage backends (local and S3) +- Error handling and edge cases + +### Test Coverage + +- **Manager Tests**: Upload, download, validation, compression +- **Local Storage Tests**: File operations, chunked uploads, directory management +- **S3 Storage Tests**: S3 operations, chunked uploads, presigned URLs (requires credentials) + +## Performance Considerations + +- **Chunked Uploads**: Use appropriate chunk sizes (1-5MB) for optimal performance +- **Image Compression**: Automatically resizes large images to reduce storage costs +- **Gzip Compression**: Reduces storage size for text-based files +- **Content Type Detection**: Efficient MIME type detection and preservation + +## Security Features + +- **File Type Validation**: Prevents upload of unauthorized file types +- **Size Limits**: Configurable file size restrictions +- **Path Sanitization**: Secure file path generation +- **Access Control**: User/Chat/Assistant-based file organization + +## License + +This package is part of the Yao project and follows the same license terms. diff --git a/neo/attachment/compresses.go b/neo/attachment/compresses.go new file mode 100644 index 00000000..5b544539 --- /dev/null +++ b/neo/attachment/compresses.go @@ -0,0 +1,76 @@ +package attachment + +import ( + "bytes" + "fmt" + "image" + "image/jpeg" + "image/png" + "io" +) + +// CompressImage compresses the image while maintaining aspect ratio +func CompressImage(reader io.Reader, contentType string, maxSize int) ([]byte, error) { + // Read all data first + data, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("failed to read image data: %w", err) + } + + // Decode image + img, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("failed to decode image: %w", err) + } + + // Calculate new dimensions + bounds := img.Bounds() + width := bounds.Dx() + height := bounds.Dy() + var newWidth, newHeight int + + if width > height { + if width > maxSize { + newWidth = maxSize + newHeight = int(float64(height) * (float64(maxSize) / float64(width))) + } else { + return data, nil // No need to resize, return original data + } + } else { + if height > maxSize { + newHeight = maxSize + newWidth = int(float64(width) * (float64(maxSize) / float64(height))) + } else { + return data, nil // No need to resize, return original data + } + } + + // Create new image with new dimensions + newImg := image.NewRGBA(image.Rect(0, 0, newWidth, newHeight)) + + // Scale the image using bilinear interpolation + for y := 0; y < newHeight; y++ { + for x := 0; x < newWidth; x++ { + srcX := float64(x) * float64(width) / float64(newWidth) + srcY := float64(y) * float64(height) / float64(newHeight) + newImg.Set(x, y, img.At(int(srcX), int(srcY))) + } + } + + // Encode image + var buf bytes.Buffer + switch contentType { + case "image/jpeg": + err = jpeg.Encode(&buf, newImg, &jpeg.Options{Quality: 85}) + case "image/png": + err = png.Encode(&buf, newImg) + default: + return data, nil // Unsupported format, return original data + } + + if err != nil { + return nil, fmt.Errorf("failed to encode image: %w", err) + } + + return buf.Bytes(), nil +} diff --git a/neo/attachment/example_usage.go b/neo/attachment/example_usage.go new file mode 100644 index 00000000..4236a4f9 --- /dev/null +++ b/neo/attachment/example_usage.go @@ -0,0 +1,445 @@ +package attachment + +import ( + "bytes" + "context" + "fmt" + "log" + "mime/multipart" + "strings" + + "github.com/yaoapp/yao/neo/attachment/s3" +) + +// ExampleUsage demonstrates how to use the attachment package +func ExampleUsage() { + // 1. Create a local storage manager + localManager, err := New(ManagerOption{ + Driver: "local", + MaxSize: "20M", + ChunkSize: "2M", + AllowedTypes: []string{"text/*", "image/*", "application/pdf", ".txt", ".jpg", ".png", ".pdf"}, + Options: map[string]interface{}{ + "path": "/var/uploads", + "base_url": "https://example.com/files", + }, + }) + if err != nil { + log.Fatalf("Failed to create local manager: %v", err) + } + + // 2. Create an S3 storage manager + s3Manager, err := New(ManagerOption{ + Driver: "s3", + MaxSize: "100M", + ChunkSize: "5M", + AllowedTypes: []string{"*"}, // Allow all types + Options: map[string]interface{}{ + "endpoint": "https://s3.amazonaws.com", + "region": "us-east-1", + "key": "your-access-key", + "secret": "your-secret-key", + "bucket": "your-bucket-name", + "prefix": "attachments/", + }, + }) + if err != nil { + log.Printf("Failed to create S3 manager (this is expected without credentials): %v", err) + } else { + fmt.Printf("Created S3 manager successfully\n") + // Demonstrate S3 manager usage if credentials are available + if s3Manager != nil { + fmt.Printf("S3 manager is ready for use with bucket: %s\n", + s3Manager.storage.(*s3.Storage).Bucket) + } + } + + // 3. Register managers globally + _, err = Register("local", "local", ManagerOption{ + Driver: "local", + MaxSize: "20M", + AllowedTypes: []string{"text/*", "image/*"}, + Options: map[string]interface{}{ + "path": "/var/uploads", + }, + }) + if err != nil { + log.Printf("Failed to register local manager: %v", err) + } + + // Try to register S3 manager (will fail without credentials) + _, err = Register("s3", "s3", ManagerOption{ + Driver: "s3", + MaxSize: "100M", + Options: map[string]interface{}{ + "bucket": "your-bucket", + "key": "your-key", + "secret": "your-secret", + }, + }) + if err != nil { + log.Printf("Failed to register S3 manager (expected without credentials): %v", err) + } + + ctx := context.Background() + + // 4. Example: Simple file upload + content := "Hello, World! This is a test file with some content to demonstrate the attachment package." + fileHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "hello.txt", + Size: int64(len(content)), + Header: make(map[string][]string), + }, + } + fileHeader.Header.Set("Content-Type", "text/plain") + + uploadOption := UploadOption{ + UserID: "user123", + ChatID: "chat456", + AssistantID: "assistant789", + Gzip: false, // No compression for small text files + } + + file, err := localManager.Upload(ctx, fileHeader, strings.NewReader(content), uploadOption) + if err != nil { + log.Fatalf("Failed to upload file: %v", err) + } + + fmt.Printf("Uploaded file: %s (ID: %s, Size: %d bytes)\n", file.Filename, file.ID, file.Bytes) + + // 5. Example: File upload with gzip compression + largeContent := strings.Repeat("This is a large text file that benefits from compression. ", 100) + gzipFileHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "large_text.txt", + Size: int64(len(largeContent)), + Header: make(map[string][]string), + }, + } + gzipFileHeader.Header.Set("Content-Type", "text/plain") + + gzipOption := UploadOption{ + UserID: "user123", + Gzip: true, // Enable compression + } + + gzipFile, err := localManager.Upload(ctx, gzipFileHeader, strings.NewReader(largeContent), gzipOption) + if err != nil { + log.Fatalf("Failed to upload gzipped file: %v", err) + } + + fmt.Printf("Uploaded compressed file: %s (ID: %s)\n", gzipFile.Filename, gzipFile.ID) + + // 6. Example: Image upload with compression + imageUploadOption := UploadOption{ + UserID: "user123", + CompressImage: true, + CompressSize: 1920, // Resize to max 1920px + Gzip: false, + } + + // Simulate image upload (you would get this from multipart form) + imageHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "photo.jpg", + Size: 1024000, // 1MB + Header: make(map[string][]string), + }, + } + imageHeader.Header.Set("Content-Type", "image/jpeg") + + fmt.Printf("Image upload option configured: compress=%v, size=%d\n", + imageUploadOption.CompressImage, imageUploadOption.CompressSize) + + // 7. Example: Chunked upload + largeContent = strings.Repeat("This is a large file content that will be uploaded in chunks. ", 1000) + chunkSize := 1024 + totalSize := len(largeContent) + uid := "unique-large-file-123" + + fmt.Printf("Starting chunked upload: total size=%d, chunk size=%d\n", totalSize, chunkSize) + + var lastFile *File + chunkCount := 0 + + // Split into chunks and upload + for i := 0; i < totalSize; i += chunkSize { + end := i + chunkSize + if end > totalSize { + end = totalSize + } + chunk := largeContent[i:end] + + chunkHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "large_file.txt", + Size: int64(len(chunk)), + Header: make(map[string][]string), + }, + } + chunkHeader.Header.Set("Content-Type", "text/plain") + chunkHeader.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", i, end-1, totalSize)) + chunkHeader.Header.Set("Content-Uid", uid) + + chunkOption := UploadOption{ + UserID: "user123", + Gzip: true, // Compress chunks + } + + chunkFile, err := localManager.Upload(ctx, chunkHeader, strings.NewReader(chunk), chunkOption) + if err != nil { + log.Fatalf("Failed to upload chunk %d: %v", chunkCount, err) + } + + chunkCount++ + lastFile = chunkFile + + // Check if this is the last chunk + if chunkHeader.Complete() { + fmt.Printf("Uploaded large file in %d chunks: %s (ID: %s)\n", chunkCount, chunkFile.Filename, chunkFile.ID) + break + } + } + + // 8. Example: Download and read files + if file != nil { + // Download as stream + response, err := localManager.Download(ctx, file.ID) + if err != nil { + log.Fatalf("Failed to download file: %v", err) + } + defer response.Reader.Close() + + fmt.Printf("Downloaded file content type: %s, extension: %s\n", response.ContentType, response.Extension) + + // Read as bytes + data, err := localManager.Read(ctx, file.ID) + if err != nil { + log.Fatalf("Failed to read file: %v", err) + } + + fmt.Printf("File content length: %d bytes\n", len(data)) + if len(data) < 100 { + fmt.Printf("File content: %s\n", string(data)) + } else { + fmt.Printf("File content preview: %s...\n", string(data[:100])) + } + + // Read as base64 + base64Data, err := localManager.ReadBase64(ctx, file.ID) + if err != nil { + log.Fatalf("Failed to read file as base64: %v", err) + } + + fmt.Printf("File as base64 (first 50 chars): %s...\n", base64Data[:min(50, len(base64Data))]) + } + + // 9. Example: Read chunked file + if lastFile != nil { + chunkData, err := localManager.Read(ctx, lastFile.ID) + if err != nil { + log.Fatalf("Failed to read chunked file: %v", err) + } + + // Since the chunks were compressed, we need to decompress + decompressed, err := Gunzip(chunkData) + if err != nil { + log.Fatalf("Failed to decompress chunked file: %v", err) + } + + fmt.Printf("Chunked file content length: %d bytes (decompressed)\n", len(decompressed)) + if len(decompressed) < 200 { + fmt.Printf("Chunked file content: %s\n", string(decompressed)) + } else { + fmt.Printf("Chunked file content preview: %s...\n", string(decompressed[:200])) + } + } + + // 10. Example: Using global managers + globalManager := Managers["local"] + if globalManager != nil { + fmt.Println("Using global manager for local storage") + + // Test a simple upload with global manager + testContent := "Test content using global manager" + testHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "global_test.txt", + Size: int64(len(testContent)), + Header: make(map[string][]string), + }, + } + testHeader.Header.Set("Content-Type", "text/plain") + + testFile, err := globalManager.Upload(ctx, testHeader, strings.NewReader(testContent), UploadOption{ + UserID: "global_user", + }) + if err != nil { + log.Printf("Failed to upload with global manager: %v", err) + } else { + fmt.Printf("Global manager upload successful: %s\n", testFile.ID) + } + } + + // 11. Example: File validation + fmt.Println("\n--- File Validation Examples ---") + + // Test file size validation + tooLargeContent := strings.Repeat("x", 25*1024*1024) // 25MB, exceeds 20MB limit + largeFileHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "too_large.txt", + Size: int64(len(tooLargeContent)), + Header: make(map[string][]string), + }, + } + largeFileHeader.Header.Set("Content-Type", "text/plain") + + _, err = localManager.Upload(ctx, largeFileHeader, strings.NewReader(tooLargeContent), UploadOption{}) + if err != nil { + fmt.Printf("Expected error for large file: %v\n", err) + } + + // Test file type validation + invalidFileHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "script.exe", + Size: 1024, + Header: make(map[string][]string), + }, + } + invalidFileHeader.Header.Set("Content-Type", "application/x-executable") + + _, err = localManager.Upload(ctx, invalidFileHeader, strings.NewReader("fake exe content"), UploadOption{}) + if err != nil { + fmt.Printf("Expected error for invalid file type: %v\n", err) + } + + fmt.Println("\n--- Example Usage Complete ---") +} + +// ExampleChunkedUpload demonstrates how to handle chunked uploads properly +func ExampleChunkedUpload(manager *Manager, filename string, totalSize int64, contentType string) error { + ctx := context.Background() + chunkSize := int64(1024 * 1024) // 1MB chunks + uid := "unique-file-" + filename + + fmt.Printf("Starting chunked upload: file=%s, total=%d bytes, chunks=%d\n", + filename, totalSize, (totalSize+chunkSize-1)/chunkSize) + + for offset := int64(0); offset < totalSize; offset += chunkSize { + end := offset + chunkSize - 1 + if end >= totalSize { + end = totalSize - 1 + } + + chunkSize := end - offset + 1 + + // Create chunk header + chunkHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: filename, + Size: chunkSize, + Header: make(map[string][]string), + }, + } + chunkHeader.Header.Set("Content-Type", contentType) + chunkHeader.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", offset, end, totalSize)) + chunkHeader.Header.Set("Content-Uid", uid) + + // In real usage, you would read the actual chunk data from the source + chunkData := make([]byte, chunkSize) + // Fill with sample data for demonstration + for i := range chunkData { + chunkData[i] = byte('A' + (i % 26)) + } + + option := UploadOption{ + UserID: "user123", + Gzip: false, // Disable compression for this example + } + + file, err := manager.Upload(ctx, chunkHeader, bytes.NewReader(chunkData), option) + if err != nil { + return fmt.Errorf("failed to upload chunk at offset %d: %w", offset, err) + } + + fmt.Printf("Uploaded chunk %d-%d/%d\n", offset, end, totalSize) + + // Check if this was the last chunk + if chunkHeader.Complete() { + fmt.Printf("File upload completed: %s (ID: %s)\n", file.Filename, file.ID) + + // Verify the uploaded file + data, err := manager.Read(ctx, file.ID) + if err != nil { + return fmt.Errorf("failed to read uploaded file: %w", err) + } + + if int64(len(data)) != totalSize { + return fmt.Errorf("uploaded file size mismatch: expected %d, got %d", totalSize, len(data)) + } + + fmt.Printf("File verification successful: %d bytes\n", len(data)) + break + } + } + + return nil +} + +// ExampleS3Upload demonstrates S3-specific features +func ExampleS3Upload() { + // This example requires actual S3 credentials + s3Manager, err := New(ManagerOption{ + Driver: "s3", + MaxSize: "50M", + Options: map[string]interface{}{ + "endpoint": "https://s3.amazonaws.com", + "region": "us-east-1", + "key": "your-access-key", + "secret": "your-secret-key", + "bucket": "your-bucket", + "prefix": "test-uploads/", + }, + }) + if err != nil { + log.Printf("S3 manager creation failed (expected without credentials): %v", err) + return + } + + ctx := context.Background() + content := "Test content for S3 upload" + + fileHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "s3_test.txt", + Size: int64(len(content)), + Header: make(map[string][]string), + }, + } + fileHeader.Header.Set("Content-Type", "text/plain") + + file, err := s3Manager.Upload(ctx, fileHeader, strings.NewReader(content), UploadOption{ + UserID: "s3_user", + }) + if err != nil { + log.Printf("S3 upload failed: %v", err) + return + } + + fmt.Printf("S3 upload successful: %s\n", file.ID) + + // Get presigned URL + url := s3Manager.storage.URL(ctx, file.ID) + fmt.Printf("Presigned URL: %s\n", url) +} + +// Helper function for min +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/neo/attachment/fileheader.go b/neo/attachment/fileheader.go new file mode 100644 index 00000000..b272895e --- /dev/null +++ b/neo/attachment/fileheader.go @@ -0,0 +1,134 @@ +package attachment + +import ( + "strconv" + "strings" +) + +// UID is the uid of the file, it is the unique identifier of the file +func (fileheader *FileHeader) UID() string { + return fileheader.Header.Get("Content-Uid") +} + +// Range is the range of the file, it is the start and end of the file +func (fileheader *FileHeader) Range() string { + return fileheader.Header.Get("Content-Range") +} + +// Sync is the sync of the file, it is the sync of the file +func (fileheader *FileHeader) Sync() bool { + return fileheader.Header.Get("Content-Sync") == "true" +} + +// IsChunk is the chunk of the file, it is the chunk of the file +func (fileheader *FileHeader) IsChunk() bool { + return fileheader.Range() != "" +} + +// Complete checks if the chunk upload is completed +// For non-chunk files, it returns true +// For chunk files, it parses the Content-Range header to determine if this is the last chunk +func (fileheader *FileHeader) Complete() bool { + if !fileheader.IsChunk() { + return true + } + + // Parse Content-Range header: "bytes start-end/total" + rangeHeader := fileheader.Range() + if rangeHeader == "" { + return false + } + + // Remove "bytes " prefix + rangeStr := strings.TrimPrefix(rangeHeader, "bytes ") + + // Split by "/" + parts := strings.Split(rangeStr, "/") + if len(parts) != 2 { + return false + } + + // Parse total size + total, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return false + } + + // Parse range "start-end" + rangeParts := strings.Split(parts[0], "-") + if len(rangeParts) != 2 { + return false + } + + end, err := strconv.ParseInt(rangeParts[1], 10, 64) + if err != nil { + return false + } + + // Check if this is the last chunk: end + 1 == total + return end+1 == total +} + +// GetChunkInfo returns the chunk information parsed from Content-Range header +// Returns start, end, total, and error +func (fileheader *FileHeader) GetChunkInfo() (start, end, total int64, err error) { + if !fileheader.IsChunk() { + return 0, 0, 0, nil + } + + rangeHeader := fileheader.Range() + if rangeHeader == "" { + return 0, 0, 0, nil + } + + // Remove "bytes " prefix + rangeStr := strings.TrimPrefix(rangeHeader, "bytes ") + + // Split by "/" + parts := strings.Split(rangeStr, "/") + if len(parts) != 2 { + return 0, 0, 0, nil + } + + // Parse total size + total, err = strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return 0, 0, 0, err + } + + // Parse range "start-end" + rangeParts := strings.Split(parts[0], "-") + if len(rangeParts) != 2 { + return 0, 0, 0, nil + } + + start, err = strconv.ParseInt(rangeParts[0], 10, 64) + if err != nil { + return 0, 0, 0, err + } + + end, err = strconv.ParseInt(rangeParts[1], 10, 64) + if err != nil { + return 0, 0, 0, err + } + + return start, end, total, nil +} + +// GetTotalSize returns the total file size from Content-Range header +func (fileheader *FileHeader) GetTotalSize() int64 { + _, _, total, err := fileheader.GetChunkInfo() + if err != nil { + return 0 + } + return total +} + +// GetChunkSize returns the current chunk size +func (fileheader *FileHeader) GetChunkSize() int64 { + start, end, _, err := fileheader.GetChunkInfo() + if err != nil { + return 0 + } + return end - start + 1 +} diff --git a/neo/attachment/gzip.go b/neo/attachment/gzip.go new file mode 100644 index 00000000..f526c468 --- /dev/null +++ b/neo/attachment/gzip.go @@ -0,0 +1,285 @@ +package attachment + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "os" +) + +// GzipCompressor supports chunked compression for Gzip +type GzipCompressor struct { + writer *gzip.Writer + buffer *bytes.Buffer + file *os.File // optional file handle +} + +// NewGzipCompressor creates a new Gzip compressor +func NewGzipCompressor() *GzipCompressor { + buf := &bytes.Buffer{} + gz := gzip.NewWriter(buf) + return &GzipCompressor{ + writer: gz, + buffer: buf, + file: nil, + } +} + +// NewGzipCompressorFromFile creates a Gzip compressor from file, supports streaming read +func NewGzipCompressorFromFile(filePath string) (*GzipCompressor, error) { + file, err := os.Open(filePath) + if err != nil { + return nil, fmt.Errorf("failed to open file %s: %w", filePath, err) + } + + buf := &bytes.Buffer{} + gz := gzip.NewWriter(buf) + return &GzipCompressor{ + writer: gz, + buffer: buf, + file: file, + }, nil +} + +// ReadChunk reads a chunk of specified size from file and compresses it +func (gc *GzipCompressor) ReadChunk(chunkSize int) (bool, error) { + if gc.file == nil { + return false, fmt.Errorf("no file associated with this compressor") + } + + chunk := make([]byte, chunkSize) + n, err := gc.file.Read(chunk) + if err != nil && err != io.EOF { + return false, fmt.Errorf("failed to read from file: %w", err) + } + + if n > 0 { + if err := gc.Write(chunk[:n]); err != nil { + return false, err + } + } + + // return whether there is more data + return err != io.EOF, nil +} + +// CompressFileInChunks compresses the entire file in chunks +func (gc *GzipCompressor) CompressFileInChunks(chunkSize int) error { + if gc.file == nil { + return fmt.Errorf("no file associated with this compressor") + } + + for { + hasMore, err := gc.ReadChunk(chunkSize) + if err != nil { + return err + } + if !hasMore { + break + } + } + return nil +} + +// Write writes data for compression (supports chunked writing) +func (gc *GzipCompressor) Write(data []byte) error { + _, err := gc.writer.Write(data) + if err != nil { + return fmt.Errorf("failed to write data to gzip: %w", err) + } + return nil +} + +// Flush flushes the buffer but does not close the compressor +func (gc *GzipCompressor) Flush() error { + return gc.writer.Flush() +} + +// Close closes the compressor and returns the final compressed data +func (gc *GzipCompressor) Close() ([]byte, error) { + err := gc.writer.Close() + if err != nil { + return nil, fmt.Errorf("failed to close gzip writer: %w", err) + } + + // if there is an associated file, close it too + if gc.file != nil { + gc.file.Close() + gc.file = nil + } + + return gc.buffer.Bytes(), nil +} + +// GetCompressedData gets the current compressed data (without closing the compressor) +func (gc *GzipCompressor) GetCompressedData() []byte { + // flush the buffer first + gc.writer.Flush() + return gc.buffer.Bytes() +} + +// Reset resets the compressor for reuse +func (gc *GzipCompressor) Reset() { + gc.buffer.Reset() + gc.writer.Reset(gc.buffer) + if gc.file != nil { + gc.file.Close() + gc.file = nil + } +} + +// Gzip compresses data in one go +func Gzip(data []byte) ([]byte, error) { + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + _, err := gz.Write(data) + if err != nil { + return nil, fmt.Errorf("failed to gzip data: %w", err) + } + err = gz.Close() + if err != nil { + return nil, fmt.Errorf("failed to close gzip writer: %w", err) + } + return buf.Bytes(), nil +} + +// GzipChunks compresses multiple data chunks +func GzipChunks(chunks [][]byte) ([]byte, error) { + compressor := NewGzipCompressor() + + for _, chunk := range chunks { + if err := compressor.Write(chunk); err != nil { + return nil, err + } + } + + return compressor.Close() +} + +// GzipFromReader compresses data from Reader stream +func GzipFromReader(reader io.Reader) ([]byte, error) { + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + + _, err := io.Copy(gz, reader) + if err != nil { + return nil, fmt.Errorf("failed to copy data to gzip: %w", err) + } + + err = gz.Close() + if err != nil { + return nil, fmt.Errorf("failed to close gzip writer: %w", err) + } + + return buf.Bytes(), nil +} + +// Gunzip decompresses gzip data +func Gunzip(data []byte) ([]byte, error) { + reader, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("failed to create gzip reader: %w", err) + } + defer reader.Close() + + var buf bytes.Buffer + _, err = io.Copy(&buf, reader) + if err != nil { + return nil, fmt.Errorf("failed to decompress data: %w", err) + } + + return buf.Bytes(), nil +} + +// GzipToWriter writes compressed data to Writer +func GzipToWriter(data []byte, writer io.Writer) error { + gz := gzip.NewWriter(writer) + defer gz.Close() + + _, err := gz.Write(data) + if err != nil { + return fmt.Errorf("failed to write gzip data: %w", err) + } + + return nil +} + +// GzipFromReaderToWriter reads data from Reader and writes compressed data to Writer +func GzipFromReaderToWriter(reader io.Reader, writer io.Writer) error { + gz := gzip.NewWriter(writer) + defer gz.Close() + + _, err := io.Copy(gz, reader) + if err != nil { + return fmt.Errorf("failed to copy and compress data: %w", err) + } + + return nil +} + +// GzipFile compresses entire file (loads into memory at once) +func GzipFile(filePath string) ([]byte, error) { + data, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read file %s: %w", filePath, err) + } + return Gzip(data) +} + +// GzipFileInChunks compresses file in chunks (memory friendly) +func GzipFileInChunks(filePath string, chunkSize int) ([]byte, error) { + compressor, err := NewGzipCompressorFromFile(filePath) + if err != nil { + return nil, err + } + defer compressor.Close() + + err = compressor.CompressFileInChunks(chunkSize) + if err != nil { + return nil, err + } + + return compressor.Close() +} + +// GzipFileToFile compresses file and saves to another file +func GzipFileToFile(srcPath, dstPath string, chunkSize int) error { + srcFile, err := os.Open(srcPath) + if err != nil { + return fmt.Errorf("failed to open source file %s: %w", srcPath, err) + } + defer srcFile.Close() + + dstFile, err := os.Create(dstPath) + if err != nil { + return fmt.Errorf("failed to create destination file %s: %w", dstPath, err) + } + defer dstFile.Close() + + return GzipFromReaderToWriter(srcFile, dstFile) +} + +// GzipFileStream compresses file in streaming mode, returns Reader interface +func GzipFileStream(filePath string) (io.Reader, error) { + file, err := os.Open(filePath) + if err != nil { + return nil, fmt.Errorf("failed to open file %s: %w", filePath, err) + } + + pr, pw := io.Pipe() + go func() { + defer pw.Close() + defer file.Close() + + gz := gzip.NewWriter(pw) + defer gz.Close() + + _, err := io.Copy(gz, file) + if err != nil { + pw.CloseWithError(err) + } + }() + + return pr, nil +} diff --git a/neo/attachment/local/storage.go b/neo/attachment/local/storage.go new file mode 100644 index 00000000..652d59ed --- /dev/null +++ b/neo/attachment/local/storage.go @@ -0,0 +1,216 @@ +package local + +import ( + "context" + "crypto/sha256" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" +) + +// MaxImageSize maximum image size (1920x1080) +const MaxImageSize = 1920 + +// Storage the local storage driver +type Storage struct { + Path string `json:"path" yaml:"path"` + Compression bool `json:"compression" yaml:"compression"` + BaseURL string `json:"base_url" yaml:"base_url"` + PreviewURL func(fileID string) string `json:"-" yaml:"-"` +} + +// New create a new local storage +func New(options map[string]interface{}) (*Storage, error) { + storage := &Storage{ + Compression: true, + } + + if path, ok := options["path"].(string); ok { + storage.Path = path + } + + if compression, ok := options["compression"].(bool); ok { + storage.Compression = compression + } + + if baseURL, ok := options["base_url"].(string); ok { + storage.BaseURL = baseURL + } + + if previewURL, ok := options["preview_url"].(func(string) string); ok { + storage.PreviewURL = previewURL + } + + if storage.Path == "" { + return nil, fmt.Errorf("path is required") + } + + // Ensure the base path exists + if err := os.MkdirAll(storage.Path, 0755); err != nil { + return nil, fmt.Errorf("failed to create base path: %w", err) + } + + return storage, nil +} + +// Upload upload file to local storage +func (storage *Storage) Upload(ctx context.Context, fileID string, reader io.Reader, contentType string) (string, error) { + path := filepath.Join(storage.Path, fileID) + + // Create directory if not exists + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0755); err != nil { + return "", err + } + + // Create and write file + file, err := os.Create(path) + if err != nil { + return "", err + } + defer file.Close() + + _, err = io.Copy(file, reader) + if err != nil { + return "", err + } + + return fileID, nil +} + +// UploadChunk uploads a chunk of a file +func (storage *Storage) UploadChunk(ctx context.Context, fileID string, chunkIndex int, reader io.Reader, contentType string) error { + // Create chunks directory + chunksDir := filepath.Join(storage.Path, ".chunks", fileID) + if err := os.MkdirAll(chunksDir, 0755); err != nil { + return err + } + + // Write chunk file + chunkPath := filepath.Join(chunksDir, fmt.Sprintf("chunk_%d", chunkIndex)) + file, err := os.Create(chunkPath) + if err != nil { + return err + } + defer file.Close() + + _, err = io.Copy(file, reader) + return err +} + +// MergeChunks merges all chunks into the final file +func (storage *Storage) MergeChunks(ctx context.Context, fileID string, totalChunks int) error { + chunksDir := filepath.Join(storage.Path, ".chunks", fileID) + finalPath := filepath.Join(storage.Path, fileID) + + // Create directory for final file + dir := filepath.Dir(finalPath) + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + + // Create final file + finalFile, err := os.Create(finalPath) + if err != nil { + return err + } + defer finalFile.Close() + + // Read and merge chunks in order + for i := 0; i < totalChunks; i++ { + chunkPath := filepath.Join(chunksDir, fmt.Sprintf("chunk_%d", i)) + chunkFile, err := os.Open(chunkPath) + if err != nil { + return fmt.Errorf("failed to read chunk %d: %w", i, err) + } + + _, err = io.Copy(finalFile, chunkFile) + chunkFile.Close() + if err != nil { + return fmt.Errorf("failed to copy chunk %d: %w", i, err) + } + } + + // Clean up chunks directory + os.RemoveAll(chunksDir) + return nil +} + +// Reader read file from local storage +func (storage *Storage) Reader(ctx context.Context, fileID string) (io.ReadCloser, error) { + fullpath := filepath.Join(storage.Path, fileID) + reader, err := os.Open(fullpath) + if err != nil { + return nil, err + } + return reader, nil +} + +// Download download file from local storage +func (storage *Storage) Download(ctx context.Context, fileID string) (io.ReadCloser, string, error) { + path := filepath.Join(storage.Path, fileID) + reader, err := os.Open(path) + if err != nil { + return nil, "", err + } + + // Try to detect content type from file extension + contentType := "application/octet-stream" + ext := filepath.Ext(path) + switch strings.ToLower(ext) { + case ".txt": + contentType = "text/plain" + case ".html": + contentType = "text/html" + case ".css": + contentType = "text/css" + case ".js": + contentType = "application/javascript" + case ".json": + contentType = "application/json" + case ".jpg", ".jpeg": + contentType = "image/jpeg" + case ".png": + contentType = "image/png" + case ".gif": + contentType = "image/gif" + case ".pdf": + contentType = "application/pdf" + } + + return reader, contentType, nil +} + +// URL get file url +func (storage *Storage) URL(ctx context.Context, fileID string) string { + if storage.PreviewURL != nil { + return storage.PreviewURL(fileID) + } + if storage.BaseURL != "" { + return fmt.Sprintf("%s/%s", strings.TrimRight(storage.BaseURL, "/"), fileID) + } + return fmt.Sprintf("%s/%s", storage.Path, fileID) +} + +// Exists checks if a file exists +func (storage *Storage) Exists(ctx context.Context, fileID string) bool { + fullpath := filepath.Join(storage.Path, fileID) + _, err := os.Stat(fullpath) + return err == nil +} + +// Delete deletes a file +func (storage *Storage) Delete(ctx context.Context, fileID string) error { + fullpath := filepath.Join(storage.Path, fileID) + return os.Remove(fullpath) +} + +func (storage *Storage) makeID(filename string, ext string) string { + date := time.Now().Format("20060102") + hash := fmt.Sprintf("%x", sha256.Sum256([]byte(filename)))[:8] + name := strings.TrimSuffix(filepath.Base(filename), ext) + return fmt.Sprintf("%s/%s-%s%s", date, name, hash, ext) +} diff --git a/neo/attachment/local/storage_test.go b/neo/attachment/local/storage_test.go new file mode 100644 index 00000000..8e1956b0 --- /dev/null +++ b/neo/attachment/local/storage_test.go @@ -0,0 +1,221 @@ +package local + +import ( + "bytes" + "context" + "image" + "image/png" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLocalStorage(t *testing.T) { + // Create a temporary directory for testing + tempDir, err := os.MkdirTemp("", "local_storage_test") + assert.NoError(t, err) + defer os.RemoveAll(tempDir) + + testPath := filepath.Join(tempDir, "test_storage") + + t.Run("Create Storage", func(t *testing.T) { + storage, err := New(map[string]interface{}{ + "path": testPath, + "compression": true, + }) + assert.NoError(t, err) + assert.NotNil(t, storage) + assert.Equal(t, testPath, storage.Path) + assert.True(t, storage.Compression) + }) + + t.Run("Upload and Download", func(t *testing.T) { + storage, err := New(map[string]interface{}{ + "path": testPath, + "compression": true, + }) + assert.NoError(t, err) + + content := []byte("test content") + reader := bytes.NewReader(content) + fileID, err := storage.Upload(context.Background(), "test.txt", reader, "text/plain") + assert.NoError(t, err) + assert.NotEmpty(t, fileID) + + // Download + reader2, contentType, err := storage.Download(context.Background(), fileID) + assert.NoError(t, err) + assert.Contains(t, contentType, "text/plain") + + downloaded, err := io.ReadAll(reader2) + assert.NoError(t, err) + assert.Equal(t, content, downloaded) + }) + + t.Run("Upload and Download Image with Compression", func(t *testing.T) { + storage, err := New(map[string]interface{}{ + "path": testPath, + "compression": true, + }) + assert.NoError(t, err) + + // Create a test image (100x100 pixels - smaller for faster testing) + img := image.NewRGBA(image.Rect(0, 0, 100, 100)) + var buf bytes.Buffer + err = png.Encode(&buf, img) + assert.NoError(t, err) + + // Upload + reader := bytes.NewReader(buf.Bytes()) + fileID, err := storage.Upload(context.Background(), "test.png", reader, "image/png") + assert.NoError(t, err) + assert.NotEmpty(t, fileID) + + // Download and verify + reader2, contentType, err := storage.Download(context.Background(), fileID) + assert.NoError(t, err) + assert.Equal(t, "image/png", contentType) + + downloaded, err := io.ReadAll(reader2) + assert.NoError(t, err) + + // Decode the downloaded image + downloadedImg, _, err := image.Decode(bytes.NewReader(downloaded)) + assert.NoError(t, err) + + // Verify image was processed + bounds := downloadedImg.Bounds() + assert.True(t, bounds.Dx() > 0) + assert.True(t, bounds.Dy() > 0) + }) + + t.Run("Upload Image without Compression", func(t *testing.T) { + storage, err := New(map[string]interface{}{ + "path": testPath, + "compression": false, + }) + assert.NoError(t, err) + + // Create a test image (100x100 pixels) + img := image.NewRGBA(image.Rect(0, 0, 100, 100)) + var buf bytes.Buffer + err = png.Encode(&buf, img) + assert.NoError(t, err) + + // Upload + reader := bytes.NewReader(buf.Bytes()) + fileID, err := storage.Upload(context.Background(), "test.png", reader, "image/png") + assert.NoError(t, err) + assert.NotEmpty(t, fileID) + + // Download and verify + reader2, contentType, err := storage.Download(context.Background(), fileID) + assert.NoError(t, err) + assert.Equal(t, "image/png", contentType) + + downloaded, err := io.ReadAll(reader2) + assert.NoError(t, err) + + // Decode the downloaded image + downloadedImg, _, err := image.Decode(bytes.NewReader(downloaded)) + assert.NoError(t, err) + + // Verify dimensions are unchanged + bounds := downloadedImg.Bounds() + assert.Equal(t, 100, bounds.Dx()) + assert.Equal(t, 100, bounds.Dy()) + }) + + t.Run("URL Generation", func(t *testing.T) { + storage, err := New(map[string]interface{}{ + "path": testPath, + "compression": true, + }) + assert.NoError(t, err) + + fileID := "20240101/test-12345678.txt" + url := storage.URL(context.Background(), fileID) + expected := filepath.Join(testPath, fileID) + assert.Equal(t, expected, url) + }) + + t.Run("Download Non-existent File", func(t *testing.T) { + storage, err := New(map[string]interface{}{ + "path": testPath, + "compression": true, + }) + assert.NoError(t, err) + + _, _, err = storage.Download(context.Background(), "non-existent.txt") + assert.Error(t, err) + }) + + t.Run("Chunked Upload", func(t *testing.T) { + storage, err := New(map[string]interface{}{ + "path": testPath, + }) + assert.NoError(t, err) + + fileID := "test-chunked.txt" + content1 := []byte("chunk1") + content2 := []byte("chunk2") + + // Upload chunks + err = storage.UploadChunk(context.Background(), fileID, 0, bytes.NewReader(content1), "text/plain") + assert.NoError(t, err) + + err = storage.UploadChunk(context.Background(), fileID, 1, bytes.NewReader(content2), "text/plain") + assert.NoError(t, err) + + // Merge chunks + err = storage.MergeChunks(context.Background(), fileID, 2) + assert.NoError(t, err) + + // Download and verify + reader, contentType, err := storage.Download(context.Background(), fileID) + assert.NoError(t, err) + assert.Equal(t, "text/plain", contentType) + + downloaded, err := io.ReadAll(reader) + assert.NoError(t, err) + assert.Equal(t, append(content1, content2...), downloaded) + }) + + t.Run("File Operations", func(t *testing.T) { + storage, err := New(map[string]interface{}{ + "path": testPath, + }) + assert.NoError(t, err) + + fileID := "test-ops.txt" + content := []byte("test content") + + // Upload file + _, err = storage.Upload(context.Background(), fileID, bytes.NewReader(content), "text/plain") + assert.NoError(t, err) + + // Check if file exists + exists := storage.Exists(context.Background(), fileID) + assert.True(t, exists) + + // Read file + reader, err := storage.Reader(context.Background(), fileID) + assert.NoError(t, err) + defer reader.Close() + + data, err := io.ReadAll(reader) + assert.NoError(t, err) + assert.Equal(t, content, data) + + // Delete file + err = storage.Delete(context.Background(), fileID) + assert.NoError(t, err) + + // Check if file no longer exists + exists = storage.Exists(context.Background(), fileID) + assert.False(t, exists) + }) +} diff --git a/neo/attachment/manager.go b/neo/attachment/manager.go new file mode 100644 index 00000000..63421328 --- /dev/null +++ b/neo/attachment/manager.go @@ -0,0 +1,452 @@ +package attachment + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "fmt" + "io" + "mime" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/yaoapp/yao/neo/attachment/local" + "github.com/yaoapp/yao/neo/attachment/s3" +) + +// Managers the managers +var Managers = map[string]*Manager{} + +// Register registers a global attachment manager +func Register(name string, driver string, option ManagerOption) (*Manager, error) { + + // Check if the manager already exists + if _, ok := Managers[name]; ok { + return nil, fmt.Errorf("manager %s already exists", name) + } + + // Create a new manager + manager, err := New(option) + if err != nil { + return nil, err + } + + // Register the manager + Managers[name] = manager + return manager, nil +} + +// New creates a new attachment manager +func New(option ManagerOption) (*Manager, error) { + manager := &Manager{ + ManagerOption: option, + allowedTypes: allowedType{mapping: make(map[string]bool), + wildcards: []string{}, + }} + + switch strings.ToLower(option.Driver) { + case "local": + storage, err := local.New(option.Options) + if err != nil { + return nil, err + } + manager.storage = storage + break + + case "s3": + storage, err := s3.New(option.Options) + if err != nil { + return nil, err + } + manager.storage = storage + break + + default: + return nil, fmt.Errorf("driver %s does not support", option.Driver) + } + + // Max size + if option.MaxSize != "" { + maxsize, err := getSize(option.MaxSize) + if err != nil { + return nil, err + } + manager.maxsize = maxsize + } + + // Chunk size + if option.ChunkSize != "" { + chunsize, err := getSize(option.ChunkSize) + if err != nil { + return nil, err + } + manager.chunsize = chunsize + } + + // init allowedTypes + if option.AllowedTypes != nil && len(option.AllowedTypes) > 0 { + for _, t := range option.AllowedTypes { + t = strings.TrimSpace(t) + if strings.HasSuffix(t, "*") { + manager.allowedTypes.wildcards = append(manager.allowedTypes.wildcards, t) + continue + } + manager.allowedTypes.mapping[t] = true + } + } + + return manager, nil +} + +// Upload uploads a file +func (manager Manager) Upload(ctx context.Context, fileheader *FileHeader, reader io.Reader, option UploadOption) (*File, error) { + + file, err := manager.makeFile(fileheader, option) + if err != nil { + return nil, err + } + + // Handle chunked upload + if fileheader.IsChunk() { + start, _, total, err := fileheader.GetChunkInfo() + if err != nil { + return nil, fmt.Errorf("invalid chunk info: %w", err) + } + + // Calculate chunk index based on start position and a standard chunk size + // We need to determine the standard chunk size from the first few chunks + standardChunkSize := int64(1024) // Default chunk size + if start > 0 { + // For non-first chunks, we can infer the standard chunk size + // by looking at the start position + if start%1024 == 0 { + standardChunkSize = 1024 + } else if start%2048 == 0 { + standardChunkSize = 2048 + } else if start%4096 == 0 { + standardChunkSize = 4096 + } else { + // Try to infer from the start position + for size := int64(512); size <= 8192; size *= 2 { + if start%size == 0 { + standardChunkSize = size + break + } + } + } + } + + chunkIndex := int(start / standardChunkSize) + totalChunks := int((total + standardChunkSize - 1) / standardChunkSize) // Ceiling division + + // Apply gzip compression if requested + if option.Gzip { + compressed, err := GzipFromReader(reader) + if err != nil { + return nil, fmt.Errorf("failed to gzip chunk: %w", err) + } + reader = bytes.NewReader(compressed) + } + + // Upload chunk + err = manager.storage.UploadChunk(ctx, file.ID, chunkIndex, reader, file.ContentType) + if err != nil { + return nil, err + } + + // If this is the last chunk, merge all chunks + if fileheader.Complete() { + err = manager.storage.MergeChunks(ctx, file.ID, totalChunks) + if err != nil { + return nil, err + } + + // Apply image compression if requested and it's the final file + if option.CompressImage && strings.HasPrefix(file.ContentType, "image/") { + err = manager.compressStoredImage(ctx, file, option) + if err != nil { + return nil, err + } + } + } + + return file, nil + } + + // Handle single file upload + var finalReader io.Reader = reader + + // Apply gzip compression if requested + if option.Gzip { + compressed, err := GzipFromReader(reader) + if err != nil { + return nil, fmt.Errorf("failed to gzip file: %w", err) + } + finalReader = bytes.NewReader(compressed) + } + + // Apply image compression if requested + if option.CompressImage && strings.HasPrefix(file.ContentType, "image/") { + size := option.CompressSize + if size == 0 { + size = 1920 + } + + // If gzip was applied, we need to decompress first + if option.Gzip { + data, err := io.ReadAll(finalReader) + if err != nil { + return nil, err + } + decompressed, err := Gunzip(data) + if err != nil { + return nil, err + } + finalReader = bytes.NewReader(decompressed) + } + + compressed, err := CompressImage(finalReader, file.ContentType, size) + if err != nil { + return nil, err + } + + // Re-apply gzip if it was requested + if option.Gzip { + gzipped, err := Gzip(compressed) + if err != nil { + return nil, err + } + finalReader = bytes.NewReader(gzipped) + } else { + finalReader = bytes.NewReader(compressed) + } + } + + // Upload the file to storage + id, err := manager.storage.Upload(ctx, file.ID, finalReader, file.ContentType) + if err != nil { + return nil, err + } + + // Update the file ID + file.ID = id + return file, nil +} + +// compressStoredImage compresses an already stored image +func (manager Manager) compressStoredImage(ctx context.Context, file *File, option UploadOption) error { + // Download the stored file + reader, err := manager.storage.Reader(ctx, file.ID) + if err != nil { + return err + } + defer reader.Close() + + size := option.CompressSize + if size == 0 { + size = 1920 + } + + // Compress the image + compressed, err := CompressImage(reader, file.ContentType, size) + if err != nil { + return err + } + + // Re-upload the compressed image + _, err = manager.storage.Upload(ctx, file.ID, bytes.NewReader(compressed), file.ContentType) + return err +} + +// Download downloads a file +func (manager Manager) Download(ctx context.Context, fileID string) (*FileResponse, error) { + reader, contentType, err := manager.storage.Download(ctx, fileID) + if err != nil { + return nil, err + } + + extension := filepath.Ext(fileID) + if extension == "" { + // Try to get extension from content type + extensions, err := mime.ExtensionsByType(contentType) + if err == nil && len(extensions) > 0 { + extension = extensions[0] + } + } + + return &FileResponse{ + Reader: reader, + ContentType: contentType, + Extension: extension, + }, nil +} + +// Read reads a file and returns the content as bytes +func (manager Manager) Read(ctx context.Context, fileID string) ([]byte, error) { + reader, err := manager.storage.Reader(ctx, fileID) + if err != nil { + return nil, err + } + defer reader.Close() + + return io.ReadAll(reader) +} + +// ReadBase64 reads a file and returns the content as base64 encoded string +func (manager Manager) ReadBase64(ctx context.Context, fileID string) (string, error) { + data, err := manager.Read(ctx, fileID) + if err != nil { + return "", err + } + + return base64.StdEncoding.EncodeToString(data), nil +} + +// validate validates the file and option +func (manager Manager) makeFile(file *FileHeader, option UploadOption) (*File, error) { + + // Validate max size + if manager.maxsize > 0 && file.Size > manager.maxsize { + return nil, fmt.Errorf("file size %d exceeds the maximum size of %d", file.Size, manager.maxsize) + } + + // Get the content type + contentType := file.Header.Get("Content-Type") + extension := filepath.Ext(file.Filename) + + // Get the extension from the content type if not available from filename + if extension == "" { + // Special handling for common types + switch contentType { + case "text/plain": + extension = ".txt" + case "image/jpeg": + extension = ".jpg" + case "image/png": + extension = ".png" + case "application/pdf": + extension = ".pdf" + default: + extensions, err := mime.ExtensionsByType(contentType) + if err == nil && len(extensions) > 0 { + // For text/plain, prefer .txt over .conf + if contentType == "text/plain" { + for _, ext := range extensions { + if ext == ".txt" { + extension = ext + break + } + } + if extension == "" { + extension = ".txt" + } + } else { + extension = extensions[0] + } + } + } + } + + // Validate allowed types + if !manager.allowed(contentType, extension) { + return nil, fmt.Errorf("%s type %s is not allowed", file.Filename, contentType) + } + + // Generate file ID + id, err := manager.generateFileID(file, extension, option) + if err != nil { + return nil, err + } + + return &File{ + ID: id, + Filename: file.Filename, + ContentType: contentType, + Bytes: int(file.Size), + CreatedAt: int(time.Now().Unix()), + }, nil +} + +func (manager Manager) allowed(contentType string, extension string) bool { + + // text/*, image/*, audio/*, video/*, application/yao-*, ... + for _, t := range manager.allowedTypes.wildcards { + prefix := strings.TrimSuffix(t, "*") + if strings.HasPrefix(contentType, prefix) { + return true + } + } + + // Accepted types + if _, ok := manager.allowedTypes.mapping[contentType]; ok { + return true + } + + // Accepted extensions + if _, ok := manager.allowedTypes.mapping[extension]; ok { + return true + } + + // Not allowed + return false +} + +// generateFileID generates a file ID with proper namespace +func (manager Manager) generateFileID(file *FileHeader, extension string, option UploadOption) (string, error) { + filename := file.Filename + if file.IsChunk() { + filename = file.UID() + } + + hash := fmt.Sprintf("%x", sha256.Sum256([]byte(filename)))[:8] + date := time.Now().Format("20060102") + path := filepath.Join("attachments", date) + if option.UserID != "" { + path = filepath.Join(path, option.UserID) + } + + if option.ChatID != "" { + path = filepath.Join(path, option.ChatID) + } + + if option.AssistantID != "" { + path = filepath.Join(path, option.AssistantID) + } + + return filepath.Join(path, hash[:2], hash[2:4], hash, extension), nil +} + +// getSize converts the size to bytes +func getSize(size string) (int64, error) { + if size == "" || size == "0" { + return 0, fmt.Errorf("size is empty") + } + + unit := strings.ToUpper(size[len(size)-1:]) + str := size[:len(size)-1] + if unit != "B" && unit != "K" && unit != "M" && unit != "G" { + unit = "B" + str = size + } + + value, err := strconv.ParseInt(str, 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid size: %s %s", size, err) + } + + switch unit { + case "B": + return value, nil + case "K": + return value * 1024, nil + case "M": + return value * 1024 * 1024, nil + case "G": + return value * 1024 * 1024 * 1024, nil + } + + return 0, fmt.Errorf("invalid size: %s", size) +} diff --git a/neo/attachment/manager_test.go b/neo/attachment/manager_test.go new file mode 100644 index 00000000..a3ba017d --- /dev/null +++ b/neo/attachment/manager_test.go @@ -0,0 +1,244 @@ +package attachment + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "mime/multipart" + "strings" + "testing" +) + +func TestManagerUpload(t *testing.T) { + // Create a local storage manager + manager, err := New(ManagerOption{ + Driver: "local", + MaxSize: "10M", + ChunkSize: "2M", + AllowedTypes: []string{"text/*", "image/*", ".txt", ".jpg", ".png"}, + Options: map[string]interface{}{ + "path": "/tmp/test_attachments", + }, + }) + if err != nil { + t.Fatalf("Failed to create manager: %v", err) + } + + // Test simple text file upload + t.Run("SimpleTextUpload", func(t *testing.T) { + content := "Hello, World!" + reader := strings.NewReader(content) + + // Create a mock file header + fileHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "test.txt", + Size: int64(len(content)), + Header: make(map[string][]string), + }, + } + fileHeader.Header.Set("Content-Type", "text/plain") + + option := UploadOption{ + UserID: "user123", + ChatID: "chat456", + AssistantID: "assistant789", + } + + file, err := manager.Upload(context.Background(), fileHeader, reader, option) + if err != nil { + t.Fatalf("Failed to upload file: %v", err) + } + + if file.Filename != "test.txt" { + t.Errorf("Expected filename 'test.txt', got '%s'", file.Filename) + } + + if file.ContentType != "text/plain" { + t.Errorf("Expected content type 'text/plain', got '%s'", file.ContentType) + } + + // Test download + response, err := manager.Download(context.Background(), file.ID) + if err != nil { + t.Fatalf("Failed to download file: %v", err) + } + defer response.Reader.Close() + + downloadedContent, err := manager.Read(context.Background(), file.ID) + if err != nil { + t.Fatalf("Failed to read file: %v", err) + } + + if string(downloadedContent) != content { + t.Errorf("Expected content '%s', got '%s'", content, string(downloadedContent)) + } + + // Test ReadBase64 + base64Content, err := manager.ReadBase64(context.Background(), file.ID) + if err != nil { + t.Fatalf("Failed to read file as base64: %v", err) + } + + expectedBase64 := base64.StdEncoding.EncodeToString([]byte(content)) + if base64Content != expectedBase64 { + t.Errorf("Expected base64 '%s', got '%s'", expectedBase64, base64Content) + } + }) + + // Test gzip compression + t.Run("GzipUpload", func(t *testing.T) { + content := "This is a test file that will be compressed with gzip." + reader := strings.NewReader(content) + + fileHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "test_gzip.txt", + Size: int64(len(content)), + Header: make(map[string][]string), + }, + } + fileHeader.Header.Set("Content-Type", "text/plain") + + option := UploadOption{ + Gzip: true, + UserID: "user123", + } + + file, err := manager.Upload(context.Background(), fileHeader, reader, option) + if err != nil { + t.Fatalf("Failed to upload gzipped file: %v", err) + } + + // The stored file should be compressed, but when we read it back, + // we should get the original content (if the storage handles decompression) + downloadedContent, err := manager.Read(context.Background(), file.ID) + if err != nil { + t.Fatalf("Failed to read gzipped file: %v", err) + } + + // Since we're storing compressed data, we need to decompress it + decompressed, err := Gunzip(downloadedContent) + if err != nil { + t.Fatalf("Failed to decompress file: %v", err) + } + + if string(decompressed) != content { + t.Errorf("Expected content '%s', got '%s'", content, string(decompressed)) + } + }) + + // Test chunked upload + t.Run("ChunkedUpload", func(t *testing.T) { + content := "This is a large file that will be uploaded in chunks. " + + strings.Repeat("Lorem ipsum dolor sit amet, consectetur adipiscing elit. ", 100) + + chunkSize := 1024 + totalSize := len(content) + + var lastFile *File + for start := 0; start < totalSize; start += chunkSize { + end := start + chunkSize - 1 + if end >= totalSize { + end = totalSize - 1 + } + + chunk := []byte(content[start : end+1]) + + fileHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "large_file.txt", + Size: int64(len(chunk)), + Header: make(map[string][]string), + }, + } + fileHeader.Header.Set("Content-Type", "text/plain") + fileHeader.Header.Set("Content-Range", + fmt.Sprintf("bytes %d-%d/%d", start, end, totalSize)) + fileHeader.Header.Set("Content-Uid", "unique-file-id-123") + + option := UploadOption{ + UserID: "user123", + } + + file, err := manager.Upload(context.Background(), fileHeader, bytes.NewReader(chunk), option) + if err != nil { + t.Fatalf("Failed to upload chunk starting at %d: %v", start, err) + } + + lastFile = file + } + + // After uploading all chunks, read the complete file + if lastFile != nil { + downloadedContent, err := manager.Read(context.Background(), lastFile.ID) + if err != nil { + t.Fatalf("Failed to read chunked file: %v", err) + } + + if string(downloadedContent) != content { + t.Errorf("Chunked upload content mismatch. Expected length %d, got %d", + len(content), len(downloadedContent)) + } + } + }) +} + +func TestManagerValidation(t *testing.T) { + manager, err := New(ManagerOption{ + Driver: "local", + MaxSize: "1K", // Very small max size for testing + AllowedTypes: []string{"text/plain"}, + Options: map[string]interface{}{ + "path": "/tmp/test_attachments", + }, + }) + if err != nil { + t.Fatalf("Failed to create manager: %v", err) + } + + // Test file size validation + t.Run("FileSizeValidation", func(t *testing.T) { + content := strings.Repeat("a", 2048) // 2KB, exceeds 1KB limit + reader := strings.NewReader(content) + + fileHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "large.txt", + Size: int64(len(content)), + Header: make(map[string][]string), + }, + } + fileHeader.Header.Set("Content-Type", "text/plain") + + option := UploadOption{} + + _, err := manager.Upload(context.Background(), fileHeader, reader, option) + if err == nil { + t.Error("Expected error for file size exceeding limit") + } + }) + + // Test file type validation + t.Run("FileTypeValidation", func(t *testing.T) { + content := "test" + reader := strings.NewReader(content) + + fileHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "test.jpg", + Size: int64(len(content)), + Header: make(map[string][]string), + }, + } + fileHeader.Header.Set("Content-Type", "image/jpeg") // Not allowed + + option := UploadOption{} + + _, err := manager.Upload(context.Background(), fileHeader, reader, option) + if err == nil { + t.Error("Expected error for disallowed file type") + } + }) +} diff --git a/neo/attachment/s3/storage.go b/neo/attachment/s3/storage.go new file mode 100644 index 00000000..14a8164f --- /dev/null +++ b/neo/attachment/s3/storage.go @@ -0,0 +1,381 @@ +package s3 + +import ( + "bytes" + "context" + "fmt" + "image" + "image/jpeg" + "image/png" + "io" + "path/filepath" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// DefaultExpiration default expiration time for presigned URLs (5 minutes) +const DefaultExpiration = 5 * time.Minute + +// MaxImageSize maximum image size (1920x1080) +const MaxImageSize = 1920 + +// Storage the S3 storage driver +type Storage struct { + Endpoint string `json:"endpoint" yaml:"endpoint"` + Region string `json:"region" yaml:"region"` + Key string `json:"key" yaml:"key"` + Secret string `json:"secret" yaml:"secret"` + Bucket string `json:"bucket" yaml:"bucket"` + Expiration time.Duration `json:"expiration" yaml:"expiration"` + client *s3.Client + prefix string + compression bool +} + +// New create a new S3 storage +func New(options map[string]interface{}) (*Storage, error) { + storage := &Storage{ + Region: "auto", + Expiration: DefaultExpiration, + compression: true, + } + + if endpoint, ok := options["endpoint"].(string); ok { + storage.Endpoint = endpoint + } + + if region, ok := options["region"].(string); ok { + storage.Region = region + } + + if key, ok := options["key"].(string); ok { + storage.Key = key + } + + if secret, ok := options["secret"].(string); ok { + storage.Secret = secret + } + + if bucket, ok := options["bucket"].(string); ok { + storage.Bucket = bucket + } + + if prefix, ok := options["prefix"].(string); ok { + storage.prefix = prefix + } + + if exp, ok := options["expiration"].(time.Duration); ok { + storage.Expiration = exp + } + + if compression, ok := options["compression"].(bool); ok { + storage.compression = compression + } + + // Validate required fields + if storage.Key == "" || storage.Secret == "" { + return nil, fmt.Errorf("key and secret are required") + } + + if storage.Bucket == "" { + return nil, fmt.Errorf("bucket is required") + } + + // Create S3 client + opts := s3.Options{ + Region: storage.Region, + Credentials: credentials.NewStaticCredentialsProvider(storage.Key, storage.Secret, ""), + UsePathStyle: true, + } + + if storage.Endpoint != "" { + // Remove bucket name from endpoint if present + endpoint := storage.Endpoint + if strings.Contains(endpoint, "/"+storage.Bucket) { + endpoint = strings.TrimSuffix(endpoint, "/"+storage.Bucket) + } + opts.BaseEndpoint = aws.String(endpoint) + } + + storage.client = s3.New(opts) + return storage, nil +} + +// Upload upload file to S3 +func (storage *Storage) Upload(ctx context.Context, fileID string, reader io.Reader, contentType string) (string, error) { + if storage.client == nil { + return "", fmt.Errorf("s3 client not initialized") + } + + key := filepath.Join(storage.prefix, fileID) + + // Upload file + _, err := storage.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(storage.Bucket), + Key: aws.String(key), + Body: reader, + ContentType: aws.String(contentType), + }) + if err != nil { + return "", fmt.Errorf("failed to upload file: %w", err) + } + + return fileID, nil +} + +// UploadChunk uploads a chunk of a file to S3 +func (storage *Storage) UploadChunk(ctx context.Context, fileID string, chunkIndex int, reader io.Reader, contentType string) error { + if storage.client == nil { + return fmt.Errorf("s3 client not initialized") + } + + // Store chunks with a special prefix + chunkKey := filepath.Join(storage.prefix, ".chunks", fileID, fmt.Sprintf("chunk_%d", chunkIndex)) + + _, err := storage.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(storage.Bucket), + Key: aws.String(chunkKey), + Body: reader, + ContentType: aws.String(contentType), + }) + if err != nil { + return fmt.Errorf("failed to upload chunk %d: %w", chunkIndex, err) + } + + return nil +} + +// MergeChunks merges all chunks into the final file in S3 +func (storage *Storage) MergeChunks(ctx context.Context, fileID string, totalChunks int) error { + if storage.client == nil { + return fmt.Errorf("s3 client not initialized") + } + + finalKey := filepath.Join(storage.prefix, fileID) + + // Create a buffer to hold the merged content + var mergedContent bytes.Buffer + var contentType string + + // Download and merge chunks in order + for i := 0; i < totalChunks; i++ { + chunkKey := filepath.Join(storage.prefix, ".chunks", fileID, fmt.Sprintf("chunk_%d", i)) + + result, err := storage.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(storage.Bucket), + Key: aws.String(chunkKey), + }) + if err != nil { + return fmt.Errorf("failed to get chunk %d: %w", i, err) + } + + // Get content type from the first chunk + if i == 0 && result.ContentType != nil { + contentType = *result.ContentType + } + + _, err = io.Copy(&mergedContent, result.Body) + result.Body.Close() + if err != nil { + return fmt.Errorf("failed to copy chunk %d: %w", i, err) + } + } + + // Default content type if not found + if contentType == "" { + contentType = "application/octet-stream" + } + + // Upload the merged content as the final file with proper content type + _, err := storage.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(storage.Bucket), + Key: aws.String(finalKey), + Body: bytes.NewReader(mergedContent.Bytes()), + ContentType: aws.String(contentType), + }) + if err != nil { + return fmt.Errorf("failed to upload merged file: %w", err) + } + + // Clean up chunks + for i := 0; i < totalChunks; i++ { + chunkKey := filepath.Join(storage.prefix, ".chunks", fileID, fmt.Sprintf("chunk_%d", i)) + storage.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(storage.Bucket), + Key: aws.String(chunkKey), + }) + } + + return nil +} + +// Reader read file from S3 +func (storage *Storage) Reader(ctx context.Context, fileID string) (io.ReadCloser, error) { + if storage.client == nil { + return nil, fmt.Errorf("s3 client not initialized") + } + + key := filepath.Join(storage.prefix, fileID) + + result, err := storage.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(storage.Bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, fmt.Errorf("failed to get file: %w", err) + } + + return result.Body, nil +} + +// Download download file from S3 +func (storage *Storage) Download(ctx context.Context, fileID string) (io.ReadCloser, string, error) { + if storage.client == nil { + return nil, "", fmt.Errorf("s3 client not initialized") + } + + key := filepath.Join(storage.prefix, fileID) + + // Get object + result, err := storage.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(storage.Bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, "", fmt.Errorf("failed to download file: %w", err) + } + + contentType := "application/octet-stream" + if result.ContentType != nil { + contentType = *result.ContentType + } + + return result.Body, contentType, nil +} + +// URL get file url with expiration +func (storage *Storage) URL(ctx context.Context, fileID string) string { + if storage.client == nil { + return "" + } + + key := filepath.Join(storage.prefix, fileID) + presignClient := s3.NewPresignClient(storage.client) + request, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(storage.Bucket), + Key: aws.String(key), + }, s3.WithPresignExpires(storage.Expiration)) + + if err != nil { + return "" + } + + return request.URL +} + +// Exists checks if a file exists in S3 +func (storage *Storage) Exists(ctx context.Context, fileID string) bool { + if storage.client == nil { + return false + } + + key := filepath.Join(storage.prefix, fileID) + _, err := storage.client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(storage.Bucket), + Key: aws.String(key), + }) + return err == nil +} + +// Delete deletes a file from S3 +func (storage *Storage) Delete(ctx context.Context, fileID string) error { + if storage.client == nil { + return fmt.Errorf("s3 client not initialized") + } + + key := filepath.Join(storage.prefix, fileID) + _, err := storage.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(storage.Bucket), + Key: aws.String(key), + }) + if err != nil { + return fmt.Errorf("failed to delete file: %w", err) + } + + return nil +} + +func (storage *Storage) makeID(filename string, ext string) string { + date := time.Now().Format("20060102") + name := strings.TrimSuffix(filepath.Base(filename), ext) + return fmt.Sprintf("%s/%s-%d%s", date, name, time.Now().UnixNano(), ext) +} + +// isImage checks if the content type is an image +func isImage(contentType string) bool { + return strings.HasPrefix(contentType, "image/") +} + +// compressImage compresses the image while maintaining aspect ratio +func compressImage(data []byte, contentType string) ([]byte, error) { + // Decode image + img, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("failed to decode image: %w", err) + } + + // Calculate new dimensions + bounds := img.Bounds() + width := bounds.Dx() + height := bounds.Dy() + var newWidth, newHeight int + + if width > height { + if width > MaxImageSize { + newWidth = MaxImageSize + newHeight = int(float64(height) * (float64(MaxImageSize) / float64(width))) + } else { + return data, nil // No need to resize + } + } else { + if height > MaxImageSize { + newHeight = MaxImageSize + newWidth = int(float64(width) * (float64(MaxImageSize) / float64(height))) + } else { + return data, nil // No need to resize + } + } + + // Create new image with new dimensions + newImg := image.NewRGBA(image.Rect(0, 0, newWidth, newHeight)) + + // Scale the image using bilinear interpolation + for y := 0; y < newHeight; y++ { + for x := 0; x < newWidth; x++ { + srcX := float64(x) * float64(width) / float64(newWidth) + srcY := float64(y) * float64(height) / float64(newHeight) + newImg.Set(x, y, img.At(int(srcX), int(srcY))) + } + } + + // Encode image + var buf bytes.Buffer + switch contentType { + case "image/jpeg": + err = jpeg.Encode(&buf, newImg, &jpeg.Options{Quality: 85}) + case "image/png": + err = png.Encode(&buf, newImg) + default: + return data, nil // Unsupported format, return original + } + + if err != nil { + return nil, fmt.Errorf("failed to encode image: %w", err) + } + + return buf.Bytes(), nil +} diff --git a/neo/attachment/s3/storage_test.go b/neo/attachment/s3/storage_test.go new file mode 100644 index 00000000..20b066c0 --- /dev/null +++ b/neo/attachment/s3/storage_test.go @@ -0,0 +1,191 @@ +package s3 + +import ( + "bytes" + "context" + "io" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func getS3Config() map[string]interface{} { + return map[string]interface{}{ + "endpoint": os.Getenv("S3_API"), + "region": "auto", + "key": os.Getenv("S3_ACCESS_KEY"), + "secret": os.Getenv("S3_SECRET_KEY"), + "bucket": os.Getenv("S3_BUCKET"), + "prefix": "attachment-test", + "expiration": 5 * time.Minute, + "compression": true, + } +} + +func skipIfNoS3Config(t *testing.T) { + if os.Getenv("S3_ACCESS_KEY") == "" || os.Getenv("S3_SECRET_KEY") == "" || os.Getenv("S3_BUCKET") == "" { + t.Skip("S3 configuration not available (set S3_ACCESS_KEY, S3_SECRET_KEY, S3_BUCKET environment variables)") + } +} + +func TestS3Storage(t *testing.T) { + t.Run("Create Storage", func(t *testing.T) { + options := getS3Config() + + storage, err := New(options) + if os.Getenv("S3_ACCESS_KEY") == "" || os.Getenv("S3_SECRET_KEY") == "" || os.Getenv("S3_BUCKET") == "" { + // Should fail without credentials + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.NotNil(t, storage) + if storage != nil { + assert.Equal(t, os.Getenv("S3_API"), storage.Endpoint) + assert.Equal(t, "auto", storage.Region) + assert.Equal(t, os.Getenv("S3_ACCESS_KEY"), storage.Key) + assert.Equal(t, os.Getenv("S3_SECRET_KEY"), storage.Secret) + assert.Equal(t, os.Getenv("S3_BUCKET"), storage.Bucket) + assert.Equal(t, "attachment-test", storage.prefix) + assert.Equal(t, 5*time.Minute, storage.Expiration) + assert.True(t, storage.compression) + } + }) + + t.Run("Upload and Download Text File", func(t *testing.T) { + skipIfNoS3Config(t) + + storage, err := New(getS3Config()) + assert.NoError(t, err) + + content := []byte("test content") + reader := bytes.NewReader(content) + fileID, err := storage.Upload(context.Background(), "test.txt", reader, "text/plain") + assert.NoError(t, err) + assert.NotEmpty(t, fileID) + + // Get presigned URL + url := storage.URL(context.Background(), fileID) + assert.NotEmpty(t, url) + assert.Contains(t, url, "X-Amz-Signature") + assert.Contains(t, url, "X-Amz-Expires") + + // Download + reader2, contentType, err := storage.Download(context.Background(), fileID) + assert.NoError(t, err) + assert.Contains(t, contentType, "text/plain") + + downloaded, err := io.ReadAll(reader2) + assert.NoError(t, err) + assert.Equal(t, content, downloaded) + reader2.Close() + + // Clean up + storage.Delete(context.Background(), fileID) + }) + + t.Run("Chunked Upload", func(t *testing.T) { + skipIfNoS3Config(t) + + storage, err := New(getS3Config()) + assert.NoError(t, err) + + fileID := "test-chunked.txt" + content1 := []byte("chunk1") + content2 := []byte("chunk2") + + // Upload chunks + err = storage.UploadChunk(context.Background(), fileID, 0, bytes.NewReader(content1), "text/plain") + assert.NoError(t, err) + + err = storage.UploadChunk(context.Background(), fileID, 1, bytes.NewReader(content2), "text/plain") + assert.NoError(t, err) + + // Merge chunks + err = storage.MergeChunks(context.Background(), fileID, 2) + assert.NoError(t, err) + + // Download and verify + reader, contentType, err := storage.Download(context.Background(), fileID) + assert.NoError(t, err) + assert.Equal(t, "text/plain", contentType) + + downloaded, err := io.ReadAll(reader) + assert.NoError(t, err) + assert.Equal(t, append(content1, content2...), downloaded) + reader.Close() + + // Clean up + storage.Delete(context.Background(), fileID) + }) + + t.Run("File Operations", func(t *testing.T) { + skipIfNoS3Config(t) + + storage, err := New(getS3Config()) + assert.NoError(t, err) + + fileID := "test-ops.txt" + content := []byte("test content") + + // Upload file + _, err = storage.Upload(context.Background(), fileID, bytes.NewReader(content), "text/plain") + assert.NoError(t, err) + + // Check if file exists + exists := storage.Exists(context.Background(), fileID) + assert.True(t, exists) + + // Read file + reader, err := storage.Reader(context.Background(), fileID) + assert.NoError(t, err) + defer reader.Close() + + data, err := io.ReadAll(reader) + assert.NoError(t, err) + assert.Equal(t, content, data) + + // Delete file + err = storage.Delete(context.Background(), fileID) + assert.NoError(t, err) + + // Check if file no longer exists + exists = storage.Exists(context.Background(), fileID) + assert.False(t, exists) + }) + + t.Run("Download Non-existent File", func(t *testing.T) { + skipIfNoS3Config(t) + + storage, err := New(getS3Config()) + assert.NoError(t, err) + + _, _, err = storage.Download(context.Background(), "non-existent.txt") + assert.Error(t, err) + }) + + t.Run("Invalid Configuration", func(t *testing.T) { + // Test with missing required fields + _, err := New(map[string]interface{}{ + "endpoint": "https://s3.amazonaws.com", + "region": "us-east-1", + // Missing key and secret + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "key and secret are required") + + // Test with missing bucket + _, err = New(map[string]interface{}{ + "endpoint": "https://s3.amazonaws.com", + "region": "us-east-1", + "key": "test-key", + "secret": "test-secret", + // Missing bucket + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "bucket is required") + }) +} diff --git a/neo/attachment/types.go b/neo/attachment/types.go new file mode 100644 index 00000000..c53559fc --- /dev/null +++ b/neo/attachment/types.go @@ -0,0 +1,93 @@ +package attachment + +import ( + "context" + "io" + "mime/multipart" +) + +// File the file +type File struct { + ID string `json:"file_id"` + Bytes int `json:"bytes"` + CreatedAt int `json:"created_at"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + Description string `json:"description,omitempty"` // Vision analysis result or other description + URL string `json:"url,omitempty"` // Vision URL for vision-capable models + DocIDs []string `json:"doc_ids,omitempty"` // RAG document IDs +} + +// FileResponse represents a file download response +type FileResponse struct { + Reader io.ReadCloser + ContentType string + Extension string +} + +// Attachment represents a file attachment +type Attachment struct { + Name string `json:"name,omitempty"` + URL string `json:"url,omitempty"` + Description string `json:"description,omitempty"` + Type string `json:"type,omitempty"` + ContentType string `json:"content_type,omitempty"` + Bytes int64 `json:"bytes,omitempty"` + CreatedAt int64 `json:"created_at,omitempty"` + FileID string `json:"file_id,omitempty"` + ChatID string `json:"chat_id,omitempty"` + AssistantID string `json:"assistant_id,omitempty"` + Gzip bool `json:"gzip,omitempty"` // Gzip the file, Optional, default is false +} + +// Manager the manager struct +type Manager struct { + ManagerOption + storage Storage + maxsize int64 + chunsize int64 + allowedTypes allowedType +} + +// Storage the storage interface +type Storage interface { + Upload(ctx context.Context, fileID string, reader io.Reader, contentType string) (string, error) + UploadChunk(ctx context.Context, fileID string, chunkIndex int, reader io.Reader, contentType string) error + MergeChunks(ctx context.Context, fileID string, totalChunks int) error + Download(ctx context.Context, fileID string) (io.ReadCloser, string, error) + Reader(ctx context.Context, fileID string) (io.ReadCloser, error) + URL(ctx context.Context, fileID string) string + Exists(ctx context.Context, fileID string) bool + Delete(ctx context.Context, fileID string) error +} + +// ManagerOption the manager option +type ManagerOption struct { + MaxSize string `json:"max_size,omitempty"` // Max size of the file, Optional, default is 20M + ChunkSize string `json:"chunk_size,omitempty"` // Chunk size of the file, Optional, default is 2M + AllowedTypes []string `json:"allowed_types,omitempty"` // Allowed types of the file, Optional, default is all + Driver string `json:"driver,omitempty"` // Driver, Optional, default is local + Options map[string]interface{} `json:"options,omitempty"` // Options, Optional +} + +type allowedType struct { + mapping map[string]bool + wildcards []string // Wildcard patterns for file types (e.g., "image/*", "text/*") +} + +// UploadOption the upload option +type UploadOption struct { + CompressImage bool `json:"compress_image,omitempty"` // Compress the file, Optional, default is true + CompressSize int `json:"compress_size,omitempty"` // Compress the file size, Optional, default is 1920, if compress_image is true, the file size will be compressed to the compress_size + Gzip bool `json:"gzip,omitempty"` // Gzip the file, Optional, default is false + Knowledge bool `json:"knowledge,omitempty"` // Push to knowledge base, Optional, default is false + ChatID string `json:"chat_id,omitempty"` // Chat ID, Optional + AssistantID string `json:"assistant_id,omitempty"` // Assistant ID, Optional + UserID string `json:"user_id,omitempty"` // User ID, Optional + ChunkSize string `json:"chunk_size,omitempty"` // Chunk size of the file, Optional, default is 2M, default is 2M +} + +// FileHeader the file header +type FileHeader struct { + *multipart.FileHeader +} diff --git a/neo/message/message.go b/neo/message/message.go index 1bb5e3f6..78a0e632 100644 --- a/neo/message/message.go +++ b/neo/message/message.go @@ -13,6 +13,7 @@ import ( "github.com/yaoapp/kun/exception" "github.com/yaoapp/kun/log" "github.com/yaoapp/kun/maps" + "github.com/yaoapp/yao/neo/attachment" "github.com/yaoapp/yao/openai" ) @@ -20,33 +21,33 @@ var locker = sync.Mutex{} // Message the message type Message struct { - ID string `json:"id,omitempty"` // id for the message - ToolID string `json:"tool_id,omitempty"` // tool_id for the message - Text string `json:"text,omitempty"` // text content - Type string `json:"type,omitempty"` // error, text, plan, table, form, page, file, video, audio, image, markdown, json ... - Props map[string]interface{} `json:"props,omitempty"` // props for the types - IsDone bool `json:"done,omitempty"` // Mark as a done message from neo - IsNew bool `json:"new,omitempty"` // Mark as a new message from neo - IsDelta bool `json:"delta,omitempty"` // Mark as a delta message from neo - Actions []Action `json:"actions,omitempty"` // Conversation Actions for frontend - Attachments []Attachment `json:"attachments,omitempty"` // File attachments - Role string `json:"role,omitempty"` // user, assistant, system ... - Name string `json:"name,omitempty"` // name for the message - AssistantID string `json:"assistant_id,omitempty"` // assistant_id (for assistant role = assistant ) - AssistantName string `json:"assistant_name,omitempty"` // assistant_name (for assistant role = assistant ) - AssistantAvatar string `json:"assistant_avatar,omitempty"` // assistant_avatar (for assistant role = assistant ) - Mentions []Mention `json:"menions,omitempty"` // Mentions for the message ( for user role = user ) - Data map[string]interface{} `json:"-"` // data for the message - Pending bool `json:"-"` // pending for the message - Hidden bool `json:"hidden,omitempty"` // hidden for the message (not show in the UI and history) - Retry bool `json:"retry,omitempty"` // retry for the message - Silent bool `json:"silent,omitempty"` // silent for the message (not show in the UI and history) - IsTool bool `json:"-"` // is tool for the message for native tool_calls - IsBeginTool bool `json:"-"` // is new tool for the message for native tool_calls - IsEndTool bool `json:"-"` // is end tool for the message for native tool_calls - Result any `json:"result,omitempty"` // result for the message - Begin int64 `json:"begin,omitempty"` // begin at for the message // timestamp - End int64 `json:"end,omitempty"` // end at for the message // timestamp + ID string `json:"id,omitempty"` // id for the message + ToolID string `json:"tool_id,omitempty"` // tool_id for the message + Text string `json:"text,omitempty"` // text content + Type string `json:"type,omitempty"` // error, text, plan, table, form, page, file, video, audio, image, markdown, json ... + Props map[string]interface{} `json:"props,omitempty"` // props for the types + IsDone bool `json:"done,omitempty"` // Mark as a done message from neo + IsNew bool `json:"new,omitempty"` // Mark as a new message from neo + IsDelta bool `json:"delta,omitempty"` // Mark as a delta message from neo + Actions []Action `json:"actions,omitempty"` // Conversation Actions for frontend + Attachments []attachment.Attachment `json:"attachments,omitempty"` // File attachments + Role string `json:"role,omitempty"` // user, assistant, system ... + Name string `json:"name,omitempty"` // name for the message + AssistantID string `json:"assistant_id,omitempty"` // assistant_id (for assistant role = assistant ) + AssistantName string `json:"assistant_name,omitempty"` // assistant_name (for assistant role = assistant ) + AssistantAvatar string `json:"assistant_avatar,omitempty"` // assistant_avatar (for assistant role = assistant ) + Mentions []Mention `json:"menions,omitempty"` // Mentions for the message ( for user role = user ) + Data map[string]interface{} `json:"-"` // data for the message + Pending bool `json:"-"` // pending for the message + Hidden bool `json:"hidden,omitempty"` // hidden for the message (not show in the UI and history) + Retry bool `json:"retry,omitempty"` // retry for the message + Silent bool `json:"silent,omitempty"` // silent for the message (not show in the UI and history) + IsTool bool `json:"-"` // is tool for the message for native tool_calls + IsBeginTool bool `json:"-"` // is new tool for the message for native tool_calls + IsEndTool bool `json:"-"` // is end tool for the message for native tool_calls + Result any `json:"result,omitempty"` // result for the message + Begin int64 `json:"begin,omitempty"` // begin at for the message // timestamp + End int64 `json:"end,omitempty"` // end at for the message // timestamp } // Mention represents a mention @@ -56,20 +57,6 @@ type Mention struct { Avatar string `json:"avatar,omitempty"` // avatar } -// Attachment represents a file attachment -type Attachment struct { - Name string `json:"name,omitempty"` - URL string `json:"url,omitempty"` - Description string `json:"description,omitempty"` - Type string `json:"type,omitempty"` - ContentType string `json:"content_type,omitempty"` - Bytes int64 `json:"bytes,omitempty"` - CreatedAt int64 `json:"created_at,omitempty"` - FileID string `json:"file_id,omitempty"` - ChatID string `json:"chat_id,omitempty"` - AssistantID string `json:"assistant_id,omitempty"` -} - // Action the action type Action struct { Name string `json:"name,omitempty"` @@ -500,7 +487,7 @@ func (m *Message) Map(msg map[string]interface{}) *Message { // attachments if attachments, has := msg["attachments"]; has { raw, _ := jsoniter.Marshal(attachments) - m.Attachments = []Attachment{} + m.Attachments = []attachment.Attachment{} if err := jsoniter.Unmarshal(raw, &m.Attachments); err != nil { color.Red("JSON parse error: %s", err.Error()) color.White(string(raw)) diff --git a/neo/neo.go b/neo/neo.go index 11315d01..8b63a7ed 100644 --- a/neo/neo.go +++ b/neo/neo.go @@ -68,8 +68,11 @@ func (neo *DSL) Upload(ctx chatctx.Context, c *gin.Context) (*assistant.File, er return nil, err } } + fmt.Println(ast) - return ast.Upload(ctx, tmpfile, reader, option) + return nil, nil + + // return ast.Upload(ctx, tmpfile, reader, option) } // Download downloads a file @@ -93,5 +96,7 @@ func (neo *DSL) Download(ctx chatctx.Context, c *gin.Context) (*assistant.FileRe } // Download file using the assistant - return ast.Download(ctx.Context, fileID) + // return ast.Download(ctx.Context, fileID) + fmt.Println(ast) + return nil, nil }