opencode/yao
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'upstream/main'

Browse source
This commit is contained in:
sunjuzhong 2025-08-23 21:20:11 +08:00
commit 904e550b75
370 changed files with 82780 additions and 3799 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
.github/workflows/build-docker-internal.yml vendored Normal file
View file

@ -0,0 +1,70 @@
name: Push docker images to internal registry
on:
# push:
# branches: [main]
# paths:
# - ".github/workflows/docker.yml"
workflow_run:
workflows: ["Build Linux Artifacts"]
types:
- completed
env:
VERSION: 0.10.5
jobs:
build:
if: ${{ github.event.workflow_run.conclusion == 'success' }}
runs-on: ubuntu-latest
timeout-minutes: 120
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Get Version
run: |
echo VERSION=$(cat share/const.go |grep 'const VERSION' | awk '{print $4}' | sed "s/\"//g")-unstable >> $GITHUB_ENV
- name: Check Version
run: echo $VERSION
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v3
with:
registry: hub.iqka.com
username: ${{ secrets.DOCKER_INTERNAL_USER }}
password: ${{ secrets.DOCKER_INTERNAL_TOKEN }}
- name: Build Production
timeout-minutes: 60
uses: docker/build-push-action@v6
env:
DOCKER_CONTENT_TRUST: 1
with:
context: ./docker/production
platforms: linux/amd64
build-args: |
VERSION=${{ env.VERSION }}
ARCH=amd64
push: true
tags: hub.iqka.com/yaoapp/yao:${{ env.VERSION }}-amd64
- name: Build Production Arm64
timeout-minutes: 60
uses: docker/build-push-action@v6
env:
DOCKER_CONTENT_TRUST: 1
with:
context: ./docker/production
platforms: linux/arm64
build-args: |
VERSION=${{ env.VERSION }}
ARCH=arm64
push: true
tags: hub.iqka.com/yaoapp/yao:${{ env.VERSION }}-arm64

92
.github/workflows/build-docker.yml vendored Normal file
View file

@ -0,0 +1,92 @@
name: Build and push docker images
on:
# push:
# branches: [main]
# paths:
# - ".github/workflows/docker.yml"
workflow_run:
workflows: ["Build Linux Artifacts"]
types:
- completed
env:
VERSION: 0.10.5
jobs:
build:
if: ${{ github.event.workflow_run.conclusion == 'success' }}
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Get Version
run: |
echo VERSION=$(cat share/const.go |grep 'const VERSION' | awk '{print $4}' | sed "s/\"//g")-unstable >> $GITHUB_ENV
- name: Check Version
run: echo $VERSION
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Build Development
uses: docker/build-push-action@v6
env:
DOCKER_CONTENT_TRUST: 1
with:
context: ./docker/development
platforms: linux/amd64
build-args: |
VERSION=${{ env.VERSION }}
ARCH=amd64
push: true
tags: yaoapp/yao:${{ env.VERSION }}-amd64-dev
- name: Build Development Arm64
uses: docker/build-push-action@v6
env:
DOCKER_CONTENT_TRUST: 1
with:
context: ./docker/development
platforms: linux/arm64
build-args: |
VERSION=${{ env.VERSION }}
ARCH=arm64
push: true
tags: yaoapp/yao:${{ env.VERSION }}-arm64-dev
- name: Build Production
uses: docker/build-push-action@v6
env:
DOCKER_CONTENT_TRUST: 1
with:
context: ./docker/production
platforms: linux/amd64
build-args: |
VERSION=${{ env.VERSION }}
ARCH=amd64
push: true
tags: yaoapp/yao:${{ env.VERSION }}-amd64
- name: Build Production Arm64
uses: docker/build-push-action@v6
env:
DOCKER_CONTENT_TRUST: 1
with:
context: ./docker/production
platforms: linux/arm64
build-args: |
VERSION=${{ env.VERSION }}
ARCH=arm64
push: true
tags: yaoapp/yao:${{ env.VERSION }}-arm64

24
.github/workflows/build-linux.yml vendored
View file

@ -9,14 +9,24 @@ on:
jobs:
build:
runs-on: "ubuntu-latest"
strategy:
matrix:
go: [1.23.4]
container:
image: yaoapp/yao-build:0.10.5
env:
CF_ACCESS_KEY_ID: ${{ secrets.CF_ACCESS_KEY_ID }}
CF_SECRET_ACCESS_KEY: ${{ secrets.CF_SECRET_ACCESS_KEY }}
R2_BUCKET: ${{ secrets.R2_BUCKET }}
R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
steps:
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: 20
- name: Configure R2 For Cloudflare
run: |
aws configure set aws_access_key_id $CF_ACCESS_KEY_ID
aws configure set aws_secret_access_key $CF_SECRET_ACCESS_KEY
aws configure set default.region us-east-1 # Update with your R2 region if different
aws configure set default.s3.signature_version s3v4
aws configure set default.s3.endpoint_url https://$R2_ACCOUNT_ID.r2.cloudflarestorage.com
aws --version
- name: Install pnpm
run: npm install -g pnpm

25
.github/workflows/build-macos.yml vendored
View file

@ -2,15 +2,18 @@ name: Build MacOS Artifacts
on:
workflow_dispatch:
push:
tags:
- 'v*'
inputs:
tags:
description: "Version tags"
env:
VERSION: 0.10.5
jobs:
build:
strategy:
matrix:
go: [1.23.4]
go: [1.24.3]
runs-on: "macos-latest"
steps:
- name: Setup Node.js
@ -65,11 +68,13 @@ jobs:
rm -rf $dir/__MACOSX
done
- name: Checkout XGen v1.0
- name: Checkout CUI v1.0
# ** XGEN will be renamed to CUI in the feature. and move to the new repository. **
# ** new repository: https://github.com/YaoApp/cui.git **
uses: actions/checkout@v4
with:
repository: sjzsdu/xgen
path: xgen-v1.0
repository: yaoapp/cui
path: cui-v1.0
- name: Checkout Yao-Init
uses: actions/checkout@v4
@ -83,12 +88,12 @@ jobs:
mv xun ../
mv gou ../
mv v8go ../
mv xgen-v1.0 ../
mv cui-v1.0 ../
mv yao-init ../
rm -f ../xgen-v1.0/packages/setup/vite.config.ts.*
rm -f ../cui-v1.0/packages/setup/vite.config.ts.*
ls -l .
ls -l ../
ls -l ../xgen-v1.0/packages/setup/
ls -l ../cui-v1.0/packages/setup/
- name: Checkout Code
uses: actions/checkout@v4

376
.github/workflows/pr-test.yml vendored Normal file
View file

@ -0,0 +1,376 @@
name: PR Unit Test
# read-write repo token
# access to secrets
on:
workflow_run:
workflows: ["Receive PR"]
types:
- completed
env:
YAO_DEV: ${{ github.WORKSPACE }}
YAO_ENV: development
YAO_ROOT: ${{ github.WORKSPACE }}/../app
YAO_HOST: 0.0.0.0
YAO_PORT: 5099
YAO_SESSION: "memory"
YAO_LOG: "./logs/application.log"
YAO_LOG_MODE: "TEXT"
YAO_JWT_SECRET: "bLp@bi!oqo-2U+hoTRUG"
YAO_DB_AESKEY: "ZLX=T&f6refeCh-ro*r@"
OSS_TEST_ID: ${{ secrets.OSS_TEST_ID}}
OSS_TEST_SECRET: ${{ secrets.OSS_TEST_SECRET}}
ROOT_PLUGIN: ${{ github.WORKSPACE }}/../../../data/gou-unit/plugins
MYSQL_TEST_HOST: "127.0.0.1"
MYSQL_TEST_PORT: "3308"
MYSQL_TEST_USER: test
MYSQL_TEST_PASS: "123456"
SQLITE_DB: "./app/db/yao.db"
REDIS_TEST_HOST: "127.0.0.1"
REDIS_TEST_PORT: "6379"
REDIS_TEST_DB: "2"
MONGO_TEST_HOST: "127.0.0.1"
MONGO_TEST_PORT: "27017"
MONGO_TEST_USER: "root"
MONGO_TEST_PASS: "123456"
OPENAI_TEST_KEY: ${{ secrets.OPENAI_TEST_KEY }}
TEST_MOAPI_SECRET: ${{ secrets.OPENAI_TEST_KEY }}
TEST_MOAPI_MIRROR: https://api.openai.com
TAB_NAME: "::PET ADMIN"
PAGE_SIZE: "20"
PAGE_LINK: "https://yaoapps.com"
PAGE_ICON: "icon-trash"
DEMO_APP: ${{ github.WORKSPACE }}/../app
# Application Setting
## Path
YAO_EXTENSION_ROOT: ${{ github.WORKSPACE }}/../extension
YAO_TEST_APPLICATION: ${{ github.WORKSPACE }}/../app
YAO_SUI_TEST_APPLICATION: ${{ github.WORKSPACE }}/../yao-startup-webapp
## Runtime
YAO_RUNTIME_MIN: 3
YAO_RUNTIME_MAX: 6
YAO_RUNTIME_HEAP_LIMIT: 1500000000
YAO_RUNTIME_HEAP_RELEASE: 10000000
YAO_RUNTIME_HEAP_AVAILABLE: 550000000
YAO_RUNTIME_PRECOMPILE: true
# Neo4j
NEO4J_TEST_URL: "neo4j://localhost:7686"
NEO4J_TEST_USER: "neo4j"
NEO4J_TEST_PASS: "Yao2026Neo4j"
# Qdrant
QDRANT_TEST_HOST: "127.0.0.1"
QDRANT_TEST_PORT: "6334"
# S3
S3_API: ${{ secrets.S3_API }}
S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
S3_BUCKET: ${{ secrets.S3_BUCKET }}
S3_PUBLIC_URL: ${{ secrets.S3_PUBLIC_URL }}
# === Openapi Signin Configs ===
## Google
GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }}
GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }}
## Microsoft
MICROSOFT_CLIENT_ID: ${{ secrets.MICROSOFT_CLIENT_ID }}
MICROSOFT_CLIENT_SECRET: ${{ secrets.MICROSOFT_CLIENT_SECRET }}
## Apple
APPLE_SERVICE_ID: ${{ secrets.APPLE_SERVICE_ID }}
APPLE_PRIVATE_KEY_PATH: "apple/signin_client_secret_key.p8"
APPLE_KEY_ID: ${{ secrets.APPLE_KEY_ID }}
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
## Github
GITHUBUSER_CLIENT_ID: ${{ secrets.GITHUBUSER_CLIENT_ID }}
GITHUBUSER_CLIENT_SECRET: ${{ secrets.GITHUBUSER_CLIENT_SECRET }}
## Cloudflare Turnstile
CLOUDFLARE_TURNSTILE_SITEKEY: ${{ secrets.CLOUDFLARE_TURNSTILE_SITEKEY }}
CLOUDFLARE_TURNSTILE_SECRET: ${{ secrets.CLOUDFLARE_TURNSTILE_SECRET }}
jobs:
UnitTest:
runs-on: ubuntu-latest
services:
qdrant:
image: qdrant/qdrant:latest
ports:
- 6333:6333 # HTTP API
- 6334:6334 # gRPC
fastembed:
image: yaoapp/fastembed:latest-amd64
env:
FASTEMBED_PASSWORD: Yao@2026
ports:
- 6001:8000
neo4j:
image: neo4j:latest
ports:
- "7687:7687"
env:
NEO4J_AUTH: neo4j/Yao2026Neo4j
mcp-everything:
image: yaoapp/mcp-everything:latest
ports:
- "3021:3021"
- "3022:3022"
strategy:
matrix:
go: [1.24]
db: [MySQL8.0, MySQL5.7, SQLite3]
redis: [4, 5, 6]
mongo: ["6.0"]
if: >
${{ github.event.workflow_run.event == 'pull_request' &&
github.event.workflow_run.conclusion == 'success' }}
steps:
- name: "Download artifact"
uses: actions/github-script@v7
with:
script: |
var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: ${{github.event.workflow_run.id }},
});
var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
return artifact.name == "pr"
})[0];
var download = await github.rest.actions.downloadArtifact({
owner: context.repo.owner,
repo: context.repo.repo,
artifact_id: matchArtifact.id,
archive_format: 'zip',
});
var fs = require('fs');
fs.writeFileSync('${{github.workspace}}/pr.zip', Buffer.from(download.data));
- name: "Read NR & SHA"
run: |
unzip pr.zip
cat NR
cat SHA
echo HEAD=$(cat SHA) >> $GITHUB_ENV
echo NR=$(cat NR) >> $GITHUB_ENV
- name: "Comment on PR"
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const { NR } = process.env
var fs = require('fs');
var issue_number = NR;
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issue_number,
body: 'Thank you for the PR! The db: ${{ matrix.db }} redis: ${{ matrix.redis }} mongo: ${{ matrix.mongo }} test workflow is running, the results of the run will be commented later.'
});
- name: Checkout Kun
uses: actions/checkout@v4
with:
repository: yaoapp/kun
path: kun
- name: Checkout Xun
uses: actions/checkout@v4
with:
repository: yaoapp/xun
path: xun
- name: Checkout Gou
uses: actions/checkout@v4
with:
repository: yaoapp/gou
path: gou
- name: Checkout V8Go
uses: actions/checkout@v4
with:
repository: yaoapp/v8go
path: v8go
- name: Unzip libv8
run: |
files=$(find ./v8go -name "libv8*.zip")
for file in $files; do
dir=$(dirname "$file") # Get the directory where the ZIP file is located
echo "Extracting $file to directory $dir"
unzip -o -d $dir $file
rm -rf $dir/__MACOSX
done
- name: Checkout Demo App
uses: actions/checkout@v4
with:
repository: yaoapp/yao-dev-app
path: app
- name: Checkout Yao Startup Webapp
uses: actions/checkout@v4
with:
repository: yaoapp/yao-startup-webapp
submodules: true
token: ${{ secrets.YAO_TEST_TOKEN }}
path: yao-startup-webapp
- name: Checkout Extension
uses: actions/checkout@v4
with:
repository: yaoapp/yao-extensions-dev
path: extension
- name: Move Kun, Xun, Gou, V8Go
run: |
mv kun ../
mv xun ../
mv gou ../
mv v8go ../
mv app ../
mv extension ../
mv yao-startup-webapp ../
ls -l .
ls -l ../
- name: Checkout pull request HEAD commit
uses: actions/checkout@v4
with:
ref: ${{ env.HEAD }}
- name: Setup Apple Private Key
run: |
mkdir -p ../app/openapi/certs/apple
echo "${{ secrets.APPLE_PRIVATE_KEY_USER }}" > ../app/openapi/certs/apple/signin_client_secret_key.p8
- name: Start Redis
uses: supercharge/redis-github-action@1.4.0
with:
redis-version: ${{ matrix.redis }}
- name: Setup Go ${{ matrix.go }}
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go }}
- name: Install FFmpeg 7.x
run: |
wget https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz
tar -xf ffmpeg-master-latest-linux64-gpl.tar.xz
sudo cp ffmpeg-master-latest-linux64-gpl/bin/ffmpeg /usr/local/bin/
sudo cp ffmpeg-master-latest-linux64-gpl/bin/ffprobe /usr/local/bin/
sudo chmod +x /usr/local/bin/ffmpeg /usr/local/bin/ffprobe
- name: Test FFmpeg
run: ffmpeg -version
- name: Install pdftoppm, mutool, imagemagick
run: |
sudo apt update
sudo apt install -y poppler-utils mupdf-tools imagemagick
- name: Test pdftoppm, mutool, imagemagick
run: |
pdftoppm -v
mutool -v
convert -version
- name: Start MongoDB
uses: supercharge/mongodb-github-action@1.8.0
with:
mongodb-version: ${{ matrix.mongo }}
mongodb-username: root
mongodb-password: 123456
mongodb-db: test
- name: Setup MySQL8.0 (connector)
uses: ./.github/actions/setup-db
with:
kind: "MySQL8.0"
db: "test"
user: "test"
password: "123456"
port: "3308"
- name: Setup ${{ matrix.db }}
uses: ./.github/actions/setup-db
with:
kind: "${{ matrix.db }}"
db: "xiang"
user: "xiang"
password: ${{ secrets.UNIT_PASS }}
- name: Setup Go Tools
run: |
make tools
- name: Setup ENV & Host
env:
PASSWORD: ${{ secrets.UNIT_PASS }}
run: |
sudo echo "127.0.0.1 local.iqka.com" | sudo tee -a /etc/hosts
echo "YAO_DB_DRIVER=$DB_DRIVER" >> $GITHUB_ENV
echo "GITHUB_WORKSPACE:\n" && ls -l $GITHUB_WORKSPACE
if [ "$DB_DRIVER" = "mysql" ]; then
echo "YAO_DB_PRIMARY=$DB_USER:$PASSWORD@$DB_HOST" >> $GITHUB_ENV
elif [ "$DB_DRIVER" = "postgres" ]; then
echo "YAO_DB_PRIMARY=postgres://$DB_USER:$PASSWORD@$DB_HOST" >> $GITHUB_ENV
else
echo "YAO_DB_PRIMARY=$YAO_ROOT/$DB_HOST" >> $GITHUB_ENV
fi
echo ".:\n" && ls -l .
echo "..:\n" && ls -l ..
ping -c 1 -t 1 local.iqka.com
- name: Test Prepare
run: |
make vet
make fmt-check
make misspell-check
- name: Run test
run: |
make test
- name: Codecov Report
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
- name: "Comment on PR"
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const { NR } = process.env
var fs = require('fs');
var issue_number = NR;
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issue_number,
body: '✨DONE✨ db: ${{ matrix.db }} redis: ${{ matrix.redis }} mongo: ${{ matrix.mongo }} passed.'
});

323
.github/workflows/unit-test.yml vendored Normal file
View file

@ -0,0 +1,323 @@
name: Unit Test
on:
workflow_dispatch:
inputs:
tags:
description: "Version"
push:
branches: [main]
env:
YAO_DEV: ${{ github.WORKSPACE }}
YAO_ENV: development
YAO_ROOT: ${{ github.WORKSPACE }}/../app
YAO_HOST: 0.0.0.0
YAO_PORT: 5099
YAO_SESSION: "memory"
YAO_LOG: "./logs/application.log"
YAO_LOG_MODE: "TEXT"
YAO_JWT_SECRET: "bLp@bi!oqo-2U+hoTRUG"
YAO_DB_AESKEY: "ZLX=T&f6refeCh-ro*r@"
OSS_TEST_ID: ${{ secrets.OSS_TEST_ID}}
OSS_TEST_SECRET: ${{ secrets.OSS_TEST_SECRET}}
ROOT_PLUGIN: ${{ github.WORKSPACE }}/../../../data/gou-unit/plugins
REPO_KUN: ${{ github.repository_owner }}/kun
REPO_XUN: ${{ github.repository_owner }}/xun
REPO_GOU: ${{ github.repository_owner }}/gou
MYSQL_TEST_HOST: "127.0.0.1"
MYSQL_TEST_PORT: "3308"
MYSQL_TEST_USER: test
MYSQL_TEST_PASS: "123456"
SQLITE_DB: "./app/db/yao.db"
REDIS_TEST_HOST: "127.0.0.1"
REDIS_TEST_PORT: "6379"
REDIS_TEST_DB: "2"
MONGO_TEST_HOST: "127.0.0.1"
MONGO_TEST_PORT: "27017"
MONGO_TEST_USER: "root"
MONGO_TEST_PASS: "123456"
OPENAI_TEST_KEY: ${{ secrets.OPENAI_TEST_KEY }}
TEST_MOAPI_SECRET: ${{ secrets.OPENAI_TEST_KEY }}
TEST_MOAPI_MIRROR: https://api.openai.com
TAB_NAME: "::PET ADMIN"
PAGE_SIZE: "20"
PAGE_LINK: "https://yaoapps.com"
PAGE_ICON: "icon-trash"
DEMO_APP: ${{ github.WORKSPACE }}/../app
# Application Setting
## Path
YAO_EXTENSION_ROOT: ${{ github.WORKSPACE }}/../extension
YAO_TEST_APPLICATION: ${{ github.WORKSPACE }}/../app
YAO_SUI_TEST_APPLICATION: ${{ github.WORKSPACE }}/../yao-startup-webapp
## Runtime
YAO_RUNTIME_MIN: 3
YAO_RUNTIME_MAX: 6
YAO_RUNTIME_HEAP_LIMIT: 1500000000
YAO_RUNTIME_HEAP_RELEASE: 10000000
YAO_RUNTIME_HEAP_AVAILABLE: 550000000
YAO_RUNTIME_PRECOMPILE: true
# Neo4j
NEO4J_TEST_URL: "neo4j://localhost:7686"
NEO4J_TEST_USER: "neo4j"
NEO4J_TEST_PASS: "Yao2026Neo4j"
# Qdrant
QDRANT_TEST_HOST: "127.0.0.1"
QDRANT_TEST_PORT: "6334"
# S3
S3_API: ${{ secrets.S3_API }}
S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
S3_BUCKET: ${{ secrets.S3_BUCKET }}
S3_PUBLIC_URL: ${{ secrets.S3_PUBLIC_URL }}
# === Openapi Signin Configs ===
## Google
GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }}
GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }}
## Microsoft
MICROSOFT_CLIENT_ID: ${{ secrets.MICROSOFT_CLIENT_ID }}
MICROSOFT_CLIENT_SECRET: ${{ secrets.MICROSOFT_CLIENT_SECRET }}
## Apple
APPLE_SERVICE_ID: ${{ secrets.APPLE_SERVICE_ID }}
APPLE_PRIVATE_KEY_PATH: "apple/signin_client_secret_key.p8"
APPLE_KEY_ID: ${{ secrets.APPLE_KEY_ID }}
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
## Github
GITHUBUSER_CLIENT_ID: ${{ secrets.GITHUBUSER_CLIENT_ID }}
GITHUBUSER_CLIENT_SECRET: ${{ secrets.GITHUBUSER_CLIENT_SECRET }}
## Cloudflare Turnstile
CLOUDFLARE_TURNSTILE_SITEKEY: ${{ secrets.CLOUDFLARE_TURNSTILE_SITEKEY }}
CLOUDFLARE_TURNSTILE_SECRET: ${{ secrets.CLOUDFLARE_TURNSTILE_SECRET }}
jobs:
unit-test:
runs-on: ubuntu-latest
services:
qdrant:
image: qdrant/qdrant:latest
ports:
- 6333:6333 # HTTP API
- 6334:6334 # gRPC
fastembed:
image: yaoapp/fastembed:latest-amd64
env:
FASTEMBED_PASSWORD: Yao@2026
ports:
- 6001:8000
neo4j:
image: neo4j:latest
ports:
- "7687:7687"
env:
NEO4J_AUTH: neo4j/Yao2026Neo4j
mcp-everything:
image: yaoapp/mcp-everything:latest
ports:
- "3021:3021"
- "3022:3022"
strategy:
matrix:
go: [1.24]
db: [MySQL8.0, MySQL5.7, SQLite3]
redis: [4, 5, 6]
mongo: ["6.0"]
steps:
- name: Checkout Kun
uses: actions/checkout@v4
with:
repository: ${{ env.REPO_KUN }}
path: kun
- name: Checkout Xun
uses: actions/checkout@v4
with:
repository: ${{ env.REPO_XUN }}
path: xun
- name: Checkout Gou
uses: actions/checkout@v4
with:
repository: ${{ env.REPO_GOU }}
path: gou
- name: Checkout V8Go
uses: actions/checkout@v4
with:
repository: yaoapp/v8go
path: v8go
- name: Unzip libv8
run: |
files=$(find ./v8go -name "libv8*.zip")
for file in $files; do
dir=$(dirname "$file") # Get the directory where the ZIP file is located
echo "Extracting $file to directory $dir"
unzip -o -d $dir $file
rm -rf $dir/__MACOSX
done
- name: Checkout Demo App
uses: actions/checkout@v4
with:
repository: yaoapp/yao-dev-app
path: app
- name: Checkout Yao Startup Webapp
uses: actions/checkout@v4
with:
repository: yaoapp/yao-startup-webapp
submodules: true
token: ${{ secrets.YAO_TEST_TOKEN }}
path: yao-startup-webapp
- name: Checkout Extension
uses: actions/checkout@v4
with:
repository: yaoapp/yao-extensions-dev
path: extension
- name: Move Kun, Xun, Gou, V8Go, Extension
run: |
mv kun ../
mv xun ../
mv gou ../
mv v8go ../
mv app ../
mv extension ../
mv yao-startup-webapp ../
ls -l .
ls -l ../
- name: Checkout Code
uses: actions/checkout@v4
- name: Setup Apple Private Key
run: |
mkdir -p ../app/openapi/certs/apple
echo "${{ secrets.APPLE_PRIVATE_KEY_USER }}" > ../app/openapi/certs/apple/signin_client_secret_key.p8
- name: Setup Go ${{ matrix.go }}
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go }}
- name: Install FFmpeg 7.x
run: |
wget https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz
tar -xf ffmpeg-master-latest-linux64-gpl.tar.xz
sudo cp ffmpeg-master-latest-linux64-gpl/bin/ffmpeg /usr/local/bin/
sudo cp ffmpeg-master-latest-linux64-gpl/bin/ffprobe /usr/local/bin/
sudo chmod +x /usr/local/bin/ffmpeg /usr/local/bin/ffprobe
- name: Test FFmpeg
run: ffmpeg -version
- name: Install pdftoppm, mutool, imagemagick
run: |
sudo apt update
sudo apt install -y poppler-utils mupdf-tools imagemagick
- name: Test pdftoppm, mutool, imagemagick
run: |
pdftoppm -v
mutool -v
convert -version
- name: Start Redis
uses: supercharge/redis-github-action@1.4.0
with:
redis-version: ${{ matrix.redis }}
- name: Start MongoDB
uses: supercharge/mongodb-github-action@1.8.0
with:
mongodb-version: ${{ matrix.mongo }}
mongodb-username: root
mongodb-password: 123456
mongodb-db: test
- name: Setup MySQL8.0 (connector)
uses: ./.github/actions/setup-db
with:
kind: "MySQL8.0"
db: "test"
user: "test"
password: "123456"
port: "3308"
- name: Setup ${{ matrix.db }}
uses: ./.github/actions/setup-db
with:
kind: "${{ matrix.db }}"
db: "xiang"
user: "xiang"
password: ${{ secrets.UNIT_PASS }}
- name: Setup Go Tools
run: |
make tools
- name: Setup ENV & Host
env:
PASSWORD: ${{ secrets.UNIT_PASS }}
run: |
sudo echo "127.0.0.1 local.iqka.com" | sudo tee -a /etc/hosts
echo "YAO_DB_DRIVER=$DB_DRIVER" >> $GITHUB_ENV
echo "GITHUB_WORKSPACE:\n" && ls -l $GITHUB_WORKSPACE
if [ "$DB_DRIVER" = "mysql" ]; then
echo "YAO_DB_PRIMARY=$DB_USER:$PASSWORD@$DB_HOST" >> $GITHUB_ENV
elif [ "$DB_DRIVER" = "postgres" ]; then
echo "YAO_DB_PRIMARY=postgres://$DB_USER:$PASSWORD@$DB_HOST" >> $GITHUB_ENV
else
echo "YAO_DB_PRIMARY=$YAO_ROOT/$DB_HOST" >> $GITHUB_ENV
fi
echo ".:\n" && ls -l .
echo "..:\n" && ls -l ..
echo "../app:\n" && ls -l ../app
ping -c 1 -t 1 local.iqka.com
- name: Test Prepare
run: |
make vet
make fmt-check
make misspell-check
- name: Inspect
run: |
go run . run utils.env.Get MONGO_TEST_HOST
go run . run utils.env.Get REDIS_TEST_HOST
go run . inspect
- name: Run test
run: |
make test
- name: Codecov Report
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos

5
.gitignore vendored
View file

@ -45,3 +45,8 @@ db
!get-yao.sh
!docker/base/**
__debug*
data/bindata.go.bak
share/const.go.bak
share/const.goe
.cursor
openapi/*.md

2
COMMERCIAL_LICENSE.md
View file

@ -1,3 +1,5 @@
> **DEPRECATED**: This license is no longer in effect. Please refer to the [LICENSE](LICENSE) file for current licensing terms.
# Commercial License for Yao
This document outlines the terms for the commercial license of the **Yao** project. While the Yao project is primarily licensed under the **Apache License, Version 2.0**, certain commercial use cases require a separate commercial license.

6
COMMERCIAL_LICENSE.zh-CN.md
View file

@ -1,3 +1,5 @@
> **已废弃**: 本许可证已不再生效。请参考 [LICENSE](LICENSE) 文件获取当前的许可条款。
# Yao 商业许可证
本文件概述了 **Yao** 项目的商业许可证条款。虽然 Yao 项目主要使用 **Apache 许可证 2.0 版** 授权，但某些商业使用场景需要单独的商业许可证。
@ -12,7 +14,7 @@
### 定义：应用托管服务
“应用托管服务”指任何涉及托管基于 Yao 的应用程序或使用 Yao 创建的 WEB 应用程序（包括 Yao 的修改版本）的服务，服务对象为第三方用户。包括但不限于：
"应用托管服务"指任何涉及托管基于 Yao 的应用程序或使用 Yao 创建的 WEB 应用程序（包括 Yao 的修改版本）的服务，服务对象为第三方用户。包括但不限于：
- **托管平台** 提供基于 Yao 的软件或服务给第三方用户。
- **SaaS 或 PaaS 服务**，在这些服务中，您管理并托管基于或利用 Yao 的应用程序，可能是原版或修改版。
@ -25,7 +27,7 @@
### 定义：AI WEB 应用生成服务
“AI WEB 应用生成服务”指任何利用 Yao（或任何分支版本或修改版本的 Yao）自动化创建具有 AI 功能的 WEB 应用程序的服务或功能。包括但不限于，为第三方用户提供以下服务：
"AI WEB 应用生成服务"指任何利用 Yao（或任何分支版本或修改版本的 Yao）自动化创建具有 AI 功能的 WEB 应用程序的服务或功能。包括但不限于，为第三方用户提供以下服务：
- **AI 驱动的自动化 WEB 应用开发**，该服务生成完整或部分 WEB 应用程序。
- **可定制的 WEB 解决方案**，这些解决方案由 AI 提供支持，并以 Yao 作为核心技术构建。

207
LICENSE
View file

@ -1,201 +1,24 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
# Open Source License
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
Yao App Engine is licensed under a modified version of the Apache License 2.0, with the following additional conditions:
1. Definitions.
1. Commercial Usage Terms:
Yao App Engine may be utilized commercially, A commercial license from the producer is required if:
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
a. Trademark and Branding Requirements
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
- The Yao App Engine console/application logo and copyright information must not be removed or modified
- Logo and copyright information can only be changed with an authorization certificate issued through Yao Developer Certificate
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
b. Authorization Verification Requirements
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
- The Yao certificate verification logic, processes, and related pages (marked in code comments) must be preserved
- The complete Yao certificate verification system must be maintained regardless of usage purpose
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
2. Contributor Agreement:
- The producer reserves the right to modify the open-source agreement terms
- Contributed code may be used for commercial purposes, including cloud business operations
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
All other rights and restrictions follow the Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0).
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
© 2025 Infinite Wisdom Software.

123
Makefile
View file

@ -129,7 +129,7 @@ bindata:
cp -r .tmp/yao-init .tmp/data/init
cp -r ui .tmp/data/
cp -r ui .tmp/data/public
cp -r xgen .tmp/data/
cp -r cui .tmp/data/
cp -r yao .tmp/data/
cp -r sui/libsui .tmp/data/
find .tmp/data -name ".DS_Store" -type f -delete
@ -142,11 +142,11 @@ bindata:
artifacts-linux: clean
mkdir -p dist/release
# Building XGEN v1.0
# Building CUI v1.0
export NODE_ENV=production
# rm -f ../xgen-v1.0/pnpm-lock.yaml
echo "BASE=__yao_admin_root" > ../xgen-v1.0/packages/xgen/.env
cd ../xgen-v1.0 && pnpm install --no-frozen-lockfile && pnpm run build
# rm -f ../cui-v1.0/pnpm-lock.yaml
echo "BASE=__yao_admin_root" > ../cui-v1.0/packages/cui/.env
cd ../cui-v1.0 && pnpm install --no-frozen-lockfile && pnpm run build
# Init Application
cd ../yao-init && rm -rf .git
@ -162,11 +162,11 @@ artifacts-linux: clean
# rm -rf .tmp/yao-builder-latest.tar.gz
# Packing
# ** XGEN will be renamed to DUI in the feature. and move to the new repository. **
# ** new repository: https://github.com/YaoApp/dui.git **
mkdir -p .tmp/data/xgen
# ** CUI will be renamed to CUI in the feature. and move to the new repository. **
# ** new repository: https://github.com/YaoApp/cui.git **
mkdir -p .tmp/data/cui
cp -r ./ui .tmp/data/ui
cp -r ../xgen-v1.0/packages/xgen/dist .tmp/data/xgen/v1.0
cp -r ../cui-v1.0/packages/cui/dist .tmp/data/cui/v1.0
cp -r ../yao-init .tmp/data/init
cp -r yao .tmp/data/
cp -r sui/libsui .tmp/data/
@ -175,6 +175,8 @@ artifacts-linux: clean
# Replace PRVERSION
sed -ie "s/const PRVERSION = \"DEV\"/const PRVERSION = \"${COMMIT}-${NOW}\"/g" share/const.go
@CUI_COMMIT=$$(cd ../cui-v1.0 && git log | head -n 1 | awk '{print substr($$2, 0, 12)}') && \
sed -ie "s/const PRCUI = \"DEV\"/const PRCUI = \"$$CUI_COMMIT-${NOW}\"/g" share/const.go
# Making artifacts
mkdir -p dist
@ -197,11 +199,11 @@ artifacts-macos: clean
mkdir -p dist/release
# Building XGEN v1.0
# Building CUI v1.0
export NODE_ENV=production
# rm -f ../xgen-v1.0/pnpm-lock.yaml
echo "BASE=__yao_admin_root" > ../xgen-v1.0/packages/xgen/.env
cd ../xgen-v1.0 && pnpm install --no-frozen-lockfile && pnpm run build
# rm -f ../cui-v1.0/pnpm-lock.yaml
echo "BASE=__yao_admin_root" > ../cui-v1.0/packages/cui/.env
cd ../cui-v1.0 && pnpm install --no-frozen-lockfile && pnpm run build
# Init Application
cd ../yao-init && rm -rf .git
@ -209,19 +211,10 @@ artifacts-macos: clean
cd ../yao-init && rm -rf LICENSE
# cd ../yao-init && rm -rf README.md
# Yao Builder
# Remove Yao Builder - DUI PageBuilder component will provide online design for pure HTML pages or SUI pages in the future.
# mkdir -p .tmp/data/builder
# curl -o .tmp/yao-builder-latest.tar.gz https://release-sv.yaoapps.com/archives/yao-builder-latest.tar.gz
# tar -zxvf .tmp/yao-builder-latest.tar.gz -C .tmp/data/builder
# rm -rf .tmp/yao-builder-latest.tar.gz
# Packing
# ** XGEN will be renamed to DUI in the feature. and move to the new repository. **
# ** new repository: https://github.com/YaoApp/dui.git **
mkdir -p .tmp/data/xgen
mkdir -p .tmp/data/cui
cp -r ./ui .tmp/data/ui
cp -r ../xgen-v1.0/packages/xgen/dist .tmp/data/xgen/v1.0
cp -r ../cui-v1.0/packages/cui/dist .tmp/data/cui/v1.0
cp -r ../yao-init .tmp/data/init
cp -r yao .tmp/data/
cp -r sui/libsui .tmp/data/
@ -230,6 +223,8 @@ artifacts-macos: clean
# Replace PRVERSION
sed -ie "s/const PRVERSION = \"DEV\"/const PRVERSION = \"${COMMIT}-${NOW}\"/g" share/const.go
@CUI_COMMIT=$$(cd ../cui-v1.0 && git log | head -n 1 | awk '{print substr($$2, 0, 12)}') && \
sed -ie "s/const PRCUI = \"DEV\"/const PRCUI = \"$$CUI_COMMIT-${NOW}\"/g" share/const.go
# Making artifacts
mkdir -p dist
@ -242,9 +237,6 @@ artifacts-macos: clean
ls -l dist/release/
dist/release/yao-${VERSION}-dev-darwin-amd64 version
# Reset const
# cp -f share/const.goe share/const.go
# rm -f share/const.goe
.PHONY: debug
debug: clean
@ -275,18 +267,19 @@ release: clean
mkdir -p dist/release
mkdir .tmp
# Building XGEN v0.9
mkdir -p .tmp/xgen/v0.9/dist
echo "XGEN v0.9" > .tmp/xgen/v0.9/dist/index.html
# Building CUI v0.9
mkdir -p .tmp/cui/v0.9/dist
echo "CUI v0.9" > .tmp/cui/v0.9/dist/index.html
# Building XGEN v1.0
# ** XGEN will be renamed to DUI in the feature. and move to the new repository. **
# ** new repository: https://github.com/YaoApp/dui.git **
# Building CUI v1.0
# ** CUI will be renamed to CUI in the feature. and move to the new repository. **
# ** new repository: https://github.com/YaoApp/cui.git **
export NODE_ENV=production
git clone https://github.com/sjzsdu/xgen.git .tmp/xgen/v1.0
# cd .tmp/xgen/v1.0 && git checkout 5002c3fded585aaa69a4366135b415ea3234964e
echo "BASE=__yao_admin_root" > .tmp/xgen/v1.0/packages/xgen/.env
cd .tmp/xgen/v1.0 && pnpm install --no-frozen-lockfile && pnpm run build
git clone https://github.com/YaoApp/cui.git .tmp/cui/v1.0
# cd .tmp/cui/v1.0 && git checkout 5002c3fded585aaa69a4366135b415ea3234964e
echo "BASE=__yao_admin_root" > .tmp/cui/v1.0/packages/cui/.env
cd .tmp/cui/v1.0 && pnpm install --no-frozen-lockfile && pnpm run build
CUI_COMMIT=$$(cd .tmp/cui/v1.0 && git rev-parse --short HEAD)
# Checkout init
git clone https://github.com/YaoApp/yao-init.git .tmp/yao-init
@ -303,57 +296,57 @@ release: clean
# rm -rf .tmp/yao-builder-latest.tar.gz
# Packing
mkdir -p .tmp/data/xgen
cp -f data/bindata.go data/bindata.go.bak
mkdir -p .tmp/data/cui
cp -r ./ui .tmp/data/ui
cp -r ./yao .tmp/data/yao
cp -r ./sui/libsui .tmp/data/libsui
cp -r .tmp/xgen/v0.9/dist .tmp/data/xgen/v0.9
cp -r .tmp/xgen/v1.0/packages/xgen/dist .tmp/data/xgen/v1.0
cp -r .tmp/cui/v0.9/dist .tmp/data/cui/v0.9
cp -r .tmp/cui/v1.0/packages/cui/dist .tmp/data/cui/v1.0
cp -r .tmp/yao-init .tmp/data/init
go-bindata -fs -pkg data -o data/bindata.go -prefix ".tmp/data/" .tmp/data/...
rm -rf .tmp/data
rm -rf .tmp/xgen
# Replace PRVERSION
cp -f share/const.go share/const.go.bak
sed -ie "s/const PRVERSION = \"DEV\"/const PRVERSION = \"${COMMIT}-${NOW}\"/g" share/const.go
@CUI_COMMIT=$$(cd .tmp/cui/v1.0 && git log | head -n 1 | awk '{print substr($$2, 0, 12)}') && \
sed -ie "s/const PRCUI = \"DEV\"/const PRCUI = \"$$CUI_COMMIT-${NOW}\"/g" share/const.go
# Making artifacts
mkdir -p dist
CGO_ENABLED=1 go build -v -o dist/release/yao
chmod +x dist/release/yao
# Clean up and restore bindata.go and const.go
cp data/bindata.go.bak data/bindata.go
cp share/const.go.bak share/const.go
rm data/bindata.go.bak
rm share/const.go.bak
rm -rf .tmp
# MacOS Application Signing
@if [ "$(OS)" = "Darwin" ]; then \
codesign --deep --force --verify --verbose --sign "${APPLE_SIGN}" dist/release/yao ; \
fi
# Reset const
cp -f share/const.goe share/const.go
rm share/const.goe
.PHONY: linux-release
linux-release: clean
mkdir -p dist/release
mkdir .tmp
# Building XGEN v0.9
git clone https://github.com/YaoApp/xgen-deprecated.git .tmp/xgen/v0.9
sed -ie "s/url('\/icon/url('\/xiang\/icon/g" .tmp/xgen/v0.9/public/icon/md_icon.css
cd .tmp/xgen/v0.9 && yarn install && yarn build
mkdir -p .tmp/xgen/v0.9
cp -r xgen/v0.9 .tmp/xgen/v0.9/dist
# Building XGEN v1.0
# ** XGEN will be renamed to DUI in the feature. and move to the new repository. **
# ** new repository: https://github.com/YaoApp/dui.git **
# Building CUI v1.0
# ** CUI will be renamed to CUI in the feature. and move to the new repository. **
# ** new repository: https://github.com/YaoApp/cui.git **
export NODE_ENV=production
git clone https://github.com/sjzsdu/xgen.git .tmp/xgen/v1.0
rm -f .tmp/xgen/v1.0/pnpm-lock.yaml
echo "BASE=__yao_admin_root" > .tmp/xgen/v1.0/packages/xgen/.env
cd .tmp/xgen/v1.0 && pnpm install --no-frozen-lockfile && pnpm run build
git clone https://github.com/YaoApp/cui.git .tmp/cui/v1.0
rm -f .tmp/cui/v1.0/pnpm-lock.yaml
echo "BASE=__yao_admin_root" > .tmp/cui/v1.0/packages/cui/.env
cd .tmp/cui/v1.0 && pnpm install --no-frozen-lockfile && pnpm run build
# Setup UI
cd .tmp/xgen/v1.0/packages/setup && pnpm install --no-frozen-lockfile && pnpm run build
cd .tmp/cui/v1.0/packages/setup && pnpm install --no-frozen-lockfile && pnpm run build
# Checkout init
@ -371,16 +364,16 @@ linux-release: clean
# rm -rf .tmp/yao-builder-latest.tar.gz
# Packing
mkdir -p .tmp/data/xgen
mkdir -p .tmp/data/cui
cp -r ./ui .tmp/data/ui
cp -r ./yao .tmp/data/yao
cp -r .tmp/xgen/v0.9/dist .tmp/data/xgen/v0.9
cp -r .tmp/xgen/v1.0/packages/setup/build .tmp/data/xgen/setup
cp -r .tmp/xgen/v1.0/packages/xgen/dist .tmp/data/xgen/v1.0
cp -r .tmp/cui/v0.9/dist .tmp/data/cui/v0.9
cp -r .tmp/cui/v1.0/packages/setup/build .tmp/data/cui/setup
cp -r .tmp/cui/v1.0/packages/cui/dist .tmp/data/cui/v1.0
cp -r .tmp/yao-init .tmp/data/init
go-bindata -fs -pkg data -o data/bindata.go -prefix ".tmp/data/" .tmp/data/...
rm -rf .tmp/data
rm -rf .tmp/xgen
rm -rf .tmp/cui
# Making artifacts
mkdir -p dist

12
aigc/load.go
View file

@ -11,9 +11,19 @@ import (
// Load load AIGC
func Load(cfg config.Config) error {
// Ignore if the aigcs directory does not exist
exists, err := application.App.Exists("aigcs")
if err != nil {
return err
}
if !exists {
return nil
}
exts := []string{"*.ai.yml", "*.ai.yaml"}
messages := []string{}
err := application.App.Walk("aigcs", func(root, file string, isdir bool) error {
err = application.App.Walk("aigcs", func(root, file string, isdir bool) error {
if isdir {
return nil
}

11
api/api.go
View file

@ -14,8 +14,17 @@ import (
func Load(cfg config.Config) error {
messages := []string{}
// Ignore if the apis directory does not exist
exists, err := application.App.Exists("apis")
if err != nil {
return err
}
if !exists {
return nil
}
exts := []string{"*.http.yao", "*.http.json", "*.http.jsonc"}
err := application.App.Walk("apis", func(root, file string, isdir bool) error {
err = application.App.Walk("apis", func(root, file string, isdir bool) error {
if isdir {
return nil
}

664
attachment/README.md Normal file
View file

@ -0,0 +1,664 @@
# Attachment Package
A comprehensive file upload package for Go that supports chunked uploads, file format validation, compression, and multiple storage backends.
## Features
- **Multiple Storage Backends**: Local filesystem and S3-compatible storage
- **Chunked Upload Support**: Handle large files with standard HTTP Content-Range headers
- **File Deduplication**: Content-based fingerprinting to avoid duplicate uploads
- **File Compression**:
- Gzip compression for any file type
- Image compression with configurable size limits
- **File Validation**:
- File size limits
- MIME type and extension validation
- Wildcard pattern support (e.g., `image/*`, `text/*`)
- **Flexible File Organization**: Hierarchical storage with multi-level group organization
- **Multiple Read Methods**: Stream, bytes, and base64 encoding
- **Global Manager Registry**: Support for registering and accessing managers globally
- **Upload Status Tracking**: Track upload progress with status field
- **Content Synchronization**: Support for synchronized uploads with Content-Sync header
## Installation
```bash
go get github.com/yaoapp/yao/neo/attachment
```
## Quick Start
### Basic Usage
```go
package main
import (
"context"
"strings"
"mime/multipart"
"github.com/yaoapp/yao/neo/attachment"
)
func main() {
// Create a manager with default settings
manager, err := attachment.RegisterDefault("uploads")
if err != nil {
panic(err)
}
// Or create a custom manager
customManager, err := attachment.New(attachment.ManagerOption{
Driver: "local",
MaxSize: "20M",
ChunkSize: "2M",
AllowedTypes: []string{"text/*", "image/*", ".pdf"},
Options: map[string]interface{}{
"path": "/var/uploads",
},
})
if err != nil {
panic(err)
}
// Upload a file
content := "Hello, World!"
fileHeader := &attachment.FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "hello.txt",
Size: int64(len(content)),
Header: make(map[string][]string),
},
}
fileHeader.Header.Set("Content-Type", "text/plain")
option := attachment.UploadOption{
Groups: []string{"user123", "chat456"}, // Multi-level groups (e.g., user, chat, knowledge, etc.)
OriginalFilename: "my_document.txt", // Preserve original filename
}
file, err := manager.Upload(context.Background(), fileHeader, strings.NewReader(content), option)
if err != nil {
panic(err)
}
// Check upload status
if file.Status == "uploaded" {
fmt.Printf("File uploaded successfully: %s\n", file.ID)
}
// Read the file back
data, err := manager.Read(context.Background(), file.ID)
if err != nil {
panic(err)
}
println(string(data)) // Output: Hello, World!
}
```
### Storage Backends
#### Local Storage
```go
manager, err := attachment.New(attachment.ManagerOption{
Driver: "local",
MaxSize: "20M",
Options: map[string]interface{}{
"path": "/var/uploads",
"base_url": "https://example.com/files",
},
})
```
#### S3 Storage
```go
manager, err := attachment.New(attachment.ManagerOption{
Driver: "s3",
MaxSize: "100M",
Options: map[string]interface{}{
"endpoint": "https://s3.amazonaws.com",
"region": "us-east-1",
"key": "your-access-key",
"secret": "your-secret-key",
"bucket": "your-bucket-name",
"prefix": "attachments/",
},
})
```
### Chunked Upload
For large files, you can upload in chunks using standard HTTP Content-Range headers:
```go
// Upload chunks
totalSize := int64(1024000) // 1MB file
chunkSize := int64(1024) // 1KB chunks
uid := "unique-file-id-123"
for start := int64(0); start < totalSize; start += chunkSize {
end := start + chunkSize - 1
if end >= totalSize {
end = totalSize - 1
}
chunkData := make([]byte, end-start+1)
// ... fill chunkData with actual data ...
chunkHeader := &attachment.FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "large_file.zip",
Size: end - start + 1,
Header: make(map[string][]string),
},
}
chunkHeader.Header.Set("Content-Type", "application/zip")
chunkHeader.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, totalSize))
chunkHeader.Header.Set("Content-Uid", uid)
file, err := manager.Upload(ctx, chunkHeader, bytes.NewReader(chunkData), option)
if err != nil {
return err
}
// File is complete when the last chunk is uploaded
if chunkHeader.Complete() {
fmt.Printf("Upload complete: %s\n", file.ID)
break
}
}
```
### Compression
#### Gzip Compression
```go
option := attachment.UploadOption{
Gzip: true, // Enable gzip compression
}
file, err := manager.Upload(ctx, fileHeader, reader, option)
```
#### Image Compression
```go
option := attachment.UploadOption{
CompressImage: true,
CompressSize: 1920, // Max dimension in pixels (default: 1920)
}
file, err := manager.Upload(ctx, imageHeader, imageReader, option)
```
### Multi-level Groups
The `Groups` field supports hierarchical file organization:
```go
// Single level grouping
option := attachment.UploadOption{
Groups: []string{"users"},
}
// Multi-level grouping
option := attachment.UploadOption{
Groups: []string{"users", "user123", "chats", "chat456"},
}
// Knowledge base organization
option := attachment.UploadOption{
Groups: []string{"knowledge", "documents", "technical"},
}
```
This creates nested directory structures for better organization and access control.
### File Validation
#### Size Limits
```go
manager, err := attachment.New(attachment.ManagerOption{
MaxSize: "20M", // Maximum file size
// Supports: B, K, M, G (e.g., "1024B", "2K", "10M", "1G")
})
```
#### Type Validation
```go
manager, err := attachment.New(attachment.ManagerOption{
AllowedTypes: []string{
"text/*", // All text types
"image/*", // All image types
"application/pdf", // Specific MIME type
".txt", // File extension
".jpg", // File extension
},
})
```
### Reading Files
#### Stream Reading
```go
response, err := manager.Download(ctx, fileID)
if err != nil {
return err
}
defer response.Reader.Close()
// Use response.Reader as io.ReadCloser
// response.ContentType contains the MIME type
// response.Extension contains the file extension
```
#### Read as Bytes
```go
data, err := manager.Read(ctx, fileID)
if err != nil {
return err
}
// data is []byte
```
#### Read as Base64
```go
base64Data, err := manager.ReadBase64(ctx, fileID)
if err != nil {
return err
}
// base64Data is string
```
### Global Managers
You can register managers globally for easy access:
```go
// Register default manager with sensible defaults
attachment.RegisterDefault("main")
// Register custom managers
attachment.Register("local", "local", attachment.ManagerOption{
Driver: "local",
Options: map[string]interface{}{
"path": "/var/uploads",
},
})
attachment.Register("s3", "s3", attachment.ManagerOption{
Driver: "s3",
Options: map[string]interface{}{
"bucket": "my-bucket",
"key": "access-key",
"secret": "secret-key",
},
})
// Use global managers
localManager := attachment.Managers["local"]
s3Manager := attachment.Managers["s3"]
defaultManager := attachment.Managers["main"]
```
## File Organization
Files are organized in a hierarchical structure:
```
attachments/
├── 20240101/ # Date (YYYYMMDD)
│ └── user123/ # First level group (optional)
│ └── chat456/ # Second level group (optional)
│ └── knowledge/ # Additional group levels (optional)
│ └── ab/ # First 2 chars of hash
│ └── cd/ # Next 2 chars of hash
│ └── abcdef12.txt # Hash + extension
```
The file ID generation includes:
- Date prefix for organization
- Multi-level groups for access control and organization
- Content hash for deduplication
- Original file extension
## API Reference
### Manager
#### `New(option ManagerOption) (*Manager, error)`
Creates a new attachment manager.
#### `Register(name string, driver string, option ManagerOption) (*Manager, error)`
Registers a global attachment manager.
#### `Upload(ctx context.Context, fileheader *FileHeader, reader io.Reader, option UploadOption) (*File, error)`
Uploads a file (supports chunked upload).
#### `Download(ctx context.Context, fileID string) (*FileResponse, error)`
Downloads a file as a stream.
#### `Read(ctx context.Context, fileID string) ([]byte, error)`
Reads a file as bytes.
#### `ReadBase64(ctx context.Context, fileID string) (string, error)`
Reads a file as base64 encoded string.
### Storage Interface
All storage backends implement the following interface:
```go
type Storage interface {
Upload(ctx context.Context, fileID string, reader io.Reader, contentType string) (string, error)
UploadChunk(ctx context.Context, fileID string, chunkIndex int, reader io.Reader, contentType string) error
MergeChunks(ctx context.Context, fileID string, totalChunks int) error
Download(ctx context.Context, fileID string) (io.ReadCloser, string, error)
Reader(ctx context.Context, fileID string) (io.ReadCloser, error)
URL(ctx context.Context, fileID string) string
Exists(ctx context.Context, fileID string) bool
Delete(ctx context.Context, fileID string) error
}
```
### Types
#### `ManagerOption`
Configuration for creating a manager:
- `Driver`: "local" or "s3"
- `MaxSize`: Maximum file size (e.g., "20M")
- `ChunkSize`: Chunk size for uploads (e.g., "2M")
- `AllowedTypes`: Array of allowed MIME types/extensions
- `Options`: Driver-specific options
#### `UploadOption`
Options for file upload:
- `CompressImage`: Enable image compression
- `CompressSize`: Maximum image dimension (default: 1920)
- `Gzip`: Enable gzip compression
- `Groups`: Multi-level group identifiers for hierarchical file organization (e.g., []string{"user123", "chat456", "knowledge"})
- `OriginalFilename`: Original filename to preserve (avoids encoding issues)
#### `File`
Uploaded file information:
- `ID`: Unique file identifier
- `Filename`: Original filename
- `ContentType`: MIME type
- `Bytes`: File size
- `CreatedAt`: Upload timestamp
- `Status`: Upload status ("uploading", "uploaded", "indexing", "indexed", "upload_failed", "index_failed")
#### `FileResponse`
Download response:
- `Reader`: io.ReadCloser for file content
- `ContentType`: MIME type
- `Extension`: File extension
## Chunked Upload Details
The package supports chunked uploads using standard HTTP headers:
- `Content-Range`: Specifies byte range (e.g., "bytes 0-1023/2048")
- `Content-Uid`: Unique identifier for the file being uploaded
### Chunk Index Calculation
The package uses a standard chunk size (1024 bytes by default) to calculate chunk indices consistently. This ensures proper chunk ordering during merge operations.
### Content Type Preservation
For chunked uploads, the content type is preserved from the first chunk and applied to the final merged file, ensuring proper MIME type handling across all storage backends.
## Error Handling
The package returns descriptive errors for common issues:
- File size exceeds limit
- Unsupported file type
- Storage backend errors
- Invalid chunk information
- Missing required configuration
## Testing
Run the tests:
```bash
# Run all tests
go test ./...
# Run with S3 credentials (optional)
export S3_ACCESS_KEY="your-key"
export S3_SECRET_KEY="your-secret"
export S3_BUCKET="your-bucket"
export S3_API="https://your-s3-endpoint"
go test ./...
```
The package includes comprehensive tests for:
- Basic file upload/download
- Chunked uploads with content type preservation
- Compression (gzip and image)
- File validation (size, type, wildcards)
- Multiple storage backends (local and S3)
- Error handling and edge cases
### Test Coverage
- **Manager Tests**: Upload, download, validation, compression
- **Local Storage Tests**: File operations, chunked uploads, directory management
- **S3 Storage Tests**: S3 operations, chunked uploads, presigned URLs (requires credentials)
## Performance Considerations
- **Chunked Uploads**: Use appropriate chunk sizes (1-5MB) for optimal performance
- **Image Compression**: Automatically resizes large images to reduce storage costs
- **Gzip Compression**: Reduces storage size for text-based files
- **Content Type Detection**: Efficient MIME type detection and preservation
## Security Features
- **File Type Validation**: Prevents upload of unauthorized file types
- **Size Limits**: Configurable file size restrictions
- **Path Sanitization**: Secure file path generation
- **Access Control**: Multi-level hierarchical file organization
## License
This package is part of the Yao project and follows the same license terms.
### File Deduplication with Fingerprints
The package supports file deduplication using content fingerprints:
```go
// Set a content fingerprint to enable deduplication
fileHeader := &attachment.FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "document.pdf",
Size: fileSize,
Header: make(map[string][]string),
},
}
fileHeader.Header.Set("Content-Type", "application/pdf")
fileHeader.Header.Set("Content-Fingerprint", "sha256:abcdef123456") // Content-based hash
file, err := manager.Upload(ctx, fileHeader, reader, option)
```
### Content Synchronization
For synchronized uploads across multiple clients:
```go
// Enable content synchronization
fileHeader.Header.Set("Content-Sync", "true")
// Each client can upload the same content with the same fingerprint
// The system will deduplicate based on the content fingerprint
```
### Chunked Upload with Enhanced Headers
For large files, you can upload in chunks using standard HTTP Content-Range headers with additional metadata:
```go
// Upload chunks with unique identifier and fingerprint
totalSize := int64(1024000) // 1MB file
chunkSize := int64(1024) // 1KB chunks
uid := "unique-file-id-123"
fingerprint := "sha256:content-hash-here"
for start := int64(0); start < totalSize; start += chunkSize {
end := start + chunkSize - 1
if end >= totalSize {
end = totalSize - 1
}
chunkData := make([]byte, end-start+1)
// ... fill chunkData with actual data ...
chunkHeader := &attachment.FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "large_file.zip",
Size: end - start + 1,
Header: make(map[string][]string),
},
}
chunkHeader.Header.Set("Content-Type", "application/zip")
chunkHeader.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, totalSize))
chunkHeader.Header.Set("Content-Uid", uid)
chunkHeader.Header.Set("Content-Fingerprint", fingerprint)
chunkHeader.Header.Set("Content-Sync", "true") // Enable synchronization
option := attachment.UploadOption{
Groups: []string{"user123", "chat456"}, // Multi-level groups
OriginalFilename: "my_large_file.zip", // Preserve original name
}
file, err := manager.Upload(ctx, chunkHeader, bytes.NewReader(chunkData), option)
if err != nil {
return err
}
// Check if upload is complete
if file.Status == "uploaded" {
fmt.Printf("Upload complete: %s\n", file.ID)
break
} else if file.Status == "uploading" {
fmt.Printf("Chunk uploaded, progress: %d/%d\n", chunkHeader.GetChunkSize(), chunkHeader.GetTotalSize())
}
}
```
### FileHeader Methods
The `FileHeader` type provides several utility methods:
```go
// Get unique identifier for chunked uploads
uid := fileHeader.UID()
// Get content fingerprint for deduplication
fingerprint := fileHeader.Fingerprint()
// Get byte range for chunked uploads
rangeHeader := fileHeader.Range()
// Check if synchronization is enabled
isSync := fileHeader.Sync()
// Check if this is a chunked upload
isChunk := fileHeader.IsChunk()
// Check if upload is complete (for chunked uploads)
isComplete := fileHeader.Complete()
// Get detailed chunk information
start, end, total, err := fileHeader.GetChunkInfo()
// Get total file size (for chunked uploads)
totalSize := fileHeader.GetTotalSize()
// Get current chunk size
chunkSize := fileHeader.GetChunkSize()
```
## File Headers and Metadata
The package supports several HTTP headers for enhanced functionality:
- `Content-Range`: Standard HTTP range header for chunked uploads (e.g., "bytes 0-1023/2048")
- `Content-Uid`: Unique identifier for file uploads (for deduplication and tracking)
- `Content-Fingerprint`: Content-based hash for deduplication (e.g., "sha256:abc123")
- `Content-Sync`: Enable synchronized uploads across multiple clients ("true"/"false")
### Header Processing
When processing uploads, headers can be extracted from both HTTP request headers and multipart file headers:
```go
// Extract headers from HTTP request and file headers
header := attachment.GetHeader(requestHeader, fileHeader, fileSize)
// The resulting FileHeader will contain merged headers from both sources
uid := header.UID()
fingerprint := header.Fingerprint()
isSync := header.Sync()
```
## Upload Status Tracking
Files have a status field that tracks the upload lifecycle:
- `"uploading"`: File upload is in progress (for chunked uploads)
- `"uploaded"`: File has been successfully uploaded
- `"indexing"`: File is being processed for search indexing
- `"indexed"`: File has been indexed and is fully processed
- `"upload_failed"`: Upload failed due to an error
- `"index_failed"`: Indexing failed but file is still accessible
```go
file, err := manager.Upload(ctx, fileHeader, reader, option)
if err != nil {
return err
}
switch file.Status {
case "uploading":
fmt.Println("Upload in progress...")
case "uploaded":
fmt.Println("Upload completed successfully")
case "upload_failed":
fmt.Println("Upload failed")
}
```
#### `RegisterDefault(name string) (*Manager, error)`
Registers a default attachment manager with sensible defaults for common file types.

76
attachment/compresses.go Normal file
View file

@ -0,0 +1,76 @@
package attachment
import (
"bytes"
"fmt"
"image"
"image/jpeg"
"image/png"
"io"
)
// CompressImage compresses the image while maintaining aspect ratio
func CompressImage(reader io.Reader, contentType string, maxSize int) ([]byte, error) {
// Read all data first
data, err := io.ReadAll(reader)
if err != nil {
return nil, fmt.Errorf("failed to read image data: %w", err)
}
// Decode image
img, _, err := image.Decode(bytes.NewReader(data))
if err != nil {
return nil, fmt.Errorf("failed to decode image: %w", err)
}
// Calculate new dimensions
bounds := img.Bounds()
width := bounds.Dx()
height := bounds.Dy()
var newWidth, newHeight int
if width > height {
if width > maxSize {
newWidth = maxSize
newHeight = int(float64(height) * (float64(maxSize) / float64(width)))
} else {
return data, nil // No need to resize, return original data
}
} else {
if height > maxSize {
newHeight = maxSize
newWidth = int(float64(width) * (float64(maxSize) / float64(height)))
} else {
return data, nil // No need to resize, return original data
}
}
// Create new image with new dimensions
newImg := image.NewRGBA(image.Rect(0, 0, newWidth, newHeight))
// Scale the image using bilinear interpolation
for y := 0; y < newHeight; y++ {
for x := 0; x < newWidth; x++ {
srcX := float64(x) * float64(width) / float64(newWidth)
srcY := float64(y) * float64(height) / float64(newHeight)
newImg.Set(x, y, img.At(int(srcX), int(srcY)))
}
}
// Encode image
var buf bytes.Buffer
switch contentType {
case "image/jpeg":
err = jpeg.Encode(&buf, newImg, &jpeg.Options{Quality: 85})
case "image/png":
err = png.Encode(&buf, newImg)
default:
return data, nil // Unsupported format, return original data
}
if err != nil {
return nil, fmt.Errorf("failed to encode image: %w", err)
}
return buf.Bytes(), nil
}

512
attachment/example_usage.go Normal file
View file

@ -0,0 +1,512 @@
package attachment
import (
"bytes"
"context"
"fmt"
"log"
"mime/multipart"
"strings"
"github.com/yaoapp/yao/attachment/s3"
)
// ExampleUsage demonstrates how to use the attachment package
func ExampleUsage() {
// 1. Create a local storage manager
localManager, err := New(ManagerOption{
Driver: "local",
MaxSize: "20M",
ChunkSize: "2M",
AllowedTypes: []string{"text/*", "image/*", "application/pdf", ".txt", ".jpg", ".png", ".pdf"},
Options: map[string]interface{}{
"path": "/var/uploads",
"base_url": "https://example.com/files",
},
})
if err != nil {
log.Fatalf("Failed to create local manager: %v", err)
}
// 2. Create an S3 storage manager
s3Manager, err := New(ManagerOption{
Driver: "s3",
MaxSize: "100M",
ChunkSize: "5M",
AllowedTypes: []string{"*"}, // Allow all types
Options: map[string]interface{}{
"endpoint": "https://s3.amazonaws.com",
"region": "us-east-1",
"key": "your-access-key",
"secret": "your-secret-key",
"bucket": "your-bucket-name",
"prefix": "attachments/",
},
})
if err != nil {
log.Printf("Failed to create S3 manager (this is expected without credentials): %v", err)
} else {
fmt.Printf("Created S3 manager successfully\n")
// Demonstrate S3 manager usage if credentials are available
if s3Manager != nil {
fmt.Printf("S3 manager is ready for use with bucket: %s\n",
s3Manager.storage.(*s3.Storage).Bucket)
}
}
// 3. Register managers globally
_, err = Register("local", "local", ManagerOption{
Driver: "local",
MaxSize: "20M",
AllowedTypes: []string{"text/*", "image/*"},
Options: map[string]interface{}{
"path": "/var/uploads",
},
})
if err != nil {
log.Printf("Failed to register local manager: %v", err)
}
// Try to register S3 manager (will fail without credentials)
_, err = Register("s3", "s3", ManagerOption{
Driver: "s3",
MaxSize: "100M",
Options: map[string]interface{}{
"bucket": "your-bucket",
"key": "your-key",
"secret": "your-secret",
},
})
if err != nil {
log.Printf("Failed to register S3 manager (expected without credentials): %v", err)
}
ctx := context.Background()
// 4. Example: Simple file upload
content := "Hello, World! This is a test file with some content to demonstrate the attachment package."
fileHeader := &FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "hello.txt",
Size: int64(len(content)),
Header: make(map[string][]string),
},
}
fileHeader.Header.Set("Content-Type", "text/plain")
uploadOption := UploadOption{
Groups: []string{"user123"},
Gzip: false, // No compression for small text files
}
file, err := localManager.Upload(ctx, fileHeader, strings.NewReader(content), uploadOption)
if err != nil {
log.Fatalf("Failed to upload file: %v", err)
}
fmt.Printf("Uploaded file: %s (ID: %s, Size: %d bytes)\n", file.Filename, file.ID, file.Bytes)
// 5. Example: File upload with gzip compression
largeContent := strings.Repeat("This is a large text file that benefits from compression. ", 100)
gzipFileHeader := &FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "large_text.txt",
Size: int64(len(largeContent)),
Header: make(map[string][]string),
},
}
gzipFileHeader.Header.Set("Content-Type", "text/plain")
gzipOption := UploadOption{
Groups: []string{"user123"},
Gzip: true, // Enable compression
}
gzipFile, err := localManager.Upload(ctx, gzipFileHeader, strings.NewReader(largeContent), gzipOption)
if err != nil {
log.Fatalf("Failed to upload gzipped file: %v", err)
}
fmt.Printf("Uploaded compressed file: %s (ID: %s)\n", gzipFile.Filename, gzipFile.ID)
// 6. Example: Image upload with compression
imageUploadOption := UploadOption{
Groups: []string{"user123"},
CompressImage: true,
CompressSize: 1920, // Resize to max 1920px
Gzip: false,
}
// Simulate image upload (you would get this from multipart form)
imageHeader := &FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "photo.jpg",
Size: 1024000, // 1MB
Header: make(map[string][]string),
},
}
imageHeader.Header.Set("Content-Type", "image/jpeg")
fmt.Printf("Image upload option configured: compress=%v, size=%d\n",
imageUploadOption.CompressImage, imageUploadOption.CompressSize)
// 6.5. Example: Multi-level groups
fmt.Println("\n--- Multi-level Groups Examples ---")
// Single level grouping
singleGroupOption := UploadOption{
Groups: []string{"knowledge"},
}
singleGroupHeader := &FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "knowledge_doc.txt",
Size: int64(len("Knowledge base document")),
Header: make(map[string][]string),
},
}
singleGroupHeader.Header.Set("Content-Type", "text/plain")
singleFile, err := localManager.Upload(ctx, singleGroupHeader, strings.NewReader("Knowledge base document"), singleGroupOption)
if err != nil {
log.Printf("Failed to upload single group file: %v", err)
} else {
fmt.Printf("Single group file uploaded: %s (ID: %s)\n", singleFile.Filename, singleFile.ID)
}
// Multi-level grouping
multiGroupOption := UploadOption{
Groups: []string{"users", "user123", "chats", "chat456", "documents"},
}
multiGroupHeader := &FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "chat_document.txt",
Size: int64(len("Document in user chat")),
Header: make(map[string][]string),
},
}
multiGroupHeader.Header.Set("Content-Type", "text/plain")
multiFile, err := localManager.Upload(ctx, multiGroupHeader, strings.NewReader("Document in user chat"), multiGroupOption)
if err != nil {
log.Printf("Failed to upload multi-group file: %v", err)
} else {
fmt.Printf("Multi-level group file uploaded: %s (ID: %s)\n", multiFile.Filename, multiFile.ID)
fmt.Printf("File path includes hierarchy: users/user123/chats/chat456/documents\n")
}
// Knowledge base organization
knowledgeOption := UploadOption{
Groups: []string{"knowledge", "technical", "api", "documentation"},
}
knowledgeHeader := &FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "api_guide.md",
Size: int64(len("# API Documentation\n\nThis is technical documentation.")),
Header: make(map[string][]string),
},
}
knowledgeHeader.Header.Set("Content-Type", "text/markdown")
knowledgeFile, err := localManager.Upload(ctx, knowledgeHeader,
strings.NewReader("# API Documentation\n\nThis is technical documentation."), knowledgeOption)
if err != nil {
log.Printf("Failed to upload knowledge file: %v", err)
} else {
fmt.Printf("Knowledge base file uploaded: %s (ID: %s)\n", knowledgeFile.Filename, knowledgeFile.ID)
fmt.Printf("Organized in: knowledge/technical/api/documentation\n")
}
// 7. Example: Chunked upload
largeContent = strings.Repeat("This is a large file content that will be uploaded in chunks. ", 1000)
chunkSize := 1024
totalSize := len(largeContent)
uid := "unique-large-file-123"
fmt.Printf("Starting chunked upload: total size=%d, chunk size=%d\n", totalSize, chunkSize)
var lastFile *File
chunkCount := 0
// Split into chunks and upload
for i := 0; i < totalSize; i += chunkSize {
end := i + chunkSize
if end > totalSize {
end = totalSize
}
chunk := largeContent[i:end]
chunkHeader := &FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "large_file.txt",
Size: int64(len(chunk)),
Header: make(map[string][]string),
},
}
chunkHeader.Header.Set("Content-Type", "text/plain")
chunkHeader.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", i, end-1, totalSize))
chunkHeader.Header.Set("Content-Uid", uid)
chunkOption := UploadOption{
Groups: []string{"user123"},
Gzip: true, // Compress chunks
}
chunkFile, err := localManager.Upload(ctx, chunkHeader, strings.NewReader(chunk), chunkOption)
if err != nil {
log.Fatalf("Failed to upload chunk %d: %v", chunkCount, err)
}
chunkCount++
lastFile = chunkFile
// Check if this is the last chunk
if chunkHeader.Complete() {
fmt.Printf("Uploaded large file in %d chunks: %s (ID: %s)\n", chunkCount, chunkFile.Filename, chunkFile.ID)
break
}
}
// 8. Example: Download and read files
if file != nil {
// Download as stream
response, err := localManager.Download(ctx, file.ID)
if err != nil {
log.Fatalf("Failed to download file: %v", err)
}
defer response.Reader.Close()
fmt.Printf("Downloaded file content type: %s, extension: %s\n", response.ContentType, response.Extension)
// Read as bytes
data, err := localManager.Read(ctx, file.ID)
if err != nil {
log.Fatalf("Failed to read file: %v", err)
}
fmt.Printf("File content length: %d bytes\n", len(data))
if len(data) < 100 {
fmt.Printf("File content: %s\n", string(data))
} else {
fmt.Printf("File content preview: %s...\n", string(data[:100]))
}
// Read as base64
base64Data, err := localManager.ReadBase64(ctx, file.ID)
if err != nil {
log.Fatalf("Failed to read file as base64: %v", err)
}
fmt.Printf("File as base64 (first 50 chars): %s...\n", base64Data[:min(50, len(base64Data))])
}
// 9. Example: Read chunked file
if lastFile != nil {
chunkData, err := localManager.Read(ctx, lastFile.ID)
if err != nil {
log.Fatalf("Failed to read chunked file: %v", err)
}
// Since the chunks were compressed, we need to decompress
decompressed, err := Gunzip(chunkData)
if err != nil {
log.Fatalf("Failed to decompress chunked file: %v", err)
}
fmt.Printf("Chunked file content length: %d bytes (decompressed)\n", len(decompressed))
if len(decompressed) < 200 {
fmt.Printf("Chunked file content: %s\n", string(decompressed))
} else {
fmt.Printf("Chunked file content preview: %s...\n", string(decompressed[:200]))
}
}
// 10. Example: Using global managers
globalManager := Managers["local"]
if globalManager != nil {
fmt.Println("Using global manager for local storage")
// Test a simple upload with global manager
testContent := "Test content using global manager"
testHeader := &FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "global_test.txt",
Size: int64(len(testContent)),
Header: make(map[string][]string),
},
}
testHeader.Header.Set("Content-Type", "text/plain")
testFile, err := globalManager.Upload(ctx, testHeader, strings.NewReader(testContent), UploadOption{
Groups: []string{"global_user"},
})
if err != nil {
log.Printf("Failed to upload with global manager: %v", err)
} else {
fmt.Printf("Global manager upload successful: %s\n", testFile.ID)
}
}
// 11. Example: File validation
fmt.Println("\n--- File Validation Examples ---")
// Test file size validation
tooLargeContent := strings.Repeat("x", 25*1024*1024) // 25MB, exceeds 20MB limit
largeFileHeader := &FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "too_large.txt",
Size: int64(len(tooLargeContent)),
Header: make(map[string][]string),
},
}
largeFileHeader.Header.Set("Content-Type", "text/plain")
_, err = localManager.Upload(ctx, largeFileHeader, strings.NewReader(tooLargeContent), UploadOption{})
if err != nil {
fmt.Printf("Expected error for large file: %v\n", err)
}
// Test file type validation
invalidFileHeader := &FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "script.exe",
Size: 1024,
Header: make(map[string][]string),
},
}
invalidFileHeader.Header.Set("Content-Type", "application/x-executable")
_, err = localManager.Upload(ctx, invalidFileHeader, strings.NewReader("fake exe content"), UploadOption{})
if err != nil {
fmt.Printf("Expected error for invalid file type: %v\n", err)
}
fmt.Println("\n--- Example Usage Complete ---")
}
// ExampleChunkedUpload demonstrates how to handle chunked uploads properly
func ExampleChunkedUpload(manager *Manager, filename string, totalSize int64, contentType string) error {
ctx := context.Background()
chunkSize := int64(1024 * 1024) // 1MB chunks
uid := "unique-file-" + filename
fmt.Printf("Starting chunked upload: file=%s, total=%d bytes, chunks=%d\n",
filename, totalSize, (totalSize+chunkSize-1)/chunkSize)
for offset := int64(0); offset < totalSize; offset += chunkSize {
end := offset + chunkSize - 1
if end >= totalSize {
end = totalSize - 1
}
chunkSize := end - offset + 1
// Create chunk header
chunkHeader := &FileHeader{
FileHeader: &multipart.FileHeader{
Filename: filename,
Size: chunkSize,
Header: make(map[string][]string),
},
}
chunkHeader.Header.Set("Content-Type", contentType)
chunkHeader.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", offset, end, totalSize))
chunkHeader.Header.Set("Content-Uid", uid)
// In real usage, you would read the actual chunk data from the source
chunkData := make([]byte, chunkSize)
// Fill with sample data for demonstration
for i := range chunkData {
chunkData[i] = byte('A' + (i % 26))
}
option := UploadOption{
Groups: []string{"user123"},
Gzip: false, // Disable compression for this example
}
file, err := manager.Upload(ctx, chunkHeader, bytes.NewReader(chunkData), option)
if err != nil {
return fmt.Errorf("failed to upload chunk at offset %d: %w", offset, err)
}
fmt.Printf("Uploaded chunk %d-%d/%d\n", offset, end, totalSize)
// Check if this was the last chunk
if chunkHeader.Complete() {
fmt.Printf("File upload completed: %s (ID: %s)\n", file.Filename, file.ID)
// Verify the uploaded file
data, err := manager.Read(ctx, file.ID)
if err != nil {
return fmt.Errorf("failed to read uploaded file: %w", err)
}
if int64(len(data)) != totalSize {
return fmt.Errorf("uploaded file size mismatch: expected %d, got %d", totalSize, len(data))
}
fmt.Printf("File verification successful: %d bytes\n", len(data))
break
}
}
return nil
}
// ExampleS3Upload demonstrates S3-specific features
func ExampleS3Upload() {
// This example requires actual S3 credentials
s3Manager, err := New(ManagerOption{
Driver: "s3",
MaxSize: "50M",
Options: map[string]interface{}{
"endpoint": "https://s3.amazonaws.com",
"region": "us-east-1",
"key": "your-access-key",
"secret": "your-secret-key",
"bucket": "your-bucket",
"prefix": "test-uploads/",
},
})
if err != nil {
log.Printf("S3 manager creation failed (expected without credentials): %v", err)
return
}
ctx := context.Background()
content := "Test content for S3 upload"
fileHeader := &FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "s3_test.txt",
Size: int64(len(content)),
Header: make(map[string][]string),
},
}
fileHeader.Header.Set("Content-Type", "text/plain")
file, err := s3Manager.Upload(ctx, fileHeader, strings.NewReader(content), UploadOption{
Groups: []string{"s3_user"},
})
if err != nil {
log.Printf("S3 upload failed: %v", err)
return
}
fmt.Printf("S3 upload successful: %s\n", file.ID)
// Get presigned URL
url := s3Manager.storage.URL(ctx, file.ID)
fmt.Printf("Presigned URL: %s\n", url)
}
// Helper function for min
func min(a, b int) int {
if a < b {
return a
}
return b
}

139
attachment/fileheader.go Normal file
View file

@ -0,0 +1,139 @@
package attachment
import (
"strconv"
"strings"
)
// UID is the uid of the file, it is the unique identifier of the file
func (fileheader *FileHeader) UID() string {
return fileheader.Header.Get("Content-Uid")
}
// Fingerprint is the fingerprint of the file, it is the fingerprint of the file
func (fileheader *FileHeader) Fingerprint() string {
return fileheader.Header.Get("Content-Fingerprint")
}
// Range is the range of the file, it is the start and end of the file
func (fileheader *FileHeader) Range() string {
return fileheader.Header.Get("Content-Range")
}
// Sync is the sync of the file, it is the sync of the file
func (fileheader *FileHeader) Sync() bool {
return fileheader.Header.Get("Content-Sync") == "true"
}
// IsChunk is the chunk of the file, it is the chunk of the file
func (fileheader *FileHeader) IsChunk() bool {
return fileheader.Range() != ""
}
// Complete checks if the chunk upload is completed
// For non-chunk files, it returns true
// For chunk files, it parses the Content-Range header to determine if this is the last chunk
func (fileheader *FileHeader) Complete() bool {
if !fileheader.IsChunk() {
return true
}
// Parse Content-Range header: "bytes start-end/total"
rangeHeader := fileheader.Range()
if rangeHeader == "" {
return false
}
// Remove "bytes " prefix
rangeStr := strings.TrimPrefix(rangeHeader, "bytes ")
// Split by "/"
parts := strings.Split(rangeStr, "/")
if len(parts) != 2 {
return false
}
// Parse total size
total, err := strconv.ParseInt(parts[1], 10, 64)
if err != nil {
return false
}
// Parse range "start-end"
rangeParts := strings.Split(parts[0], "-")
if len(rangeParts) != 2 {
return false
}
end, err := strconv.ParseInt(rangeParts[1], 10, 64)
if err != nil {
return false
}
// Check if this is the last chunk: end + 1 == total
return end+1 == total
}
// GetChunkInfo returns the chunk information parsed from Content-Range header
// Returns start, end, total, and error
func (fileheader *FileHeader) GetChunkInfo() (start, end, total int64, err error) {
if !fileheader.IsChunk() {
return 0, 0, 0, nil
}
rangeHeader := fileheader.Range()
if rangeHeader == "" {
return 0, 0, 0, nil
}
// Remove "bytes " prefix
rangeStr := strings.TrimPrefix(rangeHeader, "bytes ")
// Split by "/"
parts := strings.Split(rangeStr, "/")
if len(parts) != 2 {
return 0, 0, 0, nil
}
// Parse total size
total, err = strconv.ParseInt(parts[1], 10, 64)
if err != nil {
return 0, 0, 0, err
}
// Parse range "start-end"
rangeParts := strings.Split(parts[0], "-")
if len(rangeParts) != 2 {
return 0, 0, 0, nil
}
start, err = strconv.ParseInt(rangeParts[0], 10, 64)
if err != nil {
return 0, 0, 0, err
}
end, err = strconv.ParseInt(rangeParts[1], 10, 64)
if err != nil {
return 0, 0, 0, err
}
return start, end, total, nil
}
// GetTotalSize returns the total file size from Content-Range header
func (fileheader *FileHeader) GetTotalSize() int64 {
_, _, total, err := fileheader.GetChunkInfo()
if err != nil {
return 0
}
return total
}
// GetChunkSize returns the current chunk size
func (fileheader *FileHeader) GetChunkSize() int64 {
start, end, _, err := fileheader.GetChunkInfo()
if err != nil {
return 0
}
return end - start + 1
}

285
attachment/gzip.go Normal file
View file

@ -0,0 +1,285 @@
package attachment
import (
"bytes"
"compress/gzip"
"fmt"
"io"
"os"
)
// GzipCompressor supports chunked compression for Gzip
type GzipCompressor struct {
writer *gzip.Writer
buffer *bytes.Buffer
file *os.File // optional file handle
}
// NewGzipCompressor creates a new Gzip compressor
func NewGzipCompressor() *GzipCompressor {
buf := &bytes.Buffer{}
gz := gzip.NewWriter(buf)
return &GzipCompressor{
writer: gz,
buffer: buf,
file: nil,
}
}
// NewGzipCompressorFromFile creates a Gzip compressor from file, supports streaming read
func NewGzipCompressorFromFile(filePath string) (*GzipCompressor, error) {
file, err := os.Open(filePath)
if err != nil {
return nil, fmt.Errorf("failed to open file %s: %w", filePath, err)
}
buf := &bytes.Buffer{}
gz := gzip.NewWriter(buf)
return &GzipCompressor{
writer: gz,
buffer: buf,
file: file,
}, nil
}
// ReadChunk reads a chunk of specified size from file and compresses it
func (gc *GzipCompressor) ReadChunk(chunkSize int) (bool, error) {
if gc.file == nil {
return false, fmt.Errorf("no file associated with this compressor")
}
chunk := make([]byte, chunkSize)
n, err := gc.file.Read(chunk)
if err != nil && err != io.EOF {
return false, fmt.Errorf("failed to read from file: %w", err)
}
if n > 0 {
if err := gc.Write(chunk[:n]); err != nil {
return false, err
}
}
// return whether there is more data
return err != io.EOF, nil
}
// CompressFileInChunks compresses the entire file in chunks
func (gc *GzipCompressor) CompressFileInChunks(chunkSize int) error {
if gc.file == nil {
return fmt.Errorf("no file associated with this compressor")
}
for {
hasMore, err := gc.ReadChunk(chunkSize)
if err != nil {
return err
}
if !hasMore {
break
}
}
return nil
}
// Write writes data for compression (supports chunked writing)
func (gc *GzipCompressor) Write(data []byte) error {
_, err := gc.writer.Write(data)
if err != nil {
return fmt.Errorf("failed to write data to gzip: %w", err)
}
return nil
}
// Flush flushes the buffer but does not close the compressor
func (gc *GzipCompressor) Flush() error {
return gc.writer.Flush()
}
// Close closes the compressor and returns the final compressed data
func (gc *GzipCompressor) Close() ([]byte, error) {
err := gc.writer.Close()
if err != nil {
return nil, fmt.Errorf("failed to close gzip writer: %w", err)
}
// if there is an associated file, close it too
if gc.file != nil {
gc.file.Close()
gc.file = nil
}
return gc.buffer.Bytes(), nil
}
// GetCompressedData gets the current compressed data (without closing the compressor)
func (gc *GzipCompressor) GetCompressedData() []byte {
// flush the buffer first
gc.writer.Flush()
return gc.buffer.Bytes()
}
// Reset resets the compressor for reuse
func (gc *GzipCompressor) Reset() {
gc.buffer.Reset()
gc.writer.Reset(gc.buffer)
if gc.file != nil {
gc.file.Close()
gc.file = nil
}
}
// Gzip compresses data in one go
func Gzip(data []byte) ([]byte, error) {
var buf bytes.Buffer
gz := gzip.NewWriter(&buf)
_, err := gz.Write(data)
if err != nil {
return nil, fmt.Errorf("failed to gzip data: %w", err)
}
err = gz.Close()
if err != nil {
return nil, fmt.Errorf("failed to close gzip writer: %w", err)
}
return buf.Bytes(), nil
}
// GzipChunks compresses multiple data chunks
func GzipChunks(chunks [][]byte) ([]byte, error) {
compressor := NewGzipCompressor()
for _, chunk := range chunks {
if err := compressor.Write(chunk); err != nil {
return nil, err
}
}
return compressor.Close()
}
// GzipFromReader compresses data from Reader stream
func GzipFromReader(reader io.Reader) ([]byte, error) {
var buf bytes.Buffer
gz := gzip.NewWriter(&buf)
_, err := io.Copy(gz, reader)
if err != nil {
return nil, fmt.Errorf("failed to copy data to gzip: %w", err)
}
err = gz.Close()
if err != nil {
return nil, fmt.Errorf("failed to close gzip writer: %w", err)
}
return buf.Bytes(), nil
}
// Gunzip decompresses gzip data
func Gunzip(data []byte) ([]byte, error) {
reader, err := gzip.NewReader(bytes.NewReader(data))
if err != nil {
return nil, fmt.Errorf("failed to create gzip reader: %w", err)
}
defer reader.Close()
var buf bytes.Buffer
_, err = io.Copy(&buf, reader)
if err != nil {
return nil, fmt.Errorf("failed to decompress data: %w", err)
}
return buf.Bytes(), nil
}
// GzipToWriter writes compressed data to Writer
func GzipToWriter(data []byte, writer io.Writer) error {
gz := gzip.NewWriter(writer)
defer gz.Close()
_, err := gz.Write(data)
if err != nil {
return fmt.Errorf("failed to write gzip data: %w", err)
}
return nil
}
// GzipFromReaderToWriter reads data from Reader and writes compressed data to Writer
func GzipFromReaderToWriter(reader io.Reader, writer io.Writer) error {
gz := gzip.NewWriter(writer)
defer gz.Close()
_, err := io.Copy(gz, reader)
if err != nil {
return fmt.Errorf("failed to copy and compress data: %w", err)
}
return nil
}
// GzipFile compresses entire file (loads into memory at once)
func GzipFile(filePath string) ([]byte, error) {
data, err := os.ReadFile(filePath)
if err != nil {
return nil, fmt.Errorf("failed to read file %s: %w", filePath, err)
}
return Gzip(data)
}
// GzipFileInChunks compresses file in chunks (memory friendly)
func GzipFileInChunks(filePath string, chunkSize int) ([]byte, error) {
compressor, err := NewGzipCompressorFromFile(filePath)
if err != nil {
return nil, err
}
defer compressor.Close()
err = compressor.CompressFileInChunks(chunkSize)
if err != nil {
return nil, err
}
return compressor.Close()
}
// GzipFileToFile compresses file and saves to another file
func GzipFileToFile(srcPath, dstPath string, chunkSize int) error {
srcFile, err := os.Open(srcPath)
if err != nil {
return fmt.Errorf("failed to open source file %s: %w", srcPath, err)
}
defer srcFile.Close()
dstFile, err := os.Create(dstPath)
if err != nil {
return fmt.Errorf("failed to create destination file %s: %w", dstPath, err)
}
defer dstFile.Close()
return GzipFromReaderToWriter(srcFile, dstFile)
}
// GzipFileStream compresses file in streaming mode, returns Reader interface
func GzipFileStream(filePath string) (io.Reader, error) {
file, err := os.Open(filePath)
if err != nil {
return nil, fmt.Errorf("failed to open file %s: %w", filePath, err)
}
pr, pw := io.Pipe()
go func() {
defer pw.Close()
defer file.Close()
gz := gzip.NewWriter(pw)
defer gz.Close()
_, err := io.Copy(gz, file)
if err != nil {
pw.CloseWithError(err)
}
}()
return pr, nil
}

152
attachment/load.go Normal file
View file

@ -0,0 +1,152 @@
package attachment
import (
"fmt"
"strings"
"github.com/yaoapp/gou/application"
"github.com/yaoapp/kun/log"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/data"
"github.com/yaoapp/yao/share"
)
// SystemUploaders system uploaders
var systemUploaders = map[string]string{
"__yao.attachment": "yao/uploaders/attachment.local.yao",
}
// Load load uploaders
func Load(cfg config.Config) error {
messages := []string{}
// Load system uploaders
err := loadSystemUploaders(cfg)
if err != nil {
return err
}
// Load filesystem uploaders
exts := []string{"*.s3.yao", "*.local.yao", "*.s3.json", "*.local.json", "*.s3.jsonc", "*.local.jsonc"}
err = application.App.Walk("uploaders", func(root, file string, isdir bool) error {
if isdir {
return nil
}
// Skip if not uploader file
if !isUploaderFile(file) {
return nil
}
err := loadUploaderFile(root, file, cfg)
if err != nil {
messages = append(messages, err.Error())
}
return err
}, exts...)
if len(messages) > 0 {
for _, message := range messages {
log.Error("Load filesystem uploaders error: %s", message)
}
return fmt.Errorf(strings.Join(messages, ";\n"))
}
return nil
}
// loadSystemUploaders load system uploaders
func loadSystemUploaders(cfg config.Config) error {
for id, path := range systemUploaders {
content, err := data.Read(path)
if err != nil {
return err
}
// Parse uploader config
var option ManagerOption
err = application.Parse(path, content, &option)
if err != nil {
return err
}
// Replace environment variables and paths
option.ReplaceEnv(cfg.DataRoot)
// Register the uploader manager
_, err = Register(id, option.Driver, option)
if err != nil {
log.Error("register system uploader %s error: %s", id, err.Error())
return err
}
log.Info("loaded system uploader: %s (%s)", id, option.Label)
}
return nil
}
// loadUploaderFile load a single uploader file
func loadUploaderFile(root, file string, cfg config.Config) error {
// Generate uploader ID
id := share.ID(root, file)
// Read file content
content, err := application.App.Read(file)
if err != nil {
return fmt.Errorf("failed to read uploader file %s: %v", file, err)
}
// Parse uploader config
var option ManagerOption
err = application.Parse(file, content, &option)
if err != nil {
return fmt.Errorf("failed to parse uploader file %s: %v", file, err)
}
// Validate driver consistency between filename and config
filenameDriver := extractDriverFromFilename(file)
if filenameDriver != "" && option.Driver != "" && filenameDriver != option.Driver {
log.Warn("Driver mismatch in uploader file %s: filename suggests '%s' but config has '%s'",
file, filenameDriver, option.Driver)
}
// Replace environment variables and paths
option.ReplaceEnv(cfg.DataRoot)
// Register the uploader manager
_, err = Register(id, option.Driver, option)
if err != nil {
log.Error("register uploader %s error: %s", id, err.Error())
return fmt.Errorf("failed to register uploader %s: %v", id, err)
}
log.Info("loaded uploader: %s (%s)", id, option.Label)
return nil
}
// isUploaderFile checks if the file is an uploader configuration file
func isUploaderFile(filename string) bool {
// Accept files with specific driver patterns: *.s3.yao, *.local.yao, etc.
lower := strings.ToLower(filename)
return strings.HasSuffix(lower, ".s3.yao") ||
strings.HasSuffix(lower, ".local.yao") ||
strings.HasSuffix(lower, ".s3.json") ||
strings.HasSuffix(lower, ".local.json") ||
strings.HasSuffix(lower, ".s3.jsonc") ||
strings.HasSuffix(lower, ".local.jsonc")
}
// extractDriverFromFilename extracts the driver name from filename (e.g., "test.s3.yao" -> "s3")
func extractDriverFromFilename(filename string) string {
lower := strings.ToLower(filename)
// Extract driver from patterns like "*.s3.yao", "*.local.json", etc.
if strings.Contains(lower, ".s3.") {
return "s3"
} else if strings.Contains(lower, ".local.") {
return "local"
}
return ""
}

48
attachment/load_test.go Normal file
View file

@ -0,0 +1,48 @@
package attachment
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/test"
)
func TestLoad(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
err := Load(config.Conf)
assert.NoError(t, err)
check(t)
}
func check(t *testing.T) {
// Check that managers are loaded
assert.NotEmpty(t, Managers, "Managers should not be empty after loading")
// Check system uploader
_, exists := Managers["__yao.attachment"]
assert.True(t, exists, "System uploader __yao.attachment should be loaded")
// Check test app uploaders (must exist)
// These are the uploaders in yao-dev-app/uploaders/
_, hasData := Managers["data"]
_, hasTest := Managers["test"]
// Both test uploaders should be loaded
assert.True(t, hasData, "Test uploader 'data' should be loaded from data.local.yao")
assert.True(t, hasTest, "Test uploader 'test' should be loaded from test.s3.yao")
// Log all loaded managers for debugging
t.Logf("Loaded managers: %v", getManagerNames())
}
// getManagerNames returns a slice of manager names for testing
func getManagerNames() []string {
names := make([]string, 0, len(Managers))
for name := range Managers {
names = append(names, name)
}
return names
}

510
attachment/local/storage.go Normal file
View file

@ -0,0 +1,510 @@
package local
import (
"compress/gzip"
"context"
"crypto/sha256"
"fmt"
"io"
"mime"
"net/http"
"os"
"path/filepath"
"strings"
"time"
)
// MaxImageSize maximum image size (1920x1080)
const MaxImageSize = 1920
// Storage the local storage driver
type Storage struct {
Path string `json:"path" yaml:"path"`
Compression bool `json:"compression" yaml:"compression"`
BaseURL string `json:"base_url" yaml:"base_url"`
PreviewURL func(fileID string) string `json:"-" yaml:"-"`
}
// New create a new local storage
func New(options map[string]interface{}) (*Storage, error) {
storage := &Storage{
Compression: true,
}
if path, ok := options["path"].(string); ok {
storage.Path = path
}
if compression, ok := options["compression"].(bool); ok {
storage.Compression = compression
}
if baseURL, ok := options["base_url"].(string); ok {
storage.BaseURL = baseURL
}
if previewURL, ok := options["preview_url"].(func(string) string); ok {
storage.PreviewURL = previewURL
}
if storage.Path == "" {
return nil, fmt.Errorf("path is required")
}
// Ensure the base path exists
if err := os.MkdirAll(storage.Path, 0755); err != nil {
return nil, fmt.Errorf("failed to create base path: %w", err)
}
return storage, nil
}
// Upload upload file to local storage
func (storage *Storage) Upload(ctx context.Context, path string, reader io.Reader, contentType string) (string, error) {
fullPath := filepath.Join(storage.Path, path)
// Create directory if not exists
dir := filepath.Dir(fullPath)
if err := os.MkdirAll(dir, 0755); err != nil {
return "", err
}
// Create and write file
file, err := os.Create(fullPath)
if err != nil {
return "", err
}
defer file.Close()
_, err = io.Copy(file, reader)
if err != nil {
return "", err
}
return path, nil
}
// UploadChunk uploads a chunk of a file
func (storage *Storage) UploadChunk(ctx context.Context, path string, chunkIndex int, reader io.Reader, contentType string) error {
// Create chunks directory
chunksDir := filepath.Join(storage.Path, ".chunks", path)
if err := os.MkdirAll(chunksDir, 0755); err != nil {
return err
}
// Write chunk file
chunkPath := filepath.Join(chunksDir, fmt.Sprintf("chunk_%d", chunkIndex))
file, err := os.Create(chunkPath)
if err != nil {
return err
}
defer file.Close()
_, err = io.Copy(file, reader)
return err
}
// MergeChunks merges all chunks into the final file
func (storage *Storage) MergeChunks(ctx context.Context, path string, totalChunks int) error {
chunksDir := filepath.Join(storage.Path, ".chunks", path)
finalPath := filepath.Join(storage.Path, path)
// Create directory for final file
dir := filepath.Dir(finalPath)
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
// Create final file
finalFile, err := os.Create(finalPath)
if err != nil {
return err
}
defer finalFile.Close()
// Read and merge chunks in order
for i := 0; i < totalChunks; i++ {
chunkPath := filepath.Join(chunksDir, fmt.Sprintf("chunk_%d", i))
chunkFile, err := os.Open(chunkPath)
if err != nil {
return fmt.Errorf("failed to read chunk %d: %w", i, err)
}
_, err = io.Copy(finalFile, chunkFile)
chunkFile.Close()
if err != nil {
return fmt.Errorf("failed to copy chunk %d: %w", i, err)
}
}
// Clean up chunks directory
os.RemoveAll(chunksDir)
return nil
}
// Reader read file from local storage
func (storage *Storage) Reader(ctx context.Context, path string) (io.ReadCloser, error) {
fullpath := filepath.Join(storage.Path, path)
reader, err := os.Open(fullpath)
if err != nil {
return nil, err
}
// If the file is a gzip file, decompress it
if strings.HasSuffix(path, ".gz") {
reader, err := gzip.NewReader(reader)
if err != nil {
return nil, err
}
return reader, nil
}
return reader, nil
}
// Download download file from local storage
func (storage *Storage) Download(ctx context.Context, path string) (io.ReadCloser, string, error) {
fullPath := filepath.Join(storage.Path, path)
reader, err := os.Open(fullPath)
if err != nil {
return nil, "", err
}
// Try to detect content type from file extension
contentType := "application/octet-stream"
ext := filepath.Ext(strings.TrimSuffix(path, ".gz"))
switch strings.ToLower(ext) {
case ".txt":
contentType = "text/plain"
case ".html":
contentType = "text/html"
case ".css":
contentType = "text/css"
case ".js":
contentType = "application/javascript"
case ".json":
contentType = "application/json"
case ".jpg", ".jpeg":
contentType = "image/jpeg"
case ".png":
contentType = "image/png"
case ".gif":
contentType = "image/gif"
case ".pdf":
contentType = "application/pdf"
case ".mp4":
contentType = "video/mp4"
case ".mp3":
contentType = "audio/mpeg"
case ".wav":
contentType = "audio/wav"
case ".ogg":
contentType = "audio/ogg"
case ".webm":
contentType = "video/webm"
case ".webp":
contentType = "image/webp"
case ".zip":
}
// If the file is a gzip file, decompress it
if strings.HasSuffix(path, ".gz") {
reader, err := gzip.NewReader(reader)
if err != nil {
return nil, "", err
}
return reader, contentType, nil
}
return reader, contentType, nil
}
// URL get file url
func (storage *Storage) URL(ctx context.Context, path string) string {
if storage.PreviewURL != nil {
return storage.PreviewURL(path)
}
if storage.BaseURL != "" {
return fmt.Sprintf("%s/%s", strings.TrimRight(storage.BaseURL, "/"), path)
}
return fmt.Sprintf("%s/%s", storage.Path, path)
}
// GetContent gets file content as bytes
func (storage *Storage) GetContent(ctx context.Context, path string) ([]byte, error) {
reader, err := storage.Reader(ctx, path)
if err != nil {
return nil, err
}
defer reader.Close()
return io.ReadAll(reader)
}
// Exists checks if a file exists
func (storage *Storage) Exists(ctx context.Context, path string) bool {
fullpath := filepath.Join(storage.Path, path)
_, err := os.Stat(fullpath)
return err == nil
}
// Delete deletes a file
func (storage *Storage) Delete(ctx context.Context, path string) error {
fullpath := filepath.Join(storage.Path, path)
return os.Remove(fullpath)
}
func (storage *Storage) makeID(filename string, ext string) string {
date := time.Now().Format("20060102")
hash := fmt.Sprintf("%x", sha256.Sum256([]byte(filename)))[:8]
name := strings.TrimSuffix(filepath.Base(filename), ext)
return fmt.Sprintf("%s/%s-%s%s", date, name, hash, ext)
}
// LocalPath returns the absolute path of the file and its content type
func (storage *Storage) LocalPath(ctx context.Context, path string) (string, string, error) {
fullPath := filepath.Join(storage.Path, path)
// Check if file exists
if _, err := os.Stat(fullPath); os.IsNotExist(err) {
return "", "", fmt.Errorf("file not found: %s", path)
}
// For gzipped files, we need to detect the original content type, not the gzip wrapper
var contentType string
var err error
if strings.HasSuffix(path, ".gz") {
// For gzipped files, detect content type of the decompressed content
originalPath := strings.TrimSuffix(path, ".gz")
ext := filepath.Ext(originalPath)
// First try to detect by original file extension
contentType, err = detectContentTypeFromExtension(ext)
if err != nil || contentType == "application/octet-stream" {
// Fallback: decompress and detect from content
contentType, err = detectContentTypeFromGzippedFile(fullPath)
if err != nil {
return "", "", fmt.Errorf("failed to detect content type from gzipped file: %w", err)
}
}
} else {
// Regular file content type detection
contentType, err = detectContentType(fullPath)
if err != nil {
return "", "", fmt.Errorf("failed to detect content type: %w", err)
}
}
// Return absolute path
absPath, err := filepath.Abs(fullPath)
if err != nil {
return "", "", fmt.Errorf("failed to get absolute path: %w", err)
}
return absPath, contentType, nil
}
// detectContentType detects content type based on file extension and content
func detectContentType(filePath string) (string, error) {
// First try to detect by file extension
ext := strings.ToLower(filepath.Ext(filePath))
// Common file extensions mapping
switch ext {
case ".txt":
return "text/plain", nil
case ".html", ".htm":
return "text/html", nil
case ".css":
return "text/css", nil
case ".js":
return "application/javascript", nil
case ".json":
return "application/json", nil
case ".xml":
return "application/xml", nil
case ".jpg", ".jpeg":
return "image/jpeg", nil
case ".png":
return "image/png", nil
case ".gif":
return "image/gif", nil
case ".webp":
return "image/webp", nil
case ".svg":
return "image/svg+xml", nil
case ".pdf":
return "application/pdf", nil
case ".doc":
return "application/msword", nil
case ".docx":
return "application/vnd.openxmlformats-officedocument.wordprocessingml.document", nil
case ".xls":
return "application/vnd.ms-excel", nil
case ".xlsx":
return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", nil
case ".ppt":
return "application/vnd.ms-powerpoint", nil
case ".pptx":
return "application/vnd.openxmlformats-officedocument.presentationml.presentation", nil
case ".zip":
return "application/zip", nil
case ".tar":
return "application/x-tar", nil
case ".gz":
return "application/gzip", nil
case ".mp3":
return "audio/mpeg", nil
case ".wav":
return "audio/wav", nil
case ".m4a":
return "audio/mp4", nil
case ".ogg":
return "audio/ogg", nil
case ".mp4":
return "video/mp4", nil
case ".avi":
return "video/x-msvideo", nil
case ".mov":
return "video/quicktime", nil
case ".webm":
return "video/webm", nil
case ".md", ".mdx":
return "text/markdown", nil
case ".yao":
return "application/yao", nil
case ".csv":
return "text/csv", nil
}
// Try to detect by MIME package
if contentType := mime.TypeByExtension(ext); contentType != "" {
return contentType, nil
}
// Fallback: detect by reading file content
file, err := os.Open(filePath)
if err != nil {
return "application/octet-stream", nil // Default fallback
}
defer file.Close()
// Read first 512 bytes for content detection
buffer := make([]byte, 512)
n, err := file.Read(buffer)
if err != nil && err != io.EOF {
return "application/octet-stream", nil
}
// Use http.DetectContentType to detect based on content
contentType := http.DetectContentType(buffer[:n])
return contentType, nil
}
// detectContentTypeFromExtension detects content type based only on file extension
func detectContentTypeFromExtension(ext string) (string, error) {
ext = strings.ToLower(ext)
// Common file extensions mapping
switch ext {
case ".txt":
return "text/plain", nil
case ".html", ".htm":
return "text/html", nil
case ".css":
return "text/css", nil
case ".js":
return "application/javascript", nil
case ".json":
return "application/json", nil
case ".xml":
return "application/xml", nil
case ".jpg", ".jpeg":
return "image/jpeg", nil
case ".png":
return "image/png", nil
case ".gif":
return "image/gif", nil
case ".webp":
return "image/webp", nil
case ".svg":
return "image/svg+xml", nil
case ".pdf":
return "application/pdf", nil
case ".doc":
return "application/msword", nil
case ".docx":
return "application/vnd.openxmlformats-officedocument.wordprocessingml.document", nil
case ".xls":
return "application/vnd.ms-excel", nil
case ".xlsx":
return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", nil
case ".ppt":
return "application/vnd.ms-powerpoint", nil
case ".pptx":
return "application/vnd.openxmlformats-officedocument.presentationml.presentation", nil
case ".zip":
return "application/zip", nil
case ".tar":
return "application/x-tar", nil
case ".mp3":
return "audio/mpeg", nil
case ".wav":
return "audio/wav", nil
case ".m4a":
return "audio/mp4", nil
case ".ogg":
return "audio/ogg", nil
case ".mp4":
return "video/mp4", nil
case ".avi":
return "video/x-msvideo", nil
case ".mov":
return "video/quicktime", nil
case ".webm":
return "video/webm", nil
case ".md", ".mdx":
return "text/markdown", nil
case ".yao":
return "application/yao", nil
case ".csv":
return "text/csv", nil
}
// Try to detect by MIME package
if contentType := mime.TypeByExtension(ext); contentType != "" {
return contentType, nil
}
// Return default if not found
return "application/octet-stream", nil
}
// detectContentTypeFromGzippedFile detects content type by decompressing and reading gzipped file
func detectContentTypeFromGzippedFile(gzippedFilePath string) (string, error) {
file, err := os.Open(gzippedFilePath)
if err != nil {
return "", err
}
defer file.Close()
// Create gzip reader
gzipReader, err := gzip.NewReader(file)
if err != nil {
return "", err
}
defer gzipReader.Close()
// Read first 512 bytes of decompressed content
buffer := make([]byte, 512)
n, err := gzipReader.Read(buffer)
if err != nil && err != io.EOF {
return "", err
}
// Use http.DetectContentType to detect based on decompressed content
contentType := http.DetectContentType(buffer[:n])
return contentType, nil
}

315
attachment/local/storage_test.go Normal file
View file

@ -0,0 +1,315 @@
package local
import (
"bytes"
"context"
"image"
"image/png"
"io"
"os"
"path/filepath"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
)
// generateTestFileName generates a unique test filename with the given prefix and extension
func generateTestFileName(prefix, ext string) string {
return prefix + "-" + uuid.New().String() + ext
}
func TestLocalStorage(t *testing.T) {
// Create a temporary directory for testing
tempDir, err := os.MkdirTemp("", "local_storage_test")
assert.NoError(t, err)
defer os.RemoveAll(tempDir)
testPath := filepath.Join(tempDir, "test_storage")
t.Run("Create Storage", func(t *testing.T) {
storage, err := New(map[string]interface{}{
"path": testPath,
"compression": true,
})
assert.NoError(t, err)
assert.NotNil(t, storage)
assert.Equal(t, testPath, storage.Path)
assert.True(t, storage.Compression)
})
t.Run("Upload and Download", func(t *testing.T) {
storage, err := New(map[string]interface{}{
"path": testPath,
"compression": true,
})
assert.NoError(t, err)
content := []byte("test content")
reader := bytes.NewReader(content)
fileID := generateTestFileName("upload-download", ".txt")
_, err = storage.Upload(context.Background(), fileID, reader, "text/plain")
assert.NoError(t, err)
assert.NotEmpty(t, fileID)
// Download
reader2, contentType, err := storage.Download(context.Background(), fileID)
assert.NoError(t, err)
assert.Contains(t, contentType, "text/plain")
downloaded, err := io.ReadAll(reader2)
assert.NoError(t, err)
assert.Equal(t, content, downloaded)
})
t.Run("Upload and Download Image with Compression", func(t *testing.T) {
storage, err := New(map[string]interface{}{
"path": testPath,
"compression": true,
})
assert.NoError(t, err)
// Create a test image (100x100 pixels - smaller for faster testing)
img := image.NewRGBA(image.Rect(0, 0, 100, 100))
var buf bytes.Buffer
err = png.Encode(&buf, img)
assert.NoError(t, err)
// Upload
reader := bytes.NewReader(buf.Bytes())
fileID := generateTestFileName("image-with-compression", ".png")
_, err = storage.Upload(context.Background(), fileID, reader, "image/png")
assert.NoError(t, err)
assert.NotEmpty(t, fileID)
// Download and verify
reader2, contentType, err := storage.Download(context.Background(), fileID)
assert.NoError(t, err)
assert.Equal(t, "image/png", contentType)
downloaded, err := io.ReadAll(reader2)
assert.NoError(t, err)
// Decode the downloaded image
downloadedImg, _, err := image.Decode(bytes.NewReader(downloaded))
assert.NoError(t, err)
// Verify image was processed
bounds := downloadedImg.Bounds()
assert.True(t, bounds.Dx() > 0)
assert.True(t, bounds.Dy() > 0)
})
t.Run("Upload Image without Compression", func(t *testing.T) {
storage, err := New(map[string]interface{}{
"path": testPath,
"compression": false,
})
assert.NoError(t, err)
// Create a test image (100x100 pixels)
img := image.NewRGBA(image.Rect(0, 0, 100, 100))
var buf bytes.Buffer
err = png.Encode(&buf, img)
assert.NoError(t, err)
// Upload
reader := bytes.NewReader(buf.Bytes())
fileID := generateTestFileName("image-without-compression", ".png")
_, err = storage.Upload(context.Background(), fileID, reader, "image/png")
assert.NoError(t, err)
assert.NotEmpty(t, fileID)
// Download and verify
reader2, contentType, err := storage.Download(context.Background(), fileID)
assert.NoError(t, err)
assert.Equal(t, "image/png", contentType)
downloaded, err := io.ReadAll(reader2)
assert.NoError(t, err)
// Decode the downloaded image
downloadedImg, _, err := image.Decode(bytes.NewReader(downloaded))
assert.NoError(t, err)
// Verify dimensions are unchanged
bounds := downloadedImg.Bounds()
assert.Equal(t, 100, bounds.Dx())
assert.Equal(t, 100, bounds.Dy())
})
t.Run("URL Generation", func(t *testing.T) {
storage, err := New(map[string]interface{}{
"path": testPath,
"compression": true,
})
assert.NoError(t, err)
fileID := "20240101/test-12345678.txt"
url := storage.URL(context.Background(), fileID)
expected := filepath.Join(testPath, fileID)
assert.Equal(t, expected, url)
})
t.Run("Download Non-existent File", func(t *testing.T) {
storage, err := New(map[string]interface{}{
"path": testPath,
"compression": true,
})
assert.NoError(t, err)
_, _, err = storage.Download(context.Background(), "non-existent.txt")
assert.Error(t, err)
})
t.Run("Chunked Upload", func(t *testing.T) {
storage, err := New(map[string]interface{}{
"path": testPath,
})
assert.NoError(t, err)
fileID := "test-chunked.txt"
content1 := []byte("chunk1")
content2 := []byte("chunk2")
// Upload chunks
err = storage.UploadChunk(context.Background(), fileID, 0, bytes.NewReader(content1), "text/plain")
assert.NoError(t, err)
err = storage.UploadChunk(context.Background(), fileID, 1, bytes.NewReader(content2), "text/plain")
assert.NoError(t, err)
// Merge chunks
err = storage.MergeChunks(context.Background(), fileID, 2)
assert.NoError(t, err)
// Download and verify
reader, contentType, err := storage.Download(context.Background(), fileID)
assert.NoError(t, err)
assert.Equal(t, "text/plain", contentType)
downloaded, err := io.ReadAll(reader)
assert.NoError(t, err)
assert.Equal(t, append(content1, content2...), downloaded)
})
t.Run("File Operations", func(t *testing.T) {
storage, err := New(map[string]interface{}{
"path": testPath,
})
assert.NoError(t, err)
fileID := "test-ops.txt"
content := []byte("test content")
// Upload file
_, err = storage.Upload(context.Background(), fileID, bytes.NewReader(content), "text/plain")
assert.NoError(t, err)
// Check if file exists
exists := storage.Exists(context.Background(), fileID)
assert.True(t, exists)
// Read file
reader, err := storage.Reader(context.Background(), fileID)
assert.NoError(t, err)
defer reader.Close()
data, err := io.ReadAll(reader)
assert.NoError(t, err)
assert.Equal(t, content, data)
// Get file content directly
directContent, err := storage.GetContent(context.Background(), fileID)
assert.NoError(t, err)
assert.Equal(t, content, directContent)
// Delete file
err = storage.Delete(context.Background(), fileID)
assert.NoError(t, err)
// Check if file no longer exists
exists = storage.Exists(context.Background(), fileID)
assert.False(t, exists)
})
t.Run("LocalPath", func(t *testing.T) {
storage, err := New(map[string]interface{}{
"path": testPath,
})
assert.NoError(t, err)
// Test different file types to verify content type detection
testFiles := []struct {
ext string
content []byte
contentType string
expectedCT string
}{
{".txt", []byte("Hello World"), "text/plain", "text/plain"},
{".json", []byte(`{"key": "value"}`), "application/json", "application/json"},
{".html", []byte("<html><body>Test</body></html>"), "text/html", "text/html"},
{".csv", []byte("col1,col2\nval1,val2"), "text/csv", "text/csv"},
{".md", []byte("# Markdown Content"), "text/markdown", "text/markdown"},
{".yao", []byte("yao file content"), "application/yao", "application/yao"},
}
for _, tf := range testFiles {
// Generate unique filename with UUID to avoid conflicts
fileName := generateTestFileName("localpath-test", tf.ext)
// Upload file
_, err = storage.Upload(context.Background(), fileName, bytes.NewReader(tf.content), tf.contentType)
assert.NoError(t, err, "Failed to upload %s", fileName)
// Get local path and content type
localPath, detectedCT, err := storage.LocalPath(context.Background(), fileName)
assert.NoError(t, err, "Failed to get local path for %s", fileName)
assert.NotEmpty(t, localPath, "Local path should not be empty for %s", fileName)
assert.Equal(t, tf.expectedCT, detectedCT, "Content type mismatch for %s", fileName)
// Verify the path is absolute
assert.True(t, filepath.IsAbs(localPath), "Path should be absolute for %s", fileName)
// Verify the file exists at the returned path
_, err = os.Stat(localPath)
assert.NoError(t, err, "File should exist at local path for %s", fileName)
// Verify file content
fileContent, err := os.ReadFile(localPath)
assert.NoError(t, err, "Failed to read file at local path for %s", fileName)
assert.Equal(t, tf.content, fileContent, "File content mismatch for %s", fileName)
}
})
t.Run("LocalPath_NonExistentFile", func(t *testing.T) {
storage, err := New(map[string]interface{}{
"path": testPath,
})
assert.NoError(t, err)
// Test with non-existent file
_, _, err = storage.LocalPath(context.Background(), "non-existent.txt")
assert.Error(t, err)
assert.Contains(t, err.Error(), "file not found")
})
t.Run("LocalPath_ContentDetection", func(t *testing.T) {
storage, err := New(map[string]interface{}{
"path": testPath,
})
assert.NoError(t, err)
// Upload a file without extension but with recognizable content
htmlContent := []byte("<!DOCTYPE html><html><head><title>Test</title></head><body><h1>Hello</h1></body></html>")
_, err = storage.Upload(context.Background(), "noext", bytes.NewReader(htmlContent), "application/octet-stream")
assert.NoError(t, err)
// Get local path - should detect HTML content type
localPath, contentType, err := storage.LocalPath(context.Background(), "noext")
assert.NoError(t, err)
assert.NotEmpty(t, localPath)
// Content detection should identify this as HTML
assert.Equal(t, "text/html; charset=utf-8", contentType)
})
}

1052
attachment/manager.go Normal file
View file

File diff suppressed because it is too large Load diff

1110
attachment/manager_test.go Normal file
View file

File diff suppressed because it is too large Load diff

739
attachment/s3/storage.go Normal file
View file

@ -0,0 +1,739 @@
package s3
import (
"bytes"
"compress/gzip"
"context"
"fmt"
"image"
"image/jpeg"
"image/png"
"io"
"mime"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
// DefaultExpiration default expiration time for presigned URLs (5 minutes)
const DefaultExpiration = 5 * time.Minute
// MaxImageSize maximum image size (1920x1080)
const MaxImageSize = 1920
// Storage the S3 storage driver
type Storage struct {
Endpoint string `json:"endpoint" yaml:"endpoint"`
Region string `json:"region" yaml:"region"`
Key string `json:"key" yaml:"key"`
Secret string `json:"secret" yaml:"secret"`
Bucket string `json:"bucket" yaml:"bucket"`
Expiration time.Duration `json:"expiration" yaml:"expiration"`
CacheDir string `json:"cache_dir" yaml:"cache_dir"`
client *s3.Client
prefix string
compression bool
}
// New create a new S3 storage
func New(options map[string]interface{}) (*Storage, error) {
storage := &Storage{
Region: "auto",
Expiration: DefaultExpiration,
compression: true,
}
if endpoint, ok := options["endpoint"].(string); ok {
storage.Endpoint = endpoint
}
if region, ok := options["region"].(string); ok {
storage.Region = region
}
if key, ok := options["key"].(string); ok {
storage.Key = key
}
if secret, ok := options["secret"].(string); ok {
storage.Secret = secret
}
if bucket, ok := options["bucket"].(string); ok {
storage.Bucket = bucket
}
if prefix, ok := options["prefix"].(string); ok {
storage.prefix = prefix
}
if cacheDir, ok := options["cache_dir"].(string); ok {
storage.CacheDir = cacheDir
} else {
// Use system temp directory as default
storage.CacheDir = os.TempDir()
}
if exp, ok := options["expiration"].(time.Duration); ok {
storage.Expiration = exp
}
if compression, ok := options["compression"].(bool); ok {
storage.compression = compression
}
// Validate required fields
if storage.Key == "" || storage.Secret == "" {
return nil, fmt.Errorf("key and secret are required")
}
if storage.Bucket == "" {
return nil, fmt.Errorf("bucket is required")
}
// Create S3 client
opts := s3.Options{
Region: storage.Region,
Credentials: credentials.NewStaticCredentialsProvider(storage.Key, storage.Secret, ""),
UsePathStyle: true,
}
if storage.Endpoint != "" {
// Remove bucket name from endpoint if present
endpoint := storage.Endpoint
if strings.Contains(endpoint, "/"+storage.Bucket) {
endpoint = strings.TrimSuffix(endpoint, "/"+storage.Bucket)
}
opts.BaseEndpoint = aws.String(endpoint)
}
storage.client = s3.New(opts)
// Ensure cache directory exists
if err := os.MkdirAll(storage.CacheDir, 0755); err != nil {
return nil, fmt.Errorf("failed to create cache directory %s: %w", storage.CacheDir, err)
}
return storage, nil
}
// Upload upload file to S3
func (storage *Storage) Upload(ctx context.Context, path string, reader io.Reader, contentType string) (string, error) {
if storage.client == nil {
return "", fmt.Errorf("s3 client not initialized")
}
key := filepath.Join(storage.prefix, path)
// Upload file
_, err := storage.client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(storage.Bucket),
Key: aws.String(key),
Body: reader,
ContentType: aws.String(contentType),
})
if err != nil {
return "", fmt.Errorf("failed to upload file %s: %w", path, err)
}
return path, nil
}
// UploadChunk uploads a chunk of a file to S3
func (storage *Storage) UploadChunk(ctx context.Context, path string, chunkIndex int, reader io.Reader, contentType string) error {
if storage.client == nil {
return fmt.Errorf("s3 client not initialized")
}
// Store chunks with a special prefix
chunkKey := filepath.Join(storage.prefix, ".chunks", path, fmt.Sprintf("chunk_%d", chunkIndex))
_, err := storage.client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(storage.Bucket),
Key: aws.String(chunkKey),
Body: reader,
ContentType: aws.String(contentType),
})
if err != nil {
return fmt.Errorf("failed to upload chunk %s %d: %w", path, chunkIndex, err)
}
return nil
}
// MergeChunks merges all chunks into the final file in S3
func (storage *Storage) MergeChunks(ctx context.Context, path string, totalChunks int) error {
if storage.client == nil {
return fmt.Errorf("s3 client not initialized")
}
finalKey := filepath.Join(storage.prefix, path)
// Create a buffer to hold the merged content
var mergedContent bytes.Buffer
var contentType string
// Download and merge chunks in order
for i := 0; i < totalChunks; i++ {
chunkKey := filepath.Join(storage.prefix, ".chunks", path, fmt.Sprintf("chunk_%d", i))
result, err := storage.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(storage.Bucket),
Key: aws.String(chunkKey),
})
if err != nil {
return fmt.Errorf("failed to get chunk %d: %w", i, err)
}
// Get content type from the first chunk
if i == 0 && result.ContentType != nil {
contentType = *result.ContentType
}
_, err = io.Copy(&mergedContent, result.Body)
result.Body.Close()
if err != nil {
return fmt.Errorf("failed to copy chunk %s %d: %w", path, i, err)
}
}
// Default content type if not found
if contentType == "" {
contentType = "application/octet-stream"
}
// Upload the merged content as the final file with proper content type
_, err := storage.client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(storage.Bucket),
Key: aws.String(finalKey),
Body: bytes.NewReader(mergedContent.Bytes()),
ContentType: aws.String(contentType),
})
if err != nil {
return fmt.Errorf("failed to upload merged file %s: %w", path, err)
}
// Clean up chunks
for i := 0; i < totalChunks; i++ {
chunkKey := filepath.Join(storage.prefix, ".chunks", path, fmt.Sprintf("chunk_%d", i))
storage.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(storage.Bucket),
Key: aws.String(chunkKey),
})
}
return nil
}
// Reader read file from S3
func (storage *Storage) Reader(ctx context.Context, path string) (io.ReadCloser, error) {
if storage.client == nil {
return nil, fmt.Errorf("s3 client not initialized")
}
key := filepath.Join(storage.prefix, path)
result, err := storage.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(storage.Bucket),
Key: aws.String(key),
})
if err != nil {
return nil, fmt.Errorf("failed to get file %s: %w", path, err)
}
// If the file is a gzip file, decompress it
if strings.HasSuffix(path, ".gz") {
reader, err := gzip.NewReader(result.Body)
if err != nil {
return nil, err
}
return reader, nil
}
return result.Body, nil
}
// Download download file from S3
func (storage *Storage) Download(ctx context.Context, path string) (io.ReadCloser, string, error) {
if storage.client == nil {
return nil, "", fmt.Errorf("s3 client not initialized")
}
key := filepath.Join(storage.prefix, path)
// Get object
result, err := storage.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(storage.Bucket),
Key: aws.String(key),
})
if err != nil {
return nil, "", fmt.Errorf("failed to download file %s: %w", path, err)
}
contentType := "application/octet-stream"
if result.ContentType != nil {
contentType = *result.ContentType
}
// Try to detect content type from file extension
ext := filepath.Ext(strings.TrimSuffix(path, ".gz"))
switch strings.ToLower(ext) {
case ".txt":
contentType = "text/plain"
case ".html":
contentType = "text/html"
case ".css":
contentType = "text/css"
case ".js":
contentType = "application/javascript"
case ".json":
contentType = "application/json"
case ".jpg", ".jpeg":
contentType = "image/jpeg"
case ".png":
contentType = "image/png"
case ".gif":
contentType = "image/gif"
case ".pdf":
contentType = "application/pdf"
case ".mp4":
contentType = "video/mp4"
case ".mp3":
contentType = "audio/mpeg"
case ".wav":
contentType = "audio/wav"
case ".ogg":
contentType = "audio/ogg"
case ".webm":
contentType = "video/webm"
case ".webp":
contentType = "image/webp"
case ".zip":
}
// If the file is a gzip file, decompress it
if strings.HasSuffix(path, ".gz") {
reader, err := gzip.NewReader(result.Body)
if err != nil {
return nil, "", err
}
return reader, contentType, nil
}
return result.Body, contentType, nil
}
// GetContent gets file content as bytes
func (storage *Storage) GetContent(ctx context.Context, path string) ([]byte, error) {
reader, err := storage.Reader(ctx, path)
if err != nil {
return nil, err
}
defer reader.Close()
return io.ReadAll(reader)
}
// URL get file url with expiration
func (storage *Storage) URL(ctx context.Context, path string) string {
if storage.client == nil {
return ""
}
key := filepath.Join(storage.prefix, path)
presignClient := s3.NewPresignClient(storage.client)
request, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(storage.Bucket),
Key: aws.String(key),
}, s3.WithPresignExpires(storage.Expiration))
if err != nil {
return ""
}
return request.URL
}
// Exists checks if a file exists in S3
func (storage *Storage) Exists(ctx context.Context, path string) bool {
if storage.client == nil {
return false
}
key := filepath.Join(storage.prefix, path)
_, err := storage.client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(storage.Bucket),
Key: aws.String(key),
})
return err == nil
}
// Delete deletes a file from S3
func (storage *Storage) Delete(ctx context.Context, path string) error {
if storage.client == nil {
return fmt.Errorf("s3 client not initialized")
}
key := filepath.Join(storage.prefix, path)
_, err := storage.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(storage.Bucket),
Key: aws.String(key),
})
if err != nil {
return fmt.Errorf("failed to delete file: %w", err)
}
return nil
}
func (storage *Storage) makeID(filename string, ext string) string {
date := time.Now().Format("20060102")
name := strings.TrimSuffix(filepath.Base(filename), ext)
return fmt.Sprintf("%s/%s-%d%s", date, name, time.Now().UnixNano(), ext)
}
// isImage checks if the content type is an image
func isImage(contentType string) bool {
return strings.HasPrefix(contentType, "image/")
}
// compressImage compresses the image while maintaining aspect ratio
func compressImage(data []byte, contentType string) ([]byte, error) {
// Decode image
img, _, err := image.Decode(bytes.NewReader(data))
if err != nil {
return nil, fmt.Errorf("failed to decode image: %w", err)
}
// Calculate new dimensions
bounds := img.Bounds()
width := bounds.Dx()
height := bounds.Dy()
var newWidth, newHeight int
if width > height {
if width > MaxImageSize {
newWidth = MaxImageSize
newHeight = int(float64(height) * (float64(MaxImageSize) / float64(width)))
} else {
return data, nil // No need to resize
}
} else {
if height > MaxImageSize {
newHeight = MaxImageSize
newWidth = int(float64(width) * (float64(MaxImageSize) / float64(height)))
} else {
return data, nil // No need to resize
}
}
// Create new image with new dimensions
newImg := image.NewRGBA(image.Rect(0, 0, newWidth, newHeight))
// Scale the image using bilinear interpolation
for y := 0; y < newHeight; y++ {
for x := 0; x < newWidth; x++ {
srcX := float64(x) * float64(width) / float64(newWidth)
srcY := float64(y) * float64(height) / float64(newHeight)
newImg.Set(x, y, img.At(int(srcX), int(srcY)))
}
}
// Encode image
var buf bytes.Buffer
switch contentType {
case "image/jpeg":
err = jpeg.Encode(&buf, newImg, &jpeg.Options{Quality: 85})
case "image/png":
err = png.Encode(&buf, newImg)
default:
return data, nil // Unsupported format, return original
}
if err != nil {
return nil, fmt.Errorf("failed to encode image: %w", err)
}
return buf.Bytes(), nil
}
// LocalPath downloads the file to cache directory and returns absolute path with content type
func (storage *Storage) LocalPath(ctx context.Context, path string) (string, string, error) {
if storage.client == nil {
return "", "", fmt.Errorf("s3 client not initialized")
}
// Create cache file path using the same structure as storage path
cacheFilePath := filepath.Join(storage.CacheDir, "s3_cache", path)
// Create directory for cache file
dir := filepath.Dir(cacheFilePath)
if err := os.MkdirAll(dir, 0755); err != nil {
return "", "", fmt.Errorf("failed to create cache directory: %w", err)
}
// Check if file already exists in cache and is not outdated
if _, err := os.Stat(cacheFilePath); err == nil {
// File exists in cache, detect content type and return
contentType, err := detectContentType(cacheFilePath)
if err != nil {
return "", "", fmt.Errorf("failed to detect content type: %w", err)
}
return cacheFilePath, contentType, nil
}
// Download file from S3 to cache
key := filepath.Join(storage.prefix, path)
result, err := storage.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(storage.Bucket),
Key: aws.String(key),
})
if err != nil {
return "", "", fmt.Errorf("failed to download file %s: %w", path, err)
}
defer result.Body.Close()
// Create cache file
cacheFile, err := os.Create(cacheFilePath)
if err != nil {
return "", "", fmt.Errorf("failed to create cache file: %w", err)
}
defer cacheFile.Close()
// Handle gzipped files - decompress during download
var reader io.Reader = result.Body
if strings.HasSuffix(path, ".gz") {
gzipReader, err := gzip.NewReader(result.Body)
if err != nil {
return "", "", fmt.Errorf("failed to create gzip reader: %w", err)
}
defer gzipReader.Close()
reader = gzipReader
// Remove .gz extension from cache file path since we're decompressing
newCacheFilePath := strings.TrimSuffix(cacheFilePath, ".gz")
cacheFile.Close()
os.Remove(cacheFilePath)
cacheFile, err = os.Create(newCacheFilePath)
if err != nil {
return "", "", fmt.Errorf("failed to create decompressed cache file: %w", err)
}
defer cacheFile.Close()
cacheFilePath = newCacheFilePath
}
// Copy file content to cache
_, err = io.Copy(cacheFile, reader)
if err != nil {
return "", "", fmt.Errorf("failed to copy file to cache: %w", err)
}
// For files that were decompressed from .gz, we need to detect the original content type
var contentType string
if strings.HasSuffix(path, ".gz") {
// Original path was gzipped, detect content type of decompressed content
originalPath := strings.TrimSuffix(path, ".gz")
ext := filepath.Ext(originalPath)
// First try to detect by original file extension
contentType, err = detectContentTypeFromExtension(ext)
if err != nil || contentType == "application/octet-stream" {
// Fallback: detect from decompressed content
contentType, err = detectContentType(cacheFilePath)
if err != nil {
return "", "", fmt.Errorf("failed to detect content type: %w", err)
}
}
} else {
// Regular file content type detection
contentType, err = detectContentType(cacheFilePath)
if err != nil {
return "", "", fmt.Errorf("failed to detect content type: %w", err)
}
}
return cacheFilePath, contentType, nil
}
// detectContentType detects content type based on file extension and content
func detectContentType(filePath string) (string, error) {
// First try to detect by file extension
ext := strings.ToLower(filepath.Ext(filePath))
// Common file extensions mapping
switch ext {
case ".txt":
return "text/plain", nil
case ".html", ".htm":
return "text/html", nil
case ".css":
return "text/css", nil
case ".js":
return "application/javascript", nil
case ".json":
return "application/json", nil
case ".xml":
return "application/xml", nil
case ".jpg", ".jpeg":
return "image/jpeg", nil
case ".png":
return "image/png", nil
case ".gif":
return "image/gif", nil
case ".webp":
return "image/webp", nil
case ".svg":
return "image/svg+xml", nil
case ".pdf":
return "application/pdf", nil
case ".doc":
return "application/msword", nil
case ".docx":
return "application/vnd.openxmlformats-officedocument.wordprocessingml.document", nil
case ".xls":
return "application/vnd.ms-excel", nil
case ".xlsx":
return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", nil
case ".ppt":
return "application/vnd.ms-powerpoint", nil
case ".pptx":
return "application/vnd.openxmlformats-officedocument.presentationml.presentation", nil
case ".zip":
return "application/zip", nil
case ".tar":
return "application/x-tar", nil
case ".gz":
return "application/gzip", nil
case ".mp3":
return "audio/mpeg", nil
case ".wav":
return "audio/wav", nil
case ".m4a":
return "audio/mp4", nil
case ".ogg":
return "audio/ogg", nil
case ".mp4":
return "video/mp4", nil
case ".avi":
return "video/x-msvideo", nil
case ".mov":
return "video/quicktime", nil
case ".webm":
return "video/webm", nil
case ".md", ".mdx":
return "text/markdown", nil
case ".yao":
return "application/yao", nil
case ".csv":
return "text/csv", nil
}
// Try to detect by MIME package
if contentType := mime.TypeByExtension(ext); contentType != "" {
return contentType, nil
}
// Fallback: detect by reading file content
file, err := os.Open(filePath)
if err != nil {
return "application/octet-stream", nil // Default fallback
}
defer file.Close()
// Read first 512 bytes for content detection
buffer := make([]byte, 512)
n, err := file.Read(buffer)
if err != nil && err != io.EOF {
return "application/octet-stream", nil
}
// Use http.DetectContentType to detect based on content
contentType := http.DetectContentType(buffer[:n])
return contentType, nil
}
// detectContentTypeFromExtension detects content type based only on file extension
func detectContentTypeFromExtension(ext string) (string, error) {
ext = strings.ToLower(ext)
// Common file extensions mapping
switch ext {
case ".txt":
return "text/plain", nil
case ".html", ".htm":
return "text/html", nil
case ".css":
return "text/css", nil
case ".js":
return "application/javascript", nil
case ".json":
return "application/json", nil
case ".xml":
return "application/xml", nil
case ".jpg", ".jpeg":
return "image/jpeg", nil
case ".png":
return "image/png", nil
case ".gif":
return "image/gif", nil
case ".webp":
return "image/webp", nil
case ".svg":
return "image/svg+xml", nil
case ".pdf":
return "application/pdf", nil
case ".doc":
return "application/msword", nil
case ".docx":
return "application/vnd.openxmlformats-officedocument.wordprocessingml.document", nil
case ".xls":
return "application/vnd.ms-excel", nil
case ".xlsx":
return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", nil
case ".ppt":
return "application/vnd.ms-powerpoint", nil
case ".pptx":
return "application/vnd.openxmlformats-officedocument.presentationml.presentation", nil
case ".zip":
return "application/zip", nil
case ".tar":
return "application/x-tar", nil
case ".mp3":
return "audio/mpeg", nil
case ".wav":
return "audio/wav", nil
case ".m4a":
return "audio/mp4", nil
case ".ogg":
return "audio/ogg", nil
case ".mp4":
return "video/mp4", nil
case ".avi":
return "video/x-msvideo", nil
case ".mov":
return "video/quicktime", nil
case ".webm":
return "video/webm", nil
case ".md", ".mdx":
return "text/markdown", nil
case ".yao":
return "application/yao", nil
case ".csv":
return "text/csv", nil
}
// Try to detect by MIME package
if contentType := mime.TypeByExtension(ext); contentType != "" {
return contentType, nil
}
// Return default if not found
return "application/octet-stream", nil
}

369
attachment/s3/storage_test.go Normal file
View file

@ -0,0 +1,369 @@
package s3
import (
"bytes"
"context"
"io"
"os"
"path/filepath"
"strings"
"testing"
"time"
"compress/gzip"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
)
// generateTestFileName generates a unique test filename with the given prefix and extension
func generateTestFileName(prefix, ext string) string {
return prefix + "-" + uuid.New().String() + ext
}
func getS3Config() map[string]interface{} {
return map[string]interface{}{
"endpoint": os.Getenv("S3_API"),
"region": "auto",
"key": os.Getenv("S3_ACCESS_KEY"),
"secret": os.Getenv("S3_SECRET_KEY"),
"bucket": os.Getenv("S3_BUCKET"),
"prefix": "attachment-test",
"expiration": 5 * time.Minute,
"compression": true,
}
}
func skipIfNoS3Config(t *testing.T) {
if os.Getenv("S3_ACCESS_KEY") == "" || os.Getenv("S3_SECRET_KEY") == "" || os.Getenv("S3_BUCKET") == "" {
t.Skip("S3 configuration not available (set S3_ACCESS_KEY, S3_SECRET_KEY, S3_BUCKET environment variables)")
}
}
func TestS3Storage(t *testing.T) {
t.Run("Create Storage", func(t *testing.T) {
options := getS3Config()
storage, err := New(options)
if os.Getenv("S3_ACCESS_KEY") == "" || os.Getenv("S3_SECRET_KEY") == "" || os.Getenv("S3_BUCKET") == "" {
// Should fail without credentials
assert.Error(t, err)
return
}
assert.NoError(t, err)
assert.NotNil(t, storage)
if storage != nil {
assert.Equal(t, os.Getenv("S3_API"), storage.Endpoint)
assert.Equal(t, "auto", storage.Region)
assert.Equal(t, os.Getenv("S3_ACCESS_KEY"), storage.Key)
assert.Equal(t, os.Getenv("S3_SECRET_KEY"), storage.Secret)
assert.Equal(t, os.Getenv("S3_BUCKET"), storage.Bucket)
assert.Equal(t, "attachment-test", storage.prefix)
assert.Equal(t, 5*time.Minute, storage.Expiration)
assert.True(t, storage.compression)
}
})
t.Run("Upload and Download Text File", func(t *testing.T) {
skipIfNoS3Config(t)
storage, err := New(getS3Config())
assert.NoError(t, err)
content := []byte("test content")
reader := bytes.NewReader(content)
fileID := generateTestFileName("upload-test", ".txt")
_, err = storage.Upload(context.Background(), fileID, reader, "text/plain")
assert.NoError(t, err)
assert.NotEmpty(t, fileID)
// Get presigned URL
url := storage.URL(context.Background(), fileID)
assert.NotEmpty(t, url)
assert.Contains(t, url, "X-Amz-Signature")
assert.Contains(t, url, "X-Amz-Expires")
// Download
reader2, contentType, err := storage.Download(context.Background(), fileID)
assert.NoError(t, err)
assert.Contains(t, contentType, "text/plain")
downloaded, err := io.ReadAll(reader2)
assert.NoError(t, err)
assert.Equal(t, content, downloaded)
reader2.Close()
// Clean up
storage.Delete(context.Background(), fileID)
})
t.Run("Chunked Upload", func(t *testing.T) {
skipIfNoS3Config(t)
storage, err := New(getS3Config())
assert.NoError(t, err)
fileID := generateTestFileName("test-chunked", ".txt")
content1 := []byte("chunk1")
content2 := []byte("chunk2")
// Upload chunks
err = storage.UploadChunk(context.Background(), fileID, 0, bytes.NewReader(content1), "text/plain")
assert.NoError(t, err)
err = storage.UploadChunk(context.Background(), fileID, 1, bytes.NewReader(content2), "text/plain")
assert.NoError(t, err)
// Merge chunks
err = storage.MergeChunks(context.Background(), fileID, 2)
assert.NoError(t, err)
// Download and verify
reader, contentType, err := storage.Download(context.Background(), fileID)
assert.NoError(t, err)
assert.Equal(t, "text/plain", contentType)
downloaded, err := io.ReadAll(reader)
assert.NoError(t, err)
assert.Equal(t, append(content1, content2...), downloaded)
reader.Close()
// Clean up
storage.Delete(context.Background(), fileID)
})
t.Run("File Operations", func(t *testing.T) {
skipIfNoS3Config(t)
storage, err := New(getS3Config())
assert.NoError(t, err)
fileID := generateTestFileName("test-ops", ".txt")
content := []byte("test content")
// Upload file
_, err = storage.Upload(context.Background(), fileID, bytes.NewReader(content), "text/plain")
assert.NoError(t, err)
// Check if file exists
exists := storage.Exists(context.Background(), fileID)
assert.True(t, exists)
// Read file
reader, err := storage.Reader(context.Background(), fileID)
assert.NoError(t, err)
defer reader.Close()
data, err := io.ReadAll(reader)
assert.NoError(t, err)
assert.Equal(t, content, data)
// Get file content directly
directContent, err := storage.GetContent(context.Background(), fileID)
assert.NoError(t, err)
assert.Equal(t, content, directContent)
// Delete file
err = storage.Delete(context.Background(), fileID)
assert.NoError(t, err)
// Check if file no longer exists
exists = storage.Exists(context.Background(), fileID)
assert.False(t, exists)
})
t.Run("Download Non-existent File", func(t *testing.T) {
skipIfNoS3Config(t)
storage, err := New(getS3Config())
assert.NoError(t, err)
// Use UUID for non-existent file to avoid any potential conflicts
nonExistentFileID := generateTestFileName("non-existent", ".txt")
_, _, err = storage.Download(context.Background(), nonExistentFileID)
assert.Error(t, err)
})
t.Run("Invalid Configuration", func(t *testing.T) {
// Test with missing required fields
_, err := New(map[string]interface{}{
"endpoint": "https://s3.amazonaws.com",
"region": "us-east-1",
// Missing key and secret
})
assert.Error(t, err)
assert.Contains(t, err.Error(), "key and secret are required")
// Test with missing bucket
_, err = New(map[string]interface{}{
"endpoint": "https://s3.amazonaws.com",
"region": "us-east-1",
"key": "test-key",
"secret": "test-secret",
// Missing bucket
})
assert.Error(t, err)
assert.Contains(t, err.Error(), "bucket is required")
})
t.Run("LocalPath", func(t *testing.T) {
skipIfNoS3Config(t)
// Create storage with custom cache directory
tempCacheDir, err := os.MkdirTemp("", "s3_cache_test")
assert.NoError(t, err)
defer os.RemoveAll(tempCacheDir)
config := getS3Config()
config["cache_dir"] = tempCacheDir
storage, err := New(config)
assert.NoError(t, err)
// Test different file types
testFiles := []struct {
name string
content []byte
contentType string
expectedCT string
}{
{"localpath-test.txt", []byte("Hello S3 World"), "text/plain", "text/plain"},
{"localpath-test.json", []byte(`{"s3": "test"}`), "application/json", "application/json"},
{"localpath-test.html", []byte("<html><body>S3 Test</body></html>"), "text/html", "text/html"},
{"localpath-test.csv", []byte("s3,test\nval1,val2"), "text/csv", "text/csv"},
{"localpath-test.md", []byte("# S3 Markdown"), "text/markdown", "text/markdown"},
{"localpath-test.yao", []byte("s3 yao content"), "application/yao", "application/yao"},
}
for _, tf := range testFiles {
// Upload file to S3
fileID := generateTestFileName("s3-localpath", "-"+tf.name)
_, err = storage.Upload(context.Background(), fileID, bytes.NewReader(tf.content), tf.contentType)
assert.NoError(t, err, "Failed to upload %s", tf.name)
// Get local path - first call should download to cache
localPath1, detectedCT1, err := storage.LocalPath(context.Background(), fileID)
assert.NoError(t, err, "Failed to get local path for %s", tf.name)
assert.NotEmpty(t, localPath1, "Local path should not be empty for %s", tf.name)
assert.Equal(t, tf.expectedCT, detectedCT1, "Content type mismatch for %s", tf.name)
// Verify the path is absolute
assert.True(t, filepath.IsAbs(localPath1), "Path should be absolute for %s", tf.name)
// Verify the file exists at the returned path
_, err = os.Stat(localPath1)
assert.NoError(t, err, "File should exist at local path for %s", tf.name)
// Verify file content
fileContent, err := os.ReadFile(localPath1)
assert.NoError(t, err, "Failed to read file at local path for %s", tf.name)
assert.Equal(t, tf.content, fileContent, "File content mismatch for %s", tf.name)
// Get local path again - should use cached version
localPath2, detectedCT2, err := storage.LocalPath(context.Background(), fileID)
assert.NoError(t, err, "Failed to get cached local path for %s", tf.name)
assert.Equal(t, localPath1, localPath2, "Cached path should be same as first call for %s", tf.name)
assert.Equal(t, detectedCT1, detectedCT2, "Cached content type should be same as first call for %s", tf.name)
// Clean up from S3
storage.Delete(context.Background(), fileID)
}
})
t.Run("LocalPath_GzippedFile", func(t *testing.T) {
skipIfNoS3Config(t)
// Create storage with custom cache directory
tempCacheDir, err := os.MkdirTemp("", "s3_cache_gzip_test")
assert.NoError(t, err)
defer os.RemoveAll(tempCacheDir)
config := getS3Config()
config["cache_dir"] = tempCacheDir
storage, err := New(config)
assert.NoError(t, err)
// Create gzipped content
originalContent := []byte("This content will be gzipped and stored in S3")
var gzipBuf bytes.Buffer
gzipWriter := gzip.NewWriter(&gzipBuf)
_, err = gzipWriter.Write(originalContent)
assert.NoError(t, err)
gzipWriter.Close()
// Upload gzipped file
fileID := generateTestFileName("gzipped", ".txt.gz")
_, err = storage.Upload(context.Background(), fileID, bytes.NewReader(gzipBuf.Bytes()), "text/plain")
assert.NoError(t, err)
// Get local path - should decompress during download
localPath, contentType, err := storage.LocalPath(context.Background(), fileID)
assert.NoError(t, err)
assert.NotEmpty(t, localPath)
// Verify the file is decompressed in cache (path should not end with .gz)
assert.False(t, strings.HasSuffix(localPath, ".gz"), "Cached file should be decompressed")
// Verify content is decompressed
cachedContent, err := os.ReadFile(localPath)
assert.NoError(t, err)
assert.Equal(t, originalContent, cachedContent, "Cached file should contain decompressed content")
// Verify content type
assert.Equal(t, "text/plain", contentType)
// Clean up
storage.Delete(context.Background(), fileID)
})
t.Run("LocalPath_NonExistentFile", func(t *testing.T) {
skipIfNoS3Config(t)
storage, err := New(getS3Config())
assert.NoError(t, err)
// Test with non-existent file
nonExistentFileID := generateTestFileName("non-existent-localpath", ".txt")
_, _, err = storage.LocalPath(context.Background(), nonExistentFileID)
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to download file")
})
t.Run("LocalPath_CustomCacheDir", func(t *testing.T) {
skipIfNoS3Config(t)
// Create custom cache directory
customCacheDir, err := os.MkdirTemp("", "custom_s3_cache")
assert.NoError(t, err)
defer os.RemoveAll(customCacheDir)
config := getS3Config()
config["cache_dir"] = customCacheDir
storage, err := New(config)
assert.NoError(t, err)
// Verify cache directory is set correctly
assert.Equal(t, customCacheDir, storage.CacheDir)
// Upload a test file
content := []byte("Custom cache directory test")
fileID := generateTestFileName("custom-cache", ".txt")
_, err = storage.Upload(context.Background(), fileID, bytes.NewReader(content), "text/plain")
assert.NoError(t, err)
// Get local path
localPath, contentType, err := storage.LocalPath(context.Background(), fileID)
assert.NoError(t, err)
assert.NotEmpty(t, localPath)
assert.Equal(t, "text/plain", contentType)
// Verify the file is cached in the custom directory
assert.True(t, strings.HasPrefix(localPath, customCacheDir), "File should be cached in custom directory")
// Clean up
storage.Delete(context.Background(), fileID)
})
}

160
attachment/types.go Normal file
View file

@ -0,0 +1,160 @@
package attachment
import (
"context"
"io"
"mime/multipart"
"github.com/yaoapp/gou/types"
)
// FileManager defines the interface for file management operations.
// This interface provides abstraction for file operations, making it easier to:
// - Write unit tests with mock implementations
// - Switch between different storage backends
// - Maintain consistent API across different implementations
//
// Example usage:
//
// var fileManager FileManager = manager // Manager implements FileManager
// file, err := fileManager.Upload(ctx, header, reader, options)
// data, err := fileManager.Read(ctx, file.ID)
type FileManager interface {
// Upload uploads a file with optional chunked upload support
Upload(ctx context.Context, fileheader *FileHeader, reader io.Reader, option UploadOption) (*File, error)
// Download downloads a file by its ID
Download(ctx context.Context, fileID string) (*FileResponse, error)
// Read reads a file content as bytes
Read(ctx context.Context, fileID string) ([]byte, error)
// ReadBase64 reads a file content as base64 encoded string
ReadBase64(ctx context.Context, fileID string) (string, error)
// Info retrieves complete file information from database by file ID
Info(ctx context.Context, fileID string) (*File, error)
// List retrieves files from database with pagination and filtering
List(ctx context.Context, option ListOption) (*ListResult, error)
// Exists checks if a file exists
Exists(ctx context.Context, fileID string) bool
// Delete deletes a file
Delete(ctx context.Context, fileID string) error
// LocalPath gets the local path of the file
LocalPath(ctx context.Context, fileID string) (string, string, error)
}
// File the file
type File struct {
ID string `json:"file_id"`
UserPath string `json:"user_path"` // User-specified complete file path
Path string `json:"path"` // Actual storage path
Bytes int `json:"bytes"`
CreatedAt int `json:"created_at"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
Status string `json:"status"` // uploading, uploaded, indexing, indexed, upload_failed, index_failed
}
// FileResponse represents a file download response
type FileResponse struct {
Reader io.ReadCloser
ContentType string
Extension string
}
// Attachment represents a file attachment
type Attachment struct {
Name string `json:"name,omitempty"`
URL string `json:"url,omitempty"`
Description string `json:"description,omitempty"`
Type string `json:"type,omitempty"`
ContentType string `json:"content_type,omitempty"`
Bytes int64 `json:"bytes,omitempty"`
CreatedAt int64 `json:"created_at,omitempty"`
FileID string `json:"file_id,omitempty"`
UserPath string `json:"user_path,omitempty"` // User-specified complete file path
Path string `json:"path,omitempty"` // Actual storage path
Groups []string `json:"groups,omitempty"`
Gzip bool `json:"gzip,omitempty"` // Gzip the file, Optional, default is false
ClientID string `json:"client_id,omitempty"` // Client identifier
OpenID string `json:"openid,omitempty"` // OpenID identifier
}
// Manager the manager struct
type Manager struct {
ManagerOption
Name string // Manager name for identification
storage Storage
maxsize int64
chunsize int64
allowedTypes allowedType
}
// Storage the storage interface
type Storage interface {
Upload(ctx context.Context, path string, reader io.Reader, contentType string) (string, error)
UploadChunk(ctx context.Context, path string, chunkIndex int, reader io.Reader, contentType string) error
MergeChunks(ctx context.Context, path string, totalChunks int) error
Download(ctx context.Context, path string) (io.ReadCloser, string, error)
Reader(ctx context.Context, path string) (io.ReadCloser, error)
GetContent(ctx context.Context, path string) ([]byte, error)
URL(ctx context.Context, path string) string
Exists(ctx context.Context, path string) bool
Delete(ctx context.Context, path string) error
LocalPath(ctx context.Context, path string) (string, string, error) // Returns absolute path and content type
}
// ManagerOption the manager option
type ManagerOption struct {
types.MetaInfo
MaxSize string `json:"max_size,omitempty" yaml:"max_size,omitempty"` // Max size of the file, Optional, default is 20M
ChunkSize string `json:"chunk_size,omitempty" yaml:"chunk_size,omitempty"` // Chunk size of the file, Optional, default is 2M
AllowedTypes []string `json:"allowed_types,omitempty" yaml:"allowed_types,omitempty"` // Allowed types of the file, Optional, default is all
Gzip bool `json:"gzip,omitempty" yaml:"gzip,omitempty"` // Gzip the file, Optional, default is false
Driver string `json:"driver,omitempty" yaml:"driver,omitempty"` // Driver, Optional, default is local
Options map[string]interface{} `json:"options,omitempty" yaml:"options,omitempty"` // Options, Optional
}
type allowedType struct {
mapping map[string]bool
wildcards []string // Wildcard patterns for file types (e.g., "image/*", "text/*")
}
// UploadOption the upload option
type UploadOption struct {
CompressImage bool `json:"compress_image,omitempty" form:"compress_image"` // Compress the file, Optional, default is true
CompressSize int `json:"compress_size,omitempty" form:"compress_size"` // Compress the file size, Optional, default is 1920, if compress_image is true, the file size will be compressed to the compress_size
Gzip bool `json:"gzip,omitempty" form:"gzip"` // Gzip the file, Optional, default is false
OriginalFilename string `json:"original_filename,omitempty" form:"original_filename"` // Original filename sent separately to avoid encoding issues
Groups []string `json:"groups,omitempty" form:"groups"` // Groups, Optional, default is empty, Multi-level groups like ["user", "user123", "chat", "chat456"]
ClientID string `json:"client_id,omitempty" form:"client_id"` // Client identifier
OpenID string `json:"openid,omitempty" form:"openid"` // OpenID identifier
}
// ListOption defines options for listing files
type ListOption struct {
Page int `json:"page,omitempty"` // Page number (1-based), default is 1
PageSize int `json:"page_size,omitempty"` // Page size, default is 20
Filters map[string]interface{} `json:"filters,omitempty"` // Filter conditions, e.g., {"status": "uploaded", "content_type": "image/*"}
OrderBy string `json:"order_by,omitempty"` // Order by field, e.g., "created_at desc", "name asc"
Select []string `json:"select,omitempty"` // Fields to select, empty means select all
}
// ListResult contains the paginated list result
type ListResult struct {
Files []*File `json:"files"` // List of files
Total int64 `json:"total"` // Total count
Page int `json:"page"` // Current page
PageSize int `json:"page_size"` // Page size
TotalPages int `json:"total_pages"` // Total pages
}
// FileHeader the file header
type FileHeader struct {
*multipart.FileHeader
}

1
audit/README.md Normal file
View file

@ -0,0 +1 @@
# Audit Log

11
cert/cert.go
View file

@ -11,6 +11,17 @@ import (
// Load 加载API
func Load(cfg config.Config) error {
// Ignore if the certs directory does not exist
exists, err := application.App.Exists("certs")
if err != nil {
return err
}
if !exists {
return nil
}
exts := []string{"*.pem", "*.key", "*.pub"}
return application.App.Walk("certs", func(root, file string, isdir bool) error {
if isdir {

8
cmd/dump.go
View file

@ -57,12 +57,18 @@ var dumpCmd = &cobra.Command{
}
// Load model
err = engine.Load(config.Conf, engine.LoadOption{Action: "dump"})
loadWarnings, err := engine.Load(config.Conf, engine.LoadOption{Action: "dump"})
if err != nil {
fmt.Println(color.RedString(L("Fatal: %s"), err.Error()))
os.Exit(1)
}
if len(loadWarnings) > 0 {
for _, warning := range loadWarnings {
fmt.Println(color.YellowString("[%s] %s", warning.Widget, warning.Error))
}
}
if dumpModel != "" {
fmt.Println(color.YellowString(L("Not supported yet")))
os.Exit(1)

8
cmd/migrate.go
View file

@ -37,12 +37,18 @@ var migrateCmd = &cobra.Command{
}
// 加载数据模型
err := engine.Load(config.Conf, engine.LoadOption{Action: "migrate"})
loadWarnings, err := engine.Load(config.Conf, engine.LoadOption{Action: "migrate"})
if err != nil {
fmt.Println(color.RedString(L("Fatal: %s"), err.Error()))
os.Exit(1)
}
if len(loadWarnings) > 0 {
for _, warning := range loadWarnings {
fmt.Println(color.YellowString("[%s] %s", warning.Widget, warning.Error))
}
}
if name != "" {
mod, has := model.Models[name]
if !has {

8
cmd/restore.go
View file

@ -60,12 +60,18 @@ var restoreCmd = &cobra.Command{
})
// 加载数据模型
err = engine.Load(config.Conf, engine.LoadOption{Action: "restore"})
loadWarnings, err := engine.Load(config.Conf, engine.LoadOption{Action: "restore"})
if err != nil {
fmt.Println(color.RedString(L("Fatal: %s"), err.Error()))
os.Exit(1)
}
if len(loadWarnings) > 0 {
for _, warning := range loadWarnings {
fmt.Println(color.YellowString("[%s] %s", warning.Widget, warning.Error))
}
}
// Restore models
restoreModels(filepath.Join(dst, "model"), []model.MigrateOption{
model.WithDonotInsertValues(migrateNoInsert),

13
cmd/run.go
View file

@ -57,7 +57,7 @@ var runCmd = &cobra.Command{
return
}
err := engine.Load(cfg, engine.LoadOption{Action: "run"})
loadWarnings, err := engine.Load(cfg, engine.LoadOption{Action: "run"})
if err != nil {
if !runSilent {
color.Red(L("Engine: %s\n"), err.Error())
@ -130,6 +130,17 @@ var runCmd = &cobra.Command{
}
if !runSilent {
if len(loadWarnings) > 0 {
fmt.Println(color.YellowString("---------------------------------"))
fmt.Println(color.YellowString(L("Warnings")))
fmt.Println(color.YellowString("---------------------------------"))
for _, warning := range loadWarnings {
fmt.Println(color.YellowString("[%s] %s", warning.Widget, warning.Error))
}
fmt.Printf("\n")
}
color.White("--------------------------------------\n")
color.White(L("%s Response\n"), name)
color.White("--------------------------------------\n")

23
cmd/start.go
View file

@ -77,7 +77,7 @@ var startCmd = &cobra.Command{
}
// load the application engine
err := engine.Load(config.Conf, engine.LoadOption{Action: "start"})
loadWarnings, err := engine.Load(config.Conf, engine.LoadOption{Action: "start"})
if err != nil {
fmt.Println(color.RedString(L("Load: %s"), err.Error()))
os.Exit(1)
@ -232,6 +232,17 @@ var startCmd = &cobra.Command{
printStores(true)
}
// Print the warnings
if len(loadWarnings) > 0 {
fmt.Println(color.YellowString("---------------------------------"))
fmt.Println(color.YellowString(L("Warnings")))
fmt.Println(color.YellowString("---------------------------------"))
for _, warning := range loadWarnings {
fmt.Println(color.YellowString("[%s] %s", warning.Widget, warning.Error))
}
fmt.Printf("\n")
}
for {
select {
case v := <-srv.Event():
@ -275,11 +286,19 @@ func install() error {
Boot()
// load the application engine
err = engine.Load(config.Conf, engine.LoadOption{Action: "start"})
loadWarnings, err := engine.Load(config.Conf, engine.LoadOption{Action: "start"})
if err != nil {
return err
}
// Print the warnings
if len(loadWarnings) > 0 {
for _, warning := range loadWarnings {
fmt.Println(color.YellowString("[%s] %s", warning.Widget, warning.Error))
}
fmt.Printf("\n\n")
}
err = setup.Initialize(config.Conf.Root, config.Conf)
if err != nil {
return err

7
cmd/studio/run.go
View file

@ -18,6 +18,11 @@ import (
"github.com/yaoapp/yao/studio"
)
// *********************************************************************************
// !! Yao Studio Command has been deprecated.
// !! Do not use this command in your project.
// *********************************************************************************
// RunCmd command
var RunCmd = &cobra.Command{
Use: "run",
@ -43,7 +48,7 @@ var RunCmd = &cobra.Command{
return
}
err := engine.Load(cfg, engine.LoadOption{Action: "studio.run"})
_, err := engine.Load(cfg, engine.LoadOption{Action: "studio.run"})
if err != nil {
fmt.Println(color.RedString(L("Engine: %s"), err.Error()))
}

9
cmd/sui/build.go
View file

@ -31,7 +31,7 @@ var BuildCmd = &cobra.Command{
Boot()
cfg := config.Conf
err := engine.Load(cfg, engine.LoadOption{Action: "sui.build"})
loadWarnings, err := engine.Load(cfg, engine.LoadOption{Action: "sui.build"})
if err != nil {
fmt.Fprintln(os.Stderr, color.RedString(err.Error()))
return
@ -99,6 +99,13 @@ var BuildCmd = &cobra.Command{
fmt.Println(color.YellowString("Build succeeded for %s in %s", mode, timecost))
return
}
if len(loadWarnings) > 0 {
for _, warning := range loadWarnings {
fmt.Println(color.YellowString("[%s] %s", warning.Widget, warning.Error))
}
}
if len(warnings) > 0 {
for _, warning := range warnings {
fmt.Println(color.YellowString("Warning: %s", warning))

9
cmd/sui/trans.go
View file

@ -32,7 +32,7 @@ var TransCmd = &cobra.Command{
Boot()
cfg := config.Conf
err := engine.Load(cfg, engine.LoadOption{Action: "sui.trans"})
loadWarnings, err := engine.Load(cfg, engine.LoadOption{Action: "sui.trans"})
if err != nil {
fmt.Fprintln(os.Stderr, color.RedString(err.Error()))
return
@ -135,6 +135,13 @@ var TransCmd = &cobra.Command{
fmt.Println(color.YellowString("Translate succeeded for %s in %s", mode, timecost))
return
}
if len(loadWarnings) > 0 {
for _, warning := range loadWarnings {
fmt.Println(color.YellowString("[%s] %s", warning.Widget, warning.Error))
}
}
if len(warnings) > 0 {
for _, warning := range warnings {
fmt.Println(color.YellowString("Warning: %s", warning))

8
cmd/sui/watch.go
View file

@ -39,12 +39,18 @@ var WatchCmd = &cobra.Command{
Boot()
cfg := config.Conf
err := engine.Load(cfg, engine.LoadOption{Action: "sui.watch"})
loadWarnings, err := engine.Load(cfg, engine.LoadOption{Action: "sui.watch"})
if err != nil {
fmt.Fprintln(os.Stderr, color.RedString(err.Error()))
return
}
if len(loadWarnings) > 0 {
for _, warning := range loadWarnings {
fmt.Println(color.YellowString("[%s] %s", warning.Widget, warning.Error))
}
}
id := args[0]
template := args[1]

47
cmd/version.go
View file

@ -2,18 +2,22 @@ package cmd
import (
"fmt"
"github.com/spf13/cobra"
"github.com/yaoapp/yao/share"
"runtime"
"strings"
"github.com/fatih/color"
"github.com/spf13/cobra"
"github.com/yaoapp/yao/share"
)
var printAllVersion bool
var versionTemplate = `Version: %s
Go version: %s
Git commit: %s
Built: %s
OS/Arch: %s/%s
var versionTemplate = `Version: %s
Go version: %s
Yao commit: %s
Cui version: %s
Cui commit: %s
Built: %s
OS/Arch: %s/%s
`
var versionCmd = &cobra.Command{
Use: "version",
@ -23,16 +27,29 @@ var versionCmd = &cobra.Command{
if printAllVersion {
commit := strings.Split(share.PRVERSION, "-")[0]
buildTime := strings.TrimPrefix(share.PRVERSION, commit+"-")
fmt.Printf(versionTemplate,
share.VERSION,
runtime.Version(),
commit, buildTime,
runtime.GOOS,
runtime.GOARCH)
cuiCommit := strings.Split(share.PRCUI, "-")[0]
fmt.Printf("%s", color.WhiteString("Yao version: "))
fmt.Printf("%s\n", color.GreenString(share.VERSION))
fmt.Printf("%s", color.WhiteString("Yao commit: "))
fmt.Printf("%s\n", color.YellowString(commit))
fmt.Printf("%s", color.WhiteString("Cui commit: "))
fmt.Printf("%s\n", color.YellowString(cuiCommit))
fmt.Printf("%s", color.WhiteString("Built: "))
fmt.Printf("%s\n", color.BlueString(buildTime))
fmt.Printf("%s", color.WhiteString("OS/Arch: "))
fmt.Printf("%s\n", color.MagentaString("%s/%s", runtime.GOOS, runtime.GOARCH))
fmt.Printf("%s", color.WhiteString("Go version: "))
fmt.Printf("%s\n", color.CyanString(runtime.Version()))
return
}
// Do Stuff Here
fmt.Println(share.VERSION)
fmt.Printf("%s", color.WhiteString("Yao version: "))
fmt.Printf("%s\n", color.GreenString(share.VERSION))
},
}

1
cui/setup/index.html Normal file
View file

@ -0,0 +1 @@
CUI SETUP

1
cui/v0.9/index.html Normal file
View file

@ -0,0 +1 @@
# CUI v0.9.2

2
xgen/v1.0/index.html → cui/v1.0/index.html
View file

@ -1,3 +1,3 @@
# XGEN v1.0.1
# CUI v1.0.0
<b>## ROOT /__yao_admin_root/ </b>

0
xgen/v1.0/layouts__index.async.js → cui/v1.0/layouts__index.async.js
View file

0
xgen/v1.0/umi.js → cui/v1.0/umi.js
View file

2825
data/bindata.go
View file

File diff suppressed because one or more lines are too long

48
data/data.go
View file

@ -10,25 +10,25 @@ import (
assetfs "github.com/elazarl/go-bindata-assetfs"
)
// XgenV0 XGen 0.9
func XgenV0() *assetfs.AssetFS {
// CuiV0 CUI 0.9
func CuiV0() *assetfs.AssetFS {
assetInfo := func(path string) (os.FileInfo, error) {
return os.Stat(path)
}
for k := range _bintree.Children {
k = "xgen/v0.9"
k = "cui/v0.9"
return &assetfs.AssetFS{Asset: Asset, AssetDir: AssetDir, AssetInfo: assetInfo, Prefix: k, Fallback: "index.html"}
}
panic("unreachable")
}
// XgenV1 XGen 1.0
func XgenV1() *assetfs.AssetFS {
// CuiV1 CUI 1.0
func CuiV1() *assetfs.AssetFS {
assetInfo := func(path string) (os.FileInfo, error) {
return os.Stat(path)
}
for k := range _bintree.Children {
k = "xgen/v1.0"
k = "cui/v1.0"
return &assetfs.AssetFS{Asset: Asset, AssetDir: AssetDir, AssetInfo: assetInfo, Prefix: k, Fallback: "index.html"}
}
panic("unreachable")
@ -40,25 +40,25 @@ func Setup() *assetfs.AssetFS {
return os.Stat(path)
}
for k := range _bintree.Children {
k = "xgen/setup"
k = "cui/setup"
return &assetfs.AssetFS{Asset: Asset, AssetDir: AssetDir, AssetInfo: assetInfo, Prefix: k, Fallback: "index.html"}
}
panic("unreachable")
}
// ReplaceXGen bindata file
func ReplaceXGen(search, replace string) error {
err := replaceXGenIndex(search, replace)
// ReplaceCUI bindata file
func ReplaceCUI(search, replace string) error {
err := replaceCUIIndex(search, replace)
if err != nil {
return err
}
err = replaceXGenUmi(search, replace)
err = replaceCUIUmi(search, replace)
if err != nil {
return err
}
return replaceXGenLayouts(search, replace)
return replaceCUILayouts(search, replace)
}
// Read file from bin
@ -81,10 +81,10 @@ func RemoveApp() {
delete(_bindata, "yao/release/app.yaz")
}
// ReplaceXGenIndex bindata file
func replaceXGenIndex(search, replace string) error {
// ReplaceCUIIndex bindata file
func replaceCUIIndex(search, replace string) error {
content, err := bindataRead(_xgenV10IndexHtml, "xgen/v1.0/index.html")
content, err := bindataRead(_cuiV10IndexHtml, "cui/v1.0/index.html")
if err != nil {
return err
}
@ -100,14 +100,14 @@ func replaceXGenIndex(search, replace string) error {
return err
}
_xgenV10IndexHtml = b.Bytes()
_cuiV10IndexHtml = b.Bytes()
return nil
}
// replaceXGenUmi bindata file
func replaceXGenUmi(search, replace string) error {
// replaceCUIUmi bindata file
func replaceCUIUmi(search, replace string) error {
content, err := bindataRead(_xgenV10UmiJs, "xgen/v1.0/umi.js")
content, err := bindataRead(_cuiV10UmiJs, "cui/v1.0/umi.js")
if err != nil {
return err
}
@ -123,14 +123,14 @@ func replaceXGenUmi(search, replace string) error {
return err
}
_xgenV10UmiJs = b.Bytes()
_cuiV10UmiJs = b.Bytes()
return nil
}
// replaceXGenLayouts bindata file
func replaceXGenLayouts(search, replace string) error {
// replaceCUILayouts bindata file
func replaceCUILayouts(search, replace string) error {
content, err := bindataRead(_xgenV10Layouts__indexAsyncJs, "xgen/v1.0/layouts__index.async.js")
content, err := bindataRead(_cuiV10Layouts__indexAsyncJs, "cui/v1.0/layouts__index.async.js")
if err != nil {
return err
}
@ -146,6 +146,6 @@ func replaceXGenLayouts(search, replace string) error {
return err
}
_xgenV10Layouts__indexAsyncJs = b.Bytes()
_cuiV10Layouts__indexAsyncJs = b.Bytes()
return nil
}

8
data/data_test.go
View file

@ -6,18 +6,18 @@ import (
"github.com/stretchr/testify/assert"
)
func TestReplaceXGenIndex(t *testing.T) {
err := ReplaceXGen("__yao_admin_root", "Admin-Replaced")
func TestReplaceCUIIndex(t *testing.T) {
err := ReplaceCUI("__yao_admin_root", "Admin-Replaced")
if err != nil {
t.Fatal(err)
}
content, err := bindataRead(_xgenV10IndexHtml, "index.html")
content, err := bindataRead(_cuiV10IndexHtml, "index.html")
if err != nil {
t.Fatal(err)
}
content, err = bindataRead(_xgenV10UmiJs, "umi.js")
content, err = bindataRead(_cuiV10UmiJs, "umi.js")
if err != nil {
t.Fatal(err)
}

25
docker/build/Dockerfile
View file

@ -2,14 +2,14 @@
# Yao Build Environment (Ubuntu 24.04 AMD64)
#
# Build:
# docker build --platform linux/amd64 -t yaoapp/yao-build:0.10.4 .
# docker build --platform linux/amd64 -t yaoapp/yao-build:0.10.5 .
#
# Usage:
# docker run --rm -it -v /local/path/dist:/data yaoapp/yao-build:0.10.4
# docker run --rm -it -v /local/path/dist:/data yaoapp/yao-build:0.10.5
#
# Tests:
# docker run --rm -it yaoapp/yao-build:0.10.4 /bin/bash
# docker run --rm -it -v ./test:/data yaoapp/yao-build:0.10.4 /bin/bash
# docker run --rm -it yaoapp/yao-build:0.10.5 /bin/bash
# docker run --rm -it -v ./test:/data yaoapp/yao-build:0.10.5 /bin/bash
#
# ===========================================
FROM ubuntu:24.04
@ -29,10 +29,10 @@ RUN apt-get update && \
apt-get install -y git && \
apt-get install -y unzip
# Install Go 1.23.3
RUN wget https://golang.org/dl/go1.23.3.linux-amd64.tar.gz && \
tar -C /usr/local -xzf go1.23.3.linux-amd64.tar.gz && \
rm go1.23.3.linux-amd64.tar.gz
# Install Go 1.24.3
RUN wget https://golang.org/dl/go1.24.3.linux-amd64.tar.gz && \
tar -C /usr/local -xzf go1.24.3.linux-amd64.tar.gz && \
rm go1.24.3.linux-amd64.tar.gz
# Install Node.js 18.x
RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \
@ -40,6 +40,15 @@ RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \
RUN npm install -g pnpm
# Install AWS CLI
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.7.zip" -o "awscliv2.zip" && \
unzip awscliv2.zip && \
./aws/install && \
rm -rf awscliv2.zip && \
rm -rf aws && \
aws --version
# RUN npm install -g pnpm
RUN chmod +x /app/build.sh
VOLUME [ "/data" ]

2
docker/build/build.sh
View file

@ -4,7 +4,7 @@ git clone https://github.com/yaoapp/kun.git /app/kun && \
git clone https://github.com/yaoapp/xun.git /app/xun && \
git clone https://github.com/sjzsdu/gou.git /app/gou && \
git clone https://github.com/yaoapp/v8go.git /app/v8go && \
git clone https://github.com/sjzsdu/xgen.git /app/xgen-v1.0 && \
git clone https://github.com/yaoapp/cui.git /app/cui-v1.0 && \
git clone https://github.com/yaoapp/yao-init.git /app/yao-init && \
git clone https://github.com/sjzsdu/yao.git /app/yao

2
docker/development/Dockerfile
View file

@ -20,7 +20,7 @@ ARG ARCH
RUN groupadd -r yao && useradd -r -g yao yao && \
apt-get update && \
apt-get install -y curl sudo procps net-tools
RUN curl -fsSL "https://yao.moapi.ai/archives/yao-${VERSION}-linux-${ARCH}" > /usr/local/bin/yao && \
RUN curl -fsSL "https://pub-80136338e60643edbb55c6ca8a689cf8.r2.dev/archives/yao-${VERSION}-linux-${ARCH}" > /usr/local/bin/yao && \
chmod +x /usr/local/bin/yao && \
mkdir -p /data/app

2
docker/production/Dockerfile
View file

@ -18,7 +18,7 @@ FROM alpine:latest
ARG VERSION
ARG ARCH
RUN apk --no-cache add curl
RUN curl -fsSL "https://yao.moapi.ai/archives/yao-${VERSION}-linux-${ARCH}" > /usr/local/bin/yao && \
RUN curl -fsSL "https://pub-80136338e60643edbb55c6ca8a689cf8.r2.dev/archives/yao-${VERSION}-linux-${ARCH}" > /usr/local/bin/yao && \
chmod +x /usr/local/bin/yao && \
addgroup -S yao && adduser -S -G yao yao && \
mkdir -p /data/app && \

49
dsl/api/api.go Normal file
View file

@ -0,0 +1,49 @@
package api
import (
"context"
"github.com/yaoapp/yao/dsl/types"
)
// YaoAPI is the MCP client DSL manager
type YaoAPI struct {
root string // The relative path of the MCP client DSL
fs types.IO // The file system IO interface
db types.IO // The database IO interface
}
// New returns a new connector DSL manager
func New(root string, fs types.IO, db types.IO) types.Manager {
return &YaoAPI{root: root, fs: fs, db: db}
}
// Loaded return all loaded DSLs
func (api *YaoAPI) Loaded(ctx context.Context) (map[string]*types.Info, error) {
return nil, nil
}
// Load will unload the DSL first, then load the DSL from DB or file system
func (api *YaoAPI) Load(ctx context.Context, options *types.LoadOptions) error {
return nil
}
// Reload will unload the DSL first, then reload the DSL from DB or file system
func (api *YaoAPI) Reload(ctx context.Context, options *types.ReloadOptions) error {
return nil
}
// Unload will unload the DSL from memory
func (api *YaoAPI) Unload(ctx context.Context, options *types.UnloadOptions) error {
return nil
}
// Validate will validate the DSL from source
func (api *YaoAPI) Validate(ctx context.Context, source string) (bool, []types.LintMessage) {
return false, nil
}
// Execute will execute the DSL
func (api *YaoAPI) Execute(ctx context.Context, id string, method string, args ...any) (any, error) {
return nil, nil
}

213
dsl/connector/cases_test.go Normal file
View file

@ -0,0 +1,213 @@
package connector
import (
"fmt"
"os"
"testing"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/yaoapp/gou/application"
"github.com/yaoapp/gou/model"
"github.com/yaoapp/kun/log"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/data"
"github.com/yaoapp/yao/dsl/types"
"github.com/yaoapp/yao/test"
)
// systemModels system models
var systemModels = map[string]string{
"__yao.dsl": "yao/models/dsl.mod.yao",
}
func TestMain(m *testing.M) {
// Setup
test.Prepare(&testing.T{}, config.Conf)
defer test.Clean()
// Load system models
model.WithCrypt([]byte(fmt.Sprintf(`{"key":"%s"}`, config.Conf.DB.AESKey)), "AES")
model.WithCrypt([]byte(`{}`), "PASSWORD")
err := loadSystemModels()
if err != nil {
log.Error("Load system models error: %s", err.Error())
os.Exit(1)
}
// Load application
root := os.Getenv("GOU_TEST_APPLICATION")
app, err := application.OpenFromDisk(root) // Load app
if err != nil {
log.Error("Load application error: %s", err.Error())
os.Exit(1)
}
application.Load(app)
// Run tests
code := m.Run()
os.Exit(code)
}
// loadSystemModels load system models
func loadSystemModels() error {
for id, path := range systemModels {
content, err := data.Read(path)
if err != nil {
return err
}
// Parse model
var data map[string]interface{}
err = application.Parse(path, content, &data)
if err != nil {
return err
}
// Set prefix
if table, ok := data["table"].(map[string]interface{}); ok {
if name, ok := table["name"].(string); ok {
table["name"] = "__yao_" + name
content, err = jsoniter.Marshal(data)
if err != nil {
log.Error("failed to marshal model data: %v", err)
return fmt.Errorf("failed to marshal model data: %v", err)
}
}
}
// Load Model
mod, err := model.LoadSource(content, id, path)
if err != nil {
log.Error("load system model %s error: %s", id, err.Error())
return err
}
// Drop table first
err = mod.DropTable()
if err != nil {
log.Error("drop table error: %s", err.Error())
return err
}
// Auto migrate
err = mod.Migrate(false, model.WithDonotInsertValues(true))
if err != nil {
log.Error("migrate system model %s error: %s", id, err.Error())
return err
}
}
return nil
}
// TestCase defines a single test case
type TestCase struct {
ID string
Source string
UpdatedSource string
Tags []string
Label string
Description string
}
// NewTestCase creates a new test case
func NewTestCase() *TestCase {
id := getTestID()
return &TestCase{
ID: id,
Source: fmt.Sprintf(`{
"label": "Test OpenAI",
"description": "Test Description",
"tags": ["test_%s"],
"type": "openai",
"options": {
"proxy": "https://api.openai.com/v1",
"model": "gpt-4o-mini",
"key": "sk-test-key"
}
}`, id),
UpdatedSource: fmt.Sprintf(`{
"label": "Updated OpenAI",
"description": "Updated Description",
"tags": ["test_%s", "updated"],
"type": "openai",
"options": {
"proxy": "https://api.openai.com/v1",
"model": "gpt-4o-mini",
"key": "sk-test-key"
}
}`, id),
Tags: []string{fmt.Sprintf("test_%s", id)},
Label: "Test OpenAI",
Description: "Test Description",
}
}
// getTestID generates a unique test ID
func getTestID() string {
return fmt.Sprintf("test_%d", time.Now().UnixNano())
}
// CreateOptions returns creation options
func (tc *TestCase) CreateOptions() *types.CreateOptions {
return &types.CreateOptions{
ID: tc.ID,
Source: tc.Source,
}
}
// LoadOptions returns load options
func (tc *TestCase) LoadOptions() *types.LoadOptions {
return &types.LoadOptions{
ID: tc.ID,
Source: tc.Source,
}
}
// UnloadOptions returns unload options
func (tc *TestCase) UnloadOptions() *types.UnloadOptions {
return &types.UnloadOptions{
ID: tc.ID,
}
}
// ReloadOptions returns reload options
func (tc *TestCase) ReloadOptions() *types.ReloadOptions {
return &types.ReloadOptions{
ID: tc.ID,
Source: tc.UpdatedSource,
}
}
// AssertInfo verifies if the information is correct
func (tc *TestCase) AssertInfo(info *types.Info) bool {
if info == nil {
return false
}
return info.ID == tc.ID &&
info.Type == types.TypeConnector &&
info.Label == tc.Label &&
len(info.Tags) == len(tc.Tags) &&
info.Description == tc.Description &&
!info.Readonly &&
!info.Builtin &&
!info.Mtime.IsZero() &&
!info.Ctime.IsZero()
}
// AssertUpdatedInfo verifies if the updated information is correct
func (tc *TestCase) AssertUpdatedInfo(info *types.Info) bool {
if info == nil {
return false
}
return info.ID == tc.ID &&
info.Type == types.TypeConnector &&
info.Label == "Updated OpenAI" &&
len(info.Tags) == 2 &&
info.Description == "Updated Description" &&
!info.Readonly &&
!info.Builtin &&
!info.Mtime.IsZero() &&
!info.Ctime.IsZero()
}

175
dsl/connector/connector.go Normal file
View file

@ -0,0 +1,175 @@
package connector
import (
"context"
"fmt"
"github.com/yaoapp/gou/connector"
"github.com/yaoapp/yao/dsl/types"
)
// YaoConnector is the connector DSL manager
type YaoConnector struct {
root string // The relative path of the connector DSL
fs types.IO // The file system IO interface
db types.IO // The database IO interface
}
// New returns a new connector DSL manager
func New(root string, fs types.IO, db types.IO) types.Manager {
return &YaoConnector{root: root, fs: fs, db: db}
}
// Loaded return all loaded DSLs
func (c *YaoConnector) Loaded(ctx context.Context) (map[string]*types.Info, error) {
infos := map[string]*types.Info{}
for id, conn := range connector.Connectors {
meta := conn.GetMetaInfo()
infos[id] = &types.Info{
ID: id,
Path: conn.ID(),
Type: types.TypeConnector,
Label: meta.Label,
Sort: meta.Sort,
Description: meta.Description,
Tags: meta.Tags,
Readonly: meta.Readonly,
Builtin: meta.Builtin,
Mtime: meta.Mtime,
Ctime: meta.Ctime,
}
}
return infos, nil
}
// Load will unload the DSL first, then load the DSL from DB or file system
func (c *YaoConnector) Load(ctx context.Context, options *types.LoadOptions) error {
if options == nil {
return fmt.Errorf("load options is required")
}
if options.ID == "" {
return fmt.Errorf("load options id is required")
}
var err error
// Case 1: If Source is provided, use LoadSourceSync
if options.Source != "" {
connectorPath := types.ToPath(types.TypeConnector, options.ID)
_, err = connector.LoadSourceSync([]byte(options.Source), options.ID, connectorPath)
if err != nil {
return err
}
} else if options.Path != "" && options.Store == "fs" {
// Case 2: If Path is provided and Store is fs, use LoadSync with Path
_, err = connector.LoadSync(options.Path, options.ID)
if err != nil {
return err
}
} else if options.Store == "db" {
// Case 3: If Store is db, get Source from DB first
if c.db == nil {
return fmt.Errorf("db io is required for store type db")
}
source, exists, err := c.db.Source(options.ID)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("connector %s not found in database", options.ID)
}
connectorPath := types.ToPath(types.TypeConnector, options.ID)
_, err = connector.LoadSourceSync([]byte(source), options.ID, connectorPath)
if err != nil {
return err
}
} else {
// Case 4: Default case, use LoadSync with ID
path := types.ToPath(types.TypeConnector, options.ID)
_, err = connector.LoadSync(path, options.ID)
if err != nil {
return err
}
}
return nil
}
// Unload will unload the DSL from memory
func (c *YaoConnector) Unload(ctx context.Context, options *types.UnloadOptions) error {
if options == nil {
return fmt.Errorf("unload options is required")
}
if options.ID == "" {
return fmt.Errorf("unload options id is required")
}
return connector.Remove(options.ID)
}
// Reload will unload the DSL first, then reload the DSL from DB or file system
func (c *YaoConnector) Reload(ctx context.Context, options *types.ReloadOptions) error {
if options == nil {
return fmt.Errorf("reload options is required")
}
if options.ID == "" {
return fmt.Errorf("reload options id is required")
}
// First unload
err := connector.Remove(options.ID)
if err != nil {
return err
}
// Then load
if options.Source != "" {
connectorPath := types.ToPath(types.TypeConnector, options.ID)
_, err = connector.LoadSourceSync([]byte(options.Source), options.ID, connectorPath)
if err != nil {
return err
}
} else if options.Path != "" && options.Store == "fs" {
_, err = connector.LoadSync(options.Path, options.ID)
if err != nil {
return err
}
} else if options.Store == "db" {
if c.db == nil {
return fmt.Errorf("db io is required for store type db")
}
source, exists, err := c.db.Source(options.ID)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("connector %s not found in database", options.ID)
}
connectorPath := types.ToPath(types.TypeConnector, options.ID)
_, err = connector.LoadSourceSync([]byte(source), options.ID, connectorPath)
if err != nil {
return err
}
} else {
path := types.ToPath(types.TypeConnector, options.ID)
_, err = connector.LoadSync(path, options.ID)
if err != nil {
return err
}
}
return nil
}
// Validate will validate the DSL from source
func (c *YaoConnector) Validate(ctx context.Context, source string) (bool, []types.LintMessage) {
return true, []types.LintMessage{}
}
// Execute will execute the DSL
func (c *YaoConnector) Execute(ctx context.Context, id string, method string, args ...any) (any, error) {
return nil, fmt.Errorf("Not implemented")
}

183
dsl/connector/connector_test.go Normal file
View file

@ -0,0 +1,183 @@
package connector
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/yaoapp/yao/dsl/io"
"github.com/yaoapp/yao/dsl/types"
)
func TestConnectorLoad(t *testing.T) {
testCase := NewTestCase()
fsio := io.NewFS(types.TypeConnector)
dbio := io.NewDB(types.TypeConnector)
manager := New("", fsio, dbio)
// Test Load with nil options
err := manager.Load(context.Background(), nil)
assert.Error(t, err)
assert.Contains(t, err.Error(), "load options is required")
// Test Load with empty ID
err = manager.Load(context.Background(), &types.LoadOptions{})
assert.Error(t, err)
assert.Contains(t, err.Error(), "load options id is required")
// Test Load with Source
err = manager.Load(context.Background(), testCase.LoadOptions())
assert.NoError(t, err)
// Test Load from filesystem
err = fsio.Create(testCase.CreateOptions())
assert.NoError(t, err)
path := types.ToPath(types.TypeConnector, testCase.ID)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID,
Path: path,
Store: "fs",
})
assert.NoError(t, err)
// Test Load from database
err = dbio.Create(testCase.CreateOptions())
assert.NoError(t, err)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID,
Store: "db",
})
assert.NoError(t, err)
// Clean up
err = fsio.Delete(testCase.ID)
assert.NoError(t, err)
err = dbio.Delete(testCase.ID)
assert.NoError(t, err)
}
func TestConnectorUnload(t *testing.T) {
testCase := NewTestCase()
fsio := io.NewFS(types.TypeConnector)
dbio := io.NewDB(types.TypeConnector)
manager := New("", fsio, dbio)
// Test Unload with nil options
err := manager.Unload(context.Background(), nil)
assert.Error(t, err)
assert.Contains(t, err.Error(), "unload options is required")
// Test Unload with empty ID
err = manager.Unload(context.Background(), &types.UnloadOptions{})
assert.Error(t, err)
assert.Contains(t, err.Error(), "unload options id is required")
// Load and then unload from filesystem
err = fsio.Create(testCase.CreateOptions())
assert.NoError(t, err)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID,
Store: "fs",
})
assert.NoError(t, err)
err = manager.Unload(context.Background(), testCase.UnloadOptions())
assert.NoError(t, err)
// Clean up
err = fsio.Delete(testCase.ID)
assert.NoError(t, err)
}
func TestConnectorReload(t *testing.T) {
testCase := NewTestCase()
fsio := io.NewFS(types.TypeConnector)
dbio := io.NewDB(types.TypeConnector)
manager := New("", fsio, dbio)
// Test Reload with nil options
err := manager.Reload(context.Background(), nil)
assert.Error(t, err)
assert.Contains(t, err.Error(), "reload options is required")
// Test Reload with empty ID
err = manager.Reload(context.Background(), &types.ReloadOptions{})
assert.Error(t, err)
assert.Contains(t, err.Error(), "reload options id is required")
// Load and then reload from filesystem
err = fsio.Create(testCase.CreateOptions())
assert.NoError(t, err)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID,
Store: "fs",
})
assert.NoError(t, err)
err = manager.Reload(context.Background(), testCase.ReloadOptions())
assert.NoError(t, err)
// Clean up
err = fsio.Delete(testCase.ID)
assert.NoError(t, err)
}
func TestConnectorLoaded(t *testing.T) {
testCase := NewTestCase()
fsio := io.NewFS(types.TypeConnector)
dbio := io.NewDB(types.TypeConnector)
manager := New("", fsio, dbio)
// Load from filesystem
err := fsio.Create(testCase.CreateOptions())
assert.NoError(t, err)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID,
Store: "fs",
})
assert.NoError(t, err)
// Test Loaded
infos, err := manager.Loaded(context.Background())
assert.NoError(t, err)
assert.NotNil(t, infos)
assert.Contains(t, infos, testCase.ID)
// Verify metadata fields
fsInfo := infos[testCase.ID]
assert.Equal(t, testCase.ID, fsInfo.ID)
assert.Equal(t, types.TypeConnector, fsInfo.Type)
assert.Equal(t, testCase.Label, fsInfo.Label)
assert.Equal(t, testCase.Description, fsInfo.Description)
assert.ElementsMatch(t, testCase.Tags, fsInfo.Tags)
assert.False(t, fsInfo.Readonly)
assert.False(t, fsInfo.Builtin)
// Clean up
err = fsio.Delete(testCase.ID)
assert.NoError(t, err)
}
func TestConnectorValidate(t *testing.T) {
manager := New("", nil, nil)
// Test Validate
valid, messages := manager.Validate(context.Background(), "test source")
assert.True(t, valid)
assert.Empty(t, messages)
}
func TestConnectorExecute(t *testing.T) {
manager := New("", nil, nil)
// Test Execute
result, err := manager.Execute(context.Background(), "test_id", "test_method")
assert.Error(t, err)
assert.Contains(t, err.Error(), "Not implemented")
assert.Nil(t, result)
}

399
dsl/dsl.go Normal file
View file

@ -0,0 +1,399 @@
package dsl
import (
"context"
"fmt"
"github.com/yaoapp/yao/dsl/api"
"github.com/yaoapp/yao/dsl/connector"
"github.com/yaoapp/yao/dsl/io"
"github.com/yaoapp/yao/dsl/mcp"
"github.com/yaoapp/yao/dsl/model"
"github.com/yaoapp/yao/dsl/types"
)
// DSL is the base DSL struct
type DSL struct {
Type types.Type
exts []string
root string
manager types.Manager
db types.IO
fs types.IO
}
// New returns a new DSL manager
func New(typ types.Type) (types.DSL, error) {
var manager types.Manager
var db types.IO = io.NewDB(typ)
var fs types.IO = io.NewFS(typ)
// Get the root path and the extensions of the type
root, exts := types.TypeRootAndExts(typ)
// Create the manager
switch typ {
case types.TypeConnector:
exts = []string{".conn.yao", ".conn.jsonc", ".conn.json"}
manager = connector.New(root, fs, db)
case types.TypeModel:
exts = []string{".mod.yao", ".mod.jsonc", ".mod.json"}
manager = model.New(root, fs, db)
case types.TypeMCPClient:
exts = []string{".mcp.yao", ".mcp.jsonc", ".mcp.json"}
manager = mcp.NewClient(root, fs, db)
// case types.TypeMCPServer:
// exts = []string{".mcp.yao", ".mcp.jsonc", ".mcp.json"}
// manager = mcp.NewServer(root)
case types.TypeAPI:
exts = []string{".http.yao", ".http.jsonc", ".http.json"}
manager = api.New(root, fs, db)
default:
return nil, fmt.Errorf("dsl manager is not initialized, %s not supported", typ)
}
return &DSL{Type: typ, manager: manager, root: root, exts: exts, db: db, fs: fs}, nil
}
// Inspect DSL
func (dsl *DSL) Inspect(ctx context.Context, id string) (*types.Info, error) {
// Get the info from the db
info, exists, err := dsl.db.Inspect(id)
if err != nil {
return nil, err
}
if !exists {
// Get the info from the file
info, exists, err = dsl.fs.Inspect(id)
if err != nil {
return nil, err
}
if !exists {
return nil, fmt.Errorf("%s not found, %s", dsl.Type, id)
}
}
// Merge the status from the manager
loaded, err := dsl.manager.Loaded(ctx)
if err != nil {
fmt.Printf("DEBUG: manager.Loaded failed: %v\n", err)
return info, err
}
// Check if the DSL is loaded
if _, ok := loaded[id]; ok {
info.Status = types.StatusLoaded
}
return info, nil
}
// Path Get Path by id, ( If the DSL is saved as file, return the file path )
func (dsl *DSL) Path(ctx context.Context, id string) (string, error) {
return types.ToPath(dsl.Type, id), nil
}
// Source Get Source by id
func (dsl *DSL) Source(ctx context.Context, id string) (string, error) {
// Get the source from the db
source, exists, err := dsl.db.Source(id)
if err != nil {
return "", err
}
if !exists {
// Get the source from the file
source, exists, err = dsl.fs.Source(id)
if err != nil {
return "", err
}
if !exists {
return "", fmt.Errorf("%s DSL not found, %s", dsl.Type, id)
}
}
return source, nil
}
// List DSLs
func (dsl *DSL) List(ctx context.Context, opts *types.ListOptions) ([]*types.Info, error) {
// Get the list from the db
var dbList []*types.Info
var fileList []*types.Info
var err error
// If StoreType is not specified or is DB, get from db
if opts.Store == "" || opts.Store == types.StoreTypeDB {
dbList, err = dsl.db.List(opts)
if err != nil {
return nil, err
}
}
// If StoreType is not specified or is File, get from file
if opts.Store == "" || opts.Store == types.StoreTypeFile {
fileList, err = dsl.fs.List(opts)
if err != nil {
return nil, err
}
}
// Merge the list and unique
list := []*types.Info{}
unique := make(map[string]bool)
for _, info := range dbList {
if _, ok := unique[info.ID]; !ok {
list = append(list, info)
unique[info.ID] = true
}
}
for _, info := range fileList {
if _, ok := unique[info.ID]; !ok {
list = append(list, info)
unique[info.ID] = true
}
}
// Merge the status from the manager
loaded, err := dsl.manager.Loaded(ctx)
if err != nil {
return nil, err
}
// Merge the status from the manager
for _, info := range list {
if _, ok := loaded[info.ID]; ok {
info.Status = types.StatusLoaded
}
}
return list, nil
}
// Create DSL
func (dsl *DSL) Create(ctx context.Context, options *types.CreateOptions) error {
if options == nil {
return fmt.Errorf("create options is required")
}
// Set default store type if not specified
if options.Store == "" {
options.Store = types.StoreTypeFile
}
// Validate store type
if options.Store != types.StoreTypeDB && options.Store != types.StoreTypeFile {
return fmt.Errorf("invalid store type: %s", options.Store)
}
if options.Store == types.StoreTypeDB {
err := dsl.db.Create(options)
if err != nil {
return err
}
} else if options.Store == types.StoreTypeFile {
err := dsl.fs.Create(options)
if err != nil {
return err
}
}
var loadOptions *types.LoadOptions = &types.LoadOptions{
ID: options.ID,
Path: types.ToPath(dsl.Type, options.ID),
Source: options.Source,
Store: options.Store,
Options: options.Load,
}
// Load the DSL
err := dsl.Load(ctx, loadOptions)
if err != nil {
return err
}
return nil
}
// Exists Check if the DSL exists
func (dsl *DSL) Exists(ctx context.Context, id string) (bool, error) {
// Check if the DSL exists in the db
exists, err := dsl.db.Exists(id)
if err != nil {
return false, err
}
if exists {
return true, nil
}
// Check if the DSL exists in the file
return dsl.fs.Exists(id)
}
// Update DSL
func (dsl *DSL) Update(ctx context.Context, options *types.UpdateOptions) error {
if options == nil {
return fmt.Errorf("update options is required")
}
// Exists
info, exists, err := dsl.db.Inspect(options.ID)
if err != nil {
return err
}
if !exists {
info, exists, err = dsl.fs.Inspect(options.ID)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("%s not found, %s", dsl.Type, options.ID)
}
// Fix: If store is empty but found in fs, it should be File store
if info.Store == "" {
info.Store = types.StoreTypeFile
}
} else {
// Fix: If store is empty but found in db, it should be DB store
if info.Store == "" {
info.Store = types.StoreTypeDB
}
}
// Create the reload options
var reloadOptions *types.ReloadOptions = &types.ReloadOptions{
ID: options.ID,
Path: info.Path,
Source: options.Source,
Store: info.Store,
Options: options.Reload,
}
// Update the DSL in the db
if info.Store == types.StoreTypeDB {
err := dsl.db.Update(options)
if err != nil {
return err
}
// Reload the DSL
return dsl.manager.Reload(ctx, reloadOptions)
}
// Update the DSL in the file
err = dsl.fs.Update(options)
if err != nil {
return err
}
// Reload the DSL
return dsl.manager.Reload(ctx, reloadOptions)
}
// Delete DSL
func (dsl *DSL) Delete(ctx context.Context, options *types.DeleteOptions) error {
if options == nil {
return fmt.Errorf("delete options is required")
}
if options.ID == "" {
return fmt.Errorf("delete options id is required")
}
// Exists
info, exists, err := dsl.db.Inspect(options.ID)
if err != nil {
return err
}
if !exists {
info, exists, err = dsl.fs.Inspect(options.ID)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("%s not found, %s", dsl.Type, options.ID)
} else {
// Fix: If store is empty but found in fs, it should be File store
if info.Store == "" {
info.Store = types.StoreTypeFile
}
}
} else {
// Fix: If store is empty but found in db, it should be DB store
if info.Store == "" {
info.Store = types.StoreTypeDB
}
}
var opts map[string]interface{}
if options.Options != nil {
opts = options.Options
}
var unloadOptions *types.UnloadOptions = &types.UnloadOptions{
ID: options.ID,
Path: info.Path,
Store: info.Store,
Options: opts,
}
if info.Store == types.StoreTypeDB {
err = dsl.db.Delete(options.ID)
if err != nil {
return err
}
// Unload the DSL
return dsl.manager.Unload(ctx, unloadOptions)
}
err = dsl.fs.Delete(options.ID)
if err != nil {
return err
}
// Unload the DSL
return dsl.manager.Unload(ctx, unloadOptions)
}
// Load DSL
func (dsl *DSL) Load(ctx context.Context, options *types.LoadOptions) error {
return dsl.manager.Load(ctx, options)
}
// Unload DSL
func (dsl *DSL) Unload(ctx context.Context, options *types.UnloadOptions) error {
return dsl.manager.Unload(ctx, options)
}
// Reload DSL
func (dsl *DSL) Reload(ctx context.Context, options *types.ReloadOptions) error {
return dsl.manager.Reload(ctx, options)
}
// Execute DSL (Some DSLs can be executed)
func (dsl *DSL) Execute(ctx context.Context, id string, method string, args ...any) (any, error) {
return dsl.manager.Execute(ctx, id, method, args...)
}
// Validate DSL
func (dsl *DSL) Validate(ctx context.Context, source string) (bool, []types.LintMessage) {
return dsl.manager.Validate(ctx, source)
}

784
dsl/dsl_test.go Normal file
View file

@ -0,0 +1,784 @@
package dsl
import (
"context"
"fmt"
"os"
"testing"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/stretchr/testify/assert"
"github.com/yaoapp/gou/application"
"github.com/yaoapp/gou/model"
"github.com/yaoapp/kun/log"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/data"
"github.com/yaoapp/yao/dsl/types"
"github.com/yaoapp/yao/test"
)
// systemModels system models
var systemModels = map[string]string{
"__yao.dsl": "yao/models/dsl.mod.yao",
}
func TestMain(m *testing.M) {
// Setup
test.Prepare(&testing.T{}, config.Conf)
defer test.Clean()
// Load system models
model.WithCrypt([]byte(fmt.Sprintf(`{"key":"%s"}`, config.Conf.DB.AESKey)), "AES")
model.WithCrypt([]byte(`{}`), "PASSWORD")
err := loadSystemModels()
if err != nil {
log.Error("Load system models error: %s", err.Error())
os.Exit(1)
}
// Load application
root := os.Getenv("YAO_TEST_APPLICATION")
if root == "" {
log.Error("YAO_TEST_APPLICATION environment variable is not set")
os.Exit(1)
}
app, err := application.OpenFromDisk(root) // Load app
if err != nil {
log.Error("Load application error: %s", err.Error())
os.Exit(1)
}
application.Load(app)
// Run tests
code := m.Run()
os.Exit(code)
}
// loadSystemModels load system models
func loadSystemModels() error {
for id, path := range systemModels {
content, err := data.Read(path)
if err != nil {
return err
}
// Parse model
var data map[string]interface{}
err = application.Parse(path, content, &data)
if err != nil {
return err
}
// Set prefix
if table, ok := data["table"].(map[string]interface{}); ok {
if name, ok := table["name"].(string); ok {
table["name"] = "__yao_" + name
content, err = jsoniter.Marshal(data)
if err != nil {
log.Error("failed to marshal model data: %v", err)
return fmt.Errorf("failed to marshal model data: %v", err)
}
}
}
// Load Model
mod, err := model.LoadSource(content, id, path)
if err != nil {
log.Error("load system model %s error: %s", id, err.Error())
return err
}
// Drop table first
err = mod.DropTable()
if err != nil {
log.Error("drop table error: %s", err.Error())
return err
}
// Auto migrate
err = mod.Migrate(false, model.WithDonotInsertValues(true))
if err != nil {
log.Error("migrate system model %s error: %s", id, err.Error())
return err
}
}
return nil
}
// cleanTestData cleans test data from database
func cleanTestData() error {
m := model.Select("__yao.dsl")
err := m.DropTable()
if err != nil {
return err
}
err = m.Migrate(false, model.WithDonotInsertValues(true))
if err != nil {
return err
}
return nil
}
// getTestID generates a unique test ID
func getTestID() string {
return fmt.Sprintf("test_%d", time.Now().UnixNano())
}
// TestCase defines a unified test case for all DSL types
type TestCase struct {
ID string
Source string
UpdatedSource string
Tags []string
Label string
Description string
DSLType types.Type
}
// NewModelTestCase creates a new model test case
func NewModelTestCase() *TestCase {
id := getTestID()
return &TestCase{
ID: id,
DSLType: types.TypeModel,
Source: fmt.Sprintf(`{
"name": "%s",
"table": { "name": "%s", "comment": "Test User" },
"columns": [
{ "name": "id", "type": "ID" },
{ "name": "name", "type": "string", "length": 80, "comment": "User Name", "index": true },
{ "name": "status", "type": "enum", "option": ["active", "disabled"], "default": "active", "comment": "Status", "index": true }
],
"tags": ["test_%s"],
"label": "Test Model",
"description": "Test Model Description",
"option": { "timestamps": true, "soft_deletes": true }
}`, id, id, id),
UpdatedSource: fmt.Sprintf(`{
"name": "%s",
"table": { "name": "%s", "comment": "Updated Test User" },
"columns": [
{ "name": "id", "type": "ID" },
{ "name": "name", "type": "string", "length": 80, "comment": "User Name", "index": true },
{ "name": "status", "type": "enum", "option": ["active", "disabled", "pending"], "default": "active", "comment": "Status", "index": true }
],
"tags": ["test_%s", "updated"],
"label": "Updated Model",
"description": "Updated Model Description",
"option": { "timestamps": true, "soft_deletes": true }
}`, id, id, id),
Tags: []string{fmt.Sprintf("test_%s", id)},
Label: "Test Model",
Description: "Test Model Description",
}
}
// NewConnectorTestCase creates a new connector test case
func NewConnectorTestCase() *TestCase {
id := getTestID()
return &TestCase{
ID: id,
DSLType: types.TypeConnector,
Source: fmt.Sprintf(`{
"label": "Test Connector",
"description": "Test Connector Description",
"tags": ["test_%s"],
"type": "openai",
"options": {
"proxy": "https://api.openai.com/v1",
"model": "gpt-4o-mini",
"key": "sk-test-key"
}
}`, id),
UpdatedSource: fmt.Sprintf(`{
"label": "Updated Connector",
"description": "Updated Connector Description",
"tags": ["test_%s", "updated"],
"type": "openai",
"options": {
"proxy": "https://api.openai.com/v1",
"model": "gpt-4o-mini",
"key": "sk-test-key"
}
}`, id),
Tags: []string{fmt.Sprintf("test_%s", id)},
Label: "Test Connector",
Description: "Test Connector Description",
}
}
// NewMCPTestCase creates a new MCP test case
func NewMCPTestCase() *TestCase {
id := getTestID()
return &TestCase{
ID: id,
DSLType: types.TypeMCPClient,
Source: fmt.Sprintf(`{
"name": "Test MCP Client %s",
"label": "Test MCP Client",
"description": "Test MCP Client Description",
"tags": ["test_%s"],
"transport": "stdio",
"command": "echo",
"arguments": ["hello", "world"],
"env": {
"MCP_TEST": "true"
},
"enable_sampling": true,
"enable_roots": false,
"timeout": "30s"
}`, id, id),
UpdatedSource: fmt.Sprintf(`{
"name": "Updated MCP Client %s",
"label": "Updated MCP Client",
"description": "Updated MCP Client Description",
"tags": ["test_%s", "updated"],
"transport": "stdio",
"command": "echo",
"arguments": ["hello", "updated"],
"env": {
"MCP_TEST": "true",
"MCP_UPDATED": "true"
},
"enable_sampling": false,
"enable_roots": true,
"timeout": "60s"
}`, id, id),
Tags: []string{fmt.Sprintf("test_%s", id)},
Label: "Test MCP Client",
Description: "Test MCP Client Description",
}
}
// CreateOptions returns creation options
func (tc *TestCase) CreateOptions(store types.StoreType) *types.CreateOptions {
return &types.CreateOptions{
ID: tc.ID,
Source: tc.Source,
Store: store,
}
}
// UpdateOptions returns update options
func (tc *TestCase) UpdateOptions() *types.UpdateOptions {
return &types.UpdateOptions{
ID: tc.ID,
Source: tc.UpdatedSource,
}
}
// DeleteOptions returns delete options
func (tc *TestCase) DeleteOptions() *types.DeleteOptions {
return &types.DeleteOptions{
ID: tc.ID,
}
}
// LoadOptions returns load options
func (tc *TestCase) LoadOptions(store types.StoreType) *types.LoadOptions {
return &types.LoadOptions{
ID: tc.ID,
Source: tc.Source,
Store: store,
}
}
// UnloadOptions returns unload options
func (tc *TestCase) UnloadOptions(store types.StoreType) *types.UnloadOptions {
return &types.UnloadOptions{
ID: tc.ID,
Store: store,
}
}
// ReloadOptions returns reload options
func (tc *TestCase) ReloadOptions(store types.StoreType) *types.ReloadOptions {
return &types.ReloadOptions{
ID: tc.ID,
Source: tc.UpdatedSource,
Store: store,
}
}
// ListOptions returns list options
func (tc *TestCase) ListOptions(store types.StoreType) *types.ListOptions {
return &types.ListOptions{
Tags: tc.Tags,
Store: store,
}
}
// AssertInfo verifies if the information is correct
func (tc *TestCase) AssertInfo(info *types.Info) bool {
if info == nil {
return false
}
return info.ID == tc.ID &&
info.Type == tc.DSLType &&
info.Label == tc.Label &&
len(info.Tags) == len(tc.Tags) &&
info.Description == tc.Description &&
!info.Readonly &&
!info.Builtin &&
!info.Mtime.IsZero() &&
!info.Ctime.IsZero()
}
// AssertUpdatedInfo verifies if the updated information is correct
func (tc *TestCase) AssertUpdatedInfo(info *types.Info) bool {
if info == nil {
return false
}
expectedLabel := ""
switch tc.DSLType {
case types.TypeModel:
expectedLabel = "Updated Model"
case types.TypeConnector:
expectedLabel = "Updated Connector"
case types.TypeMCPClient:
expectedLabel = "Updated MCP Client"
}
expectedDescription := ""
switch tc.DSLType {
case types.TypeModel:
expectedDescription = "Updated Model Description"
case types.TypeConnector:
expectedDescription = "Updated Connector Description"
case types.TypeMCPClient:
expectedDescription = "Updated MCP Client Description"
}
return info.ID == tc.ID &&
info.Type == tc.DSLType &&
info.Label == expectedLabel &&
len(info.Tags) == 2 &&
info.Description == expectedDescription &&
!info.Readonly &&
!info.Builtin &&
!info.Mtime.IsZero() &&
!info.Ctime.IsZero()
}
// Test DSL creation with different types and stores
func TestDSLCreate(t *testing.T) {
ctx := context.Background()
testCases := []struct {
name string
tcFunc func() *TestCase
dslType types.Type
stores []types.StoreType
}{
{"Model", NewModelTestCase, types.TypeModel, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
{"Connector", NewConnectorTestCase, types.TypeConnector, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
{"MCP", NewMCPTestCase, types.TypeMCPClient, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
}
for _, tt := range testCases {
for _, store := range tt.stores {
t.Run(fmt.Sprintf("%s_%s", tt.name, store), func(t *testing.T) {
// Clean test data before each test
err := cleanTestData()
if err != nil {
t.Fatalf("Failed to clean test data: %v", err)
}
dsl, err := New(tt.dslType)
if !assert.Nil(t, err) {
return
}
tc := tt.tcFunc()
// Create
err = dsl.Create(ctx, tc.CreateOptions(store))
if !assert.Nil(t, err) {
return
}
// Verify exists
exists, err := dsl.Exists(ctx, tc.ID)
if !assert.Nil(t, err) {
return
}
assert.True(t, exists)
// Verify info
info, err := dsl.Inspect(ctx, tc.ID)
if !assert.Nil(t, err) {
return
}
assert.True(t, tc.AssertInfo(info))
// Cleanup
err = dsl.Delete(ctx, tc.DeleteOptions())
assert.Nil(t, err)
})
}
}
}
// Test DSL inspection
func TestDSLInspect(t *testing.T) {
ctx := context.Background()
testCases := []struct {
name string
tcFunc func() *TestCase
dslType types.Type
stores []types.StoreType
}{
{"Model", NewModelTestCase, types.TypeModel, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
{"Connector", NewConnectorTestCase, types.TypeConnector, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
{"MCP", NewMCPTestCase, types.TypeMCPClient, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
}
for _, tt := range testCases {
for _, store := range tt.stores {
t.Run(fmt.Sprintf("%s_%s", tt.name, store), func(t *testing.T) {
// Clean test data before each test
err := cleanTestData()
if err != nil {
t.Fatalf("Failed to clean test data: %v", err)
}
dsl, err := New(tt.dslType)
if !assert.Nil(t, err) {
return
}
tc := tt.tcFunc()
// Create
err = dsl.Create(ctx, tc.CreateOptions(store))
if !assert.Nil(t, err) {
return
}
// Inspect
info, err := dsl.Inspect(ctx, tc.ID)
if !assert.Nil(t, err) {
return
}
assert.True(t, tc.AssertInfo(info))
// Cleanup
err = dsl.Delete(ctx, tc.DeleteOptions())
assert.Nil(t, err)
})
}
}
}
// Test DSL source retrieval
func TestDSLSource(t *testing.T) {
ctx := context.Background()
testCases := []struct {
name string
tcFunc func() *TestCase
dslType types.Type
stores []types.StoreType
}{
{"Model", NewModelTestCase, types.TypeModel, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
{"Connector", NewConnectorTestCase, types.TypeConnector, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
{"MCP", NewMCPTestCase, types.TypeMCPClient, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
}
for _, tt := range testCases {
for _, store := range tt.stores {
t.Run(fmt.Sprintf("%s_%s", tt.name, store), func(t *testing.T) {
// Clean test data before each test
err := cleanTestData()
if err != nil {
t.Fatalf("Failed to clean test data: %v", err)
}
dsl, err := New(tt.dslType)
if !assert.Nil(t, err) {
return
}
tc := tt.tcFunc()
// Create
err = dsl.Create(ctx, tc.CreateOptions(store))
if !assert.Nil(t, err) {
return
}
// Get source
source, err := dsl.Source(ctx, tc.ID)
if !assert.Nil(t, err) {
return
}
assert.Equal(t, tc.Source, source)
// Cleanup
err = dsl.Delete(ctx, tc.DeleteOptions())
assert.Nil(t, err)
})
}
}
}
// Test DSL listing
func TestDSLList(t *testing.T) {
ctx := context.Background()
testCases := []struct {
name string
tcFunc func() *TestCase
dslType types.Type
stores []types.StoreType
}{
{"Model", NewModelTestCase, types.TypeModel, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
{"Connector", NewConnectorTestCase, types.TypeConnector, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
{"MCP", NewMCPTestCase, types.TypeMCPClient, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
}
for _, tt := range testCases {
for _, store := range tt.stores {
t.Run(fmt.Sprintf("%s_%s", tt.name, store), func(t *testing.T) {
// Clean test data before each test
err := cleanTestData()
if err != nil {
t.Fatalf("Failed to clean test data: %v", err)
}
dsl, err := New(tt.dslType)
if !assert.Nil(t, err) {
return
}
tc1 := tt.tcFunc()
tc2 := tt.tcFunc()
// Create test cases
err = dsl.Create(ctx, tc1.CreateOptions(store))
if !assert.Nil(t, err) {
return
}
err = dsl.Create(ctx, tc2.CreateOptions(store))
if !assert.Nil(t, err) {
return
}
// List all
list, err := dsl.List(ctx, &types.ListOptions{Store: store})
if !assert.Nil(t, err) {
return
}
assert.GreaterOrEqual(t, len(list), 2)
// List with tags
list, err = dsl.List(ctx, tc1.ListOptions(store))
if !assert.Nil(t, err) {
return
}
assert.GreaterOrEqual(t, len(list), 1)
// Cleanup
err = dsl.Delete(ctx, tc1.DeleteOptions())
assert.Nil(t, err)
err = dsl.Delete(ctx, tc2.DeleteOptions())
assert.Nil(t, err)
})
}
}
}
// Test DSL update
func TestDSLUpdate(t *testing.T) {
ctx := context.Background()
testCases := []struct {
name string
tcFunc func() *TestCase
dslType types.Type
stores []types.StoreType
}{
{"Model", NewModelTestCase, types.TypeModel, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
{"Connector", NewConnectorTestCase, types.TypeConnector, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
{"MCP", NewMCPTestCase, types.TypeMCPClient, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
}
for _, tt := range testCases {
for _, store := range tt.stores {
t.Run(fmt.Sprintf("%s_%s", tt.name, store), func(t *testing.T) {
// Clean test data before each test
err := cleanTestData()
if err != nil {
t.Fatalf("Failed to clean test data: %v", err)
}
dsl, err := New(tt.dslType)
if !assert.Nil(t, err) {
return
}
tc := tt.tcFunc()
// Create
err = dsl.Create(ctx, tc.CreateOptions(store))
if !assert.Nil(t, err) {
return
}
// Update
err = dsl.Update(ctx, tc.UpdateOptions())
if !assert.Nil(t, err) {
return
}
// Verify updated info
info, err := dsl.Inspect(ctx, tc.ID)
if !assert.Nil(t, err) {
return
}
assert.True(t, tc.AssertUpdatedInfo(info))
// Cleanup
err = dsl.Delete(ctx, tc.DeleteOptions())
assert.Nil(t, err)
})
}
}
}
// Test DSL delete
func TestDSLDelete(t *testing.T) {
ctx := context.Background()
testCases := []struct {
name string
tcFunc func() *TestCase
dslType types.Type
stores []types.StoreType
}{
{"Model", NewModelTestCase, types.TypeModel, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
{"Connector", NewConnectorTestCase, types.TypeConnector, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
{"MCP", NewMCPTestCase, types.TypeMCPClient, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
}
for _, tt := range testCases {
for _, store := range tt.stores {
t.Run(fmt.Sprintf("%s_%s", tt.name, store), func(t *testing.T) {
// Clean test data before each test
err := cleanTestData()
if err != nil {
t.Fatalf("Failed to clean test data: %v", err)
}
dsl, err := New(tt.dslType)
if !assert.Nil(t, err) {
return
}
tc := tt.tcFunc()
// Create
err = dsl.Create(ctx, tc.CreateOptions(store))
if !assert.Nil(t, err) {
return
}
// Delete
err = dsl.Delete(ctx, tc.DeleteOptions())
if !assert.Nil(t, err) {
return
}
// Verify deleted
exists, err := dsl.Exists(ctx, tc.ID)
if !assert.Nil(t, err) {
return
}
assert.False(t, exists)
})
}
}
}
// Test DSL full flow (create, inspect, update, delete)
func TestDSLFlow(t *testing.T) {
ctx := context.Background()
testCases := []struct {
name string
tcFunc func() *TestCase
dslType types.Type
stores []types.StoreType
}{
{"Model", NewModelTestCase, types.TypeModel, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
{"Connector", NewConnectorTestCase, types.TypeConnector, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
{"MCP", NewMCPTestCase, types.TypeMCPClient, []types.StoreType{types.StoreTypeDB, types.StoreTypeFile}},
}
for _, tt := range testCases {
for _, store := range tt.stores {
t.Run(fmt.Sprintf("%s_%s", tt.name, store), func(t *testing.T) {
// Clean test data before each test
err := cleanTestData()
if err != nil {
t.Fatalf("Failed to clean test data: %v", err)
}
dsl, err := New(tt.dslType)
if !assert.Nil(t, err) {
return
}
tc := tt.tcFunc()
// Create
err = dsl.Create(ctx, tc.CreateOptions(store))
if !assert.Nil(t, err) {
return
}
// Inspect
info, err := dsl.Inspect(ctx, tc.ID)
if !assert.Nil(t, err) {
return
}
assert.True(t, tc.AssertInfo(info))
// Get source
source, err := dsl.Source(ctx, tc.ID)
if !assert.Nil(t, err) {
return
}
assert.Equal(t, tc.Source, source)
// Update
err = dsl.Update(ctx, tc.UpdateOptions())
if !assert.Nil(t, err) {
return
}
// Verify updated
info, err = dsl.Inspect(ctx, tc.ID)
if !assert.Nil(t, err) {
return
}
assert.True(t, tc.AssertUpdatedInfo(info))
// Delete
err = dsl.Delete(ctx, tc.DeleteOptions())
if !assert.Nil(t, err) {
return
}
// Verify deleted
exists, err := dsl.Exists(ctx, tc.ID)
if !assert.Nil(t, err) {
return
}
assert.False(t, exists)
})
}
}
}

223
dsl/io/cases_test.go Normal file
View file

@ -0,0 +1,223 @@
package io
import (
"fmt"
"os"
"path/filepath"
"testing"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/yaoapp/gou/application"
"github.com/yaoapp/gou/model"
"github.com/yaoapp/kun/log"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/data"
"github.com/yaoapp/yao/dsl/types"
"github.com/yaoapp/yao/test"
)
// systemModels system models
var systemModels = map[string]string{
"__yao.dsl": "yao/models/dsl.mod.yao",
}
func TestMain(m *testing.M) {
// Setup
test.Prepare(&testing.T{}, config.Conf)
defer test.Clean()
// Load system models
model.WithCrypt([]byte(fmt.Sprintf(`{"key":"%s"}`, config.Conf.DB.AESKey)), "AES")
model.WithCrypt([]byte(`{}`), "PASSWORD")
err := loadSystemModels()
if err != nil {
log.Error("Load system models error: %s", err.Error())
os.Exit(1)
}
// Run tests
code := m.Run()
os.Exit(code)
}
// loadSystemModels load system models
func loadSystemModels() error {
for id, path := range systemModels {
content, err := data.Read(path)
if err != nil {
return err
}
// Parse model
var data map[string]interface{}
err = application.Parse(path, content, &data)
if err != nil {
return err
}
// Set prefix
if table, ok := data["table"].(map[string]interface{}); ok {
if name, ok := table["name"].(string); ok {
table["name"] = "__yao_" + name
content, err = jsoniter.Marshal(data)
if err != nil {
log.Error("failed to marshal model data: %v", err)
return fmt.Errorf("failed to marshal model data: %v", err)
}
}
}
// Load Model
mod, err := model.LoadSource(content, id, filepath.Join("__system", path))
if err != nil {
log.Error("load system model %s error: %s", id, err.Error())
return err
}
// Drop table first
err = mod.DropTable()
if err != nil {
log.Error("drop table error: %s", err.Error())
return err
}
// Auto migrate
err = mod.Migrate(false, model.WithDonotInsertValues(true))
if err != nil {
log.Error("migrate system model %s error: %s", id, err.Error())
return err
}
}
return nil
}
// cleanTestData cleans test data from database
func cleanTestData() error {
m := model.Select("__yao.dsl")
err := m.DropTable()
if err != nil {
return err
}
err = m.Migrate(false, model.WithDonotInsertValues(true))
if err != nil {
return err
}
return nil
}
// getTestID 生成唯一的测试ID
func getTestID() string {
return fmt.Sprintf("test_%d", time.Now().UnixNano())
}
// TestCase 定义单个测试用例
type TestCase struct {
ID string
Source string
UpdatedSource string
Tags []string
Label string
Description string
}
// NewTestCase 创建新的测试用例
func NewTestCase() *TestCase {
id := getTestID()
return &TestCase{
ID: id,
Source: fmt.Sprintf(`{
"name": "%s",
"table": { "name": "%s", "comment": "Test Table" },
"columns": [
{ "name": "id", "type": "ID" }
],
"tags": ["test_%s"],
"label": "Test Label",
"description": "Test Description"
}`, id, id, id),
UpdatedSource: fmt.Sprintf(`{
"name": "%s",
"table": { "name": "%s", "comment": "Updated Test Table" },
"columns": [
{ "name": "id", "type": "ID" }
],
"tags": ["test_%s", "updated"],
"label": "Updated Label",
"description": "Updated Description"
}`, id, id, id),
Tags: []string{fmt.Sprintf("test_%s", id)},
Label: "Test Label",
Description: "Test Description",
}
}
// CreateOptions 返回创建选项
func (tc *TestCase) CreateOptions() *types.CreateOptions {
return &types.CreateOptions{
ID: tc.ID,
Source: tc.Source,
}
}
// UpdateOptions 返回更新选项
func (tc *TestCase) UpdateOptions() *types.UpdateOptions {
return &types.UpdateOptions{
ID: tc.ID,
Source: tc.UpdatedSource,
}
}
// UpdateInfoOptions 返回更新信息选项
func (tc *TestCase) UpdateInfoOptions() *types.UpdateOptions {
return &types.UpdateOptions{
ID: tc.ID,
Info: &types.Info{
Label: "Updated via Info",
Tags: []string{"tag1", "info"},
Description: "Updated via info field",
},
}
}
// ListOptions 返回列表选项
func (tc *TestCase) ListOptions(withSource bool) *types.ListOptions {
return &types.ListOptions{
Source: withSource,
Tags: tc.Tags,
}
}
// AssertInfo 验证信息是否正确
func (tc *TestCase) AssertInfo(info *types.Info) bool {
if info == nil {
return false
}
return info.ID == tc.ID &&
info.Label == tc.Label &&
len(info.Tags) == len(tc.Tags) &&
info.Description == tc.Description
}
// AssertUpdatedInfo 验证更新后的信息是否正确
func (tc *TestCase) AssertUpdatedInfo(info *types.Info) bool {
if info == nil {
return false
}
return info.ID == tc.ID &&
info.Label == "Updated Label" &&
len(info.Tags) == 2 &&
info.Description == "Updated Description"
}
// AssertUpdatedInfoViaInfo 验证通过Info更新后的信息是否正确
func (tc *TestCase) AssertUpdatedInfoViaInfo(info *types.Info) bool {
if info == nil {
return false
}
return info.ID == tc.ID &&
info.Label == "Updated via Info" &&
len(info.Tags) == 2 &&
info.Description == "Updated via info field"
}

467
dsl/io/db.go Normal file
View file

@ -0,0 +1,467 @@
package io
import (
"fmt"
"strings"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/yaoapp/gou/model"
"github.com/yaoapp/yao/dsl/types"
)
// DB is the db io
type DB struct {
Type types.Type
}
// NewDB create a new db io
func NewDB(typ types.Type) types.IO {
return &DB{Type: typ}
}
// fmtRow format the row data for DSL info
func fmtRow(row map[string]interface{}) map[string]interface{} {
// Handle source field first
if source, ok := row["source"]; ok {
if str, ok := source.(string); ok {
row["source"] = str
}
}
// Map fields
if id, ok := row["dsl_id"]; ok {
row["id"] = id
delete(row, "dsl_id")
}
if readonly, ok := row["readonly"]; ok {
row["readonly"] = toBool(readonly)
delete(row, "readonly")
}
if builtin, ok := row["built_in"]; ok {
row["built_in"] = toBool(builtin)
}
// Convert time values
if mtime, ok := row["mtime"]; ok && mtime != nil {
if timeStr := toTime(mtime); timeStr != "" {
row["mtime"] = timeStr
}
}
if ctime, ok := row["ctime"]; ok && ctime != nil {
if timeStr := toTime(ctime); timeStr != "" {
row["ctime"] = timeStr
}
}
return row
}
// Inspect get the info from the db
func (db *DB) Inspect(id string) (*types.Info, bool, error) {
// Get from database
m := model.Select("__yao.dsl")
// Get the info
var info types.Info
rows, err := m.Get(model.QueryParam{
Wheres: []model.QueryWhere{
{Column: "dsl_id", Value: id},
{Column: "type", Value: db.Type},
},
Select: []interface{}{
"dsl_id",
"type",
"label",
"path",
"sort",
"tags",
"description",
"store",
"mtime",
"ctime",
"readonly",
"built_in",
},
Limit: 1,
Orders: []model.QueryOrder{{Column: "sort", Option: "asc"}, {Column: "mtime", Option: "desc"}},
})
if err != nil {
return nil, false, err
}
if len(rows) == 0 {
return nil, false, nil
}
// Format row data
row := fmtRow(rows[0])
raw, err := jsoniter.Marshal(row)
if err != nil {
return nil, false, err
}
err = jsoniter.Unmarshal(raw, &info)
if err != nil {
return nil, false, err
}
// Force set Store to DB since this record is from database
info.Store = types.StoreTypeDB
return &info, true, nil
}
// Source get the source from the db
func (db *DB) Source(id string) (string, bool, error) {
// Get from database
m := model.Select("__yao.dsl")
// Get the source
rows, err := m.Get(model.QueryParam{
Wheres: []model.QueryWhere{
{Column: "dsl_id", Value: id},
{Column: "type", Value: db.Type},
},
Select: []interface{}{"source"},
Limit: 1,
})
if err != nil {
return "", false, err
}
if len(rows) == 0 {
return "", false, nil
}
if rows[0]["source"] == nil {
return "", true, nil
}
source, ok := rows[0]["source"].(string)
if !ok {
return "", true, fmt.Errorf("%s %s source is not a string", db.Type, id)
}
return source, true, nil
}
// List get the list from the db
func (db *DB) List(options *types.ListOptions) ([]*types.Info, error) {
// Get from database
m := model.Select("__yao.dsl")
var orders []model.QueryOrder = []model.QueryOrder{{Column: "mtime", Option: "desc"}}
if options.Sort == "sort" {
orders = []model.QueryOrder{{Column: "sort", Option: "asc"}}
}
var wheres []model.QueryWhere = []model.QueryWhere{{Column: "type", Value: db.Type}}
// Filter by tags
if len(options.Tags) > 0 {
var orwheres []model.QueryWhere = []model.QueryWhere{}
for _, tag := range options.Tags {
match := "%" + strings.TrimSpace(tag) + "%"
orwheres = append(orwheres, model.QueryWhere{Column: "tags", Value: match, OP: "like", Method: "orwhere"})
}
wheres = append(wheres, model.QueryWhere{Wheres: orwheres})
}
// Select fields
fields := []interface{}{
"dsl_id",
"type",
"label",
"path",
"sort",
"tags",
"description",
"store",
"mtime",
"ctime",
"readonly",
"built_in",
}
if options.Source {
fields = append(fields, "source")
}
// Get the list
rows, err := m.Get(model.QueryParam{
Wheres: wheres,
Select: fields,
Orders: orders,
})
if err != nil {
return nil, err
}
if len(rows) == 0 {
return nil, nil
}
// Format rows data
for i := range rows {
rows[i] = fmtRow(rows[i])
}
var infos []*types.Info
raw, err := jsoniter.Marshal(rows)
if err != nil {
return nil, err
}
err = jsoniter.Unmarshal(raw, &infos)
if err != nil {
return nil, err
}
// Force set Store to DB since these records are from database
for _, info := range infos {
info.Store = types.StoreTypeDB
}
return infos, nil
}
// Create create the dsl
func (db *DB) Create(options *types.CreateOptions) error {
if options.Source == "" {
return fmt.Errorf("%s %s source is required", db.Type, options.ID)
}
// Parse the source to extract metadata
var sourceData map[string]interface{}
err := jsoniter.Unmarshal([]byte(options.Source), &sourceData)
if err != nil {
return err
}
// Extract common fields from source
var label, description string
var tags []string
var sort int
if v, ok := sourceData["label"]; ok {
if s, ok := v.(string); ok {
label = s
}
}
if v, ok := sourceData["description"]; ok {
if s, ok := v.(string); ok {
description = s
}
}
if v, ok := sourceData["tags"]; ok {
if tagsList, ok := v.([]interface{}); ok {
for _, tag := range tagsList {
if s, ok := tag.(string); ok {
tags = append(tags, s)
}
}
}
}
if v, ok := sourceData["sort"]; ok {
if s, ok := v.(float64); ok {
sort = int(s)
}
}
// Set default store type if not specified
store := options.Store
if store == "" {
store = types.StoreTypeFile
}
// Get the info
m := model.Select("__yao.dsl")
data := map[string]interface{}{
"source": options.Source,
"dsl_id": options.ID,
"type": db.Type,
"label": label,
"path": types.ToPath(db.Type, options.ID),
"sort": sort,
"tags": tags,
"description": description,
"store": store,
"mtime": time.Now(),
"ctime": time.Now(),
"readonly": 0,
"built_in": 0,
"created_at": time.Now(),
"updated_at": time.Now(),
}
_, err = m.Create(data)
if err != nil {
return err
}
return nil
}
// Update update the dsl
func (db *DB) Update(options *types.UpdateOptions) error {
if options.Source == "" && options.Info == nil {
return fmt.Errorf("%s %s one of source or info is required", db.Type, options.ID)
}
m := model.Select("__yao.dsl")
// Check if the dsl exists
rows, err := m.Get(model.QueryParam{
Wheres: []model.QueryWhere{
{Column: "dsl_id", Value: options.ID},
{Column: "type", Value: db.Type},
},
Select: []interface{}{"id"},
Limit: 1,
})
if err != nil {
return err
}
if len(rows) == 0 {
return fmt.Errorf("%s %s not found", db.Type, options.ID)
}
// update source
var data map[string]interface{} = map[string]interface{}{
"source": options.Source,
}
if options.Source != "" {
// Parse source to extract metadata
var sourceData map[string]interface{}
err = jsoniter.Unmarshal([]byte(options.Source), &sourceData)
if err != nil {
return err
}
// Extract common fields from source
if v, ok := sourceData["label"]; ok {
if s, ok := v.(string); ok {
data["label"] = s
}
}
if v, ok := sourceData["description"]; ok {
if s, ok := v.(string); ok {
data["description"] = s
}
}
if v, ok := sourceData["tags"]; ok {
if tagsList, ok := v.([]interface{}); ok {
var tags []string
for _, tag := range tagsList {
if s, ok := tag.(string); ok {
tags = append(tags, s)
}
}
data["tags"] = tags
}
}
if v, ok := sourceData["sort"]; ok {
if s, ok := v.(float64); ok {
data["sort"] = int(s)
}
}
} else {
// Update info
if options.Info.Label != "" {
data["label"] = options.Info.Label
}
if options.Info.Description != "" {
data["description"] = options.Info.Description
}
if len(options.Info.Tags) > 0 {
data["tags"] = options.Info.Tags
}
if options.Info.Sort != 0 {
data["sort"] = options.Info.Sort
}
if options.Info.Status != "" {
data["status"] = options.Info.Status
}
if options.Info.Store != "" {
data["store"] = options.Info.Store
}
if options.Info.Readonly {
data["readonly"] = 1
}
if options.Info.Builtin {
data["built_in"] = 1
}
}
data["updated_at"] = time.Now()
data["mtime"] = time.Now()
err = m.Update(rows[0]["id"], data)
if err != nil {
return err
}
return nil
}
// Delete delete the dsl
func (db *DB) Delete(id string) error {
// Get from database
m := model.Select("__yao.dsl")
// Check if the dsl exists
rows, err := m.Get(model.QueryParam{
Wheres: []model.QueryWhere{
{Column: "dsl_id", Value: id},
{Column: "type", Value: db.Type},
},
Select: []interface{}{"id", "dsl_id"},
Limit: 1,
})
if err != nil {
return err
}
if len(rows) == 0 {
return fmt.Errorf("%s %s not found", db.Type, id)
}
// Delete the dsl
row := rows[0]
return m.Delete(row["id"])
}
// Exists check if the dsl exists
func (db *DB) Exists(id string) (bool, error) {
// Get from database
m := model.Select("__yao.dsl")
// Check if the dsl exists
rows, err := m.Get(model.QueryParam{
Wheres: []model.QueryWhere{
{Column: "dsl_id", Value: id},
{Column: "type", Value: db.Type},
},
Select: []interface{}{"id", "dsl_id"},
Limit: 1,
})
if err != nil {
return false, err
}
return len(rows) > 0, nil
}

164
dsl/io/db_test.go Normal file
View file

@ -0,0 +1,164 @@
package io
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/yaoapp/yao/dsl/types"
)
func TestDBNew(t *testing.T) {
db := NewDB(types.TypeModel)
dbImpl, ok := db.(*DB)
assert.True(t, ok)
assert.Equal(t, types.TypeModel, dbImpl.Type)
}
func TestDBCreate(t *testing.T) {
prepare(t)
db := NewDB(types.TypeModel)
tc := NewTestCase()
err := db.Create(tc.CreateOptions())
assert.Nil(t, err)
// Check if exists
exists, err := db.Exists(tc.ID)
assert.Nil(t, err)
assert.True(t, exists)
// Create again should fail
err = db.Create(tc.CreateOptions())
assert.NotNil(t, err)
}
func TestDBInspect(t *testing.T) {
prepare(t)
db := NewDB(types.TypeModel)
tc := NewTestCase()
err := db.Create(tc.CreateOptions())
assert.Nil(t, err)
info, exists, err := db.Inspect(tc.ID)
assert.Nil(t, err)
assert.True(t, exists)
assert.True(t, tc.AssertInfo(info))
}
func TestDBSource(t *testing.T) {
prepare(t)
db := NewDB(types.TypeModel)
tc := NewTestCase()
err := db.Create(tc.CreateOptions())
assert.Nil(t, err)
data, exists, err := db.Source(tc.ID)
assert.Nil(t, err)
assert.True(t, exists)
assert.Equal(t, tc.Source, data)
}
func TestDBList(t *testing.T) {
prepare(t)
db := NewDB(types.TypeModel)
tc1 := NewTestCase()
tc2 := NewTestCase()
// Create test files
err := db.Create(tc1.CreateOptions())
assert.Nil(t, err)
err = db.Create(tc2.CreateOptions())
assert.Nil(t, err)
// List all
list, err := db.List(&types.ListOptions{})
assert.Nil(t, err)
assert.Equal(t, 2, len(list))
// List with tag
list, err = db.List(tc1.ListOptions(false))
assert.Nil(t, err)
assert.Equal(t, 1, len(list))
if assert.Greater(t, len(list), 0, "List should not be empty") {
assert.Equal(t, tc1.ID, list[0].ID)
}
}
func TestDBUpdate(t *testing.T) {
prepare(t)
db := NewDB(types.TypeModel)
tc := NewTestCase()
err := db.Create(tc.CreateOptions())
assert.Nil(t, err)
// Update source
err = db.Update(tc.UpdateOptions())
assert.Nil(t, err)
info, exists, err := db.Inspect(tc.ID)
assert.Nil(t, err)
assert.True(t, exists)
assert.True(t, tc.AssertUpdatedInfo(info))
// Update info
err = db.Update(tc.UpdateInfoOptions())
assert.Nil(t, err)
info, exists, err = db.Inspect(tc.ID)
assert.Nil(t, err)
assert.True(t, exists)
assert.True(t, tc.AssertUpdatedInfoViaInfo(info))
}
func TestDBDelete(t *testing.T) {
prepare(t)
db := NewDB(types.TypeModel)
tc := NewTestCase()
err := db.Create(tc.CreateOptions())
assert.Nil(t, err)
err = db.Delete(tc.ID)
assert.Nil(t, err)
exists, err := db.Exists(tc.ID)
assert.Nil(t, err)
assert.False(t, exists)
}
func TestDBFlow(t *testing.T) {
prepare(t)
db := NewDB(types.TypeModel)
tc := NewTestCase()
// Create
err := db.Create(tc.CreateOptions())
assert.Nil(t, err)
// Inspect
info, exists, err := db.Inspect(tc.ID)
assert.Nil(t, err)
assert.True(t, exists)
assert.True(t, tc.AssertInfo(info))
// Update
err = db.Update(tc.UpdateOptions())
assert.Nil(t, err)
info, exists, err = db.Inspect(tc.ID)
assert.Nil(t, err)
assert.True(t, exists)
assert.True(t, tc.AssertUpdatedInfo(info))
// Delete
err = db.Delete(tc.ID)
assert.Nil(t, err)
exists, err = db.Exists(tc.ID)
assert.Nil(t, err)
assert.False(t, exists)
}

275
dsl/io/fs.go Normal file
View file

@ -0,0 +1,275 @@
package io
import (
"fmt"
jsoniter "github.com/json-iterator/go"
"github.com/yaoapp/gou/application"
"github.com/yaoapp/yao/dsl/types"
)
// FS is the fs io
type FS struct {
Type types.Type
}
// NewFS create a new fs io
func NewFS(typ types.Type) types.IO {
return &FS{Type: typ}
}
// Inspect get the info from the file
func (fs *FS) Inspect(id string) (*types.Info, bool, error) {
file := types.ToPath(fs.Type, id)
exists, err := application.App.Exists(file)
if err != nil {
return nil, false, err
}
if !exists {
return nil, false, nil
}
// Read the file
data, err := application.App.Read(file)
if err != nil {
return nil, false, err
}
// Parse the source to extract metadata
var sourceData map[string]interface{}
err = application.Parse(file, data, &sourceData)
if err != nil {
return nil, true, err
}
// Extract common fields from source
var label, description string
var tags []string
var sort int
if v, ok := sourceData["label"]; ok {
if s, ok := v.(string); ok {
label = s
}
}
if v, ok := sourceData["description"]; ok {
if s, ok := v.(string); ok {
description = s
}
}
if v, ok := sourceData["tags"]; ok {
if tagsList, ok := v.([]interface{}); ok {
for _, tag := range tagsList {
if s, ok := tag.(string); ok {
tags = append(tags, s)
}
}
}
}
if v, ok := sourceData["sort"]; ok {
if s, ok := v.(float64); ok {
sort = int(s)
}
}
// Get file info for timestamps
fileInfo, err := application.App.Info(file)
if err != nil {
return nil, true, err
}
// Create Info structure with correct fields
info := &types.Info{
ID: id,
Type: fs.Type,
Label: label,
Description: description,
Tags: tags,
Sort: sort,
Path: file,
Store: types.StoreTypeFile,
Readonly: false,
Builtin: false,
Status: types.StatusLoading,
Mtime: fileInfo.ModTime(),
Ctime: fileInfo.ModTime(),
}
return info, true, nil
}
// Source get the source from the file
func (fs *FS) Source(id string) (string, bool, error) {
path := types.ToPath(fs.Type, id)
exists, err := application.App.Exists(path)
if err != nil {
return "", false, err
}
if !exists {
return "", false, nil
}
// Read the file
data, err := application.App.Read(path)
if err != nil {
return "", false, err
}
return string(data), true, nil
}
// List get the list from the path
func (fs *FS) List(options *types.ListOptions) ([]*types.Info, error) {
root, exts := types.TypeRootAndExts(fs.Type)
var infos []*types.Info = []*types.Info{}
patterns := []string{}
for _, ext := range exts {
patterns = append(patterns, "*"+ext)
}
var errs []error
err := application.App.Walk(root, func(root, file string, isdir bool) error {
if isdir {
return nil
}
id := types.WithTypeToID(fs.Type, file)
info, _, err := fs.Inspect(id)
if err != nil {
errs = append(errs, err)
return nil
}
// Filter by options
if len(options.Tags) > 0 {
if len(info.Tags) == 0 {
return nil
}
for _, tag := range options.Tags {
for _, t := range info.Tags {
if t == tag {
if options.Source {
source, _, err := fs.Source(id)
if err != nil {
errs = append(errs, err)
return nil
}
info.Source = source
}
infos = append(infos, info)
return nil
}
}
}
}
// Add to the list
if options.Source {
source, _, err := fs.Source(id)
if err != nil {
errs = append(errs, err)
return nil
}
info.Source = source
}
infos = append(infos, info)
return err
}, patterns...)
return infos, err
}
// Create create the file
func (fs *FS) Create(options *types.CreateOptions) error {
path := types.ToPath(fs.Type, options.ID)
// Check if the file is a directory
exists, err := application.App.Exists(path)
if err != nil {
return err
}
if exists {
return fmt.Errorf("%v %s already exists", fs.Type, options.ID)
}
// Create the file
return application.App.Write(path, []byte(options.Source))
}
// Update update the file
func (fs *FS) Update(options *types.UpdateOptions) error {
// Validate the options
if options.Source == "" && options.Info == nil {
return fmt.Errorf("%v %s one of source or info is required", fs.Type, options.ID)
}
path := types.ToPath(fs.Type, options.ID)
// Check if the file exists
exists, err := application.App.Exists(path)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("%v %s not found", fs.Type, options.ID)
}
// Update source
if options.Source != "" {
return application.App.Write(path, []byte(options.Source))
}
// Update info
var source map[string]interface{}
data, err := application.App.Read(path)
if err != nil {
return err
}
err = application.Parse(path, data, &source)
if err != nil {
return err
}
// Update the info
source["id"] = options.ID
source["label"] = options.Info.Label
source["tags"] = options.Info.Tags
source["description"] = options.Info.Description
new, err := jsoniter.MarshalIndent(source, "", " ")
if err != nil {
return err
}
return application.App.Write(path, []byte(new))
}
// Delete delete the file
func (fs *FS) Delete(id string) error {
path := types.ToPath(fs.Type, id)
// Check if the file is a directory
exists, err := application.App.Exists(path)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("%v %s not found", fs.Type, id)
}
// Delete the file
return application.App.Remove(path)
}
// Exists check if the file exists
func (fs *FS) Exists(id string) (bool, error) {
path := types.ToPath(fs.Type, id)
return application.App.Exists(path)
}

221
dsl/io/fs_test.go Normal file
View file

@ -0,0 +1,221 @@
package io
import (
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/yaoapp/gou/application"
"github.com/yaoapp/yao/dsl/types"
)
func prepare(t *testing.T) {
root := os.Getenv("YAO_TEST_APPLICATION")
if root == "" {
t.Fatal("YAO_TEST_APPLICATION environment variable is not set")
}
// Create models directory if it doesn't exist
modelsDir := filepath.Join(root, "models")
if err := os.MkdirAll(modelsDir, 0755); err != nil {
t.Fatal(err)
}
// Clean test files
files, err := os.ReadDir(modelsDir)
if err != nil {
t.Fatal(err)
}
// Remove test files
for _, file := range files {
if !file.IsDir() && strings.HasPrefix(file.Name(), "test_") && strings.HasSuffix(file.Name(), ".mod.yao") {
path := filepath.Join(modelsDir, file.Name())
if err := os.Remove(path); err != nil {
t.Fatal(err)
}
}
}
// Clean test data from database
err = cleanTestData()
if err != nil {
t.Fatal(err)
}
app, err := application.OpenFromDisk(root)
if err != nil {
t.Fatal(err)
}
application.App = app
}
func TestFSNew(t *testing.T) {
fs := NewFS(types.TypeModel)
fsImpl, ok := fs.(*FS)
assert.True(t, ok)
assert.Equal(t, types.TypeModel, fsImpl.Type)
}
func TestFSCreate(t *testing.T) {
prepare(t)
fs := NewFS(types.TypeModel)
tc := NewTestCase()
err := fs.Create(tc.CreateOptions())
assert.Nil(t, err)
// Check if exists
exists, err := fs.Exists(tc.ID)
assert.Nil(t, err)
assert.True(t, exists)
// Create again should fail
err = fs.Create(tc.CreateOptions())
assert.NotNil(t, err)
}
func TestFSInspect(t *testing.T) {
prepare(t)
fs := NewFS(types.TypeModel)
tc := NewTestCase()
err := fs.Create(tc.CreateOptions())
assert.Nil(t, err)
info, exists, err := fs.Inspect(tc.ID)
assert.Nil(t, err)
assert.True(t, exists)
assert.True(t, tc.AssertInfo(info))
}
func TestFSSource(t *testing.T) {
prepare(t)
fs := NewFS(types.TypeModel)
tc := NewTestCase()
err := fs.Create(tc.CreateOptions())
assert.Nil(t, err)
data, exists, err := fs.Source(tc.ID)
assert.Nil(t, err)
assert.True(t, exists)
assert.Equal(t, tc.Source, data)
}
func TestFSList(t *testing.T) {
prepare(t)
fs := NewFS(types.TypeModel)
// Get initial count
initialList, err := fs.List(&types.ListOptions{})
assert.Nil(t, err)
initialCount := len(initialList)
tc1 := NewTestCase()
tc2 := NewTestCase()
// Create test files
err = fs.Create(tc1.CreateOptions())
assert.Nil(t, err)
err = fs.Create(tc2.CreateOptions())
assert.Nil(t, err)
// List all
list, err := fs.List(&types.ListOptions{})
assert.Nil(t, err)
assert.Equal(t, initialCount+2, len(list))
// List with tag - should return both files since tags are OR relationship
list, err = fs.List(tc1.ListOptions(false))
assert.Nil(t, err)
assert.Equal(t, 2, len(list))
// Verify both files are in the results
found := false
for _, info := range list {
if info.ID == tc1.ID {
found = true
break
}
}
assert.True(t, found, "Should find tc1's file in results")
}
func TestFSUpdate(t *testing.T) {
prepare(t)
fs := NewFS(types.TypeModel)
tc := NewTestCase()
err := fs.Create(tc.CreateOptions())
assert.Nil(t, err)
// Update source
err = fs.Update(tc.UpdateOptions())
assert.Nil(t, err)
info, exists, err := fs.Inspect(tc.ID)
assert.Nil(t, err)
assert.True(t, exists)
assert.True(t, tc.AssertUpdatedInfo(info))
// Update info
err = fs.Update(tc.UpdateInfoOptions())
assert.Nil(t, err)
info, exists, err = fs.Inspect(tc.ID)
assert.Nil(t, err)
assert.True(t, exists)
assert.True(t, tc.AssertUpdatedInfoViaInfo(info))
}
func TestFSDelete(t *testing.T) {
prepare(t)
fs := NewFS(types.TypeModel)
tc := NewTestCase()
err := fs.Create(tc.CreateOptions())
assert.Nil(t, err)
err = fs.Delete(tc.ID)
assert.Nil(t, err)
exists, err := fs.Exists(tc.ID)
assert.Nil(t, err)
assert.False(t, exists)
}
func TestFSFlow(t *testing.T) {
prepare(t)
fs := NewFS(types.TypeModel)
tc := NewTestCase()
// Create
err := fs.Create(tc.CreateOptions())
assert.Nil(t, err)
// Inspect
info, exists, err := fs.Inspect(tc.ID)
assert.Nil(t, err)
assert.True(t, exists)
assert.True(t, tc.AssertInfo(info))
// Update
err = fs.Update(tc.UpdateOptions())
assert.Nil(t, err)
info, exists, err = fs.Inspect(tc.ID)
assert.Nil(t, err)
assert.True(t, exists)
assert.True(t, tc.AssertUpdatedInfo(info))
// Delete
err = fs.Delete(tc.ID)
assert.Nil(t, err)
exists, err = fs.Exists(tc.ID)
assert.Nil(t, err)
assert.False(t, exists)
}

51
dsl/io/utils.go Normal file
View file

@ -0,0 +1,51 @@
package io
import "time"
// toBool converts various types to boolean
func toBool(v interface{}) bool {
if v == nil {
return false
}
switch val := v.(type) {
case bool:
return val
case int:
return val == 1
case int64:
return val == 1
case float64:
return val == 1
case string:
return val == "1" || val == "true"
default:
return false
}
}
// toTime converts various time formats to RFC3339 string
func toTime(v interface{}) string {
if v == nil {
return ""
}
switch val := v.(type) {
case string:
// Try common formats
formats := []string{
"2006-01-02 15:04:05", // SQLite format
"2006-01-02T15:04:05Z07:00", // RFC3339 format
"2006-01-02T15:04:05Z", // RFC3339 without timezone
time.RFC3339,
}
for _, format := range formats {
if t, err := time.Parse(format, val); err == nil {
return t.UTC().Format(time.RFC3339) // Convert to UTC and format as RFC3339
}
}
case time.Time:
return val.UTC().Format(time.RFC3339) // Convert to UTC and format as RFC3339
}
return ""
}

325
dsl/mcp/cases_test.go Normal file
View file

@ -0,0 +1,325 @@
package mcp
import (
"fmt"
"os"
"testing"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/yaoapp/gou/application"
"github.com/yaoapp/gou/model"
"github.com/yaoapp/kun/log"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/data"
"github.com/yaoapp/yao/dsl/types"
"github.com/yaoapp/yao/test"
)
// systemModels system models
var systemModels = map[string]string{
"__yao.dsl": "yao/models/dsl.mod.yao",
}
func TestMain(m *testing.M) {
// Setup
test.Prepare(&testing.T{}, config.Conf)
defer test.Clean()
// Load system models
model.WithCrypt([]byte(fmt.Sprintf(`{"key":"%s"}`, config.Conf.DB.AESKey)), "AES")
model.WithCrypt([]byte(`{}`), "PASSWORD")
err := loadSystemModels()
if err != nil {
log.Error("Load system models error: %s", err.Error())
os.Exit(1)
}
// Load application
root := os.Getenv("GOU_TEST_APPLICATION")
app, err := application.OpenFromDisk(root) // Load app
if err != nil {
log.Error("Load application error: %s", err.Error())
os.Exit(1)
}
application.Load(app)
// Run tests
code := m.Run()
os.Exit(code)
}
// loadSystemModels load system models
func loadSystemModels() error {
for id, path := range systemModels {
content, err := data.Read(path)
if err != nil {
return err
}
// Parse model
var data map[string]interface{}
err = application.Parse(path, content, &data)
if err != nil {
return err
}
// Set prefix
if table, ok := data["table"].(map[string]interface{}); ok {
if name, ok := table["name"].(string); ok {
table["name"] = "__yao_" + name
content, err = jsoniter.Marshal(data)
if err != nil {
log.Error("failed to marshal model data: %v", err)
return fmt.Errorf("failed to marshal model data: %v", err)
}
}
}
// Load Model
mod, err := model.LoadSource(content, id, path)
if err != nil {
log.Error("load system model %s error: %s", id, err.Error())
return err
}
// Drop table first
err = mod.DropTable()
if err != nil {
log.Error("drop table error: %s", err.Error())
return err
}
// Auto migrate
err = mod.Migrate(false, model.WithDonotInsertValues(true))
if err != nil {
log.Error("migrate system model %s error: %s", id, err.Error())
return err
}
}
return nil
}
// TestCase defines a single test case
type TestCase struct {
ID string
Source string
UpdatedSource string
Tags []string
Label string
Description string
}
// NewTestCase creates a new test case
func NewTestCase() *TestCase {
id := getTestID()
return &TestCase{
ID: id,
Source: fmt.Sprintf(`{
"name": "Test MCP Client %s",
"label": "Test MCP Client",
"description": "Test MCP Client Description",
"tags": ["test_%s"],
"transport": "stdio",
"command": "echo",
"arguments": ["hello", "world"],
"env": {
"MCP_TEST": "true"
},
"enable_sampling": true,
"enable_roots": false,
"timeout": "30s"
}`, id, id),
UpdatedSource: fmt.Sprintf(`{
"name": "Updated MCP Client %s",
"label": "Updated MCP Client",
"description": "Updated MCP Client Description",
"tags": ["test_%s", "updated"],
"transport": "stdio",
"command": "echo",
"arguments": ["hello", "updated"],
"env": {
"MCP_TEST": "true",
"MCP_UPDATED": "true"
},
"enable_sampling": false,
"enable_roots": true,
"timeout": "60s"
}`, id, id),
Tags: []string{fmt.Sprintf("test_%s", id)},
Label: "Test MCP Client",
Description: "Test MCP Client Description",
}
}
// NewHTTPTestCase creates a new HTTP test case
func NewHTTPTestCase() *TestCase {
id := getTestID()
return &TestCase{
ID: id,
Source: fmt.Sprintf(`{
"name": "Test HTTP MCP Client %s",
"label": "Test HTTP MCP Client",
"description": "Test HTTP MCP Client Description",
"tags": ["test_%s", "http"],
"transport": "http",
"url": "http://localhost:8080/mcp",
"authorization_token": "Bearer test-token",
"enable_sampling": true,
"enable_roots": true,
"timeout": "30s"
}`, id, id),
UpdatedSource: fmt.Sprintf(`{
"name": "Updated HTTP MCP Client %s",
"label": "Updated HTTP MCP Client",
"description": "Updated HTTP MCP Client Description",
"tags": ["test_%s", "http", "updated"],
"transport": "http",
"url": "http://localhost:8080/mcp/v2",
"authorization_token": "Bearer updated-token",
"enable_sampling": false,
"enable_roots": false,
"timeout": "60s"
}`, id, id),
Tags: []string{fmt.Sprintf("test_%s", id), "http"},
Label: "Test HTTP MCP Client",
Description: "Test HTTP MCP Client Description",
}
}
// NewSSETestCase creates a new SSE test case
func NewSSETestCase() *TestCase {
id := getTestID()
return &TestCase{
ID: id,
Source: fmt.Sprintf(`{
"name": "Test SSE MCP Client %s",
"label": "Test SSE MCP Client",
"description": "Test SSE MCP Client Description",
"tags": ["test_%s", "sse"],
"transport": "sse",
"url": "http://localhost:8080/sse",
"authorization_token": "Bearer sse-token",
"enable_sampling": true,
"enable_elicitation": true,
"timeout": "45s"
}`, id, id),
UpdatedSource: fmt.Sprintf(`{
"name": "Updated SSE MCP Client %s",
"label": "Updated SSE MCP Client",
"description": "Updated SSE MCP Client Description",
"tags": ["test_%s", "sse", "updated"],
"transport": "sse",
"url": "http://localhost:8080/sse/v2",
"authorization_token": "Bearer updated-sse-token",
"enable_sampling": false,
"enable_elicitation": false,
"timeout": "90s"
}`, id, id),
Tags: []string{fmt.Sprintf("test_%s", id), "sse"},
Label: "Test SSE MCP Client",
Description: "Test SSE MCP Client Description",
}
}
// getTestID generates a unique test ID
func getTestID() string {
return fmt.Sprintf("test_%d", time.Now().UnixNano())
}
// CreateOptions returns creation options
func (tc *TestCase) CreateOptions() *types.CreateOptions {
return &types.CreateOptions{
ID: tc.ID,
Source: tc.Source,
}
}
// LoadOptions returns load options
func (tc *TestCase) LoadOptions() *types.LoadOptions {
return &types.LoadOptions{
ID: tc.ID,
Source: tc.Source,
}
}
// UnloadOptions returns unload options
func (tc *TestCase) UnloadOptions() *types.UnloadOptions {
return &types.UnloadOptions{
ID: tc.ID,
}
}
// ReloadOptions returns reload options
func (tc *TestCase) ReloadOptions() *types.ReloadOptions {
return &types.ReloadOptions{
ID: tc.ID,
Source: tc.UpdatedSource,
}
}
// AssertInfo verifies if the information is correct
func (tc *TestCase) AssertInfo(info *types.Info) bool {
if info == nil {
return false
}
return info.ID == tc.ID &&
info.Type == types.TypeMCPClient &&
info.Label == tc.Label &&
len(info.Tags) == len(tc.Tags) &&
info.Description == tc.Description &&
!info.Readonly &&
!info.Builtin &&
!info.Mtime.IsZero() &&
!info.Ctime.IsZero()
}
// AssertUpdatedInfo verifies if the updated information is correct
func (tc *TestCase) AssertUpdatedInfo(info *types.Info) bool {
if info == nil {
return false
}
expectedTags := append(tc.Tags, "updated")
return info.ID == tc.ID &&
info.Type == types.TypeMCPClient &&
info.Label == "Updated MCP Client" &&
len(info.Tags) == len(expectedTags) &&
info.Description == "Updated MCP Client Description" &&
!info.Readonly &&
!info.Builtin &&
!info.Mtime.IsZero() &&
!info.Ctime.IsZero()
}
// AssertHTTPInfo verifies if the HTTP client information is correct
func (tc *TestCase) AssertHTTPInfo(info *types.Info) bool {
if info == nil {
return false
}
return info.ID == tc.ID &&
info.Type == types.TypeMCPClient &&
info.Label == "Test HTTP MCP Client" &&
len(info.Tags) == 2 && // test_xxx and http
info.Description == "Test HTTP MCP Client Description" &&
!info.Readonly &&
!info.Builtin &&
!info.Mtime.IsZero() &&
!info.Ctime.IsZero()
}
// AssertSSEInfo verifies if the SSE client information is correct
func (tc *TestCase) AssertSSEInfo(info *types.Info) bool {
if info == nil {
return false
}
return info.ID == tc.ID &&
info.Type == types.TypeMCPClient &&
info.Label == "Test SSE MCP Client" &&
len(info.Tags) == 2 && // test_xxx and sse
info.Description == "Test SSE MCP Client Description" &&
!info.Readonly &&
!info.Builtin &&
!info.Mtime.IsZero() &&
!info.Ctime.IsZero()
}

184
dsl/mcp/client.go Normal file
View file

@ -0,0 +1,184 @@
package mcp
import (
"context"
"fmt"
goumcp "github.com/yaoapp/gou/mcp"
"github.com/yaoapp/yao/dsl/types"
)
// YaoMCPClient is the MCP client DSL manager
type YaoMCPClient struct {
root string // The relative path of the MCP client DSL
fs types.IO // The file system IO interface
db types.IO // The database IO interface
}
// NewClient returns a new MCP client DSL manager
func NewClient(root string, fs types.IO, db types.IO) types.Manager {
return &YaoMCPClient{root: root, fs: fs, db: db}
}
// Loaded return all loaded DSLs
func (client *YaoMCPClient) Loaded(ctx context.Context) (map[string]*types.Info, error) {
infos := map[string]*types.Info{}
// Get all loaded MCP clients
clientIDs := goumcp.ListClients()
for _, id := range clientIDs {
// Get the client
mcpClient, err := goumcp.Select(id)
if err != nil {
continue // Skip if client not found
}
// Get meta info from the client
meta := mcpClient.GetMetaInfo()
infos[id] = &types.Info{
ID: id,
Path: types.ToPath(types.TypeMCPClient, id),
Type: types.TypeMCPClient,
Label: meta.Label,
Sort: meta.Sort,
Description: meta.Description,
Tags: meta.Tags,
Readonly: meta.Readonly,
Builtin: meta.Builtin,
Mtime: meta.Mtime,
Ctime: meta.Ctime,
}
}
return infos, nil
}
// Load will unload the DSL first, then load the DSL from DB or file system
func (client *YaoMCPClient) Load(ctx context.Context, options *types.LoadOptions) error {
if options == nil {
return fmt.Errorf("load options is required")
}
if options.ID == "" {
return fmt.Errorf("load options id is required")
}
var err error
// Case 1: If Source is provided, use LoadClientSource
if options.Source != "" {
_, err = goumcp.LoadClientSource(options.Source, options.ID)
if err != nil {
return err
}
} else if options.Path != "" && options.Store == types.StoreTypeFile {
// Case 2: If Path is provided and Store is file, use LoadClient with Path
_, err = goumcp.LoadClient(options.Path, options.ID)
if err != nil {
return err
}
} else if options.Store == types.StoreTypeDB {
// Case 3: If Store is db, get Source from DB first
if client.db == nil {
return fmt.Errorf("db io is required for store type db")
}
source, exists, err := client.db.Source(options.ID)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("mcp client %s not found in database", options.ID)
}
_, err = goumcp.LoadClientSource(source, options.ID)
if err != nil {
return err
}
} else {
// Case 4: Default case, use LoadClient with ID
path := types.ToPath(types.TypeMCPClient, options.ID)
_, err = goumcp.LoadClient(path, options.ID)
if err != nil {
return err
}
}
return nil
}
// Unload will unload the DSL from memory
func (client *YaoMCPClient) Unload(ctx context.Context, options *types.UnloadOptions) error {
if options == nil {
return fmt.Errorf("unload options is required")
}
if options.ID == "" {
return fmt.Errorf("unload options id is required")
}
// Use the UnloadClient function from gou/mcp package
goumcp.UnloadClient(options.ID)
return nil
}
// Reload will unload the DSL first, then reload the DSL from DB or file system
func (client *YaoMCPClient) Reload(ctx context.Context, options *types.ReloadOptions) error {
if options == nil {
return fmt.Errorf("reload options is required")
}
if options.ID == "" {
return fmt.Errorf("reload options id is required")
}
// First unload
goumcp.UnloadClient(options.ID)
// Then load
var err error
if options.Source != "" {
_, err = goumcp.LoadClientSource(options.Source, options.ID)
if err != nil {
return err
}
} else if options.Path != "" && options.Store == types.StoreTypeFile {
_, err = goumcp.LoadClient(options.Path, options.ID)
if err != nil {
return err
}
} else if options.Store == types.StoreTypeDB {
if client.db == nil {
return fmt.Errorf("db io is required for store type db")
}
source, exists, err := client.db.Source(options.ID)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("mcp client %s not found in database", options.ID)
}
_, err = goumcp.LoadClientSource(source, options.ID)
if err != nil {
return err
}
} else {
path := types.ToPath(types.TypeMCPClient, options.ID)
_, err = goumcp.LoadClient(path, options.ID)
if err != nil {
return err
}
}
return nil
}
// Validate will validate the DSL from source
func (client *YaoMCPClient) Validate(ctx context.Context, source string) (bool, []types.LintMessage) {
return true, []types.LintMessage{}
}
// Execute will execute the DSL
func (client *YaoMCPClient) Execute(ctx context.Context, id string, method string, args ...any) (any, error) {
return nil, fmt.Errorf("Not implemented")
}

330
dsl/mcp/client_test.go Normal file
View file

@ -0,0 +1,330 @@
package mcp
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/yaoapp/yao/dsl/io"
"github.com/yaoapp/yao/dsl/types"
)
func TestMCPClientLoad(t *testing.T) {
testCase := NewTestCase()
fsio := io.NewFS(types.TypeMCPClient)
dbio := io.NewDB(types.TypeMCPClient)
manager := NewClient("mcps", fsio, dbio)
// Test Load with nil options
err := manager.Load(context.Background(), nil)
assert.Error(t, err)
assert.Contains(t, err.Error(), "load options is required")
// Test Load with empty ID
err = manager.Load(context.Background(), &types.LoadOptions{})
assert.Error(t, err)
assert.Contains(t, err.Error(), "load options id is required")
// Test Load with Source
err = manager.Load(context.Background(), testCase.LoadOptions())
assert.NoError(t, err)
// Test Load from filesystem
err = fsio.Create(testCase.CreateOptions())
assert.NoError(t, err)
path := types.ToPath(types.TypeMCPClient, testCase.ID)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID,
Path: path,
Store: types.StoreTypeFile,
})
assert.NoError(t, err)
// Test Load from database
err = dbio.Create(testCase.CreateOptions())
assert.NoError(t, err)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID,
Store: types.StoreTypeDB,
})
assert.NoError(t, err)
// Clean up
err = fsio.Delete(testCase.ID)
assert.NoError(t, err)
err = dbio.Delete(testCase.ID)
assert.NoError(t, err)
}
func TestMCPClientUnload(t *testing.T) {
testCase := NewTestCase()
fsio := io.NewFS(types.TypeMCPClient)
dbio := io.NewDB(types.TypeMCPClient)
manager := NewClient("mcps", fsio, dbio)
// Test Unload with nil options
err := manager.Unload(context.Background(), nil)
assert.Error(t, err)
assert.Contains(t, err.Error(), "unload options is required")
// Test Unload with empty ID
err = manager.Unload(context.Background(), &types.UnloadOptions{})
assert.Error(t, err)
assert.Contains(t, err.Error(), "unload options id is required")
// Load and then unload from filesystem
err = fsio.Create(testCase.CreateOptions())
assert.NoError(t, err)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID,
Store: types.StoreTypeFile,
})
assert.NoError(t, err)
err = manager.Unload(context.Background(), testCase.UnloadOptions())
assert.NoError(t, err)
// Clean up
err = fsio.Delete(testCase.ID)
assert.NoError(t, err)
}
func TestMCPClientReload(t *testing.T) {
testCase := NewTestCase()
fsio := io.NewFS(types.TypeMCPClient)
dbio := io.NewDB(types.TypeMCPClient)
manager := NewClient("mcps", fsio, dbio)
// Test Reload with nil options
err := manager.Reload(context.Background(), nil)
assert.Error(t, err)
assert.Contains(t, err.Error(), "reload options is required")
// Test Reload with empty ID
err = manager.Reload(context.Background(), &types.ReloadOptions{})
assert.Error(t, err)
assert.Contains(t, err.Error(), "reload options id is required")
// Load and then reload from filesystem
err = fsio.Create(testCase.CreateOptions())
assert.NoError(t, err)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID,
Store: types.StoreTypeFile,
})
assert.NoError(t, err)
err = manager.Reload(context.Background(), testCase.ReloadOptions())
assert.NoError(t, err)
// Clean up
err = fsio.Delete(testCase.ID)
assert.NoError(t, err)
}
func TestMCPClientLoaded(t *testing.T) {
testCase := NewTestCase()
fsio := io.NewFS(types.TypeMCPClient)
dbio := io.NewDB(types.TypeMCPClient)
manager := NewClient("mcps", fsio, dbio)
// Load from filesystem
err := fsio.Create(testCase.CreateOptions())
assert.NoError(t, err)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID,
Store: types.StoreTypeFile,
})
assert.NoError(t, err)
// Test Loaded
infos, err := manager.Loaded(context.Background())
assert.NoError(t, err)
assert.NotNil(t, infos)
assert.Contains(t, infos, testCase.ID)
// Verify metadata fields
fsInfo := infos[testCase.ID]
assert.Equal(t, testCase.ID, fsInfo.ID)
assert.Equal(t, types.TypeMCPClient, fsInfo.Type)
assert.Equal(t, testCase.Label, fsInfo.Label)
assert.Equal(t, testCase.Description, fsInfo.Description)
assert.ElementsMatch(t, testCase.Tags, fsInfo.Tags)
assert.False(t, fsInfo.Readonly)
assert.False(t, fsInfo.Builtin)
// Clean up
err = fsio.Delete(testCase.ID)
assert.NoError(t, err)
}
func TestMCPClientValidate(t *testing.T) {
manager := NewClient("mcps", nil, nil)
// Test Validate
valid, messages := manager.Validate(context.Background(), "test source")
assert.True(t, valid)
assert.Empty(t, messages)
}
func TestMCPClientExecute(t *testing.T) {
manager := NewClient("mcps", nil, nil)
// Test Execute
result, err := manager.Execute(context.Background(), "test_id", "test_method")
assert.Error(t, err)
assert.Contains(t, err.Error(), "Not implemented")
assert.Nil(t, result)
}
func TestMCPClientHTTPLoad(t *testing.T) {
testCase := NewHTTPTestCase()
fsio := io.NewFS(types.TypeMCPClient)
dbio := io.NewDB(types.TypeMCPClient)
manager := NewClient("mcps", fsio, dbio)
// Test Load with HTTP Source
err := manager.Load(context.Background(), testCase.LoadOptions())
assert.NoError(t, err)
// Test Loaded
infos, err := manager.Loaded(context.Background())
assert.NoError(t, err)
assert.NotNil(t, infos)
assert.Contains(t, infos, testCase.ID)
// Verify HTTP metadata fields
httpInfo := infos[testCase.ID]
assert.Equal(t, testCase.ID, httpInfo.ID)
assert.Equal(t, types.TypeMCPClient, httpInfo.Type)
assert.Equal(t, "Test HTTP MCP Client", httpInfo.Label)
assert.Equal(t, "Test HTTP MCP Client Description", httpInfo.Description)
assert.Contains(t, httpInfo.Tags, "http")
assert.False(t, httpInfo.Readonly)
assert.False(t, httpInfo.Builtin)
// Test Unload
err = manager.Unload(context.Background(), testCase.UnloadOptions())
assert.NoError(t, err)
}
func TestMCPClientSSELoad(t *testing.T) {
testCase := NewSSETestCase()
fsio := io.NewFS(types.TypeMCPClient)
dbio := io.NewDB(types.TypeMCPClient)
manager := NewClient("mcps", fsio, dbio)
// Test Load with SSE Source
err := manager.Load(context.Background(), testCase.LoadOptions())
assert.NoError(t, err)
// Test Loaded
infos, err := manager.Loaded(context.Background())
assert.NoError(t, err)
assert.NotNil(t, infos)
assert.Contains(t, infos, testCase.ID)
// Verify SSE metadata fields
sseInfo := infos[testCase.ID]
assert.Equal(t, testCase.ID, sseInfo.ID)
assert.Equal(t, types.TypeMCPClient, sseInfo.Type)
assert.Equal(t, "Test SSE MCP Client", sseInfo.Label)
assert.Equal(t, "Test SSE MCP Client Description", sseInfo.Description)
assert.Contains(t, sseInfo.Tags, "sse")
assert.False(t, sseInfo.Readonly)
assert.False(t, sseInfo.Builtin)
// Test Unload
err = manager.Unload(context.Background(), testCase.UnloadOptions())
assert.NoError(t, err)
}
func TestMCPClientLoadWithDatabaseStore(t *testing.T) {
testCase := NewTestCase()
fsio := io.NewFS(types.TypeMCPClient)
dbio := io.NewDB(types.TypeMCPClient)
manager := NewClient("mcps", fsio, dbio)
// Create in database first
err := dbio.Create(testCase.CreateOptions())
assert.NoError(t, err)
// Test Load from database
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID,
Store: types.StoreTypeDB,
})
assert.NoError(t, err)
// Test Loaded
infos, err := manager.Loaded(context.Background())
assert.NoError(t, err)
assert.NotNil(t, infos)
assert.Contains(t, infos, testCase.ID)
// Verify metadata fields
dbInfo := infos[testCase.ID]
assert.Equal(t, testCase.ID, dbInfo.ID)
assert.Equal(t, types.TypeMCPClient, dbInfo.Type)
assert.Equal(t, testCase.Label, dbInfo.Label)
assert.Equal(t, testCase.Description, dbInfo.Description)
assert.ElementsMatch(t, testCase.Tags, dbInfo.Tags)
assert.False(t, dbInfo.Readonly)
assert.False(t, dbInfo.Builtin)
// Test Reload from database
err = manager.Reload(context.Background(), &types.ReloadOptions{
ID: testCase.ID,
Source: testCase.UpdatedSource,
Store: types.StoreTypeDB,
})
assert.NoError(t, err)
// Clean up
err = dbio.Delete(testCase.ID)
assert.NoError(t, err)
}
func TestMCPClientLoadWithFileStore(t *testing.T) {
testCase := NewTestCase()
fsio := io.NewFS(types.TypeMCPClient)
dbio := io.NewDB(types.TypeMCPClient)
manager := NewClient("mcps", fsio, dbio)
// Create in file system first
err := fsio.Create(testCase.CreateOptions())
assert.NoError(t, err)
// Test Load from file system with explicit path
path := types.ToPath(types.TypeMCPClient, testCase.ID)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID,
Path: path,
Store: types.StoreTypeFile,
})
assert.NoError(t, err)
// Test Loaded
infos, err := manager.Loaded(context.Background())
assert.NoError(t, err)
assert.NotNil(t, infos)
assert.Contains(t, infos, testCase.ID)
// Test Reload from file system
err = manager.Reload(context.Background(), &types.ReloadOptions{
ID: testCase.ID,
Path: path,
Source: testCase.UpdatedSource,
Store: types.StoreTypeFile,
})
assert.NoError(t, err)
// Clean up
err = fsio.Delete(testCase.ID)
assert.NoError(t, err)
}

47
dsl/mcp/server.go Normal file
View file

@ -0,0 +1,47 @@
package mcp
import (
"context"
"github.com/yaoapp/yao/dsl/types"
)
// YaoMCPServer is the MCP client DSL manager
type YaoMCPServer struct {
root string // The relative path of the MCP client DSL
}
// NewServer returns a new MCP server DSL manager
func NewServer(root string) types.Manager {
return &YaoMCPServer{root: root}
}
// Loaded return all loaded DSLs
func (server *YaoMCPServer) Loaded(ctx context.Context) (map[string]*types.Info, error) {
return nil, nil
}
// Load will unload the DSL first, then load the DSL from DB or file system
func (server *YaoMCPServer) Load(ctx context.Context, options *types.LoadOptions) error {
return nil
}
// Unload will unload the DSL from memory
func (server *YaoMCPServer) Unload(ctx context.Context, options *types.UnloadOptions) error {
return nil
}
// Reload will unload the DSL first, then reload the DSL from DB or file system
func (server *YaoMCPServer) Reload(ctx context.Context, options *types.ReloadOptions) error {
return nil
}
// Validate will validate the DSL from source
func (server *YaoMCPServer) Validate(ctx context.Context, source string) (bool, []types.LintMessage) {
return false, nil
}
// Execute will execute the DSL
func (server *YaoMCPServer) Execute(ctx context.Context, id string, method string, args ...any) (any, error) {
return nil, nil
}

244
dsl/model/cases_test.go Normal file
View file

@ -0,0 +1,244 @@
package model
import (
"fmt"
"os"
"path/filepath"
"testing"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/yaoapp/gou/application"
"github.com/yaoapp/gou/model"
"github.com/yaoapp/kun/log"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/data"
"github.com/yaoapp/yao/dsl/types"
"github.com/yaoapp/yao/test"
)
// systemModels system models
var systemModels = map[string]string{
"__yao.dsl": "yao/models/dsl.mod.yao",
}
func TestMain(m *testing.M) {
// Setup
test.Prepare(&testing.T{}, config.Conf)
defer test.Clean()
// Load system models
model.WithCrypt([]byte(fmt.Sprintf(`{"key":"%s"}`, config.Conf.DB.AESKey)), "AES")
model.WithCrypt([]byte(`{}`), "PASSWORD")
err := loadSystemModels()
if err != nil {
log.Error("Load system models error: %s", err.Error())
os.Exit(1)
}
// Run tests
code := m.Run()
os.Exit(code)
}
// loadSystemModels load system models
func loadSystemModels() error {
for id, path := range systemModels {
content, err := data.Read(path)
if err != nil {
return err
}
// Parse model
var data map[string]interface{}
err = application.Parse(path, content, &data)
if err != nil {
return err
}
// Set prefix
if table, ok := data["table"].(map[string]interface{}); ok {
if name, ok := table["name"].(string); ok {
table["name"] = "__yao_" + name
content, err = jsoniter.Marshal(data)
if err != nil {
log.Error("failed to marshal model data: %v", err)
return fmt.Errorf("failed to marshal model data: %v", err)
}
}
}
// Load Model
mod, err := model.LoadSource(content, id, filepath.Join("__system", path))
if err != nil {
log.Error("load system model %s error: %s", id, err.Error())
return err
}
// Drop table first
err = mod.DropTable()
if err != nil {
log.Error("drop table error: %s", err.Error())
return err
}
// Auto migrate
err = mod.Migrate(false, model.WithDonotInsertValues(true))
if err != nil {
log.Error("migrate system model %s error: %s", id, err.Error())
return err
}
}
return nil
}
// cleanTestData cleans test data from database
func cleanTestData() error {
m := model.Select("__yao.dsl")
err := m.DropTable()
if err != nil {
return err
}
err = m.Migrate(false, model.WithDonotInsertValues(true))
if err != nil {
return err
}
return nil
}
// getTestID generates a unique test ID
func getTestID() string {
return fmt.Sprintf("test_%d", time.Now().UnixNano())
}
// TestCase defines a single test case
type TestCase struct {
ID string
Source string
UpdatedSource string
Tags []string
Label string
Description string
}
// NewTestCase creates a new test case
func NewTestCase() *TestCase {
id := getTestID()
return &TestCase{
ID: id,
Source: fmt.Sprintf(`{
"name": "%s",
"table": { "name": "%s", "comment": "Test User" },
"columns": [
{ "name": "id", "type": "ID" },
{ "name": "name", "type": "string", "length": 80, "comment": "User Name", "index": true },
{ "name": "status", "type": "enum", "option": ["active", "disabled"], "default": "active", "comment": "Status", "index": true }
],
"tags": ["test_%s"],
"label": "Test Label",
"description": "Test Description",
"option": { "timestamps": true, "soft_deletes": true }
}`, id, id, id),
UpdatedSource: fmt.Sprintf(`{
"name": "%s",
"table": { "name": "%s", "comment": "Updated Test User" },
"columns": [
{ "name": "id", "type": "ID" },
{ "name": "name", "type": "string", "length": 80, "comment": "User Name", "index": true },
{ "name": "status", "type": "enum", "option": ["active", "disabled", "pending"], "default": "active", "comment": "Status", "index": true }
],
"tags": ["test_%s", "updated"],
"label": "Updated Label",
"description": "Updated Description",
"option": { "timestamps": true, "soft_deletes": true }
}`, id, id, id),
Tags: []string{fmt.Sprintf("test_%s", id)},
Label: "Test Label",
Description: "Test Description",
}
}
// CreateOptions returns creation options
func (tc *TestCase) CreateOptions() *types.CreateOptions {
return &types.CreateOptions{
ID: tc.ID,
Source: tc.Source,
}
}
// UpdateOptions returns update options
func (tc *TestCase) UpdateOptions() *types.UpdateOptions {
return &types.UpdateOptions{
ID: tc.ID,
Source: tc.UpdatedSource,
}
}
// UpdateInfoOptions returns update info options
func (tc *TestCase) UpdateInfoOptions() *types.UpdateOptions {
return &types.UpdateOptions{
ID: tc.ID,
Info: &types.Info{
Label: "Updated via Info",
Tags: []string{"tag1", "info"},
Description: "Updated via info field",
},
}
}
// ListOptions returns list options
func (tc *TestCase) ListOptions(withSource bool) *types.ListOptions {
return &types.ListOptions{
Source: withSource,
Tags: tc.Tags,
}
}
// AssertInfo verifies if the information is correct
func (tc *TestCase) AssertInfo(info *types.Info) bool {
if info == nil {
return false
}
return info.ID == tc.ID &&
info.Type == types.TypeModel &&
info.Label == tc.Label &&
len(info.Tags) == len(tc.Tags) &&
info.Description == tc.Description &&
!info.Readonly &&
!info.Builtin &&
!info.Mtime.IsZero() &&
!info.Ctime.IsZero()
}
// AssertUpdatedInfo verifies if the updated information is correct
func (tc *TestCase) AssertUpdatedInfo(info *types.Info) bool {
if info == nil {
return false
}
return info.ID == tc.ID &&
info.Type == types.TypeModel &&
info.Label == "Updated Label" &&
len(info.Tags) == 2 &&
info.Description == "Updated Description" &&
!info.Readonly &&
!info.Builtin &&
!info.Mtime.IsZero() &&
!info.Ctime.IsZero()
}
// AssertUpdatedInfoViaInfo verifies if the information updated via Info is correct
func (tc *TestCase) AssertUpdatedInfoViaInfo(info *types.Info) bool {
if info == nil {
return false
}
return info.ID == tc.ID &&
info.Type == types.TypeModel &&
info.Label == "Updated via Info" &&
len(info.Tags) == 2 &&
info.Description == "Updated via info field" &&
!info.Readonly &&
!info.Builtin &&
!info.Mtime.IsZero() &&
!info.Ctime.IsZero()
}

256
dsl/model/model.go Normal file
View file

@ -0,0 +1,256 @@
package model
import (
"context"
"fmt"
"github.com/yaoapp/gou/model"
"github.com/yaoapp/kun/exception"
"github.com/yaoapp/yao/dsl/types"
)
// YaoModel is the MCP client DSL manager
type YaoModel struct {
root string // The relative path of the model DSL
fs types.IO // The file system IO interface
db types.IO // The database IO interface
}
// New returns a new connector DSL manager
func New(root string, fs types.IO, db types.IO) types.Manager {
return &YaoModel{root: root, fs: fs, db: db}
}
// Loaded return all loaded DSLs
func (m *YaoModel) Loaded(ctx context.Context) (map[string]*types.Info, error) {
infos := map[string]*types.Info{}
for id, mod := range model.Models {
meta := mod.GetMetaInfo()
infos[id] = &types.Info{
ID: id,
Path: mod.File,
Type: types.TypeModel,
Label: meta.Label,
Sort: meta.Sort,
Description: meta.Description,
Tags: meta.Tags,
Readonly: meta.Readonly,
Builtin: meta.Builtin,
Mtime: meta.Mtime,
Ctime: meta.Ctime,
}
}
return infos, nil
}
// Load will unload the DSL first, then load the DSL from DB or file system
func (m *YaoModel) Load(ctx context.Context, options *types.LoadOptions) error {
if options == nil {
return fmt.Errorf("load options is required")
}
if options.ID == "" {
return fmt.Errorf("load options id is required")
}
var opts map[string]interface{}
if options.Options != nil {
opts = options.Options
}
var migration bool = false
if v, ok := opts["migration"]; ok {
migration = v.(bool)
}
var reset bool = false
if v, ok := opts["reset"]; ok {
reset = v.(bool)
}
var mod *model.Model
var err error
// Case 1: If Source is provided, use LoadSource
if options.Source != "" {
mod, err = model.LoadSourceSync([]byte(options.Source), options.ID, "")
if err != nil {
return err
}
} else if options.Path != "" && options.Store == "fs" {
// Case 2: If Path is provided and Store is fs, use LoadSync with Path
mod, err = model.LoadSync(options.Path, options.ID)
if err != nil {
return err
}
} else if options.Store == "db" {
// Case 3: If Store is db, get Source from DB first
if m.db == nil {
return fmt.Errorf("db io is required for store type db")
}
source, exists, err := m.db.Source(options.ID)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("model %s not found in database", options.ID)
}
mod, err = model.LoadSourceSync([]byte(source), options.ID, "")
if err != nil {
return err
}
} else {
// Case 4: Default case, use LoadSync with ID
path := types.ToPath(types.TypeModel, options.ID)
mod, err = model.LoadSync(path, options.ID)
if err != nil {
return err
}
}
if migration || reset {
return mod.Migrate(reset, model.WithDonotInsertValues(true))
}
return nil
}
// Unload will unload the DSL from memory
func (m *YaoModel) Unload(ctx context.Context, options *types.UnloadOptions) error {
if options == nil {
return fmt.Errorf("unload options is required")
}
if options.ID == "" {
return fmt.Errorf("unload options id is required")
}
var opts map[string]interface{}
if options.Options != nil {
opts = options.Options
}
var dropTable bool = false
if v, ok := opts["dropTable"]; ok {
dropTable = v.(bool)
}
// Try to get model, handle panic
var mod *model.Model
var err error
func() {
defer func() {
if r := recover(); r != nil {
if ex, ok := r.(exception.Exception); ok {
if ex.Message == fmt.Sprintf("Model:%s; not found", options.ID) {
err = fmt.Errorf("model %s not found", options.ID)
return
}
}
panic(r)
}
}()
mod = model.Select(options.ID)
}()
if err != nil {
return err
}
if mod == nil {
return fmt.Errorf("model %s not found", options.ID)
}
if dropTable {
return mod.DropTable()
}
return nil
}
// Reload will unload the DSL first, then reload the DSL from DB or file system
func (m *YaoModel) Reload(ctx context.Context, options *types.ReloadOptions) error {
if options == nil {
return fmt.Errorf("reload options is required")
}
if options.ID == "" {
return fmt.Errorf("reload options id is required")
}
var opts map[string]interface{}
if options.Options != nil {
opts = options.Options
}
var migrate bool = false
if v, ok := opts["migrate"]; ok {
migrate = v.(bool)
}
var reset bool = false
if v, ok := opts["reset"]; ok {
reset = v.(bool)
}
var mod *model.Model
var err error
// Case 1: If Source is provided, use LoadSource
if options.Source != "" {
mod, err = model.LoadSourceSync([]byte(options.Source), options.ID, "")
if err != nil {
return err
}
} else if options.Path != "" && options.Store == "fs" {
// Case 2: If Path is provided and Store is fs, use LoadSync with Path
mod, err = model.LoadSync(options.Path, options.ID)
if err != nil {
return err
}
} else if options.Store == "db" {
// Case 3: If Store is db, get Source from DB first
if m.db == nil {
return fmt.Errorf("db io is required for store type db")
}
source, exists, err := m.db.Source(options.ID)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("model %s not found in database", options.ID)
}
mod, err = model.LoadSourceSync([]byte(source), options.ID, "")
if err != nil {
return err
}
} else {
// Case 4: Default case, use LoadSync with ID
path := types.ToPath(types.TypeModel, options.ID)
mod, err = model.LoadSync(path, options.ID)
if err != nil {
return err
}
}
if migrate || reset {
return mod.Migrate(reset, model.WithDonotInsertValues(true))
}
return nil
}
// Validate will validate the DSL from source
func (m *YaoModel) Validate(ctx context.Context, source string) (bool, []types.LintMessage) {
return true, []types.LintMessage{}
}
// Execute will execute the DSL
func (m *YaoModel) Execute(ctx context.Context, id string, method string, args ...any) (any, error) {
return nil, fmt.Errorf("Not implemented")
}

350
dsl/model/model_test.go Normal file
View file

@ -0,0 +1,350 @@
package model
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/yaoapp/yao/dsl/io"
"github.com/yaoapp/yao/dsl/types"
)
func TestModelLoad(t *testing.T) {
testCase := NewTestCase()
fsio := io.NewFS(types.TypeModel)
dbio := io.NewDB(types.TypeModel)
manager := New("", fsio, dbio)
// Test Load with nil options
err := manager.Load(context.Background(), nil)
assert.Error(t, err)
assert.Contains(t, err.Error(), "load options is required")
// Test Load with empty ID
err = manager.Load(context.Background(), &types.LoadOptions{})
assert.Error(t, err)
assert.Contains(t, err.Error(), "load options id is required")
// Test Load with Source
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID,
Source: testCase.Source,
})
assert.NoError(t, err)
// Test Load from filesystem
err = fsio.Create(&types.CreateOptions{
ID: testCase.ID + "_fs",
Source: testCase.Source,
})
assert.NoError(t, err)
path := types.ToPath(types.TypeModel, testCase.ID+"_fs")
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID + "_fs",
Path: path,
Store: "fs",
})
assert.NoError(t, err)
// Test Load from database
err = dbio.Create(&types.CreateOptions{
ID: testCase.ID + "_db",
Source: testCase.Source,
})
assert.NoError(t, err)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID + "_db",
Store: "db",
})
assert.NoError(t, err)
// Test Load with default path (should use filesystem)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID + "_fs",
})
assert.NoError(t, err)
// Test Load with migration
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID + "_fs",
Options: map[string]interface{}{"migration": true},
})
assert.NoError(t, err)
// Test Load with reset
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID + "_fs",
Options: map[string]interface{}{"reset": true},
})
assert.NoError(t, err)
// Clean up
err = fsio.Delete(testCase.ID + "_fs")
assert.NoError(t, err)
err = dbio.Delete(testCase.ID + "_db")
assert.NoError(t, err)
err = cleanTestData()
assert.NoError(t, err)
}
func TestModelLoadWithDB(t *testing.T) {
testCase := NewTestCase()
dbio := io.NewDB(types.TypeModel)
manager := New("", nil, dbio)
// Create model in DB first
err := dbio.Create(&types.CreateOptions{
ID: testCase.ID,
Source: testCase.Source,
})
assert.NoError(t, err)
// Test Load with Store=db
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID,
Store: "db",
})
assert.NoError(t, err)
// Test Load non-existent model from DB
err = manager.Load(context.Background(), &types.LoadOptions{
ID: "non-existent",
Store: "db",
})
assert.Error(t, err)
assert.Contains(t, err.Error(), "not found in database")
// Clean up
err = dbio.Delete(testCase.ID)
assert.NoError(t, err)
err = cleanTestData()
assert.NoError(t, err)
}
func TestModelUnload(t *testing.T) {
testCase := NewTestCase()
fsio := io.NewFS(types.TypeModel)
dbio := io.NewDB(types.TypeModel)
manager := New("", fsio, dbio)
// Test Unload with nil options
err := manager.Unload(context.Background(), nil)
assert.Error(t, err)
assert.Contains(t, err.Error(), "unload options is required")
// Test Unload with empty ID
err = manager.Unload(context.Background(), &types.UnloadOptions{})
assert.Error(t, err)
assert.Contains(t, err.Error(), "unload options id is required")
// Test Unload non-existent model
err = manager.Unload(context.Background(), &types.UnloadOptions{
ID: "non-existent",
})
assert.Error(t, err)
assert.Contains(t, err.Error(), "model non-existent not found")
// Test Unload from filesystem
err = fsio.Create(&types.CreateOptions{
ID: testCase.ID + "_fs",
Source: testCase.Source,
})
assert.NoError(t, err)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID + "_fs",
Store: "fs",
})
assert.NoError(t, err)
err = manager.Unload(context.Background(), &types.UnloadOptions{
ID: testCase.ID + "_fs",
Options: map[string]interface{}{"dropTable": true},
})
assert.NoError(t, err)
// Test Unload from database
err = dbio.Create(&types.CreateOptions{
ID: testCase.ID + "_db",
Source: testCase.Source,
})
assert.NoError(t, err)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID + "_db",
Store: "db",
})
assert.NoError(t, err)
err = manager.Unload(context.Background(), &types.UnloadOptions{
ID: testCase.ID + "_db",
Options: map[string]interface{}{"dropTable": true},
})
assert.NoError(t, err)
// Clean up
err = fsio.Delete(testCase.ID + "_fs")
assert.NoError(t, err)
err = dbio.Delete(testCase.ID + "_db")
assert.NoError(t, err)
err = cleanTestData()
assert.NoError(t, err)
}
func TestModelReload(t *testing.T) {
testCase := NewTestCase()
fsio := io.NewFS(types.TypeModel)
dbio := io.NewDB(types.TypeModel)
manager := New("", fsio, dbio)
// Test Reload with nil options
err := manager.Reload(context.Background(), nil)
assert.Error(t, err)
assert.Contains(t, err.Error(), "reload options is required")
// Test Reload with empty ID
err = manager.Reload(context.Background(), &types.ReloadOptions{})
assert.Error(t, err)
assert.Contains(t, err.Error(), "reload options id is required")
// Test Reload from filesystem
err = fsio.Create(&types.CreateOptions{
ID: testCase.ID + "_fs",
Source: testCase.Source,
})
assert.NoError(t, err)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID + "_fs",
Store: "fs",
})
assert.NoError(t, err)
err = manager.Reload(context.Background(), &types.ReloadOptions{
ID: testCase.ID + "_fs",
Store: "fs",
Options: map[string]interface{}{"migrate": true},
})
assert.NoError(t, err)
// Test Reload from database
err = dbio.Create(&types.CreateOptions{
ID: testCase.ID + "_db",
Source: testCase.Source,
})
assert.NoError(t, err)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID + "_db",
Store: "db",
})
assert.NoError(t, err)
err = manager.Reload(context.Background(), &types.ReloadOptions{
ID: testCase.ID + "_db",
Store: "db",
Options: map[string]interface{}{"migrate": true},
})
assert.NoError(t, err)
// Clean up
err = fsio.Delete(testCase.ID + "_fs")
assert.NoError(t, err)
err = dbio.Delete(testCase.ID + "_db")
assert.NoError(t, err)
err = cleanTestData()
assert.NoError(t, err)
}
func TestModelLoaded(t *testing.T) {
testCase := NewTestCase()
fsio := io.NewFS(types.TypeModel)
dbio := io.NewDB(types.TypeModel)
manager := New("", fsio, dbio)
// Test Load from filesystem
err := fsio.Create(&types.CreateOptions{
ID: testCase.ID + "_fs",
Source: testCase.Source,
})
assert.NoError(t, err)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID + "_fs",
Store: "fs",
})
assert.NoError(t, err)
// Test Load from database
err = dbio.Create(&types.CreateOptions{
ID: testCase.ID + "_db",
Source: testCase.Source,
})
assert.NoError(t, err)
err = manager.Load(context.Background(), &types.LoadOptions{
ID: testCase.ID + "_db",
Store: "db",
})
assert.NoError(t, err)
// Test Loaded
infos, err := manager.Loaded(context.Background())
assert.NoError(t, err)
assert.NotNil(t, infos)
assert.Contains(t, infos, testCase.ID+"_fs")
assert.Contains(t, infos, testCase.ID+"_db")
// Verify metadata fields for filesystem model
fsInfo := infos[testCase.ID+"_fs"]
assert.Equal(t, testCase.ID+"_fs", fsInfo.ID)
assert.Equal(t, types.TypeModel, fsInfo.Type)
assert.Equal(t, testCase.Label, fsInfo.Label)
assert.Equal(t, testCase.Description, fsInfo.Description)
assert.ElementsMatch(t, testCase.Tags, fsInfo.Tags)
assert.False(t, fsInfo.Readonly)
assert.False(t, fsInfo.Builtin)
// assert.False(t, fsInfo.Mtime.IsZero())
// assert.False(t, fsInfo.Ctime.IsZero())
// Verify metadata fields for database model
dbInfo := infos[testCase.ID+"_db"]
assert.Equal(t, testCase.ID+"_db", dbInfo.ID)
assert.Equal(t, types.TypeModel, dbInfo.Type)
assert.Equal(t, testCase.Label, dbInfo.Label)
assert.Equal(t, testCase.Description, dbInfo.Description)
assert.ElementsMatch(t, testCase.Tags, dbInfo.Tags)
assert.False(t, dbInfo.Readonly)
assert.False(t, dbInfo.Builtin)
// assert.False(t, dbInfo.Mtime.IsZero())
// assert.False(t, dbInfo.Ctime.IsZero())
// Clean up
err = fsio.Delete(testCase.ID + "_fs")
assert.NoError(t, err)
err = dbio.Delete(testCase.ID + "_db")
assert.NoError(t, err)
err = cleanTestData()
assert.NoError(t, err)
}
func TestModelValidate(t *testing.T) {
manager := New("", nil, nil)
// Test Validate
valid, messages := manager.Validate(context.Background(), "test source")
assert.True(t, valid)
assert.Empty(t, messages)
}
func TestModelExecute(t *testing.T) {
manager := New("", nil, nil)
// Test Execute
result, err := manager.Execute(context.Background(), "test_id", "test_method")
assert.Error(t, err)
assert.Contains(t, err.Error(), "Not implemented")
assert.Nil(t, result)
}

62
dsl/types/interfaces.go Normal file
View file

@ -0,0 +1,62 @@
package types
import (
"context"
)
// DSL interface
type DSL interface {
Inspect(ctx context.Context, id string) (*Info, error) // Inspect DSL
Path(ctx context.Context, id string) (string, error) // Get Path by id, ( If the DSL is saved as file, return the file path )
Source(ctx context.Context, id string) (string, error) // Get Source by id
List(ctx context.Context, opts *ListOptions) ([]*Info, error) // List All DSLs including unloaded/error DSLs
Exists(ctx context.Context, id string) (bool, error) // Check if the DSL exists
// DSL Operations
Create(ctx context.Context, options *CreateOptions) error // Create DSL, Create will unload the DSL first, then create the DSL to DB
Update(ctx context.Context, options *UpdateOptions) error // Update DSL, Update will unload the DSL first, then update the DSL, if update info only, will not unload the DSL
Delete(ctx context.Context, options *DeleteOptions) error // Delete DSL, Delete will unload the DSL first, then delete the DSL file
// Load manager
Load(ctx context.Context, options *LoadOptions) error // Load DSL, Load will unload the DSL first, then load the DSL from DB or file system
Reload(ctx context.Context, options *ReloadOptions) error // Reload DSL, Reload will unload the DSL first, then reload the DSL from DB or file system
Unload(ctx context.Context, options *UnloadOptions) error // Unload DSL, Unload will unload the DSL from memory
// Execute
Execute(ctx context.Context, id string, method string, args ...any) (any, error) // Execute DSL (Some DSLs can be executed)
// Validate
Validate(ctx context.Context, source string) (bool, []LintMessage) // Validate DSL, Validate will validate the DSL from source
}
// Manager interface
type Manager interface {
// Get all loaded DSLs
Loaded(ctx context.Context) (map[string]*Info, error) // Get all loaded DSLs
// Load DSL, Load will unload the DSL first, then load the DSL from DB or file system
Load(ctx context.Context, options *LoadOptions) error
// Unload DSL, Unload will unload the DSL from memory
Unload(ctx context.Context, options *UnloadOptions) error
// Reload DSL, Reload will unload the DSL first, then reload the DSL from DB or file system
Reload(ctx context.Context, options *ReloadOptions) error
// Validate DSL, Validate will validate the DSL from source
Validate(ctx context.Context, source string) (bool, []LintMessage)
// Execute DSL (Some DSLs can be executed)
Execute(ctx context.Context, id string, method string, args ...any) (any, error)
}
// IO interface
type IO interface {
Inspect(id string) (*Info, bool, error)
Source(id string) (string, bool, error)
List(options *ListOptions) ([]*Info, error)
Create(options *CreateOptions) error
Update(options *UpdateOptions) error
Delete(id string) error
Exists(id string) (bool, error)
}

170
dsl/types/types.go Normal file
View file

@ -0,0 +1,170 @@
package types
import (
"time"
)
// Type for DSL
type Type string
// Status for DSL
type Status string
// StoreType for DSL store
type StoreType string
// LintSeverity for DSL linter
type LintSeverity string
// StoreType for DSL store
const (
StoreTypeDB StoreType = "db"
StoreTypeFile StoreType = "file"
)
// Status for DSL
const (
StatusLoading Status = "loading"
StatusLoaded Status = "loaded"
StatusError Status = "error"
)
// LintSeverity for DSL linter
const (
LintSeverityError LintSeverity = "error"
LintSeverityWarning LintSeverity = "warning"
LintSeverityInfo LintSeverity = "info"
LintSeverityHint LintSeverity = "hint"
)
// Type for DSL
const (
// TypeModel for model
TypeModel Type = "model"
// TypeAPI for api
TypeAPI Type = "api"
// TypeConnector for connector
TypeConnector Type = "connector"
// TypeMCPServer for MCP server
TypeMCPServer Type = "mcp-server"
// TypeMCPClient for MCP client
TypeMCPClient Type = "mcp-client"
// TypeStore for store
TypeStore Type = "store"
// TypeSchedule for schedule
TypeSchedule Type = "schedule"
// TypeTable for table
TypeTable Type = "table"
// TypeForm for form
TypeForm Type = "form"
// TypeList for list
TypeList Type = "list"
// TypeChart for chart
TypeChart Type = "chart"
// TypeDashboard for dashboard
TypeDashboard Type = "dashboard"
// TypeFlow for flow
TypeFlow Type = "flow"
// TypePipe for pipe
TypePipe Type = "pipe"
// TypeAIGC for aigc
TypeAIGC Type = "aigc"
// TypeUnknown for unknown
TypeUnknown Type = "unknown"
)
// Info for DSL
type Info struct {
ID string `json:"id" yaml:"id"` // Unique identifier for the DSL instance
Type Type `json:"type" yaml:"type"` // DSL type (model, api, table, form, list, chart, dashboard, etc.)
Label string `json:"label,omitempty" yaml:"label,omitempty"` // Display name for the DSL
Description string `json:"description,omitempty" yaml:"description,omitempty"` // Detailed description of the DSL
Tags []string `json:"tags,omitempty" yaml:"tags,omitempty"` // Tags for categorization and filtering
Sort int `json:"sort,omitempty" yaml:"sort,omitempty"` // Sort order for display, default is 0
Path string `json:"path" yaml:"path"` // File system path or identifier
Store StoreType `json:"store" yaml:"store"` // Storage type (file or database)
Readonly bool `json:"readonly,omitempty" yaml:"readonly,omitempty"` // Whether the DSL is readonly
Builtin bool `json:"built_in,omitempty" yaml:"built_in,omitempty"` // Whether this is a built-in DSL
Status Status `json:"status,omitempty" yaml:"status,omitempty"` // Current status (loading, loaded, error)
Mtime time.Time `json:"mtime" yaml:"mtime"` // Last modification timestamp
Ctime time.Time `json:"ctime" yaml:"ctime"` // Creation timestamp
Source string `json:"source,omitempty" yaml:"source,omitempty"` // Source content, only available when explicitly requested
}
// ListOptions for DSL list
type ListOptions struct {
Sort string
Order string
Store StoreType
Source bool
Tags []string
Pattern string // Pattern for file name matching, e.g. "test_*" for test files
}
// CreateOptions for DSL upsert
type CreateOptions struct {
ID string // ID is the id of the DSL, if not provided, a new id will be generated, required
Source string // Source is the source of the DSL, if not provided, the DSL will be loaded from the file system
Store StoreType // Store is the store type of the DSL, if not provided, the DSL will be loaded from the file system
Load map[string]interface{} // LoadOptions is the options for the DSL, if not provided, the DSL will be loaded from the file system
}
// UpdateOptions for DSL upsert
type UpdateOptions struct {
ID string // ID is the id of the DSL, if not provided, a new id will be generated, required
Info *Info // Info is the info of the DSL, if not provided, the DSL will be loaded from the file system, one of info or source must be provided
Source string // Source is the source of the DSL, if not provided, the DSL will be loaded from the file system, one of info or source must be provided
Reload map[string]interface{} // ReloadOptions is the options for the DSL, if not provided, the DSL will be loaded from the file system
}
// DeleteOptions for DSL delete options
type DeleteOptions struct {
ID string // ID is the id of the DSL, if not provided, a new id will be generated, required
Path string // Path is the path of the DSL, if not provided, the DSL will be loaded from the file system
Options map[string]interface{} // Options is the options for the DSL, if not provided, the DSL will be loaded from the file system
}
// LoadOptions for DSL load options
type LoadOptions struct {
ID string
Path string
Source string
Store StoreType
Options map[string]interface{}
}
// UnloadOptions for DSL unload options
type UnloadOptions struct {
ID string
Path string
Store StoreType
Options map[string]interface{}
}
// ReloadOptions for DSL reload options
type ReloadOptions struct {
ID string
Path string
Source string
Store StoreType
Options map[string]interface{}
}
// LintMessage for DSL linter
type LintMessage struct {
File string
Line int
Column int
Message string
Severity LintSeverity
}
var lintMessages []LintMessage

194
dsl/types/utils.go Normal file
View file

@ -0,0 +1,194 @@
package types
import (
"os"
"path/filepath"
"strings"
)
// ToPath convert id to path
func ToPath(typ Type, id string) string {
// Get the root path and the extensions of the type
root, exts := TypeRootAndExts(typ)
ext := ".yao"
if len(exts) > 0 {
ext = exts[0]
}
// 1. Replace all . to /
path := strings.ReplaceAll(id, ".", string(os.PathSeparator))
// 2. Replace all __ to .
path = strings.ReplaceAll(path, "__", ".")
// 3. Join the root path
return filepath.Join(root, path) + ext
}
// ToID convert file path to id
func ToID(path string) string {
typ := DetectType(path)
return WithTypeToID(typ, path)
}
// WithTypeToID convert file path to id
func WithTypeToID(typ Type, path string) string {
// Get the root path and the extensions of the type
root, exts := TypeRootAndExts(typ)
// 0. if the first character is /, remove it
if strings.HasPrefix(path, string(os.PathSeparator)) {
path = strings.TrimPrefix(path, string(os.PathSeparator))
}
// 1. Split the path by /
parts := strings.Split(path, string(os.PathSeparator))
if len(parts) > 0 && parts[0] == root {
// Skip the root path
parts = parts[1:]
// Remove the extension only if parts is not empty
if len(parts) > 0 {
last := parts[len(parts)-1]
for _, ext := range exts {
if strings.HasSuffix(last, ext) {
parts[len(parts)-1] = strings.TrimSuffix(last, ext)
}
}
}
// Join the parts
path = strings.Join(parts, string(os.PathSeparator))
}
// 2. Replace All . to __
path = strings.ReplaceAll(path, ".", "__")
// 3. Replace all / to .
path = strings.ReplaceAll(path, string(os.PathSeparator), ".")
return path
}
// DetectType detect the type by the file path
func DetectType(path string) Type {
parts := strings.Split(path, string(os.PathSeparator))
if len(parts) < 2 {
return TypeUnknown
}
root := parts[0]
last := parts[len(parts)-1]
extParts := strings.Split(last, ".")
if len(extParts) < 2 {
return TypeUnknown
}
ext := extParts[len(extParts)-2]
// Detect the type by the extension
switch ext {
case "http":
return TypeAPI
case "sch":
return TypeSchedule
case "table":
return TypeTable
case "form":
return TypeForm
case "list":
return TypeList
case "chart":
return TypeChart
case "dash":
return TypeDashboard
case "flow":
return TypeFlow
case "pipe":
return TypePipe
case "ai":
return TypeAIGC
case "mod":
return TypeModel
case "conn":
return TypeConnector
case "lru", "redis", "mongo", "badger":
return TypeStore
}
// Detect the type by the root path
switch root {
case "models":
return TypeModel
case "connectors":
return TypeConnector
case "mcps":
return TypeMCPClient
case "apis":
if ext == "http" {
return TypeAPI
}
if ext == "mcp" {
return TypeMCPServer
}
return TypeUnknown
case "schedules":
return TypeSchedule
case "tables":
return TypeTable
case "forms":
return TypeForm
case "lists":
return TypeList
case "charts":
return TypeChart
case "dashboards":
return TypeDashboard
case "flows":
return TypeFlow
case "pipes":
return TypePipe
case "aigcs":
return TypeAIGC
case "stores":
return TypeStore
default:
return TypeUnknown
}
}
// TypeRootAndExts return the root path and the extensions of the type
func TypeRootAndExts(typ Type) (string, []string) {
switch typ {
case TypeModel:
return "models", []string{".mod.yao", ".mod.jsonc", ".mod.json"}
case TypeConnector:
return "connectors", []string{".conn.yao", ".conn.jsonc", ".conn.json"}
case TypeMCPClient, TypeMCPServer:
return "mcps", []string{".mcp.yao", ".mcp.jsonc", ".mcp.json"}
case TypeAPI:
return "apis", []string{".http.yao", ".http.jsonc", ".http.json"}
case TypeSchedule:
return "schedules", []string{".sch.yao", ".sch.jsonc", ".sch.json"}
case TypeTable:
return "tables", []string{".table.yao", ".table.jsonc", ".table.json"}
case TypeForm:
return "forms", []string{".form.yao", ".form.jsonc", ".form.json"}
case TypeList:
return "lists", []string{".list.yao", ".list.jsonc", ".list.json"}
case TypeChart:
return "charts", []string{".chart.yao", ".chart.jsonc", ".chart.json"}
case TypeDashboard:
return "dashboards", []string{".dash.yao", ".dash.jsonc", ".dash.json"}
case TypeFlow:
return "flows", []string{".flow.yao", ".flow.jsonc", ".flow.json"}
case TypePipe:
return "pipes", []string{".pipe.yao", ".pipe.jsonc", ".pipe.json"}
case TypeAIGC:
return "aigcs", []string{".ai.yao", ".ai.jsonc", ".ai.json"}
case TypeStore:
return "stores", []string{".lru.yao", ".redis.yao", ".mongo.yao", ".badger.yao", ".store.yao", ".store.jsonc", ".store.json"}
default:
return "", []string{}
}
}

601
dsl/types/utils_test.go Normal file
View file

@ -0,0 +1,601 @@
package types
import (
"os"
"path/filepath"
"testing"
)
func TestToPath(t *testing.T) {
tests := []struct {
name string
typ Type
id string
want string
}{
{
name: "Model with dots and underscores",
typ: TypeModel,
id: "user__profile.admin",
want: filepath.Join("models", "user.profile", "admin.mod.yao"),
},
{
name: "API with simple id",
typ: TypeAPI,
id: "user.login",
want: filepath.Join("apis", "user", "login.http.yao"),
},
{
name: "Unknown type (defaults to .yao)",
typ: TypeUnknown,
id: "test",
want: filepath.Join("", "test.yao"),
},
{
name: "Connector with nested path",
typ: TypeConnector,
id: "database.mysql__config",
want: filepath.Join("connectors", "database", "mysql.config.conn.yao"),
},
{
name: "Type with no extensions",
typ: Type("unknown"),
id: "test",
want: filepath.Join("", "test.yao"),
},
{
name: "Type with empty extensions",
typ: Type(""),
id: "test",
want: filepath.Join("", "test.yao"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := ToPath(tt.typ, tt.id); got != tt.want {
t.Errorf("ToPath() = %v, want %v", got, tt.want)
}
})
}
}
func TestToID(t *testing.T) {
tests := []struct {
name string
path string
want string
}{
{
name: "Model file path",
path: filepath.Join("models", "user.mod.yao"),
want: "user",
},
{
name: "API file path",
path: filepath.Join("apis", "user", "login.http.yao"),
want: "user.login",
},
{
name: "Form file path with dots",
path: filepath.Join("forms", "user.profile", "edit.form.yao"),
want: "user__profile.edit",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := ToID(tt.path); got != tt.want {
t.Errorf("ToID() = %v, want %v", got, tt.want)
}
})
}
}
func TestWithTypeToID(t *testing.T) {
tests := []struct {
name string
typ Type
path string
want string
}{
{
name: "Path with leading separator",
typ: TypeModel,
path: string(os.PathSeparator) + filepath.Join("models", "user.mod.yao"),
want: "user",
},
{
name: "Path without leading separator",
typ: TypeModel,
path: filepath.Join("models", "user.mod.yao"),
want: "user",
},
{
name: "Path with root not matching",
typ: TypeModel,
path: filepath.Join("other", "user.mod.yao"),
want: "other.user__mod__yao",
},
{
name: "Nested path with dots",
typ: TypeForm,
path: filepath.Join("forms", "user.profile", "edit.form.yao"),
want: "user__profile.edit",
},
{
name: "Multiple extensions matching",
typ: TypeModel,
path: filepath.Join("models", "user.mod.jsonc"),
want: "user",
},
{
name: "No extension matching",
typ: TypeModel,
path: filepath.Join("models", "user.txt"),
want: "user__txt",
},
{
name: "Path with single part",
typ: TypeModel,
path: "user.mod.yao",
want: "user__mod__yao",
},
{
name: "Store type with multiple extensions",
typ: TypeStore,
path: filepath.Join("stores", "cache.redis.yao"),
want: "cache",
},
{
name: "Empty path",
typ: TypeModel,
path: "",
want: "",
},
{
name: "Path with root matching but no parts",
typ: TypeModel,
path: "models",
want: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := WithTypeToID(tt.typ, tt.path); got != tt.want {
t.Errorf("WithTypeToID() = %v, want %v", got, tt.want)
}
})
}
}
func TestDetectType(t *testing.T) {
tests := []struct {
name string
path string
want Type
}{
// Test by extension
{
name: "HTTP API",
path: filepath.Join("apis", "user.http.yao"),
want: TypeAPI,
},
{
name: "Schedule",
path: filepath.Join("schedules", "backup.sch.yao"),
want: TypeSchedule,
},
{
name: "Table",
path: filepath.Join("tables", "user.table.yao"),
want: TypeTable,
},
{
name: "Form",
path: filepath.Join("forms", "user.form.yao"),
want: TypeForm,
},
{
name: "List",
path: filepath.Join("lists", "user.list.yao"),
want: TypeList,
},
{
name: "Chart",
path: filepath.Join("charts", "sales.chart.yao"),
want: TypeChart,
},
{
name: "Dashboard",
path: filepath.Join("dashboards", "main.dash.yao"),
want: TypeDashboard,
},
{
name: "Flow",
path: filepath.Join("flows", "process.flow.yao"),
want: TypeFlow,
},
{
name: "Pipe",
path: filepath.Join("pipes", "transform.pipe.yao"),
want: TypePipe,
},
{
name: "AIGC",
path: filepath.Join("aigcs", "chat.ai.yao"),
want: TypeAIGC,
},
{
name: "Model by extension",
path: filepath.Join("models", "user.mod.yao"),
want: TypeModel,
},
{
name: "Connector by extension",
path: filepath.Join("connectors", "db.conn.yao"),
want: TypeConnector,
},
{
name: "Store LRU",
path: filepath.Join("stores", "cache.lru.yao"),
want: TypeStore,
},
{
name: "LRU extension in non-stores directory",
path: filepath.Join("other", "cache.lru.yao"),
want: TypeStore,
},
{
name: "Store Redis",
path: filepath.Join("stores", "cache.redis.yao"),
want: TypeStore,
},
{
name: "Store Mongo",
path: filepath.Join("stores", "cache.mongo.yao"),
want: TypeStore,
},
{
name: "Store Badger",
path: filepath.Join("stores", "cache.badger.yao"),
want: TypeStore,
},
{
name: "Store by extension",
path: filepath.Join("stores", "cache.store.yao"),
want: TypeStore,
},
{
name: "MCP extension in non-apis directory",
path: filepath.Join("other", "service.mcp.yao"),
want: TypeUnknown,
},
// Test by root path
{
name: "Model by root",
path: filepath.Join("models", "user.yao"),
want: TypeModel,
},
{
name: "Connector by root",
path: filepath.Join("connectors", "db.yao"),
want: TypeConnector,
},
{
name: "MCP Client",
path: filepath.Join("mcps", "client.yao"),
want: TypeMCPClient,
},
{
name: "API by root with http ext",
path: filepath.Join("apis", "user.http.yao"),
want: TypeAPI,
},
{
name: "MCP Server",
path: filepath.Join("apis", "server.mcp.yao"),
want: TypeMCPServer,
},
{
name: "MCP by extension",
path: filepath.Join("mcps", "client.mcp.yao"),
want: TypeMCPClient,
},
{
name: "API by root unknown ext",
path: filepath.Join("apis", "user.unknown.yao"),
want: TypeUnknown,
},
{
name: "Schedule by root",
path: filepath.Join("schedules", "backup.yao"),
want: TypeSchedule,
},
{
name: "Table by root",
path: filepath.Join("tables", "user.yao"),
want: TypeTable,
},
{
name: "Form by root",
path: filepath.Join("forms", "user.yao"),
want: TypeForm,
},
{
name: "List by root",
path: filepath.Join("lists", "user.yao"),
want: TypeList,
},
{
name: "Chart by root",
path: filepath.Join("charts", "sales.yao"),
want: TypeChart,
},
{
name: "Dashboard by root",
path: filepath.Join("dashboards", "main.yao"),
want: TypeDashboard,
},
{
name: "Flow by root",
path: filepath.Join("flows", "process.yao"),
want: TypeFlow,
},
{
name: "Pipe by root",
path: filepath.Join("pipes", "transform.yao"),
want: TypePipe,
},
{
name: "AIGC by root",
path: filepath.Join("aigcs", "chat.yao"),
want: TypeAIGC,
},
{
name: "Store by root",
path: filepath.Join("stores", "cache.yao"),
want: TypeStore,
},
{
name: "Unknown root",
path: filepath.Join("unknown", "file.yao"),
want: TypeUnknown,
},
// Edge cases
{
name: "Path with less than 2 parts",
path: "file.yao",
want: TypeUnknown,
},
{
name: "File without extension",
path: filepath.Join("models", "user"),
want: TypeUnknown,
},
{
name: "File with single dot",
path: filepath.Join("models", "user.yao"),
want: TypeModel,
},
{
name: "Empty path",
path: "",
want: TypeUnknown,
},
{
name: "Path with single component",
path: "file",
want: TypeUnknown,
},
{
name: "File with extension parts length < 2",
path: filepath.Join("models", "user"),
want: TypeUnknown,
},
{
name: "File with extension matching filename",
path: filepath.Join("models", "http.yao"),
want: TypeAPI,
},
{
name: "File with extension matching filename - sch",
path: filepath.Join("schedules", "sch.yao"),
want: TypeSchedule,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := DetectType(tt.path); got != tt.want {
t.Errorf("DetectType() = %v, want %v", got, tt.want)
}
})
}
}
func TestTypeRootAndExts(t *testing.T) {
tests := []struct {
name string
typ Type
wantRoot string
wantExts []string
}{
{
name: "Model",
typ: TypeModel,
wantRoot: "models",
wantExts: []string{".mod.yao", ".mod.jsonc", ".mod.json"},
},
{
name: "Connector",
typ: TypeConnector,
wantRoot: "connectors",
wantExts: []string{".conn.yao", ".conn.jsonc", ".conn.json"},
},
{
name: "MCP Client",
typ: TypeMCPClient,
wantRoot: "mcps",
wantExts: []string{".mcp.yao", ".mcp.jsonc", ".mcp.json"},
},
{
name: "MCP Server",
typ: TypeMCPServer,
wantRoot: "mcps",
wantExts: []string{".mcp.yao", ".mcp.jsonc", ".mcp.json"},
},
{
name: "API",
typ: TypeAPI,
wantRoot: "apis",
wantExts: []string{".http.yao", ".http.jsonc", ".http.json"},
},
{
name: "Schedule",
typ: TypeSchedule,
wantRoot: "schedules",
wantExts: []string{".sch.yao", ".sch.jsonc", ".sch.json"},
},
{
name: "Table",
typ: TypeTable,
wantRoot: "tables",
wantExts: []string{".table.yao", ".table.jsonc", ".table.json"},
},
{
name: "Form",
typ: TypeForm,
wantRoot: "forms",
wantExts: []string{".form.yao", ".form.jsonc", ".form.json"},
},
{
name: "List",
typ: TypeList,
wantRoot: "lists",
wantExts: []string{".list.yao", ".list.jsonc", ".list.json"},
},
{
name: "Chart",
typ: TypeChart,
wantRoot: "charts",
wantExts: []string{".chart.yao", ".chart.jsonc", ".chart.json"},
},
{
name: "Dashboard",
typ: TypeDashboard,
wantRoot: "dashboards",
wantExts: []string{".dash.yao", ".dash.jsonc", ".dash.json"},
},
{
name: "Flow",
typ: TypeFlow,
wantRoot: "flows",
wantExts: []string{".flow.yao", ".flow.jsonc", ".flow.json"},
},
{
name: "Pipe",
typ: TypePipe,
wantRoot: "pipes",
wantExts: []string{".pipe.yao", ".pipe.jsonc", ".pipe.json"},
},
{
name: "AIGC",
typ: TypeAIGC,
wantRoot: "aigcs",
wantExts: []string{".ai.yao", ".ai.jsonc", ".ai.json"},
},
{
name: "Store",
typ: TypeStore,
wantRoot: "stores",
wantExts: []string{".lru.yao", ".redis.yao", ".mongo.yao", ".badger.yao", ".store.yao", ".store.jsonc", ".store.json"},
},
{
name: "Unknown",
typ: TypeUnknown,
wantRoot: "",
wantExts: []string{},
},
{
name: "Empty type",
typ: Type(""),
wantRoot: "",
wantExts: []string{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotRoot, gotExts := TypeRootAndExts(tt.typ)
if gotRoot != tt.wantRoot {
t.Errorf("TypeRootAndExts() root = %v, want %v", gotRoot, tt.wantRoot)
}
if len(gotExts) != len(tt.wantExts) {
t.Errorf("TypeRootAndExts() exts length = %v, want %v", len(gotExts), len(tt.wantExts))
return
}
for i, ext := range gotExts {
if ext != tt.wantExts[i] {
t.Errorf("TypeRootAndExts() exts[%d] = %v, want %v", i, ext, tt.wantExts[i])
}
}
})
}
}
// Test integration scenarios
func TestIntegration(t *testing.T) {
tests := []struct {
name string
typ Type
id string
wantPath string
wantID string
}{
{
name: "Model round trip",
typ: TypeModel,
id: "user__profile.admin",
wantPath: filepath.Join("models", "user.profile", "admin.mod.yao"),
wantID: "user__profile.admin",
},
{
name: "API round trip",
typ: TypeAPI,
id: "user.login",
wantPath: filepath.Join("apis", "user", "login.http.yao"),
wantID: "user.login",
},
{
name: "Complex nested path",
typ: TypeForm,
id: "admin__panel.user__management.edit",
wantPath: filepath.Join("forms", "admin.panel", "user.management", "edit.form.yao"),
wantID: "admin__panel.user__management.edit",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Test ID to Path
path := ToPath(tt.typ, tt.id)
if path != tt.wantPath {
t.Errorf("ToPath() = %v, want %v", path, tt.wantPath)
}
// Test Path to ID
id := WithTypeToID(tt.typ, path)
if id != tt.wantID {
t.Errorf("WithTypeToID() = %v, want %v", id, tt.wantID)
}
// Test DetectType
detectedType := DetectType(path)
if detectedType != tt.typ {
t.Errorf("DetectType() = %v, want %v", detectedType, tt.typ)
}
})
}
}

186
engine/load.go
View file

@ -12,6 +12,7 @@ import (
"github.com/yaoapp/kun/exception"
"github.com/yaoapp/yao/aigc"
"github.com/yaoapp/yao/api"
"github.com/yaoapp/yao/attachment"
"github.com/yaoapp/yao/cert"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/connector"
@ -19,10 +20,11 @@ import (
"github.com/yaoapp/yao/flow"
"github.com/yaoapp/yao/fs"
"github.com/yaoapp/yao/i18n"
"github.com/yaoapp/yao/importer"
"github.com/yaoapp/yao/kb"
"github.com/yaoapp/yao/moapi"
"github.com/yaoapp/yao/model"
"github.com/yaoapp/yao/neo"
"github.com/yaoapp/yao/openapi"
"github.com/yaoapp/yao/pack"
"github.com/yaoapp/yao/pipe"
"github.com/yaoapp/yao/plugin"
@ -61,8 +63,14 @@ type LoadOption struct {
IsReload bool `json:"reload"`
}
// Warning the warning
type Warning struct {
Widget string
Error error
}
// Load application engine
func Load(cfg config.Config, options LoadOption) (err error) {
func Load(cfg config.Config, options LoadOption) (warnings []Warning, err error) {
defer func() { err = exception.Catch(recover()) }()
exception.Mode = cfg.Mode
@ -80,78 +88,98 @@ func Load(cfg config.Config, options LoadOption) (err error) {
err = loadApp(cfg.AppSource)
if err != nil {
printErr(cfg.Mode, "Load Application", err)
warnings = append(warnings, Warning{Widget: "Load Application", Error: err})
}
// Make Database connections
err = share.DBConnect(cfg.DB)
if err != nil {
printErr(cfg.Mode, "DB", err)
// printErr(cfg.Mode, "DB", err)
warnings = append(warnings, Warning{Widget: "DB", Error: err})
}
// Load Certs
err = cert.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Cert", err)
// printErr(cfg.Mode, "Cert", err)
warnings = append(warnings, Warning{Widget: "Cert", Error: err})
}
// Load Connectors
err = connector.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Connector", err)
// printErr(cfg.Mode, "Connector", err)
warnings = append(warnings, Warning{Widget: "Connector", Error: err})
}
// Load FileSystem
err = fs.Load(cfg)
if err != nil {
printErr(cfg.Mode, "FileSystem", err)
// printErr(cfg.Mode, "FileSystem", err)
warnings = append(warnings, Warning{Widget: "FileSystem", Error: err})
}
// Load i18n
err = i18n.Load(cfg)
if err != nil {
printErr(cfg.Mode, "i18n", err)
// printErr(cfg.Mode, "i18n", err)
warnings = append(warnings, Warning{Widget: "i18n", Error: err})
}
// start v8 runtime
err = runtime.Start(cfg)
if err != nil {
printErr(cfg.Mode, "Runtime", err)
// printErr(cfg.Mode, "Runtime", err)
warnings = append(warnings, Warning{Widget: "Runtime", Error: err})
}
// Load Query Engine
err = query.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Query Engine", err)
// printErr(cfg.Mode, "Query Engine", err)
warnings = append(warnings, Warning{Widget: "Query Engine", Error: err})
}
// Load Scripts
err = script.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Script", err)
// printErr(cfg.Mode, "Script", err)
warnings = append(warnings, Warning{Widget: "Script", Error: err})
}
// Load Models
err = model.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Model", err)
// printErr(cfg.Mode, "Model", err)
warnings = append(warnings, Warning{Widget: "Model", Error: err})
}
// Load Data flows
err = flow.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Flow", err)
// printErr(cfg.Mode, "Flow", err)
warnings = append(warnings, Warning{Widget: "Flow", Error: err})
}
// Load Stores
err = store.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Store", err)
// printErr(cfg.Mode, "Store", err)
warnings = append(warnings, Warning{Widget: "Store", Error: err})
}
// Load Uploaders
err = attachment.Load(cfg)
if err != nil {
// printErr(cfg.Mode, "Uploader", err)
warnings = append(warnings, Warning{Widget: "Uploader", Error: err})
}
// Load Plugins
err = plugin.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Plugin", err)
// printErr(cfg.Mode, "Plugin", err)
warnings = append(warnings, Warning{Widget: "Plugin", Error: err})
}
// Load Rules
@ -165,110 +193,141 @@ func Load(cfg config.Config, options LoadOption) (err error) {
// Load build-in widgets (table / form / chart / ...)
err = widgets.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Widgets", err)
// printErr(cfg.Mode, "Widgets", err)
warnings = append(warnings, Warning{Widget: "Widgets", Error: err})
}
// Load Importers
err = importer.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Plugin", err)
}
// err = importer.Load(cfg)
// if err != nil {
// // printErr(cfg.Mode, "Plugin", err)
// warnings = append(warnings, Warning{Widget: "Plugin", Error: err})
// }
// Load Apis
err = api.Load(cfg) // 加载业务接口 API
if err != nil {
printErr(cfg.Mode, "API", err)
// printErr(cfg.Mode, "API", err)
warnings = append(warnings, Warning{Widget: "API", Error: err})
}
// Load Sockets
err = socket.Load(cfg) // Load sockets
if err != nil {
printErr(cfg.Mode, "Socket", err)
// printErr(cfg.Mode, "Socket", err)
warnings = append(warnings, Warning{Widget: "Socket", Error: err})
}
// Load websockets (client mode)
err = websocket.Load(cfg)
if err != nil {
printErr(cfg.Mode, "WebSocket", err)
// printErr(cfg.Mode, "WebSocket", err)
warnings = append(warnings, Warning{Widget: "WebSocket", Error: err})
}
// Load tasks
err = task.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Task", err)
// printErr(cfg.Mode, "Task", err)
warnings = append(warnings, Warning{Widget: "Task", Error: err})
}
// Load schedules
err = schedule.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Schedule", err)
// printErr(cfg.Mode, "Schedule", err)
warnings = append(warnings, Warning{Widget: "Schedule", Error: err})
}
// Load AIGC
err = aigc.Load(cfg)
if err != nil {
printErr(cfg.Mode, "AIGC", err)
}
// Load Neo
err = neo.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Neo", err)
// printErr(cfg.Mode, "AIGC", err)
warnings = append(warnings, Warning{Widget: "AIGC", Error: err})
}
// Load Custom Widget
err = widget.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Widget", err)
// printErr(cfg.Mode, "Widget", err)
warnings = append(warnings, Warning{Widget: "Widget", Error: err})
}
// Load Custom Widget Instances
err = widget.LoadInstances()
if err != nil {
printErr(cfg.Mode, "Widget", err)
// printErr(cfg.Mode, "Widget", err)
warnings = append(warnings, Warning{Widget: "Widget", Error: err})
}
// Load SUI
err = sui.Load(cfg)
if err != nil {
printErr(cfg.Mode, "SUI", err)
// printErr(cfg.Mode, "SUI", err)
warnings = append(warnings, Warning{Widget: "SUI", Error: err})
}
// Load Moapi
err = moapi.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Moapi", err)
// printErr(cfg.Mode, "Moapi", err)
warnings = append(warnings, Warning{Widget: "Moapi", Error: err})
}
// Load Pipe
err = pipe.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Pipe", err)
// printErr(cfg.Mode, "Pipe", err)
warnings = append(warnings, Warning{Widget: "Pipe", Error: err})
}
// Load Knowledge Base
_, err = kb.Load(cfg)
if err != nil {
// printErr(cfg.Mode, "Knowledge Base", err)
warnings = append(warnings, Warning{Widget: "Knowledge Base", Error: err})
}
// Load Neo
err = neo.Load(cfg)
if err != nil {
// printErr(cfg.Mode, "Neo", err)
warnings = append(warnings, Warning{Widget: "Neo", Error: err})
}
for name, hook := range LoadHooks {
err = hook(cfg)
if err != nil {
printErr(cfg.Mode, name, err)
// printErr(cfg.Mode, name, err)
warnings = append(warnings, Warning{Widget: name, Error: err})
}
}
// Load OpenAPI
_, err = openapi.Load(cfg)
if err != nil {
// printErr(cfg.Mode, "OpenAPI", err)
warnings = append(warnings, Warning{Widget: "OpenAPI", Error: err})
}
// Execute AfterLoad Process if exists
if share.App.AfterLoad != "" && !options.IgnoredAfterLoad {
p, err := process.Of(share.App.AfterLoad, options)
if err != nil {
printErr(cfg.Mode, "AfterLoad", err)
return err
warnings = append(warnings, Warning{Widget: "AfterLoad", Error: err})
return warnings, err
}
_, err = p.Exec()
if err != nil {
printErr(cfg.Mode, "AfterLoad", err)
return err
warnings = append(warnings, Warning{Widget: "AfterLoad", Error: err})
return warnings, err
}
}
return nil
return warnings, nil
}
// Unload application engine
@ -378,6 +437,12 @@ func Reload(cfg config.Config, options LoadOption) (err error) {
printErr(cfg.Mode, "Store", err)
}
// Load Uploaders
err = attachment.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Uploader", err)
}
// Load Plugins
err = plugin.Load(cfg)
if err != nil {
@ -434,12 +499,25 @@ func Reload(cfg config.Config, options LoadOption) (err error) {
printErr(cfg.Mode, "AIGC", err)
}
// Load Knowledge Base
_, err = kb.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Knowledge Base", err)
}
// Load Neo
err = neo.Load(cfg)
if err != nil {
printErr(cfg.Mode, "Neo", err)
}
// Load OpenAPI
_, err = openapi.Load(cfg)
if err != nil {
printErr(cfg.Mode, "OpenAPI", err)
}
// Execute AfterLoad Process if exists
if share.App.AfterLoad != "" && !options.IgnoredAfterLoad {
options.IsReload = true
@ -465,7 +543,19 @@ func Restart(cfg config.Config, options LoadOption) error {
if err != nil {
return err
}
return Load(cfg, options)
warnings, err := Load(cfg, options)
if err != nil {
return err
}
if len(warnings) > 0 {
for _, warning := range warnings {
printErr(cfg.Mode, warning.Widget, warning.Error)
}
}
return nil
}
// loadApp load the application from bindata / pkg / disk
@ -555,8 +645,20 @@ func loadApp(root string) error {
}
return []byte(val)
})
// Parse app.yao
share.App = share.AppInfo{}
return application.Parse(appFile, appData, &share.App)
err = application.Parse(appFile, appData, &share.App)
if err != nil {
return err
}
// Set default prefix
if share.App.Prefix == "" {
share.App.Prefix = "yao_"
}
return nil
}
func printErr(mode, widget string, err error) {

8
engine/load_test.go
View file

@ -13,14 +13,14 @@ import (
func TestLoad(t *testing.T) {
defer Unload()
err := Load(config.Conf, LoadOption{})
_, err := Load(config.Conf, LoadOption{})
assert.Nil(t, err)
assert.Greater(t, len(api.APIs), 0)
}
func TestReload(t *testing.T) {
defer Unload()
err := Load(config.Conf, LoadOption{})
_, err := Load(config.Conf, LoadOption{})
assert.Nil(t, err)
Reload(config.Conf, LoadOption{})
@ -41,7 +41,7 @@ func TestLoadYaz(t *testing.T) {
cfg := config.Conf
cfg.AppSource = file
err = Load(cfg, LoadOption{})
_, err = Load(cfg, LoadOption{})
if err != nil {
t.Fatal(err)
}
@ -62,7 +62,7 @@ func TestReoadYaz(t *testing.T) {
cfg := config.Conf
cfg.AppSource = file
err = Load(cfg, LoadOption{})
_, err = Load(cfg, LoadOption{})
if err != nil {
t.Fatal(err)
}

1
engine/process.go
View file

@ -39,6 +39,7 @@ func processPing(process *process.Process) interface{} {
func processInspect(process *process.Process) interface{} {
return map[string]interface{}{
"VERSION": fmt.Sprintf("%s %s", share.VERSION, share.PRVERSION),
"CUI": fmt.Sprintf("%s %s", share.CUI, share.PRCUI),
"BUILDNAME": share.BUILDNAME,
"CONFIG": config.Conf,
}

652
excel/README.md Normal file
View file

@ -0,0 +1,652 @@
# Yao Excel Module
A Go module for manipulating Excel files with TypeScript API support.
## IMPORTANT: Always Close Resources
**IMPORTANT**: Always make sure to close Excel file handles using `excel.close` when done to prevent memory leaks and file locking issues. Failing to close handles may cause file corruption or application errors.
## Quick Example
Here's a simple but complete example showing proper resource management:
```typescript
// Open an Excel file
const h = Process("excel.Open", "data.xlsx", true);
// Perform operations
const sheets = Process("excel.Sheets", h);
Process("excel.write.Cell", h, sheets[0], "A1", "Hello World");
Process("excel.Save", h);
// IMPORTANT: Always close the handle when done
Process("excel.Close", h);
```
## Usage in TypeScript
You can use the Excel module in TypeScript through the Process API. Below are examples of common operations with return type descriptions.
### Basic Operations
#### Open an Excel file
```typescript
/**
* Opens an Excel file
* @param path - Path to the Excel file
* @param writable - Whether to open in writable mode (true) or read-only mode (false)
* @returns string - Handle ID used for subsequent operations
*/
const h: string = Process("excel.Open", "file.xlsx", true);
// Open in read-only mode (false parameter or not passed)
const hRead: string = Process("excel.Open", "file.xlsx", false);
// or simply
const h2: string = Process("excel.Open", "file.xlsx");
// IMPORTANT: Don't forget to close the handle when done
// Process("excel.Close", h);
```
### Sheet Operations
#### Create a new sheet
```typescript
/**
* Creates a new sheet in the workbook
* @param handle - Handle ID from excel.open
* @param name - Name for the new sheet
* @returns number - Index of the new sheet
*/
const idx: number = Process("excel.sheet.create", h, "NewSheet");
```
#### List all sheets
```typescript
/**
* Lists all sheets in the workbook
* @param handle - Handle ID from excel.open
* @returns string[] - Array of sheet names
*/
const sheets: string[] = Process("excel.sheet.list", h);
// Example output: ["Sheet1", "Sheet2", "NewSheet"]
```
#### Read sheet data
```typescript
/**
* Reads all data from a sheet
* @param handle - Handle ID from excel.open
* @param name - Sheet name
* @returns any[][] - Two-dimensional array of cell values
*/
const data: any[][] = Process("excel.sheet.read", h, "Sheet1");
```
#### Update sheet data
```typescript
/**
* Updates data in a sheet. Creates the sheet if it doesn't exist.
* @param handle - Handle ID from excel.open
* @param name - Sheet name
* @param data - Two-dimensional array of values to write
* @returns null
*/
const data = [
["Header1", "Header2", "Header3"],
[1, "Data1", true],
[2, "Data2", false],
];
Process("excel.sheet.update", h, "Sheet1", data);
```
#### Copy a sheet
```typescript
/**
* Copies a sheet with all its content and formatting
* @param handle - Handle ID from excel.open
* @param source - Source sheet name
* @param target - Target sheet name (must not exist)
* @returns null
*/
Process("excel.sheet.copy", h, "Sheet1", "Sheet1Copy");
```
#### Delete a sheet
```typescript
/**
* Deletes a sheet from the workbook
* @param handle - Handle ID from excel.open
* @param name - Sheet name to delete
* @returns null
*/
Process("excel.sheet.delete", h, "Sheet1Copy");
```
#### Check if a sheet exists
```typescript
/**
* Checks if a sheet exists in the workbook
* @param handle - Handle ID from excel.open
* @param name - Sheet name to check
* @returns boolean - true if sheet exists, false otherwise
*/
const exists: boolean = Process("excel.sheet.exists", h, "Sheet1");
```
#### Read sheet rows with pagination
```typescript
/**
* Reads rows from a sheet with pagination support
* @param handle - Handle ID from excel.open
* @param name - Sheet name
* @param start - Starting row index (0-based)
* @param size - Number of rows to read
* @returns string[][] - Two-dimensional array of cell values
*/
const rows: string[][] = Process("excel.sheet.rows", h, "Sheet1", 0, 10); // Read first 10 rows
```
#### Get sheet dimensions
```typescript
/**
* Gets the dimensions (number of rows and columns) of a sheet
* @param handle - Handle ID from excel.open
* @param name - Sheet name
* @returns {rows: number, cols: number} - Object containing row and column counts
*/
const dim: { rows: number; cols: number } = Process(
"excel.sheet.dimension",
h,
"Sheet1"
);
console.log(`Sheet has ${dim.rows} rows and ${dim.cols} columns`);
```
### Example: Sheet Operations Workflow
```typescript
// Open Excel file in writable mode
const h: string = Process("excel.Open", "file.xlsx", true);
// Create a new sheet
const idx: number = Process("excel.sheet.create", h, "DataSheet");
// Write some data to the new sheet
const data = [
["Name", "Age", "Active"],
["John", 30, true],
["Jane", 25, false],
];
Process("excel.sheet.update", h, "DataSheet", data);
// Make a backup copy of the sheet
Process("excel.sheet.copy", h, "DataSheet", "DataSheet_Backup");
// List all sheets to verify
const sheets: string[] = Process("excel.sheet.list", h);
console.log("Available sheets:", sheets);
// Read data from the backup sheet
const backupData: any[][] = Process("excel.sheet.read", h, "DataSheet_Backup");
console.log("Backup data:", backupData);
// Delete the backup sheet when no longer needed
Process("excel.sheet.delete", h, "DataSheet_Backup");
// Save changes
Process("excel.Save", h);
// IMPORTANT: Always close the handle when done
Process("excel.Close", h);
```
#### Get all sheets in the workbook
```typescript
/**
* Gets all sheet names in the workbook
* @param handle - Handle ID from excel.open
* @returns string[] - Array of sheet names
*/
const sheets: string[] = Process("excel.Sheets", h);
// Example output: ["Sheet1", "Sheet2"]
```
#### Close a file
```typescript
/**
* Closes an Excel file
* @param handle - Handle ID from excel.open
* @returns null
*/
Process("excel.Close", h);
```
#### Save changes to file
```typescript
/**
* Saves changes to the Excel file
* @param handle - Handle ID from excel.open
* @returns null
*/
Process("excel.Save", h);
```
### Reading Data
#### Read a cell's value
```typescript
/**
* Reads a cell's value
* @param handle - Handle ID from excel.open
* @param sheet - Sheet name
* @param cell - Cell reference (e.g. "A1")
* @returns string - Cell value
*/
const value: string = Process("excel.read.Cell", h, "SheetName", "A1");
```
#### Read all rows
```typescript
/**
* Reads all rows in a sheet
* @param handle - Handle ID from excel.open
* @param sheet - Sheet name
* @returns string[][] - Two-dimensional array of cell values
*/
const rows: string[][] = Process("excel.read.Row", h, "SheetName");
```
#### Read all columns
```typescript
/**
* Reads all columns in a sheet
* @param handle - Handle ID from excel.open
* @param sheet - Sheet name
* @returns string[][] - Two-dimensional array of cell values
*/
const columns: string[][] = Process("excel.read.Column", h, "SheetName");
```
### Writing Data
#### Write to a cell
```typescript
/**
* Writes a value to a cell
* @param handle - Handle ID from excel.open
* @param sheet - Sheet name
* @param cell - Cell reference (e.g. "A1")
* @param value - Value to write (string, number, boolean, etc.)
* @returns null
*/
Process("excel.write.Cell", h, "SheetName", "A1", "Hello World");
// Can write different types of values
Process("excel.write.Cell", h, "SheetName", "A2", 123.45);
Process("excel.write.Cell", h, "SheetName", "A3", true);
```
#### Write a row
```typescript
/**
* Writes values to a row starting at the specified cell
* @param handle - Handle ID from excel.open
* @param sheet - Sheet name
* @param startCell - Starting cell reference (e.g. "A1")
* @param values - Array of values to write
* @returns null
*/
Process("excel.write.Row", h, "SheetName", "A1", ["Cell1", "Cell2", "Cell3"]);
```
#### Write a column
```typescript
/**
* Writes values to a column starting at the specified cell
* @param handle - Handle ID from excel.open
* @param sheet - Sheet name
* @param startCell - Starting cell reference (e.g. "A1")
* @param values - Array of values to write
* @returns null
*/
Process("excel.write.Column", h, "SheetName", "A1", ["Row1", "Row2", "Row3"]);
```
#### Write multiple rows
```typescript
/**
* Writes a two-dimensional array of values starting at the specified cell
* @param handle - Handle ID from excel.open
* @param sheet - Sheet name
* @param startCell - Starting cell reference (e.g. "A1")
* @param values - Two-dimensional array of values to write
* @returns null
*/
Process("excel.write.All", h, "SheetName", "A1", [
["Row1Cell1", "Row1Cell2", "Row1Cell3"],
["Row2Cell1", "Row2Cell2", "Row2Cell3"],
]);
```
### Formatting and Styling
#### Set cell style
```typescript
/**
* Sets a cell's style
* @param handle - Handle ID from excel.open
* @param sheet - Sheet name
* @param cell - Cell reference (e.g. "A1")
* @param styleID - Style ID
* @returns null
*/
Process("excel.set.Style", h, "SheetName", "A1", 1);
```
#### Style ID Constants
When using `excel.set.Style`, you need to provide a style ID. The following style IDs are supported:
```typescript
// Border styles
const BORDER_NONE = 0; // No border
const BORDER_CONTINUOUS = 1; // Continuous border (thin)
const BORDER_CONTINUOUS_2 = 2; // Continuous border (medium)
const BORDER_DASH = 3; // Dashed border
const BORDER_DOT = 4; // Dotted border
const BORDER_CONTINUOUS_3 = 5; // Continuous border (thick)
const BORDER_DOUBLE = 6; // Double line border
const BORDER_CONTINUOUS_0 = 7; // Continuous border (hair)
const BORDER_DASH_2 = 8; // Dashed border (medium)
const BORDER_DASH_DOT = 9; // Dash-dot border
const BORDER_DASH_DOT_2 = 10; // Dash-dot border (medium)
const BORDER_DASH_DOT_DOT = 11; // Dash-dot-dot border
const BORDER_DASH_DOT_DOT_2 = 12; // Dash-dot-dot border (medium)
const BORDER_SLANT_DASH_DOT = 13; // Slanted dash-dot border
// Fill patterns
const FILL_NONE = 0; // No fill
const FILL_SOLID = 1; // Solid fill
const FILL_MEDIUM_GRAY = 2; // Medium gray fill
const FILL_DARK_GRAY = 3; // Dark gray fill
const FILL_LIGHT_GRAY = 4; // Light gray fill
const FILL_DARK_HORIZONTAL = 5; // Dark horizontal line pattern
const FILL_DARK_VERTICAL = 6; // Dark vertical line pattern
const FILL_DARK_DOWN = 7; // Dark diagonal down pattern
const FILL_DARK_UP = 8; // Dark diagonal up pattern
const FILL_DARK_GRID = 9; // Dark grid pattern
const FILL_DARK_TRELLIS = 10; // Dark trellis pattern
const FILL_LIGHT_HORIZONTAL = 11; // Light horizontal line pattern
const FILL_LIGHT_VERTICAL = 12; // Light vertical line pattern
const FILL_LIGHT_DOWN = 13; // Light diagonal down pattern
const FILL_LIGHT_UP = 14; // Light diagonal up pattern
const FILL_LIGHT_GRID = 15; // Light grid pattern
const FILL_LIGHT_TRELLIS = 16; // Light trellis pattern
const FILL_GRAY_125 = 17; // 12.5% gray fill
const FILL_GRAY_0625 = 18; // 6.25% gray fill
```
Example of creating a custom style with borders and fill:
```typescript
// Create style with thick border and light gray fill
const styleID = 1; // This would typically be a custom style ID created via the NewStyle API
// Apply the style to cell A1
Process("excel.set.Style", h, "SheetName", "A1", styleID);
```
Note: The excelize library supports creating custom styles through the `NewStyle` function. Currently, in the Yao Excel module, only predefined style IDs are supported. For more complex styling needs, consider creating a custom style in the future versions of the API.
#### Set row height
```typescript
/**
* Sets a row's height
* @param handle - Handle ID from excel.open
* @param sheet - Sheet name
* @param row - Row number
* @param height - Height in points
* @returns null
*/
Process("excel.set.RowHeight", h, "SheetName", 1, 30); // Set row 1 to 30 pts height
```
#### Set column width
```typescript
/**
* Sets column width for a range of columns
* @param handle - Handle ID from excel.open
* @param sheet - Sheet name
* @param startCol - Starting column letter
* @param endCol - Ending column letter
* @param width - Width in points
* @returns null
*/
Process("excel.set.ColumnWidth", h, "SheetName", "A", "B", 20);
```
#### Merge cells
```typescript
/**
* Merges cells in a range
* @param handle - Handle ID from excel.open
* @param sheet - Sheet name
* @param startCell - Starting cell reference (e.g. "A1")
* @param endCell - Ending cell reference (e.g. "B2")
* @returns null
*/
Process("excel.set.MergeCell", h, "SheetName", "A1", "B2");
```
#### Unmerge cells
```typescript
/**
* Unmerges previously merged cells
* @param handle - Handle ID from excel.open
* @param sheet - Sheet name
* @param startCell - Starting cell reference (e.g. "A1")
* @param endCell - Ending cell reference (e.g. "B2")
* @returns null
*/
Process("excel.set.UnmergeCell", h, "SheetName", "A1", "B2");
```
#### Set a formula
```typescript
/**
* Sets a formula in a cell
* @param handle - Handle ID from excel.open
* @param sheet - Sheet name
* @param cell - Cell reference (e.g. "C1")
* @param formula - Excel formula without the leading equals sign
* @returns null
*/
Process("excel.set.Formula", h, "SheetName", "C1", "SUM(A1:B1)");
```
#### Add a hyperlink
```typescript
/**
* Adds a hyperlink to a cell
* @param handle - Handle ID from excel.open
* @param sheet - Sheet name
* @param cell - Cell reference (e.g. "A1")
* @param url - URL for the hyperlink
* @param text - Display text for the hyperlink
* @returns null
*/
Process(
"excel.set.Link",
h,
"SheetName",
"A1",
"https://example.com",
"Visit Example"
);
```
### Iterating Through Data
#### Row Iterator
```typescript
/**
* Opens a row iterator
* @param handle - Handle ID from excel.open
* @param sheet - Sheet name
* @returns string - Row iterator ID
*/
const rid: string = Process("excel.each.OpenRow", h, "SheetName");
/**
* Gets the next row from the iterator
* @param rowID - Row iterator ID from excel.each.openrow
* @returns string[] | null - Array of cell values or null if no more rows
*/
let row: string[] | null;
while ((row = Process("excel.each.NextRow", rid)) !== null) {
// Process the row
console.log(row);
}
/**
* IMPORTANT: Always close the row iterator when done
* @param rowID - Row iterator ID from excel.each.openrow
* @returns null
*/
Process("excel.each.CloseRow", rid);
```
#### Column Iterator
```typescript
/**
* Opens a column iterator
* @param handle - Handle ID from excel.open
* @param sheet - Sheet name
* @returns string - Column iterator ID
*/
const cid: string = Process("excel.each.OpenColumn", h, "SheetName");
/**
* Gets the next column from the iterator
* @param colID - Column iterator ID from excel.each.opencolumn
* @returns string[] | null - Array of cell values or null if no more columns
*/
let col: string[] | null;
while ((col = Process("excel.each.NextColumn", cid)) !== null) {
// Process the column
console.log(col);
}
/**
* IMPORTANT: Always close the column iterator when done
* @param colID - Column iterator ID from excel.each.opencolumn
* @returns null
*/
Process("excel.each.CloseColumn", cid);
```
### Utility Functions
#### Convert between column names and indices
```typescript
/**
* Converts a column name to a column number
* @param colName - Column name (e.g. "A", "AB")
* @returns number - Column number (1-based)
*/
const colNum: number = Process("excel.convert.ColumnNameToNumber", "AK"); // Returns 37
/**
* Converts a column number to a column name
* @param colNum - Column number (1-based)
* @returns string - Column name
*/
const colName: string = Process("excel.convert.ColumnNumberToName", 37); // Returns "AK"
```
#### Convert between cell references and coordinates
```typescript
/**
* Converts a cell reference to coordinates
* @param cell - Cell reference (e.g. "A1")
* @returns number[] - Array with [columnNumber, rowNumber] (1-based)
*/
const coords: number[] = Process("excel.convert.CellNameToCoordinates", "A1"); // Returns [1, 1]
/**
* Converts coordinates to a cell reference
* @param col - Column number (1-based)
* @param row - Row number (1-based)
* @returns string - Cell reference
*/
const cellName: string = Process("excel.convert.CoordinatesToCellName", 1, 1); // Returns "A1"
```
## Complete Workflow Example
```typescript
// Open Excel file in writable mode
const h: string = Process("excel.Open", "file.xlsx", true);
// Get available sheets
const sheets: string[] = Process("excel.Sheets", h);
const sheetName: string = sheets[0];
// Read some data
const value: string = Process("excel.read.Cell", h, sheetName, "A1");
console.log("Cell A1 contains:", value);
// Write data
Process("excel.write.Cell", h, sheetName, "B1", "New Value");
Process("excel.write.Row", h, sheetName, "A2", ["Data1", "Data2", "Data3"]);
// Add a formula
Process("excel.set.Formula", h, sheetName, "D1", "SUM(A1:C1)");
// Format cells
Process("excel.set.RowHeight", h, sheetName, 1, 30);
Process("excel.set.ColumnWidth", h, sheetName, "A", "D", 15);
// Save changes
Process("excel.Save", h);
// IMPORTANT: Always close the handle when done
Process("excel.Close", h);
```
## Notes
- Always make sure to close open file handles using `excel.close` when done to prevent resource leaks and file locking issues.
- Remember to save changes with `excel.save` before closing to ensure all modifications are persisted.
- For performance reasons, try to batch operations where possible instead of making many small changes.

104
excel/each.go Normal file
View file

@ -0,0 +1,104 @@
package excel
import (
"fmt"
"sync"
"time"
"github.com/google/uuid"
"github.com/xuri/excelize/v2"
)
// Cols defines an iterator to a sheet
type Cols struct {
id string
*excelize.Cols
create int64
}
// Rows defines an iterator to a sheet
type Rows struct {
id string
*excelize.Rows
create int64
}
var openCols = sync.Map{}
var openRows = sync.Map{}
// OpenRow each row of the sheet
func (excel *Excel) OpenRow(sheet string) (string, error) {
id := uuid.NewString()
rows, err := excel.Rows(sheet)
if err != nil {
return "", err
}
openRows.Store(id, &Rows{id: id, Rows: rows, create: time.Now().Unix()})
return id, nil
}
// NextRow next row of the sheet
func NextRow(id string) ([]string, error) {
value, ok := openRows.Load(id)
if !ok {
return nil, fmt.Errorf("rows %s not found", id)
}
if value.(*Rows).Next() {
row, err := value.(*Rows).Columns()
// fmt.Printf("DEBUG: %#v %v %v\n", row, err, row == nil)
if err != nil {
return nil, err
}
if row == nil {
return []string{}, nil
}
return row, nil
}
return nil, nil
}
// CloseRow done the sheet
func CloseRow(id string) {
openRows.Delete(id)
}
// OpenColumn each cols of the sheet
func (excel *Excel) OpenColumn(sheet string) (string, error) {
id := uuid.NewString()
cols, err := excel.Cols(sheet)
if err != nil {
return "", err
}
openCols.Store(id, &Cols{id: id, Cols: cols, create: time.Now().Unix()})
return id, nil
}
// NextColumn next col of the sheet
func NextColumn(id string) ([]string, error) {
value, ok := openCols.Load(id)
if !ok {
return nil, fmt.Errorf("cols %s not found", id)
}
if value.(*Cols).Next() {
col, err := value.(*Cols).Rows()
if err != nil {
return nil, err
}
if col == nil {
return []string{}, nil
}
return col, nil
}
return nil, nil
}
// CloseColumn done the sheet
func CloseColumn(id string) {
openCols.Delete(id)
}

64
excel/each_test.go Normal file
View file

@ -0,0 +1,64 @@
package excel
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestEachCols(t *testing.T) {
files := testFiles(t)
h1, err := Open(files["test-01"], false)
if err != nil {
t.Fatal(err)
}
defer Close(h1)
xls, err := Get(h1)
if err != nil {
t.Fatal(err)
}
id, err := xls.OpenColumn("供销存管理表格")
if err != nil {
t.Fatal(err)
}
defer CloseColumn(id)
res := []string{}
for col, err := NextColumn(id); err == nil && col != nil; col, err = NextColumn(id) {
res = append(res, col...)
}
assert.Contains(t, strings.Join(res, ""), "供销存管理表格产品查询")
assert.Contains(t, strings.Join(res, ""), "刘大大")
}
func TestEachRows(t *testing.T) {
files := testFiles(t)
h1, err := Open(files["test-01"], false)
if err != nil {
t.Fatal(err)
}
defer Close(h1)
xls, err := Get(h1)
if err != nil {
t.Fatal(err)
}
id, err := xls.OpenRow("供销存管理表格")
if err != nil {
t.Fatal(err)
}
defer CloseRow(id)
res := []string{}
for row, err := NextRow(id); err == nil && row != nil; row, err = NextRow(id) {
res = append(res, row...)
}
assert.Contains(t, strings.Join(res, ""), "供销存管理表格产品查询")
assert.Contains(t, strings.Join(res, ""), "刘大大")
}

115
excel/excel.go Normal file
View file

@ -0,0 +1,115 @@
package excel
import (
"fmt"
"os"
"path/filepath"
"sync"
"time"
"github.com/google/uuid"
"github.com/xuri/excelize/v2"
"github.com/yaoapp/yao/config"
)
// Excel the excel file
type Excel struct {
id string
path string
create int64
abs string
*excelize.File
}
// openFiles the open files
var openFiles = sync.Map{}
// Open open the excel file
func Open(path string, writable bool) (string, error) {
excel := &Excel{path: path}
// GET DATA ROOT
root := config.Conf.DataRoot
absPath, err := filepath.Abs(filepath.Join(root, path))
if err != nil {
return "", err
}
if writable {
// if the file not exists, create it
if _, err := os.Stat(absPath); os.IsNotExist(err) {
// Auto create dir
dir := filepath.Dir(absPath)
if _, err := os.Stat(dir); os.IsNotExist(err) {
err := os.MkdirAll(dir, 0755) // 0755 is the default permission for directories
if err != nil {
return "", err
}
}
create := excelize.NewFile()
err := create.SaveAs(absPath)
if err != nil {
return "", err
}
create.Close()
}
excelFile, err := excelize.OpenFile(absPath)
if err != nil {
return "", err
}
id := uuid.NewString()
excel.File = excelFile
excel.id = id
excel.abs = absPath
excel.create = time.Now().Unix()
openFiles.Store(id, excel)
return id, nil
}
file, err := os.Open(absPath)
if err != nil {
return "", fmt.Errorf("open file %s failed: %w", absPath, err)
}
excelFile, err := excelize.OpenReader(file)
if err != nil {
return "", err
}
id := uuid.NewString()
excel.File = excelFile
excel.id = id
excel.abs = absPath
excel.create = time.Now().Unix()
openFiles.Store(id, excel)
return id, nil
}
// Close close the excel file
func Close(handler string) error {
excel, ok := openFiles.Load(handler)
if !ok {
return fmt.Errorf("file not found")
}
err := excel.(*Excel).Close()
if err != nil {
return err
}
openFiles.Delete(handler)
return nil
}
// Get get the excel file
func Get(handler string) (*Excel, error) {
excel, ok := openFiles.Load(handler)
if !ok {
return nil, fmt.Errorf("%s not found", handler)
}
return excel.(*Excel), nil
}

252
excel/excel_test.go Normal file
View file

@ -0,0 +1,252 @@
package excel
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/test"
)
func TestOpenClose(t *testing.T) {
files := testFiles(t)
h1, err := Open(files["test-01"], false)
if err != nil {
t.Fatal(err)
}
if _, ok := openFiles.Load(h1); !ok {
t.Fatal("open file failed")
}
h2, err := Open(files["test-02"], true)
if err != nil {
t.Fatal(err)
}
if _, ok := openFiles.Load(h2); !ok {
t.Fatal("open file failed")
}
h3, err := Open(files["test-03"], false)
if err != nil {
t.Fatal(err)
}
if _, ok := openFiles.Load(h3); !ok {
t.Fatal("open file failed")
}
_, err = Open(files["test-04"], false)
assert.Error(t, err)
err = Close(h1)
if err != nil {
t.Fatal(err)
}
if _, ok := openFiles.Load(h1); ok {
t.Fatal("close file failed")
}
}
func TestGetSheetList(t *testing.T) {
files := testFiles(t)
h1, err := Open(files["test-01"], false)
if err != nil {
t.Fatal(err)
}
defer Close(h1)
xls, err := Get(h1)
if err != nil {
t.Fatal(err)
}
sheets := xls.GetSheetList()
assert.Equal(t, []string{"供销存管理表格", "使用说明"}, sheets)
_, err = Get("not found")
assert.Error(t, err)
}
func TestOpenInvalidFile(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Create an invalid excel file in the data root
root := "excel"
invalidFile := filepath.Join(root, "invalid.xlsx")
// Ensure cleanup after test
defer func() {
if err := os.Remove(filepath.Join(config.Conf.DataRoot, invalidFile)); err != nil {
t.Logf("Failed to cleanup test file: %v", err)
}
}()
err := os.WriteFile(filepath.Join(config.Conf.DataRoot, invalidFile), []byte("invalid content"), 0644)
if err != nil {
t.Fatal(err)
}
_, err = Open(invalidFile, false)
assert.Error(t, err, "should fail to open invalid excel file")
}
func TestCloseErrors(t *testing.T) {
// Test closing non-existent handler
err := Close("non-existent-handler")
assert.Error(t, err, "should fail to close non-existent file")
// Test double close
files := testFiles(t)
h1, err := Open(files["test-01"], false)
if err != nil {
t.Fatal(err)
}
// First close
err = Close(h1)
assert.NoError(t, err)
// Second close should fail
err = Close(h1)
assert.Error(t, err, "should fail on second close")
}
func TestOpenWithInvalidPath(t *testing.T) {
// Test with invalid path
_, err := Open("../invalid/path/file.xlsx", false)
assert.Error(t, err, "should fail with invalid path")
// Test with path trying to escape data root
_, err = Open("../../../../etc/file.xlsx", false)
assert.Error(t, err, "should fail with path trying to escape data root")
}
func TestWrite(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Create a new writable file for testing
root := "excel"
testFile := filepath.Join(root, "write-test.xlsx")
// Ensure cleanup after test
defer func() {
if err := os.Remove(filepath.Join(config.Conf.DataRoot, testFile)); err != nil {
t.Logf("Failed to cleanup test file: %v", err)
}
}()
h1, err := Open(testFile, true)
if err != nil {
t.Fatal(err)
}
defer Close(h1)
xls, err := Get(h1)
if err != nil {
t.Fatal(err)
}
// Test WriteCell
err = xls.WriteCell("Sheet1", "A1", "Hello")
assert.NoError(t, err)
err = xls.WriteCell("Sheet1", "B1", 123)
assert.NoError(t, err)
err = xls.WriteCell("Sheet1", "C1", true)
assert.NoError(t, err)
// Test WriteRow
row := []interface{}{"Row1", 456, false}
err = xls.WriteRow("Sheet1", "A2", row)
assert.NoError(t, err)
// Test WriteColumn
col := []interface{}{"Col1", 789, true}
err = xls.WriteColumn("Sheet1", "D1", col)
assert.NoError(t, err)
// Test WriteAll
data := [][]interface{}{
{"Name", "Age", "City"},
{"John", 30, "New York"},
{"Alice", 25, "London"},
}
err = xls.WriteAll("Sheet2", "A1", data)
assert.NoError(t, err)
// Test error cases
// Invalid cell reference
err = xls.WriteCell("Sheet1", "invalid", "test")
assert.Error(t, err)
// Save the file to verify changes
err = xls.SaveAs(filepath.Join(config.Conf.DataRoot, testFile))
assert.NoError(t, err)
// Verify written data
val, err := xls.GetCellValue("Sheet1", "A1")
assert.NoError(t, err)
assert.Equal(t, "Hello", val)
val, err = xls.GetCellValue("Sheet2", "A1")
assert.NoError(t, err)
assert.Equal(t, "Name", val)
}
func TestSetSheet(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
root := "excel"
testFile := filepath.Join(root, "sheet-test.xlsx")
// Ensure cleanup after test
defer func() {
if err := os.Remove(filepath.Join(config.Conf.DataRoot, testFile)); err != nil {
t.Logf("Failed to cleanup test file: %v", err)
}
}()
h1, err := Open(testFile, true)
if err != nil {
t.Fatal(err)
}
defer Close(h1)
xls, err := Get(h1)
if err != nil {
t.Fatal(err)
}
// Test creating new sheet
idx, err := xls.SetSheet("NewSheet")
assert.NoError(t, err)
assert.Greater(t, idx, 0)
// Test getting existing sheet
idx2, err := xls.SetSheet("NewSheet")
assert.NoError(t, err)
assert.Equal(t, idx, idx2)
// Verify sheet exists
sheets := xls.GetSheetList()
assert.Contains(t, sheets, "NewSheet")
}
func testFiles(t *testing.T) map[string]string {
test.Prepare(t, config.Conf)
defer test.Clean()
// test data root path
root := "excel"
return map[string]string{
"test-01": filepath.Join(root, "test-01.xlsx"),
"test-02": filepath.Join(root, "test-02.xlsx"),
"test-03": filepath.Join(root, "test-03.xlsx"),
}
}

782
excel/process.go Normal file
View file

@ -0,0 +1,782 @@
package excel
import (
"github.com/xuri/excelize/v2"
"github.com/yaoapp/gou/process"
"github.com/yaoapp/kun/exception"
)
func init() {
process.RegisterGroup("excel", map[string]process.Handler{
"open": processOpen,
"close": processClose,
"save": processSave,
"sheets": processSheets,
"sheet.create": processCreateSheet,
"sheet.read": processReadSheet,
"sheet.update": processUpdateSheet,
"sheet.delete": processDeleteSheet,
"sheet.copy": processCopySheet,
"sheet.list": processListSheets,
"sheet.exists": processSheetExists,
"sheet.rows": processReadSheetRows,
"sheet.dimension": processGetSheetDimension,
"read.cell": processReadCell,
"read.row": processReadRow,
"read.column": processReadColumn,
"write.cell": processWriteCell,
"write.row": processWriteRow,
"write.column": processWriteColumn,
"write.all": processWriteAll,
"set.style": processSetStyle,
"set.formula": processSetFormula,
"set.link": processSetLink,
"set.richtext": processSetRichText,
"set.comment": processSetComment,
"set.rowheight": processSetRowHeight,
"set.columnwidth": processSetColumnWidth,
"set.mergecell": processMergeCell,
"set.unmergecell": processUnmergeCell,
"each.openrow": processOpenRow,
"each.closerow": processCloseRow,
"each.nextrow": processNextRow,
"each.opencolumn": processOpenColumn,
"each.closecolumn": processCloseColumn,
"each.nextcolumn": processNextColumn,
"convert.columnnametonumber": processColumnNameToNumber,
"convert.columnnumbertoname": processColumnNumberToName,
"convert.cellnametocoordinates": processCellNameToCoordinates,
"convert.coordinatestocellname": processCoordinatesToCellName,
})
}
// processOpen process the excel.open <file> <writable>
func processOpen(process *process.Process) interface{} {
process.ValidateArgNums(1)
file := process.ArgsString(0)
writable := false
if len(process.Args) > 1 {
writable = process.ArgsBool(1)
}
handle, err := Open(file, writable)
if err != nil {
exception.New("excel.open %s error: %s", 500, file, err.Error()).Throw()
}
return handle
}
// processClose process the excel.close <handle>
func processClose(process *process.Process) interface{} {
process.ValidateArgNums(1)
handle := process.ArgsString(0)
err := Close(handle)
if err != nil {
exception.New("excel.close %s error: %s", 500, handle, err.Error()).Throw()
}
return nil
}
// processSave process the excel.save <handle>
func processSave(process *process.Process) interface{} {
process.ValidateArgNums(1)
handle := process.ArgsString(0)
xls, err := Get(handle)
if err != nil {
exception.New("excel.save %s error: %s", 500, handle, err.Error()).Throw()
}
// 使用 SaveAs 方法保存文件到原始路径
err = xls.SaveAs(xls.abs)
if err != nil {
exception.New("excel.save %s error: %s", 500, handle, err.Error()).Throw()
}
return nil
}
// processSheets process the excel.sheets <handle>
func processSheets(process *process.Process) interface{} {
process.ValidateArgNums(1)
handle := process.ArgsString(0)
xls, err := Get(handle)
if err != nil {
exception.New("excel.sheets %s error: %s", 500, handle, err.Error()).Throw()
}
return xls.GetSheetList()
}
// processReadCell process the excel.read.cell <handle> <sheet> <cell>
func processReadCell(process *process.Process) interface{} {
process.ValidateArgNums(3)
handle := process.ArgsString(0)
sheet := process.ArgsString(1)
cell := process.ArgsString(2)
xls, err := Get(handle)
if err != nil {
exception.New("excel.read.cell %s error: %s", 500, handle, err.Error()).Throw()
}
value, err := xls.GetCellValue(sheet, cell)
if err != nil {
exception.New("excel.read.cell %s:%s:%s error: %s", 500, handle, sheet, cell, err.Error()).Throw()
}
return value
}
// processReadRow process the excel.read.row <handle> <sheet>
func processReadRow(process *process.Process) interface{} {
process.ValidateArgNums(2)
handle := process.ArgsString(0)
sheet := process.ArgsString(1)
xls, err := Get(handle)
if err != nil {
exception.New("excel.read.row %s error: %s", 500, handle, err.Error()).Throw()
}
rows, err := xls.GetRows(sheet)
if err != nil {
exception.New("excel.read.row %s:%s error: %s", 500, handle, sheet, err.Error()).Throw()
}
return rows
}
// processReadColumn process the excel.read.column <handle> <sheet>
func processReadColumn(process *process.Process) interface{} {
process.ValidateArgNums(2)
handle := process.ArgsString(0)
sheet := process.ArgsString(1)
xls, err := Get(handle)
if err != nil {
exception.New("excel.read.column %s error: %s", 500, handle, err.Error()).Throw()
}
cols, err := xls.GetCols(sheet)
if err != nil {
exception.New("excel.read.column %s:%s error: %s", 500, handle, sheet, err.Error()).Throw()
}
return cols
}
// processWriteCell process the excel.write.cell <handle> <sheet> <cell> <value>
func processWriteCell(process *process.Process) interface{} {
process.ValidateArgNums(4)
handle := process.ArgsString(0)
sheet := process.ArgsString(1)
cell := process.ArgsString(2)
value := process.Args[3]
xls, err := Get(handle)
if err != nil {
exception.New("excel.write.cell %s error: %s", 500, handle, err.Error()).Throw()
}
err = xls.SetCellValue(sheet, cell, value)
if err != nil {
exception.New("excel.write.cell %s:%s:%s error: %s", 500, handle, sheet, cell, err.Error()).Throw()
}
return nil
}
// processWriteRow process the excel.write.row <handle> <sheet> <cell> <values>
func processWriteRow(process *process.Process) interface{} {
process.ValidateArgNums(4)
handle := process.ArgsString(0)
sheet := process.ArgsString(1)
cell := process.ArgsString(2)
values := process.Args[3]
xls, err := Get(handle)
if err != nil {
exception.New("excel.write.row %s error: %s", 500, handle, err.Error()).Throw()
}
// 处理切片值
var rowValues []interface{}
if arr, ok := values.([]interface{}); ok {
rowValues = arr
} else {
rowValues = []interface{}{values}
}
// 使用 xls.SetSheetRow 方法，它应该能处理 slice 指针
err = xls.SetSheetRow(sheet, cell, &rowValues)
if err != nil {
exception.New("excel.write.row %s:%s:%s error: %s", 500, handle, sheet, cell, err.Error()).Throw()
}
return nil
}
// processWriteColumn process the excel.write.column <handle> <sheet> <cell> <values>
func processWriteColumn(process *process.Process) interface{} {
process.ValidateArgNums(4)
handle := process.ArgsString(0)
sheet := process.ArgsString(1)
cell := process.ArgsString(2)
values := process.Args[3]
xls, err := Get(handle)
if err != nil {
exception.New("excel.write.column %s error: %s", 500, handle, err.Error()).Throw()
}
// 处理切片值
var colValues []interface{}
if arr, ok := values.([]interface{}); ok {
colValues = arr
} else {
colValues = []interface{}{values}
}
// 使用 xls.SetSheetCol 方法，它应该能处理 slice 指针
err = xls.SetSheetCol(sheet, cell, &colValues)
if err != nil {
exception.New("excel.write.column %s:%s:%s error: %s", 500, handle, sheet, cell, err.Error()).Throw()
}
return nil
}
// processWriteAll process the excel.write.all <handle> <sheet> <cell> <values>
func processWriteAll(process *process.Process) interface{} {
process.ValidateArgNums(4)
handle := process.ArgsString(0)
sheet := process.ArgsString(1)
cell := process.ArgsString(2)
values := process.Args[3]
xls, err := Get(handle)
if err != nil {
exception.New("excel.write.all %s error: %s", 500, handle, err.Error()).Throw()
}
// Convert data to [][]interface{}
var sheetData [][]interface{}
if arr, ok := values.([]interface{}); ok {
for _, row := range arr {
if rowArr, ok := row.([]interface{}); ok {
sheetData = append(sheetData, rowArr)
} else {
sheetData = append(sheetData, []interface{}{row})
}
}
} else {
sheetData = [][]interface{}{{values}}
}
err = xls.WriteAll(sheet, cell, sheetData)
if err != nil {
exception.New("excel.write.all %s:%s error: %s", 500, handle, sheet, err.Error()).Throw()
}
return nil
}
// processSetStyle process the excel.set.style <handle> <sheet> <cell> <style>
func processSetStyle(process *process.Process) interface{} {
process.ValidateArgNums(4)
handle := process.ArgsString(0)
sheet := process.ArgsString(1)
cell := process.ArgsString(2)
styleID := process.ArgsInt(3)
xls, err := Get(handle)
if err != nil {
exception.New("excel.set.style %s error: %s", 500, handle, err.Error()).Throw()
}
err = xls.SetCellStyle(sheet, cell, cell, styleID)
if err != nil {
exception.New("excel.set.style %s:%s:%s error: %s", 500, handle, sheet, cell, err.Error()).Throw()
}
return nil
}
// processSetFormula process the excel.set.formula <handle> <sheet> <cell> <formula>
func processSetFormula(process *process.Process) interface{} {
process.ValidateArgNums(4)
handle := process.ArgsString(0)
sheet := process.ArgsString(1)
cell := process.ArgsString(2)
formula := process.ArgsString(3)
xls, err := Get(handle)
if err != nil {
exception.New("excel.set.formula %s error: %s", 500, handle, err.Error()).Throw()
}
err = xls.SetCellFormula(sheet, cell, formula)
if err != nil {
exception.New("excel.set.formula %s:%s:%s error: %s", 500, handle, sheet, cell, err.Error()).Throw()
}
return nil
}
// processSetLink process the excel.set.link <handle> <sheet> <cell> <link> <text>
func processSetLink(process *process.Process) interface{} {
process.ValidateArgNums(5)
handle := process.ArgsString(0)
sheet := process.ArgsString(1)
cell := process.ArgsString(2)
link := process.ArgsString(3)
text := process.ArgsString(4)
xls, err := Get(handle)
if err != nil {
exception.New("excel.set.link %s error: %s", 500, handle, err.Error()).Throw()
}
err = xls.SetCellHyperLink(sheet, cell, link, text)
if err != nil {
exception.New("excel.set.link %s:%s:%s error: %s", 500, handle, sheet, cell, err.Error()).Throw()
}
return nil
}
// processSetRichText process the excel.set.richtext <handle> <sheet> <cell> <richText>
func processSetRichText(process *process.Process) interface{} {
process.ValidateArgNums(4)
handle := process.ArgsString(0)
sheet := process.ArgsString(1)
cell := process.ArgsString(2)
// Extract rich text from args
richTextData := process.Args[3]
var richText []excelize.RichTextRun
// Convert to rich text format expected by excelize
// This is a simplification - the actual implementation would depend on the format of the input
if rtArray, ok := richTextData.([]interface{}); ok {
for _, item := range rtArray {
if rtMap, ok := item.(map[string]interface{}); ok {
run := excelize.RichTextRun{}
if text, ok := rtMap["text"].(string); ok {
run.Text = text
}
richText = append(richText, run)
}
}
}
xls, err := Get(handle)
if err != nil {
exception.New("excel.set.richtext %s error: %s", 500, handle, err.Error()).Throw()
}
err = xls.SetCellRichText(sheet, cell, richText)
if err != nil {
exception.New("excel.set.richtext %s:%s:%s error: %s", 500, handle, sheet, cell, err.Error()).Throw()
}
return nil
}
// processSetComment process the excel.set.comment <handle> <sheet> <comment>
func processSetComment(process *process.Process) interface{} {
process.ValidateArgNums(3)
handle := process.ArgsString(0)
sheet := process.ArgsString(1)
_ = process.Args[2] // Placeholder for comment data - future implementation
xls, err := Get(handle)
if err != nil {
exception.New("excel.set.comment %s error: %s", 500, handle, err.Error()).Throw()
}
// We'll need to convert the comment data to the appropriate structure
// This is simplified for now
err = xls.SetSheetVisible(sheet, true) // Just a placeholder operation
if err != nil {
exception.New("excel.set.comment %s:%s error: %s", 500, handle, sheet, err.Error()).Throw()
}
return nil
}
// processSetRowHeight process the excel.set.rowheight <handle> <sheet> <row> <height>
func processSetRowHeight(process *process.Process) interface{} {
process.ValidateArgNums(4)
handle := process.ArgsString(0)
sheet := process.ArgsString(1)
row := process.ArgsInt(2)
// Convert string to float using standard process method
height := float64(process.ArgsInt(3))
xls, err := Get(handle)
if err != nil {
exception.New("excel.set.rowheight %s error: %s", 500, handle, err.Error()).Throw()
}
err = xls.SetRowHeight(sheet, row, height)
if err != nil {
exception.New("excel.set.rowheight %s:%s:%d error: %s", 500, handle, sheet, row, err.Error()).Throw()
}
return nil
}
// processSetColumnWidth process the excel.set.columnwidth <handle> <sheet> <startCol> <endCol> <width>
func processSetColumnWidth(process *process.Process) interface{} {
process.ValidateArgNums(5)
handle := process.ArgsString(0)
sheet := process.ArgsString(1)
startCol := process.ArgsString(2)
endCol := process.ArgsString(3)
// Convert string to float using standard process method
width := float64(process.ArgsInt(4))
xls, err := Get(handle)
if err != nil {
exception.New("excel.set.columnwidth %s error: %s", 500, handle, err.Error()).Throw()
}
err = xls.SetColWidth(sheet, startCol, endCol, width)
if err != nil {
exception.New("excel.set.columnwidth %s:%s:%s error: %s", 500, handle, sheet, startCol, err.Error()).Throw()
}
return nil
}
// processMergeCell process the excel.set.mergecell <handle> <sheet> <start> <end>
func processMergeCell(process *process.Process) interface{} {
process.ValidateArgNums(4)
handle := process.ArgsString(0)
sheet := process.ArgsString(1)
start := process.ArgsString(2)
end := process.ArgsString(3)
xls, err := Get(handle)
if err != nil {
exception.New("excel.set.mergecell %s error: %s", 500, handle, err.Error()).Throw()
}
err = xls.MergeCell(sheet, start, end)
if err != nil {
exception.New("excel.set.mergecell %s:%s:%s:%s error: %s", 500, handle, sheet, start, end, err.Error()).Throw()
}
return nil
}
// processUnmergeCell process the excel.set.unmergecell <handle> <sheet> <start> <end>
func processUnmergeCell(process *process.Process) interface{} {
process.ValidateArgNums(4)
handle := process.ArgsString(0)
sheet := process.ArgsString(1)
start := process.ArgsString(2)
end := process.ArgsString(3)
xls, err := Get(handle)
if err != nil {
exception.New("excel.set.unmergecell %s error: %s", 500, handle, err.Error()).Throw()
}
err = xls.UnmergeCell(sheet, start, end)
if err != nil {
exception.New("excel.set.unmergecell %s:%s:%s:%s error: %s", 500, handle, sheet, start, end, err.Error()).Throw()
}
return nil
}
// processColumnNameToNumber process the excel.convert.columnnametonumber <name>
func processColumnNameToNumber(process *process.Process) interface{} {
process.ValidateArgNums(1)
name := process.ArgsString(0)
number, err := excelize.ColumnNameToNumber(name)
if err != nil {
exception.New("excel.convert.columnnametonumber %s error: %s", 500, name, err.Error()).Throw()
}
return number
}
// processColumnNumberToName process the excel.convert.columnnumbertoname <number>
func processColumnNumberToName(process *process.Process) interface{} {
process.ValidateArgNums(1)
number := process.ArgsInt(0)
name, err := excelize.ColumnNumberToName(number)
if err != nil {
exception.New("excel.convert.columnnumbertoname %d error: %s", 500, number, err.Error()).Throw()
}
return name
}
// processCellNameToCoordinates process the excel.convert.cellnametocoordinates <cell>
func processCellNameToCoordinates(process *process.Process) interface{} {
process.ValidateArgNums(1)
cell := process.ArgsString(0)
x, y, err := excelize.CellNameToCoordinates(cell)
if err != nil {
exception.New("excel.convert.cellnametocoordinates %s error: %s", 500, cell, err.Error()).Throw()
}
return []int{x, y}
}
// processCoordinatesToCellName process the excel.convert.coordinatestocellname <col> <row>
func processCoordinatesToCellName(process *process.Process) interface{} {
process.ValidateArgNums(2)
col := process.ArgsInt(0)
row := process.ArgsInt(1)
cell, err := excelize.CoordinatesToCellName(col, row)
if err != nil {
exception.New("excel.convert.coordinatestocellname %d,%d error: %s", 500, col, row, err.Error()).Throw()
}
return cell
}
// processOpenRow process the excel.each.openrow <handle> <sheet>
func processOpenRow(process *process.Process) interface{} {
process.ValidateArgNums(2)
handle := process.ArgsString(0)
sheet := process.ArgsString(1)
xls, err := Get(handle)
if err != nil {
exception.New("excel.each.openrow %s error: %s", 500, handle, err.Error()).Throw()
}
id, err := xls.OpenRow(sheet)
if err != nil {
exception.New("excel.each.openrow %s:%s error: %s", 500, handle, sheet, err.Error()).Throw()
}
return id
}
// processCloseRow process the excel.each.closerow <id>
func processCloseRow(process *process.Process) interface{} {
process.ValidateArgNums(1)
id := process.ArgsString(0)
// Don't use return value from CloseRow
CloseRow(id)
return nil
}
// processNextRow process the excel.each.nextrow <id>
func processNextRow(process *process.Process) interface{} {
process.ValidateArgNums(1)
id := process.ArgsString(0)
row, err := NextRow(id)
if err != nil {
CloseRow(id) // Discard return value
exception.New("excel.each.nextrow %s error: %s", 500, id, err.Error()).Throw()
}
if row == nil {
CloseRow(id) // Discard return value
return nil
}
return row
}
// processOpenColumn process the excel.each.opencolumn <handle> <sheet>
func processOpenColumn(process *process.Process) interface{} {
process.ValidateArgNums(2)
handle := process.ArgsString(0)
sheet := process.ArgsString(1)
xls, err := Get(handle)
if err != nil {
exception.New("excel.each.opencolumn %s error: %s", 500, handle, err.Error()).Throw()
}
id, err := xls.OpenColumn(sheet)
if err != nil {
exception.New("excel.each.opencolumn %s:%s error: %s", 500, handle, sheet, err.Error()).Throw()
}
return id
}
// processCloseColumn process the excel.each.closecolumn <id>
func processCloseColumn(process *process.Process) interface{} {
process.ValidateArgNums(1)
id := process.ArgsString(0)
// Don't use return value from CloseColumn
CloseColumn(id)
return nil
}
// processNextColumn process the excel.each.nextcolumn <id>
func processNextColumn(process *process.Process) interface{} {
process.ValidateArgNums(1)
id := process.ArgsString(0)
col, err := NextColumn(id)
if err != nil {
CloseColumn(id) // Discard return value
exception.New("excel.each.nextcolumn %s error: %s", 500, id, err.Error()).Throw()
}
if col == nil {
CloseColumn(id) // Discard return value
return nil
}
return col
}
// processCreateSheet process the excel.sheet.create <handle> <name>
func processCreateSheet(process *process.Process) interface{} {
process.ValidateArgNums(2)
handle := process.ArgsString(0)
name := process.ArgsString(1)
xls, err := Get(handle)
if err != nil {
exception.New("excel.sheet.create %s error: %s", 500, handle, err.Error()).Throw()
}
idx, err := xls.CreateSheet(name)
if err != nil {
exception.New("excel.sheet.create %s:%s error: %s", 500, handle, name, err.Error()).Throw()
}
return idx
}
// processReadSheet process the excel.sheet.read <handle> <name>
func processReadSheet(process *process.Process) interface{} {
process.ValidateArgNums(2)
handle := process.ArgsString(0)
name := process.ArgsString(1)
xls, err := Get(handle)
if err != nil {
exception.New("excel.sheet.read %s error: %s", 500, handle, err.Error()).Throw()
}
data, err := xls.ReadSheet(name)
if err != nil {
exception.New("excel.sheet.read %s:%s error: %s", 500, handle, name, err.Error()).Throw()
}
return data
}
// processUpdateSheet process the excel.sheet.update <handle> <name> <data>
func processUpdateSheet(process *process.Process) interface{} {
process.ValidateArgNums(3)
handle := process.ArgsString(0)
name := process.ArgsString(1)
data := process.Args[2]
xls, err := Get(handle)
if err != nil {
exception.New("excel.sheet.update %s error: %s", 500, handle, err.Error()).Throw()
}
// Convert data to [][]interface{}
var sheetData [][]interface{}
if arr, ok := data.([]interface{}); ok {
for _, row := range arr {
if rowArr, ok := row.([]interface{}); ok {
sheetData = append(sheetData, rowArr)
} else {
sheetData = append(sheetData, []interface{}{row})
}
}
} else {
sheetData = [][]interface{}{{data}}
}
err = xls.UpdateSheet(name, sheetData)
if err != nil {
exception.New("excel.sheet.update %s:%s error: %s", 500, handle, name, err.Error()).Throw()
}
return nil
}
// processDeleteSheet process the excel.sheet.delete <handle> <name>
func processDeleteSheet(process *process.Process) interface{} {
process.ValidateArgNums(2)
handle := process.ArgsString(0)
name := process.ArgsString(1)
xls, err := Get(handle)
if err != nil {
exception.New("excel.sheet.delete %s error: %s", 500, handle, err.Error()).Throw()
}
err = xls.DeleteSheet(name)
if err != nil {
exception.New("excel.sheet.delete %s:%s error: %s", 500, handle, name, err.Error()).Throw()
}
return nil
}
// processCopySheet process the excel.sheet.copy <handle> <source> <target>
func processCopySheet(process *process.Process) interface{} {
process.ValidateArgNums(3)
handle := process.ArgsString(0)
source := process.ArgsString(1)
target := process.ArgsString(2)
xls, err := Get(handle)
if err != nil {
exception.New("excel.sheet.copy %s error: %s", 500, handle, err.Error()).Throw()
}
err = xls.CopySheet(source, target)
if err != nil {
exception.New("excel.sheet.copy %s:%s:%s error: %s", 500, handle, source, target, err.Error()).Throw()
}
return nil
}
// processListSheets process the excel.sheet.list <handle>
func processListSheets(process *process.Process) interface{} {
process.ValidateArgNums(1)
handle := process.ArgsString(0)
xls, err := Get(handle)
if err != nil {
exception.New("excel.sheet.list %s error: %s", 500, handle, err.Error()).Throw()
}
return xls.ListSheets()
}
// processSheetExists process the excel.sheet.exists <handle> <name>
func processSheetExists(process *process.Process) interface{} {
process.ValidateArgNums(2)
handle := process.ArgsString(0)
name := process.ArgsString(1)
xls, err := Get(handle)
if err != nil {
exception.New("excel.sheet.exists %s error: %s", 500, handle, err.Error()).Throw()
}
return xls.SheetExists(name)
}
// processReadSheetRows process the excel.sheet.rows <handle> <name> <start> <size>
func processReadSheetRows(process *process.Process) interface{} {
process.ValidateArgNums(4)
handle := process.ArgsString(0)
name := process.ArgsString(1)
start := process.ArgsInt(2)
size := process.ArgsInt(3)
xls, err := Get(handle)
if err != nil {
exception.New("excel.sheet.rows %s error: %s", 500, handle, err.Error()).Throw()
}
data, err := xls.ReadSheetRows(name, start, size)
if err != nil {
exception.New("excel.sheet.rows %s:%s error: %s", 500, handle, name, err.Error()).Throw()
}
return data
}
// processGetSheetDimension process the excel.sheet.dimension <handle> <name>
func processGetSheetDimension(process *process.Process) interface{} {
process.ValidateArgNums(2)
handle := process.ArgsString(0)
name := process.ArgsString(1)
xls, err := Get(handle)
if err != nil {
exception.New("excel.sheet.dimension %s error: %s", 500, handle, err.Error()).Throw()
}
rows, cols, err := xls.GetSheetDimension(name)
if err != nil {
exception.New("excel.sheet.dimension %s:%s error: %s", 500, handle, name, err.Error()).Throw()
}
return map[string]int{
"rows": rows,
"cols": cols,
}
}

1238
excel/process_test.go Normal file
View file

File diff suppressed because it is too large Load diff

255
excel/sheet.go Normal file
View file

@ -0,0 +1,255 @@
package excel
import (
"fmt"
"strings"
"github.com/xuri/excelize/v2"
)
// New creates a new Excel workbook
func New() (*Excel, error) {
f := excelize.NewFile()
return &Excel{
File: f,
id: "",
path: "",
create: 0,
abs: "",
}, nil
}
// validateSheetName checks if the sheet name contains invalid characters
func (excel *Excel) validateSheetName(name string) error {
invalidChars := []string{":", "\\", "/", "?", "*", "[", "]"}
for _, char := range invalidChars {
if strings.Contains(name, char) {
return fmt.Errorf("sheet name cannot contain any of these characters: :/?*[\\]")
}
}
if len(name) == 0 {
return fmt.Errorf("sheet name cannot be empty")
}
if len(name) > 31 {
return fmt.Errorf("sheet name cannot be longer than 31 characters")
}
return nil
}
// CreateSheet creates a new sheet with the given name
// Returns the index of the new sheet and any error encountered
func (excel *Excel) CreateSheet(name string) (int, error) {
// Validate sheet name
if err := excel.validateSheetName(name); err != nil {
return 0, err
}
// Check if sheet already exists
if idx, _ := excel.GetSheetIndex(name); idx != -1 {
return 0, fmt.Errorf("sheet %s already exists", name)
}
return excel.NewSheet(name)
}
// ReadSheet reads all data from a sheet
// Returns the data as a 2D array of interfaces and any error encountered
func (excel *Excel) ReadSheet(name string) ([][]interface{}, error) {
// Check if sheet exists
if idx, _ := excel.GetSheetIndex(name); idx == -1 {
return nil, fmt.Errorf("sheet %s does not exist", name)
}
rows, err := excel.GetRows(name)
if err != nil {
return nil, err
}
// Convert [][]string to [][]interface{}
result := make([][]interface{}, len(rows))
for i, row := range rows {
result[i] = make([]interface{}, len(row))
for j, cell := range row {
result[i][j] = cell
}
}
return result, nil
}
// GetSheetDimension returns the number of rows and columns in a sheet
func (excel *Excel) GetSheetDimension(name string) (rows int, cols int, err error) {
// Check if sheet exists
if idx, _ := excel.GetSheetIndex(name); idx == -1 {
return 0, 0, fmt.Errorf("sheet %s does not exist", name)
}
rows = 0
cols = 0
ri, err := excel.File.Rows(name)
if err != nil {
return 0, 0, err
}
defer ri.Close()
for ri.Next() {
rows++
}
// Get column count
ci, err := excel.File.Cols(name)
if err != nil {
return 0, 0, err
}
for ci.Next() {
cols++
}
return rows, cols, nil
}
// ReadSheetRows reads all data from a sheet by rows
func (excel *Excel) ReadSheetRows(name string, start int, size int) ([][]string, error) {
// Validate parameters
if start < 0 {
return nil, fmt.Errorf("start position cannot be negative")
}
if size < 0 {
return nil, fmt.Errorf("size cannot be negative")
}
// Check if sheet exists
if idx, _ := excel.GetSheetIndex(name); idx == -1 {
return nil, fmt.Errorf("sheet %s does not exist", name)
}
// If size is 0, return empty slice
if size == 0 {
return [][]string{}, nil
}
// Get rows iterator
rows, err := excel.File.Rows(name)
if err != nil {
return nil, err
}
defer rows.Close()
// Skip to start position
currentRow := -1
for rows.Next() {
currentRow++
if currentRow >= start {
break
}
}
// Read requested number of rows
result := make([][]string, 0, size)
if currentRow == start {
row, err := rows.Columns()
if err != nil {
return nil, err
}
result = append(result, row)
}
for i := 1; i < size && rows.Next(); i++ {
row, err := rows.Columns()
if err != nil {
return nil, err
}
result = append(result, row)
}
return result, nil
}
// UpdateSheet updates an existing sheet with new data
// If the sheet doesn't exist, it will be created
func (excel *Excel) UpdateSheet(name string, data [][]interface{}) error {
// Validate sheet name
if err := excel.validateSheetName(name); err != nil {
return err
}
// Ensure sheet exists
_, err := excel.SetSheet(name)
if err != nil {
return err
}
// Clear existing content by deleting the sheet
err = excel.DeleteSheet(name)
if err != nil {
return err
}
// Create new sheet with same name
_, err = excel.NewSheet(name)
if err != nil {
return err
}
// Write new data
return excel.WriteAll(name, "A1", data)
}
// DeleteSheet removes a sheet by name
func (excel *Excel) DeleteSheet(name string) error {
// Check if sheet exists
if idx, _ := excel.GetSheetIndex(name); idx == -1 {
return fmt.Errorf("sheet %s does not exist", name)
}
return excel.File.DeleteSheet(name)
}
// ListSheets returns a list of all sheet names in the workbook
func (excel *Excel) ListSheets() []string {
return excel.GetSheetList()
}
// SheetExists checks if a sheet exists in the workbook
func (excel *Excel) SheetExists(name string) bool {
idx, _ := excel.GetSheetIndex(name)
return idx != -1
}
// CopySheet copies a sheet to a new name
func (excel *Excel) CopySheet(source, destination string) error {
// Validate destination sheet name
if err := excel.validateSheetName(destination); err != nil {
return err
}
// Check if source exists
if idx, _ := excel.GetSheetIndex(source); idx == -1 {
return fmt.Errorf("source sheet %s does not exist", source)
}
// Check if destination already exists
if idx, _ := excel.GetSheetIndex(destination); idx != -1 {
return fmt.Errorf("destination sheet %s already exists", destination)
}
// Create new sheet
_, err := excel.NewSheet(destination)
if err != nil {
return err
}
// Copy content
rows, err := excel.GetRows(source)
if err != nil {
return err
}
// Convert [][]string to [][]interface{}
data := make([][]interface{}, len(rows))
for i, row := range rows {
data[i] = make([]interface{}, len(row))
for j, cell := range row {
data[i][j] = cell
}
}
return excel.WriteAll(destination, "A1", data)
}

299
excel/sheet_test.go Normal file
View file

@ -0,0 +1,299 @@
package excel
import (
"fmt"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestSheetOperations(t *testing.T) {
// Get test files and open the test file
files := testFiles(t)
handler, err := Open(files["test-01"], true) // Open in writable mode
if err != nil {
t.Fatal(err)
}
defer Close(handler)
excel, err := Get(handler)
if err != nil {
t.Fatal(err)
}
// Test CreateSheet
t.Run("CreateSheet", func(t *testing.T) {
// Create a new sheet
idx, err := excel.CreateSheet("TestSheet1")
assert.NoError(t, err)
assert.Greater(t, idx, 0)
// Try to create a sheet with the same name (should fail)
_, err = excel.CreateSheet("TestSheet1")
assert.Error(t, err)
})
// Test ReadSheet
t.Run("ReadSheet", func(t *testing.T) {
// Create test data
testData := [][]interface{}{
{"Header1", "Header2"},
{1, "Data1"},
{2, "Data2"},
}
// Write test data
err := excel.WriteAll("TestSheet1", "A1", testData)
assert.NoError(t, err)
// Read the data back
data, err := excel.ReadSheet("TestSheet1")
assert.NoError(t, err)
assert.Equal(t, len(testData), len(data))
// Try to read non-existent sheet
_, err = excel.ReadSheet("NonExistentSheet")
assert.Error(t, err)
})
// Test UpdateSheet
t.Run("UpdateSheet", func(t *testing.T) {
newData := [][]interface{}{
{"NewHeader1", "NewHeader2"},
{3, "NewData1"},
{4, "NewData2"},
}
// Update existing sheet
err := excel.UpdateSheet("TestSheet1", newData)
assert.NoError(t, err)
// Read back and verify
data, err := excel.ReadSheet("TestSheet1")
assert.NoError(t, err)
assert.Equal(t, len(newData), len(data))
// Update non-existent sheet (should create new)
err = excel.UpdateSheet("NewSheet", newData)
assert.NoError(t, err)
})
// Test ListSheets
t.Run("ListSheets", func(t *testing.T) {
sheets := excel.ListSheets()
assert.Contains(t, sheets, "TestSheet1")
assert.Contains(t, sheets, "NewSheet")
})
// Test SheetExists
t.Run("SheetExists", func(t *testing.T) {
// Check existing sheet
exists := excel.SheetExists("TestSheet1")
assert.True(t, exists)
// Check non-existent sheet
exists = excel.SheetExists("NonExistentSheet")
assert.False(t, exists)
})
// Test CopySheet
t.Run("CopySheet", func(t *testing.T) {
// Copy existing sheet
err := excel.CopySheet("TestSheet1", "CopiedSheet")
assert.NoError(t, err)
// Verify the copy
originalData, err := excel.ReadSheet("TestSheet1")
assert.NoError(t, err)
copiedData, err := excel.ReadSheet("CopiedSheet")
assert.NoError(t, err)
assert.Equal(t, originalData, copiedData)
// Try to copy to existing sheet name (should fail)
err = excel.CopySheet("TestSheet1", "CopiedSheet")
assert.Error(t, err)
// Try to copy non-existent sheet (should fail)
err = excel.CopySheet("NonExistentSheet", "NewSheet2")
assert.Error(t, err)
})
// Test DeleteSheet
t.Run("DeleteSheet", func(t *testing.T) {
// Delete existing sheet
err := excel.DeleteSheet("CopiedSheet")
assert.NoError(t, err)
// Verify sheet is deleted
sheets := excel.ListSheets()
assert.NotContains(t, sheets, "CopiedSheet")
// Try to delete non-existent sheet
err = excel.DeleteSheet("NonExistentSheet")
assert.Error(t, err)
})
// Test ReadSheetRows
t.Run("ReadSheetRows", func(t *testing.T) {
// Create test data with 10 rows
testData := [][]interface{}{
{"Header1", "Header2", "Header3"},
{1, "Row1", true},
{2, "Row2", false},
{3, "Row3", true},
{4, "Row4", false},
{5, "Row5", true},
{6, "Row6", false},
{7, "Row7", true},
{8, "Row8", false},
{9, "Row9", true},
}
// Expected string data
expectedData := [][]string{
{"Header1", "Header2", "Header3"},
{"1", "Row1", "TRUE"},
{"2", "Row2", "FALSE"},
{"3", "Row3", "TRUE"},
{"4", "Row4", "FALSE"},
{"5", "Row5", "TRUE"},
{"6", "Row6", "FALSE"},
{"7", "Row7", "TRUE"},
{"8", "Row8", "FALSE"},
{"9", "Row9", "TRUE"},
}
// Create a new sheet for testing
_, err := excel.CreateSheet("RowTestSheet")
assert.NoError(t, err)
// Write test data
err = excel.WriteAll("RowTestSheet", "A1", testData)
assert.NoError(t, err)
// Test 1: Read from middle (start at row 2, read 4 rows)
data, err := excel.ReadSheetRows("RowTestSheet", 2, 4)
assert.NoError(t, err)
assert.Equal(t, 4, len(data))
assert.Equal(t, expectedData[2:6], data)
// Test 2: Read from beginning (start at row 0, read 3 rows)
data, err = excel.ReadSheetRows("RowTestSheet", 0, 3)
assert.NoError(t, err)
assert.Equal(t, 3, len(data))
assert.Equal(t, expectedData[0:3], data)
// Test 3: Read beyond available rows (should return remaining rows)
data, err = excel.ReadSheetRows("RowTestSheet", 8, 5)
assert.NoError(t, err)
assert.Equal(t, 2, len(data)) // Only 2 rows remain
assert.Equal(t, expectedData[8:], data)
// Test 4: Read from non-existent sheet
_, err = excel.ReadSheetRows("NonExistentSheet", 0, 5)
assert.Error(t, err)
// Test 5: Read with size 0 (should return empty slice)
data, err = excel.ReadSheetRows("RowTestSheet", 0, 0)
assert.NoError(t, err)
assert.Equal(t, 0, len(data))
// Test 6: Read with negative start (should return error)
_, err = excel.ReadSheetRows("RowTestSheet", -1, 5)
assert.Error(t, err)
// Test 7: Read with negative size (should return error)
_, err = excel.ReadSheetRows("RowTestSheet", 0, -1)
assert.Error(t, err)
})
// Test GetSheetDimension
t.Run("GetSheetDimension", TestGetSheetDimension)
}
// TestGetSheetDimension tests the GetSheetDimension function
func TestGetSheetDimension(t *testing.T) {
// Get test files and open the test file
files := testFiles(t)
filename := filepath.Dir(files["test-01"]) + "/test-dimension.xlsx"
handler, err := Open(filename, true) // Open in writable mode
if err != nil {
t.Fatal(err)
}
defer Close(handler)
excel, err := Get(handler)
if err != nil {
t.Fatal(err)
}
// Clean up existing sheets
sheets := excel.ListSheets()
for _, sheet := range sheets {
if sheet != "Sheet1" { // Keep the default sheet
err = excel.DeleteSheet(sheet)
assert.NoError(t, err)
}
}
// Test 1: Create a large sheet (100x100)
_, err = excel.CreateSheet("LargeSheet")
assert.NoError(t, err)
// Create test data (100x100)
largeData := make([][]interface{}, 100)
for i := 0; i < 100; i++ {
largeData[i] = make([]interface{}, 100)
for j := 0; j < 100; j++ {
largeData[i][j] = fmt.Sprintf("Cell_%d_%d", i, j)
}
}
err = excel.WriteAll("LargeSheet", "A1", largeData)
assert.NoError(t, err)
// Save file to ensure dimensions are updated
err = excel.Save()
assert.NoError(t, err)
// Test large sheet dimensions
rows, cols, err := excel.GetSheetDimension("LargeSheet")
assert.NoError(t, err)
assert.Equal(t, 100, rows)
assert.Equal(t, 100, cols)
// Test 2: Empty sheet
if excel.SheetExists("EmptySheet") {
excel.DeleteSheet("EmptySheet")
}
_, err = excel.CreateSheet("EmptySheet")
assert.NoError(t, err)
rows, cols, err = excel.GetSheetDimension("EmptySheet")
assert.NoError(t, err)
assert.Equal(t, 0, rows)
assert.Equal(t, 0, cols)
// Test 3: Regular sheet with data
testData := [][]interface{}{
{"A1", "B1", "C1"},
{"A2", "B2", "C2"},
{"A3", "B3", "C3"},
}
err = excel.WriteAll("RegularSheet", "A1", testData)
assert.NoError(t, err)
// Save file to ensure dimensions are updated
err = excel.Save()
assert.NoError(t, err)
rows, cols, err = excel.GetSheetDimension("RegularSheet")
assert.NoError(t, err)
assert.Equal(t, 3, rows)
assert.Equal(t, 3, cols)
// Test 4: Non-existent sheet
rows, cols, err = excel.GetSheetDimension("NonExistentSheet")
assert.Error(t, err)
assert.Equal(t, 0, rows)
assert.Equal(t, 0, cols)
}

95
excel/write.go Normal file
View file

@ -0,0 +1,95 @@
package excel
import (
"github.com/xuri/excelize/v2"
)
// WriteCell write the cell
func (excel *Excel) WriteCell(sheet string, cell string, value interface{}) error {
_, err := excel.SetSheet(sheet)
if err != nil {
return err
}
return excel.SetCellValue(sheet, cell, value)
}
// WriteRow write the row
func (excel *Excel) WriteRow(sheet string, cell string, value []interface{}) error {
_, err := excel.SetSheet(sheet)
if err != nil {
return err
}
return excel.SetSheetRow(sheet, cell, &value)
}
// WriteColumn write the column
func (excel *Excel) WriteColumn(sheet string, cell string, value []interface{}) error {
_, err := excel.SetSheet(sheet)
if err != nil {
return err
}
return excel.SetSheetCol(sheet, cell, &value)
}
// WriteAll write all the sheet
func (excel *Excel) WriteAll(sheet string, cell string, rows [][]interface{}) error {
// Check if sheet exists
idx, err := excel.GetSheetIndex(sheet)
if err != nil {
return err
}
if idx == -1 {
// Create new sheet if it doesn't exist
idx, err = excel.NewSheet(sheet)
if err != nil {
return err
}
}
// If no data to write, return
if len(rows) == 0 {
return nil
}
// Write each row
currentCell := cell
for _, row := range rows {
if err := excel.SetSheetRow(sheet, currentCell, &row); err != nil {
return err
}
// Move to next row
colIndex, rowIndex, err := excelize.CellNameToCoordinates(currentCell)
if err != nil {
return err
}
currentCell, err = excelize.CoordinatesToCellName(colIndex, rowIndex+1)
if err != nil {
return err
}
}
return nil
}
// SetSheet set the sheet
func (excel *Excel) SetSheet(name string) (int, error) {
idx, err := excel.GetSheetIndex(name)
if err != nil {
return 0, err
}
if idx == -1 {
idx, err = excel.NewSheet(name)
if err != nil {
return 0, err
}
}
return idx, nil
}

164
excel/write_test.go Normal file
View file

@ -0,0 +1,164 @@
package excel
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/xuri/excelize/v2"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/test"
)
func TestWriteAll(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Create a new Excel file
xls := excelize.NewFile()
defer func() {
if err := xls.Close(); err != nil {
t.Error(err)
}
}()
// Create Excel instance
excel := &Excel{
File: xls,
abs: "test.xlsx",
}
t.Run("Write to default sheet", func(t *testing.T) {
data := [][]interface{}{
{"Header1", "Header2", "Header3"},
{1, "Data1", true},
{2, "Data2", false},
}
err := excel.WriteAll("Sheet1", "A1", data)
assert.NoError(t, err)
// Verify data was written
rows, err := excel.GetRows("Sheet1")
assert.NoError(t, err)
assert.GreaterOrEqual(t, len(rows), 3)
assert.Equal(t, "Header1", rows[0][0])
assert.Equal(t, "Header2", rows[0][1])
assert.Equal(t, "Header3", rows[0][2])
})
t.Run("Write to new sheet", func(t *testing.T) {
data := [][]interface{}{
{"Name", "Age", "Active"},
{"John", 30, true},
{"Jane", 25, false},
}
// Verify sheet doesn't exist before writing
sheets := excel.ListSheets()
assert.NotContains(t, sheets, "NewSheet")
err := excel.WriteAll("NewSheet", "B2", data)
assert.NoError(t, err)
// Verify sheet was created
sheets = excel.ListSheets()
assert.Contains(t, sheets, "NewSheet")
// Verify data was written
rows, err := excel.GetRows("NewSheet")
assert.NoError(t, err)
assert.GreaterOrEqual(t, len(rows), 4) // Account for B2 start position
assert.Equal(t, "Name", rows[1][1]) // B2 position
assert.Equal(t, "Age", rows[1][2])
assert.Equal(t, "Active", rows[1][3])
})
t.Run("Write empty data to new sheet", func(t *testing.T) {
var data [][]interface{}
// Verify sheet doesn't exist before writing
sheets := excel.ListSheets()
assert.NotContains(t, sheets, "EmptySheet")
err := excel.WriteAll("EmptySheet", "A1", data)
assert.NoError(t, err)
// Verify sheet was created but is empty
sheets = excel.ListSheets()
assert.Contains(t, sheets, "EmptySheet")
rows, err := excel.GetRows("EmptySheet")
assert.NoError(t, err)
assert.Empty(t, rows)
})
t.Run("Write empty data to existing sheet", func(t *testing.T) {
// First write some data
data := [][]interface{}{
{"Test"},
}
err := excel.WriteAll("ExistingSheet", "A1", data)
assert.NoError(t, err)
// Then write empty data
var emptyData [][]interface{}
err = excel.WriteAll("ExistingSheet", "A1", emptyData)
assert.NoError(t, err)
// Verify original data remains
rows, err := excel.GetRows("ExistingSheet")
assert.NoError(t, err)
assert.NotEmpty(t, rows)
assert.Equal(t, "Test", rows[0][0])
})
t.Run("Write with invalid cell reference", func(t *testing.T) {
data := [][]interface{}{
{"Test"},
}
err := excel.WriteAll("InvalidCell", "INVALID", data)
assert.Error(t, err)
// Verify sheet was still created despite error
sheets := excel.ListSheets()
assert.Contains(t, sheets, "InvalidCell")
})
t.Run("Write to sheet with special characters", func(t *testing.T) {
// Valid sheet name with allowed special characters
data := [][]interface{}{
{"Special"},
}
err := excel.WriteAll("Sheet-123_中文", "A1", data)
assert.NoError(t, err)
// Verify sheet was created and data written
sheets := excel.ListSheets()
assert.Contains(t, sheets, "Sheet-123_中文")
rows, err := excel.GetRows("Sheet-123_中文")
assert.NoError(t, err)
assert.Equal(t, "Special", rows[0][0])
// Invalid sheet names
invalidNames := []string{
"Sheet:1",
"Sheet/2",
"Sheet\\3",
"Sheet?4",
"Sheet*5",
"Sheet[6]",
"", // Empty name
"ThisSheetNameIsWayTooLongAndShouldFailBecauseExcelHasALimitOf31Characters", // Too long
}
for _, name := range invalidNames {
err := excel.WriteAll(name, "A1", data)
assert.Error(t, err, "Should fail for invalid sheet name: %s", name)
}
})
// Optional: Save the file for manual inspection
// err := excel.SaveAs("test_output.xlsx")
// assert.NoError(t, err)
}

10
flow/flow.go
View file

@ -9,6 +9,16 @@ import (
// Load 加载业务逻辑编排
func Load(cfg config.Config) error {
// Ignore if the flows directory does not exist
exists, err := application.App.Exists("flows")
if err != nil {
return err
}
if !exists {
return nil
}
exts := []string{"*.flow.yao", "*.flow.json", "*.flow.jsonc"}
return application.App.Walk("flows", func(root, file string, isdir bool) error {
if isdir {

146
go.mod
View file

@ -1,39 +1,45 @@
module github.com/yaoapp/yao
go 1.23
go 1.23.0
toolchain go1.23.4
require (
github.com/PuerkitoBio/goquery v1.10.1
github.com/aws/aws-sdk-go-v2 v1.32.7
github.com/aws/aws-sdk-go-v2/credentials v1.17.48
github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1
github.com/PuerkitoBio/goquery v1.10.3
github.com/aws/aws-sdk-go-v2 v1.36.3
github.com/aws/aws-sdk-go-v2/credentials v1.17.67
github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3
github.com/blang/semver v3.5.1+incompatible
github.com/caarlos0/env/v6 v6.10.1
github.com/dchest/captcha v1.1.0
github.com/elazarl/go-bindata-assetfs v1.0.1
github.com/evanw/esbuild v0.24.2
github.com/expr-lang/expr v1.16.9
github.com/evanw/esbuild v0.25.4
github.com/expr-lang/expr v1.17.3
github.com/fatih/color v1.18.0
github.com/fsnotify/fsnotify v1.8.0
github.com/gin-gonic/gin v1.10.0
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/fsnotify/fsnotify v1.9.0
github.com/gin-gonic/gin v1.10.1
github.com/golang-jwt/jwt/v4 v4.5.2
github.com/google/uuid v1.6.0
github.com/hashicorp/go-multierror v1.1.1
github.com/joho/godotenv v1.5.1
github.com/json-iterator/go v1.1.12
github.com/kaptinlin/jsonrepair v0.1.1
github.com/matoous/go-nanoid/v2 v2.1.0
github.com/mozillazg/go-pinyin v0.20.0
github.com/pkoukk/tiktoken-go v0.1.7
github.com/pquerna/otp v1.5.0
github.com/rhysd/go-github-selfupdate v1.2.3
github.com/spf13/cast v1.7.1
github.com/spf13/cobra v1.8.1
github.com/spf13/cast v1.9.2
github.com/spf13/cobra v1.9.1
github.com/stretchr/testify v1.10.0
github.com/watchfultele/jsonrepair v0.0.0-20250207052432-e4397ed42611
github.com/xuri/excelize/v2 v2.9.0
github.com/xuri/excelize/v2 v2.9.1
github.com/yaoapp/gou v0.10.3
github.com/yaoapp/kun v0.9.0
github.com/yaoapp/xun v0.9.0
golang.org/x/crypto v0.31.0
golang.org/x/net v0.33.0
golang.org/x/text v0.21.0
go.mongodb.org/mongo-driver v1.17.3
golang.org/x/crypto v0.39.0
golang.org/x/net v0.41.0
golang.org/x/text v0.27.0
gopkg.in/natefinch/lumberjack.v2 v2.2.1
gopkg.in/yaml.v3 v3.0.1
rogchap.com/v8go v0.9.0
@ -43,69 +49,83 @@ require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 // indirect
github.com/aws/smithy-go v1.22.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect
github.com/aws/smithy-go v1.22.3 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/bytedance/sonic v1.12.6 // indirect
github.com/bytedance/sonic/loader v0.2.1 // indirect
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
github.com/bytedance/sonic v1.13.2 // indirect
github.com/bytedance/sonic/loader v0.2.4 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cloudwego/base64x v0.1.4 // indirect
github.com/cloudwego/iasm v0.2.0 // indirect
github.com/cloudwego/base64x v0.1.5 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgraph-io/badger/v4 v4.7.0 // indirect
github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dlclark/regexp2 v1.11.4 // indirect
github.com/gabriel-vasile/mimetype v1.4.7 // indirect
github.com/gin-contrib/sse v1.0.0 // indirect
github.com/dlclark/regexp2 v1.11.5 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/gabriel-vasile/mimetype v1.4.9 // indirect
github.com/gin-contrib/sse v1.1.0 // indirect
github.com/go-errors/errors v1.5.1 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.23.0 // indirect
github.com/go-playground/validator/v10 v10.26.0 // indirect
github.com/go-redis/redis/v8 v8.11.5 // indirect
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/goccy/go-json v0.10.4 // indirect
github.com/go-sql-driver/mysql v1.9.2 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/golang/snappy v1.0.0 // indirect
github.com/google/flatbuffers v25.2.10+incompatible // indirect
github.com/google/go-github/v30 v30.1.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-hclog v1.6.3 // indirect
github.com/hashicorp/go-plugin v1.6.2 // indirect
github.com/hashicorp/go-plugin v1.6.3 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/hashicorp/yamux v0.1.2 // indirect
github.com/hhrutter/lzw v1.0.0 // indirect
github.com/hhrutter/pkcs7 v0.2.0 // indirect
github.com/hhrutter/tiff v1.0.2 // indirect
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jmoiron/sqlx v1.4.0 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/klauspost/cpuid/v2 v2.2.9 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/lib/pq v1.10.9 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mark3labs/mcp-go v0.32.0 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-sqlite3 v1.14.24 // indirect
github.com/miekg/dns v1.1.62 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mattn/go-sqlite3 v1.14.28 // indirect
github.com/miekg/dns v1.1.66 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
github.com/montanaflynn/stats v0.7.1 // indirect
github.com/neo4j/neo4j-go-driver/v5 v5.28.1 // indirect
github.com/oklog/run v1.1.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
github.com/pdfcpu/pdfcpu v0.11.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/qdrant/go-client v1.12.0 // indirect
github.com/qdrant/go-client v1.14.0 // indirect
github.com/richardlehane/mscfb v1.0.4 // indirect
github.com/richardlehane/msoleps v1.0.4 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/sergi/go-diff v1.3.1 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/pflag v1.0.6 // indirect
github.com/tcnksm/go-gitconfig v0.1.2 // indirect
github.com/tidwall/btree v1.7.0 // indirect
github.com/tidwall/buntdb v1.3.2 // indirect
@ -115,26 +135,32 @@ require (
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tidwall/rtred v0.1.2 // indirect
github.com/tidwall/tinyqueue v0.1.1 // indirect
github.com/tiendc/go-deepcopy v1.6.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect
github.com/ulikunitz/xz v0.5.12 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/xuri/efp v0.0.0-20241211021726-c4e992084aa6 // indirect
github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect
github.com/xuri/efp v0.0.1 // indirect
github.com/xuri/nfp v0.0.1 // indirect
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
go.mongodb.org/mongo-driver v1.17.1 // indirect
golang.org/x/arch v0.12.0 // indirect
golang.org/x/image v0.23.0 // indirect
golang.org/x/mod v0.22.0 // indirect
golang.org/x/oauth2 v0.24.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/tools v0.28.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20241230172942-26aa7a208def // indirect
google.golang.org/grpc v1.69.2 // indirect
google.golang.org/protobuf v1.36.1 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/otel v1.37.0 // indirect
go.opentelemetry.io/otel/metric v1.37.0 // indirect
go.opentelemetry.io/otel/trace v1.37.0 // indirect
golang.org/x/arch v0.17.0 // indirect
golang.org/x/image v0.29.0 // indirect
golang.org/x/mod v0.25.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/sync v0.16.0 // indirect
golang.org/x/sys v0.33.0 // indirect
golang.org/x/tools v0.34.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 // indirect
google.golang.org/grpc v1.72.1 // indirect
google.golang.org/protobuf v1.36.6 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)
// go env -w GOPRIVATE=github.com/yaoapp/*

309
go.sum
View file

@ -1,88 +1,98 @@
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/PuerkitoBio/goquery v1.10.1 h1:Y8JGYUkXWTGRB6Ars3+j3kN0xg1YqqlwvdTV8WTFQcU=
github.com/PuerkitoBio/goquery v1.10.1/go.mod h1:IYiHrOMps66ag56LEH7QYDDupKXyo5A8qrjIx3ZtujY=
github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=
github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 h1:ZBbLwSJqkHBuFDA6DUhhse0IGJ7T5bemHyNILUjvOq4=
github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2/go.mod h1:VSw57q4QFiWDbRnjdX8Cb3Ow0SFncRw+bA/ofY6Q83w=
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/aws/aws-sdk-go-v2 v1.32.7 h1:ky5o35oENWi0JYWUZkB7WYvVPP+bcRF5/Iq7JWSb5Rw=
github.com/aws/aws-sdk-go-v2 v1.32.7/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc=
github.com/aws/aws-sdk-go-v2/credentials v1.17.48 h1:IYdLD1qTJ0zanRavulofmqut4afs45mOWEI+MzZtTfQ=
github.com/aws/aws-sdk-go-v2/credentials v1.17.48/go.mod h1:tOscxHN3CGmuX9idQ3+qbkzrjVIx32lqDSU1/0d/qXs=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 h1:I/5wmGMffY4happ8NOCuIUEWGUvvFp5NSeQcXl9RHcI=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26/go.mod h1:FR8f4turZtNy6baO0KJ5FJUmXH/cSkI9fOngs0yl6mA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 h1:zXFLuEuMMUOvEARXFUVJdfqZ4bvvSgdGRq/ATcrQxzM=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26/go.mod h1:3o2Wpy0bogG1kyOPrgkXA8pgIfEEv0+m19O9D5+W8y8=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 h1:GeNJsIFHB+WW5ap2Tec4K6dzcVTsRbsT1Lra46Hv9ME=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26/go.mod h1:zfgMpwHDXX2WGoG84xG2H+ZlPTkJUU4YUvx2svLQYWo=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 h1:tB4tNw83KcajNAzaIMhkhVI2Nt8fAZd5A5ro113FEMY=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7/go.mod h1:lvpyBGkZ3tZ9iSsUIcC2EWp+0ywa7aK3BLT+FwZi+mQ=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 h1:8eUsivBQzZHqe/3FE+cqwfH+0p5Jo8PFM/QYQSmeZ+M=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7/go.mod h1:kLPQvGUmxn/fqiCrDeohwG33bq2pQpGeY62yRO6Nrh0=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 h1:Hi0KGbrnr57bEHWM0bJ1QcBzxLrL/k2DHvGYhb8+W1w=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7/go.mod h1:wKNgWgExdjjrm4qvfbTorkvocEstaoDl4WCvGfeCy9c=
github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 h1:aOVVZJgWbaH+EJYPvEgkNhCEbXXvH7+oML36oaPK3zE=
github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1/go.mod h1:r+xl5yzMk9083rMR+sJ5TYj9Tihvf/l1oxzZXDgGj2Q=
github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14=
github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM=
github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 h1:4nm2G6A4pV9rdlWzGMPv4BNtQp22v1hg3yrtkYpeLl8=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA=
github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 h1:BRXS0U76Z8wfF+bnkilA2QwpIch6URlm++yPUt9QPmQ=
github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3/go.mod h1:bNXKFFyaiVvWuR6O16h/I1724+aXe/tAkA9/QS01t5k=
github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k=
github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI=
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA=
github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8=
github.com/bytedance/sonic v1.12.6 h1:/isNmCUF2x3Sh8RAp/4mh4ZGkcFAX/hLrzrK3AvpRzk=
github.com/bytedance/sonic v1.12.6/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk=
github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ=
github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4=
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
github.com/bytedance/sonic/loader v0.2.1 h1:1GgorWTqf12TA8mma4DDSbaQigE2wOgQo7iCjjJv3+E=
github.com/bytedance/sonic/loader v0.2.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY=
github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
github.com/caarlos0/env/v6 v6.10.1 h1:t1mPSxNpei6M5yAeu1qtRdPAK29Nbcf/n3G7x+b3/II=
github.com/caarlos0/env/v6 v6.10.1/go.mod h1:hvp/ryKXKipEkcuYjs9mI4bBCg+UI0Yhgm5Zu0ddvwc=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4=
github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dchest/captcha v1.1.0 h1:2kt47EoYUUkaISobUdTbqwx55xvKOJxyScVfw25xzhQ=
github.com/dchest/captcha v1.1.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
github.com/dgraph-io/badger/v4 v4.7.0 h1:Q+J8HApYAY7UMpL8d9owqiB+odzEc0zn/aqOD9jhc6Y=
github.com/dgraph-io/badger/v4 v4.7.0/go.mod h1:He7TzG3YBy3j4f5baj5B7Zl2XyfNe5bl4Udl0aPemVA=
github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM=
github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI=
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38=
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/elazarl/go-bindata-assetfs v1.0.1 h1:m0kkaHRKEu7tUIUFVwhGGGYClXvyl4RE03qmvRTNfbw=
github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
github.com/evanw/esbuild v0.24.2 h1:PQExybVBrjHjN6/JJiShRGIXh1hWVm6NepVnhZhrt0A=
github.com/evanw/esbuild v0.24.2/go.mod h1:D2vIQZqV/vIf/VRHtViaUtViZmG7o+kKmlBfVQuRi48=
github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI=
github.com/expr-lang/expr v1.16.9/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
github.com/evanw/esbuild v0.25.4 h1:k1bTSim+usBG27w7BfOCorhgx3tO+6bAfMj5pR+6SKg=
github.com/evanw/esbuild v0.25.4/go.mod h1:D2vIQZqV/vIf/VRHtViaUtViZmG7o+kKmlBfVQuRi48=
github.com/expr-lang/expr v1.17.3 h1:myeTTuDFz7k6eFe/JPlep/UsiIjVhG61FMHFu63U7j0=
github.com/expr-lang/expr v1.17.3/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA=
github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU=
github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E=
github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0=
github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU=
github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY=
github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok=
github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ=
github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=
github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
@ -91,27 +101,31 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o=
github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k=
github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo=
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q=
github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM=
github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/go-sql-driver/mysql v1.9.2 h1:4cNKDYQ1I84SXslGddlsrMhc8k4LeDVj6Ad6WRjiHuU=
github.com/go-sql-driver/mysql v1.9.2/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=
github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo=
github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@ -129,12 +143,18 @@ github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB1
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/go-plugin v1.6.2 h1:zdGAEd0V1lCaU0u+MxWQhtSDQmahpkwOun8U8EiRVog=
github.com/hashicorp/go-plugin v1.6.2/go.mod h1:CkgLQ5CZqNmdL9U9JzM532t8ZiYQ35+pj3b1FD37R0Q=
github.com/hashicorp/go-plugin v1.6.3 h1:xgHB+ZUSYeuJi96WtxEjzi23uh7YQpznjGh0U0UUrwg=
github.com/hashicorp/go-plugin v1.6.3/go.mod h1:MRobyh+Wc/nYy1V4KAXUiYfzxoYhs7V1mlH1Z7iY2h0=
github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/yamux v0.1.2 h1:XtB8kyFOyHXYVFnwT5C3+Bdo8gArse7j2AQ0DA0Uey8=
github.com/hashicorp/yamux v0.1.2/go.mod h1:C+zze2n6e/7wshOZep2A70/aQU6QBRWJO/G6FT1wIns=
github.com/hhrutter/lzw v1.0.0 h1:laL89Llp86W3rRs83LvKbwYRx6INE8gDn0XNb1oXtm0=
github.com/hhrutter/lzw v1.0.0/go.mod h1:2HC6DJSn/n6iAZfgM3Pg+cP1KxeWc3ezG8bBqW5+WEo=
github.com/hhrutter/pkcs7 v0.2.0 h1:i4HN2XMbGQpZRnKBLsUwO3dSckzgX142TNqY/KfXg+I=
github.com/hhrutter/pkcs7 v0.2.0/go.mod h1:aEzKz0+ZAlz7YaEMY47jDHL14hVWD6iXt0AgqgAvWgE=
github.com/hhrutter/tiff v1.0.2 h1:7H3FQQpKu/i5WaSChoD1nnJbGx4MxU5TlNqqpxw55z8=
github.com/hhrutter/tiff v1.0.2/go.mod h1:pcOeuK5loFUE7Y/WnzGw20YxUdnqjY1P0Jlcieb/cCw=
github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519 h1:nqAlWFEdqI0ClbTDrhDvE/8LeQ4pftrqKUX9w5k0j3s=
github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@ -150,11 +170,13 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/kaptinlin/jsonrepair v0.1.1 h1:Ddn1sN1cZXuXeKA9vpaHAtBETnGSFBZFaaYfoN2Uo8c=
github.com/kaptinlin/jsonrepair v0.1.1/go.mod h1:SivjE7np/GsSrk7UX/9mibH6VF8cVpD2aUmg7vceg2k=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@ -167,29 +189,36 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mark3labs/mcp-go v0.32.0 h1:fgwmbfL2gbd67obg57OfV2Dnrhs1HtSdlY/i5fn7MU8=
github.com/mark3labs/mcp-go v0.32.0/go.mod h1:rXqOudj/djTORU/ThxYx8fqEVj/5pvTuuebQ2RC7uk4=
github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE=
github.com/matoous/go-nanoid/v2 v2.1.0/go.mod h1:KlbGNQ+FhrUNIHUxZdL63t7tl4LaPkZNpUULS8H4uVM=
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=
github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ=
github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ=
github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A=
github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/miekg/dns v1.1.66 h1:FeZXOS3VCVsKnEAd+wBkjMC3D2K+ww66Cq3VnCINuJE=
github.com/miekg/dns v1.1.66/go.mod h1:jGFzBsSNbJw6z1HYut1RKBKHA9PBdxeHrZG8J+gC2WE=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
github.com/mozillazg/go-pinyin v0.20.0 h1:BtR3DsxpApHfKReaPO1fCqF4pThRwH9uwvXzm+GnMFQ=
github.com/mozillazg/go-pinyin v0.20.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc=
github.com/neo4j/neo4j-go-driver/v5 v5.28.1 h1:RKWQW7wTgYAY2fU9S+9LaJ9OwRPbRc0I17tlT7nDmAY=
github.com/neo4j/neo4j-go-driver/v5 v5.28.1/go.mod h1:Vff8OwT7QpLm7L2yYr85XNWe9Rbqlbeb9asNXJTHO4k=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
@ -200,14 +229,20 @@ github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
github.com/pdfcpu/pdfcpu v0.11.0 h1:mL18Y3hSHzSezmnrzA21TqlayBOXuAx7BUzzZyroLGM=
github.com/pdfcpu/pdfcpu v0.11.0/go.mod h1:F1ca4GIVFdPtmgvIdvXAycAm88noyNxZwzr9CpTy+Mw=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkoukk/tiktoken-go v0.1.7 h1:qOBHXX4PHtvIvmOtyg1EeKlwFRiMKAcoMp4Q+bLQDmw=
github.com/pkoukk/tiktoken-go v0.1.7/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/qdrant/go-client v1.12.0 h1:KqsIKDAw5iQmxDzRjbzRjhvQ+Igyr7Y84vDCinf1T4M=
github.com/qdrant/go-client v1.12.0/go.mod h1:zFa6t5Y3Oqecoa0aSsGWhMqQWq3x3kTPvm0sMf5qplw=
github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs=
github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg=
github.com/qdrant/go-client v1.14.0 h1:cyz9OOooAexudw5w69LRe9vKCQFYJvaFvt9icOciI1U=
github.com/qdrant/go-client v1.14.0/go.mod h1:iO8ts78jL4x6LDHFOViyYWELVtIBDTjOykBmiOTHLnQ=
github.com/rhysd/go-github-selfupdate v1.2.3 h1:iaa+J202f+Nc+A8zi75uccC8Wg3omaM7HDeimXA22Ag=
github.com/rhysd/go-github-selfupdate v1.2.3/go.mod h1:mp/N8zj6jFfBQy/XMYoWsmfzxazpPAODuqarmPDe2Rg=
github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM=
@ -215,25 +250,27 @@ github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7
github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00=
github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/cast v1.9.2 h1:SsGfm7M8QOFtEzumm7UZrZdLLquNdzFYfIbEXntcFbE=
github.com/spf13/cast v1.9.2/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@ -241,7 +278,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tcnksm/go-gitconfig v0.1.2 h1:iiDhRitByXAEyjgBqsKi9QU4o2TNtv9kPP3RgPgXBPw=
@ -268,6 +304,8 @@ github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8=
github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ=
github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE=
github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw=
github.com/tiendc/go-deepcopy v1.6.0 h1:0UtfV/imoCwlLxVsyfUd4hNHnB3drXsfle+wzSCA5Wo=
github.com/tiendc/go-deepcopy v1.6.0/go.mod h1:toXoeQoUqXOOS/X4sKuiAoSk6elIdqc0pN7MTgOOo2I=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
@ -275,54 +313,57 @@ github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZ
github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/watchfultele/jsonrepair v0.0.0-20250207052432-e4397ed42611 h1:CWCIUJ4cqhc3ct+HbV4EJtBdRFjUVOBne8mmIHXxU6A=
github.com/watchfultele/jsonrepair v0.0.0-20250207052432-e4397ed42611/go.mod h1:63urp6bG6c9cw3DuFtNEr5g9tbjFXaPTn0sBiBDVB7g=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/xuri/efp v0.0.0-20241211021726-c4e992084aa6 h1:8m6DWBG+dlFNbx5ynvrE7NgI+Y7OlZVMVTpayoW+rCc=
github.com/xuri/efp v0.0.0-20241211021726-c4e992084aa6/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
github.com/xuri/excelize/v2 v2.9.0 h1:1tgOaEq92IOEumR1/JfYS/eR0KHOCsRv/rYXXh6YJQE=
github.com/xuri/excelize/v2 v2.9.0/go.mod h1:uqey4QBZ9gdMeWApPLdhm9x+9o2lq4iVmjiLfBS5hdE=
github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 h1:hPVCafDV85blFTabnqKgNhDCkJX25eik94Si9cTER4A=
github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8=
github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
github.com/xuri/excelize/v2 v2.9.1 h1:VdSGk+rraGmgLHGFaGG9/9IWu1nj4ufjJ7uwMDtj8Qw=
github.com/xuri/excelize/v2 v2.9.1/go.mod h1:x7L6pKz2dvo9ejrRuD8Lnl98z4JLt0TGAwjhW+EiP8s=
github.com/xuri/nfp v0.0.1 h1:MDamSGatIvp8uOmDP8FnmjuQpu90NzdJxo7242ANR9Q=
github.com/xuri/nfp v0.0.1/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM=
go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4=
go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY=
go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE=
go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=
go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk=
go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0=
go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc=
go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8=
go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=
go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=
golang.org/x/arch v0.12.0 h1:UsYJhbzPYGsT0HbEdmYcqtCv8UNGvnaL561NnIUvaKg=
golang.org/x/arch v0.12.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
go.mongodb.org/mongo-driver v1.17.3 h1:TQyXhnsWfWtgAhMtOgtYHMTkZIfBTpMTsMnd9ZBeHxQ=
go.mongodb.org/mongo-driver v1.17.3/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY=
go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg=
go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk=
go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w=
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
golang.org/x/arch v0.17.0 h1:4O3dfLzd+lQewptAHqjewQZQDyEdejz3VwgeYwkZneU=
golang.org/x/arch v0.17.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/image v0.23.0 h1:HseQ7c2OpPKTPVzNjG5fwJsOTCiiwS4QdsYi5XU6H68=
golang.org/x/image v0.23.0/go.mod h1:wJJBTdLfCCf3tiHa1fNxpZmUI4mmoZvwMCPP0ddoNKY=
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
golang.org/x/image v0.29.0 h1:HcdsyR4Gsuys/Axh0rDEmlBmB68rW1U9BUdB3UVHsas=
golang.org/x/image v0.29.0/go.mod h1:RVJROnf3SLK8d26OW91j4FrIHGbsJ8QnbEocVTOWQDA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@ -335,12 +376,13 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -348,8 +390,9 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -363,15 +406,15 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@ -392,30 +435,32 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8=
golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw=
golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo=
golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto/googleapis/rpc v0.0.0-20241230172942-26aa7a208def h1:4P81qv5JXI/sDNae2ClVx88cgDDA6DPilADkG9tYKz8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20241230172942-26aa7a208def/go.mod h1:bdAgzvd4kFrpykc5/AC2eLUiegK9T/qxZHD4hXYf/ho=
google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU=
google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4=
google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk=
google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 h1:cJfm9zPbe1e873mHJzmQ1nwVEeRDU/T1wXDK2kUSU34=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA=
google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM=
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=

61
helper/jwt.go
View file

@ -2,9 +2,10 @@ package helper
import (
"fmt"
"strings"
"time"
"github.com/golang-jwt/jwt"
"github.com/golang-jwt/jwt/v4"
"github.com/yaoapp/gou/process"
"github.com/yaoapp/gou/session"
"github.com/yaoapp/kun/any"
@ -13,12 +14,19 @@ import (
"github.com/yaoapp/yao/config"
)
const (
// MaxTokenLength is the maximum allowed length for a JWT token
MaxTokenLength = 4096
// MaxTokenParts is the maximum allowed number of parts in a JWT token (header.payload.signature)
MaxTokenParts = 3
)
// JwtClaims 用户Token
type JwtClaims struct {
ID int `json:"id"`
SID string `json:"sid"`
Data map[string]interface{} `json:"data"`
jwt.StandardClaims
jwt.RegisteredClaims
}
// JwtToken JWT令牌
@ -29,6 +37,18 @@ type JwtToken struct {
// JwtValidate JWT 校验
func JwtValidate(tokenString string, secret ...[]byte) *JwtClaims {
// Check token length
if len(tokenString) > MaxTokenLength {
exception.New("Token too long", 401).Throw()
return nil
}
// Check number of parts
parts := strings.Split(tokenString, ".")
if len(parts) > MaxTokenParts {
exception.New("Invalid token format", 401).Throw()
return nil
}
jwtSecret := []byte(config.Conf.JWTSecret)
if len(secret) > 0 {
@ -62,12 +82,12 @@ func JwtMake(id int, data map[string]interface{}, option map[string]interface{},
jwtSecret = secret[0]
}
now := time.Now().Unix()
now := time.Now()
sid := ""
timeout := int64(36000)
timeout := time.Hour
uid := fmt.Sprintf("%d", id)
subject := "User Token"
audience := "Yao Process utils.jwt.Make"
audience := []string{"Yao Process utils.jwt.Make"}
issuer := fmt.Sprintf("xiang:%d", id)
if v, has := option["subject"]; has {
@ -75,7 +95,7 @@ func JwtMake(id int, data map[string]interface{}, option map[string]interface{},
}
if v, has := option["audience"]; has {
audience = fmt.Sprintf("%v", v)
audience = []string{fmt.Sprintf("%v", v)}
}
if v, has := option["issuer"]; has {
@ -87,45 +107,42 @@ func JwtMake(id int, data map[string]interface{}, option map[string]interface{},
}
if v, has := option["timeout"]; has {
timeout = int64(any.Of(v).CInt())
timeout = time.Duration(any.Of(v).CInt()) * time.Second
}
expiresAt := now + timeout
expiresAt := now.Add(timeout)
if v, has := option["expires_at"]; has {
expiresAt = int64(any.Of(v).CInt())
expiresAt = time.Unix(int64(any.Of(v).CInt()), 0)
}
if sid == "" {
sid = session.ID()
}
// 设定会话过期时间 (并写需要加锁，这个逻辑需要优化)
// session.Global().Expire(time.Duration(timeout) * time.Second)
claims := &JwtClaims{
ID: id,
SID: sid, // 会话ID
Data: data,
StandardClaims: jwt.StandardClaims{
Id: uid, // 唯一ID
Subject: subject, // 主题
Audience: audience, // 接收人
ExpiresAt: expiresAt, // 过期时间
NotBefore: now, // 生效时间
IssuedAt: now, // 签发时间
Issuer: issuer, // 签发人
RegisteredClaims: jwt.RegisteredClaims{
ID: uid, // 唯一ID
Subject: subject, // 主题
Audience: audience, // 接收人
ExpiresAt: jwt.NewNumericDate(expiresAt), // 过期时间
NotBefore: jwt.NewNumericDate(now), // 生效时间
IssuedAt: jwt.NewNumericDate(now), // 签发时间
Issuer: issuer, // 签发人
},
}
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
tokenString, err := token.SignedString([]byte(jwtSecret))
tokenString, err := token.SignedString(jwtSecret)
if err != nil {
exception.New("JWT Make Error: %s", 500, err.Error()).Throw()
}
return JwtToken{
Token: tokenString,
ExpiresAt: expiresAt,
ExpiresAt: expiresAt.Unix(),
}
}

10
i18n/i18n.go
View file

@ -7,6 +7,7 @@ import (
"sync"
"time"
"github.com/yaoapp/gou/application"
"github.com/yaoapp/gou/lang"
"github.com/yaoapp/kun/log"
"github.com/yaoapp/yao/config"
@ -43,6 +44,15 @@ func Load(cfg config.Config) error {
return err
}
// Ignore if the langs directory does not exist
exists, err := application.App.Exists("langs")
if err != nil {
return err
}
if !exists {
return nil
}
// Load langs
err = lang.Load("langs")
if err != nil {

5
importer/importer.go
View file

@ -20,6 +20,11 @@ import (
"github.com/yaoapp/yao/share"
)
// *********************************************************************************
// !! Importer has been deprecated.
// !! Do not use this in your project.
// *********************************************************************************
// Importers 导入器
var Importers = map[string]*Importer{}

1
job/README.md Normal file
View file

@ -0,0 +1 @@
# Job

1
job/job.go Normal file
View file

@ -0,0 +1 @@
package job

12
job/types/interfaces.go Normal file
View file

@ -0,0 +1,12 @@
package types
import "context"
// Job interface
type Job interface {
Run(ctx context.Context) error
AddTask(ctx context.Context, task Task) error
}
// Task interface
type Task func(ctx context.Context, job Job) error

1
job/types/types.go Normal file
View file

@ -0,0 +1 @@
package types

1
kb/README.md Normal file
View file

@ -0,0 +1 @@
# Knowledge Base

166
kb/kb.go Normal file
View file

@ -0,0 +1,166 @@
package kb
import (
"fmt"
"path/filepath"
"slices"
"github.com/yaoapp/gou/application"
"github.com/yaoapp/gou/graphrag"
"github.com/yaoapp/gou/graphrag/types"
"github.com/yaoapp/kun/log"
"github.com/yaoapp/yao/config"
// Register the built-in providers
_ "github.com/yaoapp/yao/kb/providers"
// Import the kb types
kbtypes "github.com/yaoapp/yao/kb/types"
)
// Instance is the GraphRag instance
var Instance types.GraphRag = nil
// KnowledgeBase is the Knowledge Base instance
type KnowledgeBase struct {
Config *kbtypes.Config // Knowledge Base configuration
Providers *kbtypes.ProviderConfig // Multi-language provider configurations
*graphrag.GraphRag
}
// Load loads the GraphRag instance
func Load(appConfig config.Config) (*KnowledgeBase, error) {
configPath := filepath.Join("kb", "kb.yao")
exists, err := application.App.Exists(configPath)
if err != nil {
return nil, err
}
if !exists {
log.Warn("[Knowledge Base] kb.yao file not found, skip loading knowledge base")
return nil, nil
}
// Load providers from directories first
providers, err := kbtypes.LoadProviders("kb")
if err != nil {
return nil, err
}
// Parse the configuration
var config kbtypes.Config
raw, err := application.App.Read(filepath.Join("kb", "kb.yao"))
if err != nil {
return nil, err
}
err = application.Parse("kb.yao", raw, &config)
if err != nil {
return nil, err
}
// Assign providers to config
config.Providers = providers
// Compute features after both config and providers are loaded
config.Features = config.ComputeFeatures()
// Set global configurations for providers to use
kbtypes.SetGlobalPDF(config.PDF)
kbtypes.SetGlobalFFmpeg(config.FFmpeg)
// Create the GraphRag config
graphRagConfig, err := config.GraphRagConfig()
if err != nil {
return nil, err
}
// Create the GraphRag instance
graphRag, err := graphrag.New(graphRagConfig)
if err != nil {
return nil, err
}
// Set the instance
instance := &KnowledgeBase{Config: &config, Providers: providers, GraphRag: graphRag}
// Set the instance to the global variable
Instance = instance
return instance, nil
}
// GetProviders returns all providers
func GetProviders(typ string, ids []string, locale string) ([]kbtypes.Provider, error) {
if Instance == nil {
return nil, fmt.Errorf("knowledge base not initialized")
}
// Get the providers from the instance
knowledgeBase, ok := Instance.(*KnowledgeBase)
if !ok {
return nil, fmt.Errorf("knowledge base not initialized")
}
// Default locale to "en" if empty
if locale == "" {
locale = "en"
}
// Get providers for the requested type and language
providers := knowledgeBase.Providers.GetProviders(typ, locale)
// Filter empty ids
filteredIds := []string{}
for _, id := range ids {
if id != "" {
filteredIds = append(filteredIds, id)
}
}
// Filter the providers by ids
filteredProviders := []kbtypes.Provider{}
for _, provider := range providers {
if len(filteredIds) == 0 || slices.Contains(ids, provider.ID) {
filteredProviders = append(filteredProviders, *provider)
}
}
return filteredProviders, nil
}
// GetProvider returns a provider by id with default language "en"
func GetProvider(typ string, id string) (*kbtypes.Provider, error) {
return GetProviderWithLanguage(typ, id, "en")
}
// GetProviderWithLanguage returns a provider by id, type, and language
func GetProviderWithLanguage(typ string, id string, locale string) (*kbtypes.Provider, error) {
if Instance == nil {
return nil, fmt.Errorf("knowledge base not initialized")
}
knowledgeBase, ok := Instance.(*KnowledgeBase)
if !ok {
return nil, fmt.Errorf("knowledge base not initialized")
}
// Default locale to "en" if empty
if locale == "" {
locale = "en"
}
return knowledgeBase.Providers.GetProvider(typ, id, locale)
}
// GetConfig returns the knowledge base configuration
func GetConfig() (*kbtypes.Config, error) {
if Instance == nil {
return nil, fmt.Errorf("knowledge base not initialized")
}
knowledgeBase, ok := Instance.(*KnowledgeBase)
if !ok {
return nil, fmt.Errorf("knowledge base not initialized")
}
return knowledgeBase.Config, nil
}

Some files were not shown because too many files have changed in this diff Show more