Merge pull request #1382 from trheyi/main

Enhance CI Workflows and Makefile for Testing and Benchmarking

.github/workflows/pr-test.yml

@@ -163,7 +163,343 @@ env:
   TWILIO_TEST_PHONE: ${{ secrets.TWILIO_TEST_PHONE }}
 
 jobs:
-  UnitTest:
+  # =============================================================================
+  # AI Tests (agent, aigc) - Run once with SQLite
+  # =============================================================================
+  AITest:
+    runs-on: ubuntu-latest
+    services:
+      qdrant:
+        image: qdrant/qdrant:latest
+        ports:
+          - 6333:6333
+          - 6334:6334
+
+      fastembed:
+        image: yaoapp/fastembed:latest-amd64
+        env:
+          FASTEMBED_PASSWORD: Yao@2026
+        ports:
+          - 6001:8000
+
+      neo4j:
+        image: neo4j:latest
+        ports:
+          - "7687:7687"
+        env:
+          NEO4J_AUTH: neo4j/Yao2026Neo4j
+
+      mcp-everything:
+        image: yaoapp/mcp-everything:latest
+        ports:
+          - "3021:3021"
+          - "3022:3022"
+
+      mongodb:
+        image: mongo:6.0
+        ports:
+          - 27017:27017
+        env:
+          MONGO_INITDB_ROOT_USERNAME: root
+          MONGO_INITDB_ROOT_PASSWORD: 123456
+          MONGO_INITDB_DATABASE: test
+
+    strategy:
+      matrix:
+        go: [1.24]
+    if: >
+      ${{ github.event.workflow_run.event == 'pull_request' &&
+      github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: "Download artifact"
+        uses: actions/github-script@v7
+        with:
+          script: |
+            var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: ${{github.event.workflow_run.id }},
+            });
+            var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "pr"
+            })[0];
+            var download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            var fs = require('fs');
+            fs.writeFileSync('${{github.workspace}}/pr.zip', Buffer.from(download.data));
+
+      - name: "Read NR & SHA"
+        run: |
+          unzip pr.zip
+          cat NR
+          cat SHA
+          echo HEAD=$(cat SHA) >> $GITHUB_ENV
+          echo NR=$(cat NR) >> $GITHUB_ENV
+
+      - name: "Comment on PR"
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const { NR } = process.env
+            var issue_number = NR;
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: issue_number,
+              body: '🤖 AI Tests (agent, aigc) running with SQLite...'
+            });
+
+      - name: Checkout Kun
+        uses: actions/checkout@v4
+        with:
+          repository: yaoapp/kun
+          path: kun
+
+      - name: Checkout Xun
+        uses: actions/checkout@v4
+        with:
+          repository: yaoapp/xun
+          path: xun
+
+      - name: Checkout Gou
+        uses: actions/checkout@v4
+        with:
+          repository: yaoapp/gou
+          path: gou
+
+      - name: Checkout V8Go
+        uses: actions/checkout@v4
+        with:
+          repository: yaoapp/v8go
+          path: v8go
+
+      - name: Unzip libv8
+        run: |
+          files=$(find ./v8go -name "libv8*.zip")
+          for file in $files; do
+            dir=$(dirname "$file")
+            echo "Extracting $file to directory $dir"
+            unzip -o -d $dir $file
+            rm -rf $dir/__MACOSX
+          done
+
+      - name: Checkout Demo App
+        uses: actions/checkout@v4
+        with:
+          repository: yaoapp/yao-dev-app
+          path: app
+
+      - name: Checkout Extension
+        uses: actions/checkout@v4
+        with:
+          repository: yaoapp/yao-extensions-dev
+          path: extension
+
+      - name: Move Dependencies
+        run: |
+          mv kun ../
+          mv xun ../
+          mv gou ../
+          mv v8go ../
+          mv app ../
+          mv extension ../
+
+      - name: Checkout pull request HEAD commit
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.HEAD }}
+
+      - name: Setup Go ${{ matrix.go }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go }}
+
+      - name: Setup Go Tools
+        run: make tools
+
+      - name: Setup ENV (SQLite)
+        run: |
+          mkdir -p ${{ github.WORKSPACE }}/../app/db
+          echo "YAO_DB_DRIVER=sqlite3" >> $GITHUB_ENV
+          echo "YAO_DB_PRIMARY=${{ github.WORKSPACE }}/../app/db/yao.db" >> $GITHUB_ENV
+
+      - name: Run AI Tests (agent, aigc)
+        run: make unit-test-ai
+
+      - name: "Comment on PR - AI Tests Done"
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const { NR } = process.env
+            var issue_number = NR;
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: issue_number,
+              body: '✅ AI Tests (agent, aigc) passed!'
+            });
+
+  # =============================================================================
+  # Benchmark & Memory Leak Tests - Run with MySQL8.0 and SQLite3
+  # =============================================================================
+  PerfTest:
+    runs-on: ubuntu-latest
+    services:
+      mcp-everything:
+        image: yaoapp/mcp-everything:latest
+        ports:
+          - "3021:3021"
+          - "3022:3022"
+
+      mongodb:
+        image: mongo:6.0
+        ports:
+          - 27017:27017
+        env:
+          MONGO_INITDB_ROOT_USERNAME: root
+          MONGO_INITDB_ROOT_PASSWORD: 123456
+          MONGO_INITDB_DATABASE: test
+
+    strategy:
+      matrix:
+        go: [1.24]
+        db: [MySQL8.0, SQLite3]
+    if: >
+      ${{ github.event.workflow_run.event == 'pull_request' &&
+      github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: "Download artifact"
+        uses: actions/github-script@v7
+        with:
+          script: |
+            var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: ${{github.event.workflow_run.id }},
+            });
+            var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "pr"
+            })[0];
+            var download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            var fs = require('fs');
+            fs.writeFileSync('${{github.workspace}}/pr.zip', Buffer.from(download.data));
+
+      - name: "Read NR & SHA"
+        run: |
+          unzip pr.zip
+          cat NR
+          cat SHA
+          echo HEAD=$(cat SHA) >> $GITHUB_ENV
+          echo NR=$(cat NR) >> $GITHUB_ENV
+
+      - name: Checkout Kun
+        uses: actions/checkout@v4
+        with:
+          repository: yaoapp/kun
+          path: kun
+
+      - name: Checkout Xun
+        uses: actions/checkout@v4
+        with:
+          repository: yaoapp/xun
+          path: xun
+
+      - name: Checkout Gou
+        uses: actions/checkout@v4
+        with:
+          repository: yaoapp/gou
+          path: gou
+
+      - name: Checkout V8Go
+        uses: actions/checkout@v4
+        with:
+          repository: yaoapp/v8go
+          path: v8go
+
+      - name: Unzip libv8
+        run: |
+          files=$(find ./v8go -name "libv8*.zip")
+          for file in $files; do
+            dir=$(dirname "$file")
+            echo "Extracting $file to directory $dir"
+            unzip -o -d $dir $file
+            rm -rf $dir/__MACOSX
+          done
+
+      - name: Checkout Demo App
+        uses: actions/checkout@v4
+        with:
+          repository: yaoapp/yao-dev-app
+          path: app
+
+      - name: Checkout Extension
+        uses: actions/checkout@v4
+        with:
+          repository: yaoapp/yao-extensions-dev
+          path: extension
+
+      - name: Move Dependencies
+        run: |
+          mv kun ../
+          mv xun ../
+          mv gou ../
+          mv v8go ../
+          mv app ../
+          mv extension ../
+
+      - name: Checkout pull request HEAD commit
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.HEAD }}
+
+      - name: Setup Go ${{ matrix.go }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go }}
+
+      - name: Setup Go Tools
+        run: make tools
+
+      - name: Setup ${{ matrix.db }}
+        uses: ./.github/actions/setup-db
+        with:
+          kind: "${{ matrix.db }}"
+          db: "xiang"
+          user: "xiang"
+          password: ${{ secrets.UNIT_PASS }}
+
+      - name: Setup ENV
+        env:
+          PASSWORD: ${{ secrets.UNIT_PASS }}
+        run: |
+          echo "YAO_DB_DRIVER=$DB_DRIVER" >> $GITHUB_ENV
+          if [ "$DB_DRIVER" = "mysql" ]; then
+            echo "YAO_DB_PRIMARY=$DB_USER:$PASSWORD@$DB_HOST" >> $GITHUB_ENV
+          else
+            echo "YAO_DB_PRIMARY=${{ github.WORKSPACE }}/../app/db/yao.db" >> $GITHUB_ENV
+            mkdir -p ${{ github.WORKSPACE }}/../app/db
+          fi
+
+      - name: Run Benchmark & Memory Leak Tests
+        run: |
+          make benchmark
+          make memory-leak
+
+  # =============================================================================
+  # Core Tests - Run with DB matrix (MySQL/SQLite/Redis/Mongo combinations)
+  # =============================================================================
+  CoreTest:
     runs-on: ubuntu-latest
 
     services:
@@ -196,7 +532,7 @@ jobs:
     strategy:
       matrix:
         go: [1.24]
-        db: [MySQL8.0, MySQL5.7, SQLite3]
+        db: [MySQL8.0, SQLite3]
         redis: [4, 5, 6]
         mongo: ["6.0"]
     if: >
@@ -410,9 +746,8 @@ jobs:
           make fmt-check
           make misspell-check
 
-      - name: Run test
-        run: |
-          make test
+      - name: Run Core Tests (exclude AI)
+        run: make unit-test-core
 
       - name: Codecov Report
         uses: codecov/codecov-action@v4

.github/workflows/unit-test.yml

@@ -170,7 +170,247 @@ env:
   TWILIO_TEST_PHONE: ${{ secrets.TWILIO_TEST_PHONE }}
 
 jobs:
-  unit-test:
+  # =============================================================================
+  # AI Tests (agent, aigc) - Run once with SQLite
+  # =============================================================================
+  ai-test:
+    runs-on: ubuntu-latest
+    services:
+      qdrant:
+        image: qdrant/qdrant:latest
+        ports:
+          - 6333:6333
+          - 6334:6334
+
+      fastembed:
+        image: yaoapp/fastembed:latest-amd64
+        env:
+          FASTEMBED_PASSWORD: Yao@2026
+        ports:
+          - 6001:8000
+
+      neo4j:
+        image: neo4j:latest
+        ports:
+          - "7687:7687"
+        env:
+          NEO4J_AUTH: neo4j/Yao2026Neo4j
+
+      mcp-everything:
+        image: yaoapp/mcp-everything:latest
+        ports:
+          - "3021:3021"
+          - "3022:3022"
+
+      mongodb:
+        image: mongo:6.0
+        ports:
+          - 27017:27017
+        env:
+          MONGO_INITDB_ROOT_USERNAME: root
+          MONGO_INITDB_ROOT_PASSWORD: 123456
+          MONGO_INITDB_DATABASE: test
+
+    strategy:
+      matrix:
+        go: [1.24]
+    steps:
+      - name: Checkout Kun
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ env.REPO_KUN }}
+          path: kun
+
+      - name: Checkout Xun
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ env.REPO_XUN }}
+          path: xun
+
+      - name: Checkout Gou
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ env.REPO_GOU }}
+          path: gou
+
+      - name: Checkout V8Go
+        uses: actions/checkout@v4
+        with:
+          repository: yaoapp/v8go
+          path: v8go
+
+      - name: Unzip libv8
+        run: |
+          files=$(find ./v8go -name "libv8*.zip")
+          for file in $files; do
+            dir=$(dirname "$file")
+            echo "Extracting $file to directory $dir"
+            unzip -o -d $dir $file
+            rm -rf $dir/__MACOSX
+          done
+
+      - name: Checkout Demo App
+        uses: actions/checkout@v4
+        with:
+          repository: yaoapp/yao-dev-app
+          path: app
+
+      - name: Checkout Extension
+        uses: actions/checkout@v4
+        with:
+          repository: yaoapp/yao-extensions-dev
+          path: extension
+
+      - name: Move Dependencies
+        run: |
+          mv kun ../
+          mv xun ../
+          mv gou ../
+          mv v8go ../
+          mv app ../
+          mv extension ../
+
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Setup Go ${{ matrix.go }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go }}
+
+      - name: Setup Go Tools
+        run: make tools
+
+      - name: Setup ENV (SQLite)
+        run: |
+          mkdir -p ${{ github.WORKSPACE }}/../app/db
+          echo "YAO_DB_DRIVER=sqlite3" >> $GITHUB_ENV
+          echo "YAO_DB_PRIMARY=${{ github.WORKSPACE }}/../app/db/yao.db" >> $GITHUB_ENV
+
+      - name: Run AI Tests (agent, aigc)
+        run: make unit-test-ai
+
+  # =============================================================================
+  # Benchmark & Memory Leak Tests - Run with MySQL8.0 and SQLite3
+  # =============================================================================
+  perf-test:
+    runs-on: ubuntu-latest
+    services:
+      mcp-everything:
+        image: yaoapp/mcp-everything:latest
+        ports:
+          - "3021:3021"
+          - "3022:3022"
+
+      mongodb:
+        image: mongo:6.0
+        ports:
+          - 27017:27017
+        env:
+          MONGO_INITDB_ROOT_USERNAME: root
+          MONGO_INITDB_ROOT_PASSWORD: 123456
+          MONGO_INITDB_DATABASE: test
+
+    strategy:
+      matrix:
+        go: [1.24]
+        db: [MySQL8.0, SQLite3]
+    steps:
+      - name: Checkout Kun
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ env.REPO_KUN }}
+          path: kun
+
+      - name: Checkout Xun
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ env.REPO_XUN }}
+          path: xun
+
+      - name: Checkout Gou
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ env.REPO_GOU }}
+          path: gou
+
+      - name: Checkout V8Go
+        uses: actions/checkout@v4
+        with:
+          repository: yaoapp/v8go
+          path: v8go
+
+      - name: Unzip libv8
+        run: |
+          files=$(find ./v8go -name "libv8*.zip")
+          for file in $files; do
+            dir=$(dirname "$file")
+            echo "Extracting $file to directory $dir"
+            unzip -o -d $dir $file
+            rm -rf $dir/__MACOSX
+          done
+
+      - name: Checkout Demo App
+        uses: actions/checkout@v4
+        with:
+          repository: yaoapp/yao-dev-app
+          path: app
+
+      - name: Checkout Extension
+        uses: actions/checkout@v4
+        with:
+          repository: yaoapp/yao-extensions-dev
+          path: extension
+
+      - name: Move Dependencies
+        run: |
+          mv kun ../
+          mv xun ../
+          mv gou ../
+          mv v8go ../
+          mv app ../
+          mv extension ../
+
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Setup Go ${{ matrix.go }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go }}
+
+      - name: Setup Go Tools
+        run: make tools
+
+      - name: Setup ${{ matrix.db }}
+        uses: ./.github/actions/setup-db
+        with:
+          kind: "${{ matrix.db }}"
+          db: "xiang"
+          user: "xiang"
+          password: ${{ secrets.UNIT_PASS }}
+
+      - name: Setup ENV
+        env:
+          PASSWORD: ${{ secrets.UNIT_PASS }}
+        run: |
+          echo "YAO_DB_DRIVER=$DB_DRIVER" >> $GITHUB_ENV
+          if [ "$DB_DRIVER" = "mysql" ]; then
+            echo "YAO_DB_PRIMARY=$DB_USER:$PASSWORD@$DB_HOST" >> $GITHUB_ENV
+          else
+            echo "YAO_DB_PRIMARY=${{ github.WORKSPACE }}/../app/db/yao.db" >> $GITHUB_ENV
+            mkdir -p ${{ github.WORKSPACE }}/../app/db
+          fi
+
+      - name: Run Benchmark & Memory Leak Tests
+        run: |
+          make benchmark
+          make memory-leak
+
+  # =============================================================================
+  # Core Tests - Run with DB matrix (MySQL/SQLite/Redis/Mongo combinations)
+  # =============================================================================
+  core-test:
     runs-on: ubuntu-latest
     services:
       qdrant:
@@ -202,7 +442,7 @@ jobs:
     strategy:
       matrix:
         go: [1.24]
-        db: [MySQL8.0, MySQL5.7, SQLite3]
+        db: [MySQL8.0, SQLite3]
         redis: [4, 5, 6]
         mongo: ["6.0"]
     steps:
@@ -374,9 +614,8 @@ jobs:
           go run . run utils.env.Get REDIS_TEST_HOST
           go run . inspect
 
-      - name: Run test
-        run: |
-          make test
+      - name: Run Core Tests (exclude AI)
+        run: make unit-test-core
 
       - name: Codecov Report
         uses: codecov/codecov-action@v4

.gitignore

@@ -51,3 +51,4 @@ openapi/*.md
 coverage.html
 agent/assistant/hook/*.test.md
 agent/search/TODO.md
+agent/search/job-logs.txt

Makefile

@@ -11,11 +11,15 @@ OS := $(shell uname)
 
 # ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 TESTFOLDER := $(shell $(GO) list ./... | grep -vE 'examples|openai|aigc|neo|twilio|share*' | awk '!/\/tests\// || /openapi\/tests/')
+# Core tests (exclude AI-related: agent, aigc, openai)
+TESTFOLDER_CORE := $(shell $(GO) list ./... | grep -vE 'examples|openai|aigc|neo|twilio|share*|agent' | awk '!/\/tests\// || /openapi\/tests/')
+# AI tests (agent, aigc)
+TESTFOLDER_AI := $(shell $(GO) list ./agent/... ./aigc/...)
 TESTTAGS ?= ""
 
 # TESTWIDGETS := $(shell $(GO) list ./widgets/...)
 
-# Unit Test
+# Unit Test (all tests)
 .PHONY: unit-test
 unit-test:
 	echo "mode: count" > coverage.out
@@ -41,6 +45,64 @@ unit-test:
 		fi; \
 	done
 
+# Core Unit Test (exclude AI-related tests)
+.PHONY: unit-test-core
+unit-test-core:
+	echo "mode: count" > coverage.out
+	for d in $(TESTFOLDER_CORE); do \
+		$(GO) test -tags $(TESTTAGS) -v -covermode=count -coverprofile=profile.out -coverpkg=$$(echo $$d | sed "s/\/test$$//g") -skip='TestMemoryLeak|TestIsolateDisposal' $$d > tmp.out; \
+		cat tmp.out; \
+		if grep -q "^--- FAIL" tmp.out; then \
+			rm tmp.out; \
+			exit 1; \
+		elif grep -q "build failed" tmp.out; then \
+			rm tmp.out; \
+			exit 1; \
+		elif grep -q "setup failed" tmp.out; then \
+			rm tmp.out; \
+			exit 1; \
+		elif grep -q "runtime error" tmp.out; then \
+			rm tmp.out; \
+			exit 1; \
+		fi; \
+		if [ -f profile.out ]; then \
+			cat profile.out | grep -v "mode:" >> coverage.out; \
+			rm profile.out; \
+		fi; \
+	done
+
+# AI Unit Test (agent, aigc)
+.PHONY: unit-test-ai
+unit-test-ai:
+	echo "mode: count" > coverage-ai.out
+	for d in $(TESTFOLDER_AI); do \
+		$(GO) test -tags $(TESTTAGS) -v -timeout=20m -covermode=count -coverprofile=profile.out -coverpkg=$$(echo $$d | sed "s/\/test$$//g") -skip='TestMemoryLeak|TestIsolateDisposal' $$d > tmp.out; \
+		cat tmp.out; \
+		if grep -q "^--- FAIL" tmp.out; then \
+			rm tmp.out; \
+			exit 1; \
+		elif grep -q "^FAIL" tmp.out; then \
+			rm tmp.out; \
+			exit 1; \
+		elif grep -q "^panic:" tmp.out; then \
+			rm tmp.out; \
+			exit 1; \
+		elif grep -q "build failed" tmp.out; then \
+			rm tmp.out; \
+			exit 1; \
+		elif grep -q "setup failed" tmp.out; then \
+			rm tmp.out; \
+			exit 1; \
+		elif grep -q "runtime error" tmp.out; then \
+			rm tmp.out; \
+			exit 1; \
+		fi; \
+		if [ -f profile.out ]; then \
+			cat profile.out | grep -v "mode:" >> coverage-ai.out; \
+			rm profile.out; \
+		fi; \
+	done
+
 # Benchmark Test
 .PHONY: benchmark
 benchmark:
@@ -73,7 +135,7 @@ memory-leak:
 			echo ""; \
 			echo "🔍 Memory Leak Detection: $$d"; \
 			echo "---------------------------------------------"; \
-			$(GO) test -run='TestMemoryLeak|TestIsolateDisposal|TestGoroutineLeak' -v -timeout=60s $$d || exit 1; \
+			$(GO) test -run='TestMemoryLeak|TestIsolateDisposal|TestGoroutineLeak' -v -timeout=5m $$d || exit 1; \
 		fi; \
 	done
 	@echo ""

agent/search/search_web_test.go

@@ -145,8 +145,8 @@ func TestWebSearch_All(t *testing.T) {
 	testutils.Prepare(t)
 	defer testutils.Clean(t)
 
-	// Load the web-tavily test assistant
-	ast, err := assistant.LoadPath("/assistants/tests/web-tavily")
+	// Load the web-serper test assistant
+	ast, err := assistant.LoadPath("/assistants/tests/web-serper")
 	require.NoError(t, err)
 	require.NotNil(t, ast.Search)
 
@@ -220,8 +220,8 @@ func TestWebSearch_Race(t *testing.T) {
 	testutils.Prepare(t)
 	defer testutils.Clean(t)
 
-	// Load the web-tavily test assistant
-	ast, err := assistant.LoadPath("/assistants/tests/web-tavily")
+	// Load the web-serper test assistant
+	ast, err := assistant.LoadPath("/assistants/tests/web-serper")
 	require.NoError(t, err)
 	require.NotNil(t, ast.Search)
 
@@ -260,8 +260,8 @@ func TestWebSearch_BuildReferences(t *testing.T) {
 	testutils.Prepare(t)
 	defer testutils.Clean(t)
 
-	// Load the web-tavily test assistant
-	ast, err := assistant.LoadPath("/assistants/tests/web-tavily")
+	// Load the web-serper test assistant
+	ast, err := assistant.LoadPath("/assistants/tests/web-serper")
 	require.NoError(t, err)
 	require.NotNil(t, ast.Search)