Merge pull request #1035 from trheyi/main

Enhance CI workflows and Knowledge Base integration

.github/workflows/pr-test.yml

@@ -64,9 +64,46 @@ env:
   YAO_RUNTIME_HEAP_AVAILABLE: 550000000
   YAO_RUNTIME_PRECOMPILE: true
 
+  # Neo4j
+  NEO4J_TEST_URL: "neo4j://localhost:7686"
+  NEO4J_TEST_USER: "neo4j"
+  NEO4J_TEST_PASS: "Yao2026Neo4j"
+
+  # Qdrant
+  QDRANT_TEST_HOST: "127.0.0.1"
+  QDRANT_TEST_PORT: "6334"
+
 jobs:
   UnitTest:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
+
+    services:
+      qdrant:
+        image: qdrant/qdrant:latest
+        ports:
+          - 6333:6333 # HTTP API
+          - 6334:6334 # gRPC
+
+      fastembed:
+        image: yaoapp/fastembed:latest-amd64
+        env:
+          FASTEMBED_PASSWORD: Yao@2026
+        ports:
+          - 6001:8000
+
+      neo4j:
+        image: neo4j:latest
+        ports:
+          - "7687:7687"
+        env:
+          NEO4J_AUTH: neo4j/Yao2026Neo4j
+
+      mcp-everything:
+        image: yaoapp/mcp-everything:latest
+        ports:
+          - "3021:3021"
+          - "3022:3022"
+
     strategy:
       matrix:
         go: [1.24]
@@ -202,6 +239,28 @@ jobs:
         with:
           go-version: ${{ matrix.go }}
 
+      - name: Install FFmpeg 7.x
+        run: |
+          wget https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz
+          tar -xf ffmpeg-master-latest-linux64-gpl.tar.xz
+          sudo cp ffmpeg-master-latest-linux64-gpl/bin/ffmpeg /usr/local/bin/
+          sudo cp ffmpeg-master-latest-linux64-gpl/bin/ffprobe /usr/local/bin/
+          sudo chmod +x /usr/local/bin/ffmpeg /usr/local/bin/ffprobe
+
+      - name: Test FFmpeg
+        run: ffmpeg -version
+
+      - name: Install pdftoppm, mutool, imagemagick
+        run: |
+          sudo apt update
+          sudo apt install -y poppler-utils mupdf-tools imagemagick
+
+      - name: Test pdftoppm, mutool, imagemagick
+        run: |
+          pdftoppm -v
+          mutool -v
+          convert -version
+
       - name: Start MongoDB
         uses: supercharge/mongodb-github-action@1.8.0
         with:

.github/workflows/unit-test.yml

@@ -68,9 +68,45 @@ env:
   YAO_RUNTIME_HEAP_AVAILABLE: 550000000
   YAO_RUNTIME_PRECOMPILE: true
 
+  # Neo4j
+  NEO4J_TEST_URL: "neo4j://localhost:7686"
+  NEO4J_TEST_USER: "neo4j"
+  NEO4J_TEST_PASS: "Yao2026Neo4j"
+
+  # Qdrant
+  QDRANT_TEST_HOST: "127.0.0.1"
+  QDRANT_TEST_PORT: "6334"
+
 jobs:
   unit-test:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
+    services:
+      qdrant:
+        image: qdrant/qdrant:latest
+        ports:
+          - 6333:6333 # HTTP API
+          - 6334:6334 # gRPC
+
+      fastembed:
+        image: yaoapp/fastembed:latest-amd64
+        env:
+          FASTEMBED_PASSWORD: Yao@2026
+        ports:
+          - 6001:8000
+
+      neo4j:
+        image: neo4j:latest
+        ports:
+          - "7687:7687"
+        env:
+          NEO4J_AUTH: neo4j/Yao2026Neo4j
+
+      mcp-everything:
+        image: yaoapp/mcp-everything:latest
+        ports:
+          - "3021:3021"
+          - "3022:3022"
+
     strategy:
       matrix:
         go: [1.24]
@@ -152,6 +188,28 @@ jobs:
         with:
           go-version: ${{ matrix.go }}
 
+      - name: Install FFmpeg 7.x
+        run: |
+          wget https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz
+          tar -xf ffmpeg-master-latest-linux64-gpl.tar.xz
+          sudo cp ffmpeg-master-latest-linux64-gpl/bin/ffmpeg /usr/local/bin/
+          sudo cp ffmpeg-master-latest-linux64-gpl/bin/ffprobe /usr/local/bin/
+          sudo chmod +x /usr/local/bin/ffmpeg /usr/local/bin/ffprobe
+
+      - name: Test FFmpeg
+        run: ffmpeg -version
+
+      - name: Install pdftoppm, mutool, imagemagick
+        run: |
+          sudo apt update
+          sudo apt install -y poppler-utils mupdf-tools imagemagick
+
+      - name: Test pdftoppm, mutool, imagemagick
+        run: |
+          pdftoppm -v
+          mutool -v
+          convert -version
+
       - name: Start Redis
         uses: supercharge/redis-github-action@1.4.0
         with:

engine/load.go

@@ -19,6 +19,7 @@ import (
 	"github.com/yaoapp/yao/flow"
 	"github.com/yaoapp/yao/fs"
 	"github.com/yaoapp/yao/i18n"
+	"github.com/yaoapp/yao/kb"
 	"github.com/yaoapp/yao/moapi"
 	"github.com/yaoapp/yao/model"
 	"github.com/yaoapp/yao/neo"
@@ -280,6 +281,13 @@ func Load(cfg config.Config, options LoadOption) (warnings []Warning, err error)
 		}
 	}
 
+	// Load Knowledge Base
+	_, err = kb.Load(cfg)
+	if err != nil {
+		// printErr(cfg.Mode, "Knowledge Base", err)
+		warnings = append(warnings, Warning{Widget: "Knowledge Base", Error: err})
+	}
+
 	// Load OpenAPI
 	_, err = openapi.Load(cfg)
 	if err != nil {

kb/config.go

@@ -3,6 +3,8 @@ package kb
 import (
 	"encoding/json"
 	"os"
+	"regexp"
+	"strings"
 
 	"github.com/yaoapp/gou/graphrag"
 	"github.com/yaoapp/gou/graphrag/graph/neo4j"
@@ -114,8 +116,14 @@ func (c *Config) createGraphStore() (types.GraphStore, error) {
 
 // toVectorStoreConfig converts the vector config to VectorStoreConfig
 func (c *Config) toVectorStoreConfig() (types.VectorStoreConfig, error) {
-	// Convert map[string]interface{} to types.VectorStoreConfig via JSON
+	// Parse environment variables in config
-	jsonData, err := json.Marshal(c.Vector.Config)
+	resolvedConfig, err := c.resolveEnvVars(c.Vector.Config)
+	if err != nil {
+		return types.VectorStoreConfig{}, err
+	}
+
+	// Convert resolved config to types.VectorStoreConfig via JSON
+	jsonData, err := json.Marshal(resolvedConfig)
 	if err != nil {
 		return types.VectorStoreConfig{}, err
 	}
@@ -130,8 +138,14 @@ func (c *Config) toVectorStoreConfig() (types.VectorStoreConfig, error) {
 
 // toGraphStoreConfig converts the graph config to GraphStoreConfig
 func (c *Config) toGraphStoreConfig() (types.GraphStoreConfig, error) {
-	// Convert map[string]interface{} to types.GraphStoreConfig via JSON
+	// Parse environment variables in config
-	jsonData, err := json.Marshal(c.Graph.Config)
+	resolvedConfig, err := c.resolveEnvVars(c.Graph.Config)
+	if err != nil {
+		return types.GraphStoreConfig{}, err
+	}
+
+	// Convert resolved config to types.GraphStoreConfig via JSON
+	jsonData, err := json.Marshal(resolvedConfig)
 	if err != nil {
 		return types.GraphStoreConfig{}, err
 	}
@@ -144,6 +158,48 @@ func (c *Config) toGraphStoreConfig() (types.GraphStoreConfig, error) {
 	return config, nil
 }
 
+// resolveEnvVars resolves environment variables in configuration values
+func (c *Config) resolveEnvVars(config map[string]interface{}) (map[string]interface{}, error) {
+	resolved := make(map[string]interface{})
+
+	for key, value := range config {
+		switch v := value.(type) {
+		case string:
+			resolved[key] = c.parseEnvVar(v)
+		case map[string]interface{}:
+			// Recursively resolve nested maps
+			nestedResolved, err := c.resolveEnvVars(v)
+			if err != nil {
+				return nil, err
+			}
+			resolved[key] = nestedResolved
+		default:
+			resolved[key] = value
+		}
+	}
+
+	return resolved, nil
+}
+
+// parseEnvVar parses environment variable pattern $ENV.VAR_NAME
+func (c *Config) parseEnvVar(value string) string {
+	// Simple pattern to match $ENV.VAR_NAME
+	envPattern := regexp.MustCompile(`\$ENV\.([A-Za-z_][A-Za-z0-9_]*)`)
+
+	return envPattern.ReplaceAllStringFunc(value, func(match string) string {
+		// Extract variable name (remove $ENV. prefix)
+		varName := strings.TrimPrefix(match, "$ENV.")
+
+		// Get environment variable value
+		if envValue := os.Getenv(varName); envValue != "" {
+			return envValue
+		}
+
+		// Return original if environment variable is not set
+		return match
+	})
+}
+
 // UnmarshalJSON implements json.Unmarshaler interface
 func (c *Config) UnmarshalJSON(data []byte) error {
 	// Use alias type to avoid infinite recursion

kb/config_test.go

@@ -373,3 +373,136 @@ func TestRoundTrip(t *testing.T) {
 		t.Errorf("Features mismatch: %+v != %+v", originalConfig.Features, roundTripConfig.Features)
 	}
 }
+
+func TestConfig_ResolveEnvVars(t *testing.T) {
+	// Set test environment variables
+	os.Setenv("TEST_HOST", "localhost")
+	os.Setenv("TEST_PORT", "6333")
+	defer func() {
+		os.Unsetenv("TEST_HOST")
+		os.Unsetenv("TEST_PORT")
+	}()
+
+	config := &Config{}
+
+	tests := []struct {
+		name     string
+		input    map[string]interface{}
+		expected map[string]interface{}
+	}{
+		{
+			name: "simple environment variable",
+			input: map[string]interface{}{
+				"host": "$ENV.TEST_HOST",
+				"port": "$ENV.TEST_PORT",
+			},
+			expected: map[string]interface{}{
+				"host": "localhost",
+				"port": "6333",
+			},
+		},
+		{
+			name: "mixed values",
+			input: map[string]interface{}{
+				"host":   "$ENV.TEST_HOST",
+				"port":   6333,
+				"prefix": "test-$ENV.TEST_HOST-suffix",
+			},
+			expected: map[string]interface{}{
+				"host":   "localhost",
+				"port":   6333,
+				"prefix": "test-localhost-suffix",
+			},
+		},
+		{
+			name: "nested configuration",
+			input: map[string]interface{}{
+				"database": map[string]interface{}{
+					"host": "$ENV.TEST_HOST",
+					"port": "$ENV.TEST_PORT",
+				},
+				"name": "test",
+			},
+			expected: map[string]interface{}{
+				"database": map[string]interface{}{
+					"host": "localhost",
+					"port": "6333",
+				},
+				"name": "test",
+			},
+		},
+		{
+			name: "undefined environment variable",
+			input: map[string]interface{}{
+				"host": "$ENV.UNDEFINED_VAR",
+				"port": "$ENV.TEST_PORT",
+			},
+			expected: map[string]interface{}{
+				"host": "$ENV.UNDEFINED_VAR", // Should remain unchanged
+				"port": "6333",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := config.resolveEnvVars(tt.input)
+			if err != nil {
+				t.Errorf("resolveEnvVars() error = %v", err)
+				return
+			}
+			if !reflect.DeepEqual(result, tt.expected) {
+				t.Errorf("resolveEnvVars() = %+v, want %+v", result, tt.expected)
+			}
+		})
+	}
+}
+
+func TestConfig_ParseEnvVar(t *testing.T) {
+	// Set test environment variables
+	os.Setenv("TEST_HOST", "test_value")
+	defer os.Unsetenv("TEST_HOST")
+
+	config := &Config{}
+
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			name:     "simple env var",
+			input:    "$ENV.TEST_HOST",
+			expected: "test_value",
+		},
+		{
+			name:     "env var in string",
+			input:    "prefix-$ENV.TEST_HOST-suffix",
+			expected: "prefix-test_value-suffix",
+		},
+		{
+			name:     "multiple env vars",
+			input:    "$ENV.TEST_HOST-$ENV.TEST_HOST",
+			expected: "test_value-test_value",
+		},
+		{
+			name:     "undefined env var",
+			input:    "$ENV.UNDEFINED_VAR",
+			expected: "$ENV.UNDEFINED_VAR", // Should remain unchanged
+		},
+		{
+			name:     "no env var",
+			input:    "plain_string",
+			expected: "plain_string",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := config.parseEnvVar(tt.input)
+			if result != tt.expected {
+				t.Errorf("parseEnvVar() = %v, want %v", result, tt.expected)
+			}
+		})
+	}
+}

kb/kb.go

@@ -39,7 +39,7 @@ func Load(appConfig config.Config) (*KnowledgeBase, error) {
 		return nil, err
 	}
 
-	err = application.Parse("kb", raw, &config)
+	err = application.Parse("kb.yao", raw, &config)
 	if err != nil {
 		return nil, err
 	}

kb/kb_test.go

@@ -0,0 +1,19 @@
+package kb
+
+import (
+	"testing"
+
+	"github.com/yaoapp/yao/config"
+	"github.com/yaoapp/yao/test"
+)
+
+func TestLoad(t *testing.T) {
+	// Setup
+	test.Prepare(&testing.T{}, config.Conf)
+	defer test.Clean()
+
+	_, err := Load(config.Conf)
+	if err != nil {
+		t.Fatalf("Failed to load knowledge base: %v", err)
+	}
+}

store/store.go

@@ -21,6 +21,8 @@ var systemStores = map[string]string{
 	"__yao.oauth.client": "yao/stores/oauth/client.badger.yao", // for OAuth client store
 	"__yao.agent.memory": "yao/stores/agent/memory.badger.yao", // for agent memory store (for agent memory)
 	"__yao.agent.cache":  "yao/stores/agent/cache.lru.yao",     // for agent cache store (for agent cache)
+	"__yao.kb.store":     "yao/stores/kb/store.badger.yao",     // for knowledge base store
+	"__yao.kb.cache":     "yao/stores/kb/cache.lru.yao",        // for knowledge base cache store
 }
 
 // replaceVars replaces template variables in the JSON string

test/utils.go

@@ -54,6 +54,8 @@ var testSystemStores = map[string]string{
 	"__yao.oauth.client": "yao/stores/oauth/client.badger.yao",
 	"__yao.oauth.cache":  "yao/stores/oauth/cache.lru.yao",
 	"__yao.agent.memory": "yao/stores/agent/memory.badger.yao",
+	"__yao.kb.store":     "yao/stores/kb/store.badger.yao",
+	"__yao.kb.cache":     "yao/stores/kb/cache.lru.yao",
 }
 
 func loadSystemStores(t *testing.T, cfg config.Config) error {
@@ -255,6 +257,7 @@ func Prepare(t *testing.T, cfg config.Config, rootEnv ...string) {
 	dbconnect(t, cfg)
 	load(t, cfg)
 	startRuntime(t, cfg)
+
 }
 
 // Clean the test environment