diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 5f906f72..d763a0fc 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -64,9 +64,46 @@ env: YAO_RUNTIME_HEAP_AVAILABLE: 550000000 YAO_RUNTIME_PRECOMPILE: true + # Neo4j + NEO4J_TEST_URL: "neo4j://localhost:7686" + NEO4J_TEST_USER: "neo4j" + NEO4J_TEST_PASS: "Yao2026Neo4j" + + # Qdrant + QDRANT_TEST_HOST: "127.0.0.1" + QDRANT_TEST_PORT: "6334" + jobs: UnitTest: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest + + services: + qdrant: + image: qdrant/qdrant:latest + ports: + - 6333:6333 # HTTP API + - 6334:6334 # gRPC + + fastembed: + image: yaoapp/fastembed:latest-amd64 + env: + FASTEMBED_PASSWORD: Yao@2026 + ports: + - 6001:8000 + + neo4j: + image: neo4j:latest + ports: + - "7687:7687" + env: + NEO4J_AUTH: neo4j/Yao2026Neo4j + + mcp-everything: + image: yaoapp/mcp-everything:latest + ports: + - "3021:3021" + - "3022:3022" + strategy: matrix: go: [1.24] @@ -202,6 +239,28 @@ jobs: with: go-version: ${{ matrix.go }} + - name: Install FFmpeg 7.x + run: | + wget https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz + tar -xf ffmpeg-master-latest-linux64-gpl.tar.xz + sudo cp ffmpeg-master-latest-linux64-gpl/bin/ffmpeg /usr/local/bin/ + sudo cp ffmpeg-master-latest-linux64-gpl/bin/ffprobe /usr/local/bin/ + sudo chmod +x /usr/local/bin/ffmpeg /usr/local/bin/ffprobe + + - name: Test FFmpeg + run: ffmpeg -version + + - name: Install pdftoppm, mutool, imagemagick + run: | + sudo apt update + sudo apt install -y poppler-utils mupdf-tools imagemagick + + - name: Test pdftoppm, mutool, imagemagick + run: | + pdftoppm -v + mutool -v + convert -version + - name: Start MongoDB uses: supercharge/mongodb-github-action@1.8.0 with: diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 880b58a1..c45ba5a7 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -68,9 +68,45 @@ env: YAO_RUNTIME_HEAP_AVAILABLE: 550000000 YAO_RUNTIME_PRECOMPILE: true + # Neo4j + NEO4J_TEST_URL: "neo4j://localhost:7686" + NEO4J_TEST_USER: "neo4j" + NEO4J_TEST_PASS: "Yao2026Neo4j" + + # Qdrant + QDRANT_TEST_HOST: "127.0.0.1" + QDRANT_TEST_PORT: "6334" + jobs: unit-test: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest + services: + qdrant: + image: qdrant/qdrant:latest + ports: + - 6333:6333 # HTTP API + - 6334:6334 # gRPC + + fastembed: + image: yaoapp/fastembed:latest-amd64 + env: + FASTEMBED_PASSWORD: Yao@2026 + ports: + - 6001:8000 + + neo4j: + image: neo4j:latest + ports: + - "7687:7687" + env: + NEO4J_AUTH: neo4j/Yao2026Neo4j + + mcp-everything: + image: yaoapp/mcp-everything:latest + ports: + - "3021:3021" + - "3022:3022" + strategy: matrix: go: [1.24] @@ -152,6 +188,28 @@ jobs: with: go-version: ${{ matrix.go }} + - name: Install FFmpeg 7.x + run: | + wget https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz + tar -xf ffmpeg-master-latest-linux64-gpl.tar.xz + sudo cp ffmpeg-master-latest-linux64-gpl/bin/ffmpeg /usr/local/bin/ + sudo cp ffmpeg-master-latest-linux64-gpl/bin/ffprobe /usr/local/bin/ + sudo chmod +x /usr/local/bin/ffmpeg /usr/local/bin/ffprobe + + - name: Test FFmpeg + run: ffmpeg -version + + - name: Install pdftoppm, mutool, imagemagick + run: | + sudo apt update + sudo apt install -y poppler-utils mupdf-tools imagemagick + + - name: Test pdftoppm, mutool, imagemagick + run: | + pdftoppm -v + mutool -v + convert -version + - name: Start Redis uses: supercharge/redis-github-action@1.4.0 with: diff --git a/engine/load.go b/engine/load.go index b0681601..7f20f3c6 100644 --- a/engine/load.go +++ b/engine/load.go @@ -19,6 +19,7 @@ import ( "github.com/yaoapp/yao/flow" "github.com/yaoapp/yao/fs" "github.com/yaoapp/yao/i18n" + "github.com/yaoapp/yao/kb" "github.com/yaoapp/yao/moapi" "github.com/yaoapp/yao/model" "github.com/yaoapp/yao/neo" @@ -280,6 +281,13 @@ func Load(cfg config.Config, options LoadOption) (warnings []Warning, err error) } } + // Load Knowledge Base + _, err = kb.Load(cfg) + if err != nil { + // printErr(cfg.Mode, "Knowledge Base", err) + warnings = append(warnings, Warning{Widget: "Knowledge Base", Error: err}) + } + // Load OpenAPI _, err = openapi.Load(cfg) if err != nil { diff --git a/kb/config.go b/kb/config.go index 8e67fe09..731b8b1c 100644 --- a/kb/config.go +++ b/kb/config.go @@ -3,6 +3,8 @@ package kb import ( "encoding/json" "os" + "regexp" + "strings" "github.com/yaoapp/gou/graphrag" "github.com/yaoapp/gou/graphrag/graph/neo4j" @@ -114,8 +116,14 @@ func (c *Config) createGraphStore() (types.GraphStore, error) { // toVectorStoreConfig converts the vector config to VectorStoreConfig func (c *Config) toVectorStoreConfig() (types.VectorStoreConfig, error) { - // Convert map[string]interface{} to types.VectorStoreConfig via JSON - jsonData, err := json.Marshal(c.Vector.Config) + // Parse environment variables in config + resolvedConfig, err := c.resolveEnvVars(c.Vector.Config) + if err != nil { + return types.VectorStoreConfig{}, err + } + + // Convert resolved config to types.VectorStoreConfig via JSON + jsonData, err := json.Marshal(resolvedConfig) if err != nil { return types.VectorStoreConfig{}, err } @@ -130,8 +138,14 @@ func (c *Config) toVectorStoreConfig() (types.VectorStoreConfig, error) { // toGraphStoreConfig converts the graph config to GraphStoreConfig func (c *Config) toGraphStoreConfig() (types.GraphStoreConfig, error) { - // Convert map[string]interface{} to types.GraphStoreConfig via JSON - jsonData, err := json.Marshal(c.Graph.Config) + // Parse environment variables in config + resolvedConfig, err := c.resolveEnvVars(c.Graph.Config) + if err != nil { + return types.GraphStoreConfig{}, err + } + + // Convert resolved config to types.GraphStoreConfig via JSON + jsonData, err := json.Marshal(resolvedConfig) if err != nil { return types.GraphStoreConfig{}, err } @@ -144,6 +158,48 @@ func (c *Config) toGraphStoreConfig() (types.GraphStoreConfig, error) { return config, nil } +// resolveEnvVars resolves environment variables in configuration values +func (c *Config) resolveEnvVars(config map[string]interface{}) (map[string]interface{}, error) { + resolved := make(map[string]interface{}) + + for key, value := range config { + switch v := value.(type) { + case string: + resolved[key] = c.parseEnvVar(v) + case map[string]interface{}: + // Recursively resolve nested maps + nestedResolved, err := c.resolveEnvVars(v) + if err != nil { + return nil, err + } + resolved[key] = nestedResolved + default: + resolved[key] = value + } + } + + return resolved, nil +} + +// parseEnvVar parses environment variable pattern $ENV.VAR_NAME +func (c *Config) parseEnvVar(value string) string { + // Simple pattern to match $ENV.VAR_NAME + envPattern := regexp.MustCompile(`\$ENV\.([A-Za-z_][A-Za-z0-9_]*)`) + + return envPattern.ReplaceAllStringFunc(value, func(match string) string { + // Extract variable name (remove $ENV. prefix) + varName := strings.TrimPrefix(match, "$ENV.") + + // Get environment variable value + if envValue := os.Getenv(varName); envValue != "" { + return envValue + } + + // Return original if environment variable is not set + return match + }) +} + // UnmarshalJSON implements json.Unmarshaler interface func (c *Config) UnmarshalJSON(data []byte) error { // Use alias type to avoid infinite recursion diff --git a/kb/config_test.go b/kb/config_test.go index a754bb28..bb818fa4 100644 --- a/kb/config_test.go +++ b/kb/config_test.go @@ -373,3 +373,136 @@ func TestRoundTrip(t *testing.T) { t.Errorf("Features mismatch: %+v != %+v", originalConfig.Features, roundTripConfig.Features) } } + +func TestConfig_ResolveEnvVars(t *testing.T) { + // Set test environment variables + os.Setenv("TEST_HOST", "localhost") + os.Setenv("TEST_PORT", "6333") + defer func() { + os.Unsetenv("TEST_HOST") + os.Unsetenv("TEST_PORT") + }() + + config := &Config{} + + tests := []struct { + name string + input map[string]interface{} + expected map[string]interface{} + }{ + { + name: "simple environment variable", + input: map[string]interface{}{ + "host": "$ENV.TEST_HOST", + "port": "$ENV.TEST_PORT", + }, + expected: map[string]interface{}{ + "host": "localhost", + "port": "6333", + }, + }, + { + name: "mixed values", + input: map[string]interface{}{ + "host": "$ENV.TEST_HOST", + "port": 6333, + "prefix": "test-$ENV.TEST_HOST-suffix", + }, + expected: map[string]interface{}{ + "host": "localhost", + "port": 6333, + "prefix": "test-localhost-suffix", + }, + }, + { + name: "nested configuration", + input: map[string]interface{}{ + "database": map[string]interface{}{ + "host": "$ENV.TEST_HOST", + "port": "$ENV.TEST_PORT", + }, + "name": "test", + }, + expected: map[string]interface{}{ + "database": map[string]interface{}{ + "host": "localhost", + "port": "6333", + }, + "name": "test", + }, + }, + { + name: "undefined environment variable", + input: map[string]interface{}{ + "host": "$ENV.UNDEFINED_VAR", + "port": "$ENV.TEST_PORT", + }, + expected: map[string]interface{}{ + "host": "$ENV.UNDEFINED_VAR", // Should remain unchanged + "port": "6333", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := config.resolveEnvVars(tt.input) + if err != nil { + t.Errorf("resolveEnvVars() error = %v", err) + return + } + if !reflect.DeepEqual(result, tt.expected) { + t.Errorf("resolveEnvVars() = %+v, want %+v", result, tt.expected) + } + }) + } +} + +func TestConfig_ParseEnvVar(t *testing.T) { + // Set test environment variables + os.Setenv("TEST_HOST", "test_value") + defer os.Unsetenv("TEST_HOST") + + config := &Config{} + + tests := []struct { + name string + input string + expected string + }{ + { + name: "simple env var", + input: "$ENV.TEST_HOST", + expected: "test_value", + }, + { + name: "env var in string", + input: "prefix-$ENV.TEST_HOST-suffix", + expected: "prefix-test_value-suffix", + }, + { + name: "multiple env vars", + input: "$ENV.TEST_HOST-$ENV.TEST_HOST", + expected: "test_value-test_value", + }, + { + name: "undefined env var", + input: "$ENV.UNDEFINED_VAR", + expected: "$ENV.UNDEFINED_VAR", // Should remain unchanged + }, + { + name: "no env var", + input: "plain_string", + expected: "plain_string", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := config.parseEnvVar(tt.input) + if result != tt.expected { + t.Errorf("parseEnvVar() = %v, want %v", result, tt.expected) + } + }) + } +} diff --git a/kb/kb.go b/kb/kb.go index 8ba9a8ca..e7a66199 100644 --- a/kb/kb.go +++ b/kb/kb.go @@ -39,7 +39,7 @@ func Load(appConfig config.Config) (*KnowledgeBase, error) { return nil, err } - err = application.Parse("kb", raw, &config) + err = application.Parse("kb.yao", raw, &config) if err != nil { return nil, err } diff --git a/kb/kb_test.go b/kb/kb_test.go new file mode 100644 index 00000000..6e534087 --- /dev/null +++ b/kb/kb_test.go @@ -0,0 +1,19 @@ +package kb + +import ( + "testing" + + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +func TestLoad(t *testing.T) { + // Setup + test.Prepare(&testing.T{}, config.Conf) + defer test.Clean() + + _, err := Load(config.Conf) + if err != nil { + t.Fatalf("Failed to load knowledge base: %v", err) + } +} diff --git a/store/store.go b/store/store.go index 891ac374..fa90fc41 100644 --- a/store/store.go +++ b/store/store.go @@ -21,6 +21,8 @@ var systemStores = map[string]string{ "__yao.oauth.client": "yao/stores/oauth/client.badger.yao", // for OAuth client store "__yao.agent.memory": "yao/stores/agent/memory.badger.yao", // for agent memory store (for agent memory) "__yao.agent.cache": "yao/stores/agent/cache.lru.yao", // for agent cache store (for agent cache) + "__yao.kb.store": "yao/stores/kb/store.badger.yao", // for knowledge base store + "__yao.kb.cache": "yao/stores/kb/cache.lru.yao", // for knowledge base cache store } // replaceVars replaces template variables in the JSON string diff --git a/test/utils.go b/test/utils.go index da894f14..44f71397 100644 --- a/test/utils.go +++ b/test/utils.go @@ -54,6 +54,8 @@ var testSystemStores = map[string]string{ "__yao.oauth.client": "yao/stores/oauth/client.badger.yao", "__yao.oauth.cache": "yao/stores/oauth/cache.lru.yao", "__yao.agent.memory": "yao/stores/agent/memory.badger.yao", + "__yao.kb.store": "yao/stores/kb/store.badger.yao", + "__yao.kb.cache": "yao/stores/kb/cache.lru.yao", } func loadSystemStores(t *testing.T, cfg config.Config) error { @@ -255,6 +257,7 @@ func Prepare(t *testing.T, cfg config.Config, rootEnv ...string) { dbconnect(t, cfg) load(t, cfg) startRuntime(t, cfg) + } // Clean the test environment