Merge pull request #1385 from trheyi/main

Implement System Agents Configuration and Loading Mechanism
This commit is contained in:
Max 2025-12-16 18:15:58 +08:00 committed by GitHub
commit b9fe11eab3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
34 changed files with 3312 additions and 209 deletions

View file

@ -132,7 +132,10 @@ func (ast *Assistant) Stream(ctx *context.Context, inputMessages []context.Messa
fullMessages := historyResult.FullMessages
// Buffer user input messages (use cleaned input without overlap)
// Skip if History is disabled in options (for internal calls like needsearch)
if opts == nil || opts.Skip == nil || !opts.Skip.History {
ast.BufferUserInput(ctx, historyResult.InputMessages)
}
ctx.Logger.PhaseComplete("History")
// ================================================
@ -202,7 +205,7 @@ func (ast *Assistant) Stream(ctx *context.Context, inputMessages []context.Messa
// ================================================
// Execute Auto Search (if enabled)
// ================================================
if ast.shouldAutoSearch(ctx, createResponse) {
if ast.shouldAutoSearch(ctx, completionMessages, createResponse, opts) {
refCtx := ast.executeAutoSearch(ctx, completionMessages, createResponse, opts)
if refCtx != nil && len(refCtx.References) > 0 {
completionMessages = ast.injectSearchContext(completionMessages, refCtx)

View file

@ -124,6 +124,35 @@ func (c *Cache) Clear() {
c.items = make(map[string]*list.Element)
}
// ClearExcept removes items from the cache except those matching the keep function
// keep function returns true for items that should be preserved
func (c *Cache) ClearExcept(keep func(id string) bool) {
c.mu.Lock()
defer c.mu.Unlock()
// Collect items to remove
var toRemove []*list.Element
for element := c.list.Front(); element != nil; element = element.Next() {
item := element.Value.(*cacheItem)
if !keep(item.key) {
toRemove = append(toRemove, element)
}
}
// Remove collected items
for _, element := range toRemove {
item := element.Value.(*cacheItem)
// Unregister scripts before removing
if item.value != nil && len(item.value.Scripts) > 0 {
item.value.UnregisterScripts()
}
c.list.Remove(element)
delete(c.items, item.key)
}
}
// removeOldest removes the least recently used item from the cache
func (c *Cache) removeOldest() {
if element := c.list.Back(); element != nil {

View file

@ -33,8 +33,10 @@ var globalSearchConfig *searchTypes.Config = nil // global search config from ag
// LoadBuiltIn load the built-in assistants
func LoadBuiltIn() error {
// Clear the cache
loaded.Clear()
// Clear non-system agents from cache (preserve system agents loaded by LoadSystemAgents)
loaded.ClearExcept(func(id string) bool {
return strings.HasPrefix(id, "__yao.") // Keep system agents
})
root := `/assistants`
app, err := fs.Get("app")
@ -45,7 +47,7 @@ func LoadBuiltIn() error {
// Get all existing built-in assistants
deletedBuiltIn := map[string]bool{}
// Remove the built-in assistants
// Remove the built-in assistants (exclude system agents with __yao. prefix)
if storage != nil {
builtIn := true
@ -54,8 +56,12 @@ func LoadBuiltIn() error {
return err
}
// Get all existing built-in assistants
// Get all existing built-in assistants (exclude system agents)
for _, assistant := range res.Data {
// Skip system agents (they are managed by LoadSystemAgents)
if strings.HasPrefix(assistant.ID, "__yao.") {
continue
}
deletedBuiltIn[assistant.ID] = true
}
}
@ -582,16 +588,20 @@ func loadMap(data map[string]interface{}) (*Assistant, error) {
i18n.Locales[id] = flattened
} else {
// No locales defined, create default with name and description
// No locales defined, create default with name and description for all common locales
if assistant.Name != "" || assistant.Description != "" {
defaultLocales := make(map[string]i18n.I18n)
defaultLocales["en"] = i18n.I18n{
Locale: "en",
// Create entries for all common locales so {{name}} can be resolved
commonLocales := []string{"en", "en-us", "zh", "zh-cn", "zh-tw"}
for _, locale := range commonLocales {
defaultLocales[locale] = i18n.I18n{
Locale: locale,
Messages: map[string]any{
"name": assistant.Name,
"description": assistant.Description,
},
}
}
i18n.Locales[id] = defaultLocales
}
}

View file

@ -0,0 +1,366 @@
package assistant
import (
"fmt"
"path/filepath"
"strings"
"github.com/yaoapp/gou/application"
"github.com/yaoapp/gou/connector"
gouOpenAI "github.com/yaoapp/gou/connector/openai"
"github.com/yaoapp/kun/log"
"github.com/yaoapp/yao/agent/i18n"
store "github.com/yaoapp/yao/agent/store/types"
"github.com/yaoapp/yao/data"
"gopkg.in/yaml.v3"
)
// systemAgents defines the system agents loaded from bindata
// These are internal agents used by the system (e.g., keyword extraction, querydsl generation)
// The directory name is without __yao. prefix, prefix is added during loading
// Format: directory name -> bindata path prefix
var systemAgents = []string{
"keyword",
"querydsl",
"title",
"prompt",
"needsearch",
"entity",
}
// SystemConfig holds the system agents connector configuration
// This is set from agent.yml system block
type SystemConfig struct {
Default string // Default connector for all system agents
Keyword string // Connector for __yao.keyword agent
QueryDSL string // Connector for __yao.querydsl agent
Title string // Connector for __yao.title agent
Prompt string // Connector for __yao.prompt agent
NeedSearch string // Connector for __yao.needsearch agent
Entity string // Connector for __yao.entity agent
}
// systemConfig holds the system agents configuration (global variable like others in load.go)
var systemConfig *SystemConfig = nil
// SetSystemConfig sets the system agents configuration
func SetSystemConfig(config *SystemConfig) {
systemConfig = config
}
// GetSystemConfig returns the system agents configuration
func GetSystemConfig() *SystemConfig {
return systemConfig
}
// LoadSystemAgents loads the system agents from bindata
// These are internal agents like __yao.keyword and __yao.querydsl
// They are loaded before application assistants
// Behavior is same as LoadBuiltIn, just reads from bindata instead of filesystem
func LoadSystemAgents() error {
// Get all existing system agents (for cleanup)
deletedSystem := map[string]bool{}
if storage != nil {
// System agents have "system" tag
tags := []string{"system"}
builtIn := true
res, err := storage.GetAssistants(store.AssistantFilter{
Tags: tags,
BuiltIn: &builtIn,
Select: []string{"assistant_id", "id"},
})
if err != nil {
log.Warn("Failed to get existing system agents: %v", err)
} else {
for _, assistant := range res.Data {
deletedSystem[assistant.ID] = true
}
}
}
sort := 1
for _, name := range systemAgents {
// Build agent ID with __yao. prefix
id := "__yao." + name
pathPrefix := "yao/assistants/" + name
assistant, err := loadSystemAgent(id, pathPrefix)
if err != nil {
log.Warn("Failed to load system agent %s: %v", id, err)
continue
}
// Set sort order
if assistant.Sort == 0 {
assistant.Sort = sort
}
// Save to storage
if err := assistant.Save(); err != nil {
log.Warn("Failed to save system agent %s: %v", id, err)
continue
}
// Initialize the assistant
if err := assistant.initialize(); err != nil {
log.Warn("Failed to initialize system agent %s: %v", id, err)
continue
}
sort++
loaded.Put(assistant)
log.Trace("Loaded system agent: %s", id)
// Remove from deleted list
delete(deletedSystem, id)
}
// Remove deleted system agents
if len(deletedSystem) > 0 {
assistantIDs := []string{}
for assistantID := range deletedSystem {
assistantIDs = append(assistantIDs, assistantID)
}
if _, err := storage.DeleteAssistants(store.AssistantFilter{AssistantIDs: assistantIDs}); err != nil {
log.Warn("Failed to delete obsolete system agents: %v", err)
}
}
return nil
}
// loadSystemAgent loads a single system agent from bindata
// This follows the same pattern as LoadPath but reads from bindata
func loadSystemAgent(id, pathPrefix string) (*Assistant, error) {
// Read package.yao from bindata
pkgPath := pathPrefix + "/package.yao"
pkgContent, err := data.Read(pkgPath)
if err != nil {
return nil, fmt.Errorf("failed to read %s: %w", pkgPath, err)
}
// Parse package.yao
var pkgData map[string]interface{}
if err := application.Parse(pkgPath, pkgContent, &pkgData); err != nil {
return nil, fmt.Errorf("failed to parse %s: %w", pkgPath, err)
}
// Set assistant_id (no path - system agents are loaded from storage, not filesystem)
pkgData["assistant_id"] = id
// Set type if not specified
if _, has := pkgData["type"]; !has {
pkgData["type"] = "assistant"
}
// Resolve connector for this system agent
connectorID := resolveSystemConnector(id)
if connectorID != "" {
pkgData["connector"] = connectorID
}
// Read prompts.yml from bindata (default prompts)
promptsPath := pathPrefix + "/prompts.yml"
promptsContent, err := data.Read(promptsPath)
if err == nil {
var prompts []store.Prompt
if err := yaml.Unmarshal(promptsContent, &prompts); err == nil && len(prompts) > 0 {
pkgData["prompts"] = prompts
}
}
// Read prompt_presets from prompts directory
presets := loadSystemPromptPresets(pathPrefix)
if len(presets) > 0 {
pkgData["prompt_presets"] = presets
}
// Load scripts from src directory (hook script source and other scripts sources)
// These will be compiled by loadMap -> LoadScriptsFromData
hookScriptSource, scriptsSource := loadSystemScripts(pathPrefix)
if hookScriptSource != "" {
pkgData["script"] = hookScriptSource
}
if len(scriptsSource) > 0 {
pkgData["scripts"] = scriptsSource
}
// Read locales
locales, err := loadSystemLocales(pathPrefix)
if err == nil && len(locales) > 0 {
pkgData["locales"] = locales
}
// Mark as system agent
pkgData["readonly"] = true
pkgData["built_in"] = true
pkgData["tags"] = []string{"system"}
// Load from map (same as LoadPath, includes initialize())
return loadMap(pkgData)
}
// resolveSystemConnector resolves the connector for a system agent
// Priority: specific agent config > system.default > defaultConnector > fallback to first capable connector
func resolveSystemConnector(agentID string) string {
// Try specific agent config first
if systemConfig != nil {
switch agentID {
case "__yao.keyword":
if systemConfig.Keyword != "" {
return systemConfig.Keyword
}
case "__yao.querydsl":
if systemConfig.QueryDSL != "" {
return systemConfig.QueryDSL
}
case "__yao.title":
if systemConfig.Title != "" {
return systemConfig.Title
}
case "__yao.prompt":
if systemConfig.Prompt != "" {
return systemConfig.Prompt
}
case "__yao.needsearch":
if systemConfig.NeedSearch != "" {
return systemConfig.NeedSearch
}
case "__yao.entity":
if systemConfig.Entity != "" {
return systemConfig.Entity
}
}
// Try system default
if systemConfig.Default != "" {
return systemConfig.Default
}
}
// Try global default connector
if defaultConnector != "" {
return defaultConnector
}
// Fallback: find first connector that supports tool calling
return findCapableConnector()
}
// findCapableConnector finds the first connector that supports tool calling
func findCapableConnector() string {
// Get all registered connectors
for id, conn := range connector.Connectors {
if !conn.Is(connector.OPENAI) {
continue
}
// Check from modelCapabilities (user-defined in models.yml)
if caps, exists := modelCapabilities[id]; exists {
if caps.ToolCalls {
return id
}
}
// Check capabilities from connector's Options
if connOpenAI, ok := conn.(*gouOpenAI.Connector); ok {
if connOpenAI.Options.Capabilities != nil && connOpenAI.Options.Capabilities.ToolCalls {
return id
}
}
}
// No capable connector found, return empty
return ""
}
// loadSystemPromptPresets loads prompt presets from bindata prompts directory
func loadSystemPromptPresets(pathPrefix string) map[string][]store.Prompt {
presets := make(map[string][]store.Prompt)
promptsDir := pathPrefix + "/prompts"
// Try common preset files
presetFiles := []string{"chat.yml", "task.yml", "code.yml", "analysis.yml"}
for _, filename := range presetFiles {
presetPath := promptsDir + "/" + filename
content, err := data.Read(presetPath)
if err != nil {
continue
}
var prompts []store.Prompt
if err := yaml.Unmarshal(content, &prompts); err == nil && len(prompts) > 0 {
presetName := strings.TrimSuffix(filename, ".yml")
presets[presetName] = prompts
}
}
return presets
}
// loadSystemScripts loads scripts source from bindata src directory
// Returns hook script source and other scripts sources (as strings)
// These will be compiled by loadMap -> LoadScriptsFromData
func loadSystemScripts(pathPrefix string) (string, map[string]string) {
srcDir := pathPrefix + "/src"
// Try to load hook script (index.ts)
var hookScriptSource string
indexPath := srcDir + "/index.ts"
indexContent, err := data.Read(indexPath)
if err == nil && len(indexContent) > 0 {
hookScriptSource = string(indexContent)
}
// Try to load other scripts
scripts := make(map[string]string)
scriptFiles := []string{"utils.ts", "helpers.ts", "tools.ts"}
for _, filename := range scriptFiles {
scriptPath := srcDir + "/" + filename
content, err := data.Read(scriptPath)
if err != nil {
continue
}
scriptName := strings.TrimSuffix(filename, ".ts")
scripts[scriptName] = string(content)
}
if len(scripts) == 0 {
scripts = nil
}
return hookScriptSource, scripts
}
// loadSystemLocales loads locales from bindata
func loadSystemLocales(pathPrefix string) (i18n.Map, error) {
locales := make(i18n.Map)
// Try to load common locale files
localeFiles := []string{"en-us.yml", "zh-cn.yml", "en.yml", "zh.yml"}
localesDir := pathPrefix + "/locales"
for _, filename := range localeFiles {
localePath := filepath.Join(localesDir, filename)
content, err := data.Read(localePath)
if err != nil {
continue
}
// Parse locale file
locale := strings.TrimSuffix(filename, ".yml")
var messages map[string]any
if err := yaml.Unmarshal(content, &messages); err != nil {
continue
}
locales[locale] = i18n.I18n{
Locale: locale,
Messages: messages,
}
}
return locales, nil
}

View file

@ -1,10 +1,12 @@
package assistant
package assistant_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/yaoapp/yao/agent"
"github.com/yaoapp/yao/agent/assistant"
store "github.com/yaoapp/yao/agent/store/types"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/test"
@ -14,13 +16,19 @@ func prepare(t *testing.T) {
test.Prepare(t, config.Conf)
}
func prepareAgent(t *testing.T) {
test.Prepare(t, config.Conf)
err := agent.Load(config.Conf)
require.NoError(t, err, "agent.Load should succeed")
}
// TestLoadPath tests loading assistant from path
func TestLoadPath(t *testing.T) {
prepare(t)
defer test.Clean()
t.Run("LoadFullFieldsAssistant", func(t *testing.T) {
assistant, err := LoadPath("/assistants/tests/fullfields")
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
require.NoError(t, err)
require.NotNil(t, assistant)
@ -67,7 +75,7 @@ func TestLoadPath(t *testing.T) {
})
t.Run("LoadConnectorOptions", func(t *testing.T) {
assistant, err := LoadPath("/assistants/tests/fullfields")
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
require.NoError(t, err)
require.NotNil(t, assistant)
@ -84,7 +92,7 @@ func TestLoadPath(t *testing.T) {
})
t.Run("LoadPromptPresets", func(t *testing.T) {
assistant, err := LoadPath("/assistants/tests/fullfields")
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
require.NoError(t, err)
require.NotNil(t, assistant)
@ -116,7 +124,7 @@ func TestLoadPath(t *testing.T) {
})
t.Run("LoadKnowledgeBase", func(t *testing.T) {
assistant, err := LoadPath("/assistants/tests/fullfields")
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
require.NoError(t, err)
require.NotNil(t, assistant)
@ -129,7 +137,7 @@ func TestLoadPath(t *testing.T) {
})
t.Run("LoadMCPServers", func(t *testing.T) {
assistant, err := LoadPath("/assistants/tests/fullfields")
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
require.NoError(t, err)
require.NotNil(t, assistant)
@ -143,7 +151,7 @@ func TestLoadPath(t *testing.T) {
})
t.Run("LoadWorkflow", func(t *testing.T) {
assistant, err := LoadPath("/assistants/tests/fullfields")
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
require.NoError(t, err)
require.NotNil(t, assistant)
@ -156,7 +164,7 @@ func TestLoadPath(t *testing.T) {
})
t.Run("LoadPlaceholder", func(t *testing.T) {
assistant, err := LoadPath("/assistants/tests/fullfields")
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
require.NoError(t, err)
require.NotNil(t, assistant)
@ -169,7 +177,7 @@ func TestLoadPath(t *testing.T) {
})
t.Run("LoadLocales", func(t *testing.T) {
assistant, err := LoadPath("/assistants/tests/fullfields")
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
require.NoError(t, err)
require.NotNil(t, assistant)
@ -186,7 +194,7 @@ func TestLoadPath(t *testing.T) {
})
t.Run("LoadNonExistentAssistant", func(t *testing.T) {
_, err := LoadPath("/assistants/non-existent")
_, err := assistant.LoadPath("/assistants/non-existent")
assert.Error(t, err)
})
}
@ -196,7 +204,7 @@ func TestLoadPathMCPTest(t *testing.T) {
prepare(t)
defer test.Clean()
assistant, err := LoadPath("/assistants/tests/mcptest")
assistant, err := assistant.LoadPath("/assistants/tests/mcptest")
require.NoError(t, err)
require.NotNil(t, assistant)
@ -220,7 +228,7 @@ func TestLoadPathBuildRequest(t *testing.T) {
prepare(t)
defer test.Clean()
assistant, err := LoadPath("/assistants/tests/buildrequest")
assistant, err := assistant.LoadPath("/assistants/tests/buildrequest")
require.NoError(t, err)
require.NotNil(t, assistant)
@ -238,57 +246,57 @@ func TestLoadPathBuildRequest(t *testing.T) {
// TestCache tests the assistant cache functionality
func TestCache(t *testing.T) {
// Clear any existing cache
ClearCache()
assistant.ClearCache()
// Set small cache for testing
SetCache(3)
assert.NotNil(t, loaded)
assistant.SetCache(3)
assert.NotNil(t, assistant.GetCache())
// Create test assistants
ast1 := &Assistant{AssistantModel: store.AssistantModel{ID: "id1", Name: "Assistant 1"}}
ast2 := &Assistant{AssistantModel: store.AssistantModel{ID: "id2", Name: "Assistant 2"}}
ast3 := &Assistant{AssistantModel: store.AssistantModel{ID: "id3", Name: "Assistant 3"}}
ast4 := &Assistant{AssistantModel: store.AssistantModel{ID: "id4", Name: "Assistant 4"}}
ast1 := &assistant.Assistant{AssistantModel: store.AssistantModel{ID: "id1", Name: "Assistant 1"}}
ast2 := &assistant.Assistant{AssistantModel: store.AssistantModel{ID: "id2", Name: "Assistant 2"}}
ast3 := &assistant.Assistant{AssistantModel: store.AssistantModel{ID: "id3", Name: "Assistant 3"}}
ast4 := &assistant.Assistant{AssistantModel: store.AssistantModel{ID: "id4", Name: "Assistant 4"}}
t.Run("PutAndGet", func(t *testing.T) {
loaded.Put(ast1)
assert.Equal(t, 1, loaded.Len())
assistant.GetCache().Put(ast1)
assert.Equal(t, 1, assistant.GetCache().Len())
cached, exists := loaded.Get("id1")
cached, exists := assistant.GetCache().Get("id1")
assert.True(t, exists)
assert.Equal(t, ast1, cached)
})
t.Run("CacheEviction", func(t *testing.T) {
loaded.Put(ast2)
loaded.Put(ast3)
assert.Equal(t, 3, loaded.Len())
assistant.GetCache().Put(ast2)
assistant.GetCache().Put(ast3)
assert.Equal(t, 3, assistant.GetCache().Len())
// Access ast1 to make it recently used
loaded.Get("id1")
assistant.GetCache().Get("id1")
// Add ast4, should evict ast2 (least recently used)
loaded.Put(ast4)
assert.Equal(t, 3, loaded.Len())
assistant.GetCache().Put(ast4)
assert.Equal(t, 3, assistant.GetCache().Len())
_, exists := loaded.Get("id2")
_, exists := assistant.GetCache().Get("id2")
assert.False(t, exists, "ast2 should be evicted")
_, exists = loaded.Get("id1")
_, exists = assistant.GetCache().Get("id1")
assert.True(t, exists, "ast1 should still exist")
_, exists = loaded.Get("id4")
_, exists = assistant.GetCache().Get("id4")
assert.True(t, exists, "ast4 should exist")
})
t.Run("ClearCache", func(t *testing.T) {
ClearCache()
assert.Nil(t, loaded)
assistant.ClearCache()
assert.Nil(t, assistant.GetCache())
})
t.Run("SetCacheAfterClear", func(t *testing.T) {
SetCache(100)
assert.NotNil(t, loaded)
assistant.SetCache(100)
assert.NotNil(t, assistant.GetCache())
})
}
@ -298,7 +306,7 @@ func TestClone(t *testing.T) {
defer test.Clean()
t.Run("CloneFullFieldsAssistant", func(t *testing.T) {
original, err := LoadPath("/assistants/tests/fullfields")
original, err := assistant.LoadPath("/assistants/tests/fullfields")
require.NoError(t, err)
clone := original.Clone()
@ -328,7 +336,7 @@ func TestClone(t *testing.T) {
})
t.Run("CloneNil", func(t *testing.T) {
var nilAssistant *Assistant
var nilAssistant *assistant.Assistant
assert.Nil(t, nilAssistant.Clone())
})
}
@ -339,7 +347,7 @@ func TestUpdate(t *testing.T) {
defer test.Clean()
t.Run("UpdateBasicFields", func(t *testing.T) {
assistant, err := LoadPath("/assistants/tests/fullfields")
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
require.NoError(t, err)
updates := map[string]interface{}{
@ -357,7 +365,7 @@ func TestUpdate(t *testing.T) {
})
t.Run("UpdateConnectorOptions", func(t *testing.T) {
assistant, err := LoadPath("/assistants/tests/fullfields")
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
require.NoError(t, err)
updates := map[string]interface{}{
@ -377,7 +385,7 @@ func TestUpdate(t *testing.T) {
})
t.Run("UpdatePromptPresets", func(t *testing.T) {
assistant, err := LoadPath("/assistants/tests/fullfields")
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
require.NoError(t, err)
updates := map[string]interface{}{
@ -398,7 +406,7 @@ func TestUpdate(t *testing.T) {
})
t.Run("UpdateSource", func(t *testing.T) {
assistant, err := LoadPath("/assistants/tests/fullfields")
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
require.NoError(t, err)
updates := map[string]interface{}{
@ -412,7 +420,7 @@ func TestUpdate(t *testing.T) {
})
t.Run("UpdateNilAssistant", func(t *testing.T) {
var nilAssistant *Assistant
var nilAssistant *assistant.Assistant
err := nilAssistant.Update(map[string]interface{}{"name": "test"})
assert.Error(t, err)
})
@ -423,7 +431,7 @@ func TestMap(t *testing.T) {
prepare(t)
defer test.Clean()
assistant, err := LoadPath("/assistants/tests/fullfields")
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
require.NoError(t, err)
m := assistant.Map()
@ -451,16 +459,131 @@ func TestMap(t *testing.T) {
assert.Equal(t, assistant.Source, m["source"])
}
// TestLoadSystemAgents tests loading system agents from bindata
func TestLoadSystemAgents(t *testing.T) {
prepareAgent(t)
defer test.Clean()
// Clear cache first
assistant.ClearCache()
assistant.SetCache(200)
t.Run("LoadSystemAgents", func(t *testing.T) {
err := assistant.LoadSystemAgents()
require.NoError(t, err)
// Check __yao.keyword
keywordAst, keywordExists := assistant.GetCache().Get("__yao.keyword")
require.True(t, keywordExists, "__yao.keyword should be loaded")
assert.Equal(t, "__yao.keyword", keywordAst.ID)
assert.Equal(t, "Keyword Extractor", keywordAst.Name)
assert.True(t, keywordAst.Readonly)
assert.True(t, keywordAst.BuiltIn)
assert.Contains(t, keywordAst.Tags, "system")
assert.NotNil(t, keywordAst.Prompts)
assert.Greater(t, len(keywordAst.Prompts), 0)
// Check __yao.querydsl
querydslAst, querydslExists := assistant.GetCache().Get("__yao.querydsl")
require.True(t, querydslExists, "__yao.querydsl should be loaded")
assert.Equal(t, "__yao.querydsl", querydslAst.ID)
assert.Equal(t, "Query Builder", querydslAst.Name)
assert.True(t, querydslAst.Readonly)
assert.True(t, querydslAst.BuiltIn)
assert.Contains(t, querydslAst.Tags, "system")
assert.NotNil(t, querydslAst.Prompts)
assert.Greater(t, len(querydslAst.Prompts), 0)
// Check __yao.title
titleAst, titleExists := assistant.GetCache().Get("__yao.title")
require.True(t, titleExists, "__yao.title should be loaded")
assert.Equal(t, "__yao.title", titleAst.ID)
assert.Equal(t, "Title Generator", titleAst.Name)
assert.True(t, titleAst.Readonly)
assert.True(t, titleAst.BuiltIn)
// Check __yao.prompt
promptAst, promptExists := assistant.GetCache().Get("__yao.prompt")
require.True(t, promptExists, "__yao.prompt should be loaded")
assert.Equal(t, "__yao.prompt", promptAst.ID)
assert.Equal(t, "Prompt Optimizer", promptAst.Name)
assert.True(t, promptAst.Readonly)
assert.True(t, promptAst.BuiltIn)
// Check __yao.needsearch
needsearchAst, needsearchExists := assistant.GetCache().Get("__yao.needsearch")
require.True(t, needsearchExists, "__yao.needsearch should be loaded")
assert.Equal(t, "__yao.needsearch", needsearchAst.ID)
assert.Equal(t, "Reference Checker", needsearchAst.Name)
assert.True(t, needsearchAst.Readonly)
assert.True(t, needsearchAst.BuiltIn)
})
t.Run("SystemAgentsSavedToStorage", func(t *testing.T) {
// System agents should be saved to storage
require.NotNil(t, assistant.GetStore(), "storage should be initialized")
// Check __yao.keyword in storage
builtIn := true
tags := []string{"system"}
res, err := assistant.GetStore().GetAssistants(store.AssistantFilter{
BuiltIn: &builtIn,
Tags: tags,
Select: []string{"assistant_id", "name"},
})
require.NoError(t, err)
require.Greater(t, len(res.Data), 0, "System agents should be in storage")
// Verify at least one system agent exists
found := false
for _, ast := range res.Data {
if ast.ID == "__yao.keyword" || ast.ID == "__yao.querydsl" {
found = true
break
}
}
assert.True(t, found, "System agents should be found in storage")
})
t.Run("SystemAgentsGetFromStorage", func(t *testing.T) {
// Clear cache to force loading from storage
assistant.GetCache().Clear()
// Test Get for each system agent
systemAgents := []string{
"__yao.keyword",
"__yao.querydsl",
"__yao.title",
"__yao.prompt",
"__yao.needsearch",
"__yao.entity",
}
for _, agentID := range systemAgents {
ast, err := assistant.Get(agentID)
require.NoError(t, err, "Get(%s) should succeed", agentID)
require.NotNil(t, ast, "Get(%s) should return assistant", agentID)
assert.Equal(t, agentID, ast.ID)
assert.True(t, ast.BuiltIn, "%s should be built-in", agentID)
assert.True(t, ast.Readonly, "%s should be readonly", agentID)
assert.Contains(t, ast.Tags, "system", "%s should have system tag", agentID)
assert.Equal(t, "worker", ast.Type, "%s should be worker type", agentID)
assert.NotNil(t, ast.Prompts, "%s should have prompts", agentID)
assert.Greater(t, len(ast.Prompts), 0, "%s should have at least one prompt", agentID)
}
})
}
// TestValidate tests the assistant Validate method
func TestValidate(t *testing.T) {
tests := []struct {
name string
ast *Assistant
ast *assistant.Assistant
wantErr bool
}{
{
name: "ValidAssistant",
ast: &Assistant{
ast: &assistant.Assistant{
AssistantModel: store.AssistantModel{
ID: "test-id",
Name: "Test Assistant",
@ -471,7 +594,7 @@ func TestValidate(t *testing.T) {
},
{
name: "MissingID",
ast: &Assistant{
ast: &assistant.Assistant{
AssistantModel: store.AssistantModel{
Name: "Test Assistant",
Connector: "gpt-4o",
@ -481,7 +604,7 @@ func TestValidate(t *testing.T) {
},
{
name: "MissingName",
ast: &Assistant{
ast: &assistant.Assistant{
AssistantModel: store.AssistantModel{
ID: "test-id",
Connector: "gpt-4o",

View file

@ -1,6 +1,7 @@
package assistant
import (
"encoding/json"
"fmt"
"strings"
"time"
@ -17,9 +18,17 @@ import (
// shouldAutoSearch determines if auto search should be executed
// Returns false if:
// - opts.Skip.Search is true
// - uses.search is "disabled"
// - assistant has no search configuration
func (ast *Assistant) shouldAutoSearch(ctx *context.Context, createResponse *context.HookCreateResponse) bool {
// - needsearch intent detection returns false
func (ast *Assistant) shouldAutoSearch(ctx *context.Context, messages []context.Message, createResponse *context.HookCreateResponse, opts *context.Options) bool {
// Check if search is skipped via options
if opts != nil && opts.Skip != nil && opts.Skip.Search {
ctx.Logger.Debug("Auto search skipped by opts.Skip.Search")
return false
}
// Get merged uses configuration
uses := ast.getMergedSearchUses(createResponse)
@ -34,10 +43,194 @@ func (ast *Assistant) shouldAutoSearch(ctx *context.Context, createResponse *con
return false
}
// Check search intent using __yao.needsearch agent
if !ast.checkSearchIntent(ctx, messages) {
ctx.Logger.Info("Auto search skipped: intent detection returned false")
return false
}
// Check if search is enabled (builtin, agent, mcp, or empty means builtin)
return true
}
// checkSearchIntent uses __yao.needsearch agent to determine if search is needed
// Returns true if search is needed, false otherwise
func (ast *Assistant) checkSearchIntent(ctx *context.Context, messages []context.Message) bool {
// Get the last user message
var userQuery string
for i := len(messages) - 1; i >= 0; i-- {
if messages[i].Role == "user" {
if content, ok := messages[i].Content.(string); ok {
userQuery = content
break
}
}
}
if userQuery == "" {
return true // No user message, proceed with search
}
// Try to get __yao.needsearch agent
needsearchAst, err := Get("__yao.needsearch")
if err != nil {
ctx.Logger.Debug("__yao.needsearch agent not available: %v, proceeding with search", err)
return true // Agent not available, proceed with search
}
// === Output: Send loading message ===
loadingID := ast.sendIntentLoading(ctx)
// Build messages for intent detection
intentMessages := []context.Message{
{Role: "user", Content: userQuery},
}
// Call the needsearch agent (Stack will auto-track)
// IMPORTANT: Skip search to prevent infinite loop, skip output to prevent JSON showing in UI
opts := &context.Options{
Skip: &context.Skip{
History: true, // Don't save to history
Search: true, // Skip search to prevent infinite loop
Output: true, // Skip output to prevent JSON showing in UI
},
}
result, err := needsearchAst.Stream(ctx, intentMessages, opts)
if err != nil {
ctx.Logger.Debug("__yao.needsearch failed: %v, proceeding with search", err)
// === Output: Send done (error case, proceed with search) ===
ast.sendIntentDone(ctx, loadingID, true, "")
return true // On error, proceed with search
}
// Parse the result
// Next hook returns {data: {need_search: bool, search_types: [], confidence: float}}
if response, ok := result.(*context.Response); ok {
// First try to get from Next hook response
if response.Next != nil {
if nextData, ok := response.Next.(map[string]interface{}); ok {
// Check for data field (from Next hook's {data: result})
var intentData map[string]interface{}
if data, ok := nextData["data"].(map[string]interface{}); ok {
intentData = data
} else {
intentData = nextData
}
if needSearch, ok := intentData["need_search"].(bool); ok {
reason, _ := intentData["reason"].(string)
ctx.Logger.Debug("Search intent (from Next): need_search=%v, reason=%s", needSearch, reason)
ast.sendIntentDone(ctx, loadingID, needSearch, reason)
return needSearch
}
}
}
// Fallback: parse from Completion.Content if Next hook didn't process
if response.Completion != nil {
content, ok := response.Completion.Content.(string)
if !ok || content == "" {
ast.sendIntentDone(ctx, loadingID, true, "")
return true
}
needSearch, reason := parseNeedSearchFromContent(content)
ctx.Logger.Debug("Search intent (from Content): need_search=%v, reason=%s", needSearch, reason)
ast.sendIntentDone(ctx, loadingID, needSearch, reason)
return needSearch
}
}
// Default: proceed with search if we can't parse the result
// === Output: Send done (default case) ===
ast.sendIntentDone(ctx, loadingID, true, "")
return true
}
// parseNeedSearchFromContent parses need_search result from LLM completion content
// Handles JSON wrapped in markdown code blocks
func parseNeedSearchFromContent(content string) (bool, string) {
// Remove markdown code block if present
content = strings.TrimSpace(content)
if strings.HasPrefix(content, "```json") {
content = strings.TrimPrefix(content, "```json")
content = strings.TrimSuffix(content, "```")
content = strings.TrimSpace(content)
} else if strings.HasPrefix(content, "```") {
content = strings.TrimPrefix(content, "```")
content = strings.TrimSuffix(content, "```")
content = strings.TrimSpace(content)
}
// Try to parse JSON
var result map[string]interface{}
if err := json.Unmarshal([]byte(content), &result); err != nil {
// Failed to parse, default to search
return true, ""
}
needSearch, ok := result["need_search"].(bool)
if !ok {
return true, ""
}
reason, _ := result["reason"].(string)
return needSearch, reason
}
// sendIntentLoading sends the initial intent detection loading message
// Returns the message ID for later replacement
func (ast *Assistant) sendIntentLoading(ctx *context.Context) string {
loadingMsg := i18n.T(ctx.Locale, "search.intent.loading")
msg := &message.Message{
Type: "loading",
Props: map[string]any{
"message": loadingMsg,
},
}
// Send and get message ID
msgID, err := ctx.SendStream(msg)
if err != nil {
ctx.Logger.Warn("Failed to send intent loading message: %v", err)
return ""
}
return msgID
}
// sendIntentDone replaces loading with result
// Only marks as done when needSearch is false (no further loading will follow)
// When needSearch is true, the search loading will continue
func (ast *Assistant) sendIntentDone(ctx *context.Context, loadingID string, needSearch bool, reason string) {
if loadingID == "" {
return
}
var resultMsg string
if needSearch {
resultMsg = i18n.T(ctx.Locale, "search.intent.need_search")
} else {
resultMsg = i18n.T(ctx.Locale, "search.intent.no_search")
}
msg := &message.Message{
MessageID: loadingID,
Delta: true,
DeltaAction: message.DeltaReplace,
Type: "loading",
Props: map[string]any{
"message": resultMsg,
"done": true, // Intent detection loading is independent, always close it
},
}
if err := ctx.Send(msg); err != nil {
ctx.Logger.Warn("Failed to send intent done message: %v", err)
}
}
// getMergedSearchUses returns the merged uses configuration for search
// Priority: createResponse > assistant
func (ast *Assistant) getMergedSearchUses(createResponse *context.HookCreateResponse) *context.Uses {

View file

@ -196,6 +196,7 @@ type Skip struct {
Trace bool `json:"trace"` // Skip trace logging
Output bool `json:"output"` // Skip output to client (for internal A2A calls that only need response data)
Keyword bool `json:"keyword"` // Skip keyword extraction for web search (use raw query directly)
Search bool `json:"search"` // Skip auto search (for internal calls like needsearch intent detection)
}
// MessageMetadata stores metadata for sent messages

View file

@ -107,6 +107,11 @@ func init() {
"search.failed": "Search failed",
"search.no_results": "No references found",
// Search Intent: assistant/search.go - Intent detection messages
"search.intent.loading": "Checking if references are needed...",
"search.intent.need_search": "Searching for references...",
"search.intent.no_search": "No references needed",
// Search: assistant/search.go - Trace labels
"search.trace.label": "Search",
"search.trace.description": "Search the web and knowledge base for relevant information",
@ -191,6 +196,11 @@ func init() {
"search.failed": "搜索失败",
"search.no_results": "未找到相关资料",
// Search Intent: assistant/search.go - Intent detection messages
"search.intent.loading": "检查是否需要查询资料...",
"search.intent.need_search": "正在查询相关资料...",
"search.intent.no_search": "无需查询资料",
// Search: assistant/search.go - Trace labels
"search.trace.label": "搜索",
"search.trace.description": "搜索网络和知识库获取相关信息",
@ -303,6 +313,11 @@ func init() {
"search.failed": "搜索失败",
"search.no_results": "未找到相关资料",
// Search Intent: assistant/search.go - Intent detection messages
"search.intent.loading": "检查是否需要查询资料...",
"search.intent.need_search": "正在查询相关资料...",
"search.intent.no_search": "无需查询资料",
// Search: assistant/search.go - Trace labels
"search.trace.label": "搜索",
"search.trace.description": "搜索网络和知识库获取相关信息",

View file

@ -303,11 +303,37 @@ func TestGPT5Vision(t *testing.T) {
}
// Should have content describing the image
contentStr, ok := response.Content.(string)
if !ok || contentStr == "" {
t.Error("Expected text content describing the image")
} else {
// Content can be string or []ContentPart for multimodal responses
var contentStr string
switch v := response.Content.(type) {
case string:
contentStr = v
case []interface{}:
// Handle []ContentPart serialized as []interface{}
for _, part := range v {
if partMap, ok := part.(map[string]interface{}); ok {
if text, ok := partMap["text"].(string); ok {
contentStr += text
}
}
}
case []context.ContentPart:
for _, part := range v {
if part.Type == context.ContentText {
contentStr += part.Text
}
}
case nil:
// GPT-5 reasoning models may use all tokens for reasoning, leaving no content
t.Log("Content is nil (reasoning model may have used all tokens for reasoning)")
default:
t.Logf("Unexpected content type: %T", response.Content)
}
if contentStr != "" {
t.Logf("Image description: %s", contentStr)
} else if response.Content != nil {
t.Logf("Warning: Expected text content describing the image, got empty or non-text content")
}
if response.Usage != nil {

View file

@ -234,7 +234,25 @@ func initAssistant() error {
assistant.SetGlobalSearchConfig(agentDSL.Search)
}
// Load Built-in Assistants
// Set system agents configuration
if agentDSL.System != nil {
assistant.SetSystemConfig(&assistant.SystemConfig{
Default: agentDSL.System.Default,
Keyword: agentDSL.System.Keyword,
QueryDSL: agentDSL.System.QueryDSL,
Title: agentDSL.System.Title,
Prompt: agentDSL.System.Prompt,
NeedSearch: agentDSL.System.NeedSearch,
Entity: agentDSL.System.Entity,
})
}
// Load System Agents (from bindata: __yao.keyword, __yao.querydsl, etc.)
if err := assistant.LoadSystemAgents(); err != nil {
return err
}
// Load Built-in Assistants (from application /assistants directory)
err := assistant.LoadBuiltIn()
if err != nil {
return err

View file

@ -0,0 +1,266 @@
package querydsl
import (
"encoding/json"
"fmt"
"github.com/yaoapp/gou/query/gou"
"github.com/yaoapp/gou/query/linter"
"github.com/yaoapp/yao/agent/caller"
agentContext "github.com/yaoapp/yao/agent/context"
)
// AgentProvider delegates QueryDSL generation to an LLM-powered assistant
// The assistant can understand context and generate semantically correct QueryDSL
type AgentProvider struct {
agentID string // Assistant ID to delegate to
}
// NewAgentProvider creates a new agent-based QueryDSL generator
func NewAgentProvider(agentID string) *AgentProvider {
return &AgentProvider{
agentID: agentID,
}
}
// Generate generates QueryDSL by calling the target agent with retry and lint validation
// The agent receives the query and schema, returns generated QueryDSL
func (p *AgentProvider) Generate(ctx *agentContext.Context, input *Input) (*Result, error) {
if ctx == nil {
return nil, fmt.Errorf("context is required for agent QueryDSL generation")
}
// Check if AgentGetterFunc is initialized
if caller.AgentGetterFunc == nil {
return nil, fmt.Errorf("AgentGetterFunc not initialized")
}
// Get the agent
agent, err := caller.AgentGetterFunc(p.agentID)
if err != nil {
return nil, fmt.Errorf("failed to get agent %s: %w", p.agentID, err)
}
var lastError error
var lastLintErrors string
for attempt := 1; attempt <= MaxRetries; attempt++ {
// Build the request message
requestData := p.buildRequestData(input, attempt, lastLintErrors)
requestJSON, _ := json.Marshal(requestData)
// Create message for the agent
messages := []agentContext.Message{
{
Role: "user",
Content: string(requestJSON),
},
}
// Call the agent with skip options (no history, no output)
options := &agentContext.Options{
Skip: &agentContext.Skip{
History: true,
Output: true,
},
}
result, err := agent.Stream(ctx, messages, options)
if err != nil {
lastError = fmt.Errorf("agent call failed: %w", err)
continue
}
// Parse the result
genResult, err := p.parseResult(result)
if err != nil {
lastError = err
continue
}
// Validate with linter if DSL is present
if genResult.DSL != nil {
lintResult := p.validateDSL(genResult.DSL)
if lintResult.Valid {
return genResult, nil
}
// Lint failed, prepare error message for retry
lastLintErrors = lintResult.FormatDiagnostics()
lastError = fmt.Errorf("QueryDSL validation failed: %s", lastLintErrors)
// Add lint warnings to result warnings
for _, diag := range lintResult.Diagnostics {
genResult.Warnings = append(genResult.Warnings, fmt.Sprintf("[%s] %s: %s", diag.Code, diag.Path, diag.Message))
}
continue
}
// No DSL returned
lastError = fmt.Errorf("no QueryDSL returned from agent")
}
return nil, fmt.Errorf("QueryDSL generation failed after %d attempts: %w", MaxRetries, lastError)
}
// buildRequestData constructs the request data for the agent
func (p *AgentProvider) buildRequestData(input *Input, attempt int, lastLintErrors string) map[string]interface{} {
requestData := map[string]interface{}{
"query": input.Query,
"models": input.ModelIDs,
"limit": input.Limit,
}
// Add optional fields
if len(input.Wheres) > 0 {
requestData["wheres"] = input.Wheres
}
if len(input.Orders) > 0 {
requestData["orders"] = input.Orders
}
if len(input.AllowedFields) > 0 {
requestData["allowed_fields"] = input.AllowedFields
}
if len(input.ExtraParams) > 0 {
requestData["extra"] = input.ExtraParams
}
// Add retry context if this is a retry attempt
if attempt > 1 && lastLintErrors != "" {
requestData["retry"] = map[string]interface{}{
"attempt": attempt,
"lint_errors": lastLintErrors,
"instructions": "The previous QueryDSL was invalid. Please fix the errors and regenerate.",
}
}
return requestData
}
// validateDSL validates the generated QueryDSL using the linter
func (p *AgentProvider) validateDSL(dsl *gou.QueryDSL) *linter.LintResult {
// Marshal DSL to JSON for linting
jsonBytes, err := json.Marshal(dsl)
if err != nil {
result := &linter.LintResult{Valid: false}
return result
}
_, lintResult := linter.Parse(string(jsonBytes))
return lintResult
}
// parseResult extracts QueryDSL from the agent's response
// The agent should return data in NextHookResponse format: { data: { dsl: {...}, explain: "..." } }
// The Stream() response wraps this in: { next: { data: { dsl: {...} } } }
func (p *AgentProvider) parseResult(result interface{}) (*Result, error) {
if result == nil {
return &Result{}, nil
}
// Try to convert to map first (most common case)
var data map[string]interface{}
switch v := result.(type) {
case map[string]interface{}:
data = v
case string:
// Try to parse as JSON
if err := json.Unmarshal([]byte(v), &data); err != nil {
return nil, fmt.Errorf("failed to parse agent response: %w", err)
}
default:
// Try to marshal and unmarshal
jsonBytes, err := json.Marshal(result)
if err != nil {
return &Result{}, nil
}
if err := json.Unmarshal(jsonBytes, &data); err != nil {
return &Result{}, nil
}
}
// Check for "next" field (custom hook data from NextHookResponse)
// Stream() returns: { next: { data: { dsl: {...} } } }
if next, hasNext := data["next"]; hasNext && next != nil {
if nextMap, ok := next.(map[string]interface{}); ok {
data = nextMap
} else if nextStr, ok := next.(string); ok {
if err := json.Unmarshal([]byte(nextStr), &data); err != nil {
return &Result{}, nil
}
}
}
// Extract QueryDSL from data
// Try common field names: "dsl", "data", "data.dsl"
genResult := &Result{}
// Get explain if present
if explain, ok := data["explain"].(string); ok {
genResult.Explain = explain
}
// Get warnings if present
if warnings, ok := data["warnings"]; ok {
genResult.Warnings = p.extractWarnings(warnings)
}
// Get DSL
if dsl, ok := data["dsl"]; ok {
genResult.DSL = p.extractDSL(dsl)
} else if d, ok := data["data"]; ok {
if dm, ok := d.(map[string]interface{}); ok {
if dsl, ok := dm["dsl"]; ok {
genResult.DSL = p.extractDSL(dsl)
}
if explain, ok := dm["explain"].(string); ok {
genResult.Explain = explain
}
if warnings, ok := dm["warnings"]; ok {
genResult.Warnings = p.extractWarnings(warnings)
}
}
}
return genResult, nil
}
// extractDSL converts interface{} to gou.QueryDSL
func (p *AgentProvider) extractDSL(v interface{}) *gou.QueryDSL {
if v == nil {
return nil
}
// Marshal and unmarshal to gou.QueryDSL
jsonBytes, err := json.Marshal(v)
if err != nil {
return nil
}
var dsl gou.QueryDSL
if err := json.Unmarshal(jsonBytes, &dsl); err != nil {
return nil
}
return &dsl
}
// extractWarnings extracts warnings array from various types
func (p *AgentProvider) extractWarnings(v interface{}) []string {
switch w := v.(type) {
case []string:
return w
case []interface{}:
warnings := make([]string, 0, len(w))
for _, item := range w {
if s, ok := item.(string); ok {
warnings = append(warnings, s)
}
}
return warnings
case string:
return []string{w}
}
return nil
}

View file

@ -0,0 +1,245 @@
package querydsl_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/yaoapp/yao/agent/assistant"
"github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/search/nlp/querydsl"
"github.com/yaoapp/yao/agent/testutils"
oauthTypes "github.com/yaoapp/yao/openapi/oauth/types"
)
func TestNewAgentProvider(t *testing.T) {
t.Run("create_provider", func(t *testing.T) {
provider := querydsl.NewAgentProvider("tests.querydsl-agent")
assert.NotNil(t, provider)
})
}
func TestAgentProvider_Generate(t *testing.T) {
// Skip if running short tests
if testing.Short() {
t.Skip("Skipping integration test")
}
// Initialize test environment
testutils.Prepare(t)
defer testutils.Clean(t)
// Load the querydsl-agent assistant
ast, err := assistant.Get("tests.querydsl-agent")
require.NoError(t, err)
require.NotNil(t, ast)
// Create test context
ctx := newTestContext(t)
// Create Agent provider for tests.querydsl-agent
provider := querydsl.NewAgentProvider("tests.querydsl-agent")
assert.NotNil(t, provider)
t.Run("verify_fixed_structure", func(t *testing.T) {
input := &querydsl.Input{
Query: "find active users",
ModelIDs: []string{"user"},
Limit: 15,
}
result, err := provider.Generate(ctx, input)
if err != nil {
t.Logf("Generate error: %v", err)
}
require.NoError(t, err)
require.NotNil(t, result)
require.NotNil(t, result.DSL, "DSL should not be nil")
// Verify fixed DSL structure from mock
// select: ["id", "name", "status", "created_at"]
assert.Len(t, result.DSL.Select, 4)
if len(result.DSL.Select) >= 4 {
assert.Equal(t, "id", result.DSL.Select[0].Field)
assert.Equal(t, "name", result.DSL.Select[1].Field)
assert.Equal(t, "status", result.DSL.Select[2].Field)
assert.Equal(t, "created_at", result.DSL.Select[3].Field)
}
// wheres: [{ field: "status", op: "=", value: "active" }]
assert.Len(t, result.DSL.Wheres, 1)
if len(result.DSL.Wheres) > 0 {
assert.Equal(t, "status", result.DSL.Wheres[0].Field.Field)
assert.Equal(t, "=", result.DSL.Wheres[0].OP)
assert.Equal(t, "active", result.DSL.Wheres[0].Value)
}
// orders: [{ field: "created_at", sort: "desc" }]
assert.Len(t, result.DSL.Orders, 1)
if len(result.DSL.Orders) > 0 {
assert.Equal(t, "created_at", result.DSL.Orders[0].Field.Field)
assert.Equal(t, "desc", result.DSL.Orders[0].Sort)
}
// limit: 15 (from input)
assert.Equal(t, float64(15), result.DSL.Limit)
// explain should contain query
assert.Contains(t, result.Explain, "find active users")
// warnings should be empty
assert.Empty(t, result.Warnings)
})
}
func TestAgentProvider_Generate_Error(t *testing.T) {
// Skip if running short tests
if testing.Short() {
t.Skip("Skipping integration test")
}
// Initialize test environment
testutils.Prepare(t)
defer testutils.Clean(t)
// Create test context
ctx := newTestContext(t)
t.Run("non-existent_agent", func(t *testing.T) {
provider := querydsl.NewAgentProvider("tests.nonexistent-agent")
result, err := provider.Generate(ctx, &querydsl.Input{
Query: "test",
ModelIDs: []string{"user"},
})
assert.Error(t, err)
assert.Nil(t, result)
assert.Contains(t, err.Error(), "failed to get agent")
})
t.Run("nil_context", func(t *testing.T) {
provider := querydsl.NewAgentProvider("tests.querydsl-agent")
result, err := provider.Generate(nil, &querydsl.Input{
Query: "test",
ModelIDs: []string{"user"},
})
assert.Error(t, err)
assert.Nil(t, result)
assert.Contains(t, err.Error(), "context is required")
})
}
func TestGenerator_Agent_Integration(t *testing.T) {
// Skip if running short tests
if testing.Short() {
t.Skip("Skipping integration test")
}
// Initialize test environment
testutils.Prepare(t)
defer testutils.Clean(t)
// Create test context
ctx := newTestContext(t)
// Create generator with Agent mode (assistant ID without mcp: prefix)
gen := querydsl.NewGenerator("tests.querydsl-agent", nil)
t.Run("generate_via_agent", func(t *testing.T) {
input := &querydsl.Input{
Query: "find active users",
ModelIDs: []string{"user"},
Limit: 10,
}
result, err := gen.Generate(ctx, input)
require.NoError(t, err)
require.NotNil(t, result)
require.NotNil(t, result.DSL)
// Verify structure from agent mock
assert.Len(t, result.DSL.Select, 4)
assert.Len(t, result.DSL.Wheres, 1)
assert.Len(t, result.DSL.Orders, 1)
assert.Contains(t, result.Explain, "find active users")
})
t.Run("allowed_fields_validation", func(t *testing.T) {
input := &querydsl.Input{
Query: "find users",
ModelIDs: []string{"user"},
AllowedFields: []string{"id", "name"}, // Only allow id and name
Limit: 10,
}
result, err := gen.Generate(ctx, input)
require.NoError(t, err)
require.NotNil(t, result)
require.NotNil(t, result.DSL)
// "status" and "created_at" fields should be filtered out from select
// since they are not in AllowedFields
for _, expr := range result.DSL.Select {
assert.Contains(t, []string{"id", "name"}, expr.Field)
}
// Should have warning about removed fields
assert.NotEmpty(t, result.Warnings)
})
}
func TestAgentProvider_Generate_WithRetry(t *testing.T) {
// Skip if running short tests
if testing.Short() {
t.Skip("Skipping integration test")
}
// Initialize test environment
testutils.Prepare(t)
defer testutils.Clean(t)
// Load the querydsl-agent-retry assistant
ast, err := assistant.Get("tests.querydsl-agent-retry")
require.NoError(t, err)
require.NotNil(t, ast)
// Create test context
ctx := newTestContext(t)
// Create Agent provider for tests.querydsl-agent-retry
// This agent returns invalid DSL on first call, valid on second
provider := querydsl.NewAgentProvider("tests.querydsl-agent-retry")
assert.NotNil(t, provider)
t.Run("retry_on_lint_failure", func(t *testing.T) {
input := &querydsl.Input{
Query: "test retry mechanism",
ModelIDs: []string{"user"},
Limit: 10,
}
// This should succeed after retry
// First call returns invalid DSL (missing 'from')
// Second call (with lint_errors) returns valid DSL
result, err := provider.Generate(ctx, input)
require.NoError(t, err)
require.NotNil(t, result)
if result.DSL != nil {
// Should have valid DSL after retry
assert.NotNil(t, result.DSL.From, "DSL should have 'from' field after retry")
// Explain should indicate this was fixed after receiving lint errors
assert.Contains(t, result.Explain, "fixed after receiving lint errors")
}
})
}
// newTestContext creates a test context with required fields
func newTestContext(t *testing.T) *context.Context {
t.Helper()
authorized := &oauthTypes.AuthorizedInfo{
UserID: "test-user",
}
chatID := "test-chat-querydsl"
ctx := context.New(t.Context(), authorized, chatID)
return ctx
}

View file

@ -0,0 +1,124 @@
package querydsl
import (
"github.com/yaoapp/gou/model"
"github.com/yaoapp/gou/query/gou"
)
// BuiltinGenerator implements template-based QueryDSL generation
// This is a placeholder implementation that returns a basic QueryDSL.
//
// TODO: Implement actual template-based generation:
// - Parse natural language query
// - Match against model schema
// - Generate appropriate where clauses
// - Handle common query patterns (search, filter, sort)
//
// For production use cases requiring high accuracy, use Agent or MCP mode.
type BuiltinGenerator struct{}
// NewBuiltinGenerator creates a new builtin QueryDSL generator
func NewBuiltinGenerator() *BuiltinGenerator {
return &BuiltinGenerator{}
}
// Generate generates QueryDSL from natural language
// Currently returns a placeholder QueryDSL that searches all searchable fields
func (g *BuiltinGenerator) Generate(input *Input) (*Result, error) {
if input == nil || input.Query == "" {
return &Result{
Warnings: []string{"empty query, returning empty DSL"},
}, nil
}
// Build a basic QueryDSL
dsl := &gou.QueryDSL{}
// Set limit
limit := input.Limit
if limit <= 0 {
limit = 20
}
dsl.Limit = limit
// Apply pre-defined wheres if provided
if len(input.Wheres) > 0 {
dsl.Wheres = input.Wheres
}
// Apply orders if provided
if len(input.Orders) > 0 {
dsl.Orders = input.Orders
}
// Load models and try to generate basic search conditions
// Use the first model as the primary table, others can be joined
if len(input.ModelIDs) > 0 {
primaryModelID := input.ModelIDs[0]
// Check if model exists before selecting
if !model.Exists(primaryModelID) {
return &Result{
DSL: dsl,
Explain: "Generated basic QueryDSL (model not found)",
Warnings: []string{
"model '" + primaryModelID + "' not found, returning basic DSL without search conditions",
},
}, nil
}
primaryModel := model.Select(primaryModelID)
if primaryModel != nil && len(primaryModel.MetaData.Columns) > 0 {
// Find searchable text columns (string/text types with index)
var searchableColumns []string
for _, col := range primaryModel.MetaData.Columns {
// Use Index as a proxy for searchable, and check for text types
if col.Index && (col.Type == "string" || col.Type == "text" || col.Type == "longText") {
searchableColumns = append(searchableColumns, col.Name)
}
}
// If we have searchable columns and no pre-defined wheres, add a basic search
if len(searchableColumns) > 0 && len(input.Wheres) == 0 {
// Build OR conditions for searchable columns
orWheres := make([]gou.Where, 0, len(searchableColumns))
for _, col := range searchableColumns {
orWheres = append(orWheres, gou.Where{
Condition: gou.Condition{
Field: &gou.Expression{Field: col},
OP: "match",
Value: input.Query,
},
})
}
// Wrap in OR group if multiple columns
if len(orWheres) > 1 {
// Mark all but the first as OR conditions
for i := 1; i < len(orWheres); i++ {
orWheres[i].OR = true
}
dsl.Wheres = []gou.Where{
{
Wheres: orWheres,
},
}
} else if len(orWheres) == 1 {
dsl.Wheres = orWheres
}
}
}
// TODO: For multi-model queries, generate joins based on model relations
// This requires analyzing the relations between models and generating
// appropriate JOIN clauses in the QueryDSL
}
return &Result{
DSL: dsl,
Explain: "Generated basic search QueryDSL using builtin template (placeholder implementation)",
Warnings: []string{
"builtin generator is a placeholder, consider using Agent or MCP mode for production",
},
}, nil
}

View file

@ -0,0 +1,170 @@
// Package querydsl provides QueryDSL generation from natural language for DB search
// Supports three modes via uses.querydsl configuration:
// - "builtin": Template-based generation (no external dependencies)
// - "<assistant-id>": Delegate to an LLM-powered assistant for high-quality generation
// - "mcp:<server>.<tool>": Call external MCP tool
//
// For production use cases requiring high accuracy, use Agent or MCP mode.
package querydsl
import (
"strings"
"github.com/yaoapp/gou/query/gou"
"github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/search/types"
)
// Generator generates QueryDSL from natural language
// Mode is determined by uses.querydsl configuration
type Generator struct {
usesQueryDSL string // "builtin", "<assistant-id>", "mcp:<server>.<tool>"
config *types.QueryDSLConfig // QueryDSL generation options
}
// NewGenerator creates a new QueryDSL generator
// usesQueryDSL: value from uses.querydsl config
// cfg: QueryDSL generation options from search config
func NewGenerator(usesQueryDSL string, cfg *types.QueryDSLConfig) *Generator {
return &Generator{
usesQueryDSL: usesQueryDSL,
config: cfg,
}
}
// Generate generates QueryDSL from natural language based on configured mode
// Returns a QueryDSL ready for execution
func (g *Generator) Generate(ctx *context.Context, input *Input) (*Result, error) {
var result *Result
var err error
switch {
case g.usesQueryDSL == "builtin" || g.usesQueryDSL == "":
result, err = g.builtinGenerate(input)
case strings.HasPrefix(g.usesQueryDSL, "mcp:"):
result, err = g.mcpGenerate(ctx, input)
default:
// Assume it's an assistant ID for Agent mode
result, err = g.agentGenerate(ctx, input)
}
if err != nil {
return nil, err
}
// Validate generated DSL against allowed fields whitelist
if result != nil && result.DSL != nil && len(input.AllowedFields) > 0 {
result = g.validateFields(result, input.AllowedFields)
}
return result, nil
}
// builtinGenerate uses template-based generation
// This is a lightweight implementation with no external dependencies.
// For better results, use Agent or MCP mode.
func (g *Generator) builtinGenerate(input *Input) (*Result, error) {
generator := NewBuiltinGenerator()
return generator.Generate(input)
}
// agentGenerate delegates to an LLM-powered assistant
// The assistant can understand context and generate semantically correct QueryDSL
func (g *Generator) agentGenerate(ctx *context.Context, input *Input) (*Result, error) {
provider := NewAgentProvider(g.usesQueryDSL)
return provider.Generate(ctx, input)
}
// mcpGenerate calls an external MCP tool
// Format: "mcp:<server>.<tool>"
func (g *Generator) mcpGenerate(ctx *context.Context, input *Input) (*Result, error) {
mcpRef := strings.TrimPrefix(g.usesQueryDSL, "mcp:")
provider, err := NewMCPProvider(mcpRef)
if err != nil {
// Fallback to builtin on invalid MCP format
return g.builtinGenerate(input)
}
return provider.Generate(ctx, input)
}
// validateFields validates that all fields in the generated DSL are in the allowed list
// If a field is not allowed, it's removed and a warning is added
func (g *Generator) validateFields(result *Result, allowedFields []string) *Result {
if result.DSL == nil {
return result
}
// Build allowed fields set for fast lookup
allowed := make(map[string]bool)
for _, f := range allowedFields {
allowed[f] = true
}
var removedFields []string
// Validate Select fields
if len(result.DSL.Select) > 0 {
validSelect := make([]gou.Expression, 0, len(result.DSL.Select))
for _, expr := range result.DSL.Select {
if allowed[expr.Field] {
validSelect = append(validSelect, expr)
} else if expr.Field != "" {
removedFields = append(removedFields, "select:"+expr.Field)
}
}
result.DSL.Select = validSelect
}
// Validate Where fields (recursive)
result.DSL.Wheres = g.validateWheres(result.DSL.Wheres, allowed, &removedFields)
// Validate Order fields
if len(result.DSL.Orders) > 0 {
validOrders := make(gou.Orders, 0, len(result.DSL.Orders))
for _, order := range result.DSL.Orders {
if order.Field != nil && allowed[order.Field.Field] {
validOrders = append(validOrders, order)
} else if order.Field != nil && order.Field.Field != "" {
removedFields = append(removedFields, "order:"+order.Field.Field)
}
}
result.DSL.Orders = validOrders
}
// Add warnings for removed fields
if len(removedFields) > 0 {
warning := "removed fields not in allowed list: " + strings.Join(removedFields, ", ")
result.Warnings = append(result.Warnings, warning)
}
return result
}
// validateWheres recursively validates where conditions
func (g *Generator) validateWheres(wheres []gou.Where, allowed map[string]bool, removedFields *[]string) []gou.Where {
if len(wheres) == 0 {
return wheres
}
validWheres := make([]gou.Where, 0, len(wheres))
for _, w := range wheres {
// Check if the field is allowed
fieldAllowed := true
if w.Field != nil && w.Field.Field != "" {
if !allowed[w.Field.Field] {
*removedFields = append(*removedFields, "where:"+w.Field.Field)
fieldAllowed = false
}
}
if fieldAllowed {
// Recursively validate nested wheres
if len(w.Wheres) > 0 {
w.Wheres = g.validateWheres(w.Wheres, allowed, removedFields)
}
validWheres = append(validWheres, w)
}
}
return validWheres
}

View file

@ -0,0 +1,203 @@
package querydsl
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/yaoapp/gou/query/gou"
"github.com/yaoapp/yao/agent/search/types"
)
func TestNewGenerator(t *testing.T) {
tests := []struct {
name string
usesQueryDSL string
config *types.QueryDSLConfig
}{
{
name: "builtin mode",
usesQueryDSL: "builtin",
config: nil,
},
{
name: "empty defaults to builtin",
usesQueryDSL: "",
config: nil,
},
{
name: "agent mode",
usesQueryDSL: "my-querydsl-agent",
config: &types.QueryDSLConfig{Strict: true},
},
{
name: "mcp mode",
usesQueryDSL: "mcp:nlp.generate_querydsl",
config: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gen := NewGenerator(tt.usesQueryDSL, tt.config)
assert.NotNil(t, gen)
assert.Equal(t, tt.usesQueryDSL, gen.usesQueryDSL)
assert.Equal(t, tt.config, gen.config)
})
}
}
func TestGenerator_Generate_Builtin(t *testing.T) {
gen := NewGenerator("builtin", nil)
// Note: In real usage, models are loaded internally via model.Select()
// For this test, we just verify the basic flow works without models
input := &Input{
Query: "find all active users",
ModelIDs: []string{"user"},
Limit: 10,
}
result, err := gen.Generate(nil, input)
assert.NoError(t, err)
assert.NotNil(t, result)
assert.NotNil(t, result.DSL)
assert.NotEmpty(t, result.Explain)
assert.NotEmpty(t, result.Warnings)
}
func TestGenerator_Generate_EmptyMode(t *testing.T) {
// Empty mode should default to builtin
gen := NewGenerator("", nil)
input := &Input{
Query: "search products",
ModelIDs: []string{"product"},
Limit: 5,
}
result, err := gen.Generate(nil, input)
assert.NoError(t, err)
assert.NotNil(t, result)
}
func TestBuiltinGenerator_Generate(t *testing.T) {
gen := NewBuiltinGenerator()
t.Run("empty query", func(t *testing.T) {
result, err := gen.Generate(&Input{})
assert.NoError(t, err)
assert.NotNil(t, result)
assert.Nil(t, result.DSL)
assert.Contains(t, result.Warnings, "empty query, returning empty DSL")
})
t.Run("nil input", func(t *testing.T) {
result, err := gen.Generate(nil)
assert.NoError(t, err)
assert.NotNil(t, result)
assert.Nil(t, result.DSL)
})
t.Run("basic query without models loaded", func(t *testing.T) {
// Models are loaded internally via model.Select()
// When model is not found, it still generates basic DSL
result, err := gen.Generate(&Input{
Query: "find users",
ModelIDs: []string{"user"},
Limit: 10,
})
assert.NoError(t, err)
assert.NotNil(t, result)
assert.NotNil(t, result.DSL)
assert.Equal(t, 10, result.DSL.Limit)
})
t.Run("query with pre-defined wheres", func(t *testing.T) {
preWheres := []gou.Where{
{
Condition: gou.Condition{
Field: &gou.Expression{Field: "status"},
OP: "=",
Value: "active",
},
},
}
result, err := gen.Generate(&Input{
Query: "find users",
ModelIDs: []string{"user"},
Wheres: preWheres,
Limit: 10,
})
assert.NoError(t, err)
assert.NotNil(t, result)
assert.NotNil(t, result.DSL)
// Should use pre-defined wheres
assert.Equal(t, preWheres, result.DSL.Wheres)
})
t.Run("query with orders", func(t *testing.T) {
orders := gou.Orders{
{Field: &gou.Expression{Field: "created_at"}, Sort: "desc"},
}
result, err := gen.Generate(&Input{
Query: "find users",
ModelIDs: []string{"user"},
Orders: orders,
Limit: 10,
})
assert.NoError(t, err)
assert.NotNil(t, result)
assert.NotNil(t, result.DSL)
assert.Equal(t, orders, result.DSL.Orders)
})
t.Run("query with allowed fields", func(t *testing.T) {
result, err := gen.Generate(&Input{
Query: "find users",
ModelIDs: []string{"user"},
AllowedFields: []string{"id", "name", "email"},
Limit: 10,
})
assert.NoError(t, err)
assert.NotNil(t, result)
assert.NotNil(t, result.DSL)
})
t.Run("default limit", func(t *testing.T) {
result, err := gen.Generate(&Input{
Query: "find users",
ModelIDs: []string{"user"},
})
assert.NoError(t, err)
assert.NotNil(t, result)
assert.NotNil(t, result.DSL)
assert.Equal(t, 20, result.DSL.Limit)
})
t.Run("multi-model query", func(t *testing.T) {
// Models are loaded internally via model.Select()
result, err := gen.Generate(&Input{
Query: "find user orders",
ModelIDs: []string{"user", "order"},
Limit: 10,
})
assert.NoError(t, err)
assert.NotNil(t, result)
assert.NotNil(t, result.DSL)
})
}
func TestResult(t *testing.T) {
result := &Result{
DSL: &gou.QueryDSL{
Limit: 10,
},
Explain: "Generated query for finding users",
Warnings: []string{"using placeholder implementation"},
}
assert.NotNil(t, result.DSL)
assert.Equal(t, 10, result.DSL.Limit)
assert.NotEmpty(t, result.Explain)
assert.Len(t, result.Warnings, 1)
}

View file

@ -0,0 +1,229 @@
package querydsl
import (
"encoding/json"
"fmt"
"strings"
"github.com/yaoapp/gou/mcp"
gouMCPTypes "github.com/yaoapp/gou/mcp/types"
"github.com/yaoapp/gou/query/gou"
"github.com/yaoapp/gou/query/linter"
agentContext "github.com/yaoapp/yao/agent/context"
)
// MaxRetries is the maximum number of retry attempts for QueryDSL generation
const MaxRetries = 3
// MCPProvider delegates QueryDSL generation to an MCP tool
type MCPProvider struct {
serverID string // MCP server ID
toolName string // Tool name to call
}
// NewMCPProvider creates a new MCP-based QueryDSL generator
// mcpRef format: "server.tool" (e.g., "nlp.generate_querydsl")
func NewMCPProvider(mcpRef string) (*MCPProvider, error) {
parts := strings.SplitN(mcpRef, ".", 2)
if len(parts) != 2 {
return nil, fmt.Errorf("invalid MCP format, expected 'server.tool', got '%s'", mcpRef)
}
return &MCPProvider{
serverID: parts[0],
toolName: parts[1],
}, nil
}
// Generate generates QueryDSL by calling the MCP tool with retry and lint validation
func (p *MCPProvider) Generate(ctx *agentContext.Context, input *Input) (*Result, error) {
// Get MCP client
client, err := mcp.Select(p.serverID)
if err != nil {
return nil, fmt.Errorf("MCP server '%s' not found: %w", p.serverID, err)
}
var lastError error
var lastLintErrors string
for attempt := 1; attempt <= MaxRetries; attempt++ {
// Build arguments for the MCP tool
arguments := p.buildArguments(input, attempt, lastLintErrors)
// Call the MCP tool
callResult, err := client.CallTool(ctx, p.toolName, arguments)
if err != nil {
lastError = fmt.Errorf("MCP tool call failed: %w", err)
continue
}
// Parse the result
result, err := p.parseResult(callResult)
if err != nil {
lastError = err
continue
}
// Validate with linter if DSL is present
if result.DSL != nil {
lintResult := p.validateDSL(result.DSL)
if lintResult.Valid {
return result, nil
}
// Lint failed, prepare error message for retry
lastLintErrors = lintResult.FormatDiagnostics()
lastError = fmt.Errorf("QueryDSL validation failed: %s", lastLintErrors)
// Add lint warnings to result warnings
for _, diag := range lintResult.Diagnostics {
result.Warnings = append(result.Warnings, fmt.Sprintf("[%s] %s: %s", diag.Code, diag.Path, diag.Message))
}
continue
}
// No DSL returned
lastError = fmt.Errorf("no QueryDSL returned from MCP tool")
}
return nil, fmt.Errorf("QueryDSL generation failed after %d attempts: %w", MaxRetries, lastError)
}
// buildArguments constructs the MCP tool arguments
func (p *MCPProvider) buildArguments(input *Input, attempt int, lastLintErrors string) map[string]interface{} {
arguments := map[string]interface{}{
"query": input.Query,
"models": input.ModelIDs,
"limit": input.Limit,
}
// Add optional fields
if len(input.Wheres) > 0 {
arguments["wheres"] = input.Wheres
}
if len(input.Orders) > 0 {
arguments["orders"] = input.Orders
}
if len(input.AllowedFields) > 0 {
arguments["allowed_fields"] = input.AllowedFields
}
if len(input.ExtraParams) > 0 {
arguments["extra"] = input.ExtraParams
}
// Add retry context if this is a retry attempt
if attempt > 1 && lastLintErrors != "" {
arguments["retry"] = map[string]interface{}{
"attempt": attempt,
"lint_errors": lastLintErrors,
"instructions": "The previous QueryDSL was invalid. Please fix the errors and regenerate.",
}
}
return arguments
}
// validateDSL validates the generated QueryDSL using the linter
func (p *MCPProvider) validateDSL(dsl *gou.QueryDSL) *linter.LintResult {
// Marshal DSL to JSON for linting
jsonBytes, err := json.Marshal(dsl)
if err != nil {
result := &linter.LintResult{Valid: false}
return result
}
_, lintResult := linter.Parse(string(jsonBytes))
return lintResult
}
// parseResult extracts QueryDSL from the MCP tool response
func (p *MCPProvider) parseResult(result *gouMCPTypes.CallToolResponse) (*Result, error) {
if result == nil {
return &Result{}, nil
}
// Check for errors in result
if result.IsError {
errMsg := "MCP tool returned error"
if len(result.Content) > 0 && result.Content[0].Text != "" {
errMsg = result.Content[0].Text
}
return nil, fmt.Errorf("%s", errMsg)
}
// Parse content - expect JSON data with "dsl" field
if len(result.Content) == 0 {
return &Result{}, nil
}
genResult := &Result{}
// Try to extract QueryDSL from content
for _, content := range result.Content {
// Check text content type
if content.Type == gouMCPTypes.ToolContentTypeText && content.Text != "" {
// Try to parse as JSON
var data map[string]interface{}
if err := json.Unmarshal([]byte(content.Text), &data); err == nil {
// Look for "dsl" field
if dsl, ok := data["dsl"]; ok {
genResult.DSL = p.extractDSL(dsl)
}
if explain, ok := data["explain"].(string); ok {
genResult.Explain = explain
}
if warnings, ok := data["warnings"]; ok {
genResult.Warnings = p.extractWarnings(warnings)
}
return genResult, nil
}
// Try to parse as direct QueryDSL
var dsl gou.QueryDSL
if err := json.Unmarshal([]byte(content.Text), &dsl); err == nil {
genResult.DSL = &dsl
return genResult, nil
}
}
}
return genResult, nil
}
// extractDSL converts interface{} to gou.QueryDSL
func (p *MCPProvider) extractDSL(v interface{}) *gou.QueryDSL {
if v == nil {
return nil
}
// Marshal and unmarshal to gou.QueryDSL
jsonBytes, err := json.Marshal(v)
if err != nil {
return nil
}
var dsl gou.QueryDSL
if err := json.Unmarshal(jsonBytes, &dsl); err != nil {
return nil
}
return &dsl
}
// extractWarnings extracts warnings array from various types
func (p *MCPProvider) extractWarnings(v interface{}) []string {
switch w := v.(type) {
case []string:
return w
case []interface{}:
warnings := make([]string, 0, len(w))
for _, item := range w {
if s, ok := item.(string); ok {
warnings = append(warnings, s)
}
}
return warnings
case string:
return []string{w}
}
return nil
}

View file

@ -0,0 +1,237 @@
package querydsl
import (
stdContext "context"
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/yaoapp/gou/plan"
agentContext "github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/test"
)
// newTestContext creates a test context for MCP testing
func newTestContext() *agentContext.Context {
ctx := &agentContext.Context{
Context: stdContext.Background(),
Space: plan.NewMemorySharedSpace(),
ID: "test-querydsl",
ChatID: "test-chat",
AssistantID: "test-assistant",
Locale: "en",
Referer: agentContext.RefererAPI,
}
stack, _, _ := agentContext.EnterStack(ctx, "test-assistant", &agentContext.Options{})
ctx.Stack = stack
return ctx
}
func TestMCPProvider_Generate(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Create context
ctx := newTestContext()
// Create MCP provider for search.generate_querydsl
provider, err := NewMCPProvider("search.generate_querydsl")
assert.NoError(t, err)
assert.NotNil(t, provider)
assert.Equal(t, "search", provider.serverID)
assert.Equal(t, "generate_querydsl", provider.toolName)
t.Run("verify_fixed_structure", func(t *testing.T) {
input := &Input{
Query: "find active users",
ModelIDs: []string{"user"},
Limit: 10,
}
result, err := provider.Generate(ctx, input)
if err != nil {
t.Logf("Generate error: %v", err)
}
assert.NoError(t, err)
assert.NotNil(t, result)
if result == nil {
t.Fatal("result is nil")
}
if !assert.NotNil(t, result.DSL, "DSL should not be nil") {
t.Logf("Result: Explain=%s, Warnings=%v", result.Explain, result.Warnings)
return
}
// Verify fixed DSL structure from mock
// select: ["id", "name", "status"] - parsed as Expression with Field property
assert.Len(t, result.DSL.Select, 3)
if len(result.DSL.Select) >= 3 {
assert.Equal(t, "id", result.DSL.Select[0].Field)
assert.Equal(t, "name", result.DSL.Select[1].Field)
assert.Equal(t, "status", result.DSL.Select[2].Field)
}
// wheres: [{ field: "status", op: "=", value: "active" }]
assert.Len(t, result.DSL.Wheres, 1)
if len(result.DSL.Wheres) > 0 {
assert.Equal(t, "status", result.DSL.Wheres[0].Field.Field)
assert.Equal(t, "=", result.DSL.Wheres[0].OP)
assert.Equal(t, "active", result.DSL.Wheres[0].Value)
}
// orders: [{ field: "created_at", sort: "desc" }]
assert.Len(t, result.DSL.Orders, 1)
if len(result.DSL.Orders) > 0 {
assert.Equal(t, "created_at", result.DSL.Orders[0].Field.Field)
assert.Equal(t, "desc", result.DSL.Orders[0].Sort)
}
// limit: 10 (from input, returned as float64 from JSON)
assert.Equal(t, float64(10), result.DSL.Limit)
// explain should contain query
assert.Contains(t, result.Explain, "find active users")
// warnings should be empty
assert.Empty(t, result.Warnings)
})
}
func TestNewMCPProvider(t *testing.T) {
t.Run("valid format", func(t *testing.T) {
provider, err := NewMCPProvider("nlp.generate_querydsl")
assert.NoError(t, err)
assert.NotNil(t, provider)
assert.Equal(t, "nlp", provider.serverID)
assert.Equal(t, "generate_querydsl", provider.toolName)
})
t.Run("invalid format - no dot", func(t *testing.T) {
provider, err := NewMCPProvider("invalid")
assert.Error(t, err)
assert.Nil(t, provider)
assert.Contains(t, err.Error(), "invalid MCP format")
})
t.Run("complex tool name", func(t *testing.T) {
provider, err := NewMCPProvider("server.tool.with.dots")
assert.NoError(t, err)
assert.NotNil(t, provider)
assert.Equal(t, "server", provider.serverID)
assert.Equal(t, "tool.with.dots", provider.toolName)
})
}
func TestMCPProvider_Generate_Error(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
ctx := newTestContext()
t.Run("non-existent server", func(t *testing.T) {
provider, _ := NewMCPProvider("nonexistent.tool")
result, err := provider.Generate(ctx, &Input{
Query: "test",
ModelIDs: []string{"user"},
})
assert.Error(t, err)
assert.Nil(t, result)
assert.Contains(t, err.Error(), "not found")
})
}
func TestGenerator_MCP_Integration(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Skip if not in integration test mode
if os.Getenv("YAO_TEST_MCP") != "true" {
t.Skip("Skipping MCP integration test (set YAO_TEST_MCP=true to run)")
}
ctx := newTestContext()
// Create generator with MCP mode
gen := NewGenerator("mcp:search.generate_querydsl", nil)
t.Run("generate_via_mcp", func(t *testing.T) {
input := &Input{
Query: "find active users",
ModelIDs: []string{"user"},
Limit: 15,
}
result, err := gen.Generate(ctx, input)
assert.NoError(t, err)
assert.NotNil(t, result)
assert.NotNil(t, result.DSL)
// Verify fixed structure is correctly parsed
assert.Len(t, result.DSL.Select, 3)
assert.Len(t, result.DSL.Wheres, 1)
assert.Len(t, result.DSL.Orders, 1)
assert.Equal(t, float64(15), result.DSL.Limit)
assert.Contains(t, result.Explain, "find active users")
})
t.Run("allowed_fields_validation", func(t *testing.T) {
input := &Input{
Query: "find users",
ModelIDs: []string{"user"},
AllowedFields: []string{"id", "name"}, // Only allow id and name
Limit: 10,
}
result, err := gen.Generate(ctx, input)
assert.NoError(t, err)
assert.NotNil(t, result)
assert.NotNil(t, result.DSL)
// "status" field should be filtered out from select and wheres
// since it's not in AllowedFields
for _, expr := range result.DSL.Select {
assert.Contains(t, []string{"id", "name"}, expr.Field)
}
// Should have warning about removed fields
assert.NotEmpty(t, result.Warnings)
})
}
func TestMCPProvider_Generate_WithRetry(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
ctx := newTestContext()
// Create MCP provider for search.generate_querydsl_with_retry
// This tool returns invalid DSL on first call, valid on second
provider, err := NewMCPProvider("search.generate_querydsl_with_retry")
assert.NoError(t, err)
assert.NotNil(t, provider)
t.Run("retry_on_lint_failure", func(t *testing.T) {
input := &Input{
Query: "test retry mechanism",
ModelIDs: []string{"user"},
Limit: 10,
}
// This should succeed after retry
// First call returns invalid DSL (missing 'from')
// Second call (with lint_errors) returns valid DSL
result, err := provider.Generate(ctx, input)
assert.NoError(t, err)
assert.NotNil(t, result)
if result != nil && result.DSL != nil {
// Should have valid DSL after retry
assert.NotNil(t, result.DSL.From, "DSL should have 'from' field after retry")
// Explain should indicate this was fixed after receiving lint errors
assert.Contains(t, result.Explain, "fixed after receiving lint errors")
}
})
}

View file

@ -0,0 +1,23 @@
package querydsl
import (
"github.com/yaoapp/gou/query/gou"
)
// Input contains all information needed to generate QueryDSL
type Input struct {
Query string // Natural language query
ModelIDs []string // Target model IDs (e.g., ["user", "order", "product"])
Wheres []gou.Where // Pre-defined filters (optional)
Orders gou.Orders // Sort orders (optional)
AllowedFields []string // Allowed fields whitelist (optional, for security validation)
Limit int // Max results
ExtraParams map[string]interface{} // Additional parameters
}
// Result represents the result of QueryDSL generation
type Result struct {
DSL *gou.QueryDSL `json:"dsl"` // Generated QueryDSL (supports joins)
Explain string `json:"explain,omitempty"` // Human-readable explanation
Warnings []string `json:"warnings,omitempty"` // Any warnings during generation
}

View file

@ -16,6 +16,13 @@ type DSL struct {
StoreSetting store.Setting `json:"store" yaml:"store"` // The store setting of the assistant
Cache string `json:"cache" yaml:"cache"` // The cache store of the assistant, if not set, default is "__yao.agent.cache"
// System Agents Connector Settings
// ===============================
// System configures connectors for system agents (__yao.keyword, __yao.querydsl, __yao.title, __yao.prompt)
// Each agent can have its own connector, or use the default
// If not set, fallback to the first connector that supports the required capabilities
System *System `json:"system,omitempty" yaml:"system,omitempty"`
// Global External Settings - model capabilities, tools, etc.
// ===============================
Models map[string]openai.Capabilities `json:"models,omitempty" yaml:"models,omitempty"` // The model capabilities configuration
@ -48,6 +55,18 @@ type Uses struct {
Rerank string `json:"rerank,omitempty" yaml:"rerank,omitempty"` // Result reranking: "builtin", "<assistant-id>", "mcp:<server>.<tool>"
}
// System configures connectors for system agents
// ===============================
type System struct {
Default string `json:"default,omitempty" yaml:"default,omitempty"` // Default connector for all system agents
Keyword string `json:"keyword,omitempty" yaml:"keyword,omitempty"` // Connector for __yao.keyword agent
QueryDSL string `json:"querydsl,omitempty" yaml:"querydsl,omitempty"` // Connector for __yao.querydsl agent
Title string `json:"title,omitempty" yaml:"title,omitempty"` // Connector for __yao.title agent
Prompt string `json:"prompt,omitempty" yaml:"prompt,omitempty"` // Connector for __yao.prompt agent
NeedSearch string `json:"needsearch,omitempty" yaml:"needsearch,omitempty"` // Connector for __yao.needsearch agent
Entity string `json:"entity,omitempty" yaml:"entity,omitempty"` // Connector for __yao.entity agent
}
// Mention Structure
// ===============================
type Mention struct {

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,11 @@
{
"name": "Entity Extractor",
"description": "Extract entities and relationships",
"type": "worker",
"uses": { "search": "disabled" },
"options": {
"max_tokens": 2000,
"temperature": 0.2
}
}

View file

@ -0,0 +1,28 @@
- role: system
content: |
Extract entities and relationships from text for knowledge graph construction.
## Task
1. Identify named entities (Person, Organization, Location, Product, Event, Concept, etc.)
2. Extract relationships between entities
3. Return structured JSON
## Response Format (JSON only)
```json
{
"entities": [
{"id": "e1", "name": "Entity Name", "type": "Person|Org|Location|Product|Event|Concept", "properties": {}}
],
"relationships": [
{"source": "e1", "target": "e2", "type": "relationship_type", "properties": {}}
]
}
```
## Guidelines
- Use consistent entity IDs (e1, e2, ...)
- Normalize entity names (remove titles, standardize format)
- Common relationship types: WORKS_FOR, LOCATED_IN, OWNS, CREATED, RELATED_TO
- Keep properties minimal and relevant
- Same language as input for entity names

View file

@ -0,0 +1,10 @@
{
"name": "Keyword Extractor",
"description": "Extract search keywords",
"type": "worker",
"uses": { "search": "disabled" },
"options": {
"max_tokens": 500,
"temperature": 0.3
}
}

View file

@ -0,0 +1,21 @@
- role: system
content: |
Extract keywords from text content.
Task:
1. Analyze input text
2. Extract important keywords
3. Return JSON format
4. Match input language
Response Format (JSON only):
```json
{"keywords": ["keyword1", "keyword2", ...]}
```
Guidelines:
- Extract 5-15 keywords based on content length
- Prioritize nouns, proper nouns, key concepts
- Include single words and short phrases
- Exclude common stop words
- Keywords MUST be in the same language as input

View file

@ -0,0 +1,113 @@
/**
* Keyword Extraction Agent - Next Hook
* Parses LLM response and extracts keywords with error tolerance
*/
// @ts-nocheck
/**
* Next hook - processes keyword extraction response
* Uses json.Parse for fault-tolerant JSON parsing
*/
function Next(
ctx: agent.Context,
payload: agent.NextHookPayload
): agent.NextHookResponse | null {
const completion = payload.completion;
// No completion, return null for standard handling
if (!completion || !completion.content) {
return null;
}
// Remove markdown code block if present
let content = completion.content.trim();
if (content.startsWith("```json")) {
content = content.slice(7);
} else if (content.startsWith("```")) {
content = content.slice(3);
}
if (content.endsWith("```")) {
content = content.slice(0, -3);
}
content = content.trim();
// Try to parse JSON from completion content
let keywords: string[] = [];
try {
// Use json.Parse for fault-tolerant parsing (handles broken JSON, JSONC, etc.)
const parsed = Process("json.Parse", content) as {
keywords?: string[];
} | null;
if (parsed && Array.isArray(parsed.keywords)) {
keywords = parsed.keywords.filter(
(k) => typeof k === "string" && k.trim().length > 0
);
}
} catch (e) {
// If json.Parse fails, try to extract keywords from text
keywords = extractKeywordsFromText(content);
}
// If still no keywords, try extracting from raw text
if (keywords.length === 0) {
keywords = extractKeywordsFromText(content);
}
// Return parsed keywords
return {
data: {
keywords: keywords,
},
};
}
/**
* Extract keywords from plain text when JSON parsing fails
* Handles formats like:
* - Comma-separated: "keyword1, keyword2, keyword3"
* - Line-separated: "keyword1\nkeyword2\nkeyword3"
* - Bullet points: "- keyword1\n- keyword2"
* - Numbered: "1. keyword1\n2. keyword2"
*/
function extractKeywordsFromText(text: string): string[] {
const keywords: string[] = [];
// Remove common prefixes/suffixes
let cleaned = text
.replace(/^[\s\S]*?keywords?[\s:]*\[?/i, "") // Remove "keywords:" prefix
.replace(/\][\s\S]*$/, "") // Remove trailing ]
.trim();
// Try line-by-line extraction
const lines = cleaned.split(/[\n\r]+/);
for (const line of lines) {
// Remove bullet points, numbers, quotes
let keyword = line
.replace(/^[\s\-\*\•\d\.]+/, "") // Remove bullets/numbers
.replace(/^["'`]+|["'`]+$/g, "") // Remove quotes
.replace(/,\s*$/, "") // Remove trailing comma
.trim();
// Skip empty or too long
if (keyword.length > 0 && keyword.length < 100) {
// Split by comma if contains multiple
if (keyword.includes(",")) {
const parts = keyword.split(",").map((p) => p.trim());
for (const part of parts) {
if (part.length > 0 && part.length < 100) {
keywords.push(part);
}
}
} else {
keywords.push(keyword);
}
}
}
// Deduplicate
return [...new Set(keywords)];
}

View file

@ -0,0 +1,10 @@
{
"name": "Reference Checker",
"description": "Check if references are needed",
"type": "worker",
"uses": { "search": "disabled" },
"options": {
"max_tokens": 200,
"temperature": 0.1
}
}

View file

@ -0,0 +1,20 @@
# Need Search Agent
- role: system
content: |
Classify if user query needs external search.
## Rules
NO SEARCH: greetings, chitchat, math, code generation, text processing, general knowledge
WEB: real-time data (weather, news, prices), current events, recent info
KB: documentation, how-to, configuration, FAQ
DB: user data, orders, records, business data
## Response (JSON only)
{"need_search": bool, "search_types": ["web"|"kb"|"db"], "confidence": 0-1}
## Examples
"Hello" → {"need_search": false, "search_types": [], "confidence": 0.99}
"Today's weather" → {"need_search": true, "search_types": ["web"], "confidence": 0.95}
"Write a sort function" → {"need_search": false, "search_types": [], "confidence": 0.90}
"How to config DB" → {"need_search": true, "search_types": ["kb"], "confidence": 0.85}
"My orders" → {"need_search": true, "search_types": ["db"], "confidence": 0.95}

View file

@ -0,0 +1,117 @@
/**
* Need Search Agent - Next Hook
* Parses LLM response and extracts search intent with error tolerance
*/
// @ts-nocheck
interface SearchResult {
need_search: boolean;
search_types: string[];
confidence: number;
}
/**
* Next hook - processes search intent response
* Uses json.Parse for fault-tolerant JSON parsing
*/
function Next(
ctx: agent.Context,
payload: agent.NextHookPayload
): agent.NextHookResponse | null {
const completion = payload.completion;
// No completion, return null for standard handling
if (!completion || !completion.content) {
return null;
}
// Remove markdown code block if present
let content = completion.content.trim();
if (content.startsWith("```json")) {
content = content.slice(7); // Remove ```json
} else if (content.startsWith("```")) {
content = content.slice(3); // Remove ```
}
if (content.endsWith("```")) {
content = content.slice(0, -3); // Remove trailing ```
}
content = content.trim();
// Default result
let result: SearchResult = {
need_search: false,
search_types: [],
confidence: 0,
};
try {
// Use json.Parse for fault-tolerant parsing
const parsed = Process("json.Parse", content) as {
need_search?: boolean;
search_types?: string[];
confidence?: number;
} | null;
if (parsed) {
result.need_search = Boolean(parsed.need_search);
result.search_types = Array.isArray(parsed.search_types)
? parsed.search_types.filter(
(t) =>
typeof t === "string" &&
["web", "kb", "db"].includes(t.toLowerCase())
)
: [];
result.confidence =
typeof parsed.confidence === "number"
? Math.min(1, Math.max(0, parsed.confidence))
: 0.5;
}
} catch (e) {
// If json.Parse fails, try to extract from text
result = extractFromText(content);
}
// Return parsed result
return {
data: result,
};
}
/**
* Extract search intent from plain text when JSON parsing fails
*/
function extractFromText(text: string): SearchResult {
const lower = text.toLowerCase();
// Check for explicit indicators
const needSearch =
lower.includes("true") ||
lower.includes("need") ||
lower.includes("search") ||
lower.includes("web") ||
lower.includes("kb") ||
lower.includes("db");
const noSearch =
lower.includes("false") ||
lower.includes("no search") ||
lower.includes("not need");
// Extract search types
const searchTypes: string[] = [];
if (lower.includes("web")) searchTypes.push("web");
if (lower.includes("kb") || lower.includes("knowledge"))
searchTypes.push("kb");
if (lower.includes("db") || lower.includes("database"))
searchTypes.push("db");
// Determine need_search
const need = noSearch ? false : needSearch && searchTypes.length > 0;
return {
need_search: need,
search_types: need ? searchTypes : [],
confidence: 0.5, // Low confidence for text extraction
};
}

View file

@ -0,0 +1,9 @@
{
"name": "Prompt Optimizer",
"description": "Optimize prompts for better results",
"type": "worker",
"uses": { "search": "disabled" },
"options": {
"temperature": 0
}
}

View file

@ -0,0 +1,25 @@
- role: system
content: |
You are a prompt optimization assistant. Transform user requirements into professional prompts.
Process:
1. Extract key information and objectives
2. Reorganize with precise terminology
3. Add context and details
Include:
- Clear goal/task description
- Expected output format
- Quality requirements
- Reference information
Ensure:
- Clear and unambiguous
- Detailed and specific
- Well-structured
- Actionable
Rules:
1. Respond in same language as input
2. Output ONLY the optimized prompt
3. Ready to use as-is

View file

@ -0,0 +1,10 @@
{
"name": "Query Builder",
"description": "Build database queries",
"type": "worker",
"uses": { "search": "disabled" },
"options": {
"max_tokens": 2000,
"temperature": 0.2
}
}

View file

@ -0,0 +1,43 @@
# QueryDSL Generator Agent Prompts
- role: system
content: |
You are a QueryDSL generator. Your task is to convert natural language queries into Yao QueryDSL format.
## QueryDSL Structure
```json
{
"select": ["field1", "field2"],
"from": "table_name",
"wheres": [
{"field": "name", "op": "=", "value": "test"},
{"field": "status", "op": "in", "value": ["active", "pending"]}
],
"orders": [
{"field": "created_at", "sort": "desc"}
],
"limit": 20
}
```
## Supported Operators
- Comparison: =, !=, >, >=, <, <=
- Pattern: like, not like
- Range: in, not in, between
- Null check: is null, is not null
## Response Format
Always respond with valid JSON:
```json
{
"dsl": { ... },
"explain": "Brief explanation of the query",
"warnings": ["any warnings or notes"]
}
```
## Guidelines
- Generate valid QueryDSL based on the provided schema
- Use appropriate operators for the query intent
- Include only fields that exist in the schema
- Add helpful explanations for complex queries

View file

@ -0,0 +1,9 @@
{
"name": "Title Generator",
"description": "Generate conversation titles",
"type": "worker",
"uses": { "search": "disabled" },
"options": {
"temperature": 0
}
}

View file

@ -0,0 +1,37 @@
- role: system
content: |
Generate concise, meaningful titles for chat conversations.
Task:
1. Analyze content and identify main topic
2. Create brief, descriptive title
3. Title MUST be in the same language as user input
Output:
- Return ONLY the plain text title
- NO markdown, NO code blocks, NO quotes, NO explanation
- Just the title text itself
Length:
- English: 2-6 words, 15-50 chars
- CJK (Chinese/Japanese/Korean): 2-10 chars
- Mixed: max 50 chars
Style:
- Be specific, avoid generic titles
- Use active voice
- Start with key topic
- Sentence case for English
Examples:
Input: "How to bake cookies?"
Output: Chocolate Chip Cookie Recipe
Input: "请教如何制作曲奇"
Output: 巧克力曲奇制作
Input: "Debug my React component"
Output: React Component Debugging
Input: "帮我调试React组件"
Output: React组件调试