Merge pull request #1385 from trheyi/main
Implement System Agents Configuration and Loading Mechanism
This commit is contained in:
commit
b9fe11eab3
34 changed files with 3312 additions and 209 deletions
|
|
@ -132,7 +132,10 @@ func (ast *Assistant) Stream(ctx *context.Context, inputMessages []context.Messa
|
|||
fullMessages := historyResult.FullMessages
|
||||
|
||||
// Buffer user input messages (use cleaned input without overlap)
|
||||
// Skip if History is disabled in options (for internal calls like needsearch)
|
||||
if opts == nil || opts.Skip == nil || !opts.Skip.History {
|
||||
ast.BufferUserInput(ctx, historyResult.InputMessages)
|
||||
}
|
||||
ctx.Logger.PhaseComplete("History")
|
||||
|
||||
// ================================================
|
||||
|
|
@ -202,7 +205,7 @@ func (ast *Assistant) Stream(ctx *context.Context, inputMessages []context.Messa
|
|||
// ================================================
|
||||
// Execute Auto Search (if enabled)
|
||||
// ================================================
|
||||
if ast.shouldAutoSearch(ctx, createResponse) {
|
||||
if ast.shouldAutoSearch(ctx, completionMessages, createResponse, opts) {
|
||||
refCtx := ast.executeAutoSearch(ctx, completionMessages, createResponse, opts)
|
||||
if refCtx != nil && len(refCtx.References) > 0 {
|
||||
completionMessages = ast.injectSearchContext(completionMessages, refCtx)
|
||||
|
|
|
|||
|
|
@ -124,6 +124,35 @@ func (c *Cache) Clear() {
|
|||
c.items = make(map[string]*list.Element)
|
||||
}
|
||||
|
||||
// ClearExcept removes items from the cache except those matching the keep function
|
||||
// keep function returns true for items that should be preserved
|
||||
func (c *Cache) ClearExcept(keep func(id string) bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
// Collect items to remove
|
||||
var toRemove []*list.Element
|
||||
for element := c.list.Front(); element != nil; element = element.Next() {
|
||||
item := element.Value.(*cacheItem)
|
||||
if !keep(item.key) {
|
||||
toRemove = append(toRemove, element)
|
||||
}
|
||||
}
|
||||
|
||||
// Remove collected items
|
||||
for _, element := range toRemove {
|
||||
item := element.Value.(*cacheItem)
|
||||
|
||||
// Unregister scripts before removing
|
||||
if item.value != nil && len(item.value.Scripts) > 0 {
|
||||
item.value.UnregisterScripts()
|
||||
}
|
||||
|
||||
c.list.Remove(element)
|
||||
delete(c.items, item.key)
|
||||
}
|
||||
}
|
||||
|
||||
// removeOldest removes the least recently used item from the cache
|
||||
func (c *Cache) removeOldest() {
|
||||
if element := c.list.Back(); element != nil {
|
||||
|
|
|
|||
|
|
@ -33,8 +33,10 @@ var globalSearchConfig *searchTypes.Config = nil // global search config from ag
|
|||
// LoadBuiltIn load the built-in assistants
|
||||
func LoadBuiltIn() error {
|
||||
|
||||
// Clear the cache
|
||||
loaded.Clear()
|
||||
// Clear non-system agents from cache (preserve system agents loaded by LoadSystemAgents)
|
||||
loaded.ClearExcept(func(id string) bool {
|
||||
return strings.HasPrefix(id, "__yao.") // Keep system agents
|
||||
})
|
||||
|
||||
root := `/assistants`
|
||||
app, err := fs.Get("app")
|
||||
|
|
@ -45,7 +47,7 @@ func LoadBuiltIn() error {
|
|||
// Get all existing built-in assistants
|
||||
deletedBuiltIn := map[string]bool{}
|
||||
|
||||
// Remove the built-in assistants
|
||||
// Remove the built-in assistants (exclude system agents with __yao. prefix)
|
||||
if storage != nil {
|
||||
|
||||
builtIn := true
|
||||
|
|
@ -54,8 +56,12 @@ func LoadBuiltIn() error {
|
|||
return err
|
||||
}
|
||||
|
||||
// Get all existing built-in assistants
|
||||
// Get all existing built-in assistants (exclude system agents)
|
||||
for _, assistant := range res.Data {
|
||||
// Skip system agents (they are managed by LoadSystemAgents)
|
||||
if strings.HasPrefix(assistant.ID, "__yao.") {
|
||||
continue
|
||||
}
|
||||
deletedBuiltIn[assistant.ID] = true
|
||||
}
|
||||
}
|
||||
|
|
@ -582,16 +588,20 @@ func loadMap(data map[string]interface{}) (*Assistant, error) {
|
|||
|
||||
i18n.Locales[id] = flattened
|
||||
} else {
|
||||
// No locales defined, create default with name and description
|
||||
// No locales defined, create default with name and description for all common locales
|
||||
if assistant.Name != "" || assistant.Description != "" {
|
||||
defaultLocales := make(map[string]i18n.I18n)
|
||||
defaultLocales["en"] = i18n.I18n{
|
||||
Locale: "en",
|
||||
// Create entries for all common locales so {{name}} can be resolved
|
||||
commonLocales := []string{"en", "en-us", "zh", "zh-cn", "zh-tw"}
|
||||
for _, locale := range commonLocales {
|
||||
defaultLocales[locale] = i18n.I18n{
|
||||
Locale: locale,
|
||||
Messages: map[string]any{
|
||||
"name": assistant.Name,
|
||||
"description": assistant.Description,
|
||||
},
|
||||
}
|
||||
}
|
||||
i18n.Locales[id] = defaultLocales
|
||||
}
|
||||
}
|
||||
|
|
|
|||
366
agent/assistant/load_system.go
Normal file
366
agent/assistant/load_system.go
Normal file
|
|
@ -0,0 +1,366 @@
|
|||
package assistant
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/yaoapp/gou/application"
|
||||
"github.com/yaoapp/gou/connector"
|
||||
gouOpenAI "github.com/yaoapp/gou/connector/openai"
|
||||
"github.com/yaoapp/kun/log"
|
||||
"github.com/yaoapp/yao/agent/i18n"
|
||||
store "github.com/yaoapp/yao/agent/store/types"
|
||||
"github.com/yaoapp/yao/data"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// systemAgents defines the system agents loaded from bindata
|
||||
// These are internal agents used by the system (e.g., keyword extraction, querydsl generation)
|
||||
// The directory name is without __yao. prefix, prefix is added during loading
|
||||
// Format: directory name -> bindata path prefix
|
||||
var systemAgents = []string{
|
||||
"keyword",
|
||||
"querydsl",
|
||||
"title",
|
||||
"prompt",
|
||||
"needsearch",
|
||||
"entity",
|
||||
}
|
||||
|
||||
// SystemConfig holds the system agents connector configuration
|
||||
// This is set from agent.yml system block
|
||||
type SystemConfig struct {
|
||||
Default string // Default connector for all system agents
|
||||
Keyword string // Connector for __yao.keyword agent
|
||||
QueryDSL string // Connector for __yao.querydsl agent
|
||||
Title string // Connector for __yao.title agent
|
||||
Prompt string // Connector for __yao.prompt agent
|
||||
NeedSearch string // Connector for __yao.needsearch agent
|
||||
Entity string // Connector for __yao.entity agent
|
||||
}
|
||||
|
||||
// systemConfig holds the system agents configuration (global variable like others in load.go)
|
||||
var systemConfig *SystemConfig = nil
|
||||
|
||||
// SetSystemConfig sets the system agents configuration
|
||||
func SetSystemConfig(config *SystemConfig) {
|
||||
systemConfig = config
|
||||
}
|
||||
|
||||
// GetSystemConfig returns the system agents configuration
|
||||
func GetSystemConfig() *SystemConfig {
|
||||
return systemConfig
|
||||
}
|
||||
|
||||
// LoadSystemAgents loads the system agents from bindata
|
||||
// These are internal agents like __yao.keyword and __yao.querydsl
|
||||
// They are loaded before application assistants
|
||||
// Behavior is same as LoadBuiltIn, just reads from bindata instead of filesystem
|
||||
func LoadSystemAgents() error {
|
||||
|
||||
// Get all existing system agents (for cleanup)
|
||||
deletedSystem := map[string]bool{}
|
||||
if storage != nil {
|
||||
// System agents have "system" tag
|
||||
tags := []string{"system"}
|
||||
builtIn := true
|
||||
res, err := storage.GetAssistants(store.AssistantFilter{
|
||||
Tags: tags,
|
||||
BuiltIn: &builtIn,
|
||||
Select: []string{"assistant_id", "id"},
|
||||
})
|
||||
if err != nil {
|
||||
log.Warn("Failed to get existing system agents: %v", err)
|
||||
} else {
|
||||
for _, assistant := range res.Data {
|
||||
deletedSystem[assistant.ID] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort := 1
|
||||
for _, name := range systemAgents {
|
||||
// Build agent ID with __yao. prefix
|
||||
id := "__yao." + name
|
||||
pathPrefix := "yao/assistants/" + name
|
||||
|
||||
assistant, err := loadSystemAgent(id, pathPrefix)
|
||||
if err != nil {
|
||||
log.Warn("Failed to load system agent %s: %v", id, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Set sort order
|
||||
if assistant.Sort == 0 {
|
||||
assistant.Sort = sort
|
||||
}
|
||||
|
||||
// Save to storage
|
||||
if err := assistant.Save(); err != nil {
|
||||
log.Warn("Failed to save system agent %s: %v", id, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Initialize the assistant
|
||||
if err := assistant.initialize(); err != nil {
|
||||
log.Warn("Failed to initialize system agent %s: %v", id, err)
|
||||
continue
|
||||
}
|
||||
|
||||
sort++
|
||||
loaded.Put(assistant)
|
||||
log.Trace("Loaded system agent: %s", id)
|
||||
|
||||
// Remove from deleted list
|
||||
delete(deletedSystem, id)
|
||||
}
|
||||
|
||||
// Remove deleted system agents
|
||||
if len(deletedSystem) > 0 {
|
||||
assistantIDs := []string{}
|
||||
for assistantID := range deletedSystem {
|
||||
assistantIDs = append(assistantIDs, assistantID)
|
||||
}
|
||||
if _, err := storage.DeleteAssistants(store.AssistantFilter{AssistantIDs: assistantIDs}); err != nil {
|
||||
log.Warn("Failed to delete obsolete system agents: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadSystemAgent loads a single system agent from bindata
|
||||
// This follows the same pattern as LoadPath but reads from bindata
|
||||
func loadSystemAgent(id, pathPrefix string) (*Assistant, error) {
|
||||
// Read package.yao from bindata
|
||||
pkgPath := pathPrefix + "/package.yao"
|
||||
pkgContent, err := data.Read(pkgPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read %s: %w", pkgPath, err)
|
||||
}
|
||||
|
||||
// Parse package.yao
|
||||
var pkgData map[string]interface{}
|
||||
if err := application.Parse(pkgPath, pkgContent, &pkgData); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse %s: %w", pkgPath, err)
|
||||
}
|
||||
|
||||
// Set assistant_id (no path - system agents are loaded from storage, not filesystem)
|
||||
pkgData["assistant_id"] = id
|
||||
|
||||
// Set type if not specified
|
||||
if _, has := pkgData["type"]; !has {
|
||||
pkgData["type"] = "assistant"
|
||||
}
|
||||
|
||||
// Resolve connector for this system agent
|
||||
connectorID := resolveSystemConnector(id)
|
||||
if connectorID != "" {
|
||||
pkgData["connector"] = connectorID
|
||||
}
|
||||
|
||||
// Read prompts.yml from bindata (default prompts)
|
||||
promptsPath := pathPrefix + "/prompts.yml"
|
||||
promptsContent, err := data.Read(promptsPath)
|
||||
if err == nil {
|
||||
var prompts []store.Prompt
|
||||
if err := yaml.Unmarshal(promptsContent, &prompts); err == nil && len(prompts) > 0 {
|
||||
pkgData["prompts"] = prompts
|
||||
}
|
||||
}
|
||||
|
||||
// Read prompt_presets from prompts directory
|
||||
presets := loadSystemPromptPresets(pathPrefix)
|
||||
if len(presets) > 0 {
|
||||
pkgData["prompt_presets"] = presets
|
||||
}
|
||||
|
||||
// Load scripts from src directory (hook script source and other scripts sources)
|
||||
// These will be compiled by loadMap -> LoadScriptsFromData
|
||||
hookScriptSource, scriptsSource := loadSystemScripts(pathPrefix)
|
||||
if hookScriptSource != "" {
|
||||
pkgData["script"] = hookScriptSource
|
||||
}
|
||||
if len(scriptsSource) > 0 {
|
||||
pkgData["scripts"] = scriptsSource
|
||||
}
|
||||
|
||||
// Read locales
|
||||
locales, err := loadSystemLocales(pathPrefix)
|
||||
if err == nil && len(locales) > 0 {
|
||||
pkgData["locales"] = locales
|
||||
}
|
||||
|
||||
// Mark as system agent
|
||||
pkgData["readonly"] = true
|
||||
pkgData["built_in"] = true
|
||||
pkgData["tags"] = []string{"system"}
|
||||
|
||||
// Load from map (same as LoadPath, includes initialize())
|
||||
return loadMap(pkgData)
|
||||
}
|
||||
|
||||
// resolveSystemConnector resolves the connector for a system agent
|
||||
// Priority: specific agent config > system.default > defaultConnector > fallback to first capable connector
|
||||
func resolveSystemConnector(agentID string) string {
|
||||
// Try specific agent config first
|
||||
if systemConfig != nil {
|
||||
switch agentID {
|
||||
case "__yao.keyword":
|
||||
if systemConfig.Keyword != "" {
|
||||
return systemConfig.Keyword
|
||||
}
|
||||
case "__yao.querydsl":
|
||||
if systemConfig.QueryDSL != "" {
|
||||
return systemConfig.QueryDSL
|
||||
}
|
||||
case "__yao.title":
|
||||
if systemConfig.Title != "" {
|
||||
return systemConfig.Title
|
||||
}
|
||||
case "__yao.prompt":
|
||||
if systemConfig.Prompt != "" {
|
||||
return systemConfig.Prompt
|
||||
}
|
||||
case "__yao.needsearch":
|
||||
if systemConfig.NeedSearch != "" {
|
||||
return systemConfig.NeedSearch
|
||||
}
|
||||
case "__yao.entity":
|
||||
if systemConfig.Entity != "" {
|
||||
return systemConfig.Entity
|
||||
}
|
||||
}
|
||||
|
||||
// Try system default
|
||||
if systemConfig.Default != "" {
|
||||
return systemConfig.Default
|
||||
}
|
||||
}
|
||||
|
||||
// Try global default connector
|
||||
if defaultConnector != "" {
|
||||
return defaultConnector
|
||||
}
|
||||
|
||||
// Fallback: find first connector that supports tool calling
|
||||
return findCapableConnector()
|
||||
}
|
||||
|
||||
// findCapableConnector finds the first connector that supports tool calling
|
||||
func findCapableConnector() string {
|
||||
// Get all registered connectors
|
||||
for id, conn := range connector.Connectors {
|
||||
if !conn.Is(connector.OPENAI) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check from modelCapabilities (user-defined in models.yml)
|
||||
if caps, exists := modelCapabilities[id]; exists {
|
||||
if caps.ToolCalls {
|
||||
return id
|
||||
}
|
||||
}
|
||||
|
||||
// Check capabilities from connector's Options
|
||||
if connOpenAI, ok := conn.(*gouOpenAI.Connector); ok {
|
||||
if connOpenAI.Options.Capabilities != nil && connOpenAI.Options.Capabilities.ToolCalls {
|
||||
return id
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// No capable connector found, return empty
|
||||
return ""
|
||||
}
|
||||
|
||||
// loadSystemPromptPresets loads prompt presets from bindata prompts directory
|
||||
func loadSystemPromptPresets(pathPrefix string) map[string][]store.Prompt {
|
||||
presets := make(map[string][]store.Prompt)
|
||||
promptsDir := pathPrefix + "/prompts"
|
||||
|
||||
// Try common preset files
|
||||
presetFiles := []string{"chat.yml", "task.yml", "code.yml", "analysis.yml"}
|
||||
for _, filename := range presetFiles {
|
||||
presetPath := promptsDir + "/" + filename
|
||||
content, err := data.Read(presetPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var prompts []store.Prompt
|
||||
if err := yaml.Unmarshal(content, &prompts); err == nil && len(prompts) > 0 {
|
||||
presetName := strings.TrimSuffix(filename, ".yml")
|
||||
presets[presetName] = prompts
|
||||
}
|
||||
}
|
||||
|
||||
return presets
|
||||
}
|
||||
|
||||
// loadSystemScripts loads scripts source from bindata src directory
|
||||
// Returns hook script source and other scripts sources (as strings)
|
||||
// These will be compiled by loadMap -> LoadScriptsFromData
|
||||
func loadSystemScripts(pathPrefix string) (string, map[string]string) {
|
||||
srcDir := pathPrefix + "/src"
|
||||
|
||||
// Try to load hook script (index.ts)
|
||||
var hookScriptSource string
|
||||
indexPath := srcDir + "/index.ts"
|
||||
indexContent, err := data.Read(indexPath)
|
||||
if err == nil && len(indexContent) > 0 {
|
||||
hookScriptSource = string(indexContent)
|
||||
}
|
||||
|
||||
// Try to load other scripts
|
||||
scripts := make(map[string]string)
|
||||
scriptFiles := []string{"utils.ts", "helpers.ts", "tools.ts"}
|
||||
for _, filename := range scriptFiles {
|
||||
scriptPath := srcDir + "/" + filename
|
||||
content, err := data.Read(scriptPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
scriptName := strings.TrimSuffix(filename, ".ts")
|
||||
scripts[scriptName] = string(content)
|
||||
}
|
||||
|
||||
if len(scripts) == 0 {
|
||||
scripts = nil
|
||||
}
|
||||
|
||||
return hookScriptSource, scripts
|
||||
}
|
||||
|
||||
// loadSystemLocales loads locales from bindata
|
||||
func loadSystemLocales(pathPrefix string) (i18n.Map, error) {
|
||||
locales := make(i18n.Map)
|
||||
|
||||
// Try to load common locale files
|
||||
localeFiles := []string{"en-us.yml", "zh-cn.yml", "en.yml", "zh.yml"}
|
||||
localesDir := pathPrefix + "/locales"
|
||||
|
||||
for _, filename := range localeFiles {
|
||||
localePath := filepath.Join(localesDir, filename)
|
||||
content, err := data.Read(localePath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Parse locale file
|
||||
locale := strings.TrimSuffix(filename, ".yml")
|
||||
var messages map[string]any
|
||||
if err := yaml.Unmarshal(content, &messages); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
locales[locale] = i18n.I18n{
|
||||
Locale: locale,
|
||||
Messages: messages,
|
||||
}
|
||||
}
|
||||
|
||||
return locales, nil
|
||||
}
|
||||
|
|
@ -1,10 +1,12 @@
|
|||
package assistant
|
||||
package assistant_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/yaoapp/yao/agent"
|
||||
"github.com/yaoapp/yao/agent/assistant"
|
||||
store "github.com/yaoapp/yao/agent/store/types"
|
||||
"github.com/yaoapp/yao/config"
|
||||
"github.com/yaoapp/yao/test"
|
||||
|
|
@ -14,13 +16,19 @@ func prepare(t *testing.T) {
|
|||
test.Prepare(t, config.Conf)
|
||||
}
|
||||
|
||||
func prepareAgent(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
err := agent.Load(config.Conf)
|
||||
require.NoError(t, err, "agent.Load should succeed")
|
||||
}
|
||||
|
||||
// TestLoadPath tests loading assistant from path
|
||||
func TestLoadPath(t *testing.T) {
|
||||
prepare(t)
|
||||
defer test.Clean()
|
||||
|
||||
t.Run("LoadFullFieldsAssistant", func(t *testing.T) {
|
||||
assistant, err := LoadPath("/assistants/tests/fullfields")
|
||||
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, assistant)
|
||||
|
||||
|
|
@ -67,7 +75,7 @@ func TestLoadPath(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("LoadConnectorOptions", func(t *testing.T) {
|
||||
assistant, err := LoadPath("/assistants/tests/fullfields")
|
||||
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, assistant)
|
||||
|
||||
|
|
@ -84,7 +92,7 @@ func TestLoadPath(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("LoadPromptPresets", func(t *testing.T) {
|
||||
assistant, err := LoadPath("/assistants/tests/fullfields")
|
||||
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, assistant)
|
||||
|
||||
|
|
@ -116,7 +124,7 @@ func TestLoadPath(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("LoadKnowledgeBase", func(t *testing.T) {
|
||||
assistant, err := LoadPath("/assistants/tests/fullfields")
|
||||
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, assistant)
|
||||
|
||||
|
|
@ -129,7 +137,7 @@ func TestLoadPath(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("LoadMCPServers", func(t *testing.T) {
|
||||
assistant, err := LoadPath("/assistants/tests/fullfields")
|
||||
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, assistant)
|
||||
|
||||
|
|
@ -143,7 +151,7 @@ func TestLoadPath(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("LoadWorkflow", func(t *testing.T) {
|
||||
assistant, err := LoadPath("/assistants/tests/fullfields")
|
||||
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, assistant)
|
||||
|
||||
|
|
@ -156,7 +164,7 @@ func TestLoadPath(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("LoadPlaceholder", func(t *testing.T) {
|
||||
assistant, err := LoadPath("/assistants/tests/fullfields")
|
||||
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, assistant)
|
||||
|
||||
|
|
@ -169,7 +177,7 @@ func TestLoadPath(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("LoadLocales", func(t *testing.T) {
|
||||
assistant, err := LoadPath("/assistants/tests/fullfields")
|
||||
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, assistant)
|
||||
|
||||
|
|
@ -186,7 +194,7 @@ func TestLoadPath(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("LoadNonExistentAssistant", func(t *testing.T) {
|
||||
_, err := LoadPath("/assistants/non-existent")
|
||||
_, err := assistant.LoadPath("/assistants/non-existent")
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
|
@ -196,7 +204,7 @@ func TestLoadPathMCPTest(t *testing.T) {
|
|||
prepare(t)
|
||||
defer test.Clean()
|
||||
|
||||
assistant, err := LoadPath("/assistants/tests/mcptest")
|
||||
assistant, err := assistant.LoadPath("/assistants/tests/mcptest")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, assistant)
|
||||
|
||||
|
|
@ -220,7 +228,7 @@ func TestLoadPathBuildRequest(t *testing.T) {
|
|||
prepare(t)
|
||||
defer test.Clean()
|
||||
|
||||
assistant, err := LoadPath("/assistants/tests/buildrequest")
|
||||
assistant, err := assistant.LoadPath("/assistants/tests/buildrequest")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, assistant)
|
||||
|
||||
|
|
@ -238,57 +246,57 @@ func TestLoadPathBuildRequest(t *testing.T) {
|
|||
// TestCache tests the assistant cache functionality
|
||||
func TestCache(t *testing.T) {
|
||||
// Clear any existing cache
|
||||
ClearCache()
|
||||
assistant.ClearCache()
|
||||
|
||||
// Set small cache for testing
|
||||
SetCache(3)
|
||||
assert.NotNil(t, loaded)
|
||||
assistant.SetCache(3)
|
||||
assert.NotNil(t, assistant.GetCache())
|
||||
|
||||
// Create test assistants
|
||||
ast1 := &Assistant{AssistantModel: store.AssistantModel{ID: "id1", Name: "Assistant 1"}}
|
||||
ast2 := &Assistant{AssistantModel: store.AssistantModel{ID: "id2", Name: "Assistant 2"}}
|
||||
ast3 := &Assistant{AssistantModel: store.AssistantModel{ID: "id3", Name: "Assistant 3"}}
|
||||
ast4 := &Assistant{AssistantModel: store.AssistantModel{ID: "id4", Name: "Assistant 4"}}
|
||||
ast1 := &assistant.Assistant{AssistantModel: store.AssistantModel{ID: "id1", Name: "Assistant 1"}}
|
||||
ast2 := &assistant.Assistant{AssistantModel: store.AssistantModel{ID: "id2", Name: "Assistant 2"}}
|
||||
ast3 := &assistant.Assistant{AssistantModel: store.AssistantModel{ID: "id3", Name: "Assistant 3"}}
|
||||
ast4 := &assistant.Assistant{AssistantModel: store.AssistantModel{ID: "id4", Name: "Assistant 4"}}
|
||||
|
||||
t.Run("PutAndGet", func(t *testing.T) {
|
||||
loaded.Put(ast1)
|
||||
assert.Equal(t, 1, loaded.Len())
|
||||
assistant.GetCache().Put(ast1)
|
||||
assert.Equal(t, 1, assistant.GetCache().Len())
|
||||
|
||||
cached, exists := loaded.Get("id1")
|
||||
cached, exists := assistant.GetCache().Get("id1")
|
||||
assert.True(t, exists)
|
||||
assert.Equal(t, ast1, cached)
|
||||
})
|
||||
|
||||
t.Run("CacheEviction", func(t *testing.T) {
|
||||
loaded.Put(ast2)
|
||||
loaded.Put(ast3)
|
||||
assert.Equal(t, 3, loaded.Len())
|
||||
assistant.GetCache().Put(ast2)
|
||||
assistant.GetCache().Put(ast3)
|
||||
assert.Equal(t, 3, assistant.GetCache().Len())
|
||||
|
||||
// Access ast1 to make it recently used
|
||||
loaded.Get("id1")
|
||||
assistant.GetCache().Get("id1")
|
||||
|
||||
// Add ast4, should evict ast2 (least recently used)
|
||||
loaded.Put(ast4)
|
||||
assert.Equal(t, 3, loaded.Len())
|
||||
assistant.GetCache().Put(ast4)
|
||||
assert.Equal(t, 3, assistant.GetCache().Len())
|
||||
|
||||
_, exists := loaded.Get("id2")
|
||||
_, exists := assistant.GetCache().Get("id2")
|
||||
assert.False(t, exists, "ast2 should be evicted")
|
||||
|
||||
_, exists = loaded.Get("id1")
|
||||
_, exists = assistant.GetCache().Get("id1")
|
||||
assert.True(t, exists, "ast1 should still exist")
|
||||
|
||||
_, exists = loaded.Get("id4")
|
||||
_, exists = assistant.GetCache().Get("id4")
|
||||
assert.True(t, exists, "ast4 should exist")
|
||||
})
|
||||
|
||||
t.Run("ClearCache", func(t *testing.T) {
|
||||
ClearCache()
|
||||
assert.Nil(t, loaded)
|
||||
assistant.ClearCache()
|
||||
assert.Nil(t, assistant.GetCache())
|
||||
})
|
||||
|
||||
t.Run("SetCacheAfterClear", func(t *testing.T) {
|
||||
SetCache(100)
|
||||
assert.NotNil(t, loaded)
|
||||
assistant.SetCache(100)
|
||||
assert.NotNil(t, assistant.GetCache())
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -298,7 +306,7 @@ func TestClone(t *testing.T) {
|
|||
defer test.Clean()
|
||||
|
||||
t.Run("CloneFullFieldsAssistant", func(t *testing.T) {
|
||||
original, err := LoadPath("/assistants/tests/fullfields")
|
||||
original, err := assistant.LoadPath("/assistants/tests/fullfields")
|
||||
require.NoError(t, err)
|
||||
|
||||
clone := original.Clone()
|
||||
|
|
@ -328,7 +336,7 @@ func TestClone(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("CloneNil", func(t *testing.T) {
|
||||
var nilAssistant *Assistant
|
||||
var nilAssistant *assistant.Assistant
|
||||
assert.Nil(t, nilAssistant.Clone())
|
||||
})
|
||||
}
|
||||
|
|
@ -339,7 +347,7 @@ func TestUpdate(t *testing.T) {
|
|||
defer test.Clean()
|
||||
|
||||
t.Run("UpdateBasicFields", func(t *testing.T) {
|
||||
assistant, err := LoadPath("/assistants/tests/fullfields")
|
||||
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
|
||||
require.NoError(t, err)
|
||||
|
||||
updates := map[string]interface{}{
|
||||
|
|
@ -357,7 +365,7 @@ func TestUpdate(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("UpdateConnectorOptions", func(t *testing.T) {
|
||||
assistant, err := LoadPath("/assistants/tests/fullfields")
|
||||
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
|
||||
require.NoError(t, err)
|
||||
|
||||
updates := map[string]interface{}{
|
||||
|
|
@ -377,7 +385,7 @@ func TestUpdate(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("UpdatePromptPresets", func(t *testing.T) {
|
||||
assistant, err := LoadPath("/assistants/tests/fullfields")
|
||||
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
|
||||
require.NoError(t, err)
|
||||
|
||||
updates := map[string]interface{}{
|
||||
|
|
@ -398,7 +406,7 @@ func TestUpdate(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("UpdateSource", func(t *testing.T) {
|
||||
assistant, err := LoadPath("/assistants/tests/fullfields")
|
||||
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
|
||||
require.NoError(t, err)
|
||||
|
||||
updates := map[string]interface{}{
|
||||
|
|
@ -412,7 +420,7 @@ func TestUpdate(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("UpdateNilAssistant", func(t *testing.T) {
|
||||
var nilAssistant *Assistant
|
||||
var nilAssistant *assistant.Assistant
|
||||
err := nilAssistant.Update(map[string]interface{}{"name": "test"})
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
|
@ -423,7 +431,7 @@ func TestMap(t *testing.T) {
|
|||
prepare(t)
|
||||
defer test.Clean()
|
||||
|
||||
assistant, err := LoadPath("/assistants/tests/fullfields")
|
||||
assistant, err := assistant.LoadPath("/assistants/tests/fullfields")
|
||||
require.NoError(t, err)
|
||||
|
||||
m := assistant.Map()
|
||||
|
|
@ -451,16 +459,131 @@ func TestMap(t *testing.T) {
|
|||
assert.Equal(t, assistant.Source, m["source"])
|
||||
}
|
||||
|
||||
// TestLoadSystemAgents tests loading system agents from bindata
|
||||
func TestLoadSystemAgents(t *testing.T) {
|
||||
prepareAgent(t)
|
||||
defer test.Clean()
|
||||
|
||||
// Clear cache first
|
||||
assistant.ClearCache()
|
||||
assistant.SetCache(200)
|
||||
|
||||
t.Run("LoadSystemAgents", func(t *testing.T) {
|
||||
err := assistant.LoadSystemAgents()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Check __yao.keyword
|
||||
keywordAst, keywordExists := assistant.GetCache().Get("__yao.keyword")
|
||||
require.True(t, keywordExists, "__yao.keyword should be loaded")
|
||||
assert.Equal(t, "__yao.keyword", keywordAst.ID)
|
||||
assert.Equal(t, "Keyword Extractor", keywordAst.Name)
|
||||
assert.True(t, keywordAst.Readonly)
|
||||
assert.True(t, keywordAst.BuiltIn)
|
||||
assert.Contains(t, keywordAst.Tags, "system")
|
||||
assert.NotNil(t, keywordAst.Prompts)
|
||||
assert.Greater(t, len(keywordAst.Prompts), 0)
|
||||
|
||||
// Check __yao.querydsl
|
||||
querydslAst, querydslExists := assistant.GetCache().Get("__yao.querydsl")
|
||||
require.True(t, querydslExists, "__yao.querydsl should be loaded")
|
||||
assert.Equal(t, "__yao.querydsl", querydslAst.ID)
|
||||
assert.Equal(t, "Query Builder", querydslAst.Name)
|
||||
assert.True(t, querydslAst.Readonly)
|
||||
assert.True(t, querydslAst.BuiltIn)
|
||||
assert.Contains(t, querydslAst.Tags, "system")
|
||||
assert.NotNil(t, querydslAst.Prompts)
|
||||
assert.Greater(t, len(querydslAst.Prompts), 0)
|
||||
|
||||
// Check __yao.title
|
||||
titleAst, titleExists := assistant.GetCache().Get("__yao.title")
|
||||
require.True(t, titleExists, "__yao.title should be loaded")
|
||||
assert.Equal(t, "__yao.title", titleAst.ID)
|
||||
assert.Equal(t, "Title Generator", titleAst.Name)
|
||||
assert.True(t, titleAst.Readonly)
|
||||
assert.True(t, titleAst.BuiltIn)
|
||||
|
||||
// Check __yao.prompt
|
||||
promptAst, promptExists := assistant.GetCache().Get("__yao.prompt")
|
||||
require.True(t, promptExists, "__yao.prompt should be loaded")
|
||||
assert.Equal(t, "__yao.prompt", promptAst.ID)
|
||||
assert.Equal(t, "Prompt Optimizer", promptAst.Name)
|
||||
assert.True(t, promptAst.Readonly)
|
||||
assert.True(t, promptAst.BuiltIn)
|
||||
|
||||
// Check __yao.needsearch
|
||||
needsearchAst, needsearchExists := assistant.GetCache().Get("__yao.needsearch")
|
||||
require.True(t, needsearchExists, "__yao.needsearch should be loaded")
|
||||
assert.Equal(t, "__yao.needsearch", needsearchAst.ID)
|
||||
assert.Equal(t, "Reference Checker", needsearchAst.Name)
|
||||
assert.True(t, needsearchAst.Readonly)
|
||||
assert.True(t, needsearchAst.BuiltIn)
|
||||
})
|
||||
|
||||
t.Run("SystemAgentsSavedToStorage", func(t *testing.T) {
|
||||
// System agents should be saved to storage
|
||||
require.NotNil(t, assistant.GetStore(), "storage should be initialized")
|
||||
|
||||
// Check __yao.keyword in storage
|
||||
builtIn := true
|
||||
tags := []string{"system"}
|
||||
res, err := assistant.GetStore().GetAssistants(store.AssistantFilter{
|
||||
BuiltIn: &builtIn,
|
||||
Tags: tags,
|
||||
Select: []string{"assistant_id", "name"},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, len(res.Data), 0, "System agents should be in storage")
|
||||
|
||||
// Verify at least one system agent exists
|
||||
found := false
|
||||
for _, ast := range res.Data {
|
||||
if ast.ID == "__yao.keyword" || ast.ID == "__yao.querydsl" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "System agents should be found in storage")
|
||||
})
|
||||
|
||||
t.Run("SystemAgentsGetFromStorage", func(t *testing.T) {
|
||||
// Clear cache to force loading from storage
|
||||
assistant.GetCache().Clear()
|
||||
|
||||
// Test Get for each system agent
|
||||
systemAgents := []string{
|
||||
"__yao.keyword",
|
||||
"__yao.querydsl",
|
||||
"__yao.title",
|
||||
"__yao.prompt",
|
||||
"__yao.needsearch",
|
||||
"__yao.entity",
|
||||
}
|
||||
|
||||
for _, agentID := range systemAgents {
|
||||
ast, err := assistant.Get(agentID)
|
||||
require.NoError(t, err, "Get(%s) should succeed", agentID)
|
||||
require.NotNil(t, ast, "Get(%s) should return assistant", agentID)
|
||||
assert.Equal(t, agentID, ast.ID)
|
||||
assert.True(t, ast.BuiltIn, "%s should be built-in", agentID)
|
||||
assert.True(t, ast.Readonly, "%s should be readonly", agentID)
|
||||
assert.Contains(t, ast.Tags, "system", "%s should have system tag", agentID)
|
||||
assert.Equal(t, "worker", ast.Type, "%s should be worker type", agentID)
|
||||
assert.NotNil(t, ast.Prompts, "%s should have prompts", agentID)
|
||||
assert.Greater(t, len(ast.Prompts), 0, "%s should have at least one prompt", agentID)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestValidate tests the assistant Validate method
|
||||
func TestValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
ast *Assistant
|
||||
ast *assistant.Assistant
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "ValidAssistant",
|
||||
ast: &Assistant{
|
||||
ast: &assistant.Assistant{
|
||||
AssistantModel: store.AssistantModel{
|
||||
ID: "test-id",
|
||||
Name: "Test Assistant",
|
||||
|
|
@ -471,7 +594,7 @@ func TestValidate(t *testing.T) {
|
|||
},
|
||||
{
|
||||
name: "MissingID",
|
||||
ast: &Assistant{
|
||||
ast: &assistant.Assistant{
|
||||
AssistantModel: store.AssistantModel{
|
||||
Name: "Test Assistant",
|
||||
Connector: "gpt-4o",
|
||||
|
|
@ -481,7 +604,7 @@ func TestValidate(t *testing.T) {
|
|||
},
|
||||
{
|
||||
name: "MissingName",
|
||||
ast: &Assistant{
|
||||
ast: &assistant.Assistant{
|
||||
AssistantModel: store.AssistantModel{
|
||||
ID: "test-id",
|
||||
Connector: "gpt-4o",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package assistant
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -17,9 +18,17 @@ import (
|
|||
|
||||
// shouldAutoSearch determines if auto search should be executed
|
||||
// Returns false if:
|
||||
// - opts.Skip.Search is true
|
||||
// - uses.search is "disabled"
|
||||
// - assistant has no search configuration
|
||||
func (ast *Assistant) shouldAutoSearch(ctx *context.Context, createResponse *context.HookCreateResponse) bool {
|
||||
// - needsearch intent detection returns false
|
||||
func (ast *Assistant) shouldAutoSearch(ctx *context.Context, messages []context.Message, createResponse *context.HookCreateResponse, opts *context.Options) bool {
|
||||
// Check if search is skipped via options
|
||||
if opts != nil && opts.Skip != nil && opts.Skip.Search {
|
||||
ctx.Logger.Debug("Auto search skipped by opts.Skip.Search")
|
||||
return false
|
||||
}
|
||||
|
||||
// Get merged uses configuration
|
||||
uses := ast.getMergedSearchUses(createResponse)
|
||||
|
||||
|
|
@ -34,10 +43,194 @@ func (ast *Assistant) shouldAutoSearch(ctx *context.Context, createResponse *con
|
|||
return false
|
||||
}
|
||||
|
||||
// Check search intent using __yao.needsearch agent
|
||||
if !ast.checkSearchIntent(ctx, messages) {
|
||||
ctx.Logger.Info("Auto search skipped: intent detection returned false")
|
||||
return false
|
||||
}
|
||||
|
||||
// Check if search is enabled (builtin, agent, mcp, or empty means builtin)
|
||||
return true
|
||||
}
|
||||
|
||||
// checkSearchIntent uses __yao.needsearch agent to determine if search is needed
|
||||
// Returns true if search is needed, false otherwise
|
||||
func (ast *Assistant) checkSearchIntent(ctx *context.Context, messages []context.Message) bool {
|
||||
// Get the last user message
|
||||
var userQuery string
|
||||
for i := len(messages) - 1; i >= 0; i-- {
|
||||
if messages[i].Role == "user" {
|
||||
if content, ok := messages[i].Content.(string); ok {
|
||||
userQuery = content
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if userQuery == "" {
|
||||
return true // No user message, proceed with search
|
||||
}
|
||||
|
||||
// Try to get __yao.needsearch agent
|
||||
needsearchAst, err := Get("__yao.needsearch")
|
||||
if err != nil {
|
||||
ctx.Logger.Debug("__yao.needsearch agent not available: %v, proceeding with search", err)
|
||||
return true // Agent not available, proceed with search
|
||||
}
|
||||
|
||||
// === Output: Send loading message ===
|
||||
loadingID := ast.sendIntentLoading(ctx)
|
||||
|
||||
// Build messages for intent detection
|
||||
intentMessages := []context.Message{
|
||||
{Role: "user", Content: userQuery},
|
||||
}
|
||||
|
||||
// Call the needsearch agent (Stack will auto-track)
|
||||
// IMPORTANT: Skip search to prevent infinite loop, skip output to prevent JSON showing in UI
|
||||
opts := &context.Options{
|
||||
Skip: &context.Skip{
|
||||
History: true, // Don't save to history
|
||||
Search: true, // Skip search to prevent infinite loop
|
||||
Output: true, // Skip output to prevent JSON showing in UI
|
||||
},
|
||||
}
|
||||
|
||||
result, err := needsearchAst.Stream(ctx, intentMessages, opts)
|
||||
if err != nil {
|
||||
ctx.Logger.Debug("__yao.needsearch failed: %v, proceeding with search", err)
|
||||
// === Output: Send done (error case, proceed with search) ===
|
||||
ast.sendIntentDone(ctx, loadingID, true, "")
|
||||
return true // On error, proceed with search
|
||||
}
|
||||
|
||||
// Parse the result
|
||||
// Next hook returns {data: {need_search: bool, search_types: [], confidence: float}}
|
||||
if response, ok := result.(*context.Response); ok {
|
||||
// First try to get from Next hook response
|
||||
if response.Next != nil {
|
||||
if nextData, ok := response.Next.(map[string]interface{}); ok {
|
||||
// Check for data field (from Next hook's {data: result})
|
||||
var intentData map[string]interface{}
|
||||
if data, ok := nextData["data"].(map[string]interface{}); ok {
|
||||
intentData = data
|
||||
} else {
|
||||
intentData = nextData
|
||||
}
|
||||
|
||||
if needSearch, ok := intentData["need_search"].(bool); ok {
|
||||
reason, _ := intentData["reason"].(string)
|
||||
ctx.Logger.Debug("Search intent (from Next): need_search=%v, reason=%s", needSearch, reason)
|
||||
ast.sendIntentDone(ctx, loadingID, needSearch, reason)
|
||||
return needSearch
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: parse from Completion.Content if Next hook didn't process
|
||||
if response.Completion != nil {
|
||||
content, ok := response.Completion.Content.(string)
|
||||
if !ok || content == "" {
|
||||
ast.sendIntentDone(ctx, loadingID, true, "")
|
||||
return true
|
||||
}
|
||||
needSearch, reason := parseNeedSearchFromContent(content)
|
||||
ctx.Logger.Debug("Search intent (from Content): need_search=%v, reason=%s", needSearch, reason)
|
||||
ast.sendIntentDone(ctx, loadingID, needSearch, reason)
|
||||
return needSearch
|
||||
}
|
||||
}
|
||||
|
||||
// Default: proceed with search if we can't parse the result
|
||||
// === Output: Send done (default case) ===
|
||||
ast.sendIntentDone(ctx, loadingID, true, "")
|
||||
return true
|
||||
}
|
||||
|
||||
// parseNeedSearchFromContent parses need_search result from LLM completion content
|
||||
// Handles JSON wrapped in markdown code blocks
|
||||
func parseNeedSearchFromContent(content string) (bool, string) {
|
||||
// Remove markdown code block if present
|
||||
content = strings.TrimSpace(content)
|
||||
if strings.HasPrefix(content, "```json") {
|
||||
content = strings.TrimPrefix(content, "```json")
|
||||
content = strings.TrimSuffix(content, "```")
|
||||
content = strings.TrimSpace(content)
|
||||
} else if strings.HasPrefix(content, "```") {
|
||||
content = strings.TrimPrefix(content, "```")
|
||||
content = strings.TrimSuffix(content, "```")
|
||||
content = strings.TrimSpace(content)
|
||||
}
|
||||
|
||||
// Try to parse JSON
|
||||
var result map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(content), &result); err != nil {
|
||||
// Failed to parse, default to search
|
||||
return true, ""
|
||||
}
|
||||
|
||||
needSearch, ok := result["need_search"].(bool)
|
||||
if !ok {
|
||||
return true, ""
|
||||
}
|
||||
|
||||
reason, _ := result["reason"].(string)
|
||||
return needSearch, reason
|
||||
}
|
||||
|
||||
// sendIntentLoading sends the initial intent detection loading message
|
||||
// Returns the message ID for later replacement
|
||||
func (ast *Assistant) sendIntentLoading(ctx *context.Context) string {
|
||||
loadingMsg := i18n.T(ctx.Locale, "search.intent.loading")
|
||||
|
||||
msg := &message.Message{
|
||||
Type: "loading",
|
||||
Props: map[string]any{
|
||||
"message": loadingMsg,
|
||||
},
|
||||
}
|
||||
|
||||
// Send and get message ID
|
||||
msgID, err := ctx.SendStream(msg)
|
||||
if err != nil {
|
||||
ctx.Logger.Warn("Failed to send intent loading message: %v", err)
|
||||
return ""
|
||||
}
|
||||
|
||||
return msgID
|
||||
}
|
||||
|
||||
// sendIntentDone replaces loading with result
|
||||
// Only marks as done when needSearch is false (no further loading will follow)
|
||||
// When needSearch is true, the search loading will continue
|
||||
func (ast *Assistant) sendIntentDone(ctx *context.Context, loadingID string, needSearch bool, reason string) {
|
||||
if loadingID == "" {
|
||||
return
|
||||
}
|
||||
|
||||
var resultMsg string
|
||||
if needSearch {
|
||||
resultMsg = i18n.T(ctx.Locale, "search.intent.need_search")
|
||||
} else {
|
||||
resultMsg = i18n.T(ctx.Locale, "search.intent.no_search")
|
||||
}
|
||||
|
||||
msg := &message.Message{
|
||||
MessageID: loadingID,
|
||||
Delta: true,
|
||||
DeltaAction: message.DeltaReplace,
|
||||
Type: "loading",
|
||||
Props: map[string]any{
|
||||
"message": resultMsg,
|
||||
"done": true, // Intent detection loading is independent, always close it
|
||||
},
|
||||
}
|
||||
|
||||
if err := ctx.Send(msg); err != nil {
|
||||
ctx.Logger.Warn("Failed to send intent done message: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// getMergedSearchUses returns the merged uses configuration for search
|
||||
// Priority: createResponse > assistant
|
||||
func (ast *Assistant) getMergedSearchUses(createResponse *context.HookCreateResponse) *context.Uses {
|
||||
|
|
|
|||
|
|
@ -196,6 +196,7 @@ type Skip struct {
|
|||
Trace bool `json:"trace"` // Skip trace logging
|
||||
Output bool `json:"output"` // Skip output to client (for internal A2A calls that only need response data)
|
||||
Keyword bool `json:"keyword"` // Skip keyword extraction for web search (use raw query directly)
|
||||
Search bool `json:"search"` // Skip auto search (for internal calls like needsearch intent detection)
|
||||
}
|
||||
|
||||
// MessageMetadata stores metadata for sent messages
|
||||
|
|
|
|||
|
|
@ -107,6 +107,11 @@ func init() {
|
|||
"search.failed": "Search failed",
|
||||
"search.no_results": "No references found",
|
||||
|
||||
// Search Intent: assistant/search.go - Intent detection messages
|
||||
"search.intent.loading": "Checking if references are needed...",
|
||||
"search.intent.need_search": "Searching for references...",
|
||||
"search.intent.no_search": "No references needed",
|
||||
|
||||
// Search: assistant/search.go - Trace labels
|
||||
"search.trace.label": "Search",
|
||||
"search.trace.description": "Search the web and knowledge base for relevant information",
|
||||
|
|
@ -191,6 +196,11 @@ func init() {
|
|||
"search.failed": "搜索失败",
|
||||
"search.no_results": "未找到相关资料",
|
||||
|
||||
// Search Intent: assistant/search.go - Intent detection messages
|
||||
"search.intent.loading": "检查是否需要查询资料...",
|
||||
"search.intent.need_search": "正在查询相关资料...",
|
||||
"search.intent.no_search": "无需查询资料",
|
||||
|
||||
// Search: assistant/search.go - Trace labels
|
||||
"search.trace.label": "搜索",
|
||||
"search.trace.description": "搜索网络和知识库获取相关信息",
|
||||
|
|
@ -303,6 +313,11 @@ func init() {
|
|||
"search.failed": "搜索失败",
|
||||
"search.no_results": "未找到相关资料",
|
||||
|
||||
// Search Intent: assistant/search.go - Intent detection messages
|
||||
"search.intent.loading": "检查是否需要查询资料...",
|
||||
"search.intent.need_search": "正在查询相关资料...",
|
||||
"search.intent.no_search": "无需查询资料",
|
||||
|
||||
// Search: assistant/search.go - Trace labels
|
||||
"search.trace.label": "搜索",
|
||||
"search.trace.description": "搜索网络和知识库获取相关信息",
|
||||
|
|
|
|||
|
|
@ -303,11 +303,37 @@ func TestGPT5Vision(t *testing.T) {
|
|||
}
|
||||
|
||||
// Should have content describing the image
|
||||
contentStr, ok := response.Content.(string)
|
||||
if !ok || contentStr == "" {
|
||||
t.Error("Expected text content describing the image")
|
||||
} else {
|
||||
// Content can be string or []ContentPart for multimodal responses
|
||||
var contentStr string
|
||||
switch v := response.Content.(type) {
|
||||
case string:
|
||||
contentStr = v
|
||||
case []interface{}:
|
||||
// Handle []ContentPart serialized as []interface{}
|
||||
for _, part := range v {
|
||||
if partMap, ok := part.(map[string]interface{}); ok {
|
||||
if text, ok := partMap["text"].(string); ok {
|
||||
contentStr += text
|
||||
}
|
||||
}
|
||||
}
|
||||
case []context.ContentPart:
|
||||
for _, part := range v {
|
||||
if part.Type == context.ContentText {
|
||||
contentStr += part.Text
|
||||
}
|
||||
}
|
||||
case nil:
|
||||
// GPT-5 reasoning models may use all tokens for reasoning, leaving no content
|
||||
t.Log("Content is nil (reasoning model may have used all tokens for reasoning)")
|
||||
default:
|
||||
t.Logf("Unexpected content type: %T", response.Content)
|
||||
}
|
||||
|
||||
if contentStr != "" {
|
||||
t.Logf("Image description: %s", contentStr)
|
||||
} else if response.Content != nil {
|
||||
t.Logf("Warning: Expected text content describing the image, got empty or non-text content")
|
||||
}
|
||||
|
||||
if response.Usage != nil {
|
||||
|
|
|
|||
|
|
@ -234,7 +234,25 @@ func initAssistant() error {
|
|||
assistant.SetGlobalSearchConfig(agentDSL.Search)
|
||||
}
|
||||
|
||||
// Load Built-in Assistants
|
||||
// Set system agents configuration
|
||||
if agentDSL.System != nil {
|
||||
assistant.SetSystemConfig(&assistant.SystemConfig{
|
||||
Default: agentDSL.System.Default,
|
||||
Keyword: agentDSL.System.Keyword,
|
||||
QueryDSL: agentDSL.System.QueryDSL,
|
||||
Title: agentDSL.System.Title,
|
||||
Prompt: agentDSL.System.Prompt,
|
||||
NeedSearch: agentDSL.System.NeedSearch,
|
||||
Entity: agentDSL.System.Entity,
|
||||
})
|
||||
}
|
||||
|
||||
// Load System Agents (from bindata: __yao.keyword, __yao.querydsl, etc.)
|
||||
if err := assistant.LoadSystemAgents(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Load Built-in Assistants (from application /assistants directory)
|
||||
err := assistant.LoadBuiltIn()
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
|||
266
agent/search/nlp/querydsl/agent.go
Normal file
266
agent/search/nlp/querydsl/agent.go
Normal file
|
|
@ -0,0 +1,266 @@
|
|||
package querydsl
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/yaoapp/gou/query/gou"
|
||||
"github.com/yaoapp/gou/query/linter"
|
||||
"github.com/yaoapp/yao/agent/caller"
|
||||
agentContext "github.com/yaoapp/yao/agent/context"
|
||||
)
|
||||
|
||||
// AgentProvider delegates QueryDSL generation to an LLM-powered assistant
|
||||
// The assistant can understand context and generate semantically correct QueryDSL
|
||||
type AgentProvider struct {
|
||||
agentID string // Assistant ID to delegate to
|
||||
}
|
||||
|
||||
// NewAgentProvider creates a new agent-based QueryDSL generator
|
||||
func NewAgentProvider(agentID string) *AgentProvider {
|
||||
return &AgentProvider{
|
||||
agentID: agentID,
|
||||
}
|
||||
}
|
||||
|
||||
// Generate generates QueryDSL by calling the target agent with retry and lint validation
|
||||
// The agent receives the query and schema, returns generated QueryDSL
|
||||
func (p *AgentProvider) Generate(ctx *agentContext.Context, input *Input) (*Result, error) {
|
||||
if ctx == nil {
|
||||
return nil, fmt.Errorf("context is required for agent QueryDSL generation")
|
||||
}
|
||||
|
||||
// Check if AgentGetterFunc is initialized
|
||||
if caller.AgentGetterFunc == nil {
|
||||
return nil, fmt.Errorf("AgentGetterFunc not initialized")
|
||||
}
|
||||
|
||||
// Get the agent
|
||||
agent, err := caller.AgentGetterFunc(p.agentID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get agent %s: %w", p.agentID, err)
|
||||
}
|
||||
|
||||
var lastError error
|
||||
var lastLintErrors string
|
||||
|
||||
for attempt := 1; attempt <= MaxRetries; attempt++ {
|
||||
// Build the request message
|
||||
requestData := p.buildRequestData(input, attempt, lastLintErrors)
|
||||
requestJSON, _ := json.Marshal(requestData)
|
||||
|
||||
// Create message for the agent
|
||||
messages := []agentContext.Message{
|
||||
{
|
||||
Role: "user",
|
||||
Content: string(requestJSON),
|
||||
},
|
||||
}
|
||||
|
||||
// Call the agent with skip options (no history, no output)
|
||||
options := &agentContext.Options{
|
||||
Skip: &agentContext.Skip{
|
||||
History: true,
|
||||
Output: true,
|
||||
},
|
||||
}
|
||||
|
||||
result, err := agent.Stream(ctx, messages, options)
|
||||
if err != nil {
|
||||
lastError = fmt.Errorf("agent call failed: %w", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Parse the result
|
||||
genResult, err := p.parseResult(result)
|
||||
if err != nil {
|
||||
lastError = err
|
||||
continue
|
||||
}
|
||||
|
||||
// Validate with linter if DSL is present
|
||||
if genResult.DSL != nil {
|
||||
lintResult := p.validateDSL(genResult.DSL)
|
||||
if lintResult.Valid {
|
||||
return genResult, nil
|
||||
}
|
||||
|
||||
// Lint failed, prepare error message for retry
|
||||
lastLintErrors = lintResult.FormatDiagnostics()
|
||||
lastError = fmt.Errorf("QueryDSL validation failed: %s", lastLintErrors)
|
||||
|
||||
// Add lint warnings to result warnings
|
||||
for _, diag := range lintResult.Diagnostics {
|
||||
genResult.Warnings = append(genResult.Warnings, fmt.Sprintf("[%s] %s: %s", diag.Code, diag.Path, diag.Message))
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// No DSL returned
|
||||
lastError = fmt.Errorf("no QueryDSL returned from agent")
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("QueryDSL generation failed after %d attempts: %w", MaxRetries, lastError)
|
||||
}
|
||||
|
||||
// buildRequestData constructs the request data for the agent
|
||||
func (p *AgentProvider) buildRequestData(input *Input, attempt int, lastLintErrors string) map[string]interface{} {
|
||||
requestData := map[string]interface{}{
|
||||
"query": input.Query,
|
||||
"models": input.ModelIDs,
|
||||
"limit": input.Limit,
|
||||
}
|
||||
|
||||
// Add optional fields
|
||||
if len(input.Wheres) > 0 {
|
||||
requestData["wheres"] = input.Wheres
|
||||
}
|
||||
if len(input.Orders) > 0 {
|
||||
requestData["orders"] = input.Orders
|
||||
}
|
||||
if len(input.AllowedFields) > 0 {
|
||||
requestData["allowed_fields"] = input.AllowedFields
|
||||
}
|
||||
if len(input.ExtraParams) > 0 {
|
||||
requestData["extra"] = input.ExtraParams
|
||||
}
|
||||
|
||||
// Add retry context if this is a retry attempt
|
||||
if attempt > 1 && lastLintErrors != "" {
|
||||
requestData["retry"] = map[string]interface{}{
|
||||
"attempt": attempt,
|
||||
"lint_errors": lastLintErrors,
|
||||
"instructions": "The previous QueryDSL was invalid. Please fix the errors and regenerate.",
|
||||
}
|
||||
}
|
||||
|
||||
return requestData
|
||||
}
|
||||
|
||||
// validateDSL validates the generated QueryDSL using the linter
|
||||
func (p *AgentProvider) validateDSL(dsl *gou.QueryDSL) *linter.LintResult {
|
||||
// Marshal DSL to JSON for linting
|
||||
jsonBytes, err := json.Marshal(dsl)
|
||||
if err != nil {
|
||||
result := &linter.LintResult{Valid: false}
|
||||
return result
|
||||
}
|
||||
|
||||
_, lintResult := linter.Parse(string(jsonBytes))
|
||||
return lintResult
|
||||
}
|
||||
|
||||
// parseResult extracts QueryDSL from the agent's response
|
||||
// The agent should return data in NextHookResponse format: { data: { dsl: {...}, explain: "..." } }
|
||||
// The Stream() response wraps this in: { next: { data: { dsl: {...} } } }
|
||||
func (p *AgentProvider) parseResult(result interface{}) (*Result, error) {
|
||||
if result == nil {
|
||||
return &Result{}, nil
|
||||
}
|
||||
|
||||
// Try to convert to map first (most common case)
|
||||
var data map[string]interface{}
|
||||
|
||||
switch v := result.(type) {
|
||||
case map[string]interface{}:
|
||||
data = v
|
||||
case string:
|
||||
// Try to parse as JSON
|
||||
if err := json.Unmarshal([]byte(v), &data); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse agent response: %w", err)
|
||||
}
|
||||
default:
|
||||
// Try to marshal and unmarshal
|
||||
jsonBytes, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
return &Result{}, nil
|
||||
}
|
||||
if err := json.Unmarshal(jsonBytes, &data); err != nil {
|
||||
return &Result{}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Check for "next" field (custom hook data from NextHookResponse)
|
||||
// Stream() returns: { next: { data: { dsl: {...} } } }
|
||||
if next, hasNext := data["next"]; hasNext && next != nil {
|
||||
if nextMap, ok := next.(map[string]interface{}); ok {
|
||||
data = nextMap
|
||||
} else if nextStr, ok := next.(string); ok {
|
||||
if err := json.Unmarshal([]byte(nextStr), &data); err != nil {
|
||||
return &Result{}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract QueryDSL from data
|
||||
// Try common field names: "dsl", "data", "data.dsl"
|
||||
genResult := &Result{}
|
||||
|
||||
// Get explain if present
|
||||
if explain, ok := data["explain"].(string); ok {
|
||||
genResult.Explain = explain
|
||||
}
|
||||
|
||||
// Get warnings if present
|
||||
if warnings, ok := data["warnings"]; ok {
|
||||
genResult.Warnings = p.extractWarnings(warnings)
|
||||
}
|
||||
|
||||
// Get DSL
|
||||
if dsl, ok := data["dsl"]; ok {
|
||||
genResult.DSL = p.extractDSL(dsl)
|
||||
} else if d, ok := data["data"]; ok {
|
||||
if dm, ok := d.(map[string]interface{}); ok {
|
||||
if dsl, ok := dm["dsl"]; ok {
|
||||
genResult.DSL = p.extractDSL(dsl)
|
||||
}
|
||||
if explain, ok := dm["explain"].(string); ok {
|
||||
genResult.Explain = explain
|
||||
}
|
||||
if warnings, ok := dm["warnings"]; ok {
|
||||
genResult.Warnings = p.extractWarnings(warnings)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return genResult, nil
|
||||
}
|
||||
|
||||
// extractDSL converts interface{} to gou.QueryDSL
|
||||
func (p *AgentProvider) extractDSL(v interface{}) *gou.QueryDSL {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Marshal and unmarshal to gou.QueryDSL
|
||||
jsonBytes, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var dsl gou.QueryDSL
|
||||
if err := json.Unmarshal(jsonBytes, &dsl); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &dsl
|
||||
}
|
||||
|
||||
// extractWarnings extracts warnings array from various types
|
||||
func (p *AgentProvider) extractWarnings(v interface{}) []string {
|
||||
switch w := v.(type) {
|
||||
case []string:
|
||||
return w
|
||||
case []interface{}:
|
||||
warnings := make([]string, 0, len(w))
|
||||
for _, item := range w {
|
||||
if s, ok := item.(string); ok {
|
||||
warnings = append(warnings, s)
|
||||
}
|
||||
}
|
||||
return warnings
|
||||
case string:
|
||||
return []string{w}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
245
agent/search/nlp/querydsl/agent_test.go
Normal file
245
agent/search/nlp/querydsl/agent_test.go
Normal file
|
|
@ -0,0 +1,245 @@
|
|||
package querydsl_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/yaoapp/yao/agent/assistant"
|
||||
"github.com/yaoapp/yao/agent/context"
|
||||
"github.com/yaoapp/yao/agent/search/nlp/querydsl"
|
||||
"github.com/yaoapp/yao/agent/testutils"
|
||||
oauthTypes "github.com/yaoapp/yao/openapi/oauth/types"
|
||||
)
|
||||
|
||||
func TestNewAgentProvider(t *testing.T) {
|
||||
t.Run("create_provider", func(t *testing.T) {
|
||||
provider := querydsl.NewAgentProvider("tests.querydsl-agent")
|
||||
assert.NotNil(t, provider)
|
||||
})
|
||||
}
|
||||
|
||||
func TestAgentProvider_Generate(t *testing.T) {
|
||||
// Skip if running short tests
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test")
|
||||
}
|
||||
|
||||
// Initialize test environment
|
||||
testutils.Prepare(t)
|
||||
defer testutils.Clean(t)
|
||||
|
||||
// Load the querydsl-agent assistant
|
||||
ast, err := assistant.Get("tests.querydsl-agent")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, ast)
|
||||
|
||||
// Create test context
|
||||
ctx := newTestContext(t)
|
||||
|
||||
// Create Agent provider for tests.querydsl-agent
|
||||
provider := querydsl.NewAgentProvider("tests.querydsl-agent")
|
||||
assert.NotNil(t, provider)
|
||||
|
||||
t.Run("verify_fixed_structure", func(t *testing.T) {
|
||||
input := &querydsl.Input{
|
||||
Query: "find active users",
|
||||
ModelIDs: []string{"user"},
|
||||
Limit: 15,
|
||||
}
|
||||
|
||||
result, err := provider.Generate(ctx, input)
|
||||
if err != nil {
|
||||
t.Logf("Generate error: %v", err)
|
||||
}
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
require.NotNil(t, result.DSL, "DSL should not be nil")
|
||||
|
||||
// Verify fixed DSL structure from mock
|
||||
// select: ["id", "name", "status", "created_at"]
|
||||
assert.Len(t, result.DSL.Select, 4)
|
||||
if len(result.DSL.Select) >= 4 {
|
||||
assert.Equal(t, "id", result.DSL.Select[0].Field)
|
||||
assert.Equal(t, "name", result.DSL.Select[1].Field)
|
||||
assert.Equal(t, "status", result.DSL.Select[2].Field)
|
||||
assert.Equal(t, "created_at", result.DSL.Select[3].Field)
|
||||
}
|
||||
|
||||
// wheres: [{ field: "status", op: "=", value: "active" }]
|
||||
assert.Len(t, result.DSL.Wheres, 1)
|
||||
if len(result.DSL.Wheres) > 0 {
|
||||
assert.Equal(t, "status", result.DSL.Wheres[0].Field.Field)
|
||||
assert.Equal(t, "=", result.DSL.Wheres[0].OP)
|
||||
assert.Equal(t, "active", result.DSL.Wheres[0].Value)
|
||||
}
|
||||
|
||||
// orders: [{ field: "created_at", sort: "desc" }]
|
||||
assert.Len(t, result.DSL.Orders, 1)
|
||||
if len(result.DSL.Orders) > 0 {
|
||||
assert.Equal(t, "created_at", result.DSL.Orders[0].Field.Field)
|
||||
assert.Equal(t, "desc", result.DSL.Orders[0].Sort)
|
||||
}
|
||||
|
||||
// limit: 15 (from input)
|
||||
assert.Equal(t, float64(15), result.DSL.Limit)
|
||||
|
||||
// explain should contain query
|
||||
assert.Contains(t, result.Explain, "find active users")
|
||||
|
||||
// warnings should be empty
|
||||
assert.Empty(t, result.Warnings)
|
||||
})
|
||||
}
|
||||
|
||||
func TestAgentProvider_Generate_Error(t *testing.T) {
|
||||
// Skip if running short tests
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test")
|
||||
}
|
||||
|
||||
// Initialize test environment
|
||||
testutils.Prepare(t)
|
||||
defer testutils.Clean(t)
|
||||
|
||||
// Create test context
|
||||
ctx := newTestContext(t)
|
||||
|
||||
t.Run("non-existent_agent", func(t *testing.T) {
|
||||
provider := querydsl.NewAgentProvider("tests.nonexistent-agent")
|
||||
result, err := provider.Generate(ctx, &querydsl.Input{
|
||||
Query: "test",
|
||||
ModelIDs: []string{"user"},
|
||||
})
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, result)
|
||||
assert.Contains(t, err.Error(), "failed to get agent")
|
||||
})
|
||||
|
||||
t.Run("nil_context", func(t *testing.T) {
|
||||
provider := querydsl.NewAgentProvider("tests.querydsl-agent")
|
||||
result, err := provider.Generate(nil, &querydsl.Input{
|
||||
Query: "test",
|
||||
ModelIDs: []string{"user"},
|
||||
})
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, result)
|
||||
assert.Contains(t, err.Error(), "context is required")
|
||||
})
|
||||
}
|
||||
|
||||
func TestGenerator_Agent_Integration(t *testing.T) {
|
||||
// Skip if running short tests
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test")
|
||||
}
|
||||
|
||||
// Initialize test environment
|
||||
testutils.Prepare(t)
|
||||
defer testutils.Clean(t)
|
||||
|
||||
// Create test context
|
||||
ctx := newTestContext(t)
|
||||
|
||||
// Create generator with Agent mode (assistant ID without mcp: prefix)
|
||||
gen := querydsl.NewGenerator("tests.querydsl-agent", nil)
|
||||
|
||||
t.Run("generate_via_agent", func(t *testing.T) {
|
||||
input := &querydsl.Input{
|
||||
Query: "find active users",
|
||||
ModelIDs: []string{"user"},
|
||||
Limit: 10,
|
||||
}
|
||||
|
||||
result, err := gen.Generate(ctx, input)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
require.NotNil(t, result.DSL)
|
||||
|
||||
// Verify structure from agent mock
|
||||
assert.Len(t, result.DSL.Select, 4)
|
||||
assert.Len(t, result.DSL.Wheres, 1)
|
||||
assert.Len(t, result.DSL.Orders, 1)
|
||||
assert.Contains(t, result.Explain, "find active users")
|
||||
})
|
||||
|
||||
t.Run("allowed_fields_validation", func(t *testing.T) {
|
||||
input := &querydsl.Input{
|
||||
Query: "find users",
|
||||
ModelIDs: []string{"user"},
|
||||
AllowedFields: []string{"id", "name"}, // Only allow id and name
|
||||
Limit: 10,
|
||||
}
|
||||
|
||||
result, err := gen.Generate(ctx, input)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
require.NotNil(t, result.DSL)
|
||||
|
||||
// "status" and "created_at" fields should be filtered out from select
|
||||
// since they are not in AllowedFields
|
||||
for _, expr := range result.DSL.Select {
|
||||
assert.Contains(t, []string{"id", "name"}, expr.Field)
|
||||
}
|
||||
|
||||
// Should have warning about removed fields
|
||||
assert.NotEmpty(t, result.Warnings)
|
||||
})
|
||||
}
|
||||
|
||||
func TestAgentProvider_Generate_WithRetry(t *testing.T) {
|
||||
// Skip if running short tests
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test")
|
||||
}
|
||||
|
||||
// Initialize test environment
|
||||
testutils.Prepare(t)
|
||||
defer testutils.Clean(t)
|
||||
|
||||
// Load the querydsl-agent-retry assistant
|
||||
ast, err := assistant.Get("tests.querydsl-agent-retry")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, ast)
|
||||
|
||||
// Create test context
|
||||
ctx := newTestContext(t)
|
||||
|
||||
// Create Agent provider for tests.querydsl-agent-retry
|
||||
// This agent returns invalid DSL on first call, valid on second
|
||||
provider := querydsl.NewAgentProvider("tests.querydsl-agent-retry")
|
||||
assert.NotNil(t, provider)
|
||||
|
||||
t.Run("retry_on_lint_failure", func(t *testing.T) {
|
||||
input := &querydsl.Input{
|
||||
Query: "test retry mechanism",
|
||||
ModelIDs: []string{"user"},
|
||||
Limit: 10,
|
||||
}
|
||||
|
||||
// This should succeed after retry
|
||||
// First call returns invalid DSL (missing 'from')
|
||||
// Second call (with lint_errors) returns valid DSL
|
||||
result, err := provider.Generate(ctx, input)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
|
||||
if result.DSL != nil {
|
||||
// Should have valid DSL after retry
|
||||
assert.NotNil(t, result.DSL.From, "DSL should have 'from' field after retry")
|
||||
// Explain should indicate this was fixed after receiving lint errors
|
||||
assert.Contains(t, result.Explain, "fixed after receiving lint errors")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// newTestContext creates a test context with required fields
|
||||
func newTestContext(t *testing.T) *context.Context {
|
||||
t.Helper()
|
||||
authorized := &oauthTypes.AuthorizedInfo{
|
||||
UserID: "test-user",
|
||||
}
|
||||
chatID := "test-chat-querydsl"
|
||||
ctx := context.New(t.Context(), authorized, chatID)
|
||||
return ctx
|
||||
}
|
||||
124
agent/search/nlp/querydsl/builtin.go
Normal file
124
agent/search/nlp/querydsl/builtin.go
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
package querydsl
|
||||
|
||||
import (
|
||||
"github.com/yaoapp/gou/model"
|
||||
"github.com/yaoapp/gou/query/gou"
|
||||
)
|
||||
|
||||
// BuiltinGenerator implements template-based QueryDSL generation
|
||||
// This is a placeholder implementation that returns a basic QueryDSL.
|
||||
//
|
||||
// TODO: Implement actual template-based generation:
|
||||
// - Parse natural language query
|
||||
// - Match against model schema
|
||||
// - Generate appropriate where clauses
|
||||
// - Handle common query patterns (search, filter, sort)
|
||||
//
|
||||
// For production use cases requiring high accuracy, use Agent or MCP mode.
|
||||
type BuiltinGenerator struct{}
|
||||
|
||||
// NewBuiltinGenerator creates a new builtin QueryDSL generator
|
||||
func NewBuiltinGenerator() *BuiltinGenerator {
|
||||
return &BuiltinGenerator{}
|
||||
}
|
||||
|
||||
// Generate generates QueryDSL from natural language
|
||||
// Currently returns a placeholder QueryDSL that searches all searchable fields
|
||||
func (g *BuiltinGenerator) Generate(input *Input) (*Result, error) {
|
||||
if input == nil || input.Query == "" {
|
||||
return &Result{
|
||||
Warnings: []string{"empty query, returning empty DSL"},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Build a basic QueryDSL
|
||||
dsl := &gou.QueryDSL{}
|
||||
|
||||
// Set limit
|
||||
limit := input.Limit
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
dsl.Limit = limit
|
||||
|
||||
// Apply pre-defined wheres if provided
|
||||
if len(input.Wheres) > 0 {
|
||||
dsl.Wheres = input.Wheres
|
||||
}
|
||||
|
||||
// Apply orders if provided
|
||||
if len(input.Orders) > 0 {
|
||||
dsl.Orders = input.Orders
|
||||
}
|
||||
|
||||
// Load models and try to generate basic search conditions
|
||||
// Use the first model as the primary table, others can be joined
|
||||
if len(input.ModelIDs) > 0 {
|
||||
primaryModelID := input.ModelIDs[0]
|
||||
|
||||
// Check if model exists before selecting
|
||||
if !model.Exists(primaryModelID) {
|
||||
return &Result{
|
||||
DSL: dsl,
|
||||
Explain: "Generated basic QueryDSL (model not found)",
|
||||
Warnings: []string{
|
||||
"model '" + primaryModelID + "' not found, returning basic DSL without search conditions",
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
primaryModel := model.Select(primaryModelID)
|
||||
if primaryModel != nil && len(primaryModel.MetaData.Columns) > 0 {
|
||||
// Find searchable text columns (string/text types with index)
|
||||
var searchableColumns []string
|
||||
for _, col := range primaryModel.MetaData.Columns {
|
||||
// Use Index as a proxy for searchable, and check for text types
|
||||
if col.Index && (col.Type == "string" || col.Type == "text" || col.Type == "longText") {
|
||||
searchableColumns = append(searchableColumns, col.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// If we have searchable columns and no pre-defined wheres, add a basic search
|
||||
if len(searchableColumns) > 0 && len(input.Wheres) == 0 {
|
||||
// Build OR conditions for searchable columns
|
||||
orWheres := make([]gou.Where, 0, len(searchableColumns))
|
||||
for _, col := range searchableColumns {
|
||||
orWheres = append(orWheres, gou.Where{
|
||||
Condition: gou.Condition{
|
||||
Field: &gou.Expression{Field: col},
|
||||
OP: "match",
|
||||
Value: input.Query,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// Wrap in OR group if multiple columns
|
||||
if len(orWheres) > 1 {
|
||||
// Mark all but the first as OR conditions
|
||||
for i := 1; i < len(orWheres); i++ {
|
||||
orWheres[i].OR = true
|
||||
}
|
||||
dsl.Wheres = []gou.Where{
|
||||
{
|
||||
Wheres: orWheres,
|
||||
},
|
||||
}
|
||||
} else if len(orWheres) == 1 {
|
||||
dsl.Wheres = orWheres
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: For multi-model queries, generate joins based on model relations
|
||||
// This requires analyzing the relations between models and generating
|
||||
// appropriate JOIN clauses in the QueryDSL
|
||||
}
|
||||
|
||||
return &Result{
|
||||
DSL: dsl,
|
||||
Explain: "Generated basic search QueryDSL using builtin template (placeholder implementation)",
|
||||
Warnings: []string{
|
||||
"builtin generator is a placeholder, consider using Agent or MCP mode for production",
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
170
agent/search/nlp/querydsl/generator.go
Normal file
170
agent/search/nlp/querydsl/generator.go
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
// Package querydsl provides QueryDSL generation from natural language for DB search
|
||||
// Supports three modes via uses.querydsl configuration:
|
||||
// - "builtin": Template-based generation (no external dependencies)
|
||||
// - "<assistant-id>": Delegate to an LLM-powered assistant for high-quality generation
|
||||
// - "mcp:<server>.<tool>": Call external MCP tool
|
||||
//
|
||||
// For production use cases requiring high accuracy, use Agent or MCP mode.
|
||||
package querydsl
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/yaoapp/gou/query/gou"
|
||||
"github.com/yaoapp/yao/agent/context"
|
||||
"github.com/yaoapp/yao/agent/search/types"
|
||||
)
|
||||
|
||||
// Generator generates QueryDSL from natural language
|
||||
// Mode is determined by uses.querydsl configuration
|
||||
type Generator struct {
|
||||
usesQueryDSL string // "builtin", "<assistant-id>", "mcp:<server>.<tool>"
|
||||
config *types.QueryDSLConfig // QueryDSL generation options
|
||||
}
|
||||
|
||||
// NewGenerator creates a new QueryDSL generator
|
||||
// usesQueryDSL: value from uses.querydsl config
|
||||
// cfg: QueryDSL generation options from search config
|
||||
func NewGenerator(usesQueryDSL string, cfg *types.QueryDSLConfig) *Generator {
|
||||
return &Generator{
|
||||
usesQueryDSL: usesQueryDSL,
|
||||
config: cfg,
|
||||
}
|
||||
}
|
||||
|
||||
// Generate generates QueryDSL from natural language based on configured mode
|
||||
// Returns a QueryDSL ready for execution
|
||||
func (g *Generator) Generate(ctx *context.Context, input *Input) (*Result, error) {
|
||||
var result *Result
|
||||
var err error
|
||||
|
||||
switch {
|
||||
case g.usesQueryDSL == "builtin" || g.usesQueryDSL == "":
|
||||
result, err = g.builtinGenerate(input)
|
||||
case strings.HasPrefix(g.usesQueryDSL, "mcp:"):
|
||||
result, err = g.mcpGenerate(ctx, input)
|
||||
default:
|
||||
// Assume it's an assistant ID for Agent mode
|
||||
result, err = g.agentGenerate(ctx, input)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Validate generated DSL against allowed fields whitelist
|
||||
if result != nil && result.DSL != nil && len(input.AllowedFields) > 0 {
|
||||
result = g.validateFields(result, input.AllowedFields)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// builtinGenerate uses template-based generation
|
||||
// This is a lightweight implementation with no external dependencies.
|
||||
// For better results, use Agent or MCP mode.
|
||||
func (g *Generator) builtinGenerate(input *Input) (*Result, error) {
|
||||
generator := NewBuiltinGenerator()
|
||||
return generator.Generate(input)
|
||||
}
|
||||
|
||||
// agentGenerate delegates to an LLM-powered assistant
|
||||
// The assistant can understand context and generate semantically correct QueryDSL
|
||||
func (g *Generator) agentGenerate(ctx *context.Context, input *Input) (*Result, error) {
|
||||
provider := NewAgentProvider(g.usesQueryDSL)
|
||||
return provider.Generate(ctx, input)
|
||||
}
|
||||
|
||||
// mcpGenerate calls an external MCP tool
|
||||
// Format: "mcp:<server>.<tool>"
|
||||
func (g *Generator) mcpGenerate(ctx *context.Context, input *Input) (*Result, error) {
|
||||
mcpRef := strings.TrimPrefix(g.usesQueryDSL, "mcp:")
|
||||
provider, err := NewMCPProvider(mcpRef)
|
||||
if err != nil {
|
||||
// Fallback to builtin on invalid MCP format
|
||||
return g.builtinGenerate(input)
|
||||
}
|
||||
return provider.Generate(ctx, input)
|
||||
}
|
||||
|
||||
// validateFields validates that all fields in the generated DSL are in the allowed list
|
||||
// If a field is not allowed, it's removed and a warning is added
|
||||
func (g *Generator) validateFields(result *Result, allowedFields []string) *Result {
|
||||
if result.DSL == nil {
|
||||
return result
|
||||
}
|
||||
|
||||
// Build allowed fields set for fast lookup
|
||||
allowed := make(map[string]bool)
|
||||
for _, f := range allowedFields {
|
||||
allowed[f] = true
|
||||
}
|
||||
|
||||
var removedFields []string
|
||||
|
||||
// Validate Select fields
|
||||
if len(result.DSL.Select) > 0 {
|
||||
validSelect := make([]gou.Expression, 0, len(result.DSL.Select))
|
||||
for _, expr := range result.DSL.Select {
|
||||
if allowed[expr.Field] {
|
||||
validSelect = append(validSelect, expr)
|
||||
} else if expr.Field != "" {
|
||||
removedFields = append(removedFields, "select:"+expr.Field)
|
||||
}
|
||||
}
|
||||
result.DSL.Select = validSelect
|
||||
}
|
||||
|
||||
// Validate Where fields (recursive)
|
||||
result.DSL.Wheres = g.validateWheres(result.DSL.Wheres, allowed, &removedFields)
|
||||
|
||||
// Validate Order fields
|
||||
if len(result.DSL.Orders) > 0 {
|
||||
validOrders := make(gou.Orders, 0, len(result.DSL.Orders))
|
||||
for _, order := range result.DSL.Orders {
|
||||
if order.Field != nil && allowed[order.Field.Field] {
|
||||
validOrders = append(validOrders, order)
|
||||
} else if order.Field != nil && order.Field.Field != "" {
|
||||
removedFields = append(removedFields, "order:"+order.Field.Field)
|
||||
}
|
||||
}
|
||||
result.DSL.Orders = validOrders
|
||||
}
|
||||
|
||||
// Add warnings for removed fields
|
||||
if len(removedFields) > 0 {
|
||||
warning := "removed fields not in allowed list: " + strings.Join(removedFields, ", ")
|
||||
result.Warnings = append(result.Warnings, warning)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// validateWheres recursively validates where conditions
|
||||
func (g *Generator) validateWheres(wheres []gou.Where, allowed map[string]bool, removedFields *[]string) []gou.Where {
|
||||
if len(wheres) == 0 {
|
||||
return wheres
|
||||
}
|
||||
|
||||
validWheres := make([]gou.Where, 0, len(wheres))
|
||||
for _, w := range wheres {
|
||||
// Check if the field is allowed
|
||||
fieldAllowed := true
|
||||
if w.Field != nil && w.Field.Field != "" {
|
||||
if !allowed[w.Field.Field] {
|
||||
*removedFields = append(*removedFields, "where:"+w.Field.Field)
|
||||
fieldAllowed = false
|
||||
}
|
||||
}
|
||||
|
||||
if fieldAllowed {
|
||||
// Recursively validate nested wheres
|
||||
if len(w.Wheres) > 0 {
|
||||
w.Wheres = g.validateWheres(w.Wheres, allowed, removedFields)
|
||||
}
|
||||
validWheres = append(validWheres, w)
|
||||
}
|
||||
}
|
||||
|
||||
return validWheres
|
||||
}
|
||||
203
agent/search/nlp/querydsl/generator_test.go
Normal file
203
agent/search/nlp/querydsl/generator_test.go
Normal file
|
|
@ -0,0 +1,203 @@
|
|||
package querydsl
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/yaoapp/gou/query/gou"
|
||||
"github.com/yaoapp/yao/agent/search/types"
|
||||
)
|
||||
|
||||
func TestNewGenerator(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
usesQueryDSL string
|
||||
config *types.QueryDSLConfig
|
||||
}{
|
||||
{
|
||||
name: "builtin mode",
|
||||
usesQueryDSL: "builtin",
|
||||
config: nil,
|
||||
},
|
||||
{
|
||||
name: "empty defaults to builtin",
|
||||
usesQueryDSL: "",
|
||||
config: nil,
|
||||
},
|
||||
{
|
||||
name: "agent mode",
|
||||
usesQueryDSL: "my-querydsl-agent",
|
||||
config: &types.QueryDSLConfig{Strict: true},
|
||||
},
|
||||
{
|
||||
name: "mcp mode",
|
||||
usesQueryDSL: "mcp:nlp.generate_querydsl",
|
||||
config: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
gen := NewGenerator(tt.usesQueryDSL, tt.config)
|
||||
assert.NotNil(t, gen)
|
||||
assert.Equal(t, tt.usesQueryDSL, gen.usesQueryDSL)
|
||||
assert.Equal(t, tt.config, gen.config)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerator_Generate_Builtin(t *testing.T) {
|
||||
gen := NewGenerator("builtin", nil)
|
||||
|
||||
// Note: In real usage, models are loaded internally via model.Select()
|
||||
// For this test, we just verify the basic flow works without models
|
||||
input := &Input{
|
||||
Query: "find all active users",
|
||||
ModelIDs: []string{"user"},
|
||||
Limit: 10,
|
||||
}
|
||||
|
||||
result, err := gen.Generate(nil, input)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
assert.NotNil(t, result.DSL)
|
||||
assert.NotEmpty(t, result.Explain)
|
||||
assert.NotEmpty(t, result.Warnings)
|
||||
}
|
||||
|
||||
func TestGenerator_Generate_EmptyMode(t *testing.T) {
|
||||
// Empty mode should default to builtin
|
||||
gen := NewGenerator("", nil)
|
||||
|
||||
input := &Input{
|
||||
Query: "search products",
|
||||
ModelIDs: []string{"product"},
|
||||
Limit: 5,
|
||||
}
|
||||
|
||||
result, err := gen.Generate(nil, input)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
}
|
||||
|
||||
func TestBuiltinGenerator_Generate(t *testing.T) {
|
||||
gen := NewBuiltinGenerator()
|
||||
|
||||
t.Run("empty query", func(t *testing.T) {
|
||||
result, err := gen.Generate(&Input{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
assert.Nil(t, result.DSL)
|
||||
assert.Contains(t, result.Warnings, "empty query, returning empty DSL")
|
||||
})
|
||||
|
||||
t.Run("nil input", func(t *testing.T) {
|
||||
result, err := gen.Generate(nil)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
assert.Nil(t, result.DSL)
|
||||
})
|
||||
|
||||
t.Run("basic query without models loaded", func(t *testing.T) {
|
||||
// Models are loaded internally via model.Select()
|
||||
// When model is not found, it still generates basic DSL
|
||||
result, err := gen.Generate(&Input{
|
||||
Query: "find users",
|
||||
ModelIDs: []string{"user"},
|
||||
Limit: 10,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
assert.NotNil(t, result.DSL)
|
||||
assert.Equal(t, 10, result.DSL.Limit)
|
||||
})
|
||||
|
||||
t.Run("query with pre-defined wheres", func(t *testing.T) {
|
||||
preWheres := []gou.Where{
|
||||
{
|
||||
Condition: gou.Condition{
|
||||
Field: &gou.Expression{Field: "status"},
|
||||
OP: "=",
|
||||
Value: "active",
|
||||
},
|
||||
},
|
||||
}
|
||||
result, err := gen.Generate(&Input{
|
||||
Query: "find users",
|
||||
ModelIDs: []string{"user"},
|
||||
Wheres: preWheres,
|
||||
Limit: 10,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
assert.NotNil(t, result.DSL)
|
||||
// Should use pre-defined wheres
|
||||
assert.Equal(t, preWheres, result.DSL.Wheres)
|
||||
})
|
||||
|
||||
t.Run("query with orders", func(t *testing.T) {
|
||||
orders := gou.Orders{
|
||||
{Field: &gou.Expression{Field: "created_at"}, Sort: "desc"},
|
||||
}
|
||||
result, err := gen.Generate(&Input{
|
||||
Query: "find users",
|
||||
ModelIDs: []string{"user"},
|
||||
Orders: orders,
|
||||
Limit: 10,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
assert.NotNil(t, result.DSL)
|
||||
assert.Equal(t, orders, result.DSL.Orders)
|
||||
})
|
||||
|
||||
t.Run("query with allowed fields", func(t *testing.T) {
|
||||
result, err := gen.Generate(&Input{
|
||||
Query: "find users",
|
||||
ModelIDs: []string{"user"},
|
||||
AllowedFields: []string{"id", "name", "email"},
|
||||
Limit: 10,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
assert.NotNil(t, result.DSL)
|
||||
})
|
||||
|
||||
t.Run("default limit", func(t *testing.T) {
|
||||
result, err := gen.Generate(&Input{
|
||||
Query: "find users",
|
||||
ModelIDs: []string{"user"},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
assert.NotNil(t, result.DSL)
|
||||
assert.Equal(t, 20, result.DSL.Limit)
|
||||
})
|
||||
|
||||
t.Run("multi-model query", func(t *testing.T) {
|
||||
// Models are loaded internally via model.Select()
|
||||
result, err := gen.Generate(&Input{
|
||||
Query: "find user orders",
|
||||
ModelIDs: []string{"user", "order"},
|
||||
Limit: 10,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
assert.NotNil(t, result.DSL)
|
||||
})
|
||||
}
|
||||
|
||||
func TestResult(t *testing.T) {
|
||||
result := &Result{
|
||||
DSL: &gou.QueryDSL{
|
||||
Limit: 10,
|
||||
},
|
||||
Explain: "Generated query for finding users",
|
||||
Warnings: []string{"using placeholder implementation"},
|
||||
}
|
||||
|
||||
assert.NotNil(t, result.DSL)
|
||||
assert.Equal(t, 10, result.DSL.Limit)
|
||||
assert.NotEmpty(t, result.Explain)
|
||||
assert.Len(t, result.Warnings, 1)
|
||||
}
|
||||
229
agent/search/nlp/querydsl/mcp.go
Normal file
229
agent/search/nlp/querydsl/mcp.go
Normal file
|
|
@ -0,0 +1,229 @@
|
|||
package querydsl
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/yaoapp/gou/mcp"
|
||||
gouMCPTypes "github.com/yaoapp/gou/mcp/types"
|
||||
"github.com/yaoapp/gou/query/gou"
|
||||
"github.com/yaoapp/gou/query/linter"
|
||||
agentContext "github.com/yaoapp/yao/agent/context"
|
||||
)
|
||||
|
||||
// MaxRetries is the maximum number of retry attempts for QueryDSL generation
|
||||
const MaxRetries = 3
|
||||
|
||||
// MCPProvider delegates QueryDSL generation to an MCP tool
|
||||
type MCPProvider struct {
|
||||
serverID string // MCP server ID
|
||||
toolName string // Tool name to call
|
||||
}
|
||||
|
||||
// NewMCPProvider creates a new MCP-based QueryDSL generator
|
||||
// mcpRef format: "server.tool" (e.g., "nlp.generate_querydsl")
|
||||
func NewMCPProvider(mcpRef string) (*MCPProvider, error) {
|
||||
parts := strings.SplitN(mcpRef, ".", 2)
|
||||
if len(parts) != 2 {
|
||||
return nil, fmt.Errorf("invalid MCP format, expected 'server.tool', got '%s'", mcpRef)
|
||||
}
|
||||
return &MCPProvider{
|
||||
serverID: parts[0],
|
||||
toolName: parts[1],
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Generate generates QueryDSL by calling the MCP tool with retry and lint validation
|
||||
func (p *MCPProvider) Generate(ctx *agentContext.Context, input *Input) (*Result, error) {
|
||||
// Get MCP client
|
||||
client, err := mcp.Select(p.serverID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("MCP server '%s' not found: %w", p.serverID, err)
|
||||
}
|
||||
|
||||
var lastError error
|
||||
var lastLintErrors string
|
||||
|
||||
for attempt := 1; attempt <= MaxRetries; attempt++ {
|
||||
// Build arguments for the MCP tool
|
||||
arguments := p.buildArguments(input, attempt, lastLintErrors)
|
||||
|
||||
// Call the MCP tool
|
||||
callResult, err := client.CallTool(ctx, p.toolName, arguments)
|
||||
if err != nil {
|
||||
lastError = fmt.Errorf("MCP tool call failed: %w", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Parse the result
|
||||
result, err := p.parseResult(callResult)
|
||||
if err != nil {
|
||||
lastError = err
|
||||
continue
|
||||
}
|
||||
|
||||
// Validate with linter if DSL is present
|
||||
if result.DSL != nil {
|
||||
lintResult := p.validateDSL(result.DSL)
|
||||
if lintResult.Valid {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Lint failed, prepare error message for retry
|
||||
lastLintErrors = lintResult.FormatDiagnostics()
|
||||
lastError = fmt.Errorf("QueryDSL validation failed: %s", lastLintErrors)
|
||||
|
||||
// Add lint warnings to result warnings
|
||||
for _, diag := range lintResult.Diagnostics {
|
||||
result.Warnings = append(result.Warnings, fmt.Sprintf("[%s] %s: %s", diag.Code, diag.Path, diag.Message))
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// No DSL returned
|
||||
lastError = fmt.Errorf("no QueryDSL returned from MCP tool")
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("QueryDSL generation failed after %d attempts: %w", MaxRetries, lastError)
|
||||
}
|
||||
|
||||
// buildArguments constructs the MCP tool arguments
|
||||
func (p *MCPProvider) buildArguments(input *Input, attempt int, lastLintErrors string) map[string]interface{} {
|
||||
arguments := map[string]interface{}{
|
||||
"query": input.Query,
|
||||
"models": input.ModelIDs,
|
||||
"limit": input.Limit,
|
||||
}
|
||||
|
||||
// Add optional fields
|
||||
if len(input.Wheres) > 0 {
|
||||
arguments["wheres"] = input.Wheres
|
||||
}
|
||||
if len(input.Orders) > 0 {
|
||||
arguments["orders"] = input.Orders
|
||||
}
|
||||
if len(input.AllowedFields) > 0 {
|
||||
arguments["allowed_fields"] = input.AllowedFields
|
||||
}
|
||||
if len(input.ExtraParams) > 0 {
|
||||
arguments["extra"] = input.ExtraParams
|
||||
}
|
||||
|
||||
// Add retry context if this is a retry attempt
|
||||
if attempt > 1 && lastLintErrors != "" {
|
||||
arguments["retry"] = map[string]interface{}{
|
||||
"attempt": attempt,
|
||||
"lint_errors": lastLintErrors,
|
||||
"instructions": "The previous QueryDSL was invalid. Please fix the errors and regenerate.",
|
||||
}
|
||||
}
|
||||
|
||||
return arguments
|
||||
}
|
||||
|
||||
// validateDSL validates the generated QueryDSL using the linter
|
||||
func (p *MCPProvider) validateDSL(dsl *gou.QueryDSL) *linter.LintResult {
|
||||
// Marshal DSL to JSON for linting
|
||||
jsonBytes, err := json.Marshal(dsl)
|
||||
if err != nil {
|
||||
result := &linter.LintResult{Valid: false}
|
||||
return result
|
||||
}
|
||||
|
||||
_, lintResult := linter.Parse(string(jsonBytes))
|
||||
return lintResult
|
||||
}
|
||||
|
||||
// parseResult extracts QueryDSL from the MCP tool response
|
||||
func (p *MCPProvider) parseResult(result *gouMCPTypes.CallToolResponse) (*Result, error) {
|
||||
if result == nil {
|
||||
return &Result{}, nil
|
||||
}
|
||||
|
||||
// Check for errors in result
|
||||
if result.IsError {
|
||||
errMsg := "MCP tool returned error"
|
||||
if len(result.Content) > 0 && result.Content[0].Text != "" {
|
||||
errMsg = result.Content[0].Text
|
||||
}
|
||||
return nil, fmt.Errorf("%s", errMsg)
|
||||
}
|
||||
|
||||
// Parse content - expect JSON data with "dsl" field
|
||||
if len(result.Content) == 0 {
|
||||
return &Result{}, nil
|
||||
}
|
||||
|
||||
genResult := &Result{}
|
||||
|
||||
// Try to extract QueryDSL from content
|
||||
for _, content := range result.Content {
|
||||
// Check text content type
|
||||
if content.Type == gouMCPTypes.ToolContentTypeText && content.Text != "" {
|
||||
// Try to parse as JSON
|
||||
var data map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(content.Text), &data); err == nil {
|
||||
// Look for "dsl" field
|
||||
if dsl, ok := data["dsl"]; ok {
|
||||
genResult.DSL = p.extractDSL(dsl)
|
||||
}
|
||||
if explain, ok := data["explain"].(string); ok {
|
||||
genResult.Explain = explain
|
||||
}
|
||||
if warnings, ok := data["warnings"]; ok {
|
||||
genResult.Warnings = p.extractWarnings(warnings)
|
||||
}
|
||||
return genResult, nil
|
||||
}
|
||||
|
||||
// Try to parse as direct QueryDSL
|
||||
var dsl gou.QueryDSL
|
||||
if err := json.Unmarshal([]byte(content.Text), &dsl); err == nil {
|
||||
genResult.DSL = &dsl
|
||||
return genResult, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return genResult, nil
|
||||
}
|
||||
|
||||
// extractDSL converts interface{} to gou.QueryDSL
|
||||
func (p *MCPProvider) extractDSL(v interface{}) *gou.QueryDSL {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Marshal and unmarshal to gou.QueryDSL
|
||||
jsonBytes, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var dsl gou.QueryDSL
|
||||
if err := json.Unmarshal(jsonBytes, &dsl); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &dsl
|
||||
}
|
||||
|
||||
// extractWarnings extracts warnings array from various types
|
||||
func (p *MCPProvider) extractWarnings(v interface{}) []string {
|
||||
switch w := v.(type) {
|
||||
case []string:
|
||||
return w
|
||||
case []interface{}:
|
||||
warnings := make([]string, 0, len(w))
|
||||
for _, item := range w {
|
||||
if s, ok := item.(string); ok {
|
||||
warnings = append(warnings, s)
|
||||
}
|
||||
}
|
||||
return warnings
|
||||
case string:
|
||||
return []string{w}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
237
agent/search/nlp/querydsl/mcp_test.go
Normal file
237
agent/search/nlp/querydsl/mcp_test.go
Normal file
|
|
@ -0,0 +1,237 @@
|
|||
package querydsl
|
||||
|
||||
import (
|
||||
stdContext "context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/yaoapp/gou/plan"
|
||||
agentContext "github.com/yaoapp/yao/agent/context"
|
||||
"github.com/yaoapp/yao/config"
|
||||
"github.com/yaoapp/yao/test"
|
||||
)
|
||||
|
||||
// newTestContext creates a test context for MCP testing
|
||||
func newTestContext() *agentContext.Context {
|
||||
ctx := &agentContext.Context{
|
||||
Context: stdContext.Background(),
|
||||
Space: plan.NewMemorySharedSpace(),
|
||||
ID: "test-querydsl",
|
||||
ChatID: "test-chat",
|
||||
AssistantID: "test-assistant",
|
||||
Locale: "en",
|
||||
Referer: agentContext.RefererAPI,
|
||||
}
|
||||
stack, _, _ := agentContext.EnterStack(ctx, "test-assistant", &agentContext.Options{})
|
||||
ctx.Stack = stack
|
||||
return ctx
|
||||
}
|
||||
|
||||
func TestMCPProvider_Generate(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Create context
|
||||
ctx := newTestContext()
|
||||
|
||||
// Create MCP provider for search.generate_querydsl
|
||||
provider, err := NewMCPProvider("search.generate_querydsl")
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, provider)
|
||||
assert.Equal(t, "search", provider.serverID)
|
||||
assert.Equal(t, "generate_querydsl", provider.toolName)
|
||||
|
||||
t.Run("verify_fixed_structure", func(t *testing.T) {
|
||||
input := &Input{
|
||||
Query: "find active users",
|
||||
ModelIDs: []string{"user"},
|
||||
Limit: 10,
|
||||
}
|
||||
|
||||
result, err := provider.Generate(ctx, input)
|
||||
if err != nil {
|
||||
t.Logf("Generate error: %v", err)
|
||||
}
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
|
||||
if result == nil {
|
||||
t.Fatal("result is nil")
|
||||
}
|
||||
|
||||
if !assert.NotNil(t, result.DSL, "DSL should not be nil") {
|
||||
t.Logf("Result: Explain=%s, Warnings=%v", result.Explain, result.Warnings)
|
||||
return
|
||||
}
|
||||
|
||||
// Verify fixed DSL structure from mock
|
||||
// select: ["id", "name", "status"] - parsed as Expression with Field property
|
||||
assert.Len(t, result.DSL.Select, 3)
|
||||
if len(result.DSL.Select) >= 3 {
|
||||
assert.Equal(t, "id", result.DSL.Select[0].Field)
|
||||
assert.Equal(t, "name", result.DSL.Select[1].Field)
|
||||
assert.Equal(t, "status", result.DSL.Select[2].Field)
|
||||
}
|
||||
|
||||
// wheres: [{ field: "status", op: "=", value: "active" }]
|
||||
assert.Len(t, result.DSL.Wheres, 1)
|
||||
if len(result.DSL.Wheres) > 0 {
|
||||
assert.Equal(t, "status", result.DSL.Wheres[0].Field.Field)
|
||||
assert.Equal(t, "=", result.DSL.Wheres[0].OP)
|
||||
assert.Equal(t, "active", result.DSL.Wheres[0].Value)
|
||||
}
|
||||
|
||||
// orders: [{ field: "created_at", sort: "desc" }]
|
||||
assert.Len(t, result.DSL.Orders, 1)
|
||||
if len(result.DSL.Orders) > 0 {
|
||||
assert.Equal(t, "created_at", result.DSL.Orders[0].Field.Field)
|
||||
assert.Equal(t, "desc", result.DSL.Orders[0].Sort)
|
||||
}
|
||||
|
||||
// limit: 10 (from input, returned as float64 from JSON)
|
||||
assert.Equal(t, float64(10), result.DSL.Limit)
|
||||
|
||||
// explain should contain query
|
||||
assert.Contains(t, result.Explain, "find active users")
|
||||
|
||||
// warnings should be empty
|
||||
assert.Empty(t, result.Warnings)
|
||||
})
|
||||
}
|
||||
|
||||
func TestNewMCPProvider(t *testing.T) {
|
||||
t.Run("valid format", func(t *testing.T) {
|
||||
provider, err := NewMCPProvider("nlp.generate_querydsl")
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, provider)
|
||||
assert.Equal(t, "nlp", provider.serverID)
|
||||
assert.Equal(t, "generate_querydsl", provider.toolName)
|
||||
})
|
||||
|
||||
t.Run("invalid format - no dot", func(t *testing.T) {
|
||||
provider, err := NewMCPProvider("invalid")
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, provider)
|
||||
assert.Contains(t, err.Error(), "invalid MCP format")
|
||||
})
|
||||
|
||||
t.Run("complex tool name", func(t *testing.T) {
|
||||
provider, err := NewMCPProvider("server.tool.with.dots")
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, provider)
|
||||
assert.Equal(t, "server", provider.serverID)
|
||||
assert.Equal(t, "tool.with.dots", provider.toolName)
|
||||
})
|
||||
}
|
||||
|
||||
func TestMCPProvider_Generate_Error(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
ctx := newTestContext()
|
||||
|
||||
t.Run("non-existent server", func(t *testing.T) {
|
||||
provider, _ := NewMCPProvider("nonexistent.tool")
|
||||
result, err := provider.Generate(ctx, &Input{
|
||||
Query: "test",
|
||||
ModelIDs: []string{"user"},
|
||||
})
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, result)
|
||||
assert.Contains(t, err.Error(), "not found")
|
||||
})
|
||||
}
|
||||
|
||||
func TestGenerator_MCP_Integration(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Skip if not in integration test mode
|
||||
if os.Getenv("YAO_TEST_MCP") != "true" {
|
||||
t.Skip("Skipping MCP integration test (set YAO_TEST_MCP=true to run)")
|
||||
}
|
||||
|
||||
ctx := newTestContext()
|
||||
|
||||
// Create generator with MCP mode
|
||||
gen := NewGenerator("mcp:search.generate_querydsl", nil)
|
||||
|
||||
t.Run("generate_via_mcp", func(t *testing.T) {
|
||||
input := &Input{
|
||||
Query: "find active users",
|
||||
ModelIDs: []string{"user"},
|
||||
Limit: 15,
|
||||
}
|
||||
|
||||
result, err := gen.Generate(ctx, input)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
assert.NotNil(t, result.DSL)
|
||||
|
||||
// Verify fixed structure is correctly parsed
|
||||
assert.Len(t, result.DSL.Select, 3)
|
||||
assert.Len(t, result.DSL.Wheres, 1)
|
||||
assert.Len(t, result.DSL.Orders, 1)
|
||||
assert.Equal(t, float64(15), result.DSL.Limit)
|
||||
assert.Contains(t, result.Explain, "find active users")
|
||||
})
|
||||
|
||||
t.Run("allowed_fields_validation", func(t *testing.T) {
|
||||
input := &Input{
|
||||
Query: "find users",
|
||||
ModelIDs: []string{"user"},
|
||||
AllowedFields: []string{"id", "name"}, // Only allow id and name
|
||||
Limit: 10,
|
||||
}
|
||||
|
||||
result, err := gen.Generate(ctx, input)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
assert.NotNil(t, result.DSL)
|
||||
|
||||
// "status" field should be filtered out from select and wheres
|
||||
// since it's not in AllowedFields
|
||||
for _, expr := range result.DSL.Select {
|
||||
assert.Contains(t, []string{"id", "name"}, expr.Field)
|
||||
}
|
||||
|
||||
// Should have warning about removed fields
|
||||
assert.NotEmpty(t, result.Warnings)
|
||||
})
|
||||
}
|
||||
|
||||
func TestMCPProvider_Generate_WithRetry(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
ctx := newTestContext()
|
||||
|
||||
// Create MCP provider for search.generate_querydsl_with_retry
|
||||
// This tool returns invalid DSL on first call, valid on second
|
||||
provider, err := NewMCPProvider("search.generate_querydsl_with_retry")
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, provider)
|
||||
|
||||
t.Run("retry_on_lint_failure", func(t *testing.T) {
|
||||
input := &Input{
|
||||
Query: "test retry mechanism",
|
||||
ModelIDs: []string{"user"},
|
||||
Limit: 10,
|
||||
}
|
||||
|
||||
// This should succeed after retry
|
||||
// First call returns invalid DSL (missing 'from')
|
||||
// Second call (with lint_errors) returns valid DSL
|
||||
result, err := provider.Generate(ctx, input)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
|
||||
if result != nil && result.DSL != nil {
|
||||
// Should have valid DSL after retry
|
||||
assert.NotNil(t, result.DSL.From, "DSL should have 'from' field after retry")
|
||||
// Explain should indicate this was fixed after receiving lint errors
|
||||
assert.Contains(t, result.Explain, "fixed after receiving lint errors")
|
||||
}
|
||||
})
|
||||
}
|
||||
23
agent/search/nlp/querydsl/types.go
Normal file
23
agent/search/nlp/querydsl/types.go
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
package querydsl
|
||||
|
||||
import (
|
||||
"github.com/yaoapp/gou/query/gou"
|
||||
)
|
||||
|
||||
// Input contains all information needed to generate QueryDSL
|
||||
type Input struct {
|
||||
Query string // Natural language query
|
||||
ModelIDs []string // Target model IDs (e.g., ["user", "order", "product"])
|
||||
Wheres []gou.Where // Pre-defined filters (optional)
|
||||
Orders gou.Orders // Sort orders (optional)
|
||||
AllowedFields []string // Allowed fields whitelist (optional, for security validation)
|
||||
Limit int // Max results
|
||||
ExtraParams map[string]interface{} // Additional parameters
|
||||
}
|
||||
|
||||
// Result represents the result of QueryDSL generation
|
||||
type Result struct {
|
||||
DSL *gou.QueryDSL `json:"dsl"` // Generated QueryDSL (supports joins)
|
||||
Explain string `json:"explain,omitempty"` // Human-readable explanation
|
||||
Warnings []string `json:"warnings,omitempty"` // Any warnings during generation
|
||||
}
|
||||
|
|
@ -16,6 +16,13 @@ type DSL struct {
|
|||
StoreSetting store.Setting `json:"store" yaml:"store"` // The store setting of the assistant
|
||||
Cache string `json:"cache" yaml:"cache"` // The cache store of the assistant, if not set, default is "__yao.agent.cache"
|
||||
|
||||
// System Agents Connector Settings
|
||||
// ===============================
|
||||
// System configures connectors for system agents (__yao.keyword, __yao.querydsl, __yao.title, __yao.prompt)
|
||||
// Each agent can have its own connector, or use the default
|
||||
// If not set, fallback to the first connector that supports the required capabilities
|
||||
System *System `json:"system,omitempty" yaml:"system,omitempty"`
|
||||
|
||||
// Global External Settings - model capabilities, tools, etc.
|
||||
// ===============================
|
||||
Models map[string]openai.Capabilities `json:"models,omitempty" yaml:"models,omitempty"` // The model capabilities configuration
|
||||
|
|
@ -48,6 +55,18 @@ type Uses struct {
|
|||
Rerank string `json:"rerank,omitempty" yaml:"rerank,omitempty"` // Result reranking: "builtin", "<assistant-id>", "mcp:<server>.<tool>"
|
||||
}
|
||||
|
||||
// System configures connectors for system agents
|
||||
// ===============================
|
||||
type System struct {
|
||||
Default string `json:"default,omitempty" yaml:"default,omitempty"` // Default connector for all system agents
|
||||
Keyword string `json:"keyword,omitempty" yaml:"keyword,omitempty"` // Connector for __yao.keyword agent
|
||||
QueryDSL string `json:"querydsl,omitempty" yaml:"querydsl,omitempty"` // Connector for __yao.querydsl agent
|
||||
Title string `json:"title,omitempty" yaml:"title,omitempty"` // Connector for __yao.title agent
|
||||
Prompt string `json:"prompt,omitempty" yaml:"prompt,omitempty"` // Connector for __yao.prompt agent
|
||||
NeedSearch string `json:"needsearch,omitempty" yaml:"needsearch,omitempty"` // Connector for __yao.needsearch agent
|
||||
Entity string `json:"entity,omitempty" yaml:"entity,omitempty"` // Connector for __yao.entity agent
|
||||
}
|
||||
|
||||
// Mention Structure
|
||||
// ===============================
|
||||
type Mention struct {
|
||||
|
|
|
|||
626
data/bindata.go
626
data/bindata.go
File diff suppressed because it is too large
Load diff
11
yao/assistants/entity/package.yao
Normal file
11
yao/assistants/entity/package.yao
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"name": "Entity Extractor",
|
||||
"description": "Extract entities and relationships",
|
||||
"type": "worker",
|
||||
"uses": { "search": "disabled" },
|
||||
"options": {
|
||||
"max_tokens": 2000,
|
||||
"temperature": 0.2
|
||||
}
|
||||
}
|
||||
|
||||
28
yao/assistants/entity/prompts.yml
Normal file
28
yao/assistants/entity/prompts.yml
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
- role: system
|
||||
content: |
|
||||
Extract entities and relationships from text for knowledge graph construction.
|
||||
|
||||
## Task
|
||||
1. Identify named entities (Person, Organization, Location, Product, Event, Concept, etc.)
|
||||
2. Extract relationships between entities
|
||||
3. Return structured JSON
|
||||
|
||||
## Response Format (JSON only)
|
||||
```json
|
||||
{
|
||||
"entities": [
|
||||
{"id": "e1", "name": "Entity Name", "type": "Person|Org|Location|Product|Event|Concept", "properties": {}}
|
||||
],
|
||||
"relationships": [
|
||||
{"source": "e1", "target": "e2", "type": "relationship_type", "properties": {}}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Guidelines
|
||||
- Use consistent entity IDs (e1, e2, ...)
|
||||
- Normalize entity names (remove titles, standardize format)
|
||||
- Common relationship types: WORKS_FOR, LOCATED_IN, OWNS, CREATED, RELATED_TO
|
||||
- Keep properties minimal and relevant
|
||||
- Same language as input for entity names
|
||||
|
||||
10
yao/assistants/keyword/package.yao
Normal file
10
yao/assistants/keyword/package.yao
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"name": "Keyword Extractor",
|
||||
"description": "Extract search keywords",
|
||||
"type": "worker",
|
||||
"uses": { "search": "disabled" },
|
||||
"options": {
|
||||
"max_tokens": 500,
|
||||
"temperature": 0.3
|
||||
}
|
||||
}
|
||||
21
yao/assistants/keyword/prompts.yml
Normal file
21
yao/assistants/keyword/prompts.yml
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
- role: system
|
||||
content: |
|
||||
Extract keywords from text content.
|
||||
|
||||
Task:
|
||||
1. Analyze input text
|
||||
2. Extract important keywords
|
||||
3. Return JSON format
|
||||
4. Match input language
|
||||
|
||||
Response Format (JSON only):
|
||||
```json
|
||||
{"keywords": ["keyword1", "keyword2", ...]}
|
||||
```
|
||||
|
||||
Guidelines:
|
||||
- Extract 5-15 keywords based on content length
|
||||
- Prioritize nouns, proper nouns, key concepts
|
||||
- Include single words and short phrases
|
||||
- Exclude common stop words
|
||||
- Keywords MUST be in the same language as input
|
||||
113
yao/assistants/keyword/src/index.ts
Normal file
113
yao/assistants/keyword/src/index.ts
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
/**
|
||||
* Keyword Extraction Agent - Next Hook
|
||||
* Parses LLM response and extracts keywords with error tolerance
|
||||
*/
|
||||
|
||||
// @ts-nocheck
|
||||
|
||||
/**
|
||||
* Next hook - processes keyword extraction response
|
||||
* Uses json.Parse for fault-tolerant JSON parsing
|
||||
*/
|
||||
function Next(
|
||||
ctx: agent.Context,
|
||||
payload: agent.NextHookPayload
|
||||
): agent.NextHookResponse | null {
|
||||
const completion = payload.completion;
|
||||
|
||||
// No completion, return null for standard handling
|
||||
if (!completion || !completion.content) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Remove markdown code block if present
|
||||
let content = completion.content.trim();
|
||||
if (content.startsWith("```json")) {
|
||||
content = content.slice(7);
|
||||
} else if (content.startsWith("```")) {
|
||||
content = content.slice(3);
|
||||
}
|
||||
if (content.endsWith("```")) {
|
||||
content = content.slice(0, -3);
|
||||
}
|
||||
content = content.trim();
|
||||
|
||||
// Try to parse JSON from completion content
|
||||
let keywords: string[] = [];
|
||||
|
||||
try {
|
||||
// Use json.Parse for fault-tolerant parsing (handles broken JSON, JSONC, etc.)
|
||||
const parsed = Process("json.Parse", content) as {
|
||||
keywords?: string[];
|
||||
} | null;
|
||||
|
||||
if (parsed && Array.isArray(parsed.keywords)) {
|
||||
keywords = parsed.keywords.filter(
|
||||
(k) => typeof k === "string" && k.trim().length > 0
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
// If json.Parse fails, try to extract keywords from text
|
||||
keywords = extractKeywordsFromText(content);
|
||||
}
|
||||
|
||||
// If still no keywords, try extracting from raw text
|
||||
if (keywords.length === 0) {
|
||||
keywords = extractKeywordsFromText(content);
|
||||
}
|
||||
|
||||
// Return parsed keywords
|
||||
return {
|
||||
data: {
|
||||
keywords: keywords,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract keywords from plain text when JSON parsing fails
|
||||
* Handles formats like:
|
||||
* - Comma-separated: "keyword1, keyword2, keyword3"
|
||||
* - Line-separated: "keyword1\nkeyword2\nkeyword3"
|
||||
* - Bullet points: "- keyword1\n- keyword2"
|
||||
* - Numbered: "1. keyword1\n2. keyword2"
|
||||
*/
|
||||
function extractKeywordsFromText(text: string): string[] {
|
||||
const keywords: string[] = [];
|
||||
|
||||
// Remove common prefixes/suffixes
|
||||
let cleaned = text
|
||||
.replace(/^[\s\S]*?keywords?[\s::]*\[?/i, "") // Remove "keywords:" prefix
|
||||
.replace(/\][\s\S]*$/, "") // Remove trailing ]
|
||||
.trim();
|
||||
|
||||
// Try line-by-line extraction
|
||||
const lines = cleaned.split(/[\n\r]+/);
|
||||
|
||||
for (const line of lines) {
|
||||
// Remove bullet points, numbers, quotes
|
||||
let keyword = line
|
||||
.replace(/^[\s\-\*\•\d\.]+/, "") // Remove bullets/numbers
|
||||
.replace(/^["'`]+|["'`]+$/g, "") // Remove quotes
|
||||
.replace(/,\s*$/, "") // Remove trailing comma
|
||||
.trim();
|
||||
|
||||
// Skip empty or too long
|
||||
if (keyword.length > 0 && keyword.length < 100) {
|
||||
// Split by comma if contains multiple
|
||||
if (keyword.includes(",")) {
|
||||
const parts = keyword.split(",").map((p) => p.trim());
|
||||
for (const part of parts) {
|
||||
if (part.length > 0 && part.length < 100) {
|
||||
keywords.push(part);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
keywords.push(keyword);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplicate
|
||||
return [...new Set(keywords)];
|
||||
}
|
||||
10
yao/assistants/needsearch/package.yao
Normal file
10
yao/assistants/needsearch/package.yao
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"name": "Reference Checker",
|
||||
"description": "Check if references are needed",
|
||||
"type": "worker",
|
||||
"uses": { "search": "disabled" },
|
||||
"options": {
|
||||
"max_tokens": 200,
|
||||
"temperature": 0.1
|
||||
}
|
||||
}
|
||||
20
yao/assistants/needsearch/prompts.yml
Normal file
20
yao/assistants/needsearch/prompts.yml
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
# Need Search Agent
|
||||
- role: system
|
||||
content: |
|
||||
Classify if user query needs external search.
|
||||
|
||||
## Rules
|
||||
NO SEARCH: greetings, chitchat, math, code generation, text processing, general knowledge
|
||||
WEB: real-time data (weather, news, prices), current events, recent info
|
||||
KB: documentation, how-to, configuration, FAQ
|
||||
DB: user data, orders, records, business data
|
||||
|
||||
## Response (JSON only)
|
||||
{"need_search": bool, "search_types": ["web"|"kb"|"db"], "confidence": 0-1}
|
||||
|
||||
## Examples
|
||||
"Hello" → {"need_search": false, "search_types": [], "confidence": 0.99}
|
||||
"Today's weather" → {"need_search": true, "search_types": ["web"], "confidence": 0.95}
|
||||
"Write a sort function" → {"need_search": false, "search_types": [], "confidence": 0.90}
|
||||
"How to config DB" → {"need_search": true, "search_types": ["kb"], "confidence": 0.85}
|
||||
"My orders" → {"need_search": true, "search_types": ["db"], "confidence": 0.95}
|
||||
117
yao/assistants/needsearch/src/index.ts
Normal file
117
yao/assistants/needsearch/src/index.ts
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
/**
|
||||
* Need Search Agent - Next Hook
|
||||
* Parses LLM response and extracts search intent with error tolerance
|
||||
*/
|
||||
|
||||
// @ts-nocheck
|
||||
|
||||
interface SearchResult {
|
||||
need_search: boolean;
|
||||
search_types: string[];
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Next hook - processes search intent response
|
||||
* Uses json.Parse for fault-tolerant JSON parsing
|
||||
*/
|
||||
function Next(
|
||||
ctx: agent.Context,
|
||||
payload: agent.NextHookPayload
|
||||
): agent.NextHookResponse | null {
|
||||
const completion = payload.completion;
|
||||
|
||||
// No completion, return null for standard handling
|
||||
if (!completion || !completion.content) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Remove markdown code block if present
|
||||
let content = completion.content.trim();
|
||||
if (content.startsWith("```json")) {
|
||||
content = content.slice(7); // Remove ```json
|
||||
} else if (content.startsWith("```")) {
|
||||
content = content.slice(3); // Remove ```
|
||||
}
|
||||
if (content.endsWith("```")) {
|
||||
content = content.slice(0, -3); // Remove trailing ```
|
||||
}
|
||||
content = content.trim();
|
||||
|
||||
// Default result
|
||||
let result: SearchResult = {
|
||||
need_search: false,
|
||||
search_types: [],
|
||||
confidence: 0,
|
||||
};
|
||||
|
||||
try {
|
||||
// Use json.Parse for fault-tolerant parsing
|
||||
const parsed = Process("json.Parse", content) as {
|
||||
need_search?: boolean;
|
||||
search_types?: string[];
|
||||
confidence?: number;
|
||||
} | null;
|
||||
|
||||
if (parsed) {
|
||||
result.need_search = Boolean(parsed.need_search);
|
||||
result.search_types = Array.isArray(parsed.search_types)
|
||||
? parsed.search_types.filter(
|
||||
(t) =>
|
||||
typeof t === "string" &&
|
||||
["web", "kb", "db"].includes(t.toLowerCase())
|
||||
)
|
||||
: [];
|
||||
result.confidence =
|
||||
typeof parsed.confidence === "number"
|
||||
? Math.min(1, Math.max(0, parsed.confidence))
|
||||
: 0.5;
|
||||
}
|
||||
} catch (e) {
|
||||
// If json.Parse fails, try to extract from text
|
||||
result = extractFromText(content);
|
||||
}
|
||||
|
||||
// Return parsed result
|
||||
return {
|
||||
data: result,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract search intent from plain text when JSON parsing fails
|
||||
*/
|
||||
function extractFromText(text: string): SearchResult {
|
||||
const lower = text.toLowerCase();
|
||||
|
||||
// Check for explicit indicators
|
||||
const needSearch =
|
||||
lower.includes("true") ||
|
||||
lower.includes("need") ||
|
||||
lower.includes("search") ||
|
||||
lower.includes("web") ||
|
||||
lower.includes("kb") ||
|
||||
lower.includes("db");
|
||||
|
||||
const noSearch =
|
||||
lower.includes("false") ||
|
||||
lower.includes("no search") ||
|
||||
lower.includes("not need");
|
||||
|
||||
// Extract search types
|
||||
const searchTypes: string[] = [];
|
||||
if (lower.includes("web")) searchTypes.push("web");
|
||||
if (lower.includes("kb") || lower.includes("knowledge"))
|
||||
searchTypes.push("kb");
|
||||
if (lower.includes("db") || lower.includes("database"))
|
||||
searchTypes.push("db");
|
||||
|
||||
// Determine need_search
|
||||
const need = noSearch ? false : needSearch && searchTypes.length > 0;
|
||||
|
||||
return {
|
||||
need_search: need,
|
||||
search_types: need ? searchTypes : [],
|
||||
confidence: 0.5, // Low confidence for text extraction
|
||||
};
|
||||
}
|
||||
9
yao/assistants/prompt/package.yao
Normal file
9
yao/assistants/prompt/package.yao
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"name": "Prompt Optimizer",
|
||||
"description": "Optimize prompts for better results",
|
||||
"type": "worker",
|
||||
"uses": { "search": "disabled" },
|
||||
"options": {
|
||||
"temperature": 0
|
||||
}
|
||||
}
|
||||
25
yao/assistants/prompt/prompts.yml
Normal file
25
yao/assistants/prompt/prompts.yml
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
- role: system
|
||||
content: |
|
||||
You are a prompt optimization assistant. Transform user requirements into professional prompts.
|
||||
|
||||
Process:
|
||||
1. Extract key information and objectives
|
||||
2. Reorganize with precise terminology
|
||||
3. Add context and details
|
||||
|
||||
Include:
|
||||
- Clear goal/task description
|
||||
- Expected output format
|
||||
- Quality requirements
|
||||
- Reference information
|
||||
|
||||
Ensure:
|
||||
- Clear and unambiguous
|
||||
- Detailed and specific
|
||||
- Well-structured
|
||||
- Actionable
|
||||
|
||||
Rules:
|
||||
1. Respond in same language as input
|
||||
2. Output ONLY the optimized prompt
|
||||
3. Ready to use as-is
|
||||
10
yao/assistants/querydsl/package.yao
Normal file
10
yao/assistants/querydsl/package.yao
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"name": "Query Builder",
|
||||
"description": "Build database queries",
|
||||
"type": "worker",
|
||||
"uses": { "search": "disabled" },
|
||||
"options": {
|
||||
"max_tokens": 2000,
|
||||
"temperature": 0.2
|
||||
}
|
||||
}
|
||||
43
yao/assistants/querydsl/prompts.yml
Normal file
43
yao/assistants/querydsl/prompts.yml
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
# QueryDSL Generator Agent Prompts
|
||||
- role: system
|
||||
content: |
|
||||
You are a QueryDSL generator. Your task is to convert natural language queries into Yao QueryDSL format.
|
||||
|
||||
## QueryDSL Structure
|
||||
```json
|
||||
{
|
||||
"select": ["field1", "field2"],
|
||||
"from": "table_name",
|
||||
"wheres": [
|
||||
{"field": "name", "op": "=", "value": "test"},
|
||||
{"field": "status", "op": "in", "value": ["active", "pending"]}
|
||||
],
|
||||
"orders": [
|
||||
{"field": "created_at", "sort": "desc"}
|
||||
],
|
||||
"limit": 20
|
||||
}
|
||||
```
|
||||
|
||||
## Supported Operators
|
||||
- Comparison: =, !=, >, >=, <, <=
|
||||
- Pattern: like, not like
|
||||
- Range: in, not in, between
|
||||
- Null check: is null, is not null
|
||||
|
||||
## Response Format
|
||||
Always respond with valid JSON:
|
||||
```json
|
||||
{
|
||||
"dsl": { ... },
|
||||
"explain": "Brief explanation of the query",
|
||||
"warnings": ["any warnings or notes"]
|
||||
}
|
||||
```
|
||||
|
||||
## Guidelines
|
||||
- Generate valid QueryDSL based on the provided schema
|
||||
- Use appropriate operators for the query intent
|
||||
- Include only fields that exist in the schema
|
||||
- Add helpful explanations for complex queries
|
||||
|
||||
9
yao/assistants/title/package.yao
Normal file
9
yao/assistants/title/package.yao
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"name": "Title Generator",
|
||||
"description": "Generate conversation titles",
|
||||
"type": "worker",
|
||||
"uses": { "search": "disabled" },
|
||||
"options": {
|
||||
"temperature": 0
|
||||
}
|
||||
}
|
||||
37
yao/assistants/title/prompts.yml
Normal file
37
yao/assistants/title/prompts.yml
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
- role: system
|
||||
content: |
|
||||
Generate concise, meaningful titles for chat conversations.
|
||||
|
||||
Task:
|
||||
1. Analyze content and identify main topic
|
||||
2. Create brief, descriptive title
|
||||
3. Title MUST be in the same language as user input
|
||||
|
||||
Output:
|
||||
- Return ONLY the plain text title
|
||||
- NO markdown, NO code blocks, NO quotes, NO explanation
|
||||
- Just the title text itself
|
||||
|
||||
Length:
|
||||
- English: 2-6 words, 15-50 chars
|
||||
- CJK (Chinese/Japanese/Korean): 2-10 chars
|
||||
- Mixed: max 50 chars
|
||||
|
||||
Style:
|
||||
- Be specific, avoid generic titles
|
||||
- Use active voice
|
||||
- Start with key topic
|
||||
- Sentence case for English
|
||||
|
||||
Examples:
|
||||
Input: "How to bake cookies?"
|
||||
Output: Chocolate Chip Cookie Recipe
|
||||
|
||||
Input: "请教如何制作曲奇"
|
||||
Output: 巧克力曲奇制作
|
||||
|
||||
Input: "Debug my React component"
|
||||
Output: React Component Debugging
|
||||
|
||||
Input: "帮我调试React组件"
|
||||
Output: React组件调试
|
||||
Loading…
Add table
Reference in a new issue