Enhance Search Functionality with Keyword Extraction and Intent Detection
- Updated the search handling to incorporate keyword extraction with weights, improving the relevance of search results. - Refactored the `shouldAutoSearch` method to return a `SearchIntent` struct, allowing for more nuanced control over search execution based on context. - Enhanced the `buildSearchRequests` function to utilize extracted keywords, optimizing search queries based on user input. - Improved the handling of search types and conditions, ensuring that the system can dynamically adjust search behavior based on intent and configuration. - Updated documentation and prompts to reflect changes in keyword extraction and search intent classification, providing clearer guidelines for usage.
This commit is contained in:
Max
parent
99ac7abdf1
commit
3cf3060e0c
16 changed files with 820 additions and 394 deletions
|
|
@ -205,8 +205,8 @@ func (ast *Assistant) Stream(ctx *context.Context, inputMessages []context.Messa
|
|||
// ================================================
|
||||
// Execute Auto Search (if enabled)
|
||||
// ================================================
|
||||
if ast.shouldAutoSearch(ctx, completionMessages, createResponse, opts) {
|
||||
refCtx := ast.executeAutoSearch(ctx, completionMessages, createResponse, opts)
|
||||
if intent := ast.shouldAutoSearch(ctx, completionMessages, createResponse, opts); intent != nil {
|
||||
refCtx := ast.executeAutoSearch(ctx, completionMessages, createResponse, intent, opts)
|
||||
if refCtx != nil && len(refCtx.References) > 0 {
|
||||
completionMessages = ast.injectSearchContext(completionMessages, refCtx)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,16 +17,32 @@ import (
|
|||
)
|
||||
|
||||
// shouldAutoSearch determines if auto search should be executed
|
||||
// Returns false if:
|
||||
// Returns nil if search should be skipped, otherwise returns SearchIntent with types to search
|
||||
// Search is skipped if:
|
||||
// - opts.Skip.Search is true
|
||||
// - createResponse.Search is false
|
||||
// - uses.search is "disabled"
|
||||
// - assistant has no search configuration
|
||||
// - needsearch intent detection returns false
|
||||
func (ast *Assistant) shouldAutoSearch(ctx *context.Context, messages []context.Message, createResponse *context.HookCreateResponse, opts *context.Options) bool {
|
||||
func (ast *Assistant) shouldAutoSearch(ctx *context.Context, messages []context.Message, createResponse *context.HookCreateResponse, opts *context.Options) *SearchIntent {
|
||||
// Check if search is skipped via options
|
||||
if opts != nil && opts.Skip != nil && opts.Skip.Search {
|
||||
ctx.Logger.Debug("Auto search skipped by opts.Skip.Search")
|
||||
return false
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check createResponse.Search field (highest priority from Create hook)
|
||||
// Supports: bool | SearchIntent | nil
|
||||
if createResponse != nil && createResponse.Search != nil {
|
||||
intent := parseSearchField(createResponse.Search)
|
||||
if intent != nil {
|
||||
if !intent.NeedSearch {
|
||||
ctx.Logger.Info("Auto search disabled by createResponse.Search")
|
||||
return nil
|
||||
}
|
||||
ctx.Logger.Info("Auto search controlled by createResponse.Search: types=%v", intent.SearchTypes)
|
||||
return intent
|
||||
}
|
||||
}
|
||||
|
||||
// Get merged uses configuration
|
||||
|
|
@ -35,27 +51,101 @@ func (ast *Assistant) shouldAutoSearch(ctx *context.Context, messages []context.
|
|||
// Check if search is explicitly disabled
|
||||
if uses != nil && uses.Search == "disabled" {
|
||||
ctx.Logger.Info("Auto search disabled by uses.search=disabled")
|
||||
return false
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check if assistant has search configuration
|
||||
if ast.Search == nil && (uses == nil || uses.Search == "") {
|
||||
return false
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check search intent using __yao.needsearch agent
|
||||
if !ast.checkSearchIntent(ctx, messages) {
|
||||
intent := ast.checkSearchIntent(ctx, messages)
|
||||
if intent == nil || !intent.NeedSearch {
|
||||
ctx.Logger.Info("Auto search skipped: intent detection returned false")
|
||||
return false
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check if search is enabled (builtin, agent, mcp, or empty means builtin)
|
||||
return true
|
||||
return intent
|
||||
}
|
||||
|
||||
// parseSearchField parses the Search field from HookCreateResponse
|
||||
// Supports: bool | SearchIntent | map[string]any | nil
|
||||
func parseSearchField(search any) *SearchIntent {
|
||||
if search == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch v := search.(type) {
|
||||
case bool:
|
||||
// bool: true = enable all, false = disable all
|
||||
if v {
|
||||
return &SearchIntent{
|
||||
NeedSearch: true,
|
||||
SearchTypes: []string{"web", "kb", "db"},
|
||||
Confidence: 1.0,
|
||||
Reason: "enabled by hook",
|
||||
}
|
||||
}
|
||||
return &SearchIntent{
|
||||
NeedSearch: false,
|
||||
SearchTypes: []string{},
|
||||
Confidence: 1.0,
|
||||
Reason: "disabled by hook",
|
||||
}
|
||||
|
||||
case *SearchIntent:
|
||||
// SearchIntent is alias for context.SearchIntent, so this covers both
|
||||
return v
|
||||
|
||||
case SearchIntent:
|
||||
return &v
|
||||
|
||||
case map[string]any:
|
||||
// Parse from map (e.g., from JSON)
|
||||
intent := &SearchIntent{
|
||||
NeedSearch: false,
|
||||
SearchTypes: []string{},
|
||||
Confidence: 0.5,
|
||||
}
|
||||
|
||||
if needSearch, ok := v["need_search"].(bool); ok {
|
||||
intent.NeedSearch = needSearch
|
||||
}
|
||||
|
||||
if types, ok := v["search_types"].([]any); ok {
|
||||
for _, t := range types {
|
||||
if typeStr, ok := t.(string); ok {
|
||||
intent.SearchTypes = append(intent.SearchTypes, typeStr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if confidence, ok := v["confidence"].(float64); ok {
|
||||
intent.Confidence = confidence
|
||||
}
|
||||
|
||||
if reason, ok := v["reason"].(string); ok {
|
||||
intent.Reason = reason
|
||||
}
|
||||
|
||||
return intent
|
||||
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// checkSearchIntent uses __yao.needsearch agent to determine if search is needed
|
||||
// Returns true if search is needed, false otherwise
|
||||
func (ast *Assistant) checkSearchIntent(ctx *context.Context, messages []context.Message) bool {
|
||||
// Returns SearchIntent with search types and confidence
|
||||
func (ast *Assistant) checkSearchIntent(ctx *context.Context, messages []context.Message) *SearchIntent {
|
||||
// Default intent: no search needed (fallback when agent unavailable or fails)
|
||||
defaultIntent := &SearchIntent{
|
||||
NeedSearch: false,
|
||||
SearchTypes: []string{},
|
||||
Confidence: 0,
|
||||
}
|
||||
|
||||
// Filter out system messages and pass full conversation context
|
||||
var intentMessages []context.Message
|
||||
for _, msg := range messages {
|
||||
|
|
@ -65,14 +155,14 @@ func (ast *Assistant) checkSearchIntent(ctx *context.Context, messages []context
|
|||
}
|
||||
|
||||
if len(intentMessages) == 0 {
|
||||
return true // No messages, proceed with search
|
||||
return defaultIntent // No messages, skip search
|
||||
}
|
||||
|
||||
// Try to get __yao.needsearch agent
|
||||
needsearchAst, err := Get("__yao.needsearch")
|
||||
if err != nil {
|
||||
ctx.Logger.Debug("__yao.needsearch agent not available: %v, proceeding with search", err)
|
||||
return true // Agent not available, proceed with search
|
||||
ctx.Logger.Debug("__yao.needsearch agent not available: %v, skipping search", err)
|
||||
return defaultIntent // Agent not available, skip search
|
||||
}
|
||||
|
||||
// === Output: Send loading message ===
|
||||
|
|
@ -90,10 +180,10 @@ func (ast *Assistant) checkSearchIntent(ctx *context.Context, messages []context
|
|||
|
||||
result, err := needsearchAst.Stream(ctx, intentMessages, opts)
|
||||
if err != nil {
|
||||
ctx.Logger.Debug("__yao.needsearch failed: %v, proceeding with search", err)
|
||||
// === Output: Send done (error case, proceed with search) ===
|
||||
ast.sendIntentDone(ctx, loadingID, true, "")
|
||||
return true // On error, proceed with search
|
||||
ctx.Logger.Debug("__yao.needsearch failed: %v, skipping search", err)
|
||||
// === Output: Send done (error case, skip search) ===
|
||||
ast.sendIntentDone(ctx, loadingID, false, "")
|
||||
return defaultIntent // On error, skip search
|
||||
}
|
||||
|
||||
// Parse the result
|
||||
|
|
@ -109,11 +199,12 @@ func (ast *Assistant) checkSearchIntent(ctx *context.Context, messages []context
|
|||
intentData = nextData
|
||||
}
|
||||
|
||||
if needSearch, ok := intentData["need_search"].(bool); ok {
|
||||
reason, _ := intentData["reason"].(string)
|
||||
ctx.Logger.Debug("Search intent (from Next): need_search=%v, reason=%s", needSearch, reason)
|
||||
ast.sendIntentDone(ctx, loadingID, needSearch, reason)
|
||||
return needSearch
|
||||
intent := parseSearchIntent(intentData)
|
||||
if intent != nil {
|
||||
ctx.Logger.Debug("Search intent (from Next): need_search=%v, types=%v, confidence=%.2f, reason=%s",
|
||||
intent.NeedSearch, intent.SearchTypes, intent.Confidence, intent.Reason)
|
||||
ast.sendIntentDone(ctx, loadingID, intent.NeedSearch, intent.Reason)
|
||||
return intent
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -122,24 +213,75 @@ func (ast *Assistant) checkSearchIntent(ctx *context.Context, messages []context
|
|||
if result.Completion != nil {
|
||||
content, ok := result.Completion.Content.(string)
|
||||
if !ok || content == "" {
|
||||
ast.sendIntentDone(ctx, loadingID, true, "")
|
||||
return true
|
||||
ast.sendIntentDone(ctx, loadingID, false, "")
|
||||
return defaultIntent
|
||||
}
|
||||
needSearch, reason := parseNeedSearchFromContent(content)
|
||||
ctx.Logger.Debug("Search intent (from Content): need_search=%v, reason=%s", needSearch, reason)
|
||||
ast.sendIntentDone(ctx, loadingID, needSearch, reason)
|
||||
return needSearch
|
||||
intent := parseSearchIntentFromContent(content)
|
||||
ctx.Logger.Debug("Search intent (from Content): need_search=%v, types=%v, confidence=%.2f, reason=%s",
|
||||
intent.NeedSearch, intent.SearchTypes, intent.Confidence, intent.Reason)
|
||||
ast.sendIntentDone(ctx, loadingID, intent.NeedSearch, intent.Reason)
|
||||
return intent
|
||||
}
|
||||
|
||||
// Default: proceed with search if we can't parse the result
|
||||
// Default: skip search if we can't parse the result
|
||||
// === Output: Send done (default case) ===
|
||||
ast.sendIntentDone(ctx, loadingID, true, "")
|
||||
return true
|
||||
ast.sendIntentDone(ctx, loadingID, false, "")
|
||||
return defaultIntent
|
||||
}
|
||||
|
||||
// parseNeedSearchFromContent parses need_search result from LLM completion content
|
||||
// parseSearchIntent parses SearchIntent from intent data map
|
||||
func parseSearchIntent(intentData map[string]interface{}) *SearchIntent {
|
||||
if intentData == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
needSearch, ok := intentData["need_search"].(bool)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
intent := &SearchIntent{
|
||||
NeedSearch: needSearch,
|
||||
SearchTypes: []string{},
|
||||
Confidence: 0.5, // Default confidence
|
||||
}
|
||||
|
||||
// Parse search_types
|
||||
if types, ok := intentData["search_types"].([]interface{}); ok {
|
||||
for _, t := range types {
|
||||
if typeStr, ok := t.(string); ok {
|
||||
// Validate type
|
||||
typeStr = strings.ToLower(typeStr)
|
||||
if typeStr == "web" || typeStr == "kb" || typeStr == "db" {
|
||||
intent.SearchTypes = append(intent.SearchTypes, typeStr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parse confidence
|
||||
if confidence, ok := intentData["confidence"].(float64); ok {
|
||||
intent.Confidence = confidence
|
||||
}
|
||||
|
||||
// Parse reason
|
||||
if reason, ok := intentData["reason"].(string); ok {
|
||||
intent.Reason = reason
|
||||
}
|
||||
|
||||
return intent
|
||||
}
|
||||
|
||||
// parseSearchIntentFromContent parses SearchIntent from LLM completion content
|
||||
// Handles JSON wrapped in markdown code blocks
|
||||
func parseNeedSearchFromContent(content string) (bool, string) {
|
||||
func parseSearchIntentFromContent(content string) *SearchIntent {
|
||||
// Default intent: no search needed
|
||||
defaultIntent := &SearchIntent{
|
||||
NeedSearch: false,
|
||||
SearchTypes: []string{},
|
||||
Confidence: 0,
|
||||
}
|
||||
|
||||
// Remove markdown code block if present
|
||||
content = strings.TrimSpace(content)
|
||||
if strings.HasPrefix(content, "```json") {
|
||||
|
|
@ -155,17 +297,16 @@ func parseNeedSearchFromContent(content string) (bool, string) {
|
|||
// Try to parse JSON
|
||||
var result map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(content), &result); err != nil {
|
||||
// Failed to parse, default to search
|
||||
return true, ""
|
||||
// Failed to parse, default to no search
|
||||
return defaultIntent
|
||||
}
|
||||
|
||||
needSearch, ok := result["need_search"].(bool)
|
||||
if !ok {
|
||||
return true, ""
|
||||
intent := parseSearchIntent(result)
|
||||
if intent == nil {
|
||||
return defaultIntent
|
||||
}
|
||||
|
||||
reason, _ := result["reason"].(string)
|
||||
return needSearch, reason
|
||||
return intent
|
||||
}
|
||||
|
||||
// sendIntentLoading sends the initial intent detection loading message
|
||||
|
|
@ -261,10 +402,11 @@ func (ast *Assistant) getMergedSearchUses(createResponse *context.HookCreateResp
|
|||
return uses
|
||||
}
|
||||
|
||||
// executeAutoSearch executes auto search based on configuration
|
||||
// executeAutoSearch executes auto search based on configuration and intent
|
||||
// Returns ReferenceContext with results and formatted context
|
||||
// intent specifies which search types to execute (from needsearch agent)
|
||||
// opts is optional, used to check Skip.Keyword
|
||||
func (ast *Assistant) executeAutoSearch(ctx *context.Context, messages []context.Message, createResponse *context.HookCreateResponse, opts ...*context.Options) *searchTypes.ReferenceContext {
|
||||
func (ast *Assistant) executeAutoSearch(ctx *context.Context, messages []context.Message, createResponse *context.HookCreateResponse, intent *SearchIntent, opts ...*context.Options) *searchTypes.ReferenceContext {
|
||||
ctx.Logger.Phase("Search")
|
||||
defer ctx.Logger.PhaseComplete("Search")
|
||||
|
||||
|
|
@ -301,33 +443,23 @@ func (ast *Assistant) executeAutoSearch(ctx *context.Context, messages []context
|
|||
skipKeyword = opts[0].Skip.Keyword
|
||||
}
|
||||
|
||||
// Extract keywords for web search if:
|
||||
// 1. uses.keyword is configured (not empty)
|
||||
// 2. Skip.Keyword is not true
|
||||
// 3. Web search is enabled
|
||||
var extractedKeywords []string
|
||||
webSearchEnabled := searchConfig != nil && searchConfig.Web != nil
|
||||
if webSearchEnabled && !skipKeyword && searchUses.Keyword != "" {
|
||||
extractor := keyword.NewExtractor(searchUses.Keyword, searchConfig.Keyword)
|
||||
keywords, err := extractor.Extract(ctx, query, nil)
|
||||
if err != nil {
|
||||
ctx.Logger.Warn("Keyword extraction failed, using original query: %v", err)
|
||||
} else if len(keywords) > 0 {
|
||||
extractedKeywords = keywords
|
||||
// Use extracted keywords as the search query for web search
|
||||
optimizedQuery := strings.Join(keywords, " ")
|
||||
ctx.Logger.Info("Extracted keywords for web search: %s -> %s", truncateString(query, 30), optimizedQuery)
|
||||
query = optimizedQuery
|
||||
}
|
||||
// Build search requests based on configuration and intent
|
||||
// Keyword extraction is done inside buildSearchRequests for web search
|
||||
buildOpts := &buildSearchRequestsOptions{
|
||||
skipKeyword: skipKeyword,
|
||||
usesKeyword: searchUses.Keyword,
|
||||
}
|
||||
|
||||
// Build search requests based on configuration
|
||||
requests := ast.buildSearchRequests(query, searchConfig)
|
||||
requests, extractedKeywords := ast.buildSearchRequests(ctx, query, searchConfig, intent, buildOpts)
|
||||
if len(requests) == 0 {
|
||||
ctx.Logger.Info("No search requests to execute")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Update query if keywords were extracted (for web search)
|
||||
if len(extractedKeywords) > 0 {
|
||||
query = keywordsToQuery(extractedKeywords)
|
||||
}
|
||||
|
||||
// === Output: Send loading message ===
|
||||
loadingID := ast.sendSearchLoading(ctx)
|
||||
|
||||
|
|
@ -425,6 +557,52 @@ func (ast *Assistant) sendSearchLoading(ctx *context.Context) string {
|
|||
return msgID
|
||||
}
|
||||
|
||||
// sendKeywordLoading sends the keyword extraction loading message
|
||||
// Returns the message ID for later replacement
|
||||
func (ast *Assistant) sendKeywordLoading(ctx *context.Context) string {
|
||||
loadingMsg := i18n.T(ctx.Locale, "search.keyword.loading")
|
||||
|
||||
msg := &message.Message{
|
||||
Type: "loading",
|
||||
Props: map[string]any{
|
||||
"message": loadingMsg,
|
||||
},
|
||||
}
|
||||
|
||||
// Send and get message ID
|
||||
msgID, err := ctx.SendStream(msg)
|
||||
if err != nil {
|
||||
ctx.Logger.Warn("Failed to send keyword loading message: %v", err)
|
||||
return ""
|
||||
}
|
||||
|
||||
return msgID
|
||||
}
|
||||
|
||||
// sendKeywordDone replaces keyword loading with done message
|
||||
func (ast *Assistant) sendKeywordDone(ctx *context.Context, loadingID string, success bool) {
|
||||
if loadingID == "" {
|
||||
return
|
||||
}
|
||||
|
||||
resultMsg := i18n.T(ctx.Locale, "search.keyword.done")
|
||||
|
||||
msg := &message.Message{
|
||||
MessageID: loadingID,
|
||||
Delta: true,
|
||||
DeltaAction: message.DeltaReplace,
|
||||
Type: "loading",
|
||||
Props: map[string]any{
|
||||
"message": resultMsg,
|
||||
"done": true,
|
||||
},
|
||||
}
|
||||
|
||||
if err := ctx.Send(msg); err != nil {
|
||||
ctx.Logger.Warn("Failed to send keyword done message: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// sendSearchResult replaces loading with result message (without done flag)
|
||||
func (ast *Assistant) sendSearchResult(ctx *context.Context, loadingID string, count int) {
|
||||
if loadingID == "" {
|
||||
|
|
@ -554,22 +732,67 @@ func (ast *Assistant) completeSearchTrace(node traceTypes.Node, resultCount int,
|
|||
})
|
||||
}
|
||||
|
||||
// buildSearchRequests builds search requests based on assistant configuration
|
||||
func (ast *Assistant) buildSearchRequests(query string, config *searchTypes.Config) []*searchTypes.Request {
|
||||
var requests []*searchTypes.Request
|
||||
// buildSearchRequestsOptions contains options for building search requests
|
||||
type buildSearchRequestsOptions struct {
|
||||
skipKeyword bool // Skip keyword extraction
|
||||
usesKeyword string // Keyword extractor config: "builtin", "<assistant-id>", "mcp:<server>.<tool>"
|
||||
}
|
||||
|
||||
// buildSearchRequests builds search requests based on assistant configuration and intent
|
||||
// intent specifies which search types to execute (from needsearch agent)
|
||||
// Returns requests and extracted keywords (if any)
|
||||
func (ast *Assistant) buildSearchRequests(ctx *context.Context, query string, config *searchTypes.Config, intent *SearchIntent, opts *buildSearchRequestsOptions) ([]*searchTypes.Request, []searchTypes.Keyword) {
|
||||
var requests []*searchTypes.Request
|
||||
var extractedKeywords []searchTypes.Keyword
|
||||
|
||||
// Helper to check if a search type is allowed by intent
|
||||
isTypeAllowed := func(searchType string) bool {
|
||||
if intent == nil || len(intent.SearchTypes) == 0 {
|
||||
return true // No intent or empty types means all types allowed
|
||||
}
|
||||
for _, t := range intent.SearchTypes {
|
||||
if t == searchType {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Web search - check if web search is configured and allowed by intent
|
||||
if config != nil && config.Web != nil && isTypeAllowed("web") {
|
||||
webQuery := query
|
||||
|
||||
// Extract keywords for web search if configured
|
||||
if opts != nil && !opts.skipKeyword && opts.usesKeyword != "" {
|
||||
// === Output: Send keyword extraction loading ===
|
||||
keywordLoadingID := ast.sendKeywordLoading(ctx)
|
||||
|
||||
extractor := keyword.NewExtractor(opts.usesKeyword, config.Keyword)
|
||||
keywords, err := extractor.Extract(ctx, query, nil)
|
||||
if err != nil {
|
||||
ctx.Logger.Warn("Keyword extraction failed, using original query: %v", err)
|
||||
ast.sendKeywordDone(ctx, keywordLoadingID, false)
|
||||
} else if len(keywords) > 0 {
|
||||
extractedKeywords = keywords
|
||||
// Use extracted keywords as the search query for web search
|
||||
webQuery = keywordsToQuery(keywords)
|
||||
ctx.Logger.Info("Extracted keywords for web search: %s -> %s", truncateString(query, 30), webQuery)
|
||||
ast.sendKeywordDone(ctx, keywordLoadingID, true)
|
||||
} else {
|
||||
ast.sendKeywordDone(ctx, keywordLoadingID, true)
|
||||
}
|
||||
}
|
||||
|
||||
// Web search - check if web search is configured
|
||||
if config != nil && config.Web != nil {
|
||||
requests = append(requests, &searchTypes.Request{
|
||||
Type: searchTypes.SearchTypeWeb,
|
||||
Query: query,
|
||||
Query: webQuery,
|
||||
Source: searchTypes.SourceAuto,
|
||||
Limit: config.Web.MaxResults,
|
||||
})
|
||||
}
|
||||
|
||||
// KB search - check if KB is configured
|
||||
if ast.KB != nil && len(ast.KB.Collections) > 0 {
|
||||
// KB search - check if KB is configured and allowed by intent
|
||||
if ast.KB != nil && len(ast.KB.Collections) > 0 && isTypeAllowed("kb") {
|
||||
limit := 10
|
||||
threshold := 0.7
|
||||
if config != nil && config.KB != nil {
|
||||
|
|
@ -579,7 +802,7 @@ func (ast *Assistant) buildSearchRequests(query string, config *searchTypes.Conf
|
|||
}
|
||||
requests = append(requests, &searchTypes.Request{
|
||||
Type: searchTypes.SearchTypeKB,
|
||||
Query: query,
|
||||
Query: query, // KB uses original query for semantic search
|
||||
Source: searchTypes.SourceAuto,
|
||||
Limit: limit,
|
||||
Collections: ast.KB.Collections,
|
||||
|
|
@ -588,22 +811,22 @@ func (ast *Assistant) buildSearchRequests(query string, config *searchTypes.Conf
|
|||
})
|
||||
}
|
||||
|
||||
// DB search - check if DB is configured
|
||||
if ast.DB != nil && len(ast.DB.Models) > 0 {
|
||||
// DB search - check if DB is configured and allowed by intent
|
||||
if ast.DB != nil && len(ast.DB.Models) > 0 && isTypeAllowed("db") {
|
||||
limit := 20
|
||||
if config != nil && config.DB != nil && config.DB.MaxResults > 0 {
|
||||
limit = config.DB.MaxResults
|
||||
}
|
||||
requests = append(requests, &searchTypes.Request{
|
||||
Type: searchTypes.SearchTypeDB,
|
||||
Query: query,
|
||||
Query: query, // DB uses original query for QueryDSL generation
|
||||
Source: searchTypes.SourceAuto,
|
||||
Limit: limit,
|
||||
Models: ast.DB.Models,
|
||||
})
|
||||
}
|
||||
|
||||
return requests
|
||||
return requests, extractedKeywords
|
||||
}
|
||||
|
||||
// injectSearchContext injects search results into messages
|
||||
|
|
@ -699,7 +922,7 @@ func truncateString(s string, maxLen int) string {
|
|||
// SearchExecutionResult holds all data from search execution for storage
|
||||
type SearchExecutionResult struct {
|
||||
Query string // Original query (before keyword optimization)
|
||||
Keywords []string // Extracted keywords
|
||||
Keywords []searchTypes.Keyword // Extracted keywords with weights
|
||||
Config map[string]any // Search config used
|
||||
RefCtx *searchTypes.ReferenceContext // Reference context with results
|
||||
Duration int64 // Search duration in ms
|
||||
|
|
@ -707,6 +930,54 @@ type SearchExecutionResult struct {
|
|||
SearchType string // "auto", "web", "kb", "db"
|
||||
}
|
||||
|
||||
// keywordsToQuery converts keywords with weights to a search query string
|
||||
// Keywords are sorted by weight (descending) and joined with spaces
|
||||
func keywordsToQuery(keywords []searchTypes.Keyword) string {
|
||||
if len(keywords) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Sort by weight descending (higher weight first)
|
||||
sorted := make([]searchTypes.Keyword, len(keywords))
|
||||
copy(sorted, keywords)
|
||||
for i := 0; i < len(sorted)-1; i++ {
|
||||
for j := i + 1; j < len(sorted); j++ {
|
||||
if sorted[j].W > sorted[i].W {
|
||||
sorted[i], sorted[j] = sorted[j], sorted[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Join keywords
|
||||
parts := make([]string, len(sorted))
|
||||
for i, kw := range sorted {
|
||||
parts[i] = kw.K
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
// keywordsToStrings converts keywords to string slice for storage
|
||||
func keywordsToStrings(keywords []searchTypes.Keyword) []string {
|
||||
if len(keywords) == 0 {
|
||||
return nil
|
||||
}
|
||||
result := make([]string, len(keywords))
|
||||
for i, kw := range keywords {
|
||||
result[i] = kw.K
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// containsSearchType checks if a search type is in the list
|
||||
func containsSearchType(types []string, searchType string) bool {
|
||||
for _, t := range types {
|
||||
if t == searchType {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// saveSearch saves search results to storage
|
||||
// Called after search execution completes (success or failure)
|
||||
func (ast *Assistant) saveSearch(ctx *context.Context, execResult *SearchExecutionResult) {
|
||||
|
|
@ -722,7 +993,7 @@ func (ast *Assistant) saveSearch(ctx *context.Context, execResult *SearchExecuti
|
|||
RequestID: ctx.RequestID(),
|
||||
ChatID: ctx.ChatID,
|
||||
Query: execResult.Query,
|
||||
Keywords: execResult.Keywords,
|
||||
Keywords: keywordsToStrings(execResult.Keywords),
|
||||
Config: execResult.Config,
|
||||
Source: execResult.SearchType,
|
||||
Duration: execResult.Duration,
|
||||
|
|
|
|||
|
|
@ -85,6 +85,10 @@ type NextProcessContext struct {
|
|||
CreateResponse *chatctx.HookCreateResponse // Create hook response
|
||||
}
|
||||
|
||||
// SearchIntent is an alias for context.SearchIntent
|
||||
// Used for search intent detection from __yao.needsearch agent
|
||||
type SearchIntent = chatctx.SearchIntent
|
||||
|
||||
// ParsedContent extracts the actual tool return value from MCP ToolContent array
|
||||
// According to MCP protocol:
|
||||
// - Content is []ToolContent array
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ func (opts *Options) ToMap() map[string]interface{} {
|
|||
result["mode"] = opts.Mode
|
||||
}
|
||||
if opts.Search != nil {
|
||||
result["search"] = *opts.Search
|
||||
result["search"] = opts.Search
|
||||
}
|
||||
if opts.Skip != nil {
|
||||
result["skip"] = opts.Skip
|
||||
|
|
@ -50,8 +50,9 @@ func OptionsFromMap(m map[string]interface{}) *Options {
|
|||
if mode, ok := m["mode"].(string); ok {
|
||||
opts.Mode = mode
|
||||
}
|
||||
if search, ok := m["search"].(bool); ok {
|
||||
opts.Search = &search
|
||||
// Search supports: bool | SearchIntent | map[string]any | nil
|
||||
if search := m["search"]; search != nil {
|
||||
opts.Search = search
|
||||
}
|
||||
if skipMap, ok := m["skip"].(map[string]interface{}); ok {
|
||||
skip := &Skip{}
|
||||
|
|
|
|||
|
|
@ -266,6 +266,15 @@ type Context struct {
|
|||
Metadata map[string]interface{} `json:"metadata,omitempty"` // The metadata of the request, it will be used to pass data to the page
|
||||
}
|
||||
|
||||
// SearchIntent represents the result of search intent detection
|
||||
// Used by Create hook to specify fine-grained search behavior
|
||||
type SearchIntent struct {
|
||||
NeedSearch bool `json:"need_search"` // Whether search is needed
|
||||
SearchTypes []string `json:"search_types,omitempty"` // Types of search to perform: "web", "kb", "db"
|
||||
Confidence float64 `json:"confidence,omitempty"` // Confidence level (0-1)
|
||||
Reason string `json:"reason,omitempty"` // Reason for the decision
|
||||
}
|
||||
|
||||
// Options represents the options for the context
|
||||
type Options struct {
|
||||
|
||||
|
|
@ -284,8 +293,11 @@ type Options struct {
|
|||
// Disable global prompts, default is false
|
||||
DisableGlobalPrompts bool `json:"disable_global_prompts,omitempty"` // Temporarily disable global prompts for this request
|
||||
|
||||
// Search mode, default is true
|
||||
Search *bool `json:"search,omitempty"` // Search mode, default is true
|
||||
// Search controls search behavior, supports multiple types:
|
||||
// - bool: true = enable all search types, false = disable all search
|
||||
// - SearchIntent: fine-grained control with specific types, confidence, etc.
|
||||
// - nil: use default behavior (determined by __yao.needsearch agent)
|
||||
Search any `json:"search,omitempty"` // Search mode: bool | SearchIntent | nil
|
||||
|
||||
// Agent mode, use to select the mode of the request, default is "chat"
|
||||
Mode string `json:"mode,omitempty"` // Agent mode, use to select the mode of the request, default is "chat"
|
||||
|
|
@ -372,6 +384,12 @@ type HookCreateResponse struct {
|
|||
|
||||
// ForceUses controls whether to force using Uses tools even when model has native capabilities
|
||||
ForceUses *bool `json:"force_uses,omitempty"` // Force using Uses tools regardless of model capabilities
|
||||
|
||||
// Search controls search behavior, supports multiple types:
|
||||
// - bool: true = enable all search types, false = disable all search
|
||||
// - SearchIntent: fine-grained control with specific types, confidence, etc.
|
||||
// - nil: use default behavior (determined by __yao.needsearch agent)
|
||||
Search any `json:"search,omitempty"` // Search mode: bool | SearchIntent | nil
|
||||
}
|
||||
|
||||
// NextHookPayload payload for the next hook
|
||||
|
|
|
|||
|
|
@ -112,6 +112,10 @@ func init() {
|
|||
"search.intent.need_search": "Searching for references...",
|
||||
"search.intent.no_search": "No references needed",
|
||||
|
||||
// Keyword Extraction: assistant/search.go - Keyword extraction messages
|
||||
"search.keyword.loading": "Analyzing conversation...",
|
||||
"search.keyword.done": "Analysis complete",
|
||||
|
||||
// Search: assistant/search.go - Trace labels
|
||||
"search.trace.label": "Search",
|
||||
"search.trace.description": "Search the web and knowledge base for relevant information",
|
||||
|
|
@ -201,6 +205,10 @@ func init() {
|
|||
"search.intent.need_search": "正在查询相关资料...",
|
||||
"search.intent.no_search": "无需查询资料",
|
||||
|
||||
// Keyword Extraction: assistant/search.go - Keyword extraction messages
|
||||
"search.keyword.loading": "正在分析对话内容...",
|
||||
"search.keyword.done": "分析完成",
|
||||
|
||||
// Search: assistant/search.go - Trace labels
|
||||
"search.trace.label": "搜索",
|
||||
"search.trace.description": "搜索网络和知识库获取相关信息",
|
||||
|
|
@ -318,6 +326,10 @@ func init() {
|
|||
"search.intent.need_search": "正在查询相关资料...",
|
||||
"search.intent.no_search": "无需查询资料",
|
||||
|
||||
// Keyword Extraction: assistant/search.go - Keyword extraction messages
|
||||
"search.keyword.loading": "正在分析对话内容...",
|
||||
"search.keyword.done": "分析完成",
|
||||
|
||||
// Search: assistant/search.go - Trace labels
|
||||
"search.trace.label": "搜索",
|
||||
"search.trace.description": "搜索网络和知识库获取相关信息",
|
||||
|
|
|
|||
|
|
@ -209,10 +209,14 @@ func initAssistant() error {
|
|||
// Set global Uses configuration
|
||||
if agentDSL.Uses != nil {
|
||||
globalUses := &context.Uses{
|
||||
Vision: agentDSL.Uses.Vision,
|
||||
Audio: agentDSL.Uses.Audio,
|
||||
Search: agentDSL.Uses.Search,
|
||||
Fetch: agentDSL.Uses.Fetch,
|
||||
Vision: agentDSL.Uses.Vision,
|
||||
Audio: agentDSL.Uses.Audio,
|
||||
Search: agentDSL.Uses.Search,
|
||||
Fetch: agentDSL.Uses.Fetch,
|
||||
Web: agentDSL.Uses.Web,
|
||||
Keyword: agentDSL.Uses.Keyword,
|
||||
QueryDSL: agentDSL.Uses.QueryDSL,
|
||||
Rerank: agentDSL.Uses.Rerank,
|
||||
}
|
||||
assistant.SetGlobalUses(globalUses)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,8 +23,8 @@ func NewAgentProvider(agentID string) *AgentProvider {
|
|||
}
|
||||
|
||||
// Extract extracts keywords by calling the target agent
|
||||
// The agent receives the content and returns extracted keywords
|
||||
func (p *AgentProvider) Extract(ctx *agentContext.Context, content string, opts *types.KeywordOptions) ([]string, error) {
|
||||
// The agent receives the content and returns extracted keywords with weights
|
||||
func (p *AgentProvider) Extract(ctx *agentContext.Context, content string, opts *types.KeywordOptions) ([]types.Keyword, error) {
|
||||
if ctx == nil {
|
||||
return nil, fmt.Errorf("context is required for agent keyword extraction")
|
||||
}
|
||||
|
|
@ -77,19 +77,20 @@ func (p *AgentProvider) Extract(ctx *agentContext.Context, content string, opts
|
|||
// Now that agent.Stream() returns *context.Response directly,
|
||||
// we can access fields without type assertions.
|
||||
//
|
||||
// The agent returns keywords in response.Next field
|
||||
func (p *AgentProvider) parseResponse(response *agentContext.Response) ([]string, error) {
|
||||
// The agent returns keywords in response.Next field as {data: {keywords: [{k, w}, ...]}}
|
||||
func (p *AgentProvider) parseResponse(response *agentContext.Response) ([]types.Keyword, error) {
|
||||
if response == nil || response.Next == nil {
|
||||
return []string{}, nil
|
||||
return []types.Keyword{}, nil
|
||||
}
|
||||
|
||||
return p.parseNextData(response.Next)
|
||||
}
|
||||
|
||||
// parseNextData extracts keywords from Next hook data
|
||||
func (p *AgentProvider) parseNextData(next interface{}) ([]string, error) {
|
||||
// Expected format: {data: {keywords: [{k: "keyword", w: 0.9}, ...]}}
|
||||
func (p *AgentProvider) parseNextData(next interface{}) ([]types.Keyword, error) {
|
||||
if next == nil {
|
||||
return []string{}, nil
|
||||
return []types.Keyword{}, nil
|
||||
}
|
||||
|
||||
// Try to convert to map first (most common case)
|
||||
|
|
@ -101,32 +102,26 @@ func (p *AgentProvider) parseNextData(next interface{}) ([]string, error) {
|
|||
case string:
|
||||
// Try to parse as JSON
|
||||
if err := json.Unmarshal([]byte(v), &data); err != nil {
|
||||
// Not a JSON object, try as array
|
||||
var keywords []string
|
||||
// Not a JSON object, try as array of keywords
|
||||
var keywords []types.Keyword
|
||||
if err := json.Unmarshal([]byte(v), &keywords); err == nil {
|
||||
return keywords, nil
|
||||
}
|
||||
// Return as single keyword
|
||||
return []string{v}, nil
|
||||
// Return as single keyword with default weight
|
||||
return []types.Keyword{{K: v, W: 0.5}}, nil
|
||||
}
|
||||
case []string:
|
||||
case []types.Keyword:
|
||||
return v, nil
|
||||
case []interface{}:
|
||||
keywords := make([]string, 0, len(v))
|
||||
for _, item := range v {
|
||||
if s, ok := item.(string); ok {
|
||||
keywords = append(keywords, s)
|
||||
}
|
||||
}
|
||||
return keywords, nil
|
||||
return p.extractKeywordsFromArray(v)
|
||||
default:
|
||||
// Try to marshal and unmarshal
|
||||
jsonBytes, err := json.Marshal(next)
|
||||
if err != nil {
|
||||
return []string{}, nil
|
||||
return []types.Keyword{}, nil
|
||||
}
|
||||
if err := json.Unmarshal(jsonBytes, &data); err != nil {
|
||||
return []string{}, nil
|
||||
return []types.Keyword{}, nil
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -144,28 +139,48 @@ func (p *AgentProvider) parseNextData(next interface{}) ([]string, error) {
|
|||
return p.extractKeywordsFromValue(d)
|
||||
}
|
||||
|
||||
return []string{}, nil
|
||||
return []types.Keyword{}, nil
|
||||
}
|
||||
|
||||
// extractKeywordsFromValue extracts string array from various types
|
||||
func (p *AgentProvider) extractKeywordsFromValue(v interface{}) ([]string, error) {
|
||||
// extractKeywordsFromValue extracts Keyword array from various types
|
||||
func (p *AgentProvider) extractKeywordsFromValue(v interface{}) ([]types.Keyword, error) {
|
||||
switch kw := v.(type) {
|
||||
case []string:
|
||||
case []types.Keyword:
|
||||
return kw, nil
|
||||
case []interface{}:
|
||||
keywords := make([]string, 0, len(kw))
|
||||
for _, item := range kw {
|
||||
if s, ok := item.(string); ok {
|
||||
keywords = append(keywords, s)
|
||||
}
|
||||
}
|
||||
return keywords, nil
|
||||
return p.extractKeywordsFromArray(kw)
|
||||
case string:
|
||||
var keywords []string
|
||||
var keywords []types.Keyword
|
||||
if err := json.Unmarshal([]byte(kw), &keywords); err == nil {
|
||||
return keywords, nil
|
||||
}
|
||||
return []string{kw}, nil
|
||||
return []types.Keyword{{K: kw, W: 0.5}}, nil
|
||||
}
|
||||
return []string{}, nil
|
||||
return []types.Keyword{}, nil
|
||||
}
|
||||
|
||||
// extractKeywordsFromArray extracts keywords from []interface{}
|
||||
// Handles both {k, w} objects and plain strings
|
||||
func (p *AgentProvider) extractKeywordsFromArray(items []interface{}) ([]types.Keyword, error) {
|
||||
keywords := make([]types.Keyword, 0, len(items))
|
||||
for _, item := range items {
|
||||
switch v := item.(type) {
|
||||
case map[string]interface{}:
|
||||
// Handle {k: "keyword", w: 0.9} format
|
||||
k, _ := v["k"].(string)
|
||||
w, _ := v["w"].(float64)
|
||||
if k != "" {
|
||||
if w == 0 {
|
||||
w = 0.5 // Default weight
|
||||
}
|
||||
keywords = append(keywords, types.Keyword{K: k, W: w})
|
||||
}
|
||||
case string:
|
||||
// Plain string, use default weight
|
||||
if v != "" {
|
||||
keywords = append(keywords, types.Keyword{K: v, W: 0.5})
|
||||
}
|
||||
}
|
||||
}
|
||||
return keywords, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,8 +34,8 @@ func NewExtractor(usesKeyword string, cfg *types.KeywordConfig) *Extractor {
|
|||
}
|
||||
|
||||
// Extract extracts keywords from content based on configured mode
|
||||
// Returns a list of keywords optimized for search queries
|
||||
func (e *Extractor) Extract(ctx *context.Context, content string, opts *types.KeywordOptions) ([]string, error) {
|
||||
// Returns a list of keywords with weights optimized for search queries
|
||||
func (e *Extractor) Extract(ctx *context.Context, content string, opts *types.KeywordOptions) ([]types.Keyword, error) {
|
||||
// Merge options with config defaults
|
||||
mergedOpts := e.mergeOptions(opts)
|
||||
|
||||
|
|
@ -83,7 +83,7 @@ func (e *Extractor) mergeOptions(opts *types.KeywordOptions) *types.KeywordOptio
|
|||
|
||||
// agentExtract delegates to an LLM-powered assistant
|
||||
// The assistant can understand context and extract semantically relevant keywords
|
||||
func (e *Extractor) agentExtract(ctx *context.Context, content string, agentID string, opts *types.KeywordOptions) ([]string, error) {
|
||||
func (e *Extractor) agentExtract(ctx *context.Context, content string, agentID string, opts *types.KeywordOptions) ([]types.Keyword, error) {
|
||||
if ctx == nil {
|
||||
return nil, fmt.Errorf("context is required for keyword extraction")
|
||||
}
|
||||
|
|
@ -93,7 +93,7 @@ func (e *Extractor) agentExtract(ctx *context.Context, content string, agentID s
|
|||
|
||||
// mcpExtract calls an external MCP tool
|
||||
// Format: "mcp:<server>.<tool>"
|
||||
func (e *Extractor) mcpExtract(ctx *context.Context, content string, opts *types.KeywordOptions) ([]string, error) {
|
||||
func (e *Extractor) mcpExtract(ctx *context.Context, content string, opts *types.KeywordOptions) ([]types.Keyword, error) {
|
||||
mcpRef := strings.TrimPrefix(e.usesKeyword, "mcp:")
|
||||
provider, err := NewMCPProvider(mcpRef)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ func NewMCPProvider(mcpRef string) (*MCPProvider, error) {
|
|||
}
|
||||
|
||||
// Extract extracts keywords by calling the MCP tool
|
||||
func (p *MCPProvider) Extract(ctx *agentContext.Context, content string, opts *types.KeywordOptions) ([]string, error) {
|
||||
func (p *MCPProvider) Extract(ctx *agentContext.Context, content string, opts *types.KeywordOptions) ([]types.Keyword, error) {
|
||||
// Get MCP client
|
||||
client, err := mcp.Select(p.serverID)
|
||||
if err != nil {
|
||||
|
|
@ -56,9 +56,9 @@ func (p *MCPProvider) Extract(ctx *agentContext.Context, content string, opts *t
|
|||
}
|
||||
|
||||
// parseResult extracts keywords from the MCP tool response
|
||||
func (p *MCPProvider) parseResult(result *gouMCPTypes.CallToolResponse) ([]string, error) {
|
||||
func (p *MCPProvider) parseResult(result *gouMCPTypes.CallToolResponse) ([]types.Keyword, error) {
|
||||
if result == nil {
|
||||
return []string{}, nil
|
||||
return []types.Keyword{}, nil
|
||||
}
|
||||
|
||||
// Check for errors in result
|
||||
|
|
@ -72,7 +72,7 @@ func (p *MCPProvider) parseResult(result *gouMCPTypes.CallToolResponse) ([]strin
|
|||
|
||||
// Parse content - expect JSON data with "keywords" field
|
||||
if len(result.Content) == 0 {
|
||||
return []string{}, nil
|
||||
return []types.Keyword{}, nil
|
||||
}
|
||||
|
||||
// Try to extract keywords from content
|
||||
|
|
@ -88,36 +88,50 @@ func (p *MCPProvider) parseResult(result *gouMCPTypes.CallToolResponse) ([]strin
|
|||
}
|
||||
}
|
||||
|
||||
// Try to parse as direct array
|
||||
var keywords []string
|
||||
// Try to parse as direct array of keywords
|
||||
var keywords []types.Keyword
|
||||
if err := json.Unmarshal([]byte(content.Text), &keywords); err == nil {
|
||||
return keywords, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return []string{}, nil
|
||||
return []types.Keyword{}, nil
|
||||
}
|
||||
|
||||
// extractKeywordsFromValue extracts string array from various types
|
||||
func (p *MCPProvider) extractKeywordsFromValue(v interface{}) ([]string, error) {
|
||||
// extractKeywordsFromValue extracts Keyword array from various types
|
||||
func (p *MCPProvider) extractKeywordsFromValue(v interface{}) ([]types.Keyword, error) {
|
||||
switch kw := v.(type) {
|
||||
case []string:
|
||||
case []types.Keyword:
|
||||
return kw, nil
|
||||
case []interface{}:
|
||||
keywords := make([]string, 0, len(kw))
|
||||
keywords := make([]types.Keyword, 0, len(kw))
|
||||
for _, item := range kw {
|
||||
if s, ok := item.(string); ok {
|
||||
keywords = append(keywords, s)
|
||||
switch v := item.(type) {
|
||||
case map[string]interface{}:
|
||||
// Handle {k: "keyword", w: 0.9} format
|
||||
k, _ := v["k"].(string)
|
||||
w, _ := v["w"].(float64)
|
||||
if k != "" {
|
||||
if w == 0 {
|
||||
w = 0.5 // Default weight
|
||||
}
|
||||
keywords = append(keywords, types.Keyword{K: k, W: w})
|
||||
}
|
||||
case string:
|
||||
// Plain string, use default weight
|
||||
if v != "" {
|
||||
keywords = append(keywords, types.Keyword{K: v, W: 0.5})
|
||||
}
|
||||
}
|
||||
}
|
||||
return keywords, nil
|
||||
case string:
|
||||
var keywords []string
|
||||
var keywords []types.Keyword
|
||||
if err := json.Unmarshal([]byte(kw), &keywords); err == nil {
|
||||
return keywords, nil
|
||||
}
|
||||
return []string{kw}, nil
|
||||
return []types.Keyword{{K: kw, W: 0.5}}, nil
|
||||
}
|
||||
return []string{}, nil
|
||||
return []types.Keyword{}, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -120,6 +120,12 @@ type ProcessedQuery struct {
|
|||
DSL *gou.QueryDSL `json:"dsl,omitempty"` // For DB search, uses GOU QueryDSL
|
||||
}
|
||||
|
||||
// Keyword represents an extracted keyword with weight
|
||||
type Keyword struct {
|
||||
K string `json:"k"` // Keyword text
|
||||
W float64 `json:"w"` // Weight (0.1-1.0), higher means more relevant
|
||||
}
|
||||
|
||||
// Note: For QueryDSL and Model types, use GOU types directly:
|
||||
// - github.com/yaoapp/gou/query/gou.QueryDSL
|
||||
// - github.com/yaoapp/gou/model.Model
|
||||
|
|
|
|||
|
|
@ -254,18 +254,9 @@ func (a *Asserter) assertJSONPath(assertion *Assertion, output interface{}) *Ass
|
|||
actual := a.extractPath(jsonData, path)
|
||||
result.Actual = actual
|
||||
|
||||
// Support array of expected values (OR logic - any match passes)
|
||||
if expectedArr, ok := assertion.Value.([]interface{}); ok && len(expectedArr) > 0 {
|
||||
for _, expected := range expectedArr {
|
||||
if validateOutput(actual, expected) {
|
||||
result.Passed = true
|
||||
result.Message = fmt.Sprintf("path '%s' equals one of expected values", assertion.Path)
|
||||
return result
|
||||
}
|
||||
}
|
||||
result.Passed = false
|
||||
result.Message = fmt.Sprintf("path '%s': expected one of %v, got %v", assertion.Path, assertion.Value, actual)
|
||||
} else if validateOutput(actual, assertion.Value) {
|
||||
// Compare expected value with actual value
|
||||
// First, try direct comparison (handles both primitive values and arrays)
|
||||
if validateOutput(actual, assertion.Value) {
|
||||
result.Passed = true
|
||||
result.Message = fmt.Sprintf("path '%s' equals expected value", assertion.Path)
|
||||
} else {
|
||||
|
|
|
|||
330
data/bindata.go
330
data/bindata.go
File diff suppressed because one or more lines are too long
|
|
@ -1,21 +1,25 @@
|
|||
- role: system
|
||||
content: |
|
||||
Extract keywords from text content.
|
||||
You are a keyword extraction tool, NOT a chatbot. Do NOT answer questions or provide explanations.
|
||||
Your ONLY job: analyze text and output a JSON array of keywords with weights.
|
||||
|
||||
Task:
|
||||
1. Analyze input text
|
||||
2. Extract important keywords
|
||||
3. Return JSON format
|
||||
4. Match input language
|
||||
Output format: ["keyword:weight", ...]
|
||||
|
||||
Response Format (JSON only):
|
||||
```json
|
||||
{"keywords": ["keyword1", "keyword2", ...]}
|
||||
```
|
||||
Weight:
|
||||
- 1.0: Core topic
|
||||
- 0.8-0.9: Key concepts
|
||||
- 0.6-0.7: Supporting themes
|
||||
- 0.4-0.5: Peripheral concepts
|
||||
|
||||
Guidelines:
|
||||
- Extract 5-15 keywords based on content length
|
||||
- Prioritize nouns, proper nouns, key concepts
|
||||
- Include single words and short phrases
|
||||
- Exclude common stop words
|
||||
- Keywords MUST be in the same language as input
|
||||
Examples:
|
||||
- Input(EN): "Developers frustrated with callback hell. ES2017 async/await improved readability."
|
||||
- Output: ["async/await:1", "asynchronous programming:0.9", "ES2017:0.8", "code readability:0.7"]
|
||||
|
||||
- Input(中文): "用户反馈APP启动慢、页面卡顿。需要优化首屏加载和内存占用。"
|
||||
- Output: ["性能优化:1", "启动速度:0.9", "内存管理:0.8", "用户体验:0.7"]
|
||||
|
||||
Rules:
|
||||
- ONLY output JSON array, nothing else
|
||||
- Max 5 keywords, sorted by weight
|
||||
- Summarize related concepts
|
||||
- Match input language (EN→EN, 中文→中文)
|
||||
|
|
|
|||
|
|
@ -1,13 +1,20 @@
|
|||
/**
|
||||
* Keyword Extraction Agent - Next Hook
|
||||
* Parses LLM response and extracts keywords with error tolerance
|
||||
* Parses LLM response and extracts keywords with weight
|
||||
* Format: ["keyword:weight", ...] -> [{k, w}, ...]
|
||||
*/
|
||||
|
||||
// @ts-nocheck
|
||||
|
||||
/** Keyword with weight */
|
||||
interface Keyword {
|
||||
k: string; // keyword
|
||||
w: number; // weight (0.1-1.0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Next hook - processes keyword extraction response
|
||||
* Uses text.ExtractJSON for fault-tolerant JSON extraction from LLM output
|
||||
* Parses format: ["keyword1:0.9", "keyword2:0.8", ...]
|
||||
*/
|
||||
function Next(
|
||||
ctx: agent.Context,
|
||||
|
|
@ -21,22 +28,17 @@ function Next(
|
|||
}
|
||||
|
||||
const content = completion.content;
|
||||
let keywords: string[] = [];
|
||||
let keywords: Keyword[] = [];
|
||||
|
||||
try {
|
||||
// Use text.ExtractJSON for fault-tolerant extraction
|
||||
// Handles markdown code blocks, broken JSON, etc.
|
||||
const parsed = Process("text.ExtractJSON", content) as {
|
||||
keywords?: string[];
|
||||
} | null;
|
||||
// Extract JSON array from response
|
||||
const parsed = Process("text.ExtractJSON", content) as string[] | null;
|
||||
|
||||
if (parsed && Array.isArray(parsed.keywords)) {
|
||||
keywords = parsed.keywords.filter(
|
||||
(k) => typeof k === "string" && k.trim().length > 0
|
||||
);
|
||||
if (parsed && Array.isArray(parsed)) {
|
||||
keywords = parseKeywordArray(parsed);
|
||||
}
|
||||
} catch (e) {
|
||||
// If extraction fails, try to extract keywords from text
|
||||
// If extraction fails, try to extract from text
|
||||
keywords = extractKeywordsFromText(content);
|
||||
}
|
||||
|
||||
|
|
@ -45,6 +47,9 @@ function Next(
|
|||
keywords = extractKeywordsFromText(content);
|
||||
}
|
||||
|
||||
// Sort by weight descending and limit to 5
|
||||
keywords = keywords.sort((a, b) => b.w - a.w).slice(0, 5);
|
||||
|
||||
// Return parsed keywords
|
||||
return {
|
||||
data: {
|
||||
|
|
@ -54,49 +59,99 @@ function Next(
|
|||
}
|
||||
|
||||
/**
|
||||
* Extract keywords from plain text when JSON parsing fails
|
||||
* Handles formats like:
|
||||
* - Comma-separated: "keyword1, keyword2, keyword3"
|
||||
* - Line-separated: "keyword1\nkeyword2\nkeyword3"
|
||||
* - Bullet points: "- keyword1\n- keyword2"
|
||||
* - Numbered: "1. keyword1\n2. keyword2"
|
||||
* Parse keyword array format: ["keyword:weight", ...]
|
||||
* Examples: ["AI:0.9", "机器学习:0.8", "deep learning:0.7"]
|
||||
*/
|
||||
function extractKeywordsFromText(text: string): string[] {
|
||||
const keywords: string[] = [];
|
||||
function parseKeywordArray(items: (string | any)[]): Keyword[] {
|
||||
const keywords: Keyword[] = [];
|
||||
|
||||
// Remove common prefixes/suffixes
|
||||
let cleaned = text
|
||||
.replace(/^[\s\S]*?keywords?[\s:：]*\[?/i, "") // Remove "keywords:" prefix
|
||||
.replace(/\][\s\S]*$/, "") // Remove trailing ]
|
||||
.trim();
|
||||
|
||||
// Try line-by-line extraction
|
||||
const lines = cleaned.split(/[\n\r]+/);
|
||||
|
||||
for (const line of lines) {
|
||||
// Remove bullet points, numbers, quotes
|
||||
let keyword = line
|
||||
.replace(/^[\s\-\*\•\d\.]+/, "") // Remove bullets/numbers
|
||||
.replace(/^["'`]+|["'`]+$/g, "") // Remove quotes
|
||||
.replace(/,\s*$/, "") // Remove trailing comma
|
||||
.trim();
|
||||
|
||||
// Skip empty or too long
|
||||
if (keyword.length > 0 && keyword.length < 100) {
|
||||
// Split by comma if contains multiple
|
||||
if (keyword.includes(",")) {
|
||||
const parts = keyword.split(",").map((p) => p.trim());
|
||||
for (const part of parts) {
|
||||
if (part.length > 0 && part.length < 100) {
|
||||
keywords.push(part);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
keywords.push(keyword);
|
||||
for (const item of items) {
|
||||
if (typeof item === "string") {
|
||||
const parsed = parseKeywordString(item);
|
||||
if (parsed) {
|
||||
keywords.push(parsed);
|
||||
}
|
||||
} else if (item && typeof item === "object" && item.k) {
|
||||
// Fallback: handle {k, w} format
|
||||
const k = String(item.k).trim();
|
||||
const w =
|
||||
typeof item.w === "number" ? Math.min(1.0, Math.max(0.1, item.w)) : 0.5;
|
||||
if (k.length > 0) {
|
||||
keywords.push({ k, w });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplicate
|
||||
return [...new Set(keywords)];
|
||||
return keywords;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse single keyword string: "keyword:weight" or "keyword"
|
||||
*/
|
||||
function parseKeywordString(str: string): Keyword | null {
|
||||
const trimmed = str.trim().replace(/^["']+|["']+$/g, ""); // Remove quotes
|
||||
if (!trimmed) return null;
|
||||
|
||||
// Try to split by last colon (keyword may contain colons)
|
||||
const lastColonIdx = trimmed.lastIndexOf(":");
|
||||
if (lastColonIdx > 0) {
|
||||
const keyword = trimmed.substring(0, lastColonIdx).trim();
|
||||
const weightStr = trimmed.substring(lastColonIdx + 1).trim();
|
||||
const weight = parseFloat(weightStr);
|
||||
|
||||
if (keyword && !isNaN(weight)) {
|
||||
return {
|
||||
k: keyword,
|
||||
w: Math.min(1.0, Math.max(0.1, weight)),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// No weight found, return with default weight
|
||||
return { k: trimmed, w: 0.5 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract keywords from plain text when JSON parsing fails
|
||||
*/
|
||||
function extractKeywordsFromText(text: string): Keyword[] {
|
||||
const keywords: Keyword[] = [];
|
||||
|
||||
// Try to find array-like content
|
||||
const arrayMatch = text.match(/\[([^\]]+)\]/);
|
||||
if (arrayMatch) {
|
||||
const items = arrayMatch[1].split(",");
|
||||
for (const item of items) {
|
||||
const parsed = parseKeywordString(item);
|
||||
if (parsed) {
|
||||
keywords.push(parsed);
|
||||
}
|
||||
}
|
||||
if (keywords.length > 0) return keywords;
|
||||
}
|
||||
|
||||
// Fallback: line-by-line extraction
|
||||
const lines = text.split(/[\n\r,]+/);
|
||||
let defaultWeight = 1.0;
|
||||
|
||||
for (const line of lines) {
|
||||
let cleaned = line
|
||||
.replace(/^[\s\-\*\•\d\.\[\]"'`]+/, "") // Remove prefixes
|
||||
.replace(/[\]"'`]+$/, "") // Remove suffixes
|
||||
.trim();
|
||||
|
||||
if (cleaned.length > 0 && cleaned.length < 100) {
|
||||
const parsed = parseKeywordString(cleaned);
|
||||
if (parsed) {
|
||||
// Use parsed weight or assign decreasing default
|
||||
if (parsed.w === 0.5) {
|
||||
parsed.w = Math.max(0.1, defaultWeight);
|
||||
defaultWeight -= 0.1;
|
||||
}
|
||||
keywords.push(parsed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return keywords;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,10 +9,41 @@
|
|||
- DO NOT extract keywords, DO NOT answer the question, DO NOT add explanations
|
||||
|
||||
## Classification Rules
|
||||
need_search=false: greetings, chitchat, math, code requests, text processing, general knowledge, philosophy
|
||||
need_search=true with search_types=["web"]: weather, news, prices, exchange rates, live events, real-time info
|
||||
need_search=true with search_types=["kb"]: docs, how-to, config, FAQ, product info, policies
|
||||
need_search=true with search_types=["db"]: user data (my orders, my balance), account info, business records
|
||||
|
||||
### need_search=false (No search needed)
|
||||
Use when the question can be answered from LLM's internal knowledge:
|
||||
- Greetings & chitchat: "hello", "how are you", casual conversation
|
||||
- Math & calculations: arithmetic, equations, formulas
|
||||
- Code generation: write code, debug, explain code, algorithms
|
||||
- Text processing: translate, summarize, rewrite, format
|
||||
- General knowledge: history, science, concepts (not time-sensitive)
|
||||
- Creative tasks: write stories, poems, brainstorm ideas
|
||||
- Reasoning & logic: philosophy, opinions, hypothetical questions
|
||||
|
||||
### need_search=true with search_types=["web"] (Web search)
|
||||
Use when real-time or frequently changing information is needed:
|
||||
- Current events: news, breaking stories, recent happenings
|
||||
- Time-sensitive data: weather, stock prices, exchange rates, sports scores
|
||||
- Live information: event schedules, store hours, availability
|
||||
- Recent updates: latest versions, new releases, current status
|
||||
- Location-based: nearby places, local info, addresses
|
||||
|
||||
### need_search=true with search_types=["kb"] (Knowledge base)
|
||||
Use when querying internal documentation or product knowledge:
|
||||
- Documentation: how-to guides, tutorials, setup instructions
|
||||
- Configuration: settings, parameters, options explained
|
||||
- Product info: features, specifications, capabilities
|
||||
- Policies: terms, rules, guidelines, compliance
|
||||
- FAQ: common questions about the system/product
|
||||
- Troubleshooting: error messages, known issues, solutions
|
||||
|
||||
### need_search=true with search_types=["db"] (Database)
|
||||
Use when querying user-specific or transactional data:
|
||||
- Personal data: "my orders", "my profile", "my history"
|
||||
- Account info: balance, subscription, membership status
|
||||
- Business records: invoices, transactions, payments
|
||||
- User preferences: settings, saved items, favorites
|
||||
- Keywords: "my", "mine", specific order/ID numbers
|
||||
|
||||
## Required Output Format (JSON only, no markdown)
|
||||
{"need_search": true/false, "search_types": [], "confidence": 0.0-1.0}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue