8dcb78fc37
- Modified the 'Type' field in the serpAPIKnowledge struct to use an interface{}, enabling it to accept either a string or an object based on the query context.
- This change enhances the flexibility of the knowledge graph data representation in the search handler.
302 lines
8 KiB
Go
302 lines
8 KiB
Go
package web
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"time"
|
|
|
|
"github.com/yaoapp/yao/agent/search/types"
|
|
)
|
|
|
|
const (
|
|
serpAPIURL = "https://serpapi.com/search.json"
|
|
serpAPITimeout = 30 * time.Second
|
|
)
|
|
|
|
// SerpAPIProvider implements web search using SerpAPI (supports multiple search engines)
|
|
type SerpAPIProvider struct {
|
|
apiKey string
|
|
maxResults int
|
|
engine string // Search engine: "google", "bing", "baidu", "yandex", "duckduckgo", etc.
|
|
}
|
|
|
|
// NewSerpAPIProvider creates a new SerpAPI provider
|
|
func NewSerpAPIProvider(cfg *types.WebConfig) *SerpAPIProvider {
|
|
apiKey := ""
|
|
if cfg != nil && cfg.APIKeyEnv != "" {
|
|
// Support both "$ENV.VAR_NAME" and "VAR_NAME" formats
|
|
envName := cfg.APIKeyEnv
|
|
if len(envName) > 5 && envName[:5] == "$ENV." {
|
|
envName = envName[5:]
|
|
}
|
|
apiKey = os.Getenv(envName)
|
|
}
|
|
|
|
maxResults := 10
|
|
if cfg != nil && cfg.MaxResults > 0 {
|
|
maxResults = cfg.MaxResults
|
|
}
|
|
|
|
engine := "google" // Default to Google
|
|
if cfg != nil && cfg.Engine != "" {
|
|
engine = cfg.Engine
|
|
}
|
|
|
|
return &SerpAPIProvider{
|
|
apiKey: apiKey,
|
|
maxResults: maxResults,
|
|
engine: engine,
|
|
}
|
|
}
|
|
|
|
// serpAPIResponse represents the response from SerpAPI
|
|
type serpAPIResponse struct {
|
|
SearchMetadata serpAPIMetadata `json:"search_metadata"`
|
|
SearchParameters serpAPIParams `json:"search_parameters"`
|
|
SearchInformation serpAPIInfo `json:"search_information"`
|
|
OrganicResults []serpAPIResult `json:"organic_results"`
|
|
AnswerBox *serpAPIAnswerBox `json:"answer_box,omitempty"`
|
|
KnowledgeGraph *serpAPIKnowledge `json:"knowledge_graph,omitempty"`
|
|
RelatedSearches []serpAPIRelated `json:"related_searches,omitempty"`
|
|
RelatedQuestions []serpAPIQuestion `json:"related_questions,omitempty"`
|
|
}
|
|
|
|
// serpAPIMetadata contains metadata from response
|
|
type serpAPIMetadata struct {
|
|
ID string `json:"id"`
|
|
Status string `json:"status"`
|
|
CreatedAt string `json:"created_at"`
|
|
ProcessedAt string `json:"processed_at"`
|
|
TotalTimeTaken float64 `json:"total_time_taken"`
|
|
}
|
|
|
|
// serpAPIParams contains search parameters from response
|
|
type serpAPIParams struct {
|
|
Engine string `json:"engine"`
|
|
Q string `json:"q"`
|
|
Location string `json:"location_used"`
|
|
GoogleDomain string `json:"google_domain"`
|
|
HL string `json:"hl"`
|
|
GL string `json:"gl"`
|
|
Device string `json:"device"`
|
|
}
|
|
|
|
// serpAPIInfo contains search information
|
|
type serpAPIInfo struct {
|
|
QueryDisplayed string `json:"query_displayed"`
|
|
TotalResults int64 `json:"total_results"`
|
|
TimeTakenDisplayed float64 `json:"time_taken_displayed"`
|
|
OrganicResultsState string `json:"organic_results_state"`
|
|
}
|
|
|
|
// serpAPIResult represents a single organic search result
|
|
type serpAPIResult struct {
|
|
Position int `json:"position"`
|
|
Title string `json:"title"`
|
|
Link string `json:"link"`
|
|
RedirectLink string `json:"redirect_link,omitempty"`
|
|
DisplayedLink string `json:"displayed_link"`
|
|
Snippet string `json:"snippet"`
|
|
Date string `json:"date,omitempty"`
|
|
CachedPageLink string `json:"cached_page_link,omitempty"`
|
|
}
|
|
|
|
// serpAPIAnswerBox represents the answer box (featured snippet)
|
|
type serpAPIAnswerBox struct {
|
|
Type string `json:"type,omitempty"`
|
|
Title string `json:"title,omitempty"`
|
|
Snippet string `json:"snippet,omitempty"`
|
|
Link string `json:"link,omitempty"`
|
|
}
|
|
|
|
// serpAPIKnowledge represents knowledge graph data
|
|
type serpAPIKnowledge struct {
|
|
Title string `json:"title,omitempty"`
|
|
Type interface{} `json:"type,omitempty"` // Can be string or object depending on query
|
|
Description string `json:"description,omitempty"`
|
|
}
|
|
|
|
// serpAPIRelated represents related searches
|
|
type serpAPIRelated struct {
|
|
Query string `json:"query"`
|
|
Link string `json:"link"`
|
|
}
|
|
|
|
// serpAPIQuestion represents related questions (People Also Ask)
|
|
type serpAPIQuestion struct {
|
|
Question string `json:"question"`
|
|
Snippet string `json:"snippet,omitempty"`
|
|
Title string `json:"title,omitempty"`
|
|
Link string `json:"link,omitempty"`
|
|
}
|
|
|
|
// Search executes a web search using SerpAPI
|
|
func (p *SerpAPIProvider) Search(req *types.Request) (*types.Result, error) {
|
|
startTime := time.Now()
|
|
|
|
// Validate API key
|
|
if p.apiKey == "" {
|
|
return &types.Result{
|
|
Type: types.SearchTypeWeb,
|
|
Query: req.Query,
|
|
Source: req.Source,
|
|
Items: []*types.ResultItem{},
|
|
Total: 0,
|
|
Error: "SerpAPI API key not configured",
|
|
}, nil
|
|
}
|
|
|
|
// Determine max results
|
|
maxResults := p.maxResults
|
|
if req.Limit > 0 {
|
|
maxResults = req.Limit
|
|
}
|
|
|
|
// Build query parameters
|
|
params := url.Values{}
|
|
params.Set("engine", p.engine)
|
|
params.Set("api_key", p.apiKey)
|
|
params.Set("num", fmt.Sprintf("%d", maxResults))
|
|
|
|
// Build search query with site restrictions if specified
|
|
query := req.Query
|
|
if len(req.Sites) > 0 {
|
|
siteQuery := ""
|
|
for i, site := range req.Sites {
|
|
if i > 0 {
|
|
siteQuery += " OR "
|
|
}
|
|
siteQuery += "site:" + site
|
|
}
|
|
query = "(" + siteQuery + ") " + req.Query
|
|
}
|
|
params.Set("q", query)
|
|
|
|
// Add time range if specified (tbs parameter)
|
|
if req.TimeRange != "" {
|
|
tbs := convertSerpAPITimeRange(req.TimeRange)
|
|
if tbs != "" {
|
|
params.Set("tbs", tbs)
|
|
}
|
|
}
|
|
|
|
// Execute API call
|
|
serpResp, err := p.callAPI(params)
|
|
if err != nil {
|
|
return &types.Result{
|
|
Type: types.SearchTypeWeb,
|
|
Query: req.Query,
|
|
Source: req.Source,
|
|
Items: []*types.ResultItem{},
|
|
Total: 0,
|
|
Duration: time.Since(startTime).Milliseconds(),
|
|
Error: fmt.Sprintf("SerpAPI error: %v", err),
|
|
}, nil
|
|
}
|
|
|
|
// Convert results
|
|
items := make([]*types.ResultItem, 0, len(serpResp.OrganicResults))
|
|
|
|
// Add answer box as first result if available
|
|
if serpResp.AnswerBox != nil && serpResp.AnswerBox.Snippet != "" {
|
|
items = append(items, &types.ResultItem{
|
|
Type: types.SearchTypeWeb,
|
|
Title: serpResp.AnswerBox.Title,
|
|
Content: serpResp.AnswerBox.Snippet,
|
|
URL: serpResp.AnswerBox.Link,
|
|
Score: 1.0, // Featured snippet gets highest score
|
|
Source: req.Source,
|
|
Metadata: map[string]interface{}{
|
|
"type": "answer_box",
|
|
},
|
|
})
|
|
}
|
|
|
|
// Add organic results
|
|
for _, r := range serpResp.OrganicResults {
|
|
// Calculate score based on position (1st = 0.95, 2nd = 0.90, etc.)
|
|
score := 1.0 - float64(r.Position)*0.05
|
|
if score < 0.1 {
|
|
score = 0.1
|
|
}
|
|
|
|
items = append(items, &types.ResultItem{
|
|
Type: types.SearchTypeWeb,
|
|
Title: r.Title,
|
|
Content: r.Snippet,
|
|
URL: r.Link,
|
|
Score: score,
|
|
Source: req.Source,
|
|
})
|
|
}
|
|
|
|
return &types.Result{
|
|
Type: types.SearchTypeWeb,
|
|
Query: req.Query,
|
|
Source: req.Source,
|
|
Items: items,
|
|
Total: len(items),
|
|
Duration: time.Since(startTime).Milliseconds(),
|
|
}, nil
|
|
}
|
|
|
|
// callAPI makes the HTTP GET request to SerpAPI
|
|
func (p *SerpAPIProvider) callAPI(params url.Values) (*serpAPIResponse, error) {
|
|
// Build URL with query parameters
|
|
reqURL := serpAPIURL + "?" + params.Encode()
|
|
|
|
// Create HTTP request
|
|
httpReq, err := http.NewRequest(http.MethodGet, reqURL, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create request: %w", err)
|
|
}
|
|
|
|
// Execute request
|
|
client := &http.Client{Timeout: serpAPITimeout}
|
|
resp, err := client.Do(httpReq)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("request failed: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
// Read response body
|
|
respBody, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read response: %w", err)
|
|
}
|
|
|
|
// Check status code
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(respBody))
|
|
}
|
|
|
|
// Parse response
|
|
var serpResp serpAPIResponse
|
|
if err := json.Unmarshal(respBody, &serpResp); err != nil {
|
|
return nil, fmt.Errorf("failed to parse response: %w", err)
|
|
}
|
|
|
|
return &serpResp, nil
|
|
}
|
|
|
|
// convertSerpAPITimeRange converts time range to SerpAPI tbs format
|
|
func convertSerpAPITimeRange(timeRange string) string {
|
|
switch timeRange {
|
|
case "hour":
|
|
return "qdr:h"
|
|
case "day":
|
|
return "qdr:d"
|
|
case "week":
|
|
return "qdr:w"
|
|
case "month":
|
|
return "qdr:m"
|
|
case "year":
|
|
return "qdr:y"
|
|
default:
|
|
return ""
|
|
}
|
|
}
|