feat: add Mistral AI provider, Firecrawl and SerpAPI tools
Added new LLM provider: - Mistral AI (https://api.mistral.ai/v1) Supports models: mistral-tiny, mistral-small, mistral-medium, etc. Added new tools: - firecrawl: Advanced web scraping with markdown extraction, screenshots, structured data extraction using LLM. Supports Firecrawl API. - serp_api: Google search via SerpAPI with organic results, knowledge graph, related questions, and more. Changes: - pkg/config/config.go: Added Mistral provider config and tool configs - pkg/providers/http_provider.go: Added Mistral routing and auto-detection - pkg/agent/loop.go: Registered Firecrawl and SerpAPI tools - pkg/tools/firecrawl.go: New tool implementation with tests - pkg/tools/serpapi.go: New tool implementation with tests - config/config.example.json: Added configuration examples - pkg/providers/mistral_test.go: Integration test for Mistral provider
This commit is contained in:
parent
9a3f3611c3
commit
f29d0615a5
9 changed files with 661 additions and 3 deletions
|
|
@ -107,14 +107,33 @@
|
||||||
"moonshot": {
|
"moonshot": {
|
||||||
"api_key": "sk-xxx",
|
"api_key": "sk-xxx",
|
||||||
"api_base": ""
|
"api_base": ""
|
||||||
|
},
|
||||||
|
"mistral": {
|
||||||
|
"api_key": "YOUR_MISTRAL_API_KEY",
|
||||||
|
"api_base": "https://api.mistral.ai/v1"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"tools": {
|
"tools": {
|
||||||
"web": {
|
"web": {
|
||||||
"search": {
|
"brave": {
|
||||||
|
"enabled": false,
|
||||||
"api_key": "YOUR_BRAVE_API_KEY",
|
"api_key": "YOUR_BRAVE_API_KEY",
|
||||||
"max_results": 5
|
"max_results": 5
|
||||||
|
},
|
||||||
|
"duckduckgo": {
|
||||||
|
"enabled": true,
|
||||||
|
"max_results": 5
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"firecrawl": {
|
||||||
|
"enabled": false,
|
||||||
|
"api_key": "YOUR_FIRECRAWL_API_KEY",
|
||||||
|
"api_base": "https://api.firecrawl.dev/v1"
|
||||||
|
},
|
||||||
|
"serpapi": {
|
||||||
|
"enabled": false,
|
||||||
|
"api_key": "YOUR_SERPAPI_KEY",
|
||||||
|
"max_results": 10
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"heartbeat": {
|
"heartbeat": {
|
||||||
|
|
|
||||||
|
|
@ -82,6 +82,16 @@ func createToolRegistry(workspace string, restrict bool, cfg *config.Config, msg
|
||||||
}
|
}
|
||||||
registry.Register(tools.NewWebFetchTool(50000))
|
registry.Register(tools.NewWebFetchTool(50000))
|
||||||
|
|
||||||
|
// Firecrawl tool for advanced web scraping
|
||||||
|
if cfg.Tools.Firecrawl.Enabled && cfg.Tools.Firecrawl.APIKey != "" {
|
||||||
|
registry.Register(tools.NewFirecrawlTool(cfg.Tools.Firecrawl.APIKey, cfg.Tools.Firecrawl.APIBase))
|
||||||
|
}
|
||||||
|
|
||||||
|
// SerpAPI tool for Google search results
|
||||||
|
if cfg.Tools.SerpAPI.Enabled && cfg.Tools.SerpAPI.APIKey != "" {
|
||||||
|
registry.Register(tools.NewSerpAPITool(cfg.Tools.SerpAPI.APIKey, cfg.Tools.SerpAPI.MaxResults))
|
||||||
|
}
|
||||||
|
|
||||||
// Hardware tools (I2C, SPI) - Linux only, returns error on other platforms
|
// Hardware tools (I2C, SPI) - Linux only, returns error on other platforms
|
||||||
registry.Register(tools.NewI2CTool())
|
registry.Register(tools.NewI2CTool())
|
||||||
registry.Register(tools.NewSPITool())
|
registry.Register(tools.NewSPITool())
|
||||||
|
|
|
||||||
|
|
@ -179,6 +179,7 @@ type ProvidersConfig struct {
|
||||||
ShengSuanYun ProviderConfig `json:"shengsuanyun"`
|
ShengSuanYun ProviderConfig `json:"shengsuanyun"`
|
||||||
DeepSeek ProviderConfig `json:"deepseek"`
|
DeepSeek ProviderConfig `json:"deepseek"`
|
||||||
GitHubCopilot ProviderConfig `json:"github_copilot"`
|
GitHubCopilot ProviderConfig `json:"github_copilot"`
|
||||||
|
Mistral ProviderConfig `json:"mistral"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ProviderConfig struct {
|
type ProviderConfig struct {
|
||||||
|
|
@ -210,8 +211,22 @@ type WebToolsConfig struct {
|
||||||
DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"`
|
DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type FirecrawlConfig struct {
|
||||||
|
Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_FIRECRAWL_ENABLED"`
|
||||||
|
APIKey string `json:"api_key" env:"PICOCLAW_TOOLS_FIRECRAWL_API_KEY"`
|
||||||
|
APIBase string `json:"api_base" env:"PICOCLAW_TOOLS_FIRECRAWL_API_BASE"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type SerpAPIConfig struct {
|
||||||
|
Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_SERPAPI_ENABLED"`
|
||||||
|
APIKey string `json:"api_key" env:"PICOCLAW_TOOLS_SERPAPI_API_KEY"`
|
||||||
|
MaxResults int `json:"max_results" env:"PICOCLAW_TOOLS_SERPAPI_MAX_RESULTS"`
|
||||||
|
}
|
||||||
|
|
||||||
type ToolsConfig struct {
|
type ToolsConfig struct {
|
||||||
Web WebToolsConfig `json:"web"`
|
Web WebToolsConfig `json:"web"`
|
||||||
|
Firecrawl FirecrawlConfig `json:"firecrawl"`
|
||||||
|
SerpAPI SerpAPIConfig `json:"serpapi"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func DefaultConfig() *Config {
|
func DefaultConfig() *Config {
|
||||||
|
|
@ -304,6 +319,7 @@ func DefaultConfig() *Config {
|
||||||
Nvidia: ProviderConfig{},
|
Nvidia: ProviderConfig{},
|
||||||
Moonshot: ProviderConfig{},
|
Moonshot: ProviderConfig{},
|
||||||
ShengSuanYun: ProviderConfig{},
|
ShengSuanYun: ProviderConfig{},
|
||||||
|
Mistral: ProviderConfig{},
|
||||||
},
|
},
|
||||||
Gateway: GatewayConfig{
|
Gateway: GatewayConfig{
|
||||||
Host: "0.0.0.0",
|
Host: "0.0.0.0",
|
||||||
|
|
@ -321,6 +337,16 @@ func DefaultConfig() *Config {
|
||||||
MaxResults: 5,
|
MaxResults: 5,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
Firecrawl: FirecrawlConfig{
|
||||||
|
Enabled: false,
|
||||||
|
APIKey: "",
|
||||||
|
APIBase: "https://api.firecrawl.dev/v1",
|
||||||
|
},
|
||||||
|
SerpAPI: SerpAPIConfig{
|
||||||
|
Enabled: false,
|
||||||
|
APIKey: "",
|
||||||
|
MaxResults: 10,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
Heartbeat: HeartbeatConfig{
|
Heartbeat: HeartbeatConfig{
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
|
|
|
||||||
|
|
@ -56,7 +56,7 @@ func (p *HTTPProvider) Chat(ctx context.Context, messages []Message, tools []Too
|
||||||
// Strip provider prefix from model name (e.g., moonshot/kimi-k2.5 -> kimi-k2.5)
|
// Strip provider prefix from model name (e.g., moonshot/kimi-k2.5 -> kimi-k2.5)
|
||||||
if idx := strings.Index(model, "/"); idx != -1 {
|
if idx := strings.Index(model, "/"); idx != -1 {
|
||||||
prefix := model[:idx]
|
prefix := model[:idx]
|
||||||
if prefix == "moonshot" || prefix == "nvidia" {
|
if prefix == "moonshot" || prefix == "nvidia" || prefix == "mistral" {
|
||||||
model = model[idx+1:]
|
model = model[idx+1:]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -322,6 +322,15 @@ func CreateProvider(cfg *config.Config) (LLMProvider, error) {
|
||||||
apiBase = "localhost:4321"
|
apiBase = "localhost:4321"
|
||||||
}
|
}
|
||||||
return NewGitHubCopilotProvider(apiBase, cfg.Providers.GitHubCopilot.ConnectMode, model)
|
return NewGitHubCopilotProvider(apiBase, cfg.Providers.GitHubCopilot.ConnectMode, model)
|
||||||
|
case "mistral":
|
||||||
|
if cfg.Providers.Mistral.APIKey != "" {
|
||||||
|
apiKey = cfg.Providers.Mistral.APIKey
|
||||||
|
apiBase = cfg.Providers.Mistral.APIBase
|
||||||
|
proxy = cfg.Providers.Mistral.Proxy
|
||||||
|
if apiBase == "" {
|
||||||
|
apiBase = "https://api.mistral.ai/v1"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -406,6 +415,14 @@ func CreateProvider(cfg *config.Config) (LLMProvider, error) {
|
||||||
apiBase = cfg.Providers.VLLM.APIBase
|
apiBase = cfg.Providers.VLLM.APIBase
|
||||||
proxy = cfg.Providers.VLLM.Proxy
|
proxy = cfg.Providers.VLLM.Proxy
|
||||||
|
|
||||||
|
case (strings.Contains(lowerModel, "mistral") || strings.HasPrefix(model, "mistral/")) && cfg.Providers.Mistral.APIKey != "":
|
||||||
|
apiKey = cfg.Providers.Mistral.APIKey
|
||||||
|
apiBase = cfg.Providers.Mistral.APIBase
|
||||||
|
proxy = cfg.Providers.Mistral.Proxy
|
||||||
|
if apiBase == "" {
|
||||||
|
apiBase = "https://api.mistral.ai/v1"
|
||||||
|
}
|
||||||
|
|
||||||
default:
|
default:
|
||||||
if cfg.Providers.OpenRouter.APIKey != "" {
|
if cfg.Providers.OpenRouter.APIKey != "" {
|
||||||
apiKey = cfg.Providers.OpenRouter.APIKey
|
apiKey = cfg.Providers.OpenRouter.APIKey
|
||||||
|
|
|
||||||
34
pkg/providers/mistral_test.go
Normal file
34
pkg/providers/mistral_test.go
Normal file
|
|
@ -0,0 +1,34 @@
|
||||||
|
package providers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMistralProvider_Integration(t *testing.T) {
|
||||||
|
apiKey := os.Getenv("MISTRAL_API_KEY")
|
||||||
|
if apiKey == "" {
|
||||||
|
t.Skip("MISTRAL_API_KEY not set")
|
||||||
|
}
|
||||||
|
|
||||||
|
provider := NewHTTPProvider(apiKey, "https://api.mistral.ai/v1", "")
|
||||||
|
|
||||||
|
messages := []Message{
|
||||||
|
{Role: "user", Content: "Say 'Hello from Mistral' in exactly 3 words"},
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := provider.Chat(context.Background(), messages, nil, "mistral-tiny", map[string]interface{}{
|
||||||
|
"max_tokens": 50,
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Mistral chat failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp.Content == "" {
|
||||||
|
t.Errorf("Expected non-empty response")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("Mistral response: %s", resp.Content)
|
||||||
|
}
|
||||||
201
pkg/tools/firecrawl.go
Normal file
201
pkg/tools/firecrawl.go
Normal file
|
|
@ -0,0 +1,201 @@
|
||||||
|
package tools
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type FirecrawlTool struct {
|
||||||
|
apiKey string
|
||||||
|
apiBase string
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewFirecrawlTool(apiKey, apiBase string) *FirecrawlTool {
|
||||||
|
if apiBase == "" {
|
||||||
|
apiBase = "https://api.firecrawl.dev/v1"
|
||||||
|
}
|
||||||
|
return &FirecrawlTool{
|
||||||
|
apiKey: apiKey,
|
||||||
|
apiBase: apiBase,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *FirecrawlTool) Name() string {
|
||||||
|
return "firecrawl"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *FirecrawlTool) Description() string {
|
||||||
|
return "Scrape web pages and extract structured data using Firecrawl. Supports markdown extraction, screenshot capture, and structured data extraction with LLM."
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *FirecrawlTool) Parameters() map[string]interface{} {
|
||||||
|
return map[string]interface{}{
|
||||||
|
"type": "object",
|
||||||
|
"properties": map[string]interface{}{
|
||||||
|
"url": map[string]interface{}{
|
||||||
|
"type": "string",
|
||||||
|
"description": "URL to scrape",
|
||||||
|
},
|
||||||
|
"formats": map[string]interface{}{
|
||||||
|
"type": "array",
|
||||||
|
"description": "Output formats (markdown, html, screenshot, links, extract)",
|
||||||
|
"items": map[string]interface{}{
|
||||||
|
"type": "string",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"only_main_content": map[string]interface{}{
|
||||||
|
"type": "boolean",
|
||||||
|
"description": "Extract only main content, excluding navigation, headers, footers",
|
||||||
|
},
|
||||||
|
"extract": map[string]interface{}{
|
||||||
|
"type": "object",
|
||||||
|
"description": "Schema for structured data extraction using LLM",
|
||||||
|
"properties": map[string]interface{}{
|
||||||
|
"prompt": map[string]interface{}{
|
||||||
|
"type": "string",
|
||||||
|
"description": "Prompt describing what to extract",
|
||||||
|
},
|
||||||
|
"schema": map[string]interface{}{
|
||||||
|
"type": "object",
|
||||||
|
"description": "JSON schema for structured output",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"required": []string{"url"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *FirecrawlTool) Execute(ctx context.Context, args map[string]interface{}) *ToolResult {
|
||||||
|
if t.apiKey == "" {
|
||||||
|
return ErrorResult("Firecrawl API key not configured")
|
||||||
|
}
|
||||||
|
|
||||||
|
url, ok := args["url"].(string)
|
||||||
|
if !ok || url == "" {
|
||||||
|
return ErrorResult("url is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build request body
|
||||||
|
requestBody := map[string]interface{}{
|
||||||
|
"url": url,
|
||||||
|
}
|
||||||
|
|
||||||
|
if formats, ok := args["formats"].([]interface{}); ok && len(formats) > 0 {
|
||||||
|
requestBody["formats"] = formats
|
||||||
|
} else {
|
||||||
|
// Default to markdown
|
||||||
|
requestBody["formats"] = []string{"markdown"}
|
||||||
|
}
|
||||||
|
|
||||||
|
if onlyMain, ok := args["only_main_content"].(bool); ok {
|
||||||
|
requestBody["onlyMainContent"] = onlyMain
|
||||||
|
}
|
||||||
|
|
||||||
|
if extract, ok := args["extract"].(map[string]interface{}); ok {
|
||||||
|
requestBody["extract"] = extract
|
||||||
|
}
|
||||||
|
|
||||||
|
jsonData, err := json.Marshal(requestBody)
|
||||||
|
if err != nil {
|
||||||
|
return ErrorResult(fmt.Sprintf("failed to marshal request: %v", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, "POST", t.apiBase+"/scrape", bytes.NewReader(jsonData))
|
||||||
|
if err != nil {
|
||||||
|
return ErrorResult(fmt.Sprintf("failed to create request: %v", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("Authorization", "Bearer "+t.apiKey)
|
||||||
|
|
||||||
|
client := &http.Client{Timeout: 60 * time.Second}
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return ErrorResult(fmt.Sprintf("request failed: %v", err))
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return ErrorResult(fmt.Sprintf("failed to read response: %v", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return ErrorResult(fmt.Sprintf("API error (status %d): %s", resp.StatusCode, string(body)))
|
||||||
|
}
|
||||||
|
|
||||||
|
var result struct {
|
||||||
|
Success bool `json:"success"`
|
||||||
|
Data struct {
|
||||||
|
Markdown string `json:"markdown"`
|
||||||
|
HTML string `json:"html"`
|
||||||
|
Metadata map[string]interface{} `json:"metadata"`
|
||||||
|
Extract map[string]interface{} `json:"extract"`
|
||||||
|
Screenshot string `json:"screenshot"`
|
||||||
|
Links []string `json:"links"`
|
||||||
|
} `json:"data"`
|
||||||
|
Error string `json:"error"`
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := json.Unmarshal(body, &result); err != nil {
|
||||||
|
return ErrorResult(fmt.Sprintf("failed to parse response: %v", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
if !result.Success {
|
||||||
|
return ErrorResult(fmt.Sprintf("Firecrawl error: %s", result.Error))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build response
|
||||||
|
var output strings.Builder
|
||||||
|
output.WriteString(fmt.Sprintf("# Scraped: %s\n\n", url))
|
||||||
|
|
||||||
|
if result.Data.Metadata != nil {
|
||||||
|
if title, ok := result.Data.Metadata["title"].(string); ok && title != "" {
|
||||||
|
output.WriteString(fmt.Sprintf("**Title:** %s\n\n", title))
|
||||||
|
}
|
||||||
|
if description, ok := result.Data.Metadata["description"].(string); ok && description != "" {
|
||||||
|
output.WriteString(fmt.Sprintf("**Description:** %s\n\n", description))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.Data.Markdown != "" {
|
||||||
|
output.WriteString("## Content (Markdown)\n\n")
|
||||||
|
output.WriteString(result.Data.Markdown)
|
||||||
|
output.WriteString("\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.Data.Extract != nil && len(result.Data.Extract) > 0 {
|
||||||
|
output.WriteString("## Extracted Data\n\n")
|
||||||
|
extractJSON, _ := json.MarshalIndent(result.Data.Extract, "", " ")
|
||||||
|
output.WriteString("```json\n")
|
||||||
|
output.WriteString(string(extractJSON))
|
||||||
|
output.WriteString("\n```\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(result.Data.Links) > 0 {
|
||||||
|
output.WriteString(fmt.Sprintf("## Links Found: %d\n\n", len(result.Data.Links)))
|
||||||
|
for i, link := range result.Data.Links {
|
||||||
|
if i >= 20 { // Limit to 20 links
|
||||||
|
output.WriteString(fmt.Sprintf("\n... and %d more links\n", len(result.Data.Links)-20))
|
||||||
|
break
|
||||||
|
}
|
||||||
|
output.WriteString(fmt.Sprintf("- %s\n", link))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.Data.Screenshot != "" {
|
||||||
|
output.WriteString("\n**Screenshot:** [Base64 encoded image available]\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
return &ToolResult{
|
||||||
|
ForLLM: output.String(),
|
||||||
|
ForUser: output.String(),
|
||||||
|
}
|
||||||
|
}
|
||||||
27
pkg/tools/firecrawl_test.go
Normal file
27
pkg/tools/firecrawl_test.go
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
package tools
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFirecrawlTool_Integration(t *testing.T) {
|
||||||
|
apiKey := os.Getenv("FIRECRAWL_API_KEY")
|
||||||
|
if apiKey == "" {
|
||||||
|
t.Skip("FIRECRAWL_API_KEY not set")
|
||||||
|
}
|
||||||
|
|
||||||
|
tool := NewFirecrawlTool(apiKey, "")
|
||||||
|
|
||||||
|
result := tool.Execute(context.Background(), map[string]interface{}{
|
||||||
|
"url": "https://example.com",
|
||||||
|
"formats": []string{"markdown"},
|
||||||
|
})
|
||||||
|
|
||||||
|
if result.IsError {
|
||||||
|
t.Errorf("Firecrawl execution failed: %s", result.ForLLM)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("Firecrawl result:\n%s", result.ForUser)
|
||||||
|
}
|
||||||
296
pkg/tools/serpapi.go
Normal file
296
pkg/tools/serpapi.go
Normal file
|
|
@ -0,0 +1,296 @@
|
||||||
|
package tools
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type SerpAPITool struct {
|
||||||
|
apiKey string
|
||||||
|
maxResults int
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewSerpAPITool(apiKey string, maxResults int) *SerpAPITool {
|
||||||
|
if maxResults <= 0 {
|
||||||
|
maxResults = 10
|
||||||
|
}
|
||||||
|
return &SerpAPITool{
|
||||||
|
apiKey: apiKey,
|
||||||
|
maxResults: maxResults,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *SerpAPITool) Name() string {
|
||||||
|
return "serp_api"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *SerpAPITool) Description() string {
|
||||||
|
return "Search Google and other search engines using SerpAPI. Returns organic results, knowledge graph, related questions, and more."
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *SerpAPITool) Parameters() map[string]interface{} {
|
||||||
|
return map[string]interface{}{
|
||||||
|
"type": "object",
|
||||||
|
"properties": map[string]interface{}{
|
||||||
|
"query": map[string]interface{}{
|
||||||
|
"type": "string",
|
||||||
|
"description": "Search query",
|
||||||
|
},
|
||||||
|
"engine": map[string]interface{}{
|
||||||
|
"type": "string",
|
||||||
|
"description": "Search engine (google, bing, yahoo, duckduckgo, yandex)",
|
||||||
|
"enum": []string{"google", "bing", "yahoo", "duckduckgo", "yandex"},
|
||||||
|
},
|
||||||
|
"location": map[string]interface{}{
|
||||||
|
"type": "string",
|
||||||
|
"description": "Location for localized results (e.g., 'Austin, Texas, United States')",
|
||||||
|
},
|
||||||
|
"hl": map[string]interface{}{
|
||||||
|
"type": "string",
|
||||||
|
"description": "Language code (e.g., 'en', 'es', 'fr')",
|
||||||
|
},
|
||||||
|
"gl": map[string]interface{}{
|
||||||
|
"type": "string",
|
||||||
|
"description": "Country code (e.g., 'us', 'uk', 'ca')",
|
||||||
|
},
|
||||||
|
"num": map[string]interface{}{
|
||||||
|
"type": "integer",
|
||||||
|
"description": "Number of results (1-100)",
|
||||||
|
"minimum": 1,
|
||||||
|
"maximum": 100,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"required": []string{"query"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *SerpAPITool) Execute(ctx context.Context, args map[string]interface{}) *ToolResult {
|
||||||
|
if t.apiKey == "" {
|
||||||
|
return ErrorResult("SerpAPI key not configured. Get one at https://serpapi.com/")
|
||||||
|
}
|
||||||
|
|
||||||
|
query, ok := args["query"].(string)
|
||||||
|
if !ok || query == "" {
|
||||||
|
return ErrorResult("query is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build query parameters
|
||||||
|
params := url.Values{}
|
||||||
|
params.Set("q", query)
|
||||||
|
params.Set("api_key", t.apiKey)
|
||||||
|
params.Set("output", "json")
|
||||||
|
|
||||||
|
// Set engine (default to google)
|
||||||
|
engine := "google"
|
||||||
|
if e, ok := args["engine"].(string); ok && e != "" {
|
||||||
|
engine = e
|
||||||
|
}
|
||||||
|
params.Set("engine", engine)
|
||||||
|
|
||||||
|
// Set number of results
|
||||||
|
num := t.maxResults
|
||||||
|
if n, ok := args["num"].(float64); ok && n > 0 {
|
||||||
|
num = int(n)
|
||||||
|
if num > 100 {
|
||||||
|
num = 100
|
||||||
|
}
|
||||||
|
}
|
||||||
|
params.Set("num", fmt.Sprintf("%d", num))
|
||||||
|
|
||||||
|
// Optional parameters
|
||||||
|
if location, ok := args["location"].(string); ok && location != "" {
|
||||||
|
params.Set("location", location)
|
||||||
|
}
|
||||||
|
if hl, ok := args["hl"].(string); ok && hl != "" {
|
||||||
|
params.Set("hl", hl)
|
||||||
|
}
|
||||||
|
if gl, ok := args["gl"].(string); ok && gl != "" {
|
||||||
|
params.Set("gl", gl)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Make request
|
||||||
|
apiURL := fmt.Sprintf("https://serpapi.com/search?%s", params.Encode())
|
||||||
|
req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return ErrorResult(fmt.Sprintf("failed to create request: %v", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
client := &http.Client{Timeout: 30 * time.Second}
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return ErrorResult(fmt.Sprintf("request failed: %v", err))
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return ErrorResult(fmt.Sprintf("failed to read response: %v", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return ErrorResult(fmt.Sprintf("API error (status %d): %s", resp.StatusCode, string(body)))
|
||||||
|
}
|
||||||
|
|
||||||
|
var result map[string]interface{}
|
||||||
|
if err := json.Unmarshal(body, &result); err != nil {
|
||||||
|
return ErrorResult(fmt.Sprintf("failed to parse response: %v", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for SerpAPI error
|
||||||
|
if errMsg, ok := result["error"].(string); ok && errMsg != "" {
|
||||||
|
return ErrorResult(fmt.Sprintf("SerpAPI error: %s", errMsg))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build output
|
||||||
|
var output strings.Builder
|
||||||
|
output.WriteString(fmt.Sprintf("# Search Results for: %s\n\n", query))
|
||||||
|
|
||||||
|
// Search metadata
|
||||||
|
if searchMetadata, ok := result["search_metadata"].(map[string]interface{}); ok {
|
||||||
|
if status, ok := searchMetadata["status"].(string); ok {
|
||||||
|
output.WriteString(fmt.Sprintf("**Status:** %s\n", status))
|
||||||
|
}
|
||||||
|
if totalTime, ok := searchMetadata["total_time_taken"].(float64); ok {
|
||||||
|
output.WriteString(fmt.Sprintf("**Time:** %.2fs\n", totalTime))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Search parameters info
|
||||||
|
if searchParams, ok := result["search_parameters"].(map[string]interface{}); ok {
|
||||||
|
if engine, ok := searchParams["engine"].(string); ok {
|
||||||
|
output.WriteString(fmt.Sprintf("**Engine:** %s\n", engine))
|
||||||
|
}
|
||||||
|
if location, ok := searchParams["location"].(string); ok && location != "" {
|
||||||
|
output.WriteString(fmt.Sprintf("**Location:** %s\n", location))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
output.WriteString("\n")
|
||||||
|
|
||||||
|
// Organic results
|
||||||
|
if organicResults, ok := result["organic_results"].([]interface{}); ok && len(organicResults) > 0 {
|
||||||
|
output.WriteString(fmt.Sprintf("## Organic Results (%d)\n\n", len(organicResults)))
|
||||||
|
|
||||||
|
for i, r := range organicResults {
|
||||||
|
if i >= t.maxResults {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
result, ok := r.(map[string]interface{})
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
position := i + 1
|
||||||
|
if pos, ok := result["position"].(float64); ok {
|
||||||
|
position = int(pos)
|
||||||
|
}
|
||||||
|
|
||||||
|
title := ""
|
||||||
|
if t, ok := result["title"].(string); ok {
|
||||||
|
title = t
|
||||||
|
}
|
||||||
|
|
||||||
|
link := ""
|
||||||
|
if l, ok := result["link"].(string); ok {
|
||||||
|
link = l
|
||||||
|
}
|
||||||
|
|
||||||
|
snippet := ""
|
||||||
|
if s, ok := result["snippet"].(string); ok {
|
||||||
|
snippet = s
|
||||||
|
}
|
||||||
|
|
||||||
|
output.WriteString(fmt.Sprintf("### %d. %s\n", position, title))
|
||||||
|
output.WriteString(fmt.Sprintf("**URL:** %s\n", link))
|
||||||
|
if snippet != "" {
|
||||||
|
output.WriteString(fmt.Sprintf("%s\n", snippet))
|
||||||
|
}
|
||||||
|
output.WriteString("\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Knowledge Graph
|
||||||
|
if kg, ok := result["knowledge_graph"].(map[string]interface{}); ok && len(kg) > 0 {
|
||||||
|
output.WriteString("## Knowledge Graph\n\n")
|
||||||
|
|
||||||
|
if title, ok := kg["title"].(string); ok {
|
||||||
|
output.WriteString(fmt.Sprintf("**Title:** %s\n", title))
|
||||||
|
}
|
||||||
|
if description, ok := kg["description"].(string); ok {
|
||||||
|
output.WriteString(fmt.Sprintf("**Description:** %s\n", description))
|
||||||
|
}
|
||||||
|
if link, ok := kg["knowledge_graph_search_link"].(string); ok {
|
||||||
|
output.WriteString(fmt.Sprintf("**More Info:** %s\n", link))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Additional attributes
|
||||||
|
for key, value := range kg {
|
||||||
|
if key == "title" || key == "description" || key == "knowledge_graph_search_link" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strVal, ok := value.(string); ok {
|
||||||
|
output.WriteString(fmt.Sprintf("- **%s:** %s\n", strings.Title(key), strVal))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
output.WriteString("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Related Questions (People Also Ask)
|
||||||
|
if relatedQuestions, ok := result["related_questions"].([]interface{}); ok && len(relatedQuestions) > 0 {
|
||||||
|
output.WriteString(fmt.Sprintf("## People Also Ask (%d)\n\n", len(relatedQuestions)))
|
||||||
|
|
||||||
|
for i, q := range relatedQuestions {
|
||||||
|
if i >= 5 { // Limit to 5 questions
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
question, ok := q.(map[string]interface{})
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if questionText, ok := question["question"].(string); ok {
|
||||||
|
output.WriteString(fmt.Sprintf("**Q:** %s\n", questionText))
|
||||||
|
}
|
||||||
|
if snippet, ok := question["snippet"].(string); ok {
|
||||||
|
output.WriteString(fmt.Sprintf("**A:** %s\n", snippet))
|
||||||
|
}
|
||||||
|
if link, ok := question["link"].(string); ok {
|
||||||
|
output.WriteString(fmt.Sprintf("*Source: %s*\n", link))
|
||||||
|
}
|
||||||
|
output.WriteString("\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Related Searches
|
||||||
|
if relatedSearches, ok := result["related_searches"].([]interface{}); ok && len(relatedSearches) > 0 {
|
||||||
|
output.WriteString("## Related Searches\n\n")
|
||||||
|
|
||||||
|
for i, s := range relatedSearches {
|
||||||
|
if i >= 10 { // Limit to 10
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
search, ok := s.(map[string]interface{})
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if query, ok := search["query"].(string); ok {
|
||||||
|
output.WriteString(fmt.Sprintf("- %s\n", query))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
output.WriteString("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
return &ToolResult{
|
||||||
|
ForLLM: output.String(),
|
||||||
|
ForUser: output.String(),
|
||||||
|
}
|
||||||
|
}
|
||||||
28
pkg/tools/serpapi_test.go
Normal file
28
pkg/tools/serpapi_test.go
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
package tools
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSerpAPITool_Integration(t *testing.T) {
|
||||||
|
apiKey := os.Getenv("SERPAPI_API_KEY")
|
||||||
|
if apiKey == "" {
|
||||||
|
t.Skip("SERPAPI_API_KEY not set")
|
||||||
|
}
|
||||||
|
|
||||||
|
tool := NewSerpAPITool(apiKey, 5)
|
||||||
|
|
||||||
|
result := tool.Execute(context.Background(), map[string]interface{}{
|
||||||
|
"query": "golang programming",
|
||||||
|
"engine": "google",
|
||||||
|
"num": float64(3),
|
||||||
|
})
|
||||||
|
|
||||||
|
if result.IsError {
|
||||||
|
t.Errorf("SerpAPI execution failed: %s", result.ForLLM)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("SerpAPI result:\n%s", result.ForUser)
|
||||||
|
}
|
||||||
Loading…
Add table
Reference in a new issue