feat: add Exa search provider and .env file loading
Add Exa AI (exa.ai) as a web search provider in the priority chain (Perplexity > Exa > Brave > Tavily > DuckDuckGo). Also add automatic .env file loading from the config directory and provider environment variable overrides for cleaner secret management. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
9c9524f934
commit
479958b250
6 changed files with 159 additions and 1 deletions
1
go.mod
1
go.mod
|
|
@ -36,6 +36,7 @@ require (
|
||||||
github.com/gdamore/encoding v1.0.1 // indirect
|
github.com/gdamore/encoding v1.0.1 // indirect
|
||||||
github.com/gdamore/tcell/v2 v2.13.8 // indirect
|
github.com/gdamore/tcell/v2 v2.13.8 // indirect
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
|
github.com/joho/godotenv v1.5.1 // indirect
|
||||||
github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
|
github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
|
||||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||||
|
|
|
||||||
2
go.sum
2
go.sum
|
|
@ -101,6 +101,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyf
|
||||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||||
|
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||||
|
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||||
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
|
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
|
||||||
|
|
|
||||||
|
|
@ -112,6 +112,9 @@ func registerSharedTools(
|
||||||
PerplexityAPIKey: cfg.Tools.Web.Perplexity.APIKey,
|
PerplexityAPIKey: cfg.Tools.Web.Perplexity.APIKey,
|
||||||
PerplexityMaxResults: cfg.Tools.Web.Perplexity.MaxResults,
|
PerplexityMaxResults: cfg.Tools.Web.Perplexity.MaxResults,
|
||||||
PerplexityEnabled: cfg.Tools.Web.Perplexity.Enabled,
|
PerplexityEnabled: cfg.Tools.Web.Perplexity.Enabled,
|
||||||
|
ExaAPIKey: cfg.Tools.Web.Exa.APIKey,
|
||||||
|
ExaMaxResults: cfg.Tools.Web.Exa.MaxResults,
|
||||||
|
ExaEnabled: cfg.Tools.Web.Exa.Enabled,
|
||||||
Proxy: cfg.Tools.Web.Proxy,
|
Proxy: cfg.Tools.Web.Proxy,
|
||||||
}); searchTool != nil {
|
}); searchTool != nil {
|
||||||
agent.Tools.Register(searchTool)
|
agent.Tools.Register(searchTool)
|
||||||
|
|
|
||||||
|
|
@ -3,10 +3,14 @@ package config
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
|
||||||
"github.com/caarlos0/env/v11"
|
"github.com/caarlos0/env/v11"
|
||||||
|
"github.com/joho/godotenv"
|
||||||
|
|
||||||
"github.com/sipeed/picoclaw/pkg/fileutil"
|
"github.com/sipeed/picoclaw/pkg/fileutil"
|
||||||
)
|
)
|
||||||
|
|
@ -516,11 +520,18 @@ type PerplexityConfig struct {
|
||||||
MaxResults int `json:"max_results" env:"PICOCLAW_TOOLS_WEB_PERPLEXITY_MAX_RESULTS"`
|
MaxResults int `json:"max_results" env:"PICOCLAW_TOOLS_WEB_PERPLEXITY_MAX_RESULTS"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ExaConfig struct {
|
||||||
|
Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_WEB_EXA_ENABLED"`
|
||||||
|
APIKey string `json:"api_key" env:"PICOCLAW_TOOLS_WEB_EXA_API_KEY"`
|
||||||
|
MaxResults int `json:"max_results" env:"PICOCLAW_TOOLS_WEB_EXA_MAX_RESULTS"`
|
||||||
|
}
|
||||||
|
|
||||||
type WebToolsConfig struct {
|
type WebToolsConfig struct {
|
||||||
Brave BraveConfig `json:"brave"`
|
Brave BraveConfig `json:"brave"`
|
||||||
Tavily TavilyConfig `json:"tavily"`
|
Tavily TavilyConfig `json:"tavily"`
|
||||||
DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"`
|
DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"`
|
||||||
Perplexity PerplexityConfig `json:"perplexity"`
|
Perplexity PerplexityConfig `json:"perplexity"`
|
||||||
|
Exa ExaConfig `json:"exa"`
|
||||||
// Proxy is an optional proxy URL for web tools (http/https/socks5/socks5h).
|
// Proxy is an optional proxy URL for web tools (http/https/socks5/socks5h).
|
||||||
// For authenticated proxies, prefer HTTP_PROXY/HTTPS_PROXY env vars instead of embedding credentials in config.
|
// For authenticated proxies, prefer HTTP_PROXY/HTTPS_PROXY env vars instead of embedding credentials in config.
|
||||||
Proxy string `json:"proxy,omitempty" env:"PICOCLAW_TOOLS_WEB_PROXY"`
|
Proxy string `json:"proxy,omitempty" env:"PICOCLAW_TOOLS_WEB_PROXY"`
|
||||||
|
|
@ -605,10 +616,19 @@ func LoadConfig(path string) (*Config, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Load .env file from config directory (secrets, API keys, etc.)
|
||||||
|
envFile := filepath.Join(filepath.Dir(path), ".env")
|
||||||
|
if err := godotenv.Load(envFile); err != nil {
|
||||||
|
log.Printf("[INFO] No .env file loaded from %s: %v", envFile, err)
|
||||||
|
}
|
||||||
|
|
||||||
if err := env.Parse(cfg); err != nil {
|
if err := env.Parse(cfg); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Load provider-specific env overrides (PICOCLAW_PROVIDERS_<NAME>_API_KEY, etc.)
|
||||||
|
loadProviderEnvOverrides(cfg)
|
||||||
|
|
||||||
// Migrate legacy channel config fields to new unified structures
|
// Migrate legacy channel config fields to new unified structures
|
||||||
cfg.migrateChannelConfigs()
|
cfg.migrateChannelConfigs()
|
||||||
|
|
||||||
|
|
@ -774,3 +794,52 @@ func (c *Config) ValidateModelList() error {
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// loadProviderEnvOverrides reads PICOCLAW_PROVIDERS_<NAME>_API_KEY and _API_BASE
|
||||||
|
// environment variables and sets them on the corresponding provider config fields.
|
||||||
|
// This enables storing provider secrets in .env files without using struct tags.
|
||||||
|
func loadProviderEnvOverrides(cfg *Config) {
|
||||||
|
providers := []struct {
|
||||||
|
name string
|
||||||
|
apiKey *string
|
||||||
|
base *string
|
||||||
|
}{
|
||||||
|
{"ANTHROPIC", &cfg.Providers.Anthropic.APIKey, &cfg.Providers.Anthropic.APIBase},
|
||||||
|
{"OPENAI", &cfg.Providers.OpenAI.APIKey, &cfg.Providers.OpenAI.APIBase},
|
||||||
|
{"OPENROUTER", &cfg.Providers.OpenRouter.APIKey, &cfg.Providers.OpenRouter.APIBase},
|
||||||
|
{"GROQ", &cfg.Providers.Groq.APIKey, &cfg.Providers.Groq.APIBase},
|
||||||
|
{"ZHIPU", &cfg.Providers.Zhipu.APIKey, &cfg.Providers.Zhipu.APIBase},
|
||||||
|
{"GEMINI", &cfg.Providers.Gemini.APIKey, &cfg.Providers.Gemini.APIBase},
|
||||||
|
{"NVIDIA", &cfg.Providers.Nvidia.APIKey, &cfg.Providers.Nvidia.APIBase},
|
||||||
|
{"OLLAMA", &cfg.Providers.Ollama.APIKey, &cfg.Providers.Ollama.APIBase},
|
||||||
|
{"MOONSHOT", &cfg.Providers.Moonshot.APIKey, &cfg.Providers.Moonshot.APIBase},
|
||||||
|
{"SHENGSUANYUN", &cfg.Providers.ShengSuanYun.APIKey, &cfg.Providers.ShengSuanYun.APIBase},
|
||||||
|
{"DEEPSEEK", &cfg.Providers.DeepSeek.APIKey, &cfg.Providers.DeepSeek.APIBase},
|
||||||
|
{"MISTRAL", &cfg.Providers.Mistral.APIKey, &cfg.Providers.Mistral.APIBase},
|
||||||
|
}
|
||||||
|
for _, p := range providers {
|
||||||
|
if v := os.Getenv("PICOCLAW_PROVIDERS_" + p.name + "_API_KEY"); v != "" {
|
||||||
|
*p.apiKey = v
|
||||||
|
}
|
||||||
|
if v := os.Getenv("PICOCLAW_PROVIDERS_" + p.name + "_API_BASE"); v != "" {
|
||||||
|
*p.base = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SanitizeForLog masks sensitive API key values for safe logging.
|
||||||
|
func SanitizeForLog(s string) string {
|
||||||
|
prefixes := []string{"sk-", "xoxb-", "xoxp-", "gsk_", "AIza"}
|
||||||
|
for _, p := range prefixes {
|
||||||
|
if strings.HasPrefix(s, p) {
|
||||||
|
if len(s) > len(p)+4 {
|
||||||
|
return s[:len(p)+4] + "****"
|
||||||
|
}
|
||||||
|
return p + "****"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(s) > 8 {
|
||||||
|
return s[:8] + "****"
|
||||||
|
}
|
||||||
|
return "****"
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -314,6 +314,11 @@ func DefaultConfig() *Config {
|
||||||
APIKey: "",
|
APIKey: "",
|
||||||
MaxResults: 5,
|
MaxResults: 5,
|
||||||
},
|
},
|
||||||
|
Exa: ExaConfig{
|
||||||
|
Enabled: false,
|
||||||
|
APIKey: "",
|
||||||
|
MaxResults: 5,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
Cron: CronToolsConfig{
|
Cron: CronToolsConfig{
|
||||||
ExecTimeoutMinutes: 5,
|
ExecTimeoutMinutes: 5,
|
||||||
|
|
|
||||||
|
|
@ -412,6 +412,9 @@ type WebSearchToolOptions struct {
|
||||||
PerplexityAPIKey string
|
PerplexityAPIKey string
|
||||||
PerplexityMaxResults int
|
PerplexityMaxResults int
|
||||||
PerplexityEnabled bool
|
PerplexityEnabled bool
|
||||||
|
ExaAPIKey string
|
||||||
|
ExaMaxResults int
|
||||||
|
ExaEnabled bool
|
||||||
Proxy string
|
Proxy string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -419,12 +422,17 @@ func NewWebSearchTool(opts WebSearchToolOptions) *WebSearchTool {
|
||||||
var provider SearchProvider
|
var provider SearchProvider
|
||||||
maxResults := 5
|
maxResults := 5
|
||||||
|
|
||||||
// Priority: Perplexity > Brave > Tavily > DuckDuckGo
|
// Priority: Perplexity > Exa > Brave > Tavily > DuckDuckGo
|
||||||
if opts.PerplexityEnabled && opts.PerplexityAPIKey != "" {
|
if opts.PerplexityEnabled && opts.PerplexityAPIKey != "" {
|
||||||
provider = &PerplexitySearchProvider{apiKey: opts.PerplexityAPIKey, proxy: opts.Proxy}
|
provider = &PerplexitySearchProvider{apiKey: opts.PerplexityAPIKey, proxy: opts.Proxy}
|
||||||
if opts.PerplexityMaxResults > 0 {
|
if opts.PerplexityMaxResults > 0 {
|
||||||
maxResults = opts.PerplexityMaxResults
|
maxResults = opts.PerplexityMaxResults
|
||||||
}
|
}
|
||||||
|
} else if opts.ExaEnabled && opts.ExaAPIKey != "" {
|
||||||
|
provider = &ExaSearchProvider{apiKey: opts.ExaAPIKey, proxy: opts.Proxy}
|
||||||
|
if opts.ExaMaxResults > 0 {
|
||||||
|
maxResults = opts.ExaMaxResults
|
||||||
|
}
|
||||||
} else if opts.BraveEnabled && opts.BraveAPIKey != "" {
|
} else if opts.BraveEnabled && opts.BraveAPIKey != "" {
|
||||||
provider = &BraveSearchProvider{apiKey: opts.BraveAPIKey, proxy: opts.Proxy}
|
provider = &BraveSearchProvider{apiKey: opts.BraveAPIKey, proxy: opts.Proxy}
|
||||||
if opts.BraveMaxResults > 0 {
|
if opts.BraveMaxResults > 0 {
|
||||||
|
|
@ -684,3 +692,73 @@ func (t *WebFetchTool) extractText(htmlContent string) string {
|
||||||
|
|
||||||
return strings.Join(cleanLines, "\n")
|
return strings.Join(cleanLines, "\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ExaSearchProvider uses the Exa AI search API (https://exa.ai).
|
||||||
|
type ExaSearchProvider struct {
|
||||||
|
apiKey string
|
||||||
|
proxy string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *ExaSearchProvider) Search(ctx context.Context, query string, count int) (string, error) {
|
||||||
|
client, err := createHTTPClient(p.proxy, 30*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("exa: failed to create HTTP client: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
reqBody := map[string]any{
|
||||||
|
"query": query,
|
||||||
|
"num_results": count,
|
||||||
|
"type": "neural",
|
||||||
|
}
|
||||||
|
jsonData, err := json.Marshal(reqBody)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("exa: marshal error: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.exa.ai/search", bytes.NewReader(jsonData))
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("exa: request error: %w", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("x-api-key", p.apiKey)
|
||||||
|
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("exa: search failed: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("exa: read error: %w", err)
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return "", fmt.Errorf("exa: API error %d: %s", resp.StatusCode, string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
var result struct {
|
||||||
|
Results []struct {
|
||||||
|
Title string `json:"title"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
Text string `json:"text"`
|
||||||
|
} `json:"results"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, &result); err != nil {
|
||||||
|
return "", fmt.Errorf("exa: parse error: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var sb strings.Builder
|
||||||
|
for i, r := range result.Results {
|
||||||
|
sb.WriteString(fmt.Sprintf("%d. %s\n URL: %s\n", i+1, r.Title, r.URL))
|
||||||
|
if r.Text != "" {
|
||||||
|
snippet := r.Text
|
||||||
|
if len(snippet) > 200 {
|
||||||
|
snippet = snippet[:200] + "..."
|
||||||
|
}
|
||||||
|
sb.WriteString(fmt.Sprintf(" %s\n", snippet))
|
||||||
|
}
|
||||||
|
sb.WriteString("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
return sb.String(), nil
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue