From 479958b2500acb119a2a9f5d4e9dffa7e28ea8a4 Mon Sep 17 00:00:00 2001 From: I Putu Eddy Irawan Date: Sun, 1 Mar 2026 08:54:24 +0700 Subject: [PATCH] feat: add Exa search provider and .env file loading Add Exa AI (exa.ai) as a web search provider in the priority chain (Perplexity > Exa > Brave > Tavily > DuckDuckGo). Also add automatic .env file loading from the config directory and provider environment variable overrides for cleaner secret management. Co-Authored-By: Claude Opus 4.6 --- go.mod | 1 + go.sum | 2 ++ pkg/agent/loop.go | 3 ++ pkg/config/config.go | 69 ++++++++++++++++++++++++++++++++++++ pkg/config/defaults.go | 5 +++ pkg/tools/web.go | 80 +++++++++++++++++++++++++++++++++++++++++- 6 files changed, 159 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 7892cade6..be3c73ef0 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/gdamore/encoding v1.0.1 // indirect github.com/gdamore/tcell/v2 v2.13.8 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/joho/godotenv v1.5.1 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect diff --git a/go.sum b/go.sum index d1ee1d629..00f2b1600 100644 --- a/go.sum +++ b/go.sum @@ -101,6 +101,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyf github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c= diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 8fd7328d1..ea7520c98 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -112,6 +112,9 @@ func registerSharedTools( PerplexityAPIKey: cfg.Tools.Web.Perplexity.APIKey, PerplexityMaxResults: cfg.Tools.Web.Perplexity.MaxResults, PerplexityEnabled: cfg.Tools.Web.Perplexity.Enabled, + ExaAPIKey: cfg.Tools.Web.Exa.APIKey, + ExaMaxResults: cfg.Tools.Web.Exa.MaxResults, + ExaEnabled: cfg.Tools.Web.Exa.Enabled, Proxy: cfg.Tools.Web.Proxy, }); searchTool != nil { agent.Tools.Register(searchTool) diff --git a/pkg/config/config.go b/pkg/config/config.go index d84772d2b..ba2adc9ae 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -3,10 +3,14 @@ package config import ( "encoding/json" "fmt" + "log" "os" + "path/filepath" + "strings" "sync/atomic" "github.com/caarlos0/env/v11" + "github.com/joho/godotenv" "github.com/sipeed/picoclaw/pkg/fileutil" ) @@ -516,11 +520,18 @@ type PerplexityConfig struct { MaxResults int `json:"max_results" env:"PICOCLAW_TOOLS_WEB_PERPLEXITY_MAX_RESULTS"` } +type ExaConfig struct { + Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_WEB_EXA_ENABLED"` + APIKey string `json:"api_key" env:"PICOCLAW_TOOLS_WEB_EXA_API_KEY"` + MaxResults int `json:"max_results" env:"PICOCLAW_TOOLS_WEB_EXA_MAX_RESULTS"` +} + type WebToolsConfig struct { Brave BraveConfig `json:"brave"` Tavily TavilyConfig `json:"tavily"` DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"` Perplexity PerplexityConfig `json:"perplexity"` + Exa ExaConfig `json:"exa"` // Proxy is an optional proxy URL for web tools (http/https/socks5/socks5h). // For authenticated proxies, prefer HTTP_PROXY/HTTPS_PROXY env vars instead of embedding credentials in config. Proxy string `json:"proxy,omitempty" env:"PICOCLAW_TOOLS_WEB_PROXY"` @@ -605,10 +616,19 @@ func LoadConfig(path string) (*Config, error) { return nil, err } + // Load .env file from config directory (secrets, API keys, etc.) + envFile := filepath.Join(filepath.Dir(path), ".env") + if err := godotenv.Load(envFile); err != nil { + log.Printf("[INFO] No .env file loaded from %s: %v", envFile, err) + } + if err := env.Parse(cfg); err != nil { return nil, err } + // Load provider-specific env overrides (PICOCLAW_PROVIDERS__API_KEY, etc.) + loadProviderEnvOverrides(cfg) + // Migrate legacy channel config fields to new unified structures cfg.migrateChannelConfigs() @@ -774,3 +794,52 @@ func (c *Config) ValidateModelList() error { } return nil } + +// loadProviderEnvOverrides reads PICOCLAW_PROVIDERS__API_KEY and _API_BASE +// environment variables and sets them on the corresponding provider config fields. +// This enables storing provider secrets in .env files without using struct tags. +func loadProviderEnvOverrides(cfg *Config) { + providers := []struct { + name string + apiKey *string + base *string + }{ + {"ANTHROPIC", &cfg.Providers.Anthropic.APIKey, &cfg.Providers.Anthropic.APIBase}, + {"OPENAI", &cfg.Providers.OpenAI.APIKey, &cfg.Providers.OpenAI.APIBase}, + {"OPENROUTER", &cfg.Providers.OpenRouter.APIKey, &cfg.Providers.OpenRouter.APIBase}, + {"GROQ", &cfg.Providers.Groq.APIKey, &cfg.Providers.Groq.APIBase}, + {"ZHIPU", &cfg.Providers.Zhipu.APIKey, &cfg.Providers.Zhipu.APIBase}, + {"GEMINI", &cfg.Providers.Gemini.APIKey, &cfg.Providers.Gemini.APIBase}, + {"NVIDIA", &cfg.Providers.Nvidia.APIKey, &cfg.Providers.Nvidia.APIBase}, + {"OLLAMA", &cfg.Providers.Ollama.APIKey, &cfg.Providers.Ollama.APIBase}, + {"MOONSHOT", &cfg.Providers.Moonshot.APIKey, &cfg.Providers.Moonshot.APIBase}, + {"SHENGSUANYUN", &cfg.Providers.ShengSuanYun.APIKey, &cfg.Providers.ShengSuanYun.APIBase}, + {"DEEPSEEK", &cfg.Providers.DeepSeek.APIKey, &cfg.Providers.DeepSeek.APIBase}, + {"MISTRAL", &cfg.Providers.Mistral.APIKey, &cfg.Providers.Mistral.APIBase}, + } + for _, p := range providers { + if v := os.Getenv("PICOCLAW_PROVIDERS_" + p.name + "_API_KEY"); v != "" { + *p.apiKey = v + } + if v := os.Getenv("PICOCLAW_PROVIDERS_" + p.name + "_API_BASE"); v != "" { + *p.base = v + } + } +} + +// SanitizeForLog masks sensitive API key values for safe logging. +func SanitizeForLog(s string) string { + prefixes := []string{"sk-", "xoxb-", "xoxp-", "gsk_", "AIza"} + for _, p := range prefixes { + if strings.HasPrefix(s, p) { + if len(s) > len(p)+4 { + return s[:len(p)+4] + "****" + } + return p + "****" + } + } + if len(s) > 8 { + return s[:8] + "****" + } + return "****" +} diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index ebb924859..67fdabb5c 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -314,6 +314,11 @@ func DefaultConfig() *Config { APIKey: "", MaxResults: 5, }, + Exa: ExaConfig{ + Enabled: false, + APIKey: "", + MaxResults: 5, + }, }, Cron: CronToolsConfig{ ExecTimeoutMinutes: 5, diff --git a/pkg/tools/web.go b/pkg/tools/web.go index 8ba2a723a..63eb470f4 100644 --- a/pkg/tools/web.go +++ b/pkg/tools/web.go @@ -412,6 +412,9 @@ type WebSearchToolOptions struct { PerplexityAPIKey string PerplexityMaxResults int PerplexityEnabled bool + ExaAPIKey string + ExaMaxResults int + ExaEnabled bool Proxy string } @@ -419,12 +422,17 @@ func NewWebSearchTool(opts WebSearchToolOptions) *WebSearchTool { var provider SearchProvider maxResults := 5 - // Priority: Perplexity > Brave > Tavily > DuckDuckGo + // Priority: Perplexity > Exa > Brave > Tavily > DuckDuckGo if opts.PerplexityEnabled && opts.PerplexityAPIKey != "" { provider = &PerplexitySearchProvider{apiKey: opts.PerplexityAPIKey, proxy: opts.Proxy} if opts.PerplexityMaxResults > 0 { maxResults = opts.PerplexityMaxResults } + } else if opts.ExaEnabled && opts.ExaAPIKey != "" { + provider = &ExaSearchProvider{apiKey: opts.ExaAPIKey, proxy: opts.Proxy} + if opts.ExaMaxResults > 0 { + maxResults = opts.ExaMaxResults + } } else if opts.BraveEnabled && opts.BraveAPIKey != "" { provider = &BraveSearchProvider{apiKey: opts.BraveAPIKey, proxy: opts.Proxy} if opts.BraveMaxResults > 0 { @@ -684,3 +692,73 @@ func (t *WebFetchTool) extractText(htmlContent string) string { return strings.Join(cleanLines, "\n") } + +// ExaSearchProvider uses the Exa AI search API (https://exa.ai). +type ExaSearchProvider struct { + apiKey string + proxy string +} + +func (p *ExaSearchProvider) Search(ctx context.Context, query string, count int) (string, error) { + client, err := createHTTPClient(p.proxy, 30*time.Second) + if err != nil { + return "", fmt.Errorf("exa: failed to create HTTP client: %w", err) + } + + reqBody := map[string]any{ + "query": query, + "num_results": count, + "type": "neural", + } + jsonData, err := json.Marshal(reqBody) + if err != nil { + return "", fmt.Errorf("exa: marshal error: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", "https://api.exa.ai/search", bytes.NewReader(jsonData)) + if err != nil { + return "", fmt.Errorf("exa: request error: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-api-key", p.apiKey) + + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("exa: search failed: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("exa: read error: %w", err) + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("exa: API error %d: %s", resp.StatusCode, string(body)) + } + + var result struct { + Results []struct { + Title string `json:"title"` + URL string `json:"url"` + Text string `json:"text"` + } `json:"results"` + } + if err := json.Unmarshal(body, &result); err != nil { + return "", fmt.Errorf("exa: parse error: %w", err) + } + + var sb strings.Builder + for i, r := range result.Results { + sb.WriteString(fmt.Sprintf("%d. %s\n URL: %s\n", i+1, r.Title, r.URL)) + if r.Text != "" { + snippet := r.Text + if len(snippet) > 200 { + snippet = snippet[:200] + "..." + } + sb.WriteString(fmt.Sprintf(" %s\n", snippet)) + } + sb.WriteString("\n") + } + + return sb.String(), nil +}