From e7d8975f1c3420733d1e34411c6f352c0e00b3d2 Mon Sep 17 00:00:00 2001 From: Truong Vinh Tran Date: Fri, 20 Feb 2026 12:02:00 +0100 Subject: [PATCH 1/9] feat: Add SearXNG search provider support Implements SearXNG as a third web search provider to address Oracle Cloud datacenter IP blocking issues and provide a cost-free, self-hosted alternative to commercial search APIs. Changes: - Add SearXNGConfig struct with Enabled, BaseURL, and MaxResults fields - Implement SearXNGSearchProvider with JSON API integration - Update provider priority: Perplexity > Brave > SearXNG > DuckDuckGo - Wire SearXNG configuration through agent tool registration - Add default config values (disabled by default, empty BaseURL) Benefits: - Solves DuckDuckGo datacenter IP blocking (138 bytes redirect responses) - Zero-cost alternative to Brave Search API ($5/1000 queries) - Self-hosted solution with 70+ aggregated search engines - Privacy-focused with no rate limits or API keys required - Ideal for Oracle Cloud, GCP, AWS, and Azure VM deployments The implementation follows the existing provider interface pattern and maintains backward compatibility with all existing search providers. Co-Authored-By: Claude Sonnet 4.5 --- pkg/agent/loop.go | 3 ++ pkg/config/config.go | 7 ++++ pkg/config/defaults.go | 5 +++ pkg/tools/web.go | 72 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 86 insertions(+), 1 deletion(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index e7b48d47a..4cdc1fe90 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -96,6 +96,9 @@ func registerSharedTools(cfg *config.Config, msgBus *bus.MessageBus, registry *A PerplexityAPIKey: cfg.Tools.Web.Perplexity.APIKey, PerplexityMaxResults: cfg.Tools.Web.Perplexity.MaxResults, PerplexityEnabled: cfg.Tools.Web.Perplexity.Enabled, + SearXNGBaseURL: cfg.Tools.Web.SearXNG.BaseURL, + SearXNGMaxResults: cfg.Tools.Web.SearXNG.MaxResults, + SearXNGEnabled: cfg.Tools.Web.SearXNG.Enabled, }); searchTool != nil { agent.Tools.Register(searchTool) } diff --git a/pkg/config/config.go b/pkg/config/config.go index 0d41796a4..87a1186a8 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -400,10 +400,17 @@ type PerplexityConfig struct { MaxResults int `json:"max_results" env:"PICOCLAW_TOOLS_WEB_PERPLEXITY_MAX_RESULTS"` } +type SearXNGConfig struct { + Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_WEB_SEARXNG_ENABLED"` + BaseURL string `json:"base_url" env:"PICOCLAW_TOOLS_WEB_SEARXNG_BASE_URL"` + MaxResults int `json:"max_results" env:"PICOCLAW_TOOLS_WEB_SEARXNG_MAX_RESULTS"` +} + type WebToolsConfig struct { Brave BraveConfig `json:"brave"` DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"` Perplexity PerplexityConfig `json:"perplexity"` + SearXNG SearXNGConfig `json:"searxng"` } type CronToolsConfig struct { diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index 54d6d68c3..a8c5bee58 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -258,6 +258,11 @@ func DefaultConfig() *Config { APIKey: "", MaxResults: 5, }, + SearXNG: SearXNGConfig{ + Enabled: false, + BaseURL: "", + MaxResults: 5, + }, }, Cron: CronToolsConfig{ ExecTimeoutMinutes: 5, diff --git a/pkg/tools/web.go b/pkg/tools/web.go index 1f5c58ea5..e1a640ff0 100644 --- a/pkg/tools/web.go +++ b/pkg/tools/web.go @@ -241,6 +241,68 @@ func (p *PerplexitySearchProvider) Search(ctx context.Context, query string, cou return fmt.Sprintf("Results for: %s (via Perplexity)\n%s", query, searchResp.Choices[0].Message.Content), nil } +type SearXNGSearchProvider struct { + baseURL string +} + +func (p *SearXNGSearchProvider) Search(ctx context.Context, query string, count int) (string, error) { + searchURL := fmt.Sprintf("%s/search?q=%s&format=json&categories=general", + strings.TrimSuffix(p.baseURL, "/"), + url.QueryEscape(query)) + + req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil) + if err != nil { + return "", fmt.Errorf("failed to create request: %w", err) + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("SearXNG returned status %d", resp.StatusCode) + } + + var result struct { + Results []struct { + Title string `json:"title"` + URL string `json:"url"` + Content string `json:"content"` + Engine string `json:"engine"` + Score float64 `json:"score"` + } `json:"results"` + } + + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", fmt.Errorf("failed to parse response: %w", err) + } + + if len(result.Results) == 0 { + return fmt.Sprintf("No results for: %s", query), nil + } + + // Limit results to requested count + if len(result.Results) > count { + result.Results = result.Results[:count] + } + + // Format results in standard PicoClaw format + var b strings.Builder + b.WriteString(fmt.Sprintf("Results for: %s (via SearXNG)\n", query)) + for i, r := range result.Results { + b.WriteString(fmt.Sprintf("%d. %s\n", i+1, r.Title)) + b.WriteString(fmt.Sprintf(" %s\n", r.URL)) + if r.Content != "" { + b.WriteString(fmt.Sprintf(" %s\n", r.Content)) + } + } + + return b.String(), nil +} + type WebSearchTool struct { provider SearchProvider maxResults int @@ -255,13 +317,16 @@ type WebSearchToolOptions struct { PerplexityAPIKey string PerplexityMaxResults int PerplexityEnabled bool + SearXNGBaseURL string + SearXNGMaxResults int + SearXNGEnabled bool } func NewWebSearchTool(opts WebSearchToolOptions) *WebSearchTool { var provider SearchProvider maxResults := 5 - // Priority: Perplexity > Brave > DuckDuckGo + // Priority: Perplexity > Brave > SearXNG > DuckDuckGo if opts.PerplexityEnabled && opts.PerplexityAPIKey != "" { provider = &PerplexitySearchProvider{apiKey: opts.PerplexityAPIKey} if opts.PerplexityMaxResults > 0 { @@ -272,6 +337,11 @@ func NewWebSearchTool(opts WebSearchToolOptions) *WebSearchTool { if opts.BraveMaxResults > 0 { maxResults = opts.BraveMaxResults } + } else if opts.SearXNGEnabled && opts.SearXNGBaseURL != "" { + provider = &SearXNGSearchProvider{baseURL: opts.SearXNGBaseURL} + if opts.SearXNGMaxResults > 0 { + maxResults = opts.SearXNGMaxResults + } } else if opts.DuckDuckGoEnabled { provider = &DuckDuckGoSearchProvider{} if opts.DuckDuckGoMaxResults > 0 { From 25d8f0e1ca075cc505bef96456757b9f46690ed8 Mon Sep 17 00:00:00 2001 From: Truong Vinh Tran Date: Fri, 20 Feb 2026 12:37:58 +0100 Subject: [PATCH 2/9] docs: Add SearXNG web search provider documentation Update README to document the new SearXNG search provider option alongside existing Brave, DuckDuckGo, and Perplexity providers. Changes: - Document provider priority order: Perplexity > Brave > SearXNG > DuckDuckGo - Add SearXNG configuration examples in Quick Start and Full Config sections - Expand "Get API Keys" section with all 4 search provider options - Enhance troubleshooting section with detailed setup instructions for each provider - Add SearXNG to API Key Comparison table (unlimited/self-hosted) SearXNG benefits documented: - Zero cost with no API fees or rate limits - Privacy-focused self-hosted solution - Aggregates 70+ search engines for comprehensive results - Solves datacenter IP blocking issues on Oracle Cloud, GCP, AWS, Azure - No API key required, just deploy and configure base URL This documentation complements the code implementation in commit e7d8975. Co-Authored-By: Claude Sonnet 4.5 --- README.md | 116 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 105 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 468350409..83a533973 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,16 @@ picoclaw onboard "duckduckgo": { "enabled": true, "max_results": 5 + }, + "perplexity": { + "enabled": false, + "api_key": "YOUR_PERPLEXITY_API_KEY", + "max_results": 5 + }, + "searxng": { + "enabled": false, + "base_url": "http://your-searxng-instance:8888", + "max_results": 5 } } } @@ -248,7 +258,11 @@ picoclaw onboard **3. Get API Keys** * **LLM Provider**: [OpenRouter](https://openrouter.ai/keys) 路 [Zhipu](https://open.bigmodel.cn/usercenter/proj-mgmt/apikeys) 路 [Anthropic](https://console.anthropic.com) 路 [OpenAI](https://platform.openai.com) 路 [Gemini](https://aistudio.google.com/api-keys) -* **Web Search** (optional): [Brave Search](https://brave.com/search/api) - Free tier available (2000 requests/month) +* **Web Search** (optional): + * [Brave Search](https://brave.com/search/api) - Free tier (2000 requests/month) + * [Perplexity](https://www.perplexity.ai) - AI-powered search with chat interface + * [SearXNG](https://github.com/searxng/searxng) - Self-hosted metasearch engine (free, no API key needed) + * DuckDuckGo - Built-in fallback (no API key required) > **Note**: See `config.example.json` for a complete configuration template. @@ -977,6 +991,16 @@ picoclaw agent -m "Hello" "duckduckgo": { "enabled": true, "max_results": 5 + }, + "perplexity": { + "enabled": false, + "api_key": "", + "max_results": 5 + }, + "searxng": { + "enabled": false, + "base_url": "http://localhost:8888", + "max_results": 5 } }, "cron": { @@ -1034,10 +1058,69 @@ discord: This is normal if you haven't configured a search API key yet. PicoClaw will provide helpful links for manual searching. -To enable web search: +#### Search Provider Priority -1. **Option 1 (Recommended)**: Get a free API key at [https://brave.com/search/api](https://brave.com/search/api) (2000 free queries/month) for the best results. -2. **Option 2 (No Credit Card)**: If you don't have a key, we automatically fall back to **DuckDuckGo** (no key required). +PicoClaw automatically selects the best available search provider in this order: +1. **Perplexity** (if enabled and API key configured) - AI-powered search with citations +2. **Brave Search** (if enabled and API key configured) - Privacy-focused with 2000 free queries/month +3. **SearXNG** (if enabled and base_url configured) - Self-hosted metasearch aggregating 70+ engines +4. **DuckDuckGo** (if enabled, default fallback) - No API key required + +#### Web Search Configuration Options + +**Option 1 (Best Results)**: Perplexity AI Search +```json +{ + "tools": { + "web": { + "perplexity": { + "enabled": true, + "api_key": "YOUR_PERPLEXITY_API_KEY", + "max_results": 5 + } + } + } +} +``` + +**Option 2 (Free Tier)**: Get a free API key at [https://brave.com/search/api](https://brave.com/search/api) (2000 free queries/month) +```json +{ + "tools": { + "web": { + "brave": { + "enabled": true, + "api_key": "YOUR_BRAVE_API_KEY", + "max_results": 5 + } + } + } +} +``` + +**Option 3 (Self-Hosted)**: Deploy your own [SearXNG](https://github.com/searxng/searxng) instance +```json +{ + "tools": { + "web": { + "searxng": { + "enabled": true, + "base_url": "http://your-server:8888", + "max_results": 5 + } + } + } +} +``` + +Benefits of SearXNG: +- **Zero cost**: No API fees or rate limits +- **Privacy-focused**: Self-hosted, no tracking +- **Aggregate results**: Queries 70+ search engines simultaneously +- **Perfect for cloud VMs**: Solves datacenter IP blocking issues (Oracle Cloud, GCP, AWS, Azure) +- **No API key needed**: Just deploy and configure the base URL + +**Option 4 (No Setup Required)**: DuckDuckGo is enabled by default as fallback (no API key needed) Add the key to `~/.picoclaw/config.json` if using Brave: @@ -1053,6 +1136,16 @@ Add the key to `~/.picoclaw/config.json` if using Brave: "duckduckgo": { "enabled": true, "max_results": 5 + }, + "perplexity": { + "enabled": false, + "api_key": "YOUR_PERPLEXITY_API_KEY", + "max_results": 5 + }, + "searxng": { + "enabled": false, + "base_url": "http://your-searxng-instance:8888", + "max_results": 5 } } } @@ -1071,10 +1164,11 @@ This happens when another instance of the bot is running. Make sure only one `pi ## 馃摑 API Key Comparison -| Service | Free Tier | Use Case | -| ---------------- | ------------------- | ------------------------------------- | -| **OpenRouter** | 200K tokens/month | Multiple models (Claude, GPT-4, etc.) | -| **Zhipu** | 200K tokens/month | Best for Chinese users | -| **Brave Search** | 2000 queries/month | Web search functionality | -| **Groq** | Free tier available | Fast inference (Llama, Mixtral) | -| **Cerebras** | Free tier available | Fast inference (Llama, Qwen, etc.) | +| Service | Free Tier | Use Case | +| ---------------- | ------------------------ | ------------------------------------- | +| **OpenRouter** | 200K tokens/month | Multiple models (Claude, GPT-4, etc.) | +| **Zhipu** | 200K tokens/month | Best for Chinese users | +| **Brave Search** | 2000 queries/month | Web search functionality | +| **SearXNG** | Unlimited (self-hosted) | Privacy-focused metasearch (70+ engines) | +| **Groq** | Free tier available | Fast inference (Llama, Mixtral) | +| **Cerebras** | Free tier available | Fast inference (Llama, Qwen, etc.) | From a5043854c38ec1ee2eb7995e0123b5b06f32fde0 Mon Sep 17 00:00:00 2001 From: Truong Vinh Tran Date: Fri, 20 Feb 2026 12:39:25 +0100 Subject: [PATCH 3/9] docs: Add SearXNG to example configuration file Update config.example.json to include SearXNG web search provider configuration alongside existing Brave, DuckDuckGo, and Perplexity options. This ensures users have a complete reference for all available search providers when setting up their PicoClaw instance. Co-Authored-By: Claude Sonnet 4.5 --- config/config.example.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/config.example.json b/config/config.example.json index abc928e92..e046f7b76 100644 --- a/config/config.example.json +++ b/config/config.example.json @@ -186,6 +186,11 @@ "enabled": false, "api_key": "pplx-xxx", "max_results": 5 + }, + "searxng": { + "enabled": false, + "base_url": "http://localhost:8888", + "max_results": 5 } }, "cron": { From 5d2674b336ca8fb8e89856974a8f80452d637c0b Mon Sep 17 00:00:00 2001 From: Truong Vinh Tran Date: Fri, 20 Feb 2026 14:02:46 +0100 Subject: [PATCH 4/9] docs: Update Brave Search pricing - now $5/1000 queries (no free tier) Brave Search discontinued free tier on Feb 12, 2026. Updated all README references to reflect paid pricing. Emphasized SearXNG as free alternative. Co-Authored-By: Claude Sonnet 4.5 --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 83a533973..1f7200d15 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,7 @@ docker compose --profile gateway up -d > [!TIP] > Set your API key in `~/.picoclaw/config.json`. > Get API keys: [OpenRouter](https://openrouter.ai/keys) (LLM) 路 [Zhipu](https://open.bigmodel.cn/usercenter/proj-mgmt/apikeys) (LLM) -> Web search is **optional** - get free [Brave Search API](https://brave.com/search/api) (2000 free queries/month) or use built-in auto fallback. +> Web search is **optional** - [Brave Search API](https://brave.com/search/api) ($5/1000 queries, ~$5-6/month), [SearXNG](https://github.com/searxng/searxng) (free, self-hosted), or use built-in DuckDuckGo fallback. **1. Initialize** @@ -259,7 +259,7 @@ picoclaw onboard * **LLM Provider**: [OpenRouter](https://openrouter.ai/keys) 路 [Zhipu](https://open.bigmodel.cn/usercenter/proj-mgmt/apikeys) 路 [Anthropic](https://console.anthropic.com) 路 [OpenAI](https://platform.openai.com) 路 [Gemini](https://aistudio.google.com/api-keys) * **Web Search** (optional): - * [Brave Search](https://brave.com/search/api) - Free tier (2000 requests/month) + * [Brave Search](https://brave.com/search/api) - Paid ($5/1000 queries, ~$5-6/month) * [Perplexity](https://www.perplexity.ai) - AI-powered search with chat interface * [SearXNG](https://github.com/searxng/searxng) - Self-hosted metasearch engine (free, no API key needed) * DuckDuckGo - Built-in fallback (no API key required) @@ -1062,9 +1062,9 @@ This is normal if you haven't configured a search API key yet. PicoClaw will pro PicoClaw automatically selects the best available search provider in this order: 1. **Perplexity** (if enabled and API key configured) - AI-powered search with citations -2. **Brave Search** (if enabled and API key configured) - Privacy-focused with 2000 free queries/month -3. **SearXNG** (if enabled and base_url configured) - Self-hosted metasearch aggregating 70+ engines -4. **DuckDuckGo** (if enabled, default fallback) - No API key required +2. **Brave Search** (if enabled and API key configured) - Privacy-focused paid API ($5/1000 queries) +3. **SearXNG** (if enabled and base_url configured) - Self-hosted metasearch aggregating 70+ engines (free) +4. **DuckDuckGo** (if enabled, default fallback) - No API key required (free) #### Web Search Configuration Options @@ -1083,7 +1083,7 @@ PicoClaw automatically selects the best available search provider in this order: } ``` -**Option 2 (Free Tier)**: Get a free API key at [https://brave.com/search/api](https://brave.com/search/api) (2000 free queries/month) +**Option 2 (Paid API)**: Get an API key at [https://brave.com/search/api](https://brave.com/search/api) ($5/1000 queries, ~$5-6/month) ```json { "tools": { @@ -1168,7 +1168,7 @@ This happens when another instance of the bot is running. Make sure only one `pi | ---------------- | ------------------------ | ------------------------------------- | | **OpenRouter** | 200K tokens/month | Multiple models (Claude, GPT-4, etc.) | | **Zhipu** | 200K tokens/month | Best for Chinese users | -| **Brave Search** | 2000 queries/month | Web search functionality | +| **Brave Search** | Paid ($5/1000 queries) | Web search functionality | | **SearXNG** | Unlimited (self-hosted) | Privacy-focused metasearch (70+ engines) | | **Groq** | Free tier available | Fast inference (Llama, Mixtral) | | **Cerebras** | Free tier available | Fast inference (Llama, Qwen, etc.) | From de0f15d548f7a62521531423083e38a58b12fb1a Mon Sep 17 00:00:00 2001 From: Truong Vinh Tran Date: Wed, 4 Mar 2026 21:48:36 +0100 Subject: [PATCH 5/9] style: fix golines struct tag alignment in SearXNGConfig Co-Authored-By: Claude Opus 4.6 --- pkg/config/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 6cabddafc..8b0fd8098 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -548,8 +548,8 @@ type PerplexityConfig struct { } type SearXNGConfig struct { - Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_WEB_SEARXNG_ENABLED"` - BaseURL string `json:"base_url" env:"PICOCLAW_TOOLS_WEB_SEARXNG_BASE_URL"` + Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_WEB_SEARXNG_ENABLED"` + BaseURL string `json:"base_url" env:"PICOCLAW_TOOLS_WEB_SEARXNG_BASE_URL"` MaxResults int `json:"max_results" env:"PICOCLAW_TOOLS_WEB_SEARXNG_MAX_RESULTS"` } From 7a2fdc24dc202012db849722ba92df8238257b3e Mon Sep 17 00:00:00 2001 From: qs3c <2749950753@qq.com> Date: Thu, 5 Mar 2026 15:00:06 +0800 Subject: [PATCH 6/9] fix(skills): retry ClawHub requests on 429 --- docs/tools_configuration.md | 2 + pkg/skills/clawhub_registry.go | 162 ++++++++++++++++++++++++---- pkg/skills/clawhub_registry_test.go | 81 ++++++++++++++ 3 files changed, 225 insertions(+), 20 deletions(-) diff --git a/docs/tools_configuration.md b/docs/tools_configuration.md index 6204fb0c8..e64a3a107 100644 --- a/docs/tools_configuration.md +++ b/docs/tools_configuration.md @@ -180,6 +180,7 @@ The skills tool configures skill discovery and installation via registries like | ---------------------------------- | ------ | -------------------- | ----------------------- | | `registries.clawhub.enabled` | bool | true | Enable ClawHub registry | | `registries.clawhub.base_url` | string | `https://clawhub.ai` | ClawHub base URL | +| `registries.clawhub.auth_token` | string | `""` | Optional Bearer token for higher rate limits | | `registries.clawhub.search_path` | string | `/api/v1/search` | Search API path | | `registries.clawhub.skills_path` | string | `/api/v1/skills` | Skills API path | | `registries.clawhub.download_path` | string | `/api/v1/download` | Download API path | @@ -194,6 +195,7 @@ The skills tool configures skill discovery and installation via registries like "clawhub": { "enabled": true, "base_url": "https://clawhub.ai", + "auth_token": "", "search_path": "/api/v1/search", "skills_path": "/api/v1/skills", "download_path": "/api/v1/download" diff --git a/pkg/skills/clawhub_registry.go b/pkg/skills/clawhub_registry.go index f78197bbe..b520f3260 100644 --- a/pkg/skills/clawhub_registry.go +++ b/pkg/skills/clawhub_registry.go @@ -8,6 +8,8 @@ import ( "net/http" "net/url" "os" + "strconv" + "strings" "time" "github.com/sipeed/picoclaw/pkg/utils" @@ -17,6 +19,7 @@ const ( defaultClawHubTimeout = 30 * time.Second defaultMaxZipSize = 50 * 1024 * 1024 // 50 MB defaultMaxResponseSize = 2 * 1024 * 1024 // 2 MB + defaultMaxRetries = 3 ) // ClawHubRegistry implements SkillRegistry for the ClawHub platform. @@ -259,15 +262,7 @@ func (c *ClawHubRegistry) DownloadAndInstall( } u.RawQuery = q.Encode() - req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - if c.authToken != "" { - req.Header.Set("Authorization", "Bearer "+c.authToken) - } - - tmpPath, err := utils.DownloadToFile(ctx, c.client, req, int64(c.maxZipSize)) + tmpPath, err := c.downloadToTempFileWithRetry(ctx, u.String()) if err != nil { return nil, fmt.Errorf("download failed: %w", err) } @@ -284,17 +279,7 @@ func (c *ClawHubRegistry) DownloadAndInstall( // --- HTTP helper --- func (c *ClawHubRegistry) doGet(ctx context.Context, urlStr string) ([]byte, error) { - req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) - if err != nil { - return nil, err - } - - req.Header.Set("Accept", "application/json") - if c.authToken != "" { - req.Header.Set("Authorization", "Bearer "+c.authToken) - } - - resp, err := c.client.Do(req) + resp, err := c.doGetWithRetry(ctx, urlStr, "application/json") if err != nil { return nil, err } @@ -312,3 +297,140 @@ func (c *ClawHubRegistry) doGet(ctx context.Context, urlStr string) ([]byte, err return body, nil } + +func (c *ClawHubRegistry) doGetWithRetry(ctx context.Context, urlStr, accept string) (*http.Response, error) { + var lastErr error + for attempt := 0; attempt < defaultMaxRetries; attempt++ { + req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", accept) + if c.authToken != "" { + req.Header.Set("Authorization", "Bearer "+c.authToken) + } + + resp, err := c.client.Do(req) + if err != nil { + lastErr = err + } else { + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return resp, nil + } + + if !isRetryableStatus(resp.StatusCode) || attempt == defaultMaxRetries-1 { + return resp, nil + } + + delay := retryDelay(resp.Header.Get("Retry-After"), attempt) + resp.Body.Close() + if err := sleepWithContext(ctx, delay); err != nil { + return nil, err + } + continue + } + + if attempt == defaultMaxRetries-1 { + return nil, lastErr + } + if err := sleepWithContext(ctx, retryDelay("", attempt)); err != nil { + return nil, err + } + } + return nil, lastErr +} + +func (c *ClawHubRegistry) downloadToTempFileWithRetry(ctx context.Context, urlStr string) (string, error) { + resp, err := c.doGetWithRetry(ctx, urlStr, "application/zip") + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + errBody := make([]byte, 512) + n, _ := io.ReadFull(resp.Body, errBody) + return "", fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(errBody[:n])) + } + + tmpFile, err := os.CreateTemp("", "picoclaw-dl-*") + if err != nil { + return "", fmt.Errorf("failed to create temp file: %w", err) + } + tmpPath := tmpFile.Name() + + cleanup := func() { + _ = tmpFile.Close() + _ = os.Remove(tmpPath) + } + + src := io.LimitReader(resp.Body, int64(c.maxZipSize)+1) + written, err := io.Copy(tmpFile, src) + if err != nil { + cleanup() + return "", fmt.Errorf("download write failed: %w", err) + } + + if written > int64(c.maxZipSize) { + cleanup() + return "", fmt.Errorf("download too large: %d bytes (max %d)", written, c.maxZipSize) + } + + if err := tmpFile.Close(); err != nil { + _ = os.Remove(tmpPath) + return "", fmt.Errorf("failed to close temp file: %w", err) + } + + return tmpPath, nil +} + +func isRetryableStatus(statusCode int) bool { + return statusCode == http.StatusTooManyRequests || statusCode >= http.StatusInternalServerError +} + +func retryDelay(retryAfter string, attempt int) time.Duration { + if d, ok := parseRetryAfter(retryAfter); ok { + return d + } + return time.Duration(attempt+1) * time.Second +} + +func parseRetryAfter(headerValue string) (time.Duration, bool) { + headerValue = strings.TrimSpace(headerValue) + if headerValue == "" { + return 0, false + } + + if sec, err := strconv.Atoi(headerValue); err == nil { + if sec < 0 { + sec = 0 + } + return time.Duration(sec) * time.Second, true + } + + if resetAt, err := http.ParseTime(headerValue); err == nil { + d := time.Until(resetAt) + if d < 0 { + d = 0 + } + return d, true + } + + return 0, false +} + +func sleepWithContext(ctx context.Context, delay time.Duration) error { + if delay <= 0 { + return nil + } + + timer := time.NewTimer(delay) + defer timer.Stop() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/pkg/skills/clawhub_registry_test.go b/pkg/skills/clawhub_registry_test.go index 65ee638da..055da22dc 100644 --- a/pkg/skills/clawhub_registry_test.go +++ b/pkg/skills/clawhub_registry_test.go @@ -54,6 +54,39 @@ func TestClawHubRegistrySearch(t *testing.T) { assert.Equal(t, "clawhub", results[0].RegistryName) } +func TestClawHubRegistrySearchRetries429(t *testing.T) { + attempts := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts++ + if attempts == 1 { + w.Header().Set("Retry-After", "0") + w.WriteHeader(http.StatusTooManyRequests) + w.Write([]byte("rate limited")) + return + } + + slug := "github" + name := "GitHub Integration" + summary := "Interact with GitHub repos" + version := "1.0.0" + + json.NewEncoder(w).Encode(clawhubSearchResponse{ + Results: []clawhubSearchResult{ + {Score: 0.95, Slug: &slug, DisplayName: &name, Summary: &summary, Version: &version}, + }, + }) + })) + defer srv.Close() + + reg := newTestRegistry(srv.URL, "") + results, err := reg.Search(context.Background(), "github", 5) + + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, 2, attempts) + assert.Equal(t, "github", results[0].Slug) +} + func TestClawHubRegistryGetSkillMeta(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, "/api/v1/skills/github", r.URL.Path) @@ -137,6 +170,54 @@ func TestClawHubRegistryDownloadAndInstall(t *testing.T) { assert.Contains(t, string(readmeContent), "# Test Skill") } +func TestClawHubRegistryDownloadAndInstallRetries429(t *testing.T) { + zipBuf := createTestZip(t, map[string]string{ + "SKILL.md": "---\nname: retry-skill\ndescription: A test\n---\nHello skill", + }) + + downloadAttempts := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/skills/retry-skill": + json.NewEncoder(w).Encode(clawhubSkillResponse{ + Slug: "retry-skill", + DisplayName: "Retry Skill", + Summary: "A retry test skill", + LatestVersion: &clawhubVersionInfo{Version: "1.0.0"}, + }) + case "/api/v1/download": + downloadAttempts++ + if downloadAttempts == 1 { + w.Header().Set("Retry-After", "0") + w.WriteHeader(http.StatusTooManyRequests) + w.Write([]byte("rate limited")) + return + } + assert.Equal(t, "retry-skill", r.URL.Query().Get("slug")) + w.Header().Set("Content-Type", "application/zip") + w.Write(zipBuf) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer srv.Close() + + tmpDir := t.TempDir() + targetDir := filepath.Join(tmpDir, "retry-skill") + + reg := newTestRegistry(srv.URL, "") + result, err := reg.DownloadAndInstall(context.Background(), "retry-skill", "", targetDir) + + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, "1.0.0", result.Version) + assert.Equal(t, 2, downloadAttempts) + + skillContent, err := os.ReadFile(filepath.Join(targetDir, "SKILL.md")) + require.NoError(t, err) + assert.Contains(t, string(skillContent), "Hello skill") +} + func TestClawHubRegistryAuthToken(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { authHeader := r.Header.Get("Authorization") From ab120af64906537409c297c9c56d1bf80492926e Mon Sep 17 00:00:00 2001 From: cornjosh Date: Thu, 5 Mar 2026 17:10:04 +0800 Subject: [PATCH 7/9] fix(skills): use --registry flag value as registry name The --registry flag value was previously ignored and only used as a switch. Now the flag value is properly used as the registry name. Fixes #1104 Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> --- cmd/picoclaw/internal/skills/install.go | 6 +- cmd/picoclaw/internal/skills/install_test.go | 69 ++++++++++++++++++++ 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/cmd/picoclaw/internal/skills/install.go b/cmd/picoclaw/internal/skills/install.go index a30f68632..78bc421db 100644 --- a/cmd/picoclaw/internal/skills/install.go +++ b/cmd/picoclaw/internal/skills/install.go @@ -21,8 +21,8 @@ picoclaw skills install --registry clawhub github `, Args: func(cmd *cobra.Command, args []string) error { if registry != "" { - if len(args) != 2 { - return fmt.Errorf("when --registry is set, exactly 2 arguments are required: ") + if len(args) != 1 { + return fmt.Errorf("when --registry is set, exactly 1 argument is required: ") } return nil } @@ -45,7 +45,7 @@ picoclaw skills install --registry clawhub github return err } - return skillsInstallFromRegistry(cfg, args[0], args[1]) + return skillsInstallFromRegistry(cfg, registry, args[0]) } return skillsInstallCmd(installer, args[0]) diff --git a/cmd/picoclaw/internal/skills/install_test.go b/cmd/picoclaw/internal/skills/install_test.go index 97787a986..6b362822d 100644 --- a/cmd/picoclaw/internal/skills/install_test.go +++ b/cmd/picoclaw/internal/skills/install_test.go @@ -26,3 +26,72 @@ func TestNewInstallSubcommand(t *testing.T) { assert.Len(t, cmd.Aliases, 0) } + +func TestInstallCommandArgs(t *testing.T) { + tests := []struct { + name string + args []string + registry string + expectError bool + errorMsg string + }{ + { + name: "no registry, one arg", + args: []string{"sipeed/picoclaw-skills/weather"}, + registry: "", + expectError: false, + }, + { + name: "no registry, no args", + args: []string{}, + registry: "", + expectError: true, + errorMsg: "exactly 1 argument is required: ", + }, + { + name: "no registry, too many args", + args: []string{"arg1", "arg2"}, + registry: "", + expectError: true, + errorMsg: "exactly 1 argument is required: ", + }, + { + name: "with registry, one arg", + args: []string{"weather-skill"}, + registry: "clawhub", + expectError: false, + }, + { + name: "with registry, no args", + args: []string{}, + registry: "clawhub", + expectError: true, + errorMsg: "when --registry is set, exactly 1 argument is required: ", + }, + { + name: "with registry, too many args", + args: []string{"arg1", "arg2"}, + registry: "clawhub", + expectError: true, + errorMsg: "when --registry is set, exactly 1 argument is required: ", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cmd := newInstallCommand(nil) + + if tt.registry != "" { + require.NoError(t, cmd.Flags().Set("registry", tt.registry)) + } + + err := cmd.Args(cmd, tt.args) + if tt.expectError { + require.Error(t, err) + assert.Equal(t, tt.errorMsg, err.Error()) + } else { + require.NoError(t, err) + } + }) + } +} From 536e9ac9de6aadf97ea23fbe21f7c6126589a625 Mon Sep 17 00:00:00 2001 From: qs3c <2749950753@qq.com> Date: Thu, 5 Mar 2026 19:10:36 +0800 Subject: [PATCH 8/9] refactor(skills): reuse shared HTTP retry helper --- pkg/skills/clawhub_registry.go | 116 ++++++--------------------------- 1 file changed, 21 insertions(+), 95 deletions(-) diff --git a/pkg/skills/clawhub_registry.go b/pkg/skills/clawhub_registry.go index b520f3260..bd4bed8fb 100644 --- a/pkg/skills/clawhub_registry.go +++ b/pkg/skills/clawhub_registry.go @@ -8,8 +8,6 @@ import ( "net/http" "net/url" "os" - "strconv" - "strings" "time" "github.com/sipeed/picoclaw/pkg/utils" @@ -19,7 +17,6 @@ const ( defaultClawHubTimeout = 30 * time.Second defaultMaxZipSize = 50 * 1024 * 1024 // 50 MB defaultMaxResponseSize = 2 * 1024 * 1024 // 2 MB - defaultMaxRetries = 3 ) // ClawHubRegistry implements SkillRegistry for the ClawHub platform. @@ -279,7 +276,12 @@ func (c *ClawHubRegistry) DownloadAndInstall( // --- HTTP helper --- func (c *ClawHubRegistry) doGet(ctx context.Context, urlStr string) ([]byte, error) { - resp, err := c.doGetWithRetry(ctx, urlStr, "application/json") + req, err := c.newGetRequest(ctx, urlStr, "application/json") + if err != nil { + return nil, err + } + + resp, err := utils.DoRequestWithRetry(c.client, req) if err != nil { return nil, err } @@ -298,50 +300,25 @@ func (c *ClawHubRegistry) doGet(ctx context.Context, urlStr string) ([]byte, err return body, nil } -func (c *ClawHubRegistry) doGetWithRetry(ctx context.Context, urlStr, accept string) (*http.Response, error) { - var lastErr error - for attempt := 0; attempt < defaultMaxRetries; attempt++ { - req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) - if err != nil { - return nil, err - } - req.Header.Set("Accept", accept) - if c.authToken != "" { - req.Header.Set("Authorization", "Bearer "+c.authToken) - } - - resp, err := c.client.Do(req) - if err != nil { - lastErr = err - } else { - if resp.StatusCode >= 200 && resp.StatusCode < 300 { - return resp, nil - } - - if !isRetryableStatus(resp.StatusCode) || attempt == defaultMaxRetries-1 { - return resp, nil - } - - delay := retryDelay(resp.Header.Get("Retry-After"), attempt) - resp.Body.Close() - if err := sleepWithContext(ctx, delay); err != nil { - return nil, err - } - continue - } - - if attempt == defaultMaxRetries-1 { - return nil, lastErr - } - if err := sleepWithContext(ctx, retryDelay("", attempt)); err != nil { - return nil, err - } +func (c *ClawHubRegistry) newGetRequest(ctx context.Context, urlStr, accept string) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil) + if err != nil { + return nil, err } - return nil, lastErr + req.Header.Set("Accept", accept) + if c.authToken != "" { + req.Header.Set("Authorization", "Bearer "+c.authToken) + } + return req, nil } func (c *ClawHubRegistry) downloadToTempFileWithRetry(ctx context.Context, urlStr string) (string, error) { - resp, err := c.doGetWithRetry(ctx, urlStr, "application/zip") + req, err := c.newGetRequest(ctx, urlStr, "application/zip") + if err != nil { + return "", err + } + + resp, err := utils.DoRequestWithRetry(c.client, req) if err != nil { return "", err } @@ -383,54 +360,3 @@ func (c *ClawHubRegistry) downloadToTempFileWithRetry(ctx context.Context, urlSt return tmpPath, nil } - -func isRetryableStatus(statusCode int) bool { - return statusCode == http.StatusTooManyRequests || statusCode >= http.StatusInternalServerError -} - -func retryDelay(retryAfter string, attempt int) time.Duration { - if d, ok := parseRetryAfter(retryAfter); ok { - return d - } - return time.Duration(attempt+1) * time.Second -} - -func parseRetryAfter(headerValue string) (time.Duration, bool) { - headerValue = strings.TrimSpace(headerValue) - if headerValue == "" { - return 0, false - } - - if sec, err := strconv.Atoi(headerValue); err == nil { - if sec < 0 { - sec = 0 - } - return time.Duration(sec) * time.Second, true - } - - if resetAt, err := http.ParseTime(headerValue); err == nil { - d := time.Until(resetAt) - if d < 0 { - d = 0 - } - return d, true - } - - return 0, false -} - -func sleepWithContext(ctx context.Context, delay time.Duration) error { - if delay <= 0 { - return nil - } - - timer := time.NewTimer(delay) - defer timer.Stop() - - select { - case <-ctx.Done(): - return ctx.Err() - case <-timer.C: - return nil - } -} From f046ba59e80e6f901067513913490af83affa117 Mon Sep 17 00:00:00 2001 From: esubaalew Date: Thu, 5 Mar 2026 15:40:06 +0300 Subject: [PATCH 9/9] fix(agent): respect global skills toggle for skill tools --- pkg/agent/loop.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 1ab79f3ca..fa001a9c9 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -168,9 +168,10 @@ func registerSharedTools( } // Skill discovery and installation tools + skills_enabled := cfg.Tools.IsToolEnabled("skills") find_skills_enable := cfg.Tools.IsToolEnabled("find_skills") install_skills_enable := cfg.Tools.IsToolEnabled("install_skill") - if find_skills_enable || install_skills_enable { + if skills_enabled && (find_skills_enable || install_skills_enable) { registryMgr := skills.NewRegistryManagerFromConfig(skills.RegistryConfig{ MaxConcurrentSearches: cfg.Tools.Skills.MaxConcurrentSearches, ClawHub: skills.ClawHubConfig(cfg.Tools.Skills.Registries.ClawHub),