Merge branch 'main' into feat/mutil_keys
This commit is contained in:
commit
8ac3294872
11 changed files with 427 additions and 32 deletions
119
README.md
119
README.md
|
|
@ -216,7 +216,7 @@ docker compose -f docker/docker-compose.yml --profile gateway up -d
|
|||
> [!TIP]
|
||||
> Set your API key in `~/.picoclaw/config.json`.
|
||||
> Get API keys: [OpenRouter](https://openrouter.ai/keys) (LLM) · [Zhipu](https://open.bigmodel.cn/usercenter/proj-mgmt/apikeys) (LLM)
|
||||
> Web Search is **optional** - get free [Tavily API](https://tavily.com) (1000 free queries/month) or [Brave Search API](https://brave.com/search/api) (2000 free queries/month) or use built-in auto fallback.
|
||||
> Web Search is **optional** - get free [Tavily API](https://tavily.com) (1000 free queries/month), [SearXNG](https://github.com/searxng/searxng) (free, self-hosted) or [Brave Search API](https://brave.com/search/api) (2000 free queries/month) or use built-in auto fallback.
|
||||
|
||||
**1. Initialize**
|
||||
|
||||
|
|
@ -265,6 +265,16 @@ picoclaw onboard
|
|||
"duckduckgo": {
|
||||
"enabled": true,
|
||||
"max_results": 5
|
||||
},
|
||||
"perplexity": {
|
||||
"enabled": false,
|
||||
"api_key": "YOUR_PERPLEXITY_API_KEY",
|
||||
"max_results": 5
|
||||
},
|
||||
"searxng": {
|
||||
"enabled": false,
|
||||
"base_url": "http://your-searxng-instance:8888",
|
||||
"max_results": 5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -277,7 +287,12 @@ picoclaw onboard
|
|||
**3. Get API Keys**
|
||||
|
||||
* **LLM Provider**: [OpenRouter](https://openrouter.ai/keys) · [Zhipu](https://open.bigmodel.cn/usercenter/proj-mgmt/apikeys) · [Anthropic](https://console.anthropic.com) · [OpenAI](https://platform.openai.com) · [Gemini](https://aistudio.google.com/api-keys)
|
||||
* **Web Search** (optional): [Tavily](https://tavily.com) - Optimized for AI Agents (1000 requests/month) · [Brave Search](https://brave.com/search/api) - Free tier available (2000 requests/month)
|
||||
* **Web Search** (optional):
|
||||
* [Brave Search](https://brave.com/search/api) - Paid ($5/1000 queries, ~$5-6/month)
|
||||
* [Perplexity](https://www.perplexity.ai) - AI-powered search with chat interface
|
||||
* [SearXNG](https://github.com/searxng/searxng) - Self-hosted metasearch engine (free, no API key needed)
|
||||
* [Tavily](https://tavily.com) - Optimized for AI Agents (1000 requests/month)
|
||||
* DuckDuckGo - Built-in fallback (no API key required)
|
||||
|
||||
> **Note**: See `config.example.json` for a complete configuration template.
|
||||
|
||||
|
|
@ -1241,6 +1256,16 @@ picoclaw agent -m "Hello"
|
|||
"duckduckgo": {
|
||||
"enabled": true,
|
||||
"max_results": 5
|
||||
},
|
||||
"perplexity": {
|
||||
"enabled": false,
|
||||
"api_key": "",
|
||||
"max_results": 5
|
||||
},
|
||||
"searxng": {
|
||||
"enabled": false,
|
||||
"base_url": "http://localhost:8888",
|
||||
"max_results": 5
|
||||
}
|
||||
},
|
||||
"cron": {
|
||||
|
|
@ -1298,10 +1323,69 @@ discord: <https://discord.gg/V4sAZ9XWpN>
|
|||
|
||||
This is normal if you haven't configured a search API key yet. PicoClaw will provide helpful links for manual searching.
|
||||
|
||||
To enable web search:
|
||||
#### Search Provider Priority
|
||||
|
||||
1. **Option 1 (Recommended)**: Get a free API key at [https://brave.com/search/api](https://brave.com/search/api) (2000 free queries/month) for the best results.
|
||||
2. **Option 2 (No Credit Card)**: If you don't have a key, we automatically fall back to **DuckDuckGo** (no key required).
|
||||
PicoClaw automatically selects the best available search provider in this order:
|
||||
1. **Perplexity** (if enabled and API key configured) - AI-powered search with citations
|
||||
2. **Brave Search** (if enabled and API key configured) - Privacy-focused paid API ($5/1000 queries)
|
||||
3. **SearXNG** (if enabled and base_url configured) - Self-hosted metasearch aggregating 70+ engines (free)
|
||||
4. **DuckDuckGo** (if enabled, default fallback) - No API key required (free)
|
||||
|
||||
#### Web Search Configuration Options
|
||||
|
||||
**Option 1 (Best Results)**: Perplexity AI Search
|
||||
```json
|
||||
{
|
||||
"tools": {
|
||||
"web": {
|
||||
"perplexity": {
|
||||
"enabled": true,
|
||||
"api_key": "YOUR_PERPLEXITY_API_KEY",
|
||||
"max_results": 5
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Option 2 (Paid API)**: Get an API key at [https://brave.com/search/api](https://brave.com/search/api) ($5/1000 queries, ~$5-6/month)
|
||||
```json
|
||||
{
|
||||
"tools": {
|
||||
"web": {
|
||||
"brave": {
|
||||
"enabled": true,
|
||||
"api_key": "YOUR_BRAVE_API_KEY",
|
||||
"max_results": 5
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Option 3 (Self-Hosted)**: Deploy your own [SearXNG](https://github.com/searxng/searxng) instance
|
||||
```json
|
||||
{
|
||||
"tools": {
|
||||
"web": {
|
||||
"searxng": {
|
||||
"enabled": true,
|
||||
"base_url": "http://your-server:8888",
|
||||
"max_results": 5
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Benefits of SearXNG:
|
||||
- **Zero cost**: No API fees or rate limits
|
||||
- **Privacy-focused**: Self-hosted, no tracking
|
||||
- **Aggregate results**: Queries 70+ search engines simultaneously
|
||||
- **Perfect for cloud VMs**: Solves datacenter IP blocking issues (Oracle Cloud, GCP, AWS, Azure)
|
||||
- **No API key needed**: Just deploy and configure the base URL
|
||||
|
||||
**Option 4 (No Setup Required)**: DuckDuckGo is enabled by default as fallback (no API key needed)
|
||||
|
||||
Add the key to `~/.picoclaw/config.json` if using Brave:
|
||||
|
||||
|
|
@ -1317,6 +1401,16 @@ Add the key to `~/.picoclaw/config.json` if using Brave:
|
|||
"duckduckgo": {
|
||||
"enabled": true,
|
||||
"max_results": 5
|
||||
},
|
||||
"perplexity": {
|
||||
"enabled": false,
|
||||
"api_key": "YOUR_PERPLEXITY_API_KEY",
|
||||
"max_results": 5
|
||||
},
|
||||
"searxng": {
|
||||
"enabled": false,
|
||||
"base_url": "http://your-searxng-instance:8888",
|
||||
"max_results": 5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1335,10 +1429,11 @@ This happens when another instance of the bot is running. Make sure only one `pi
|
|||
|
||||
## 📝 API Key Comparison
|
||||
|
||||
| Service | Free Tier | Use Case |
|
||||
| ---------------- | ------------------- | ------------------------------------- |
|
||||
| **OpenRouter** | 200K tokens/month | Multiple models (Claude, GPT-4, etc.) |
|
||||
| **Zhipu** | 200K tokens/month | Best for Chinese users |
|
||||
| **Brave Search** | 2000 queries/month | Web search functionality |
|
||||
| **Groq** | Free tier available | Fast inference (Llama, Mixtral) |
|
||||
| **Cerebras** | Free tier available | Fast inference (Llama, Qwen, etc.) |
|
||||
| Service | Free Tier | Use Case |
|
||||
| ---------------- | ------------------------ | ------------------------------------- |
|
||||
| **OpenRouter** | 200K tokens/month | Multiple models (Claude, GPT-4, etc.) |
|
||||
| **Zhipu** | 200K tokens/month | Best for Chinese users |
|
||||
| **Brave Search** | Paid ($5/1000 queries) | Web search functionality |
|
||||
| **SearXNG** | Unlimited (self-hosted) | Privacy-focused metasearch (70+ engines) |
|
||||
| **Groq** | Free tier available | Fast inference (Llama, Mixtral) |
|
||||
| **Cerebras** | Free tier available | Fast inference (Llama, Qwen, etc.) |
|
||||
|
|
|
|||
|
|
@ -21,8 +21,8 @@ picoclaw skills install --registry clawhub github
|
|||
`,
|
||||
Args: func(cmd *cobra.Command, args []string) error {
|
||||
if registry != "" {
|
||||
if len(args) != 2 {
|
||||
return fmt.Errorf("when --registry is set, exactly 2 arguments are required: <name> <slug>")
|
||||
if len(args) != 1 {
|
||||
return fmt.Errorf("when --registry is set, exactly 1 argument is required: <slug>")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -45,7 +45,7 @@ picoclaw skills install --registry clawhub github
|
|||
return err
|
||||
}
|
||||
|
||||
return skillsInstallFromRegistry(cfg, args[0], args[1])
|
||||
return skillsInstallFromRegistry(cfg, registry, args[0])
|
||||
}
|
||||
|
||||
return skillsInstallCmd(installer, args[0])
|
||||
|
|
|
|||
|
|
@ -26,3 +26,72 @@ func TestNewInstallSubcommand(t *testing.T) {
|
|||
|
||||
assert.Len(t, cmd.Aliases, 0)
|
||||
}
|
||||
|
||||
func TestInstallCommandArgs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
registry string
|
||||
expectError bool
|
||||
errorMsg string
|
||||
}{
|
||||
{
|
||||
name: "no registry, one arg",
|
||||
args: []string{"sipeed/picoclaw-skills/weather"},
|
||||
registry: "",
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "no registry, no args",
|
||||
args: []string{},
|
||||
registry: "",
|
||||
expectError: true,
|
||||
errorMsg: "exactly 1 argument is required: <github>",
|
||||
},
|
||||
{
|
||||
name: "no registry, too many args",
|
||||
args: []string{"arg1", "arg2"},
|
||||
registry: "",
|
||||
expectError: true,
|
||||
errorMsg: "exactly 1 argument is required: <github>",
|
||||
},
|
||||
{
|
||||
name: "with registry, one arg",
|
||||
args: []string{"weather-skill"},
|
||||
registry: "clawhub",
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "with registry, no args",
|
||||
args: []string{},
|
||||
registry: "clawhub",
|
||||
expectError: true,
|
||||
errorMsg: "when --registry is set, exactly 1 argument is required: <slug>",
|
||||
},
|
||||
{
|
||||
name: "with registry, too many args",
|
||||
args: []string{"arg1", "arg2"},
|
||||
registry: "clawhub",
|
||||
expectError: true,
|
||||
errorMsg: "when --registry is set, exactly 1 argument is required: <slug>",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cmd := newInstallCommand(nil)
|
||||
|
||||
if tt.registry != "" {
|
||||
require.NoError(t, cmd.Flags().Set("registry", tt.registry))
|
||||
}
|
||||
|
||||
err := cmd.Args(cmd, tt.args)
|
||||
if tt.expectError {
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, tt.errorMsg, err.Error())
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -262,6 +262,11 @@
|
|||
],
|
||||
"max_results": 5
|
||||
},
|
||||
"searxng": {
|
||||
"enabled": false,
|
||||
"base_url": "http://localhost:8888",
|
||||
"max_results": 5
|
||||
},
|
||||
"glm_search": {
|
||||
"enabled": false,
|
||||
"api_key": "",
|
||||
|
|
|
|||
|
|
@ -180,6 +180,7 @@ The skills tool configures skill discovery and installation via registries like
|
|||
| ---------------------------------- | ------ | -------------------- | ----------------------- |
|
||||
| `registries.clawhub.enabled` | bool | true | Enable ClawHub registry |
|
||||
| `registries.clawhub.base_url` | string | `https://clawhub.ai` | ClawHub base URL |
|
||||
| `registries.clawhub.auth_token` | string | `""` | Optional Bearer token for higher rate limits |
|
||||
| `registries.clawhub.search_path` | string | `/api/v1/search` | Search API path |
|
||||
| `registries.clawhub.skills_path` | string | `/api/v1/skills` | Skills API path |
|
||||
| `registries.clawhub.download_path` | string | `/api/v1/download` | Download API path |
|
||||
|
|
@ -194,6 +195,7 @@ The skills tool configures skill discovery and installation via registries like
|
|||
"clawhub": {
|
||||
"enabled": true,
|
||||
"base_url": "https://clawhub.ai",
|
||||
"auth_token": "",
|
||||
"search_path": "/api/v1/search",
|
||||
"skills_path": "/api/v1/skills",
|
||||
"download_path": "/api/v1/download"
|
||||
|
|
|
|||
|
|
@ -124,6 +124,9 @@ func registerSharedTools(
|
|||
),
|
||||
PerplexityMaxResults: cfg.Tools.Web.Perplexity.MaxResults,
|
||||
PerplexityEnabled: cfg.Tools.Web.Perplexity.Enabled,
|
||||
SearXNGBaseURL: cfg.Tools.Web.SearXNG.BaseURL,
|
||||
SearXNGMaxResults: cfg.Tools.Web.SearXNG.MaxResults,
|
||||
SearXNGEnabled: cfg.Tools.Web.SearXNG.Enabled,
|
||||
GLMSearchAPIKey: cfg.Tools.Web.GLMSearch.APIKey,
|
||||
GLMSearchBaseURL: cfg.Tools.Web.GLMSearch.BaseURL,
|
||||
GLMSearchEngine: cfg.Tools.Web.GLMSearch.SearchEngine,
|
||||
|
|
@ -170,9 +173,10 @@ func registerSharedTools(
|
|||
}
|
||||
|
||||
// Skill discovery and installation tools
|
||||
skills_enabled := cfg.Tools.IsToolEnabled("skills")
|
||||
find_skills_enable := cfg.Tools.IsToolEnabled("find_skills")
|
||||
install_skills_enable := cfg.Tools.IsToolEnabled("install_skill")
|
||||
if find_skills_enable || install_skills_enable {
|
||||
if skills_enabled && (find_skills_enable || install_skills_enable) {
|
||||
registryMgr := skills.NewRegistryManagerFromConfig(skills.RegistryConfig{
|
||||
MaxConcurrentSearches: cfg.Tools.Skills.MaxConcurrentSearches,
|
||||
ClawHub: skills.ClawHubConfig(cfg.Tools.Skills.Registries.ClawHub),
|
||||
|
|
|
|||
|
|
@ -558,6 +558,12 @@ type PerplexityConfig struct {
|
|||
MaxResults int `json:"max_results" env:"PICOCLAW_TOOLS_WEB_PERPLEXITY_MAX_RESULTS"`
|
||||
}
|
||||
|
||||
type SearXNGConfig struct {
|
||||
Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_WEB_SEARXNG_ENABLED"`
|
||||
BaseURL string `json:"base_url" env:"PICOCLAW_TOOLS_WEB_SEARXNG_BASE_URL"`
|
||||
MaxResults int `json:"max_results" env:"PICOCLAW_TOOLS_WEB_SEARXNG_MAX_RESULTS"`
|
||||
}
|
||||
|
||||
type GLMSearchConfig struct {
|
||||
Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_WEB_GLM_ENABLED"`
|
||||
APIKey string `json:"api_key" env:"PICOCLAW_TOOLS_WEB_GLM_API_KEY"`
|
||||
|
|
@ -574,6 +580,7 @@ type WebToolsConfig struct {
|
|||
Tavily TavilyConfig ` json:"tavily"`
|
||||
DuckDuckGo DuckDuckGoConfig ` json:"duckduckgo"`
|
||||
Perplexity PerplexityConfig ` json:"perplexity"`
|
||||
SearXNG SearXNGConfig ` json:"searxng"`
|
||||
GLMSearch GLMSearchConfig ` json:"glm_search"`
|
||||
// Proxy is an optional proxy URL for web tools (http/https/socks5/socks5h).
|
||||
// For authenticated proxies, prefer HTTP_PROXY/HTTPS_PROXY env vars instead of embedding credentials in config.
|
||||
|
|
|
|||
|
|
@ -370,6 +370,11 @@ func DefaultConfig() *Config {
|
|||
APIKeys: nil,
|
||||
MaxResults: 5,
|
||||
},
|
||||
SearXNG: SearXNGConfig{
|
||||
Enabled: false,
|
||||
BaseURL: "",
|
||||
MaxResults: 5,
|
||||
},
|
||||
GLMSearch: GLMSearchConfig{
|
||||
Enabled: false,
|
||||
APIKey: "",
|
||||
|
|
|
|||
|
|
@ -259,15 +259,7 @@ func (c *ClawHubRegistry) DownloadAndInstall(
|
|||
}
|
||||
u.RawQuery = q.Encode()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
if c.authToken != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.authToken)
|
||||
}
|
||||
|
||||
tmpPath, err := utils.DownloadToFile(ctx, c.client, req, int64(c.maxZipSize))
|
||||
tmpPath, err := c.downloadToTempFileWithRetry(ctx, u.String())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("download failed: %w", err)
|
||||
}
|
||||
|
|
@ -284,17 +276,12 @@ func (c *ClawHubRegistry) DownloadAndInstall(
|
|||
// --- HTTP helper ---
|
||||
|
||||
func (c *ClawHubRegistry) doGet(ctx context.Context, urlStr string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
|
||||
req, err := c.newGetRequest(ctx, urlStr, "application/json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Accept", "application/json")
|
||||
if c.authToken != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.authToken)
|
||||
}
|
||||
|
||||
resp, err := c.client.Do(req)
|
||||
resp, err := utils.DoRequestWithRetry(c.client, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -312,3 +299,64 @@ func (c *ClawHubRegistry) doGet(ctx context.Context, urlStr string) ([]byte, err
|
|||
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (c *ClawHubRegistry) newGetRequest(ctx context.Context, urlStr, accept string) (*http.Request, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Accept", accept)
|
||||
if c.authToken != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.authToken)
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
func (c *ClawHubRegistry) downloadToTempFileWithRetry(ctx context.Context, urlStr string) (string, error) {
|
||||
req, err := c.newGetRequest(ctx, urlStr, "application/zip")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
resp, err := utils.DoRequestWithRetry(c.client, req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
errBody := make([]byte, 512)
|
||||
n, _ := io.ReadFull(resp.Body, errBody)
|
||||
return "", fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(errBody[:n]))
|
||||
}
|
||||
|
||||
tmpFile, err := os.CreateTemp("", "picoclaw-dl-*")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create temp file: %w", err)
|
||||
}
|
||||
tmpPath := tmpFile.Name()
|
||||
|
||||
cleanup := func() {
|
||||
_ = tmpFile.Close()
|
||||
_ = os.Remove(tmpPath)
|
||||
}
|
||||
|
||||
src := io.LimitReader(resp.Body, int64(c.maxZipSize)+1)
|
||||
written, err := io.Copy(tmpFile, src)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return "", fmt.Errorf("download write failed: %w", err)
|
||||
}
|
||||
|
||||
if written > int64(c.maxZipSize) {
|
||||
cleanup()
|
||||
return "", fmt.Errorf("download too large: %d bytes (max %d)", written, c.maxZipSize)
|
||||
}
|
||||
|
||||
if err := tmpFile.Close(); err != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
return "", fmt.Errorf("failed to close temp file: %w", err)
|
||||
}
|
||||
|
||||
return tmpPath, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -54,6 +54,39 @@ func TestClawHubRegistrySearch(t *testing.T) {
|
|||
assert.Equal(t, "clawhub", results[0].RegistryName)
|
||||
}
|
||||
|
||||
func TestClawHubRegistrySearchRetries429(t *testing.T) {
|
||||
attempts := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
w.Header().Set("Retry-After", "0")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
w.Write([]byte("rate limited"))
|
||||
return
|
||||
}
|
||||
|
||||
slug := "github"
|
||||
name := "GitHub Integration"
|
||||
summary := "Interact with GitHub repos"
|
||||
version := "1.0.0"
|
||||
|
||||
json.NewEncoder(w).Encode(clawhubSearchResponse{
|
||||
Results: []clawhubSearchResult{
|
||||
{Score: 0.95, Slug: &slug, DisplayName: &name, Summary: &summary, Version: &version},
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
reg := newTestRegistry(srv.URL, "")
|
||||
results, err := reg.Search(context.Background(), "github", 5)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, results, 1)
|
||||
assert.Equal(t, 2, attempts)
|
||||
assert.Equal(t, "github", results[0].Slug)
|
||||
}
|
||||
|
||||
func TestClawHubRegistryGetSkillMeta(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "/api/v1/skills/github", r.URL.Path)
|
||||
|
|
@ -137,6 +170,54 @@ func TestClawHubRegistryDownloadAndInstall(t *testing.T) {
|
|||
assert.Contains(t, string(readmeContent), "# Test Skill")
|
||||
}
|
||||
|
||||
func TestClawHubRegistryDownloadAndInstallRetries429(t *testing.T) {
|
||||
zipBuf := createTestZip(t, map[string]string{
|
||||
"SKILL.md": "---\nname: retry-skill\ndescription: A test\n---\nHello skill",
|
||||
})
|
||||
|
||||
downloadAttempts := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/v1/skills/retry-skill":
|
||||
json.NewEncoder(w).Encode(clawhubSkillResponse{
|
||||
Slug: "retry-skill",
|
||||
DisplayName: "Retry Skill",
|
||||
Summary: "A retry test skill",
|
||||
LatestVersion: &clawhubVersionInfo{Version: "1.0.0"},
|
||||
})
|
||||
case "/api/v1/download":
|
||||
downloadAttempts++
|
||||
if downloadAttempts == 1 {
|
||||
w.Header().Set("Retry-After", "0")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
w.Write([]byte("rate limited"))
|
||||
return
|
||||
}
|
||||
assert.Equal(t, "retry-skill", r.URL.Query().Get("slug"))
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
w.Write(zipBuf)
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
targetDir := filepath.Join(tmpDir, "retry-skill")
|
||||
|
||||
reg := newTestRegistry(srv.URL, "")
|
||||
result, err := reg.DownloadAndInstall(context.Background(), "retry-skill", "", targetDir)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
assert.Equal(t, "1.0.0", result.Version)
|
||||
assert.Equal(t, 2, downloadAttempts)
|
||||
|
||||
skillContent, err := os.ReadFile(filepath.Join(targetDir, "SKILL.md"))
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, string(skillContent), "Hello skill")
|
||||
}
|
||||
|
||||
func TestClawHubRegistryAuthToken(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
authHeader := r.Header.Get("Authorization")
|
||||
|
|
|
|||
|
|
@ -499,6 +499,68 @@ func (p *PerplexitySearchProvider) Search(ctx context.Context, query string, cou
|
|||
return "", fmt.Errorf("all api keys failed, last error: %w", lastErr)
|
||||
}
|
||||
|
||||
type SearXNGSearchProvider struct {
|
||||
baseURL string
|
||||
}
|
||||
|
||||
func (p *SearXNGSearchProvider) Search(ctx context.Context, query string, count int) (string, error) {
|
||||
searchURL := fmt.Sprintf("%s/search?q=%s&format=json&categories=general",
|
||||
strings.TrimSuffix(p.baseURL, "/"),
|
||||
url.QueryEscape(query))
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("SearXNG returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Results []struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Content string `json:"content"`
|
||||
Engine string `json:"engine"`
|
||||
Score float64 `json:"score"`
|
||||
} `json:"results"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return "", fmt.Errorf("failed to parse response: %w", err)
|
||||
}
|
||||
|
||||
if len(result.Results) == 0 {
|
||||
return fmt.Sprintf("No results for: %s", query), nil
|
||||
}
|
||||
|
||||
// Limit results to requested count
|
||||
if len(result.Results) > count {
|
||||
result.Results = result.Results[:count]
|
||||
}
|
||||
|
||||
// Format results in standard PicoClaw format
|
||||
var b strings.Builder
|
||||
b.WriteString(fmt.Sprintf("Results for: %s (via SearXNG)\n", query))
|
||||
for i, r := range result.Results {
|
||||
b.WriteString(fmt.Sprintf("%d. %s\n", i+1, r.Title))
|
||||
b.WriteString(fmt.Sprintf(" %s\n", r.URL))
|
||||
if r.Content != "" {
|
||||
b.WriteString(fmt.Sprintf(" %s\n", r.Content))
|
||||
}
|
||||
}
|
||||
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
type GLMSearchProvider struct {
|
||||
apiKey string
|
||||
baseURL string
|
||||
|
|
@ -599,6 +661,9 @@ type WebSearchToolOptions struct {
|
|||
PerplexityAPIKeys []string
|
||||
PerplexityMaxResults int
|
||||
PerplexityEnabled bool
|
||||
SearXNGBaseURL string
|
||||
SearXNGMaxResults int
|
||||
SearXNGEnabled bool
|
||||
GLMSearchAPIKey string
|
||||
GLMSearchBaseURL string
|
||||
GLMSearchEngine string
|
||||
|
|
@ -611,8 +676,13 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
|
|||
var provider SearchProvider
|
||||
maxResults := 5
|
||||
|
||||
<<<<<<< feat/mutil_keys
|
||||
// Priority: Perplexity > Brave > Tavily > DuckDuckGo
|
||||
if opts.PerplexityEnabled && len(opts.PerplexityAPIKeys) > 0 {
|
||||
=======
|
||||
// Priority: Perplexity > Brave > SearXNG > Tavily > DuckDuckGo > GLM Search
|
||||
if opts.PerplexityEnabled && opts.PerplexityAPIKey != "" {
|
||||
>>>>>>> main
|
||||
client, err := createHTTPClient(opts.Proxy, perplexityTimeout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create HTTP client for Perplexity: %w", err)
|
||||
|
|
@ -634,7 +704,16 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
|
|||
if opts.BraveMaxResults > 0 {
|
||||
maxResults = opts.BraveMaxResults
|
||||
}
|
||||
<<<<<<< feat/mutil_keys
|
||||
} else if opts.TavilyEnabled && len(opts.TavilyAPIKeys) > 0 {
|
||||
=======
|
||||
} else if opts.SearXNGEnabled && opts.SearXNGBaseURL != "" {
|
||||
provider = &SearXNGSearchProvider{baseURL: opts.SearXNGBaseURL}
|
||||
if opts.SearXNGMaxResults > 0 {
|
||||
maxResults = opts.SearXNGMaxResults
|
||||
}
|
||||
} else if opts.TavilyEnabled && opts.TavilyAPIKey != "" {
|
||||
>>>>>>> main
|
||||
client, err := createHTTPClient(opts.Proxy, searchTimeout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create HTTP client for Tavily: %w", err)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue