feat(skills): add IndexRegistry with url_mappings for local dev
- Add IndexRegistry that fetches skills from any JSON index URL - Add url_mappings to redirect downloads to local files or forks - Add symlink_local to symlink local directories instead of copying - Add allowed_prefixes for security restrictions
This commit is contained in:
keithy
Keith Patrick
parent
7a360f726e
commit
a5b88dce5e
7 changed files with 920 additions and 10 deletions
|
|
@ -17,6 +17,15 @@ import (
|
|||
|
||||
const skillsSearchMaxResults = 20
|
||||
|
||||
// ConvertConfig converts config.IndexRegistryConfig map to skills.IndexRegistryConfig map
|
||||
func ConvertConfig(c map[string]config.IndexRegistryConfig) map[string]skills.IndexRegistryConfig {
|
||||
result := make(map[string]skills.IndexRegistryConfig)
|
||||
for k, v := range c {
|
||||
result[k] = skills.IndexRegistryConfig(v)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func skillsListCmd(loader *skills.SkillsLoader) {
|
||||
allSkills := loader.ListSkills()
|
||||
|
||||
|
|
@ -67,6 +76,7 @@ func skillsInstallFromRegistry(cfg *config.Config, registryName, slug string) er
|
|||
registryMgr := skills.NewRegistryManagerFromConfig(skills.RegistryConfig{
|
||||
MaxConcurrentSearches: cfg.Tools.Skills.MaxConcurrentSearches,
|
||||
ClawHub: skills.ClawHubConfig(cfg.Tools.Skills.Registries.ClawHub),
|
||||
Index: ConvertConfig(cfg.Tools.Skills.Registries.Index),
|
||||
})
|
||||
|
||||
registry := registryMgr.GetRegistry(registryName)
|
||||
|
|
@ -229,6 +239,7 @@ func skillsSearchCmd(query string) {
|
|||
registryMgr := skills.NewRegistryManagerFromConfig(skills.RegistryConfig{
|
||||
MaxConcurrentSearches: cfg.Tools.Skills.MaxConcurrentSearches,
|
||||
ClawHub: skills.ClawHubConfig(cfg.Tools.Skills.Registries.ClawHub),
|
||||
Index: ConvertConfig(cfg.Tools.Skills.Registries.Index),
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
|
|
|
|||
|
|
@ -373,13 +373,26 @@
|
|||
"clawhub": {
|
||||
"enabled": true,
|
||||
"base_url": "https://clawhub.ai",
|
||||
"auth_token": "",
|
||||
"search_path": "",
|
||||
"skills_path": "",
|
||||
"download_path": "",
|
||||
"timeout": 0,
|
||||
"max_zip_size": 0,
|
||||
"max_response_size": 0
|
||||
"search_path": "/api/v1/search",
|
||||
"skills_path": "/api/v1/skills",
|
||||
"download_path": "/api/v1/download"
|
||||
},
|
||||
"index:angelhub": {
|
||||
"enabled": false,
|
||||
"index_url": "https://raw.githubusercontent.com/wiki/keithy/angelhub/picoclaw-skills-index.json",
|
||||
"extra_header": "X-Custom-Header: value",
|
||||
"authorization_header": "Bearer token",
|
||||
"agent_header": "picoclaw/1.0",
|
||||
"allowed_prefixes": [
|
||||
"https://raw.githubusercontent.com/wiki/keithy/angelhub/",
|
||||
"https://raw.githubusercontent.com/keithy/angelhub/"
|
||||
],
|
||||
"url_mappings": {
|
||||
"https://raw.githubusercontent.com/keithy/angelhub/main/": "https://raw.githubusercontent.com/keithy/angelhub/beta/",
|
||||
"https://raw.githubusercontent.com/keithy/angelhub/": "file:///home/me/repos/angelhub/"
|
||||
},
|
||||
"symlink_local": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"max_concurrent_searches": 2,
|
||||
|
|
|
|||
247
docs/SKILLS_REGISTRIES.md
Normal file
247
docs/SKILLS_REGISTRIES.md
Normal file
|
|
@ -0,0 +1,247 @@
|
|||
# Skills Registries
|
||||
|
||||
PicoClaw supports installing skills from multiple registries. This guide covers how to use and create registries.
|
||||
|
||||
## Using Registries
|
||||
|
||||
### List Available Skills
|
||||
|
||||
```bash
|
||||
picoclaw skills search <query>
|
||||
```
|
||||
|
||||
### Install a Skill
|
||||
|
||||
```bash
|
||||
# Install from a specific registry
|
||||
picoclaw skills install --registry index:angelhub self-config
|
||||
picoclaw skills install --registry clawhub github
|
||||
|
||||
# Install directly from GitHub hosted SKILL.md
|
||||
picoclaw skills install owner/repo/skill-name
|
||||
```
|
||||
|
||||
### List Installed Skills
|
||||
|
||||
```bash
|
||||
picoclaw skills list
|
||||
```
|
||||
|
||||
## Supported Registries
|
||||
|
||||
### ClawHub Registry
|
||||
|
||||
The default registry at [clawhub.ai](https://clawhub.ai). Enable in config:
|
||||
|
||||
```json
|
||||
{
|
||||
"tools": {
|
||||
"skills": {
|
||||
"registries": {
|
||||
"clawhub": {
|
||||
"enabled": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Index Fronted Registry
|
||||
|
||||
Install skills from any HTTP endpoint that serves a skills-index.json.
|
||||
|
||||
**Basic Configuration:**
|
||||
|
||||
```json
|
||||
{
|
||||
"tools": {
|
||||
"skills": {
|
||||
"registries": {
|
||||
"index:myorg": {
|
||||
"enabled": true,
|
||||
"index_url": "https://example.com/skills-index.json"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Configuration with security options:**
|
||||
|
||||
```json
|
||||
{
|
||||
"tools": {
|
||||
"skills": {
|
||||
"registries": {
|
||||
"index:angelhub": {
|
||||
"enabled": true,
|
||||
"index_url": "https://raw.githubusercontent.com/wiki/keithy/angelhub/picoclaw-skills-index.json",
|
||||
"extra_header": "X-Custom-Header: value",
|
||||
"authorization_header": "Bearer token",
|
||||
"agent_header": "picoclaw/1.0",
|
||||
"allowed_prefixes": [
|
||||
"https://raw.githubusercontent.com/wiki/keithy/angelhub/",
|
||||
"https://raw.githubusercontent.com/keithy/angelhub/"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Configuration for local development:**
|
||||
|
||||
Use `url_mappings` to redirect downloads to local files or forks, and `symlink_local` to create symlinks instead of copying:
|
||||
|
||||
```json
|
||||
{
|
||||
"tools": {
|
||||
"skills": {
|
||||
"registries": {
|
||||
"index:angelhub": {
|
||||
"enabled": true,
|
||||
"index_url": "https://raw.githubusercontent.com/wiki/keithy/angelhub/picoclaw-skills-index.json",
|
||||
"allowed_prefixes": [
|
||||
"https://raw.githubusercontent.com/wiki/keithy/angelhub/",
|
||||
"https://raw.githubusercontent.com/keithy/angelhub/"
|
||||
],
|
||||
"url_mappings": {
|
||||
"https://raw.githubusercontent.com/keithy/angelhub/main/": "https://raw.githubusercontent.com/keithy/angelhub/beta/",
|
||||
"https://raw.githubusercontent.com/keithy/angelhub/": "file:///home/me/repos/angelhub/"
|
||||
},
|
||||
"symlink_local": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
| Option | Description |
|
||||
|--------|-------------|
|
||||
| `url_mappings` | Map URL prefixes to redirect downloads. Useful for testing forks (`https://.../main/` → `https://.../beta/`) or local development (`https://...` → `file:///path/to/repo/`) |
|
||||
| `symlink_local` | When using `file://` URLs in mappings, create a symlink to the local directory instead of copying files. Useful for development |
|
||||
|
||||
## Creating Your Own Registry
|
||||
|
||||
To create a skill registry:
|
||||
|
||||
### 1. Create a Repository
|
||||
|
||||
Create a public repository to host your skills.
|
||||
|
||||
### 2. Add Skills
|
||||
|
||||
Add skills in the `picoclaw/skills/` directory (or `skills/` for ecosystem-agnostic). Each skill needs a `SKILL.md` file:
|
||||
|
||||
```
|
||||
picoclaw/
|
||||
└── skills/
|
||||
├── self/
|
||||
│ ├── self-config/
|
||||
│ │ └── SKILL.md
|
||||
│ └── self-debug/
|
||||
│ └── SKILL.md
|
||||
└── weather/
|
||||
└── SKILL.md
|
||||
```
|
||||
|
||||
### 3. Create the Index Workflow
|
||||
|
||||
Add a workflow to generate the skills index. See [AngelHub's workflow](https://github.com/keithy/angelhub/blob/main/.github/workflows/picoclaw-skills-index.yml) for a complete example.
|
||||
|
||||
### 4. Enable in PicoClaw
|
||||
|
||||
```json
|
||||
{
|
||||
"tools": {
|
||||
"skills": {
|
||||
"registries": {
|
||||
"index:myorg": {
|
||||
"enabled": true,
|
||||
"index_url": "https://example.com/skills-index.json"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Skill Format
|
||||
|
||||
Each skill should have a `SKILL.md` file with frontmatter:
|
||||
|
||||
```markdown
|
||||
---
|
||||
name: skill-name
|
||||
description: What the skill does
|
||||
---
|
||||
|
||||
# Skill Name
|
||||
|
||||
Your skill documentation here...
|
||||
```
|
||||
|
||||
## Index JSON Format
|
||||
|
||||
The `skills-index.json` should look like:
|
||||
|
||||
```json
|
||||
{
|
||||
"version": 1,
|
||||
"skills": [
|
||||
{
|
||||
"slug": "my-skill",
|
||||
"name": "My Skill",
|
||||
"description": "Does something useful",
|
||||
"_path": "skills/my-skill",
|
||||
"download_url": "https://raw.githubusercontent.com/owner/repo/main/skills/my-skill",
|
||||
"files": ["SKILL.md", "scripts/helper.sh"]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Fields
|
||||
|
||||
| Field | Required | Description |
|
||||
|-------|----------|-------------|
|
||||
| `slug` | Yes | Unique identifier for the skill |
|
||||
| `name` | No | Display name (defaults to slug) |
|
||||
| `description` | No | Short description for search results |
|
||||
| `_path` | No | Derived path to skill folder (for categorization) |
|
||||
| `download_url` | Yes* | URL to download skill files from |
|
||||
| `files` | No | List of files to download (if omitted, downloads from download_url directly) |
|
||||
|
||||
* A direct download url can reference a ZIP archive.
|
||||
|
||||
### Including External Skills
|
||||
|
||||
You can include skills from other sources by placing `.json` files in your skills directory:
|
||||
|
||||
```
|
||||
skills/
|
||||
├── self/
|
||||
│ ├── self-config/
|
||||
│ │ └── SKILL.md
|
||||
│ └── external-skill/
|
||||
│ └── skills.json <-- included skills files obtained from any url.*
|
||||
```
|
||||
|
||||
* Hence the `allowed_prefixes` config option.
|
||||
|
||||
The JSON file can contain an array of skill objects:
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"slug": "external-skill",
|
||||
"name": "External Skill",
|
||||
"description": "Skill defined in external JSON",
|
||||
"download_url": "https://raw.githubusercontent.com/other/repo/main/skill"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
|
@ -172,10 +172,12 @@ The MCP tool enables integration with external Model Context Protocol servers.
|
|||
|
||||
## Skills Tool
|
||||
|
||||
The skills tool configures skill discovery and installation via registries like ClawHub.
|
||||
The skills tool configures skill discovery and installation via registries like ClawHub and GitHub.
|
||||
|
||||
### Registries
|
||||
|
||||
#### ClawHub
|
||||
|
||||
| Config | Type | Default | Description |
|
||||
| ---------------------------------- | ------ | -------------------- | ----------------------- |
|
||||
| `registries.clawhub.enabled` | bool | true | Enable ClawHub registry |
|
||||
|
|
@ -185,6 +187,17 @@ The skills tool configures skill discovery and installation via registries like
|
|||
| `registries.clawhub.skills_path` | string | `/api/v1/skills` | Skills API path |
|
||||
| `registries.clawhub.download_path` | string | `/api/v1/download` | Download API path |
|
||||
|
||||
#### Index (Remote)
|
||||
|
||||
Index registries fetch `skills-index.json` from any HTTP URL.
|
||||
|
||||
| Config | Type | Default | Description |
|
||||
| ------------------------------- | ------ | ------------- | -------------------------------------------------------- |
|
||||
| `registries.index:<name>.enabled` | bool | false | Enable index registry |
|
||||
| `registries.index:<name>.index_url` | string | - | Full URL to skills-index.json |
|
||||
|
||||
The index URL can point to any publicly accessible `skills-index.json` file (e.g., GitHub wiki raw URL).
|
||||
|
||||
### Configuration Example
|
||||
|
||||
```json
|
||||
|
|
@ -199,6 +212,11 @@ The skills tool configures skill discovery and installation via registries like
|
|||
"search_path": "/api/v1/search",
|
||||
"skills_path": "/api/v1/skills",
|
||||
"download_path": "/api/v1/download"
|
||||
},
|
||||
"index:angelhub": {
|
||||
"enabled": true,
|
||||
"index_url": "https://raw.githubusercontent.com/wiki/keithy/angelhub/picoclaw-skills-index.json"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/caarlos0/env/v11"
|
||||
|
|
@ -674,7 +675,56 @@ type SearchCacheConfig struct {
|
|||
}
|
||||
|
||||
type SkillsRegistriesConfig struct {
|
||||
ClawHub ClawHubRegistryConfig `json:"clawhub"`
|
||||
ClawHub ClawHubRegistryConfig `json:"clawhub"`
|
||||
Index map[string]IndexRegistryConfig `json:"index"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON supports both flat "index:*" keys and nested "index" object.
|
||||
// Keys like "index:angelhub" are auto-detected as index registries.
|
||||
func (s *SkillsRegistriesConfig) UnmarshalJSON(data []byte) error {
|
||||
type Alias SkillsRegistriesConfig
|
||||
aux := &struct {
|
||||
*Alias
|
||||
}{
|
||||
Alias: (*Alias)(s),
|
||||
}
|
||||
if err := json.Unmarshal(data, &aux); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Initialize Index map if nil
|
||||
if s.Index == nil {
|
||||
s.Index = make(map[string]IndexRegistryConfig)
|
||||
}
|
||||
|
||||
// Parse again to find flat "index:*" keys
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.Unmarshal(data, &raw); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for key, value := range raw {
|
||||
if strings.HasPrefix(key, "index:") {
|
||||
var cfg IndexRegistryConfig
|
||||
if err := json.Unmarshal(value, &cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
s.Index[key] = cfg
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type IndexRegistryConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
IndexURL string `json:"index_url"`
|
||||
ExtraHeader string `json:"extra_header"`
|
||||
AuthorizationHeader string `json:"authorization_header"`
|
||||
AgentHeader string `json:"agent_header"`
|
||||
AllowedPrefixes []string `json:"allowed_prefixes"`
|
||||
URLMappings map[string]string `json:"url_mappings"`
|
||||
SymlinkLocal bool `json:"symlink_local"`
|
||||
}
|
||||
|
||||
type ClawHubRegistryConfig struct {
|
||||
|
|
|
|||
535
pkg/skills/index_registry.go
Normal file
535
pkg/skills/index_registry.go
Normal file
|
|
@ -0,0 +1,535 @@
|
|||
package skills
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/sipeed/picoclaw/pkg/utils"
|
||||
)
|
||||
|
||||
// IndexRegistry discovers skills from any index.json URL.
|
||||
type IndexRegistry struct {
|
||||
name string
|
||||
indexURL string
|
||||
extraHeader string
|
||||
authorizationHeader string
|
||||
agentHeader string
|
||||
allowedPrefixes []string
|
||||
urlMappings map[string]string
|
||||
symlinkLocal bool
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewIndexRegistry creates a new Index registry.
|
||||
func NewIndexRegistry(name string, config IndexRegistryConfig) *IndexRegistry {
|
||||
return &IndexRegistry{
|
||||
name: name,
|
||||
indexURL: config.IndexURL,
|
||||
extraHeader: config.ExtraHeader,
|
||||
authorizationHeader: config.AuthorizationHeader,
|
||||
agentHeader: config.AgentHeader,
|
||||
allowedPrefixes: config.AllowedPrefixes,
|
||||
urlMappings: config.URLMappings,
|
||||
symlinkLocal: config.SymlinkLocal,
|
||||
httpClient: http.DefaultClient,
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the registry name.
|
||||
func (r *IndexRegistry) Name() string {
|
||||
return r.name
|
||||
}
|
||||
|
||||
// isURLAllowed checks if the URL matches any of the allowed prefixes.
|
||||
// If no prefixes are configured, all URLs are allowed.
|
||||
func (r *IndexRegistry) isURLAllowed(url string) bool {
|
||||
if len(r.allowedPrefixes) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, prefix := range r.allowedPrefixes {
|
||||
if strings.HasPrefix(url, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// mapURL maps a URL to another URL if a mapping exists.
|
||||
// This allows redirecting to local files (file://) or alternative sources (forks).
|
||||
func (r *IndexRegistry) mapURL(url string) string {
|
||||
if len(r.urlMappings) == 0 {
|
||||
return url
|
||||
}
|
||||
for prefix, replacement := range r.urlMappings {
|
||||
if strings.HasPrefix(url, prefix) {
|
||||
return strings.Replace(url, prefix, replacement, 1)
|
||||
}
|
||||
}
|
||||
return url
|
||||
}
|
||||
|
||||
// Search searches for skills in the Index registry.
|
||||
func (r *IndexRegistry) Search(ctx context.Context, query string, limit int) ([]SearchResult, error) {
|
||||
index, err := r.getSkillIndex(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get skill index: %w", err)
|
||||
}
|
||||
|
||||
var results []SearchResult
|
||||
queryLower := strings.ToLower(query)
|
||||
for _, skill := range index.Skills {
|
||||
if query == "" || strings.Contains(strings.ToLower(skill.Slug), queryLower) {
|
||||
results = append(results, SearchResult{
|
||||
Score: 1.0,
|
||||
Slug: skill.Slug,
|
||||
DisplayName: skill.Name,
|
||||
Summary: skill.Description,
|
||||
RegistryName: r.name,
|
||||
})
|
||||
if len(results) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// GetSkillMeta retrieves metadata for a specific skill.
|
||||
func (r *IndexRegistry) GetSkillMeta(ctx context.Context, slug string) (*SkillMeta, error) {
|
||||
index, err := r.getSkillIndex(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get skill index: %w", err)
|
||||
}
|
||||
|
||||
for _, skill := range index.Skills {
|
||||
if skill.Slug == slug {
|
||||
return &SkillMeta{
|
||||
Slug: skill.Slug,
|
||||
DisplayName: skill.Name,
|
||||
Summary: skill.Description,
|
||||
RegistryName: r.name,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("skill not found: %s", slug)
|
||||
}
|
||||
|
||||
// DownloadAndInstall downloads and installs a skill from GitHub.
|
||||
func (r *IndexRegistry) DownloadAndInstall(
|
||||
ctx context.Context, slug, version, targetDir string,
|
||||
) (*InstallResult, error) {
|
||||
index, err := r.getSkillIndex(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get skill index: %w", err)
|
||||
}
|
||||
|
||||
var skillDef SkillDefinition
|
||||
var found bool
|
||||
for _, skill := range index.Skills {
|
||||
if skill.Slug == slug {
|
||||
skillDef = skill
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("skill not found: %s", slug)
|
||||
}
|
||||
|
||||
// Get download URL
|
||||
if skillDef.DownloadURL == "" {
|
||||
return nil, fmt.Errorf("no download URL for skill: %s", slug)
|
||||
}
|
||||
|
||||
// Apply URL mapping for development (local files or forks)
|
||||
mappedURL := r.mapURL(skillDef.DownloadURL)
|
||||
|
||||
// If mapped URL is a local file:// directory and symlink_local enabled,
|
||||
// symlink the entire directory instead of downloading individual files
|
||||
if r.symlinkLocal && strings.HasPrefix(mappedURL, "file://") {
|
||||
localPath := strings.TrimPrefix(mappedURL, "file://")
|
||||
info, err := os.Stat(localPath)
|
||||
if err == nil && info.IsDir() {
|
||||
if err := r.copyLocalPath(localPath, targetDir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &InstallResult{
|
||||
Summary: fmt.Sprintf("Symlinked skill from %s", mappedURL),
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// If files list provided, fetch each file from download_url + file path
|
||||
// Otherwise, fetch directly from download_url
|
||||
if len(skillDef.Files) > 0 {
|
||||
for _, filePath := range skillDef.Files {
|
||||
url := strings.TrimRight(mappedURL, "/") + "/" + filePath
|
||||
if err := r.downloadFromURL(ctx, url, targetDir); err != nil {
|
||||
return nil, fmt.Errorf("failed to download %s: %w", filePath, err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if err := r.downloadFromURL(ctx, mappedURL, targetDir); err != nil {
|
||||
return nil, fmt.Errorf("failed to download skill: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return &InstallResult{
|
||||
Summary: fmt.Sprintf("Installed skill from %s", mappedURL),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// downloadFromURL fetches from download_url and handles both file content and directory listings.
|
||||
func (r *IndexRegistry) downloadFromURL(ctx context.Context, url, targetDir string) error {
|
||||
// Apply URL mapping (for local files or alternative sources)
|
||||
url = r.mapURL(url)
|
||||
|
||||
// Handle file:// URLs for local development
|
||||
if strings.HasPrefix(url, "file://") {
|
||||
localPath := strings.TrimPrefix(url, "file://")
|
||||
return r.copyLocalPath(localPath, targetDir)
|
||||
}
|
||||
|
||||
// Validate URL against allowed prefixes
|
||||
if !r.isURLAllowed(url) {
|
||||
return fmt.Errorf("URL not allowed by registry restrictions: %s", url)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := r.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("failed to download: %s", resp.Status)
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check if it's a ZIP archive
|
||||
if strings.HasSuffix(url, ".zip") || strings.Contains(contentType, "zip") {
|
||||
tmpPath, tmpErr := os.CreateTemp("", "skill-*.zip")
|
||||
if tmpErr != nil {
|
||||
return fmt.Errorf("failed to create temp file: %w", tmpErr)
|
||||
}
|
||||
defer os.Remove(tmpPath.Name())
|
||||
|
||||
if _, err := tmpPath.Write(data); err != nil {
|
||||
tmpPath.Close()
|
||||
return fmt.Errorf("failed to write temp file: %w", err)
|
||||
}
|
||||
tmpPath.Close()
|
||||
|
||||
// Extract to temp dir first, then strip root component
|
||||
tmpDir, err := os.MkdirTemp("", "skill-extract-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create temp dir: %w", err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
if err := utils.ExtractZipFile(tmpPath.Name(), tmpDir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Strip first path component (common in archives)
|
||||
return r.stripRootAndMove(tmpDir, targetDir)
|
||||
}
|
||||
|
||||
// Check if it's JSON (directory listing) or raw content
|
||||
if strings.Contains(contentType, "application/json") || strings.HasPrefix(strings.TrimSpace(string(data)), "[") {
|
||||
// It's a JSON listing - parse and fetch each item
|
||||
return r.parseAndDownloadListing(ctx, data, targetDir)
|
||||
}
|
||||
|
||||
// It's raw content - save directly to targetDir
|
||||
return r.saveFile(data, targetDir, filepath.Base(url))
|
||||
}
|
||||
|
||||
// parseAndDownloadListing parses JSON listing and downloads each file.
|
||||
func (r *IndexRegistry) parseAndDownloadListing(ctx context.Context, data []byte, targetDir string) error {
|
||||
// Try parsing as array first
|
||||
var items []struct {
|
||||
Name string `json:"name"`
|
||||
DownloadURL string `json:"download_url"`
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(data, &items); err != nil {
|
||||
// Try single object
|
||||
var item struct {
|
||||
Name string `json:"name"`
|
||||
DownloadURL string `json:"download_url"`
|
||||
Type string `json:"type"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &item); err != nil {
|
||||
return fmt.Errorf("failed to parse listing: %w", err)
|
||||
}
|
||||
if item.Name != "" {
|
||||
items = []struct {
|
||||
Name string `json:"name"`
|
||||
DownloadURL string `json:"download_url"`
|
||||
Type string `json:"type"`
|
||||
}{item}
|
||||
}
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
if item.Type == "dir" || item.DownloadURL == "" {
|
||||
subDir := filepath.Join(targetDir, item.Name)
|
||||
if err := os.MkdirAll(subDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.downloadFromURL(ctx, item.DownloadURL, subDir); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// Download file
|
||||
if err := r.downloadFile(ctx, item.DownloadURL, targetDir, item.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// downloadFile downloads a single file.
|
||||
func (r *IndexRegistry) downloadFile(ctx context.Context, url, targetDir, filename string) error {
|
||||
// Apply URL mapping
|
||||
url = r.mapURL(url)
|
||||
|
||||
// Handle file:// URLs for local development
|
||||
if strings.HasPrefix(url, "file://") {
|
||||
localPath := strings.TrimPrefix(url, "file://")
|
||||
return r.copyLocalPath(localPath, targetDir)
|
||||
}
|
||||
|
||||
if !r.isURLAllowed(url) {
|
||||
return fmt.Errorf("URL not allowed by registry restrictions: %s", url)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := r.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("failed to download file: %s", resp.Status)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return r.saveFile(data, targetDir, filename)
|
||||
}
|
||||
|
||||
// saveFile saves content to a file in targetDir.
|
||||
func (r *IndexRegistry) saveFile(data []byte, targetDir, filename string) error {
|
||||
if err := os.MkdirAll(targetDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
path := filepath.Join(targetDir, filename)
|
||||
return os.WriteFile(path, data, 0o644)
|
||||
}
|
||||
|
||||
// copyLocalPath copies a local file or directory to targetDir.
|
||||
// If symlinkLocal is true, creates a symlink to the local directory instead of copying.
|
||||
func (r *IndexRegistry) copyLocalPath(localPath, targetDir string) error {
|
||||
// If symlink enabled, create a symlink to the local directory
|
||||
if r.symlinkLocal {
|
||||
absPath, err := filepath.Abs(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get absolute path: %w", err)
|
||||
}
|
||||
// Remove targetDir if it exists (we're creating a symlink, not a directory)
|
||||
os.RemoveAll(targetDir)
|
||||
return os.Symlink(absPath, targetDir)
|
||||
}
|
||||
|
||||
info, err := os.Stat(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to stat local path: %w", err)
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
return filepath.Walk(localPath, func(path string, fi os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
relPath, err := filepath.Rel(localPath, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if relPath == "." {
|
||||
return nil
|
||||
}
|
||||
destPath := filepath.Join(targetDir, relPath)
|
||||
if fi.IsDir() {
|
||||
return os.MkdirAll(destPath, 0o755)
|
||||
}
|
||||
return copyFile(path, destPath)
|
||||
})
|
||||
}
|
||||
|
||||
// Single file
|
||||
if err := os.MkdirAll(targetDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
return copyFile(localPath, filepath.Join(targetDir, filepath.Base(localPath)))
|
||||
}
|
||||
|
||||
// copyFile copies a single file from src to dst.
|
||||
func copyFile(src, dst string) error {
|
||||
srcFile, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer srcFile.Close()
|
||||
|
||||
dstFile, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dstFile.Close()
|
||||
|
||||
_, err = io.Copy(dstFile, srcFile)
|
||||
return err
|
||||
}
|
||||
|
||||
// stripRootAndMove moves files from srcDir to targetDir, stripping the first path component.
|
||||
func (r *IndexRegistry) stripRootAndMove(srcDir, targetDir string) error {
|
||||
entries, err := os.ReadDir(srcDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// If there's a single directory, use it as the root
|
||||
var rootDir string
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
rootDir = entry.Name()
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if rootDir == "" {
|
||||
// No subdirectory, just move everything up
|
||||
} else {
|
||||
srcDir = filepath.Join(srcDir, rootDir)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(targetDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
relPath, err := filepath.Rel(srcDir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if relPath == "." {
|
||||
return nil
|
||||
}
|
||||
|
||||
destPath := filepath.Join(targetDir, relPath)
|
||||
|
||||
if info.IsDir() {
|
||||
return os.MkdirAll(destPath, 0o755)
|
||||
}
|
||||
|
||||
return os.Rename(path, destPath)
|
||||
})
|
||||
}
|
||||
|
||||
// getSkillIndex fetches the skill index from the configured URL.
|
||||
func (r *IndexRegistry) getSkillIndex(ctx context.Context) (*SkillIndex, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", r.indexURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if r.extraHeader != "" {
|
||||
parts := strings.SplitN(r.extraHeader, ":", 2)
|
||||
if len(parts) == 2 {
|
||||
req.Header.Set(strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]))
|
||||
}
|
||||
}
|
||||
if r.authorizationHeader != "" {
|
||||
req.Header.Set("Authorization", r.authorizationHeader)
|
||||
}
|
||||
if r.agentHeader != "" {
|
||||
req.Header.Set("User-Agent", r.agentHeader)
|
||||
} else {
|
||||
req.Header.Set("User-Agent", "picoclaw")
|
||||
}
|
||||
|
||||
resp, err := r.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("failed to fetch skills index from %s: HTTP %d", r.indexURL, resp.StatusCode)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var index SkillIndex
|
||||
if err := json.Unmarshal(data, &index); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse skills index from %s: %w", r.indexURL, err)
|
||||
}
|
||||
|
||||
if len(index.Skills) == 0 {
|
||||
return nil, fmt.Errorf("no skills found in index from %s (check format/version)", r.indexURL)
|
||||
}
|
||||
|
||||
// Validate at least one skill has a download URL
|
||||
hasValidSkill := false
|
||||
for _, s := range index.Skills {
|
||||
if s.DownloadURL != "" {
|
||||
hasValidSkill = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasValidSkill {
|
||||
return nil, fmt.Errorf("index from %s has no skills with download_url (check format)", r.indexURL)
|
||||
}
|
||||
|
||||
return &index, nil
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
|
@ -60,10 +61,23 @@ type SkillRegistry interface {
|
|||
// RegistryConfig holds configuration for all skill registries.
|
||||
// This is the input to NewRegistryManagerFromConfig.
|
||||
type RegistryConfig struct {
|
||||
Index map[string]IndexRegistryConfig
|
||||
ClawHub ClawHubConfig
|
||||
MaxConcurrentSearches int
|
||||
}
|
||||
|
||||
// IndexRegistryConfig configures an index-based registry.
|
||||
type IndexRegistryConfig struct {
|
||||
Enabled bool
|
||||
IndexURL string
|
||||
ExtraHeader string
|
||||
AuthorizationHeader string
|
||||
AgentHeader string
|
||||
AllowedPrefixes []string
|
||||
URLMappings map[string]string // URL prefix -> local file path mapping
|
||||
SymlinkLocal bool // Symlink instead of copy for local file:// URLs
|
||||
}
|
||||
|
||||
// ClawHubConfig configures the ClawHub registry.
|
||||
type ClawHubConfig struct {
|
||||
Enabled bool
|
||||
|
|
@ -100,6 +114,12 @@ func NewRegistryManagerFromConfig(cfg RegistryConfig) *RegistryManager {
|
|||
if cfg.MaxConcurrentSearches > 0 {
|
||||
rm.maxConcurrent = cfg.MaxConcurrentSearches
|
||||
}
|
||||
// Add index registries from map
|
||||
for name, indexCfg := range cfg.Index {
|
||||
if indexCfg.Enabled {
|
||||
rm.AddRegistry(NewIndexRegistry(name, indexCfg))
|
||||
}
|
||||
}
|
||||
if cfg.ClawHub.Enabled {
|
||||
rm.AddRegistry(NewClawHubRegistry(cfg.ClawHub))
|
||||
}
|
||||
|
|
@ -114,11 +134,12 @@ func (rm *RegistryManager) AddRegistry(r SkillRegistry) {
|
|||
}
|
||||
|
||||
// GetRegistry returns a registry by name, or nil if not found.
|
||||
// Supports exact match and prefix match (e.g., "github" matches "github:angelhub").
|
||||
func (rm *RegistryManager) GetRegistry(name string) SkillRegistry {
|
||||
rm.mu.RLock()
|
||||
defer rm.mu.RUnlock()
|
||||
for _, r := range rm.registries {
|
||||
if r.Name() == name {
|
||||
if r.Name() == name || strings.HasPrefix(r.Name(), name+":") || strings.HasPrefix(name, r.Name()+":") {
|
||||
return r
|
||||
}
|
||||
}
|
||||
|
|
@ -221,3 +242,18 @@ func sortByScoreDesc(results []SearchResult) {
|
|||
results[j+1] = key
|
||||
}
|
||||
}
|
||||
|
||||
// SkillIndex represents the index published by a GitHub workflow.
|
||||
type SkillIndex struct {
|
||||
Skills []SkillDefinition `json:"skills"`
|
||||
}
|
||||
|
||||
// SkillDefinition defines a skill in the index.
|
||||
type SkillDefinition struct {
|
||||
Slug string `json:"slug"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Path string `json:"path"`
|
||||
DownloadURL string `json:"download_url"`
|
||||
Files []string `json:"files"`
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue