diff --git a/cmd/picoclaw/internal/skills/helpers.go b/cmd/picoclaw/internal/skills/helpers.go index a59a2013a..72ba00cf7 100644 --- a/cmd/picoclaw/internal/skills/helpers.go +++ b/cmd/picoclaw/internal/skills/helpers.go @@ -17,6 +17,15 @@ import ( const skillsSearchMaxResults = 20 +// ConvertConfig converts config.IndexRegistryConfig map to skills.IndexRegistryConfig map +func ConvertConfig(c map[string]config.IndexRegistryConfig) map[string]skills.IndexRegistryConfig { + result := make(map[string]skills.IndexRegistryConfig) + for k, v := range c { + result[k] = skills.IndexRegistryConfig(v) + } + return result +} + func skillsListCmd(loader *skills.SkillsLoader) { allSkills := loader.ListSkills() @@ -67,6 +76,7 @@ func skillsInstallFromRegistry(cfg *config.Config, registryName, slug string) er registryMgr := skills.NewRegistryManagerFromConfig(skills.RegistryConfig{ MaxConcurrentSearches: cfg.Tools.Skills.MaxConcurrentSearches, ClawHub: skills.ClawHubConfig(cfg.Tools.Skills.Registries.ClawHub), + Index: ConvertConfig(cfg.Tools.Skills.Registries.Index), }) registry := registryMgr.GetRegistry(registryName) @@ -229,6 +239,7 @@ func skillsSearchCmd(query string) { registryMgr := skills.NewRegistryManagerFromConfig(skills.RegistryConfig{ MaxConcurrentSearches: cfg.Tools.Skills.MaxConcurrentSearches, ClawHub: skills.ClawHubConfig(cfg.Tools.Skills.Registries.ClawHub), + Index: ConvertConfig(cfg.Tools.Skills.Registries.Index), }) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) diff --git a/config/config.example.json b/config/config.example.json index 8e58e3252..172af2b82 100644 --- a/config/config.example.json +++ b/config/config.example.json @@ -373,13 +373,26 @@ "clawhub": { "enabled": true, "base_url": "https://clawhub.ai", - "auth_token": "", - "search_path": "", - "skills_path": "", - "download_path": "", - "timeout": 0, - "max_zip_size": 0, - "max_response_size": 0 + "search_path": "/api/v1/search", + "skills_path": "/api/v1/skills", + "download_path": "/api/v1/download" + }, + "index:angelhub": { + "enabled": false, + "index_url": "https://raw.githubusercontent.com/wiki/keithy/angelhub/picoclaw-skills-index.json", + "extra_header": "X-Custom-Header: value", + "authorization_header": "Bearer token", + "agent_header": "picoclaw/1.0", + "allowed_prefixes": [ + "https://raw.githubusercontent.com/wiki/keithy/angelhub/", + "https://raw.githubusercontent.com/keithy/angelhub/" + ], + "url_mappings": { + "https://raw.githubusercontent.com/keithy/angelhub/main/": "https://raw.githubusercontent.com/keithy/angelhub/beta/", + "https://raw.githubusercontent.com/keithy/angelhub/": "file:///home/me/repos/angelhub/" + }, + "symlink_local": true + } } }, "max_concurrent_searches": 2, diff --git a/docs/SKILLS_REGISTRIES.md b/docs/SKILLS_REGISTRIES.md new file mode 100644 index 000000000..051a5648c --- /dev/null +++ b/docs/SKILLS_REGISTRIES.md @@ -0,0 +1,247 @@ +# Skills Registries + +PicoClaw supports installing skills from multiple registries. This guide covers how to use and create registries. + +## Using Registries + +### List Available Skills + +```bash +picoclaw skills search +``` + +### Install a Skill + +```bash +# Install from a specific registry +picoclaw skills install --registry index:angelhub self-config +picoclaw skills install --registry clawhub github + +# Install directly from GitHub hosted SKILL.md +picoclaw skills install owner/repo/skill-name +``` + +### List Installed Skills + +```bash +picoclaw skills list +``` + +## Supported Registries + +### ClawHub Registry + +The default registry at [clawhub.ai](https://clawhub.ai). Enable in config: + +```json +{ + "tools": { + "skills": { + "registries": { + "clawhub": { + "enabled": true + } + } + } + } +} +``` + +### Index Fronted Registry + +Install skills from any HTTP endpoint that serves a skills-index.json. + +**Basic Configuration:** + +```json +{ + "tools": { + "skills": { + "registries": { + "index:myorg": { + "enabled": true, + "index_url": "https://example.com/skills-index.json" + } + } + } + } +} +``` + +**Configuration with security options:** + +```json +{ + "tools": { + "skills": { + "registries": { + "index:angelhub": { + "enabled": true, + "index_url": "https://raw.githubusercontent.com/wiki/keithy/angelhub/picoclaw-skills-index.json", + "extra_header": "X-Custom-Header: value", + "authorization_header": "Bearer token", + "agent_header": "picoclaw/1.0", + "allowed_prefixes": [ + "https://raw.githubusercontent.com/wiki/keithy/angelhub/", + "https://raw.githubusercontent.com/keithy/angelhub/" + ] + } + } + } + } +} +``` + +**Configuration for local development:** + +Use `url_mappings` to redirect downloads to local files or forks, and `symlink_local` to create symlinks instead of copying: + +```json +{ + "tools": { + "skills": { + "registries": { + "index:angelhub": { + "enabled": true, + "index_url": "https://raw.githubusercontent.com/wiki/keithy/angelhub/picoclaw-skills-index.json", + "allowed_prefixes": [ + "https://raw.githubusercontent.com/wiki/keithy/angelhub/", + "https://raw.githubusercontent.com/keithy/angelhub/" + ], + "url_mappings": { + "https://raw.githubusercontent.com/keithy/angelhub/main/": "https://raw.githubusercontent.com/keithy/angelhub/beta/", + "https://raw.githubusercontent.com/keithy/angelhub/": "file:///home/me/repos/angelhub/" + }, + "symlink_local": true + } + } + } + } +} +``` + +| Option | Description | +|--------|-------------| +| `url_mappings` | Map URL prefixes to redirect downloads. Useful for testing forks (`https://.../main/` → `https://.../beta/`) or local development (`https://...` → `file:///path/to/repo/`) | +| `symlink_local` | When using `file://` URLs in mappings, create a symlink to the local directory instead of copying files. Useful for development | + +## Creating Your Own Registry + +To create a skill registry: + +### 1. Create a Repository + +Create a public repository to host your skills. + +### 2. Add Skills + +Add skills in the `picoclaw/skills/` directory (or `skills/` for ecosystem-agnostic). Each skill needs a `SKILL.md` file: + +``` +picoclaw/ +└── skills/ + ├── self/ + │ ├── self-config/ + │ │ └── SKILL.md + │ └── self-debug/ + │ └── SKILL.md + └── weather/ + └── SKILL.md +``` + +### 3. Create the Index Workflow + +Add a workflow to generate the skills index. See [AngelHub's workflow](https://github.com/keithy/angelhub/blob/main/.github/workflows/picoclaw-skills-index.yml) for a complete example. + +### 4. Enable in PicoClaw + +```json +{ + "tools": { + "skills": { + "registries": { + "index:myorg": { + "enabled": true, + "index_url": "https://example.com/skills-index.json" + } + } + } + } +} +``` + +## Skill Format + +Each skill should have a `SKILL.md` file with frontmatter: + +```markdown +--- +name: skill-name +description: What the skill does +--- + +# Skill Name + +Your skill documentation here... +``` + +## Index JSON Format + +The `skills-index.json` should look like: + +```json +{ + "version": 1, + "skills": [ + { + "slug": "my-skill", + "name": "My Skill", + "description": "Does something useful", + "_path": "skills/my-skill", + "download_url": "https://raw.githubusercontent.com/owner/repo/main/skills/my-skill", + "files": ["SKILL.md", "scripts/helper.sh"] + } + ] +} +``` + +### Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `slug` | Yes | Unique identifier for the skill | +| `name` | No | Display name (defaults to slug) | +| `description` | No | Short description for search results | +| `_path` | No | Derived path to skill folder (for categorization) | +| `download_url` | Yes* | URL to download skill files from | +| `files` | No | List of files to download (if omitted, downloads from download_url directly) | + +* A direct download url can reference a ZIP archive. + +### Including External Skills + +You can include skills from other sources by placing `.json` files in your skills directory: + +``` +skills/ +├── self/ +│ ├── self-config/ +│ │ └── SKILL.md +│ └── external-skill/ +│ └── skills.json <-- included skills files obtained from any url.* +``` + +* Hence the `allowed_prefixes` config option. + +The JSON file can contain an array of skill objects: + +```json +[ + { + "slug": "external-skill", + "name": "External Skill", + "description": "Skill defined in external JSON", + "download_url": "https://raw.githubusercontent.com/other/repo/main/skill" + } +] +``` diff --git a/docs/tools_configuration.md b/docs/tools_configuration.md index e64a3a107..f2552728c 100644 --- a/docs/tools_configuration.md +++ b/docs/tools_configuration.md @@ -172,10 +172,12 @@ The MCP tool enables integration with external Model Context Protocol servers. ## Skills Tool -The skills tool configures skill discovery and installation via registries like ClawHub. +The skills tool configures skill discovery and installation via registries like ClawHub and GitHub. ### Registries +#### ClawHub + | Config | Type | Default | Description | | ---------------------------------- | ------ | -------------------- | ----------------------- | | `registries.clawhub.enabled` | bool | true | Enable ClawHub registry | @@ -185,6 +187,17 @@ The skills tool configures skill discovery and installation via registries like | `registries.clawhub.skills_path` | string | `/api/v1/skills` | Skills API path | | `registries.clawhub.download_path` | string | `/api/v1/download` | Download API path | +#### Index (Remote) + +Index registries fetch `skills-index.json` from any HTTP URL. + +| Config | Type | Default | Description | +| ------------------------------- | ------ | ------------- | -------------------------------------------------------- | +| `registries.index:.enabled` | bool | false | Enable index registry | +| `registries.index:.index_url` | string | - | Full URL to skills-index.json | + +The index URL can point to any publicly accessible `skills-index.json` file (e.g., GitHub wiki raw URL). + ### Configuration Example ```json @@ -199,6 +212,11 @@ The skills tool configures skill discovery and installation via registries like "search_path": "/api/v1/search", "skills_path": "/api/v1/skills", "download_path": "/api/v1/download" + }, + "index:angelhub": { + "enabled": true, + "index_url": "https://raw.githubusercontent.com/wiki/keithy/angelhub/picoclaw-skills-index.json" + } } } } diff --git a/pkg/config/config.go b/pkg/config/config.go index 5c53c08ad..1cc2bde3b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "os" + "strings" "sync/atomic" "github.com/caarlos0/env/v11" @@ -674,7 +675,56 @@ type SearchCacheConfig struct { } type SkillsRegistriesConfig struct { - ClawHub ClawHubRegistryConfig `json:"clawhub"` + ClawHub ClawHubRegistryConfig `json:"clawhub"` + Index map[string]IndexRegistryConfig `json:"index"` +} + +// UnmarshalJSON supports both flat "index:*" keys and nested "index" object. +// Keys like "index:angelhub" are auto-detected as index registries. +func (s *SkillsRegistriesConfig) UnmarshalJSON(data []byte) error { + type Alias SkillsRegistriesConfig + aux := &struct { + *Alias + }{ + Alias: (*Alias)(s), + } + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + + // Initialize Index map if nil + if s.Index == nil { + s.Index = make(map[string]IndexRegistryConfig) + } + + // Parse again to find flat "index:*" keys + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + + for key, value := range raw { + if strings.HasPrefix(key, "index:") { + var cfg IndexRegistryConfig + if err := json.Unmarshal(value, &cfg); err != nil { + return err + } + s.Index[key] = cfg + } + } + + return nil +} + +type IndexRegistryConfig struct { + Enabled bool `json:"enabled"` + IndexURL string `json:"index_url"` + ExtraHeader string `json:"extra_header"` + AuthorizationHeader string `json:"authorization_header"` + AgentHeader string `json:"agent_header"` + AllowedPrefixes []string `json:"allowed_prefixes"` + URLMappings map[string]string `json:"url_mappings"` + SymlinkLocal bool `json:"symlink_local"` } type ClawHubRegistryConfig struct { diff --git a/pkg/skills/index_registry.go b/pkg/skills/index_registry.go new file mode 100644 index 000000000..2a3260eae --- /dev/null +++ b/pkg/skills/index_registry.go @@ -0,0 +1,535 @@ +package skills + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/sipeed/picoclaw/pkg/utils" +) + +// IndexRegistry discovers skills from any index.json URL. +type IndexRegistry struct { + name string + indexURL string + extraHeader string + authorizationHeader string + agentHeader string + allowedPrefixes []string + urlMappings map[string]string + symlinkLocal bool + httpClient *http.Client +} + +// NewIndexRegistry creates a new Index registry. +func NewIndexRegistry(name string, config IndexRegistryConfig) *IndexRegistry { + return &IndexRegistry{ + name: name, + indexURL: config.IndexURL, + extraHeader: config.ExtraHeader, + authorizationHeader: config.AuthorizationHeader, + agentHeader: config.AgentHeader, + allowedPrefixes: config.AllowedPrefixes, + urlMappings: config.URLMappings, + symlinkLocal: config.SymlinkLocal, + httpClient: http.DefaultClient, + } +} + +// Name returns the registry name. +func (r *IndexRegistry) Name() string { + return r.name +} + +// isURLAllowed checks if the URL matches any of the allowed prefixes. +// If no prefixes are configured, all URLs are allowed. +func (r *IndexRegistry) isURLAllowed(url string) bool { + if len(r.allowedPrefixes) == 0 { + return true + } + for _, prefix := range r.allowedPrefixes { + if strings.HasPrefix(url, prefix) { + return true + } + } + return false +} + +// mapURL maps a URL to another URL if a mapping exists. +// This allows redirecting to local files (file://) or alternative sources (forks). +func (r *IndexRegistry) mapURL(url string) string { + if len(r.urlMappings) == 0 { + return url + } + for prefix, replacement := range r.urlMappings { + if strings.HasPrefix(url, prefix) { + return strings.Replace(url, prefix, replacement, 1) + } + } + return url +} + +// Search searches for skills in the Index registry. +func (r *IndexRegistry) Search(ctx context.Context, query string, limit int) ([]SearchResult, error) { + index, err := r.getSkillIndex(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get skill index: %w", err) + } + + var results []SearchResult + queryLower := strings.ToLower(query) + for _, skill := range index.Skills { + if query == "" || strings.Contains(strings.ToLower(skill.Slug), queryLower) { + results = append(results, SearchResult{ + Score: 1.0, + Slug: skill.Slug, + DisplayName: skill.Name, + Summary: skill.Description, + RegistryName: r.name, + }) + if len(results) >= limit { + break + } + } + } + + return results, nil +} + +// GetSkillMeta retrieves metadata for a specific skill. +func (r *IndexRegistry) GetSkillMeta(ctx context.Context, slug string) (*SkillMeta, error) { + index, err := r.getSkillIndex(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get skill index: %w", err) + } + + for _, skill := range index.Skills { + if skill.Slug == slug { + return &SkillMeta{ + Slug: skill.Slug, + DisplayName: skill.Name, + Summary: skill.Description, + RegistryName: r.name, + }, nil + } + } + + return nil, fmt.Errorf("skill not found: %s", slug) +} + +// DownloadAndInstall downloads and installs a skill from GitHub. +func (r *IndexRegistry) DownloadAndInstall( + ctx context.Context, slug, version, targetDir string, +) (*InstallResult, error) { + index, err := r.getSkillIndex(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get skill index: %w", err) + } + + var skillDef SkillDefinition + var found bool + for _, skill := range index.Skills { + if skill.Slug == slug { + skillDef = skill + found = true + break + } + } + if !found { + return nil, fmt.Errorf("skill not found: %s", slug) + } + + // Get download URL + if skillDef.DownloadURL == "" { + return nil, fmt.Errorf("no download URL for skill: %s", slug) + } + + // Apply URL mapping for development (local files or forks) + mappedURL := r.mapURL(skillDef.DownloadURL) + + // If mapped URL is a local file:// directory and symlink_local enabled, + // symlink the entire directory instead of downloading individual files + if r.symlinkLocal && strings.HasPrefix(mappedURL, "file://") { + localPath := strings.TrimPrefix(mappedURL, "file://") + info, err := os.Stat(localPath) + if err == nil && info.IsDir() { + if err := r.copyLocalPath(localPath, targetDir); err != nil { + return nil, err + } + return &InstallResult{ + Summary: fmt.Sprintf("Symlinked skill from %s", mappedURL), + }, nil + } + } + + // If files list provided, fetch each file from download_url + file path + // Otherwise, fetch directly from download_url + if len(skillDef.Files) > 0 { + for _, filePath := range skillDef.Files { + url := strings.TrimRight(mappedURL, "/") + "/" + filePath + if err := r.downloadFromURL(ctx, url, targetDir); err != nil { + return nil, fmt.Errorf("failed to download %s: %w", filePath, err) + } + } + } else { + if err := r.downloadFromURL(ctx, mappedURL, targetDir); err != nil { + return nil, fmt.Errorf("failed to download skill: %w", err) + } + } + + return &InstallResult{ + Summary: fmt.Sprintf("Installed skill from %s", mappedURL), + }, nil +} + +// downloadFromURL fetches from download_url and handles both file content and directory listings. +func (r *IndexRegistry) downloadFromURL(ctx context.Context, url, targetDir string) error { + // Apply URL mapping (for local files or alternative sources) + url = r.mapURL(url) + + // Handle file:// URLs for local development + if strings.HasPrefix(url, "file://") { + localPath := strings.TrimPrefix(url, "file://") + return r.copyLocalPath(localPath, targetDir) + } + + // Validate URL against allowed prefixes + if !r.isURLAllowed(url) { + return fmt.Errorf("URL not allowed by registry restrictions: %s", url) + } + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return err + } + + resp, err := r.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to download: %s", resp.Status) + } + + contentType := resp.Header.Get("Content-Type") + data, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + // Check if it's a ZIP archive + if strings.HasSuffix(url, ".zip") || strings.Contains(contentType, "zip") { + tmpPath, tmpErr := os.CreateTemp("", "skill-*.zip") + if tmpErr != nil { + return fmt.Errorf("failed to create temp file: %w", tmpErr) + } + defer os.Remove(tmpPath.Name()) + + if _, err := tmpPath.Write(data); err != nil { + tmpPath.Close() + return fmt.Errorf("failed to write temp file: %w", err) + } + tmpPath.Close() + + // Extract to temp dir first, then strip root component + tmpDir, err := os.MkdirTemp("", "skill-extract-*") + if err != nil { + return fmt.Errorf("failed to create temp dir: %w", err) + } + defer os.RemoveAll(tmpDir) + + if err := utils.ExtractZipFile(tmpPath.Name(), tmpDir); err != nil { + return err + } + + // Strip first path component (common in archives) + return r.stripRootAndMove(tmpDir, targetDir) + } + + // Check if it's JSON (directory listing) or raw content + if strings.Contains(contentType, "application/json") || strings.HasPrefix(strings.TrimSpace(string(data)), "[") { + // It's a JSON listing - parse and fetch each item + return r.parseAndDownloadListing(ctx, data, targetDir) + } + + // It's raw content - save directly to targetDir + return r.saveFile(data, targetDir, filepath.Base(url)) +} + +// parseAndDownloadListing parses JSON listing and downloads each file. +func (r *IndexRegistry) parseAndDownloadListing(ctx context.Context, data []byte, targetDir string) error { + // Try parsing as array first + var items []struct { + Name string `json:"name"` + DownloadURL string `json:"download_url"` + Type string `json:"type"` + } + + if err := json.Unmarshal(data, &items); err != nil { + // Try single object + var item struct { + Name string `json:"name"` + DownloadURL string `json:"download_url"` + Type string `json:"type"` + } + if err := json.Unmarshal(data, &item); err != nil { + return fmt.Errorf("failed to parse listing: %w", err) + } + if item.Name != "" { + items = []struct { + Name string `json:"name"` + DownloadURL string `json:"download_url"` + Type string `json:"type"` + }{item} + } + } + + for _, item := range items { + if item.Type == "dir" || item.DownloadURL == "" { + subDir := filepath.Join(targetDir, item.Name) + if err := os.MkdirAll(subDir, 0o755); err != nil { + return err + } + if err := r.downloadFromURL(ctx, item.DownloadURL, subDir); err != nil { + return err + } + } else { + // Download file + if err := r.downloadFile(ctx, item.DownloadURL, targetDir, item.Name); err != nil { + return err + } + } + } + + return nil +} + +// downloadFile downloads a single file. +func (r *IndexRegistry) downloadFile(ctx context.Context, url, targetDir, filename string) error { + // Apply URL mapping + url = r.mapURL(url) + + // Handle file:// URLs for local development + if strings.HasPrefix(url, "file://") { + localPath := strings.TrimPrefix(url, "file://") + return r.copyLocalPath(localPath, targetDir) + } + + if !r.isURLAllowed(url) { + return fmt.Errorf("URL not allowed by registry restrictions: %s", url) + } + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return err + } + + resp, err := r.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to download file: %s", resp.Status) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + return r.saveFile(data, targetDir, filename) +} + +// saveFile saves content to a file in targetDir. +func (r *IndexRegistry) saveFile(data []byte, targetDir, filename string) error { + if err := os.MkdirAll(targetDir, 0o755); err != nil { + return err + } + + path := filepath.Join(targetDir, filename) + return os.WriteFile(path, data, 0o644) +} + +// copyLocalPath copies a local file or directory to targetDir. +// If symlinkLocal is true, creates a symlink to the local directory instead of copying. +func (r *IndexRegistry) copyLocalPath(localPath, targetDir string) error { + // If symlink enabled, create a symlink to the local directory + if r.symlinkLocal { + absPath, err := filepath.Abs(localPath) + if err != nil { + return fmt.Errorf("failed to get absolute path: %w", err) + } + // Remove targetDir if it exists (we're creating a symlink, not a directory) + os.RemoveAll(targetDir) + return os.Symlink(absPath, targetDir) + } + + info, err := os.Stat(localPath) + if err != nil { + return fmt.Errorf("failed to stat local path: %w", err) + } + + if info.IsDir() { + return filepath.Walk(localPath, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + relPath, err := filepath.Rel(localPath, path) + if err != nil { + return err + } + if relPath == "." { + return nil + } + destPath := filepath.Join(targetDir, relPath) + if fi.IsDir() { + return os.MkdirAll(destPath, 0o755) + } + return copyFile(path, destPath) + }) + } + + // Single file + if err := os.MkdirAll(targetDir, 0o755); err != nil { + return err + } + return copyFile(localPath, filepath.Join(targetDir, filepath.Base(localPath))) +} + +// copyFile copies a single file from src to dst. +func copyFile(src, dst string) error { + srcFile, err := os.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + + dstFile, err := os.Create(dst) + if err != nil { + return err + } + defer dstFile.Close() + + _, err = io.Copy(dstFile, srcFile) + return err +} + +// stripRootAndMove moves files from srcDir to targetDir, stripping the first path component. +func (r *IndexRegistry) stripRootAndMove(srcDir, targetDir string) error { + entries, err := os.ReadDir(srcDir) + if err != nil { + return err + } + + // If there's a single directory, use it as the root + var rootDir string + for _, entry := range entries { + if entry.IsDir() { + rootDir = entry.Name() + break + } + } + + if rootDir == "" { + // No subdirectory, just move everything up + } else { + srcDir = filepath.Join(srcDir, rootDir) + } + + if err := os.MkdirAll(targetDir, 0o755); err != nil { + return err + } + + return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + relPath, err := filepath.Rel(srcDir, path) + if err != nil { + return err + } + + if relPath == "." { + return nil + } + + destPath := filepath.Join(targetDir, relPath) + + if info.IsDir() { + return os.MkdirAll(destPath, 0o755) + } + + return os.Rename(path, destPath) + }) +} + +// getSkillIndex fetches the skill index from the configured URL. +func (r *IndexRegistry) getSkillIndex(ctx context.Context) (*SkillIndex, error) { + req, err := http.NewRequestWithContext(ctx, "GET", r.indexURL, nil) + if err != nil { + return nil, err + } + + if r.extraHeader != "" { + parts := strings.SplitN(r.extraHeader, ":", 2) + if len(parts) == 2 { + req.Header.Set(strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1])) + } + } + if r.authorizationHeader != "" { + req.Header.Set("Authorization", r.authorizationHeader) + } + if r.agentHeader != "" { + req.Header.Set("User-Agent", r.agentHeader) + } else { + req.Header.Set("User-Agent", "picoclaw") + } + + resp, err := r.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to fetch skills index from %s: HTTP %d", r.indexURL, resp.StatusCode) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var index SkillIndex + if err := json.Unmarshal(data, &index); err != nil { + return nil, fmt.Errorf("failed to parse skills index from %s: %w", r.indexURL, err) + } + + if len(index.Skills) == 0 { + return nil, fmt.Errorf("no skills found in index from %s (check format/version)", r.indexURL) + } + + // Validate at least one skill has a download URL + hasValidSkill := false + for _, s := range index.Skills { + if s.DownloadURL != "" { + hasValidSkill = true + break + } + } + if !hasValidSkill { + return nil, fmt.Errorf("index from %s has no skills with download_url (check format)", r.indexURL) + } + + return &index, nil +} diff --git a/pkg/skills/registry.go b/pkg/skills/registry.go index 45ae72253..5a9eaa007 100644 --- a/pkg/skills/registry.go +++ b/pkg/skills/registry.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log/slog" + "strings" "sync" "time" ) @@ -60,10 +61,23 @@ type SkillRegistry interface { // RegistryConfig holds configuration for all skill registries. // This is the input to NewRegistryManagerFromConfig. type RegistryConfig struct { + Index map[string]IndexRegistryConfig ClawHub ClawHubConfig MaxConcurrentSearches int } +// IndexRegistryConfig configures an index-based registry. +type IndexRegistryConfig struct { + Enabled bool + IndexURL string + ExtraHeader string + AuthorizationHeader string + AgentHeader string + AllowedPrefixes []string + URLMappings map[string]string // URL prefix -> local file path mapping + SymlinkLocal bool // Symlink instead of copy for local file:// URLs +} + // ClawHubConfig configures the ClawHub registry. type ClawHubConfig struct { Enabled bool @@ -100,6 +114,12 @@ func NewRegistryManagerFromConfig(cfg RegistryConfig) *RegistryManager { if cfg.MaxConcurrentSearches > 0 { rm.maxConcurrent = cfg.MaxConcurrentSearches } + // Add index registries from map + for name, indexCfg := range cfg.Index { + if indexCfg.Enabled { + rm.AddRegistry(NewIndexRegistry(name, indexCfg)) + } + } if cfg.ClawHub.Enabled { rm.AddRegistry(NewClawHubRegistry(cfg.ClawHub)) } @@ -114,11 +134,12 @@ func (rm *RegistryManager) AddRegistry(r SkillRegistry) { } // GetRegistry returns a registry by name, or nil if not found. +// Supports exact match and prefix match (e.g., "github" matches "github:angelhub"). func (rm *RegistryManager) GetRegistry(name string) SkillRegistry { rm.mu.RLock() defer rm.mu.RUnlock() for _, r := range rm.registries { - if r.Name() == name { + if r.Name() == name || strings.HasPrefix(r.Name(), name+":") || strings.HasPrefix(name, r.Name()+":") { return r } } @@ -221,3 +242,18 @@ func sortByScoreDesc(results []SearchResult) { results[j+1] = key } } + +// SkillIndex represents the index published by a GitHub workflow. +type SkillIndex struct { + Skills []SkillDefinition `json:"skills"` +} + +// SkillDefinition defines a skill in the index. +type SkillDefinition struct { + Slug string `json:"slug"` + Name string `json:"name"` + Description string `json:"description"` + Path string `json:"path"` + DownloadURL string `json:"download_url"` + Files []string `json:"files"` +}