azure skills whitelisting: fix skills loader, security config, and tests
This commit is contained in:
parent
3d7ea15702
commit
60f3b7419f
19 changed files with 86 additions and 44 deletions
2
Makefile
2
Makefile
|
|
@ -254,7 +254,7 @@ test: generate
|
||||||
|
|
||||||
## fmt: Format Go code
|
## fmt: Format Go code
|
||||||
fmt:
|
fmt:
|
||||||
@$(GOLANGCI_LINT) fmt
|
@gofmt -s -w $$(find . -name "*.go" -not -path "./web/*" -not -path "./vendor/*")
|
||||||
|
|
||||||
## lint: Run linters
|
## lint: Run linters
|
||||||
lint:
|
lint:
|
||||||
|
|
|
||||||
|
|
@ -43,7 +43,7 @@ func NewSkillsCommand() *cobra.Command {
|
||||||
globalDir := filepath.Dir(internal.GetConfigPath())
|
globalDir := filepath.Dir(internal.GetConfigPath())
|
||||||
globalSkillsDir := filepath.Join(globalDir, "skills")
|
globalSkillsDir := filepath.Join(globalDir, "skills")
|
||||||
builtinSkillsDir := filepath.Join(globalDir, "picoclaw", "skills")
|
builtinSkillsDir := filepath.Join(globalDir, "picoclaw", "skills")
|
||||||
d.skillsLoader = skills.NewSkillsLoader(d.workspace, globalSkillsDir, builtinSkillsDir)
|
d.skillsLoader = skills.NewSkillsLoader(d.workspace, globalSkillsDir, builtinSkillsDir, nil, false)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -81,7 +81,7 @@ func NewContextBuilder(workspace string) *ContextBuilder {
|
||||||
|
|
||||||
return &ContextBuilder{
|
return &ContextBuilder{
|
||||||
workspace: workspace,
|
workspace: workspace,
|
||||||
skillsLoader: skills.NewSkillsLoader(workspace, globalSkillsDir, builtinSkillsDir),
|
skillsLoader: skills.NewSkillsLoader(workspace, globalSkillsDir, builtinSkillsDir, nil, false),
|
||||||
memory: NewMemoryStore(workspace),
|
memory: NewMemoryStore(workspace),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -272,11 +272,25 @@ func registerSharedTools(
|
||||||
cfg.Tools.Skills.SearchCache.MaxSize,
|
cfg.Tools.Skills.SearchCache.MaxSize,
|
||||||
time.Duration(cfg.Tools.Skills.SearchCache.TTLSeconds)*time.Second,
|
time.Duration(cfg.Tools.Skills.SearchCache.TTLSeconds)*time.Second,
|
||||||
)
|
)
|
||||||
agent.Tools.Register(tools.NewFindSkillsTool(registryMgr, searchCache, cfg.Tools.Skills.Whitelist, cfg.Tools.Skills.WhitelistEnabled))
|
agent.Tools.Register(
|
||||||
|
tools.NewFindSkillsTool(
|
||||||
|
registryMgr,
|
||||||
|
searchCache,
|
||||||
|
cfg.Tools.Skills.Whitelist,
|
||||||
|
cfg.Tools.Skills.WhitelistEnabled,
|
||||||
|
),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if install_skills_enable {
|
if install_skills_enable {
|
||||||
agent.Tools.Register(tools.NewInstallSkillTool(registryMgr, agent.Workspace, cfg.Tools.Skills.Whitelist, cfg.Tools.Skills.WhitelistEnabled))
|
agent.Tools.Register(
|
||||||
|
tools.NewInstallSkillTool(
|
||||||
|
registryMgr,
|
||||||
|
agent.Workspace,
|
||||||
|
cfg.Tools.Skills.Whitelist,
|
||||||
|
cfg.Tools.Skills.WhitelistEnabled,
|
||||||
|
),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -706,7 +706,7 @@ func (c *ModelConfig) UnmarshalJSON(data []byte) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
c.apiKeys = MergeAPIKeys(aux.APIKey, aux.APIKeys)
|
c.APIKeys = SimpleSecureStrings(MergeAPIKeys(aux.APIKey, aux.APIKeys)...)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -938,7 +938,7 @@ type ToolsConfig struct {
|
||||||
MediaCleanup MediaCleanupConfig `json:"media_cleanup" yaml:"-"`
|
MediaCleanup MediaCleanupConfig `json:"media_cleanup" yaml:"-"`
|
||||||
Whitelist FlexibleStringSlice `json:"whitelist,omitempty" yaml:"-" env:"PICOCLAW_TOOLS_WHITELIST"`
|
Whitelist FlexibleStringSlice `json:"whitelist,omitempty" yaml:"-" env:"PICOCLAW_TOOLS_WHITELIST"`
|
||||||
WhitelistEnabled bool `json:"whitelist_enabled,omitempty" yaml:"-" env:"PICOCLAW_TOOLS_WHITELIST_ENABLED"`
|
WhitelistEnabled bool `json:"whitelist_enabled,omitempty" yaml:"-" env:"PICOCLAW_TOOLS_WHITELIST_ENABLED"`
|
||||||
MCP MCPConfig `json:"mcp" yaml:"-""`
|
MCP MCPConfig `json:"mcp" yaml:"-"`
|
||||||
AppendFile ToolConfig `json:"append_file" yaml:"-" envPrefix:"PICOCLAW_TOOLS_APPEND_FILE_"`
|
AppendFile ToolConfig `json:"append_file" yaml:"-" envPrefix:"PICOCLAW_TOOLS_APPEND_FILE_"`
|
||||||
EditFile ToolConfig `json:"edit_file" yaml:"-" envPrefix:"PICOCLAW_TOOLS_EDIT_FILE_"`
|
EditFile ToolConfig `json:"edit_file" yaml:"-" envPrefix:"PICOCLAW_TOOLS_EDIT_FILE_"`
|
||||||
FindSkills ToolConfig `json:"find_skills" yaml:"-" envPrefix:"PICOCLAW_TOOLS_FIND_SKILLS_"`
|
FindSkills ToolConfig `json:"find_skills" yaml:"-" envPrefix:"PICOCLAW_TOOLS_FIND_SKILLS_"`
|
||||||
|
|
|
||||||
|
|
@ -34,8 +34,9 @@ func TestJSONUnmarshalPrivateFields(t *testing.T) {
|
||||||
if s.PublicField != "pub" {
|
if s.PublicField != "pub" {
|
||||||
t.Errorf("PublicField = %q, want 'pub'", s.PublicField)
|
t.Errorf("PublicField = %q, want 'pub'", s.PublicField)
|
||||||
}
|
}
|
||||||
|
// Private fields cannot be unmarshaled from JSON
|
||||||
if s.privateField != "" {
|
if s.privateField != "" {
|
||||||
t.Errorf("privateField = %q, want empty because unexported fields are ignored", s.privateField)
|
t.Errorf("privateField = %q, want empty string (private fields are not unmarshaled)", s.privateField)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -168,7 +168,6 @@ func Run(debug bool, homePath, configPath string, allowEmptyStartup bool) error
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
fmt.Printf("✓ Gateway started on %s:%d\n", cfg.Gateway.Host, cfg.Gateway.Port)
|
fmt.Printf("✓ Gateway started on %s:%d\n", cfg.Gateway.Host, cfg.Gateway.Port)
|
||||||
fmt.Println("Press Ctrl+C to stop")
|
fmt.Println("Press Ctrl+C to stop")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/sipeed/picoclaw/pkg/logger"
|
"github.com/sipeed/picoclaw/pkg/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -206,7 +206,6 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err
|
||||||
cfg.RequestTimeout,
|
cfg.RequestTimeout,
|
||||||
), modelID, nil
|
), modelID, nil
|
||||||
|
|
||||||
|
|
||||||
case "minimax":
|
case "minimax":
|
||||||
// Minimax requires reasoning_split: true in the request body
|
// Minimax requires reasoning_split: true in the request body
|
||||||
if cfg.APIKey() == "" && cfg.APIBase == "" {
|
if cfg.APIKey() == "" && cfg.APIBase == "" {
|
||||||
|
|
|
||||||
|
|
@ -90,4 +90,3 @@ func (p *HTTPProvider) SetUseAzureHeaders(use bool) {
|
||||||
func (p *HTTPProvider) SupportsNativeSearch() bool {
|
func (p *HTTPProvider) SupportsNativeSearch() bool {
|
||||||
return p.delegate.SupportsNativeSearch()
|
return p.delegate.SupportsNativeSearch()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -31,12 +31,12 @@ type (
|
||||||
)
|
)
|
||||||
|
|
||||||
type Provider struct {
|
type Provider struct {
|
||||||
apiKey string
|
apiKey string
|
||||||
apiBase string
|
apiBase string
|
||||||
maxTokensField string // Field name for max tokens (e.g., "max_completion_tokens" for o1/glm models)
|
maxTokensField string // Field name for max tokens (e.g., "max_completion_tokens" for o1/glm models)
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
extraBody map[string]any // Additional fields to inject into request body
|
extraBody map[string]any // Additional fields to inject into request body
|
||||||
useAzureHeaders bool // Use api-key header instead of Authorization: Bearer
|
useAzureHeaders bool // Use api-key header instead of Authorization: Bearer
|
||||||
}
|
}
|
||||||
|
|
||||||
type Option func(*Provider)
|
type Option func(*Provider)
|
||||||
|
|
|
||||||
|
|
@ -59,10 +59,10 @@ func (info SkillInfo) validate() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
type SkillsLoader struct {
|
type SkillsLoader struct {
|
||||||
workspace string
|
workspace string
|
||||||
workspaceSkills string // workspace skills (project-level)
|
workspaceSkills string // workspace skills (project-level)
|
||||||
globalSkills string // global skills (~/.picoclaw/skills)
|
globalSkills string // global skills (~/.picoclaw/skills)
|
||||||
builtinSkills string // builtin skills
|
builtinSkills string // builtin skills
|
||||||
whitelist []string
|
whitelist []string
|
||||||
whitelistEnabled bool
|
whitelistEnabled bool
|
||||||
}
|
}
|
||||||
|
|
@ -90,13 +90,19 @@ func (sl *SkillsLoader) SkillRoots() []string {
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewSkillsLoader(workspace string, globalSkills string, builtinSkills string, whitelist []string, whitelistEnabled bool) *SkillsLoader {
|
func NewSkillsLoader(
|
||||||
|
workspace string,
|
||||||
|
globalSkills string,
|
||||||
|
builtinSkills string,
|
||||||
|
whitelist []string,
|
||||||
|
whitelistEnabled bool,
|
||||||
|
) *SkillsLoader {
|
||||||
return &SkillsLoader{
|
return &SkillsLoader{
|
||||||
workspace: workspace,
|
workspace: workspace,
|
||||||
workspaceSkills: filepath.Join(workspace, "skills"),
|
workspaceSkills: filepath.Join(workspace, "skills"),
|
||||||
globalSkills: globalSkills, // ~/.picoclaw/skills
|
globalSkills: globalSkills, // ~/.picoclaw/skills
|
||||||
builtinSkills: builtinSkills,
|
builtinSkills: builtinSkills,
|
||||||
whitelist: whitelist,
|
whitelist: whitelist,
|
||||||
whitelistEnabled: whitelistEnabled,
|
whitelistEnabled: whitelistEnabled,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -196,7 +202,7 @@ func (sl *SkillsLoader) LoadSkill(name string) (string, bool) {
|
||||||
return sl.stripFrontmatter(string(content)), true
|
return sl.stripFrontmatter(string(content)), true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// ...
|
// ...
|
||||||
|
|
||||||
// 2. then load from global skills (~/.picoclaw/skills)
|
// 2. then load from global skills (~/.picoclaw/skills)
|
||||||
if sl.globalSkills != "" {
|
if sl.globalSkills != "" {
|
||||||
|
|
|
||||||
|
|
@ -417,6 +417,7 @@ func TestGetSkillMetadata_IgnoresHTMLCommentBlocks(t *testing.T) {
|
||||||
assert.Equal(t, "biomed-skill", meta.Name)
|
assert.Equal(t, "biomed-skill", meta.Name)
|
||||||
assert.Equal(t, "Summarize biomedical papers.", meta.Description)
|
assert.Equal(t, "Summarize biomedical papers.", meta.Description)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestListSkillsWithWhitelist(t *testing.T) {
|
func TestListSkillsWithWhitelist(t *testing.T) {
|
||||||
tmp := t.TempDir()
|
tmp := t.TempDir()
|
||||||
ws := filepath.Join(tmp, "workspace")
|
ws := filepath.Join(tmp, "workspace")
|
||||||
|
|
|
||||||
|
|
@ -26,13 +26,18 @@ type InstallSkillTool struct {
|
||||||
// NewInstallSkillTool creates a new InstallSkillTool.
|
// NewInstallSkillTool creates a new InstallSkillTool.
|
||||||
// registryMgr is the shared registry manager (same instance as FindSkillsTool).
|
// registryMgr is the shared registry manager (same instance as FindSkillsTool).
|
||||||
// workspace is the root workspace directory; skills install to {workspace}/skills/{slug}/.
|
// workspace is the root workspace directory; skills install to {workspace}/skills/{slug}/.
|
||||||
func NewInstallSkillTool(registryMgr *skills.RegistryManager, workspace string, whitelist []string, whitelistEnabled bool) *InstallSkillTool {
|
func NewInstallSkillTool(
|
||||||
|
registryMgr *skills.RegistryManager,
|
||||||
|
workspace string,
|
||||||
|
whitelist []string,
|
||||||
|
whitelistEnabled bool,
|
||||||
|
) *InstallSkillTool {
|
||||||
return &InstallSkillTool{
|
return &InstallSkillTool{
|
||||||
registryMgr: registryMgr,
|
registryMgr: registryMgr,
|
||||||
workspace: workspace,
|
workspace: workspace,
|
||||||
whitelist: whitelist,
|
whitelist: whitelist,
|
||||||
whitelistEnabled: whitelistEnabled,
|
whitelistEnabled: whitelistEnabled,
|
||||||
mu: sync.Mutex{},
|
mu: sync.Mutex{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -102,6 +102,7 @@ func TestInstallSkillToolMissingRegistry(t *testing.T) {
|
||||||
assert.True(t, result.IsError)
|
assert.True(t, result.IsError)
|
||||||
assert.Contains(t, result.ForLLM, "invalid registry")
|
assert.Contains(t, result.ForLLM, "invalid registry")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestInstallSkillToolWhitelist(t *testing.T) {
|
func TestInstallSkillToolWhitelist(t *testing.T) {
|
||||||
workspace := t.TempDir()
|
workspace := t.TempDir()
|
||||||
rm := skills.NewRegistryManager()
|
rm := skills.NewRegistryManager()
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,12 @@ type FindSkillsTool struct {
|
||||||
// NewFindSkillsTool creates a new FindSkillsTool.
|
// NewFindSkillsTool creates a new FindSkillsTool.
|
||||||
// registryMgr is the shared registry manager (built from config in createToolRegistry).
|
// registryMgr is the shared registry manager (built from config in createToolRegistry).
|
||||||
// cache is the search cache for deduplicating similar queries.
|
// cache is the search cache for deduplicating similar queries.
|
||||||
func NewFindSkillsTool(registryMgr *skills.RegistryManager, cache *skills.SearchCache, whitelist []string, enabled bool) *FindSkillsTool {
|
func NewFindSkillsTool(
|
||||||
|
registryMgr *skills.RegistryManager,
|
||||||
|
cache *skills.SearchCache,
|
||||||
|
whitelist []string,
|
||||||
|
enabled bool,
|
||||||
|
) *FindSkillsTool {
|
||||||
return &FindSkillsTool{
|
return &FindSkillsTool{
|
||||||
registryMgr: registryMgr,
|
registryMgr: registryMgr,
|
||||||
cache: cache,
|
cache: cache,
|
||||||
|
|
@ -98,7 +103,6 @@ func (t *FindSkillsTool) Execute(ctx context.Context, args map[string]any) *Tool
|
||||||
results = filtered
|
results = filtered
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// Cache the results.
|
// Cache the results.
|
||||||
if t.cache != nil && len(results) > 0 {
|
if t.cache != nil && len(results) > 0 {
|
||||||
t.cache.Put(query, results)
|
t.cache.Put(query, results)
|
||||||
|
|
|
||||||
|
|
@ -83,7 +83,11 @@ build:
|
||||||
# Run all tests
|
# Run all tests
|
||||||
test:
|
test:
|
||||||
cd backend && ${WEB_GO} test ./...
|
cd backend && ${WEB_GO} test ./...
|
||||||
cd frontend && pnpm lint
|
@if command -v pnpm >/dev/null 2>&1; then \
|
||||||
|
cd frontend && pnpm lint; \
|
||||||
|
else \
|
||||||
|
echo "pnpm not found, skipping frontend linting"; \
|
||||||
|
fi
|
||||||
|
|
||||||
# Lint and format
|
# Lint and format
|
||||||
lint:
|
lint:
|
||||||
|
|
|
||||||
|
|
@ -126,8 +126,12 @@ func (h *Handler) handleAddModel(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if mc.APIKey != "" {
|
apiKey := mc.APIKey
|
||||||
mc.ModelConfig.SetAPIKey(mc.APIKey)
|
if apiKey == "" {
|
||||||
|
apiKey = mc.ModelConfig.APIKey()
|
||||||
|
}
|
||||||
|
if apiKey != "" {
|
||||||
|
mc.ModelConfig.SetAPIKey(apiKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg, err := config.LoadConfig(h.configPath)
|
cfg, err := config.LoadConfig(h.configPath)
|
||||||
|
|
@ -197,13 +201,15 @@ func (h *Handler) handleUpdateModel(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Preserve the existing API key when the caller omits it (empty string).
|
apiKey := mc.APIKey
|
||||||
// This lets the UI update api_base / proxy without clearing the stored secret.
|
if apiKey == "" {
|
||||||
if mc.APIKey == "" {
|
apiKey = mc.ModelConfig.APIKey()
|
||||||
mc.ModelConfig.SetAPIKey(cfg.ModelList[idx].APIKey())
|
|
||||||
} else {
|
|
||||||
mc.ModelConfig.SetAPIKey(mc.APIKey)
|
|
||||||
}
|
}
|
||||||
|
if apiKey == "" {
|
||||||
|
apiKey = cfg.ModelList[idx].APIKey()
|
||||||
|
}
|
||||||
|
mc.ModelConfig.SetAPIKey(apiKey)
|
||||||
|
|
||||||
// Preserve existing ExtraBody when omitted (nil), but clear it when
|
// Preserve existing ExtraBody when omitted (nil), but clear it when
|
||||||
// the frontend sends an empty object {} to indicate the field should
|
// the frontend sends an empty object {} to indicate the field should
|
||||||
// be removed.
|
// be removed.
|
||||||
|
|
|
||||||
|
|
@ -194,6 +194,8 @@ func newSkillsLoader(workspace string) *skills.SkillsLoader {
|
||||||
workspace,
|
workspace,
|
||||||
filepath.Join(globalConfigDir(), "skills"),
|
filepath.Join(globalConfigDir(), "skills"),
|
||||||
builtinSkillsDir(),
|
builtinSkillsDir(),
|
||||||
|
nil,
|
||||||
|
false,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue