feat(plugin): add policy demo plugin with runtime enforcement cases
This commit is contained in:
xj
parent
c9ca0835ae
commit
d8cca6810a
3 changed files with 547 additions and 0 deletions
71
docs/design/plugin-code-plan-demo.md
Normal file
71
docs/design/plugin-code-plan-demo.md
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
# Plugin System Demo Code Plan
|
||||
|
||||
## Goal
|
||||
Build a minimal, executable demo to prove plugin value with measurable behavior (not conceptual discussion).
|
||||
|
||||
## Why this demo
|
||||
This demo validates runtime capabilities that skill text cannot reliably enforce:
|
||||
- deterministic tool-call blocking at runtime
|
||||
- deterministic outbound content rewrite at runtime
|
||||
- reversible behavior (no plugin config => no effect)
|
||||
|
||||
## Scope (1-day demo)
|
||||
In scope:
|
||||
- add one compile-time plugin: `policy-demo`
|
||||
- add tests for:
|
||||
- global tool block
|
||||
- channel-specific tool allowlist
|
||||
- outbound redaction
|
||||
- outbound deny-pattern guard
|
||||
- tool argument normalization (timeout clamp)
|
||||
- hook-based audit counters
|
||||
- no-config no-effect path
|
||||
- provide a reviewer-friendly verification command
|
||||
|
||||
Out of scope:
|
||||
- dynamic plugin loading
|
||||
- plugin marketplace/distribution
|
||||
- UI/config schema work
|
||||
|
||||
## Code Changes
|
||||
1. `pkg/plugin/demoplugin/policy_demo.go`
|
||||
- add `PolicyDemoPlugin`
|
||||
- register `before_tool_call` hook to block configured tools
|
||||
- support channel-specific tool allowlist
|
||||
- normalize timeout-like tool args (`timeout`, `timeout_seconds`)
|
||||
- register `message_sending` hook for redaction and deny-pattern guard
|
||||
- register `session_start`, `session_end`, `after_tool_call` audit hooks
|
||||
- expose `Snapshot()` for deterministic verification
|
||||
|
||||
2. `pkg/plugin/demoplugin/policy_demo_test.go`
|
||||
- `TestPolicyDemoPluginBlocksConfiguredTool`
|
||||
- `TestPolicyDemoPluginRedactsOutboundContent`
|
||||
- `TestPolicyDemoPluginChannelAllowlist`
|
||||
- `TestPolicyDemoPluginOutboundGuard`
|
||||
- `TestPolicyDemoPluginNormalizesTimeoutArg`
|
||||
- `TestPolicyDemoPluginAuditHooks`
|
||||
- `TestPolicyDemoPluginNoConfigNoEffect`
|
||||
|
||||
## Verification
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./pkg/plugin/... ./pkg/agent/...
|
||||
```
|
||||
|
||||
Expected:
|
||||
- all tests pass
|
||||
- plugin test suite demonstrates deterministic runtime interception and rewrite
|
||||
|
||||
## Acceptance Criteria
|
||||
- plugin can enforce a hard runtime policy (`before_tool_call` cancel)
|
||||
- plugin can enforce channel-level runtime policy without core code changes
|
||||
- plugin can enforce outbound transformation (`message_sending` rewrite)
|
||||
- plugin can enforce outbound hard-block (`message_sending` cancel)
|
||||
- plugin can mutate tool args before execution in a deterministic way
|
||||
- plugin can collect lifecycle audit counters
|
||||
- empty plugin config causes zero behavior change
|
||||
- behavior is covered by automated tests
|
||||
|
||||
## Reviewer Notes
|
||||
If this demo is accepted, next step is wiring a config-driven enable/disable path (Roadmap Phase 2), while keeping current compile-time contract (`pkg/plugin`) unchanged.
|
||||
303
pkg/plugin/demoplugin/policy_demo.go
Normal file
303
pkg/plugin/demoplugin/policy_demo.go
Normal file
|
|
@ -0,0 +1,303 @@
|
|||
package demoplugin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/sipeed/picoclaw/pkg/hooks"
|
||||
)
|
||||
|
||||
// PolicyDemoConfig controls the demo plugin behavior.
|
||||
type PolicyDemoConfig struct {
|
||||
BlockedTools []string
|
||||
RedactPrefixes []string
|
||||
ChannelToolAllowlist map[string][]string
|
||||
DenyOutboundPatterns []string
|
||||
MaxToolTimeoutSecond int
|
||||
}
|
||||
|
||||
// PolicyDemoStats provides basic evidence that hook paths were executed.
|
||||
type PolicyDemoStats struct {
|
||||
BeforeToolCalls int
|
||||
BlockedToolCalls int
|
||||
MessageSends int
|
||||
RedactedMessages int
|
||||
BlockedMessages int
|
||||
SessionStarts int
|
||||
SessionEnds int
|
||||
AfterToolCalls int
|
||||
TotalToolDuration time.Duration
|
||||
}
|
||||
|
||||
// PolicyDemoPlugin demonstrates why plugins are needed: it enforces runtime policy
|
||||
// at tool-call and outbound-message lifecycle points and collects audit metrics.
|
||||
type PolicyDemoPlugin struct {
|
||||
blockedTools map[string]struct{}
|
||||
prefixes []string
|
||||
channelAllowlist map[string]map[string]struct{}
|
||||
denyPatterns []string
|
||||
maxTimeout int
|
||||
|
||||
mu sync.Mutex
|
||||
stats PolicyDemoStats
|
||||
}
|
||||
|
||||
func NewPolicyDemoPlugin(cfg PolicyDemoConfig) *PolicyDemoPlugin {
|
||||
blocked := make(map[string]struct{}, len(cfg.BlockedTools))
|
||||
for _, t := range cfg.BlockedTools {
|
||||
t = normalizeLower(t)
|
||||
if t == "" {
|
||||
continue
|
||||
}
|
||||
blocked[t] = struct{}{}
|
||||
}
|
||||
|
||||
prefixes := make([]string, 0, len(cfg.RedactPrefixes))
|
||||
for _, p := range cfg.RedactPrefixes {
|
||||
p = strings.TrimSpace(p)
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
prefixes = append(prefixes, p)
|
||||
}
|
||||
|
||||
allowlist := make(map[string]map[string]struct{}, len(cfg.ChannelToolAllowlist))
|
||||
for channel, tools := range cfg.ChannelToolAllowlist {
|
||||
channel = normalizeLower(channel)
|
||||
if channel == "" {
|
||||
continue
|
||||
}
|
||||
toolSet := make(map[string]struct{}, len(tools))
|
||||
for _, t := range tools {
|
||||
t = normalizeLower(t)
|
||||
if t == "" {
|
||||
continue
|
||||
}
|
||||
toolSet[t] = struct{}{}
|
||||
}
|
||||
allowlist[channel] = toolSet
|
||||
}
|
||||
|
||||
patterns := make([]string, 0, len(cfg.DenyOutboundPatterns))
|
||||
for _, p := range cfg.DenyOutboundPatterns {
|
||||
p = strings.TrimSpace(p)
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
patterns = append(patterns, p)
|
||||
}
|
||||
|
||||
maxTimeout := cfg.MaxToolTimeoutSecond
|
||||
if maxTimeout < 0 {
|
||||
maxTimeout = 0
|
||||
}
|
||||
|
||||
return &PolicyDemoPlugin{
|
||||
blockedTools: blocked,
|
||||
prefixes: prefixes,
|
||||
channelAllowlist: allowlist,
|
||||
denyPatterns: patterns,
|
||||
maxTimeout: maxTimeout,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *PolicyDemoPlugin) Name() string {
|
||||
return "policy-demo"
|
||||
}
|
||||
|
||||
func (p *PolicyDemoPlugin) Snapshot() PolicyDemoStats {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.stats
|
||||
}
|
||||
|
||||
func (p *PolicyDemoPlugin) Register(r *hooks.HookRegistry) error {
|
||||
r.OnBeforeToolCall("policy-demo-tool-policy", 100, func(_ context.Context, e *hooks.BeforeToolCallEvent) error {
|
||||
tool := normalizeLower(e.ToolName)
|
||||
p.incBeforeToolCalls()
|
||||
|
||||
if _, blocked := p.blockedTools[tool]; blocked {
|
||||
e.Cancel = true
|
||||
e.CancelReason = "blocked by policy-demo plugin"
|
||||
p.incBlockedToolCalls()
|
||||
return nil
|
||||
}
|
||||
|
||||
channel := normalizeLower(e.Channel)
|
||||
if allow, ok := p.channelAllowlist[channel]; ok {
|
||||
if _, allowed := allow[tool]; !allowed {
|
||||
e.Cancel = true
|
||||
e.CancelReason = fmt.Sprintf("tool %q is not allowed on channel %q", e.ToolName, e.Channel)
|
||||
p.incBlockedToolCalls()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if p.maxTimeout > 0 {
|
||||
clampArgNumber(e.Args, "timeout", p.maxTimeout)
|
||||
clampArgNumber(e.Args, "timeout_seconds", p.maxTimeout)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
r.OnMessageSending("policy-demo-redact-and-guard", 50, func(_ context.Context, e *hooks.MessageSendingEvent) error {
|
||||
p.incMessageSends()
|
||||
|
||||
for _, pattern := range p.denyPatterns {
|
||||
if strings.Contains(e.Content, pattern) {
|
||||
e.Cancel = true
|
||||
e.CancelReason = "blocked by policy-demo outbound guard"
|
||||
p.incBlockedMessages()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
content := e.Content
|
||||
redacted := false
|
||||
for _, prefix := range p.prefixes {
|
||||
next := strings.ReplaceAll(content, prefix, "[redacted]-")
|
||||
if next != content {
|
||||
redacted = true
|
||||
}
|
||||
content = next
|
||||
}
|
||||
e.Content = content
|
||||
if redacted {
|
||||
p.incRedactedMessages()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
r.OnSessionStart("policy-demo-session-start-audit", 0, func(_ context.Context, _ *hooks.SessionEvent) error {
|
||||
p.incSessionStarts()
|
||||
return nil
|
||||
})
|
||||
|
||||
r.OnSessionEnd("policy-demo-session-end-audit", 0, func(_ context.Context, _ *hooks.SessionEvent) error {
|
||||
p.incSessionEnds()
|
||||
return nil
|
||||
})
|
||||
|
||||
r.OnAfterToolCall("policy-demo-after-tool-audit", 0, func(_ context.Context, e *hooks.AfterToolCallEvent) error {
|
||||
p.incAfterToolCall(e.Duration)
|
||||
return nil
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeLower(s string) string {
|
||||
return strings.ToLower(strings.TrimSpace(s))
|
||||
}
|
||||
|
||||
func clampArgNumber(args map[string]any, key string, max int) {
|
||||
if args == nil || max <= 0 {
|
||||
return
|
||||
}
|
||||
v, ok := args[key]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
n, ok := toInt(v)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if n > max {
|
||||
args[key] = max
|
||||
}
|
||||
}
|
||||
|
||||
func toInt(v any) (int, bool) {
|
||||
maxInt := int(^uint(0) >> 1)
|
||||
maxIntU64 := uint64(maxInt)
|
||||
|
||||
switch n := v.(type) {
|
||||
case int:
|
||||
return n, true
|
||||
case int8:
|
||||
return int(n), true
|
||||
case int16:
|
||||
return int(n), true
|
||||
case int32:
|
||||
return int(n), true
|
||||
case int64:
|
||||
return int(n), true
|
||||
case uint:
|
||||
if uint64(n) > maxIntU64 {
|
||||
return 0, false
|
||||
}
|
||||
return int(n), true
|
||||
case uint8:
|
||||
return int(n), true
|
||||
case uint16:
|
||||
return int(n), true
|
||||
case uint32:
|
||||
if uint64(n) > maxIntU64 {
|
||||
return 0, false
|
||||
}
|
||||
return int(n), true
|
||||
case uint64:
|
||||
if n > maxIntU64 {
|
||||
return 0, false
|
||||
}
|
||||
return int(n), true
|
||||
case float32:
|
||||
return int(n), true
|
||||
case float64:
|
||||
return int(n), true
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func (p *PolicyDemoPlugin) incBeforeToolCalls() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.stats.BeforeToolCalls++
|
||||
}
|
||||
|
||||
func (p *PolicyDemoPlugin) incBlockedToolCalls() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.stats.BlockedToolCalls++
|
||||
}
|
||||
|
||||
func (p *PolicyDemoPlugin) incMessageSends() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.stats.MessageSends++
|
||||
}
|
||||
|
||||
func (p *PolicyDemoPlugin) incRedactedMessages() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.stats.RedactedMessages++
|
||||
}
|
||||
|
||||
func (p *PolicyDemoPlugin) incBlockedMessages() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.stats.BlockedMessages++
|
||||
}
|
||||
|
||||
func (p *PolicyDemoPlugin) incSessionStarts() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.stats.SessionStarts++
|
||||
}
|
||||
|
||||
func (p *PolicyDemoPlugin) incSessionEnds() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.stats.SessionEnds++
|
||||
}
|
||||
|
||||
func (p *PolicyDemoPlugin) incAfterToolCall(d time.Duration) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.stats.AfterToolCalls++
|
||||
p.stats.TotalToolDuration += d
|
||||
}
|
||||
173
pkg/plugin/demoplugin/policy_demo_test.go
Normal file
173
pkg/plugin/demoplugin/policy_demo_test.go
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
package demoplugin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/sipeed/picoclaw/pkg/hooks"
|
||||
"github.com/sipeed/picoclaw/pkg/plugin"
|
||||
)
|
||||
|
||||
func TestPolicyDemoPluginBlocksConfiguredTool(t *testing.T) {
|
||||
pm := plugin.NewManager()
|
||||
p := NewPolicyDemoPlugin(PolicyDemoConfig{
|
||||
BlockedTools: []string{"shell"},
|
||||
})
|
||||
if err := pm.Register(p); err != nil {
|
||||
t.Fatalf("register plugin: %v", err)
|
||||
}
|
||||
|
||||
e := &hooks.BeforeToolCallEvent{ToolName: "shell", Args: map[string]any{}, Channel: "cli"}
|
||||
pm.HookRegistry().TriggerBeforeToolCall(context.Background(), e)
|
||||
|
||||
if !e.Cancel {
|
||||
t.Fatal("expected tool call to be canceled")
|
||||
}
|
||||
if e.CancelReason == "" {
|
||||
t.Fatal("expected cancel reason")
|
||||
}
|
||||
|
||||
stats := p.Snapshot()
|
||||
if stats.BeforeToolCalls != 1 || stats.BlockedToolCalls != 1 {
|
||||
t.Fatalf("unexpected stats: %+v", stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPolicyDemoPluginRedactsOutboundContent(t *testing.T) {
|
||||
pm := plugin.NewManager()
|
||||
p := NewPolicyDemoPlugin(PolicyDemoConfig{
|
||||
RedactPrefixes: []string{"sk-"},
|
||||
})
|
||||
if err := pm.Register(p); err != nil {
|
||||
t.Fatalf("register plugin: %v", err)
|
||||
}
|
||||
|
||||
e := &hooks.MessageSendingEvent{Content: "token=sk-abc123"}
|
||||
pm.HookRegistry().TriggerMessageSending(context.Background(), e)
|
||||
|
||||
if e.Cancel {
|
||||
t.Fatal("did not expect cancellation")
|
||||
}
|
||||
if e.Content != "token=[redacted]-abc123" {
|
||||
t.Fatalf("unexpected redaction result: %q", e.Content)
|
||||
}
|
||||
|
||||
stats := p.Snapshot()
|
||||
if stats.MessageSends != 1 || stats.RedactedMessages != 1 {
|
||||
t.Fatalf("unexpected stats: %+v", stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPolicyDemoPluginChannelAllowlist(t *testing.T) {
|
||||
pm := plugin.NewManager()
|
||||
p := NewPolicyDemoPlugin(PolicyDemoConfig{
|
||||
ChannelToolAllowlist: map[string][]string{
|
||||
"telegram": {"web_search"},
|
||||
},
|
||||
})
|
||||
if err := pm.Register(p); err != nil {
|
||||
t.Fatalf("register plugin: %v", err)
|
||||
}
|
||||
|
||||
blocked := &hooks.BeforeToolCallEvent{ToolName: "shell", Args: map[string]any{}, Channel: "telegram"}
|
||||
pm.HookRegistry().TriggerBeforeToolCall(context.Background(), blocked)
|
||||
if !blocked.Cancel {
|
||||
t.Fatal("expected tool to be blocked by channel allowlist")
|
||||
}
|
||||
|
||||
allowed := &hooks.BeforeToolCallEvent{ToolName: "web_search", Args: map[string]any{}, Channel: "telegram"}
|
||||
pm.HookRegistry().TriggerBeforeToolCall(context.Background(), allowed)
|
||||
if allowed.Cancel {
|
||||
t.Fatalf("did not expect allowlisted tool to be blocked: %s", allowed.CancelReason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPolicyDemoPluginOutboundGuard(t *testing.T) {
|
||||
pm := plugin.NewManager()
|
||||
p := NewPolicyDemoPlugin(PolicyDemoConfig{
|
||||
DenyOutboundPatterns: []string{"4111-1111-1111-1111", "@corp.internal"},
|
||||
})
|
||||
if err := pm.Register(p); err != nil {
|
||||
t.Fatalf("register plugin: %v", err)
|
||||
}
|
||||
|
||||
e := &hooks.MessageSendingEvent{Content: "card=4111-1111-1111-1111"}
|
||||
pm.HookRegistry().TriggerMessageSending(context.Background(), e)
|
||||
if !e.Cancel {
|
||||
t.Fatal("expected outbound message to be blocked")
|
||||
}
|
||||
if e.CancelReason == "" {
|
||||
t.Fatal("expected block reason")
|
||||
}
|
||||
|
||||
stats := p.Snapshot()
|
||||
if stats.BlockedMessages != 1 {
|
||||
t.Fatalf("expected blocked message count to be 1, got %+v", stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPolicyDemoPluginNormalizesTimeoutArg(t *testing.T) {
|
||||
pm := plugin.NewManager()
|
||||
p := NewPolicyDemoPlugin(PolicyDemoConfig{MaxToolTimeoutSecond: 30})
|
||||
if err := pm.Register(p); err != nil {
|
||||
t.Fatalf("register plugin: %v", err)
|
||||
}
|
||||
|
||||
e := &hooks.BeforeToolCallEvent{
|
||||
ToolName: "web_fetch",
|
||||
Channel: "cli",
|
||||
Args: map[string]any{
|
||||
"timeout": 120,
|
||||
"timeout_seconds": 90.0,
|
||||
},
|
||||
}
|
||||
pm.HookRegistry().TriggerBeforeToolCall(context.Background(), e)
|
||||
|
||||
if got, ok := e.Args["timeout"].(int); !ok || got != 30 {
|
||||
t.Fatalf("expected timeout to be clamped to 30, got %#v", e.Args["timeout"])
|
||||
}
|
||||
if got, ok := e.Args["timeout_seconds"].(int); !ok || got != 30 {
|
||||
t.Fatalf("expected timeout_seconds to be clamped to 30, got %#v", e.Args["timeout_seconds"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPolicyDemoPluginAuditHooks(t *testing.T) {
|
||||
pm := plugin.NewManager()
|
||||
p := NewPolicyDemoPlugin(PolicyDemoConfig{})
|
||||
if err := pm.Register(p); err != nil {
|
||||
t.Fatalf("register plugin: %v", err)
|
||||
}
|
||||
|
||||
pm.HookRegistry().TriggerSessionStart(context.Background(), &hooks.SessionEvent{AgentID: "a1", SessionKey: "s1"})
|
||||
pm.HookRegistry().TriggerAfterToolCall(context.Background(), &hooks.AfterToolCallEvent{ToolName: "web_search", Duration: 45 * time.Millisecond})
|
||||
pm.HookRegistry().TriggerSessionEnd(context.Background(), &hooks.SessionEvent{AgentID: "a1", SessionKey: "s1"})
|
||||
|
||||
stats := p.Snapshot()
|
||||
if stats.SessionStarts != 1 || stats.SessionEnds != 1 {
|
||||
t.Fatalf("unexpected session stats: %+v", stats)
|
||||
}
|
||||
if stats.AfterToolCalls != 1 || stats.TotalToolDuration != 45*time.Millisecond {
|
||||
t.Fatalf("unexpected after_tool_call stats: %+v", stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPolicyDemoPluginNoConfigNoEffect(t *testing.T) {
|
||||
pm := plugin.NewManager()
|
||||
p := NewPolicyDemoPlugin(PolicyDemoConfig{})
|
||||
if err := pm.Register(p); err != nil {
|
||||
t.Fatalf("register plugin: %v", err)
|
||||
}
|
||||
|
||||
toolEvent := &hooks.BeforeToolCallEvent{ToolName: "shell", Args: map[string]any{}, Channel: "telegram"}
|
||||
pm.HookRegistry().TriggerBeforeToolCall(context.Background(), toolEvent)
|
||||
if toolEvent.Cancel {
|
||||
t.Fatal("did not expect cancellation with empty config")
|
||||
}
|
||||
|
||||
msgEvent := &hooks.MessageSendingEvent{Content: "token=sk-abc123"}
|
||||
pm.HookRegistry().TriggerMessageSending(context.Background(), msgEvent)
|
||||
if msgEvent.Content != "token=sk-abc123" {
|
||||
t.Fatalf("did not expect content rewrite, got %q", msgEvent.Content)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue