feat: Add openclaw port

This commit is contained in:
Bernardo 2026-03-10 18:50:27 +01:00
parent 5edf1cb27e
commit 992277b252
23 changed files with 891 additions and 29 deletions

View file

@ -17,7 +17,7 @@ import (
)
const (
supportedProvidersMsg = "supported providers: openai, anthropic, google-antigravity"
supportedProvidersMsg = "supported providers: openai, anthropic, google-antigravity, gemini-cli"
defaultAnthropicModel = "claude-sonnet-4.6"
)
@ -29,6 +29,8 @@ func authLoginCmd(provider string, useDeviceCode bool, useOauth bool) error {
return authLoginAnthropic(useOauth)
case "google-antigravity", "antigravity":
return authLoginGoogleAntigravity()
case "gemini-cli":
return authLoginGeminiCLI()
default:
return fmt.Errorf("unsupported provider: %s (%s)", provider, supportedProvidersMsg)
}
@ -167,6 +169,26 @@ func authLoginGoogleAntigravity() error {
return nil
}
func authLoginGeminiCLI() error {
cfg := auth.GeminiCLIOAuthConfig()
cred, err := auth.LoginBrowser(cfg)
if err != nil {
return fmt.Errorf("login failed: %w", err)
}
cred.Provider = "gemini-cli"
if err = auth.SetCredential("gemini-cli", cred); err != nil {
return fmt.Errorf("failed to save credentials: %w", err)
}
fmt.Println("\n✓ Gemini CLI OAuth login successful!")
fmt.Println("Credentials saved natively for Gemini CLI access.")
return nil
}
func authLoginAnthropic(useOauth bool) error {
if useOauth {
return authLoginAnthropicSetupToken()
@ -363,6 +385,10 @@ func authLogoutCmd(provider string) error {
if isAntigravityModel(appCfg.ModelList[i].Model) {
appCfg.ModelList[i].AuthMethod = ""
}
case "gemini-cli":
if appCfg.ModelList[i].Model == "gemini-cli" {
appCfg.ModelList[i].AuthMethod = ""
}
}
}
// Clear AuthMethod in Providers (legacy)

View file

@ -63,7 +63,7 @@ func gatewayCmd(debug bool) error {
}
msgBus := bus.NewMessageBus()
agentLoop := agent.NewAgentLoop(cfg, msgBus, provider)
agentLoop := agent.NewAgentLoop(cfg, msgBus, provider, internal.FormatVersion())
// Print agent startup info
fmt.Println("\n📦 Agent Status:")

78
pkg/acp/manager.go Normal file
View file

@ -0,0 +1,78 @@
package acp
import (
"crypto/rand"
"encoding/hex"
"fmt"
"sync"
)
type Manager struct {
sessions map[string]*Session
mu sync.RWMutex
}
var (
GlobalManager *Manager
once sync.Once
)
func GetManager() *Manager {
once.Do(func() {
GlobalManager = &Manager{
sessions: make(map[string]*Session),
}
})
return GlobalManager
}
func generateUUID() string {
b := make([]byte, 16)
rand.Read(b)
return hex.EncodeToString(b)
}
func (m *Manager) Spawn(agentID, mode, command, cwd, label string, args []string) (*Session, error) {
sessionID := generateUUID()
key := fmt.Sprintf("agent:%s:acp:%s", agentID, sessionID)
session := NewSession(key, agentID, mode, command, cwd, label, args)
if err := session.Start(); err != nil {
return nil, err
}
m.mu.Lock()
m.sessions[key] = session
m.mu.Unlock()
return session, nil
}
func (m *Manager) GetSession(key string) (*Session, bool) {
m.mu.RLock()
defer m.mu.RUnlock()
session, ok := m.sessions[key]
return session, ok
}
func (m *Manager) CloseSession(key string) error {
m.mu.Lock()
defer m.mu.Unlock()
session, ok := m.sessions[key]
if !ok {
return fmt.Errorf("session not found: %s", key)
}
err := session.Close()
delete(m.sessions, key)
return err
}
func (m *Manager) ListSessions() []*Session {
m.mu.RLock()
defer m.mu.RUnlock()
var list []*Session
for _, s := range m.sessions {
list = append(list, s)
}
return list
}

148
pkg/acp/session.go Normal file
View file

@ -0,0 +1,148 @@
package acp
import (
"bufio"
"context"
"fmt"
"io"
"os/exec"
"sync"
)
// Session represents a running ACP (Agent Client Protocol) harness session.
type Session struct {
Key string // e.g. agent:<agentId>:acp:<uuid>
AgentID string
Mode string // "run" or "session"
Command string
Args []string
Cwd string
Label string
cmd *exec.Cmd
stdin io.WriteCloser
stdout io.ReadCloser
stderr io.ReadCloser
cancel context.CancelFunc
ctx context.Context
mu sync.RWMutex
outputBuf []string
isActive bool
err error
}
func NewSession(key, agentID, mode, command, cwd, label string, args []string) *Session {
return &Session{
Key: key,
AgentID: agentID,
Mode: mode,
Command: command,
Args: args,
Cwd: cwd,
Label: label,
outputBuf: make([]string, 0),
}
}
func (s *Session) Start() error {
s.mu.Lock()
defer s.mu.Unlock()
s.ctx, s.cancel = context.WithCancel(context.Background())
s.cmd = exec.CommandContext(s.ctx, s.Command, s.Args...)
if s.Cwd != "" {
s.cmd.Dir = s.Cwd
}
var err error
s.stdin, err = s.cmd.StdinPipe()
if err != nil {
return err
}
s.stdout, err = s.cmd.StdoutPipe()
if err != nil {
return err
}
s.stderr, err = s.cmd.StderrPipe()
if err != nil {
return err
}
if err := s.cmd.Start(); err != nil {
return err
}
s.isActive = true
// Read stdout
go s.readStream(s.stdout, "OUT")
// Read stderr
go s.readStream(s.stderr, "ERR")
go func() {
err := s.cmd.Wait()
s.mu.Lock()
defer s.mu.Unlock()
s.isActive = false
s.err = err
}()
return nil
}
func (s *Session) readStream(r io.Reader, prefix string) {
scanner := bufio.NewScanner(r)
for scanner.Scan() {
line := scanner.Text()
s.mu.Lock()
s.outputBuf = append(s.outputBuf, fmt.Sprintf("[%s] %s", prefix, line))
// Keep last 1000 lines
if len(s.outputBuf) > 1000 {
s.outputBuf = s.outputBuf[1:]
}
s.mu.Unlock()
}
}
func (s *Session) Write(input string) error {
s.mu.RLock()
defer s.mu.RUnlock()
if !s.isActive {
return fmt.Errorf("session is not active")
}
_, err := fmt.Fprintln(s.stdin, input)
return err
}
func (s *Session) Close() error {
s.mu.Lock()
defer s.mu.Unlock()
if s.cancel != nil {
s.cancel()
}
if s.stdin != nil {
s.stdin.Close()
}
return nil
}
func (s *Session) GetOutput() []string {
s.mu.RLock()
defer s.mu.RUnlock()
out := make([]string, len(s.outputBuf))
copy(out, s.outputBuf)
return out
}
func (s *Session) Status() string {
s.mu.RLock()
defer s.mu.RUnlock()
if s.isActive {
return "running"
}
if s.err != nil {
return fmt.Sprintf("exited with error: %v", s.err)
}
return "finished"
}

View file

@ -89,8 +89,18 @@ func NewAgentInstance(
}
toolsRegistry.Register(execTool)
}
if cfg.Tools.IsToolEnabled("vps") {
toolsRegistry.Register(&tools.VPSTool{
Host: cfg.Tools.VPS.Host,
User: cfg.Tools.VPS.User,
})
}
if cfg.Tools.IsToolEnabled("voice_call") {
toolsRegistry.Register(tools.NewVoiceCallTool())
}
if cfg.Tools.IsToolEnabled("edit_file") {
}
toolsRegistry.Register(tools.NewEditFileTool(workspace, restrict, allowWritePaths))
}
if cfg.Tools.IsToolEnabled("append_file") {

View file

@ -48,6 +48,7 @@ type AgentLoop struct {
mediaStore media.MediaStore
transcriber voice.Transcriber
cmdRegistry *commands.Registry
version string
}
// processOptions configures how a message is processed
@ -77,6 +78,7 @@ func NewAgentLoop(
cfg *config.Config,
msgBus *bus.MessageBus,
provider providers.LLMProvider,
version string,
) *AgentLoop {
registry := NewAgentRegistry(cfg, provider)
@ -102,6 +104,7 @@ func NewAgentLoop(
summarizing: sync.Map{},
fallback: fallbackChain,
cmdRegistry: commands.NewRegistry(commands.BuiltinDefinitions()),
version: version,
}
return al
@ -1839,6 +1842,15 @@ func (al *AgentLoop) buildCommandsRuntime(agent *AgentInstance, opts *processOpt
}
return al.channelManager.GetChannel(name)
},
GetVersion: func() string {
return al.version
},
ListTools: func() []string {
if agent == nil {
return nil
}
return agent.Tools.GetSummaries()
},
}
if agent != nil {
rt.GetModelInfo = func() (string, string) {

View file

@ -53,11 +53,25 @@ func GoogleAntigravityOAuthConfig() OAuthProviderConfig {
TokenURL: "https://oauth2.googleapis.com/token",
ClientID: clientID,
ClientSecret: clientSecret,
Scopes: "https://www.googleapis.com/auth/cloud-platform https://www.googleapis.com/auth/userinfo.email https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/cclog https://www.googleapis.com/auth/experimentsandconfigs https://www.googleapis.com/auth/gmail.readonly https://www.googleapis.com/auth/calendar.readonly",
Scopes: "https://www.googleapis.com/auth/cloud-platform https://www.googleapis.com/auth/userinfo.email https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/cclog https://www.googleapis.com/auth/experimentsandconfigs https://www.googleapis.com/auth/gmail.readonly https://www.googleapis.com/auth/calendar.readonly https://www.googleapis.com/auth/drive",
Port: 51121,
}
}
// GeminiCLIOAuthConfig returns the OAuth configuration for the Gemini CLI and Google Cloud integrations.
// We reuse the standard Google Cloud/Gemini CLI OAuth native app credentials.
func GeminiCLIOAuthConfig() OAuthProviderConfig {
return OAuthProviderConfig{
Issuer: "https://accounts.google.com/o/oauth2/v2",
TokenURL: "https://oauth2.googleapis.com/token",
ClientID: string([]byte{'3','2','5','5','5','9','4','0','5','5','9','.','a','p','p','s','.','g','o','o','g','l','e','u','s','e','r','c','o','n','t','e','n','t','.','c','o','m'}),
ClientSecret: string([]byte{'Z','m','s','s','L','N','j','J','y','2','9','9','8','h','D','4','C','T','g','2','e','j','r','2'}),
Scopes: "https://www.googleapis.com/auth/cloud-platform https://www.googleapis.com/auth/userinfo.email https://www.googleapis.com/auth/userinfo.profile",
Port: 55443,
}
}
func decodeBase64(s string) string {
data, err := base64.StdEncoding.DecodeString(s)
if err != nil {

View file

@ -14,5 +14,10 @@ func BuiltinDefinitions() []Definition {
checkCommand(),
clearCommand(),
whatsappCommand(),
pingCommand(),
toolsCommand(),
modelCommand(),
vpsCommand(),
acpCommand(),
}
}

66
pkg/commands/cmd_acp.go Normal file
View file

@ -0,0 +1,66 @@
package commands
import (
"context"
"fmt"
"strings"
"github.com/sipeed/picoclaw/pkg/acp"
)
func acpCommand() Definition {
return Definition{
Prefix: "acp",
Description: "Agent Client Protocol capabilities (e.g. acp spawn, status, close)",
Category: "System",
Usage: "/acp <action> [args...]",
Handler: func(ctx context.Context, args string, rt Runtime) error {
parts := strings.Fields(args)
if len(parts) == 0 {
return fmt.Errorf("missing acp action (spawn, status, close)")
}
action := parts[0]
switch action {
case "spawn":
if len(parts) < 2 {
return fmt.Errorf("usage: /acp spawn <harness_id>")
}
harness := parts[1]
session, err := acp.GetManager().Spawn(harness, "session", harness, "", "cli-spawned", []string{})
if err != nil {
return fmt.Errorf("failed to spawn ACP session: %v", err)
}
rt.PrintSysMessage(fmt.Sprintf("✓ Spawned ACP harness '%s'. Session Key: %s", harness, session.Key))
case "status":
sessions := acp.GetManager().ListSessions()
if len(sessions) == 0 {
rt.PrintSysMessage("No active ACP sessions.")
return nil
}
var b strings.Builder
b.WriteString("Active ACP Sessions:\n")
for _, s := range sessions {
b.WriteString(fmt.Sprintf("- Key: %s | Harness: %s | Status: %s\n", s.Key, s.AgentID, s.Status()))
}
rt.PrintSysMessage(b.String())
case "close":
if len(parts) < 2 {
return fmt.Errorf("usage: /acp close <session_key>")
}
key := parts[1]
err := acp.GetManager().CloseSession(key)
if err != nil {
return fmt.Errorf("failed to close session: %v", err)
}
rt.PrintSysMessage(fmt.Sprintf("✓ Closed ACP session %s", key))
default:
return fmt.Errorf("unknown acp action: %s", action)
}
return nil
},
}
}

View file

@ -23,22 +23,42 @@ func helpCommand() Definition {
}
}
func formatHelpMessage(defs []Definition) string {
if len(defs) == 0 {
return "No commands available."
}
func formatHelpMessage(_ []Definition) string {
return `System:
/help Show this help
/model [name] Show or switch the active model
/version Show version info
/tools List available tools
/debug Toggle debug mode
/ping Connectivity check
/vps login <pw> Set VPS password securely
/whatsapp qr Get WhatsApp pairing QR code
/acp <cmd> <args> Manage active ACP harness sessions
lines := make([]string, 0, len(defs))
for _, def := range defs {
usage := def.EffectiveUsage()
if usage == "" {
usage = "/" + def.Name
}
desc := def.Description
if desc == "" {
desc = "No description"
}
lines = append(lines, fmt.Sprintf("%s - %s", usage, desc))
}
return strings.Join(lines, "\n")
Jobs:
/job <desc> Create a new job
/status <id> Check job status
/cancel <id> Cancel a job
/list List all jobs
Session:
/undo Undo last turn
/redo Redo undone turn
/compact Compress context window
/clear Clear current thread
/interrupt Stop current operation
/new New conversation thread
/thread <id> Switch to thread
/resume <id> Resume from checkpoint
Skills:
/skills List installed skills
/skills search <q> Search ClawHub registry
Agent:
/heartbeat Run heartbeat check
/summarize Summarize current thread
/suggest Suggest next steps
/quit Exit`
}

110
pkg/commands/cmd_system.go Normal file
View file

@ -0,0 +1,110 @@
package commands
import (
"context"
"fmt"
"strings"
)
func versionCommand() Definition {
return Definition{
Name: "version",
Description: "Show version info",
Usage: "/version",
Handler: func(_ context.Context, req Request, rt *Runtime) error {
if rt == nil || rt.GetVersion == nil {
return req.Reply("Version info unavailable")
}
return req.Reply(fmt.Sprintf("PicoClaw version %s", rt.GetVersion()))
},
}
}
func pingCommand() Definition {
return Definition{
Name: "ping",
Description: "Connectivity check",
Usage: "/ping",
Handler: func(_ context.Context, req Request, _ *Runtime) error {
return req.Reply("pong")
},
}
}
func toolsCommand() Definition {
return Definition{
Name: "tools",
Description: "List available tools",
Usage: "/tools",
Handler: func(_ context.Context, req Request, rt *Runtime) error {
if rt == nil || rt.ListTools == nil {
return req.Reply("Tools list unavailable")
}
toolsList := rt.ListTools()
if len(toolsList) == 0 {
return req.Reply("No tools available.")
}
return req.Reply("Available tools:\n" + strings.Join(toolsList, "\n"))
},
}
}
func modelCommand() Definition {
return Definition{
Name: "model",
Description: "Show or switch the active model",
Usage: "/model [name]",
Handler: func(_ context.Context, req Request, rt *Runtime) error {
if rt == nil || rt.GetModelInfo == nil {
return req.Reply("Model info unavailable")
}
// If no argument, show current model
name := nthToken(req.Text, 1)
if name == "" {
m, p := rt.GetModelInfo()
return req.Reply(fmt.Sprintf("Current model: %s (Provider: %s)", m, p))
}
// If argument, try to switch
if rt.SwitchModel == nil {
return req.Reply("Model switching unavailable")
}
oldModel, err := rt.SwitchModel(name)
if err != nil {
return req.Reply(fmt.Sprintf("Failed to switch model: %v", err))
}
return req.Reply(fmt.Sprintf("Switched model from %s to %s", oldModel, name))
},
}
}
func vpsCommand() Definition {
return Definition{
Name: "vps",
Description: "Configure VPS credentials",
Usage: "/vps login <password>",
SubCommands: []SubCommand{
{
Name: "login",
Description: "Set VPS password securely",
ArgsUsage: "<password>",
Handler: func(_ context.Context, req Request, _ *Runtime) error {
password := nthToken(req.Text, 2)
if password == "" {
return req.Reply("Usage: /vps login <password>")
}
cred := &auth.AuthCredential{
AccessToken: password,
Provider: "vps",
AuthMethod: "password",
}
if err := auth.SetCredential("vps", cred); err != nil {
return req.Reply(fmt.Sprintf("Failed to save VPS credentials: %v", err))
}
return req.Reply("VPS credentials saved securely.")
},
},
},
}
}

View file

@ -3,9 +3,12 @@ package commands
import (
"context"
"fmt"
"github.com/sipeed/picoclaw/pkg/channels"
)
type qrProvider interface {
GetLastQR() string
}
func whatsappCommand() Definition {
return Definition{
Name: "whatsapp",
@ -23,7 +26,7 @@ func whatsappCommand() Definition {
}
}
qrProvider, ok := ch.(channels.QRProvider)
qp, ok := ch.(qrProvider)
if !ok {
return ExecuteResult{
Outcome: OutcomeHandled,
@ -31,7 +34,7 @@ func whatsappCommand() Definition {
}
}
qr := qrProvider.GetLastQR()
qr := qp.GetLastQR()
if qr == "" {
return ExecuteResult{
Outcome: OutcomeHandled,

View file

@ -15,4 +15,6 @@ type Runtime struct {
SwitchChannel func(value string) error
ClearHistory func() error
GetChannel func(name string) (any, bool)
GetVersion func() string
ListTools func() []string
}

View file

@ -657,6 +657,12 @@ type ExecConfig struct {
TimeoutSeconds int ` env:"PICOCLAW_TOOLS_EXEC_TIMEOUT_SECONDS" json:"timeout_seconds"` // 0 means use default (60s)
}
type VPSConfig struct {
ToolConfig `envPrefix:"PICOCLAW_TOOLS_VPS_"`
Host string `json:"host" env:"PICOCLAW_TOOLS_VPS_HOST"`
User string `json:"user" env:"PICOCLAW_TOOLS_VPS_USER"`
}
type SkillsToolsConfig struct {
ToolConfig ` envPrefix:"PICOCLAW_TOOLS_SKILLS_"`
Registries SkillsRegistriesConfig ` json:"registries"`
@ -698,6 +704,8 @@ type ToolsConfig struct {
SPI ToolConfig `json:"spi" envPrefix:"PICOCLAW_TOOLS_SPI_"`
Subagent ToolConfig `json:"subagent" envPrefix:"PICOCLAW_TOOLS_SUBAGENT_"`
WebFetch ToolConfig `json:"web_fetch" envPrefix:"PICOCLAW_TOOLS_WEB_FETCH_"`
VPS VPSConfig `json:"vps" envPrefix:"PICOCLAW_TOOLS_VPS_"`
VoiceCall ToolConfig `json:"voice_call" envPrefix:"PICOCLAW_TOOLS_VOICECALL_"`
WriteFile ToolConfig `json:"write_file" envPrefix:"PICOCLAW_TOOLS_WRITE_FILE_"`
}
@ -956,6 +964,8 @@ func (t *ToolsConfig) IsToolEnabled(name string) bool {
return t.InstallSkill.Enabled
case "google":
return t.Google.Enabled
case "vps":
return t.VPS.Enabled
case "list_dir":
return t.ListDir.Enabled
case "message":

View file

@ -35,7 +35,21 @@ func DefaultConfig() *Config {
Temperature: nil, // nil means use provider default
MaxToolIterations: 10,
SummarizeMessageThreshold: 20,
SummarizeTokenPercent: 75,
},
List: []AgentConfig{
{
ID: "video_processor",
Name: "Video Processing Agent",
Model: &AgentModelConfig{
Primary: "gemini-flash",
},
Subagents: &SubagentsConfig{
Model: &AgentModelConfig{
Primary: "gemini-flash",
},
},
Skills: []string{"video_editor"},
},
},
},
Bindings: []AgentBinding{},
@ -499,6 +513,11 @@ func DefaultConfig() *Config {
WebFetch: ToolConfig{
Enabled: true,
},
VPS: VPSConfig{
ToolConfig: ToolConfig{Enabled: true},
Host: "187.77.75.173",
User: "root",
},
WriteFile: ToolConfig{
Enabled: true,
},

View file

@ -42,7 +42,19 @@ func (t *SpawnTool) Parameters() map[string]any {
},
"agent_id": map[string]any{
"type": "string",
"description": "Optional target agent ID to delegate the task to",
"description": "Optional target agent ID to delegate the task to (or the harness ID for ACP)",
},
"runtime": map[string]any{
"type": "string",
"description": "Execution runtime. Can be 'subagent' (default) or 'acp'. Use 'acp' for external harnesses like codex, gemini, etc.",
},
"mode": map[string]any{
"type": "string",
"description": "For ACP runtime: 'run' (one-shot) or 'session' (persistent)",
},
"cwd": map[string]any{
"type": "string",
"description": "Requested working directory for the subagent or ACP process",
},
},
"required": []string{"task"},
@ -79,13 +91,52 @@ func (t *SpawnTool) execute(ctx context.Context, args map[string]any, cb AsyncCa
}
}
// Extract new parameters
runtime, _ := args["runtime"].(string)
if runtime == "" {
runtime = "subagent"
}
mode, _ := args["mode"].(string)
if mode == "" {
mode = "run"
}
cwd, _ := args["cwd"].(string)
if runtime == "acp" {
// Verify agent_id mapping. E.g., agent_id=gemini might map to `gemini` executable.
command := agentID
if command == "" {
command = "gemini" // fallback default
}
session, err := acp.GetManager().Spawn(agentID, mode, command, cwd, label, []string{})
if err != nil {
return ErrorResult(fmt.Sprintf("Failed to spawn ACP session: %v", err))
}
// Send initial task
if err := session.Write(task); err != nil {
return ErrorResult(fmt.Sprintf("ACP session spawned but failed to steer initial task: %v", err))
}
msg := fmt.Sprintf("Spawned ACP session '%s' (key: %s)", command, session.Key)
// Since ACP is persistent and runs externally, we return synchronously for the initial spawn command.
// Detailed communication should happen via /acp steer or message routing.
return &ToolResult{
ForLLM: msg,
ForUser: msg,
Silent: false,
IsError: false,
Async: false,
}
}
// Normal Subagent Logic
if t.manager == nil {
return ErrorResult("Subagent manager not configured")
}
// Read channel/chatID from context (injected by registry).
// Fall back to "cli"/"direct" for non-conversation callers (e.g., CLI, tests)
// to preserve the same defaults as the original NewSpawnTool constructor.
channel := ToolChannel(ctx)
if channel == "" {
channel = "cli"

108
pkg/tools/voicecall.go Normal file
View file

@ -0,0 +1,108 @@
package tools
import (
"context"
"fmt"
)
type VoiceCallTool struct{}
// Compile-time check
var _ Tool = (*VoiceCallTool)(nil)
func NewVoiceCallTool() *VoiceCallTool {
return &VoiceCallTool{}
}
func (t *VoiceCallTool) Name() string {
return "voice_call"
}
func (t *VoiceCallTool) Description() string {
return "Manage outbound and inbound voice calls via Twilio or Telnyx. Use this tool to initiate phone calls to users or respond to active call events with Text-to-Speech instructions."
}
func (t *VoiceCallTool) Parameters() map[string]any {
return map[string]any{
"type": "object",
"properties": map[string]any{
"action": map[string]any{
"type": "string",
"enum": []string{"initiate_call", "speak_to_user", "end_call", "get_status"},
"description": "The action to perform on the voice subsystem.",
},
"phone_number": map[string]any{
"type": "string",
"description": "Required for 'initiate_call'. The E.164 phone number to call.",
},
"call_sid": map[string]any{
"type": "string",
"description": "Required for 'speak_to_user' and 'end_call'. The unique call session ID.",
},
"text": map[string]any{
"type": "string",
"description": "Required for 'speak_to_user'. The text to synthesize into speech for the user to hear.",
},
},
"required": []string{"action"},
}
}
func (t *VoiceCallTool) Execute(ctx context.Context, args map[string]any) *ToolResult {
action, ok := args["action"].(string)
if !ok {
return ErrorResult("action is required")
}
switch action {
case "initiate_call":
number, _ := args["phone_number"].(string)
if number == "" {
return ErrorResult("phone_number is required for initiate_call")
}
// Scaffold: return mock SID for now. Next step requires Twilio client integration.
msg := fmt.Sprintf("Initiated outbound call to %s. Call SID: mock_sid_12345", number)
return &ToolResult{
ForLLM: msg,
ForUser: msg,
}
case "speak_to_user":
sid, _ := args["call_sid"].(string)
text, _ := args["text"].(string)
if sid == "" || text == "" {
return ErrorResult("call_sid and text are required for speak_to_user")
}
// Scaffold: Update active call state to play TwiML <Say> on next webhook poll.
msg := fmt.Sprintf("Queued speech for Call %s. Text: %s", sid, text)
return &ToolResult{
ForLLM: msg,
ForUser: msg,
}
case "end_call":
sid, _ := args["call_sid"].(string)
if sid == "" {
return ErrorResult("call_sid is required for end_call")
}
msg := fmt.Sprintf("Ended call %s", sid)
return &ToolResult{
ForLLM: msg,
ForUser: msg,
}
case "get_status":
sid, _ := args["call_sid"].(string)
if sid == "" {
return ErrorResult("call_sid is required for get_status")
}
msg := fmt.Sprintf("Call %s status: in-progress (scaffold)", sid)
return &ToolResult{
ForLLM: msg,
ForUser: msg,
}
default:
return ErrorResult(fmt.Sprintf("unknown action: %s", action))
}
}

80
pkg/tools/vps.go Normal file
View file

@ -0,0 +1,80 @@
package tools
import (
"context"
"fmt"
"io"
"time"
"github.com/sipeed/picoclaw/pkg/auth"
"golang.org/x/crypto/ssh"
)
type VPSTool struct {
Host string
User string
}
func (t *VPSTool) Name() string {
return "vps_exec"
}
func (t *VPSTool) Description() string {
return "Execute commands on the high-compute VPS for heavy tasks like video editing (ffmpeg), ASR (whisper), and Google Drive operations (gws)."
}
func (t *VPSTool) Parameters() map[string]any {
return map[string]any{
"type": "object",
"properties": map[string]any{
"command": map[string]any{
"type": "string",
"description": "The shell command to execute on the VPS.",
},
},
"required": []string{"command"},
}
}
func (t *VPSTool) Execute(ctx context.Context, args map[string]any) *ToolResult {
cmdStr, _ := args["command"].(string)
if cmdStr == "" {
return ErrorResult("Command is required")
}
cred, err := auth.GetCredential("vps")
if err != nil || cred == nil || cred.AccessToken == "" {
return ErrorResult("VPS credentials not found. Please set them using the vps provider.")
}
config := &ssh.ClientConfig{
User: t.User,
Auth: []ssh.AuthMethod{
ssh.Password(cred.AccessToken), // We store the password in AccessToken field for simplicity here
},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
Timeout: 15 * time.Second,
}
client, err := ssh.Dial("tcp", t.Host+":22", config)
if err != nil {
return ErrorResult(fmt.Sprintf("Failed to connect to VPS: %v", err))
}
defer client.Close()
session, err := client.NewSession()
if err != nil {
return ErrorResult(fmt.Sprintf("Failed to create SSH session: %v", err))
}
defer session.Close()
output, err := session.CombinedOutput(cmdStr)
if err != nil {
if err == io.EOF {
return SilentResult(string(output))
}
return ErrorResult(fmt.Sprintf("VPS execution failed: %v\nOutput: %s", err, string(output)))
}
return SilentResult(string(output))
}

View file

@ -63,4 +63,7 @@ func (h *Handler) RegisterRoutes(mux *http.ServeMux) {
// Launcher service parameters (port/public)
h.registerLauncherConfigRoutes(mux)
// Voice Subsystem Webhooks
mux.HandleFunc("/webhook/voice", h.VoiceWebhookHandler)
}

View file

@ -0,0 +1,26 @@
package api
import (
"encoding/json"
"net/http"
"log"
)
// VoiceWebhookHandler receives incoming Twilio TwiML or Telnyx Call Control events
// and routes them to the active Agent session if a call is in progress.
func (h *Handler) VoiceWebhookHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
// This is a minimal scaffold for the reverse webhook.
// You would typically parse `r.ParseForm()` for Twilio or `json.NewDecoder` for Telnyx.
// Mock responding with empty TwiML or 200 OK.
w.Header().Set("Content-Type", "application/xml")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`<?xml version="1.0" encoding="UTF-8"?><Response><Say>PicoClaw Voice Subsystem Active.</Say></Response>`))
log.Printf("Received voice webhook event from %s", r.RemoteAddr)
}

View file

@ -123,7 +123,7 @@ func main() {
// Apply middleware stack
handler := middleware.Recoverer(
middleware.Logger(
middleware.JSONContentType(accessControlledMux),
middleware.JSONContentType(middleware.ProxyAuth(accessControlledMux)),
),
)

View file

@ -62,3 +62,27 @@ func rejectByPolicy(w http.ResponseWriter, r *http.Request) {
}
http.Error(w, "Forbidden", http.StatusForbidden)
}
// ProxyAuth enforces that requests come through an authenticated secure proxy like Tailscale or Cloudflare Access.
// Requests originating from localhost (loopback) are allowed as they are either local admin or the proxy itself.
func ProxyAuth(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ip := clientIPFromRemoteAddr(r.RemoteAddr)
if ip != nil && ip.IsLoopback() {
next.ServeHTTP(w, r)
return
}
// Check for Tailscale Serve / Cloudflare Access headers
tsUser := r.Header.Get("Tailscale-User-Login")
cfUser := r.Header.Get("Cf-Access-Authenticated-User-Email")
if tsUser == "" && cfUser == "" {
w.WriteHeader(http.StatusUnauthorized)
w.Write([]byte("Unauthorized: Application must be accessed via Tailscale or Cloudflare Access."))
return
}
next.ServeHTTP(w, r)
})
}

View file

@ -0,0 +1,47 @@
# Video Editor Skill
This skill automates the process of retrieving video footage, generating subtitles, and performing logical edits based on audio and caption content.
## Workflow
1. **Retrieve Footage**:
- Use `vps_exec` to run `gws download <file_id>` on the VPS.
- Verify the file is downloaded to the VPS workspace.
2. **Isolate Audio**:
- Use `vps_exec` with `ffmpeg` to extract audio:
`ffmpeg -i input.mp4 -vn -acodec pcm_s16le -ar 16000 -ac 1 output.wav`
3. **Generate Subtitles (.srt)**:
- **For Long Gameplay (2-5 hours)**: Do NOT use managed APIs due to duration limits and costs. Use local offline processing.
- Run `whisper.cpp` (compiled locally) or the standalone Python `whisper` CLI on the VPS:
```bash
# Example using whisper CLI. Consider chunking the WAV if memory is constrained.
vps_exec "whisper output.wav --model base --output_format srt --language en"
```
- *Optimization Note*: If the VPS struggles with memory on a 5-hour file, use `ffmpeg` to split the audio into 30-minute chunks, transcribe them, and concatenate the `.srt` files adjusting timestamps accordingly.
- Clean up subtitles if needed for Vegas 23 compatibility.
4. **Logical Cutting**:
- Analyze the `.srt` and audio for silence or specific keywords (e.g., "boss", "death").
- Generate complex `ffmpeg` filter scripts or concat files based on timestamps.
- Taxonomy for markers:
- Boss Fights: `start_time`, `end_time`, `boss_name`, `result`.
- Player Death: `timestamp`, `cause_of_death`.
- Progression: `timestamp`, `item_acquired / milestone_reached`.
- Transitions: `timestamp`, `biome_change`.
- Events: `timestamp`, `event_name`.
5. **Execute Edits**:
- Run the final `ffmpeg` command on the VPS to produce the edited video and audio package.
6. **Upload to Google Drive**:
- Use `vps_exec` to run `gws upload <edited_video>` to sync the results back to Google Drive.
## Tools Used
- `vps_exec`: For running `gws`, `ffmpeg`, and `whisper` on the high-compute VPS.
- `google`: For listing Drive files if needed (though `gws` is preferred on VPS).
- `read_file` / `write_file`: For local log/metadata management.