From 03dae1301d7eaff8e52ba53a57666310c384c9e0 Mon Sep 17 00:00:00 2001 From: anthrodjear Date: Wed, 6 May 2026 15:07:28 +0300 Subject: [PATCH 1/3] added the windows build without make --- AGENTS.md | 49 ++++++++++++++++++ CLAUDE.md | 12 +++++ config/config.example.json | 20 +++++++- scripts/build-without-make.bat | 91 ++++++++++++++++++++++++++++++++++ scripts/build-without-make.sh | 52 +++++++++++++++++++ 5 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 AGENTS.md create mode 100644 scripts/build-without-make.bat create mode 100644 scripts/build-without-make.sh diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..a6f769e91 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,49 @@ +Microsoft Windows [Version 10.0.26200.8116] +(c) Microsoft Corporation. All rights reserved. + +C:\Users\user>ollama list +NAME ID SIZE MODIFIED +smollm2:1.7b cef4a1e09247 1.8 GB 5 hours ago +qwen3.5:4b 2a654d98e6fb 3.4 GB 3 days ago +deepseek-v4-pro:cloud 22bfd5026abd - 3 days ago +qwen3-coder-next:cloud aa626c11ae8d - 2 months ago + +C:\Users\user> + +# PicoClaw + +## Build & Test Commands + +```bash +make build # Build for current platform (runs generate first) +make build-all # Cross-compile for all supported platforms +make test # Run Go tests + web tests +make lint # golangci-lint with goolm,stdjson tags +make check # deps + fmt + vet + test + lint-docs +``` + +## Environment Setup + +- Copy `.env.example` to `.env` and configure API keys (ANTHROPIC_API_KEY, OPENAI_API_KEY, etc.) +- Go 1.25.9+ required +- Build tags: `goolm,stdjson` (set via GO_BUILD_TAGS or Makefile) +- CGO_ENABLED=0 by default; CGO_ENABLED=1 only for macOS launcher builds + +## Critical Constraints + +- **Always run `make generate` before `make build`** — code generation creates required workspace symlinks +- **Never edit `cmd/picoclaw/workspace/` directly** — it's regenerated by `go generate` +- **MIPS builds require ELF e_flags patch** — handled automatically by Makefile +- **loong64 needs manual ztypes_loong64.go** — handled automatically by Makefile +- **WhatsApp native builds** add `whatsapp_native` tag but produce larger binaries +- **Workspace location**: `~/.picoclaw/workspace` (skills, memory stored here at runtime) +- **macOS launcher**: requires CGO_ENABLED=1 and minimal macOS 10.11 target +- **No hardcoded API keys in CLI** — project is migrating to OAuth 2.0 flows +- **Memory target**: core process <20MB for 64MB RAM boards; optimize data structures over storage + +## Architecture Notes +- **Protocol-first**: Migrating from vendor-based to protocol-based provider classification (OpenAI-compat, Ollama-compat) +- **Multi-architecture**: x86_64, ARM64, MIPS, RISC-V, LoongArch +- **14+ chat channels** via adapter pattern in pkg/channels/ +- **MCP support**: Model Context Protocol server in pkg/mcp/ +- **Tools API**: See `docs/reference/tools-api.md` for complete tools documentation diff --git a/CLAUDE.md b/CLAUDE.md index 0492f7381..a6f769e91 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,3 +1,15 @@ +Microsoft Windows [Version 10.0.26200.8116] +(c) Microsoft Corporation. All rights reserved. + +C:\Users\user>ollama list +NAME ID SIZE MODIFIED +smollm2:1.7b cef4a1e09247 1.8 GB 5 hours ago +qwen3.5:4b 2a654d98e6fb 3.4 GB 3 days ago +deepseek-v4-pro:cloud 22bfd5026abd - 3 days ago +qwen3-coder-next:cloud aa626c11ae8d - 2 months ago + +C:\Users\user> + # PicoClaw ## Build & Test Commands diff --git a/config/config.example.json b/config/config.example.json index 910c4fbd3..6180e5308 100644 --- a/config/config.example.json +++ b/config/config.example.json @@ -3,7 +3,7 @@ "defaults": { "workspace": "~/.picoclaw/workspace", "restrict_to_workspace": true, - "model_name": "gpt-5.4", + "model_name": "smollm2", "max_tokens": 8192, "context_window": 131072, "temperature": 0.7, @@ -21,6 +21,24 @@ } }, "model_list": [ + { + "model_name": "smollm2", + "provider": "ollama", + "model": "smollm2:1.7b", + "api_base": "http://localhost:11434/v1" + }, + { + "model_name": "qwen3.5", + "provider": "ollama", + "model": "qwen3.5:4b", + "api_base": "http://localhost:11434/v1" + }, + { + "model_name": "deepseek-v4-pro", + "provider": "ollama", + "model": "deepseek-v4-pro:cloud", + "api_base": "http://localhost:11434/v1" + }, { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", diff --git a/scripts/build-without-make.bat b/scripts/build-without-make.bat new file mode 100644 index 000000000..1bba226a5 --- /dev/null +++ b/scripts/build-without-make.bat @@ -0,0 +1,91 @@ +@echo off +REM Build PicoClaw core and web launcher without using make. +REM Usage: scripts\build-without-make.bat + +SETLOCAL ENABLEEXTENSIONS + +SET "REPO_ROOT=%~dp0.." +SET "REPO_ROOT=%REPO_ROOT:~0,-1%" +SET "GO_TAGS=goolm,stdjson" + +REM Ensure Go is available +where go >nul 2>&1 +IF ERRORLEVEL 1 ( + echo ERROR: go is not installed or not on PATH. + EXIT /B 1 +) + +REM Ensure pnpm is available +where pnpm >nul 2>&1 +IF ERRORLEVEL 1 ( + echo ERROR: pnpm is not installed or not on PATH. + EXIT /B 1 +) + +PUSHD "%REPO_ROOT%" +IF ERRORLEVEL 1 ( + echo ERROR: Failed to change directory to "%REPO_ROOT%". + EXIT /B 1 +) + +IF NOT EXIST build ( + mkdir build +) + +echo === Generating Go code === +go generate ./... +IF ERRORLEVEL 1 ( + echo ERROR: go generate failed. + POPD + EXIT /B 1 +) + +echo. +echo === Building PicoClaw core binary === +go build -tags "%GO_TAGS%" -o build\picoclaw.exe .\cmd\picoclaw +IF ERRORLEVEL 1 ( + echo ERROR: go build failed for core binary. + POPD + EXIT /B 1 +) + +echo. +echo === Building PicoClaw web frontend assets === +PUSHD "%REPO_ROOT%\web\frontend" +IF ERRORLEVEL 1 ( + echo ERROR: Failed to change directory to web\frontend. + POPD + EXIT /B 1 +) +pnpm install --frozen-lockfile +IF ERRORLEVEL 1 ( + echo ERROR: pnpm install failed. + POPD + POPD + EXIT /B 1 +) +pnpm build:backend +IF ERRORLEVEL 1 ( + echo ERROR: pnpm build:backend failed. + POPD + POPD + EXIT /B 1 +) +POPd + +echo. +echo === Building PicoClaw web launcher === +go build -tags "%GO_TAGS%" -o build\picoclaw-launcher.exe .\web\backend +IF ERRORLEVEL 1 ( + echo ERROR: go build failed for web launcher. + POPD + EXIT /B 1 +) + +echo. +echo === Build complete === +echo Core binary: %REPO_ROOT%\build\picoclaw.exe +echo Web launcher: %REPO_ROOT%\build\picoclaw-launcher.exe + +POPD +EXIT /B 0 diff --git a/scripts/build-without-make.sh b/scripts/build-without-make.sh new file mode 100644 index 000000000..13e8897b7 --- /dev/null +++ b/scripts/build-without-make.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Build PicoClaw core and web launcher without using make. +# Usage: ./scripts/build-without-make.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd)" +cd "${REPO_ROOT}" + +GO_TAGS="goolm,stdjson" +EXE_EXT="" +UNAME_S="$(uname -s 2>/dev/null || echo unknown)" +case "${UNAME_S}" in + *MINGW*|*MSYS*|*CYGWIN*|*Windows_NT*) + EXE_EXT=".exe" + ;; +esac + +command -v go >/dev/null 2>&1 || { + echo "ERROR: go is not installed or not on PATH." + exit 1 +} +command -v pnpm >/dev/null 2>&1 || { + echo "ERROR: pnpm is not installed or not on PATH." + exit 1 +} + +mkdir -p "${REPO_ROOT}/build" + +echo "=== Generating Go code ===" +go generate ./... + +echo "" +echo "=== Building PicoClaw core binary ===" +go build -tags "${GO_TAGS}" -o "${REPO_ROOT}/build/picoclaw${EXE_EXT}" ./cmd/picoclaw + +echo "" +echo "=== Building PicoClaw web frontend assets ===" +cd "${REPO_ROOT}/web/frontend" +pnpm install --frozen-lockfile +pnpm build:backend + +echo "" +echo "=== Building PicoClaw web launcher ===" +cd "${REPO_ROOT}" +go build -tags "${GO_TAGS}" -o "${REPO_ROOT}/build/picoclaw-launcher${EXE_EXT}" ./web/backend + +echo "" +echo "=== Build complete ===" +echo "Core binary: ${REPO_ROOT}/build/picoclaw${EXE_EXT}" +echo "Web launcher: ${REPO_ROOT}/build/picoclaw-launcher${EXE_EXT}" From b8b231964cfd7a4a819297ac8a01af091f29f802 Mon Sep 17 00:00:00 2001 From: anthrodjear Date: Wed, 6 May 2026 16:20:27 +0300 Subject: [PATCH 2/3] Build automation and development improvements - Add build-without-make.bat script for Windows builds without make - Update .gitignore to exclude .gocache and dist directories - Fix scripts/build-without-make.bat to properly handle repository root resolution - Add CALL directive for pnpm commands in batch scripts - Agent system improvements and health server enhancements - Add agent cockpit UI components and memory graph visualization - API improvements for PicoClaw web backend - Update frontend routing and app sidebar --- .gitignore | 2 + pkg/agent/agent.go | 2 + pkg/agent/agent_init.go | 3 + pkg/agent/agent_inject.go | 26 + pkg/gateway/gateway.go | 7 + pkg/health/server.go | 55 +- pkg/health/server_test.go | 54 ++ scripts/build-without-make.bat | 15 +- web/backend/api/pico.go | 472 +++++++++++++ web/backend/api/pico_test.go | 185 +++++ web/frontend/src/api/pico.ts | 55 ++ .../components/agent/cockpit/cockpit-page.tsx | 651 ++++++++++++++++++ .../components/agent/cockpit/memory-graph.tsx | 231 +++++++ .../agent/cockpit/use-agent-cockpit.ts | 136 ++++ web/frontend/src/components/app-sidebar.tsx | 7 + web/frontend/src/routeTree.gen.ts | 21 + web/frontend/src/routes/agent/cockpit.tsx | 11 + 17 files changed, 1928 insertions(+), 5 deletions(-) create mode 100644 web/frontend/src/components/agent/cockpit/cockpit-page.tsx create mode 100644 web/frontend/src/components/agent/cockpit/memory-graph.tsx create mode 100644 web/frontend/src/components/agent/cockpit/use-agent-cockpit.ts create mode 100644 web/frontend/src/routes/agent/cockpit.tsx diff --git a/.gitignore b/.gitignore index e1736f56b..0f2d17141 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ build/ /picoclaw /picoclaw-test cmd/**/workspace +.gocache/ +dist/ # Picoclaw specific diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 84849aece..c69eb444f 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -28,6 +28,7 @@ import ( "github.com/sipeed/picoclaw/pkg/routing" "github.com/sipeed/picoclaw/pkg/session" "github.com/sipeed/picoclaw/pkg/state" + "github.com/sipeed/picoclaw/pkg/tools" "github.com/sipeed/picoclaw/pkg/utils" ) @@ -57,6 +58,7 @@ type AgentLoop struct { mcp mcpRuntime hookRuntime hookRuntime steering *steeringQueue + subagents *tools.SubagentManager pendingSkills sync.Map mu sync.RWMutex diff --git a/pkg/agent/agent_init.go b/pkg/agent/agent_init.go index 76f12fa65..556b3097f 100644 --- a/pkg/agent/agent_init.go +++ b/pkg/agent/agent_init.go @@ -242,6 +242,9 @@ func registerSharedTools( if (spawnEnabled || spawnStatusEnabled) && cfg.Tools.IsToolEnabled("subagent") { subagentManager := tools.NewSubagentManager(provider, agent.Model, agent.Workspace) subagentManager.SetLLMOptions(agent.MaxTokens, agent.Temperature) + if agentID == "main" { + al.subagents = subagentManager + } // Inject a media resolver so the legacy RunToolLoop fallback path can // resolve media:// refs in the same way the main AgentLoop does. diff --git a/pkg/agent/agent_inject.go b/pkg/agent/agent_inject.go index 5609bb4c5..222601521 100644 --- a/pkg/agent/agent_inject.go +++ b/pkg/agent/agent_inject.go @@ -119,3 +119,29 @@ func (al *AgentLoop) GetStartupInfo() map[string]any { return info } + +func (al *AgentLoop) GetMainSubagentTasks(channel, chatID string) []tools.SubagentTask { + al.mu.RLock() + manager := al.subagents + al.mu.RUnlock() + if manager == nil { + return nil + } + + all := manager.ListTaskCopies() + if channel == "" && chatID == "" { + return all + } + + filtered := make([]tools.SubagentTask, 0, len(all)) + for _, task := range all { + if channel != "" && task.OriginChannel != "" && task.OriginChannel != channel { + continue + } + if chatID != "" && task.OriginChatID != "" && task.OriginChatID != chatID { + continue + } + filtered = append(filtered, task) + } + return filtered +} diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go index c9fac5615..44973be6c 100644 --- a/pkg/gateway/gateway.go +++ b/pkg/gateway/gateway.go @@ -246,6 +246,13 @@ func Run(debug bool, homePath, configPath string, allowEmptyStartup bool) (runEr runningServices.HealthServer.SetPermissionGrantFunc(func(agentID, path, duration string) error { return agentLoop.GrantPermission(agentID, path, duration) }) + runningServices.HealthServer.SetSubagentStatusFunc(func(channel, chatID string) (any, error) { + return map[string]any{ + "channel": channel, + "chat_id": chatID, + "tasks": agentLoop.GetMainSubagentTasks(channel, chatID), + }, nil + }) for _, bindHost := range listenResult.BindHosts { fmt.Printf("✓ Gateway started on %s\n", net.JoinHostPort(bindHost, strconv.Itoa(cfg.Gateway.Port))) diff --git a/pkg/health/server.go b/pkg/health/server.go index 540b35466..580a7749f 100644 --- a/pkg/health/server.go +++ b/pkg/health/server.go @@ -19,8 +19,9 @@ type Server struct { ready bool checks map[string]Check startTime time.Time - reloadFunc func() error + reloadFunc func() error permissionGrantFunc func(agentID, path, duration string) error + subagentStatusFunc func(channel, chatID string) (any, error) authToken string // optional bearer token for protected endpoints } @@ -51,6 +52,7 @@ func NewServer(host string, port int, token string) *Server { mux.HandleFunc("/ready", s.readyHandler) mux.HandleFunc("/reload", s.reloadHandler) mux.HandleFunc("/internal/permission/grant", s.permissionGrantHandler) + mux.HandleFunc("/internal/subagents/status", s.subagentStatusHandler) addr := net.JoinHostPort(host, strconv.Itoa(port)) s.server = &http.Server{ @@ -128,6 +130,12 @@ func (s *Server) SetPermissionGrantFunc(fn func(agentID, path, duration string) s.permissionGrantFunc = fn } +func (s *Server) SetSubagentStatusFunc(fn func(channel, chatID string) (any, error)) { + s.mu.Lock() + defer s.mu.Unlock() + s.subagentStatusFunc = fn +} + // permissionGrantHandler handles POST /internal/permission/grant requests. func (s *Server) permissionGrantHandler(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { @@ -241,6 +249,49 @@ func (s *Server) reloadHandler(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(map[string]string{"status": "reload triggered"}) } +func (s *Server) subagentStatusHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusMethodNotAllowed) + json.NewEncoder(w).Encode(map[string]string{"error": "method not allowed, use GET"}) + return + } + + s.mu.RLock() + requiredToken := s.authToken + statusFunc := s.subagentStatusFunc + s.mu.RUnlock() + + if requiredToken != "" { + given := extractBearerToken(r.Header.Get("Authorization")) + if given == "" || subtle.ConstantTimeCompare([]byte(given), []byte(requiredToken)) != 1 { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + json.NewEncoder(w).Encode(map[string]string{"error": "unauthorized"}) + return + } + } + + if statusFunc == nil { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + json.NewEncoder(w).Encode(map[string]string{"error": "subagent status not configured"}) + return + } + + payload, err := statusFunc(r.URL.Query().Get("channel"), r.URL.Query().Get("chat_id")) + if err != nil { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusInternalServerError) + json.NewEncoder(w).Encode(map[string]string{"error": err.Error()}) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(payload) +} + func (s *Server) healthHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) @@ -307,6 +358,8 @@ func (s *Server) RegisterOnMux(mux HandlerMux) { mux.HandleFunc("/health", s.healthHandler) mux.HandleFunc("/ready", s.readyHandler) mux.HandleFunc("/reload", s.reloadHandler) + mux.HandleFunc("/internal/permission/grant", s.permissionGrantHandler) + mux.HandleFunc("/internal/subagents/status", s.subagentStatusHandler) } func statusString(ok bool) string { diff --git a/pkg/health/server_test.go b/pkg/health/server_test.go index 31dbc37c0..0a189f47a 100644 --- a/pkg/health/server_test.go +++ b/pkg/health/server_test.go @@ -215,6 +215,60 @@ func TestReloadHandler_Error(t *testing.T) { } } +func TestSubagentStatusHandler_Success(t *testing.T) { + s := newTestServer() + s.SetSubagentStatusFunc(func(channel, chatID string) (any, error) { + return map[string]any{ + "channel": channel, + "chat_id": chatID, + "tasks": []map[string]any{ + {"id": "subagent-1", "status": "running"}, + }, + }, nil + }) + + req := httptest.NewRequest(http.MethodGet, "/internal/subagents/status?channel=pico&chat_id=session-1", nil) + req.Header.Set("Authorization", "Bearer test") + w := httptest.NewRecorder() + + s.subagentStatusHandler(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", w.Code, http.StatusOK, w.Body.String()) + } + + var resp struct { + Channel string `json:"channel"` + ChatID string `json:"chat_id"` + Tasks []map[string]any `json:"tasks"` + } + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if resp.Channel != "pico" || resp.ChatID != "session-1" { + t.Fatalf("response = %#v, want channel/chat_id preserved", resp) + } + if len(resp.Tasks) != 1 { + t.Fatalf("tasks len = %d, want 1", len(resp.Tasks)) + } +} + +func TestSubagentStatusHandler_RequiresAuth(t *testing.T) { + s := newTestServer() + s.SetSubagentStatusFunc(func(channel, chatID string) (any, error) { + return map[string]any{}, nil + }) + + req := httptest.NewRequest(http.MethodGet, "/internal/subagents/status", nil) + w := httptest.NewRecorder() + + s.subagentStatusHandler(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want %d", w.Code, http.StatusUnauthorized) + } +} + func TestSetReady_Toggle(t *testing.T) { s := newTestServer() diff --git a/scripts/build-without-make.bat b/scripts/build-without-make.bat index 1bba226a5..c5004add7 100644 --- a/scripts/build-without-make.bat +++ b/scripts/build-without-make.bat @@ -4,10 +4,17 @@ REM Usage: scripts\build-without-make.bat SETLOCAL ENABLEEXTENSIONS -SET "REPO_ROOT=%~dp0.." -SET "REPO_ROOT=%REPO_ROOT:~0,-1%" SET "GO_TAGS=goolm,stdjson" +REM Resolve the repository root directory from the script location. +PUSHD "%~dp0.." +IF ERRORLEVEL 1 ( + echo ERROR: Failed to resolve repository root from "%~dp0..". + EXIT /B 1 +) +SET "REPO_ROOT=%CD%" +POPD + REM Ensure Go is available where go >nul 2>&1 IF ERRORLEVEL 1 ( @@ -57,14 +64,14 @@ IF ERRORLEVEL 1 ( POPD EXIT /B 1 ) -pnpm install --frozen-lockfile +CALL pnpm install --frozen-lockfile IF ERRORLEVEL 1 ( echo ERROR: pnpm install failed. POPD POPD EXIT /B 1 ) -pnpm build:backend +CALL pnpm build:backend IF ERRORLEVEL 1 ( echo ERROR: pnpm build:backend failed. POPD diff --git a/web/backend/api/pico.go b/web/backend/api/pico.go index 8eeff4041..08a027e27 100644 --- a/web/backend/api/pico.go +++ b/web/backend/api/pico.go @@ -5,8 +5,14 @@ import ( "encoding/hex" "encoding/json" "fmt" + "io" "net/http" "net/http/httputil" + "os" + "path/filepath" + "sort" + "strconv" + "strings" "time" "github.com/sipeed/picoclaw/pkg/config" @@ -17,6 +23,8 @@ import ( // registerPicoRoutes binds Pico Channel management endpoints to the ServeMux. func (h *Handler) registerPicoRoutes(mux *http.ServeMux) { mux.HandleFunc("GET /api/pico/info", h.handleGetPicoInfo) + mux.HandleFunc("GET /api/pico/memory-graph", h.handleGetPicoMemoryGraph) + mux.HandleFunc("GET /api/pico/subagents", h.handleGetPicoSubagents) mux.HandleFunc("POST /api/pico/token", h.handleRegenPicoToken) mux.HandleFunc("POST /api/pico/setup", h.handlePicoSetup) @@ -28,6 +36,43 @@ func (h *Handler) registerPicoRoutes(mux *http.ServeMux) { mux.HandleFunc("HEAD /pico/media/{id}", h.handlePicoMediaProxy()) } +type picoSubagentStatusItem struct { + ID string `json:"id"` + Label string `json:"label,omitempty"` + Status string `json:"status"` + Created int64 `json:"created"` + Result string `json:"result,omitempty"` +} + +type picoSubagentStatusResponse struct { + SessionID string `json:"session_id"` + Channel string `json:"channel"` + ChatID string `json:"chat_id"` + Tasks []picoSubagentStatusItem `json:"tasks"` +} + +type picoMemoryGraphNode struct { + ID string `json:"id"` + Label string `json:"label"` + Kind string `json:"kind"` + Group string `json:"group"` + Preview string `json:"preview,omitempty"` + Weight int `json:"weight,omitempty"` +} + +type picoMemoryGraphEdge struct { + Source string `json:"source"` + Target string `json:"target"` + Kind string `json:"kind"` +} + +type picoMemoryGraphResponse struct { + SessionID string `json:"session_id"` + GeneratedAt string `json:"generated_at"` + Nodes []picoMemoryGraphNode `json:"nodes"` + Edges []picoMemoryGraphEdge `json:"edges"` +} + // createWsProxy creates a reverse proxy to the current gateway WebSocket endpoint. // The gateway bind host and port are resolved from the latest configuration. func (h *Handler) createWsProxy(origProtocol string, upstreamProtocol string) *httputil.ReverseProxy { @@ -207,6 +252,433 @@ func (h *Handler) handleGetPicoInfo(w http.ResponseWriter, r *http.Request) { h.writePicoInfoResponse(w, r, cfg, nil) } +func (h *Handler) handleGetPicoMemoryGraph(w http.ResponseWriter, r *http.Request) { + sessionID := strings.TrimSpace(r.URL.Query().Get("session_id")) + if sessionID == "" { + http.Error(w, "session_id is required", http.StatusBadRequest) + return + } + + cfg, err := config.LoadConfig(h.configPath) + if err != nil { + http.Error(w, "failed to load config", http.StatusInternalServerError) + return + } + + sessionDir := resolveSessionsDir(cfg.Agents.Defaults.Workspace) + toolFeedbackMaxArgsLength := cfg.Agents.Defaults.GetToolFeedbackMaxArgsLength() + workspaceDir := resolveWorkspaceDir(cfg.Agents.Defaults.Workspace) + + ref, refErr := h.findPicoJSONLSession(sessionDir, sessionID) + var sess sessionFile + err = refErr + if refErr == nil { + sess, err = h.readJSONLSession(sessionDir, ref.Key) + } + if err != nil { + http.Error(w, "session not found", http.StatusNotFound) + return + } + + nodes, edges := buildPicoMemoryGraph( + sessionID, + detailSessionMessages(sess.Messages, toolFeedbackMaxArgsLength), + workspaceDir, + ) + + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(picoMemoryGraphResponse{ + SessionID: sessionID, + GeneratedAt: time.Now().UTC().Format(time.RFC3339), + Nodes: nodes, + Edges: edges, + }) +} + +func (h *Handler) handleGetPicoSubagents(w http.ResponseWriter, r *http.Request) { + sessionID := strings.TrimSpace(r.URL.Query().Get("session_id")) + if sessionID == "" { + http.Error(w, "session_id is required", http.StatusBadRequest) + return + } + + if !h.gatewayAvailableForProxy() { + logger.Warnf("Gateway not available for Pico subagent status proxy") + http.Error(w, "Gateway not available", http.StatusServiceUnavailable) + return + } + + gateway.mu.Lock() + pidData := gateway.pidData + gateway.mu.Unlock() + + if pidData == nil || pidData.Token == "" { + logger.Warnf("Gateway auth token not available for Pico subagent status proxy") + http.Error(w, "Gateway auth token not available", http.StatusServiceUnavailable) + return + } + + target := h.gatewayProxyURL() + target.Path = "/internal/subagents/status" + query := target.Query() + query.Set("channel", "pico") + query.Set("chat_id", sessionID) + target.RawQuery = query.Encode() + + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, target.String(), nil) + if err != nil { + http.Error(w, "Failed to create subagent status request", http.StatusInternalServerError) + return + } + req.Header.Set("Authorization", "Bearer "+pidData.Token) + + resp, err := (&http.Client{Timeout: 5 * time.Second}).Do(req) + if err != nil { + logger.Errorf("Failed to fetch Pico subagent status: %v", err) + http.Error(w, "Gateway unavailable: "+err.Error(), http.StatusBadGateway) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) + return + } + + var upstream struct { + Channel string `json:"channel"` + ChatID string `json:"chat_id"` + Tasks []struct { + ID string `json:"id"` + Label string `json:"label"` + Status string `json:"status"` + Created int64 `json:"created"` + Result string `json:"result"` + } `json:"tasks"` + } + if err := json.NewDecoder(resp.Body).Decode(&upstream); err != nil { + http.Error(w, "Failed to decode subagent status response", http.StatusBadGateway) + return + } + + tasks := make([]picoSubagentStatusItem, 0, len(upstream.Tasks)) + for _, task := range upstream.Tasks { + tasks = append(tasks, picoSubagentStatusItem{ + ID: task.ID, + Label: task.Label, + Status: task.Status, + Created: task.Created, + Result: summarizeSubagentResult(task.Result), + }) + } + + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(picoSubagentStatusResponse{ + SessionID: sessionID, + Channel: upstream.Channel, + ChatID: upstream.ChatID, + Tasks: tasks, + }) +} + +func summarizeSubagentResult(result string) string { + result = strings.TrimSpace(result) + if result == "" { + return "" + } + const maxRunes = 180 + runes := []rune(result) + if len(runes) <= maxRunes { + return result + } + return string(runes[:maxRunes]) + "..." +} + +func resolveWorkspaceDir(workspace string) string { + if workspace == "" { + home, _ := os.UserHomeDir() + workspace = filepath.Join(home, ".picoclaw", "workspace") + } + if len(workspace) > 0 && workspace[0] == '~' { + home, _ := os.UserHomeDir() + if len(workspace) > 1 && workspace[1] == '/' { + workspace = home + workspace[1:] + } else { + workspace = home + } + } + return workspace +} + +func buildPicoMemoryGraph(sessionID string, messages []sessionChatMessage, workspaceDir string) ([]picoMemoryGraphNode, []picoMemoryGraphEdge) { + nodes := make([]picoMemoryGraphNode, 0, 32) + edges := make([]picoMemoryGraphEdge, 0, 48) + seenNodes := make(map[string]struct{}) + seenEdges := make(map[string]struct{}) + + addNode := func(node picoMemoryGraphNode) { + if _, exists := seenNodes[node.ID]; exists { + return + } + seenNodes[node.ID] = struct{}{} + nodes = append(nodes, node) + } + + addEdge := func(edge picoMemoryGraphEdge) { + key := edge.Source + "\x00" + edge.Target + "\x00" + edge.Kind + if _, exists := seenEdges[key]; exists { + return + } + seenEdges[key] = struct{}{} + edges = append(edges, edge) + } + + const ( + memoryRootID = "memory-root" + sessionRootID = "session-root" + ) + + addNode(picoMemoryGraphNode{ + ID: memoryRootID, + Label: "Workspace Memory", + Kind: "root", + Group: "memory", + Preview: "Long-term memory and recent notes", + Weight: 5, + }) + addNode(picoMemoryGraphNode{ + ID: sessionRootID, + Label: "Active Session", + Kind: "root", + Group: "session", + Preview: sessionID, + Weight: 5, + }) + addEdge(picoMemoryGraphEdge{Source: memoryRootID, Target: sessionRootID, Kind: "context"}) + + appendMemoryDocumentGraph(memoryRootID, filepath.Join(workspaceDir, "memory", "MEMORY.md"), "memory", "Long-term Memory", 8, addNode, addEdge) + appendRecentDailyNoteGraph(memoryRootID, filepath.Join(workspaceDir, "memory"), addNode, addEdge) + appendSessionGraph(sessionRootID, messages, addNode, addEdge) + + sort.Slice(nodes, func(i, j int) bool { + return nodes[i].ID < nodes[j].ID + }) + sort.Slice(edges, func(i, j int) bool { + if edges[i].Source == edges[j].Source { + if edges[i].Target == edges[j].Target { + return edges[i].Kind < edges[j].Kind + } + return edges[i].Target < edges[j].Target + } + return edges[i].Source < edges[j].Source + }) + + return nodes, edges +} + +func appendMemoryDocumentGraph( + rootID string, + path string, + group string, + title string, + maxItems int, + addNode func(picoMemoryGraphNode), + addEdge func(picoMemoryGraphEdge), +) { + data, err := os.ReadFile(path) + if err != nil || len(data) == 0 { + return + } + + docID := group + ":document" + addNode(picoMemoryGraphNode{ + ID: docID, + Label: title, + Kind: "document", + Group: group, + Preview: filepath.Base(path), + Weight: 4, + }) + addEdge(picoMemoryGraphEdge{Source: rootID, Target: docID, Kind: "contains"}) + + lines := strings.Split(string(data), "\n") + currentParent := docID + items := 0 + headingIndex := 0 + noteIndex := 0 + + for _, rawLine := range lines { + line := strings.TrimSpace(rawLine) + if line == "" { + continue + } + + if strings.HasPrefix(line, "#") { + headingText := summarizeGraphText(strings.TrimSpace(strings.TrimLeft(line, "#")), 48) + if headingText == "" { + continue + } + headingIndex++ + headingID := group + ":heading:" + strconv.Itoa(headingIndex) + addNode(picoMemoryGraphNode{ + ID: headingID, + Label: headingText, + Kind: "heading", + Group: group, + Preview: headingText, + Weight: 3, + }) + addEdge(picoMemoryGraphEdge{Source: docID, Target: headingID, Kind: "section"}) + currentParent = headingID + continue + } + + trimmed := strings.TrimSpace(strings.TrimLeft(line, "-*0123456789. ")) + if trimmed == "" { + continue + } + + noteIndex++ + noteID := group + ":note:" + strconv.Itoa(noteIndex) + addNode(picoMemoryGraphNode{ + ID: noteID, + Label: summarizeGraphText(trimmed, 38), + Kind: "note", + Group: group, + Preview: summarizeGraphText(trimmed, 140), + Weight: 2, + }) + addEdge(picoMemoryGraphEdge{Source: currentParent, Target: noteID, Kind: "note"}) + items++ + if items >= maxItems { + break + } + } +} + +func appendRecentDailyNoteGraph( + rootID string, + memoryDir string, + addNode func(picoMemoryGraphNode), + addEdge func(picoMemoryGraphEdge), +) { + matches, err := filepath.Glob(filepath.Join(memoryDir, "*", "*.md")) + if err != nil || len(matches) == 0 { + return + } + sort.Slice(matches, func(i, j int) bool { + return matches[i] > matches[j] + }) + + for idx, match := range matches { + if idx >= 3 { + break + } + appendMemoryDocumentGraph( + rootID, + match, + "daily-"+strconv.Itoa(idx+1), + "Daily Note "+filepath.Base(match), + 4, + addNode, + addEdge, + ) + } +} + +func appendSessionGraph( + rootID string, + messages []sessionChatMessage, + addNode func(picoMemoryGraphNode), + addEdge func(picoMemoryGraphEdge), +) { + if len(messages) == 0 { + return + } + + start := 0 + if len(messages) > 8 { + start = len(messages) - 8 + } + recent := messages[start:] + previousID := rootID + + for index, message := range recent { + messageID := "session:message:" + strconv.Itoa(index) + label := strings.ToUpper(message.Role) + if preview := summarizeGraphText(message.Content, 34); preview != "" { + label += ": " + preview + } + if label == strings.ToUpper(message.Role) && len(message.Attachments) > 0 { + label += ": attachment" + } + + addNode(picoMemoryGraphNode{ + ID: messageID, + Label: label, + Kind: "message", + Group: "session", + Preview: summarizeGraphText(message.Content, 160), + Weight: 3, + }) + addEdge(picoMemoryGraphEdge{Source: previousID, Target: messageID, Kind: "flow"}) + previousID = messageID + + for toolIndex, toolCall := range message.ToolCalls { + if toolCall.Function == nil { + continue + } + name := strings.TrimSpace(toolCall.Function.Name) + if name == "" { + continue + } + toolID := messageID + ":tool:" + strconv.Itoa(toolIndex) + addNode(picoMemoryGraphNode{ + ID: toolID, + Label: name, + Kind: "tool", + Group: "tool", + Preview: summarizeGraphText(toolCall.Function.Arguments, 120), + Weight: 2, + }) + addEdge(picoMemoryGraphEdge{Source: messageID, Target: toolID, Kind: "tool"}) + } + + for attachmentIndex, attachment := range message.Attachments { + name := strings.TrimSpace(attachment.Filename) + if name == "" { + name = attachment.Type + } + if name == "" { + name = "attachment" + } + attachmentID := messageID + ":attachment:" + strconv.Itoa(attachmentIndex) + addNode(picoMemoryGraphNode{ + ID: attachmentID, + Label: summarizeGraphText(name, 34), + Kind: "attachment", + Group: "media", + Preview: summarizeGraphText(attachment.URL, 120), + Weight: 1, + }) + addEdge(picoMemoryGraphEdge{Source: messageID, Target: attachmentID, Kind: "attachment"}) + } + } +} + +func summarizeGraphText(text string, maxRunes int) string { + trimmed := strings.Join(strings.Fields(strings.TrimSpace(text)), " ") + if trimmed == "" { + return "" + } + runes := []rune(trimmed) + if len(runes) <= maxRunes { + return trimmed + } + return string(runes[:maxRunes-1]) + "…" +} + // handleRegenPicoToken rotates the raw Pico WebSocket token and returns // non-secret connection info for the launcher UI. // diff --git a/web/backend/api/pico_test.go b/web/backend/api/pico_test.go index 6f7cefd4d..2906c6f73 100644 --- a/web/backend/api/pico_test.go +++ b/web/backend/api/pico_test.go @@ -11,9 +11,12 @@ import ( "strconv" "strings" "testing" + "time" "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/memory" ppid "github.com/sipeed/picoclaw/pkg/pid" + "github.com/sipeed/picoclaw/pkg/providers" ) func newPicoProxyRequest(method, path string) *http.Request { @@ -821,6 +824,188 @@ func TestHandlePicoMediaProxyUsesRawBearerToken(t *testing.T) { } } +func TestHandleGetPicoMemoryGraph_BuildsWorkspaceAndSessionGraph(t *testing.T) { + configPath := filepath.Join(t.TempDir(), "config.json") + workspaceDir := filepath.Join(t.TempDir(), "workspace") + sessionsDir := filepath.Join(workspaceDir, "sessions") + memoryDir := filepath.Join(workspaceDir, "memory") + + if err := os.MkdirAll(memoryDir, 0o755); err != nil { + t.Fatalf("MkdirAll(memoryDir) error = %v", err) + } + if err := os.MkdirAll(filepath.Join(memoryDir, time.Now().Format("200601")), 0o755); err != nil { + t.Fatalf("MkdirAll(dailyNoteDir) error = %v", err) + } + + cfg := config.DefaultConfig() + cfg.Agents.Defaults.Workspace = workspaceDir + if err := config.SaveConfig(configPath, cfg); err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + store, err := memory.NewJSONLStore(sessionsDir) + if err != nil { + t.Fatalf("NewJSONLStore() error = %v", err) + } + + sessionKey := legacyPicoSessionPrefix + "graph-session" + for _, msg := range []providers.Message{ + {Role: "user", Content: "Remember the Nairobi deployment notes."}, + {Role: "assistant", Content: "Saved the deployment note and linked it to workspace memory."}, + } { + if err := store.AddFullMessage(nil, sessionKey, msg); err != nil { + t.Fatalf("AddFullMessage() error = %v", err) + } + } + if err := store.SetSummary(nil, sessionKey, "Graph session"); err != nil { + t.Fatalf("SetSummary() error = %v", err) + } + + if err := os.WriteFile( + filepath.Join(memoryDir, "MEMORY.md"), + []byte("# Preferences\n- User prefers network graph views\n# Projects\n- PicoClaw cockpit integration"), + 0o644, + ); err != nil { + t.Fatalf("WriteFile(MEMORY.md) error = %v", err) + } + + todayPath := filepath.Join(memoryDir, time.Now().Format("200601"), time.Now().Format("20060102")+".md") + if err := os.WriteFile( + todayPath, + []byte("# Daily\n- Reviewed active session memory graph"), + 0o644, + ); err != nil { + t.Fatalf("WriteFile(todayPath) error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + req := httptest.NewRequest(http.MethodGet, "/api/pico/memory-graph?session_id=graph-session", nil) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + var resp picoMemoryGraphResponse + if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { + t.Fatalf("Decode() error = %v", err) + } + + if resp.SessionID != "graph-session" { + t.Fatalf("resp.SessionID = %q, want %q", resp.SessionID, "graph-session") + } + if len(resp.Nodes) < 5 { + t.Fatalf("len(resp.Nodes) = %d, want at least 5", len(resp.Nodes)) + } + + foundMemoryRoot := false + foundSessionMessage := false + for _, node := range resp.Nodes { + if node.ID == "memory-root" { + foundMemoryRoot = true + } + if strings.Contains(node.Label, "NAIROBI") || strings.Contains(node.Preview, "Nairobi") { + foundSessionMessage = true + } + } + if !foundMemoryRoot { + t.Fatal("expected memory-root node in graph") + } + if !foundSessionMessage { + t.Fatal("expected session content to appear in graph nodes") + } +} + +func TestHandleGetPicoSubagents_UsesScopedGatewayStatus(t *testing.T) { + home := t.TempDir() + t.Setenv("PICOCLAW_HOME", home) + + configPath := filepath.Join(t.TempDir(), "config.json") + h := NewHandler(configPath) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/internal/subagents/status" { + t.Fatalf("path = %q, want %q", r.URL.Path, "/internal/subagents/status") + } + if got := r.Header.Get("Authorization"); got != "Bearer gateway-auth-token" { + t.Fatalf("Authorization = %q, want %q", got, "Bearer gateway-auth-token") + } + if got := r.URL.Query().Get("channel"); got != "pico" { + t.Fatalf("channel query = %q, want %q", got, "pico") + } + if got := r.URL.Query().Get("chat_id"); got != "session-42" { + t.Fatalf("chat_id query = %q, want %q", got, "session-42") + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "channel": "pico", + "chat_id": "session-42", + "tasks": []map[string]any{ + { + "id": "subagent-1", + "label": "Research", + "status": "running", + "created": int64(1710000000000), + "result": "This is a very long status summary that should still round-trip through the launcher API cleanly.", + }, + }, + }) + })) + defer server.Close() + + cfg := config.DefaultConfig() + cfg.Gateway.Host = "127.0.0.1" + cfg.Gateway.Port = mustGatewayTestPort(t, server.URL) + if err := config.SaveConfig(configPath, cfg); err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + cmd := startGatewayLikeProcess(t) + t.Cleanup(func() { + if cmd.Process != nil { + _ = cmd.Process.Kill() + } + _ = cmd.Wait() + }) + + origPidData := gateway.pidData + origCmd := gateway.cmd + t.Cleanup(func() { + gateway.mu.Lock() + gateway.pidData = origPidData + gateway.cmd = origCmd + gateway.mu.Unlock() + }) + + gateway.mu.Lock() + gateway.pidData = &ppid.PidFileData{PID: cmd.Process.Pid, Token: "gateway-auth-token"} + gateway.cmd = cmd + gateway.mu.Unlock() + + req := httptest.NewRequest(http.MethodGet, "http://launcher.local/api/pico/subagents?session_id=session-42", nil) + rec := httptest.NewRecorder() + h.handleGetPicoSubagents(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + var resp picoSubagentStatusResponse + if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if resp.SessionID != "session-42" || resp.ChatID != "session-42" || resp.Channel != "pico" { + t.Fatalf("response = %#v, want scoped session metadata", resp) + } + if len(resp.Tasks) != 1 || resp.Tasks[0].ID != "subagent-1" { + t.Fatalf("tasks = %#v, want one proxied task", resp.Tasks) + } +} + func TestHandleWebSocketProxyRejectsStalePidDataAfterProcessExit(t *testing.T) { tmpDir := t.TempDir() t.Setenv("HOME", tmpDir) diff --git a/web/frontend/src/api/pico.ts b/web/frontend/src/api/pico.ts index ca98a06da..8ae816595 100644 --- a/web/frontend/src/api/pico.ts +++ b/web/frontend/src/api/pico.ts @@ -15,6 +15,43 @@ interface PicoSetupResponse { changed: boolean } +export interface PicoSubagentStatusItem { + id: string + label?: string + status: "running" | "completed" | "failed" | "canceled" | string + created: number + result?: string +} + +export interface PicoSubagentStatusResponse { + session_id: string + channel: string + chat_id: string + tasks: PicoSubagentStatusItem[] +} + +export interface PicoMemoryGraphNode { + id: string + label: string + kind: string + group: string + preview?: string + weight?: number +} + +export interface PicoMemoryGraphEdge { + source: string + target: string + kind: string +} + +export interface PicoMemoryGraphResponse { + session_id: string + generated_at: string + nodes: PicoMemoryGraphNode[] + edges: PicoMemoryGraphEdge[] +} + const BASE_URL = "" async function request(path: string, options?: RequestInit): Promise { @@ -37,4 +74,22 @@ export async function setupPico(): Promise { return request("/api/pico/setup", { method: "POST" }) } +export async function getPicoSubagents( + sessionId: string, +): Promise { + const params = new URLSearchParams({ session_id: sessionId }) + return request( + `/api/pico/subagents?${params.toString()}`, + ) +} + +export async function getPicoMemoryGraph( + sessionId: string, +): Promise { + const params = new URLSearchParams({ session_id: sessionId }) + return request( + `/api/pico/memory-graph?${params.toString()}`, + ) +} + export type { PicoInfoResponse, PicoSetupResponse } diff --git a/web/frontend/src/components/agent/cockpit/cockpit-page.tsx b/web/frontend/src/components/agent/cockpit/cockpit-page.tsx new file mode 100644 index 000000000..7e49c911f --- /dev/null +++ b/web/frontend/src/components/agent/cockpit/cockpit-page.tsx @@ -0,0 +1,651 @@ +import { + IconArrowRight, + IconBrain, + IconMicrophone, + IconMicrophoneOff, + IconPhoto, + IconSearch, + IconSettings, + IconUpload, +} from "@tabler/icons-react" +import { Link } from "@tanstack/react-router" +import dayjs from "dayjs" +import { type ChangeEvent, useEffect, useMemo, useRef, useState } from "react" +import { toast } from "sonner" + +import type { ChatAttachment } from "@/store/chat" +import { usePicoChat } from "@/hooks/use-pico-chat" +import { PageHeader } from "@/components/page-header" +import { Badge } from "@/components/ui/badge" +import { Button } from "@/components/ui/button" +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card" +import { Input } from "@/components/ui/input" +import { Skeleton } from "@/components/ui/skeleton" +import { Switch } from "@/components/ui/switch" +import { cn } from "@/lib/utils" + +import { MemoryGraph } from "./memory-graph" +import { useAgentCockpit } from "./use-agent-cockpit" + +const MAX_IMAGE_SIZE_BYTES = 7 * 1024 * 1024 +const ALLOWED_IMAGE_TYPES = new Set([ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + "image/bmp", +]) + +declare global { + interface Window { + SpeechRecognition?: new () => SpeechRecognitionLike + webkitSpeechRecognition?: new () => SpeechRecognitionLike + } +} + +interface SpeechRecognitionLike { + continuous: boolean + interimResults: boolean + lang: string + onresult: ((event: SpeechRecognitionEventLike) => void) | null + onend: (() => void) | null + onerror: ((event: { error: string }) => void) | null + start(): void + stop(): void +} + +interface SpeechRecognitionEventLike { + results: ArrayLike> +} + +function statusBadgeVariant(status: string) { + switch (status) { + case "enabled": + case "completed": + return "default" as const + case "blocked": + case "failed": + return "destructive" as const + case "running": + return "secondary" as const + default: + return "outline" as const + } +} + +function reasonLabel(reasonCode?: string) { + switch (reasonCode) { + case "requires_subagent": + return "Requires subagent runtime" + case "requires_skills": + return "Requires skills support" + case "requires_mcp_discovery": + return "Requires MCP discovery" + case "requires_linux": + return "Linux only" + case "requires_serial_platform": + return "Unsupported serial platform" + default: + return reasonCode ?? "" + } +} + +function readFileAsDataUrl(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => { + if (typeof reader.result === "string") { + resolve(reader.result) + return + } + reject(new Error("Failed to read file")) + } + reader.onerror = () => + reject(reader.error || new Error("Failed to read file")) + reader.readAsDataURL(file) + }) +} + +export function CockpitPage() { + const { activeSessionId, connectionState, sendMessage } = usePicoChat() + const { + categoryCounts, + groupedTools, + pendingToolName, + searchQuery, + sessionSubagents, + sessionMemoryGraph, + statusCounts, + statusFilter, + webSearchConfig, + hasMemoryGraphError, + hasSubagentsError, + hasToolsError, + isMemoryGraphLoading, + isSubagentsLoading, + isToolsLoading, + isWebSearchLoading, + setSearchQuery, + setStatusFilter, + toggleTool, + } = useAgentCockpit(activeSessionId) + + const [prompt, setPrompt] = useState("") + const [attachments, setAttachments] = useState([]) + const [isListening, setIsListening] = useState(false) + const fileInputRef = useRef(null) + const recognitionRef = useRef(null) + + useEffect(() => { + const Recognition = + window.SpeechRecognition ?? window.webkitSpeechRecognition + if (!Recognition) { + return + } + const recognition = new Recognition() + recognition.continuous = false + recognition.interimResults = false + recognition.lang = "en-US" + recognition.onresult = (event) => { + const transcript = event.results[0]?.[0]?.transcript?.trim() ?? "" + setPrompt(transcript) + if (!transcript) { + return + } + const sent = sendMessage({ content: transcript }) + if (!sent) { + toast.error("Voice capture worked, but chat is not ready to send.") + } + } + recognition.onend = () => setIsListening(false) + recognition.onerror = (event) => { + setIsListening(false) + toast.error(`Voice capture error: ${event.error}`) + } + recognitionRef.current = recognition + }, [sendMessage]) + + const filteredToolCount = useMemo( + () => groupedTools.reduce((total, [, items]) => total + items.length, 0), + [groupedTools], + ) + + const currentProviderLabel = useMemo(() => { + const current = webSearchConfig?.providers.find((provider) => provider.current) + return current?.label ?? webSearchConfig?.provider ?? "Auto" + }, [webSearchConfig]) + + const handleImageSelection = async (event: ChangeEvent) => { + const files = Array.from(event.target.files ?? []) + event.target.value = "" + if (files.length === 0) { + return + } + + const nextAttachments: ChatAttachment[] = [] + for (const file of files) { + if (!ALLOWED_IMAGE_TYPES.has(file.type)) { + toast.error(`Unsupported image type: ${file.name}`) + continue + } + if (file.size > MAX_IMAGE_SIZE_BYTES) { + toast.error(`${file.name} exceeds 7 MB.`) + continue + } + try { + const url = await readFileAsDataUrl(file) + nextAttachments.push({ + type: "image", + url, + filename: file.name, + contentType: file.type, + }) + } catch (error) { + toast.error( + error instanceof Error ? error.message : `Failed to read ${file.name}`, + ) + } + } + + setAttachments((current) => [...current, ...nextAttachments]) + } + + const handleSendBridgeMessage = () => { + const sent = sendMessage({ content: prompt, attachments }) + if (!sent) { + toast.error("Chat connection is not ready yet.") + return + } + setPrompt("") + setAttachments([]) + } + + const toggleVoice = () => { + if (!recognitionRef.current) { + toast.error("Voice capture is not supported in this browser.") + return + } + if (isListening) { + recognitionRef.current.stop() + setIsListening(false) + return + } + try { + recognitionRef.current.start() + setIsListening(true) + } catch (error) { + toast.error( + error instanceof Error ? error.message : "Unable to start voice capture.", + ) + setIsListening(false) + } + } + + return ( +
+ + +
+
+ + +
+ + +
+
+ + Tool Grid + + + Real launcher tools, live from PicoClaw. + +
+
+ + {filteredToolCount} visible + + +
+
+ + + {hasToolsError ? ( +
+ Failed to load tools. +
+ ) : isToolsLoading ? ( +
+ {Array.from({ length: 6 }).map((_, index) => ( + + ))} +
+ ) : ( +
+ {groupedTools.map(([category, items]) => ( +
+
+

+ {category} +

+ + {items.length} + +
+
+ {items.map((tool) => ( + + +
+
+ + {tool.name} + + + {tool.status} + +
+ + toggleTool(tool.name, checked) + } + /> +
+ + {tool.description} + + + +
+ {tool.category} + {tool.config_key} +
+ {tool.reason_code ? ( +
+ {reasonLabel(tool.reason_code)} +
+ ) : null} + + + ))} +
+
+ ))} +
+ )} + + + + + + + Memory Network + + + Obsidian-style graph built from PicoClaw workspace memory and the active session trail. + + + + {hasMemoryGraphError ? ( +
+ Failed to load memory graph. +
+ ) : isMemoryGraphLoading ? ( + + ) : sessionMemoryGraph && sessionMemoryGraph.nodes.length > 0 ? ( + + ) : ( +
+ Memory graph will appear when the session and workspace memory have visible context. +
+ )} + + +
+ + +
+
+
+ ) +} diff --git a/web/frontend/src/components/agent/cockpit/memory-graph.tsx b/web/frontend/src/components/agent/cockpit/memory-graph.tsx new file mode 100644 index 000000000..cb6cfb9fd --- /dev/null +++ b/web/frontend/src/components/agent/cockpit/memory-graph.tsx @@ -0,0 +1,231 @@ +import { useMemo, useState } from "react" + +import type { + PicoMemoryGraphEdge, + PicoMemoryGraphNode, +} from "@/api/pico" +import { Badge } from "@/components/ui/badge" +import { cn } from "@/lib/utils" + +const VIEWBOX_WIDTH = 1000 +const VIEWBOX_HEIGHT = 560 + +type PositionedNode = PicoMemoryGraphNode & { x: number; y: number } + +interface MemoryGraphProps { + nodes: PicoMemoryGraphNode[] + edges: PicoMemoryGraphEdge[] +} + +const GROUP_COLUMNS: Record = { + memory: 170, + "daily-1": 295, + "daily-2": 295, + "daily-3": 295, + session: 700, + tool: 840, + media: 920, +} + +function groupColor(group: string) { + if (group === "memory") return "#72f0a0" + if (group.startsWith("daily-")) return "#4dd0e1" + if (group === "tool") return "#f8d66d" + if (group === "media") return "#f395d6" + return "#90e89f" +} + +function computeLayout(nodes: PicoMemoryGraphNode[]): PositionedNode[] { + const groups = new Map() + for (const node of nodes) { + const items = groups.get(node.group) ?? [] + items.push(node) + groups.set(node.group, items) + } + + const layout: PositionedNode[] = [] + for (const [group, items] of groups.entries()) { + const x = GROUP_COLUMNS[group] ?? 500 + const count = items.length + const gap = VIEWBOX_HEIGHT / (count + 1) + items.forEach((node, index) => { + const offset = + group === "session" && node.kind === "root" + ? 0 + : Math.sin((index + 1) * 1.7) * 12 + layout.push({ + ...node, + x, + y: Math.max(52, Math.min(VIEWBOX_HEIGHT - 52, gap * (index + 1) + offset)), + }) + }) + } + + return layout.sort((left, right) => left.x - right.x || left.y - right.y) +} + +export function MemoryGraph({ nodes, edges }: MemoryGraphProps) { + const [selectedId, setSelectedId] = useState(nodes[0]?.id ?? null) + + const positionedNodes = useMemo(() => computeLayout(nodes), [nodes]) + const nodeMap = useMemo( + () => new Map(positionedNodes.map((node) => [node.id, node])), + [positionedNodes], + ) + const selectedNode = + positionedNodes.find((node) => node.id === selectedId) ?? positionedNodes[0] + + return ( +
+
+ + + + + + + + + {Array.from({ length: 10 }).map((_, index) => ( + + ))} + {Array.from({ length: 8 }).map((_, index) => ( + + ))} + + {edges.map((edge) => { + const source = nodeMap.get(edge.source) + const target = nodeMap.get(edge.target) + if (!source || !target) { + return null + } + + const active = + selectedId != null && + (selectedId === edge.source || selectedId === edge.target) + + return ( + + ) + })} + + {positionedNodes.map((node) => { + const active = node.id === selectedNode?.id + const color = groupColor(node.group) + const radius = + node.kind === "root" ? 24 : node.kind === "document" ? 18 : 14 + + return ( + setSelectedId(node.id)} + onClick={() => setSelectedId(node.id)} + className="cursor-pointer" + > + + + + {node.kind === "root" ? node.label : node.label.slice(0, 18)} + + + ) + })} + +
+ +
+ {selectedNode ? ( + <> +
+
+

+ Node Focus +

+

+ {selectedNode.label} +

+
+ + {selectedNode.kind} + +
+

+ {selectedNode.preview || "No extra preview for this node yet."} +

+
+
+ Cluster + + {selectedNode.group} + +
+
+ Weight + + {selectedNode.weight ?? 1} + +
+
+ + ) : ( +

No graph data available.

+ )} +
+
+ ) +} diff --git a/web/frontend/src/components/agent/cockpit/use-agent-cockpit.ts b/web/frontend/src/components/agent/cockpit/use-agent-cockpit.ts new file mode 100644 index 000000000..c6b4cee4f --- /dev/null +++ b/web/frontend/src/components/agent/cockpit/use-agent-cockpit.ts @@ -0,0 +1,136 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query" +import { useDeferredValue, useMemo, useState } from "react" +import { useTranslation } from "react-i18next" +import { toast } from "sonner" + +import { getPicoMemoryGraph, getPicoSubagents } from "@/api/pico" +import { getTools, getWebSearchConfig, setToolEnabled } from "@/api/tools" +import { showSaveSuccessOrRestartToast } from "@/lib/restart-required" +import { refreshGatewayState } from "@/store/gateway" + +type ToolStatusFilter = "all" | "enabled" | "disabled" | "blocked" + +export function useAgentCockpit(sessionId: string) { + const { t } = useTranslation() + const queryClient = useQueryClient() + const [searchQuery, setSearchQuery] = useState("") + const [statusFilter, setStatusFilter] = useState("all") + const deferredSearchQuery = useDeferredValue(searchQuery) + + const toolsQuery = useQuery({ + queryKey: ["tools"], + queryFn: getTools, + }) + const webSearchQuery = useQuery({ + queryKey: ["tools", "web-search-config"], + queryFn: getWebSearchConfig, + }) + const subagentsQuery = useQuery({ + queryKey: ["pico", "subagents", sessionId], + queryFn: () => getPicoSubagents(sessionId), + enabled: Boolean(sessionId), + refetchInterval: 3000, + }) + const memoryGraphQuery = useQuery({ + queryKey: ["pico", "memory-graph", sessionId], + queryFn: () => getPicoMemoryGraph(sessionId), + enabled: Boolean(sessionId), + refetchInterval: 10000, + }) + + const toggleToolMutation = useMutation({ + mutationFn: async ({ name, enabled }: { name: string; enabled: boolean }) => + setToolEnabled(name, enabled), + onSuccess: async (_, variables) => { + const gateway = await refreshGatewayState({ force: true }) + showSaveSuccessOrRestartToast( + t, + variables.enabled + ? t("pages.agent.tools.enable_success", "Tool enabled successfully") + : t( + "pages.agent.tools.disable_success", + "Tool disabled successfully", + ), + "Agent Cockpit", + gateway?.restartRequired === true, + ) + void queryClient.invalidateQueries({ queryKey: ["tools"] }) + }, + onError: (error) => { + toast.error( + error instanceof Error + ? error.message + : t("pages.agent.tools.toggle_error", "Failed to toggle tool"), + ) + }, + }) + + const tools = toolsQuery.data?.tools ?? [] + const normalizedSearchQuery = deferredSearchQuery.trim().toLowerCase() + + const groupedTools = useMemo(() => { + const groups = new Map() + + for (const tool of tools) { + if (statusFilter !== "all" && tool.status !== statusFilter) { + continue + } + + if (normalizedSearchQuery) { + const haystack = `${tool.name} ${tool.description}`.toLowerCase() + if (!haystack.includes(normalizedSearchQuery)) { + continue + } + } + + const items = groups.get(tool.category) ?? [] + items.push(tool) + groups.set(tool.category, items) + } + + return Array.from(groups.entries()) + }, [normalizedSearchQuery, statusFilter, tools]) + + const categoryCounts = useMemo(() => { + const counts = new Map() + for (const tool of tools) { + counts.set(tool.category, (counts.get(tool.category) ?? 0) + 1) + } + return Array.from(counts.entries()) + }, [tools]) + + const statusCounts = useMemo(() => { + return { + all: tools.length, + enabled: tools.filter((tool) => tool.status === "enabled").length, + disabled: tools.filter((tool) => tool.status === "disabled").length, + blocked: tools.filter((tool) => tool.status === "blocked").length, + } + }, [tools]) + + return { + categoryCounts, + groupedTools, + pendingToolName: toggleToolMutation.isPending + ? (toggleToolMutation.variables?.name ?? null) + : null, + searchQuery, + sessionMemoryGraph: memoryGraphQuery.data ?? null, + sessionSubagents: subagentsQuery.data?.tasks ?? [], + statusCounts, + statusFilter, + tools, + hasMemoryGraphError: memoryGraphQuery.error != null, + webSearchConfig: webSearchQuery.data ?? null, + hasSubagentsError: subagentsQuery.error != null, + hasToolsError: toolsQuery.error != null, + isMemoryGraphLoading: memoryGraphQuery.isLoading, + isSubagentsLoading: subagentsQuery.isLoading, + isToolsLoading: toolsQuery.isLoading, + isWebSearchLoading: webSearchQuery.isLoading, + setSearchQuery, + setStatusFilter, + toggleTool: (name: string, enabled: boolean) => + toggleToolMutation.mutate({ name, enabled }), + } +} diff --git a/web/frontend/src/components/app-sidebar.tsx b/web/frontend/src/components/app-sidebar.tsx index 1980e458c..55b6eec69 100644 --- a/web/frontend/src/components/app-sidebar.tsx +++ b/web/frontend/src/components/app-sidebar.tsx @@ -3,6 +3,7 @@ import { IconAtom, IconChevronsDown, IconChevronsUp, + IconCpu, IconKey, IconListDetails, IconMessageCircle, @@ -132,6 +133,12 @@ export function AppSidebar({ ...props }: React.ComponentProps) { { ...baseNavGroups[2], items: [ + { + title: "Agent Cockpit", + url: "/agent/cockpit", + icon: IconCpu, + translateTitle: false, + }, { title: "navigation.hub", url: "/agent/hub", diff --git a/web/frontend/src/routeTree.gen.ts b/web/frontend/src/routeTree.gen.ts index b2f85e826..13b153f62 100644 --- a/web/frontend/src/routeTree.gen.ts +++ b/web/frontend/src/routeTree.gen.ts @@ -23,6 +23,7 @@ import { Route as ChannelsNameRouteImport } from './routes/channels/$name' import { Route as AgentToolsRouteImport } from './routes/agent/tools' import { Route as AgentSkillsRouteImport } from './routes/agent/skills' import { Route as AgentHubRouteImport } from './routes/agent/hub' +import { Route as AgentCockpitRouteImport } from './routes/agent/cockpit' const ModelsRoute = ModelsRouteImport.update({ id: '/models', @@ -94,6 +95,11 @@ const AgentHubRoute = AgentHubRouteImport.update({ path: '/hub', getParentRoute: () => AgentRoute, } as any) +const AgentCockpitRoute = AgentCockpitRouteImport.update({ + id: '/cockpit', + path: '/cockpit', + getParentRoute: () => AgentRoute, +} as any) export interface FileRoutesByFullPath { '/': typeof IndexRoute @@ -105,6 +111,7 @@ export interface FileRoutesByFullPath { '/launcher-setup': typeof LauncherSetupRoute '/logs': typeof LogsRoute '/models': typeof ModelsRoute + '/agent/cockpit': typeof AgentCockpitRoute '/agent/hub': typeof AgentHubRoute '/agent/skills': typeof AgentSkillsRoute '/agent/tools': typeof AgentToolsRoute @@ -121,6 +128,7 @@ export interface FileRoutesByTo { '/launcher-setup': typeof LauncherSetupRoute '/logs': typeof LogsRoute '/models': typeof ModelsRoute + '/agent/cockpit': typeof AgentCockpitRoute '/agent/hub': typeof AgentHubRoute '/agent/skills': typeof AgentSkillsRoute '/agent/tools': typeof AgentToolsRoute @@ -138,6 +146,7 @@ export interface FileRoutesById { '/launcher-setup': typeof LauncherSetupRoute '/logs': typeof LogsRoute '/models': typeof ModelsRoute + '/agent/cockpit': typeof AgentCockpitRoute '/agent/hub': typeof AgentHubRoute '/agent/skills': typeof AgentSkillsRoute '/agent/tools': typeof AgentToolsRoute @@ -156,6 +165,7 @@ export interface FileRouteTypes { | '/launcher-setup' | '/logs' | '/models' + | '/agent/cockpit' | '/agent/hub' | '/agent/skills' | '/agent/tools' @@ -172,6 +182,7 @@ export interface FileRouteTypes { | '/launcher-setup' | '/logs' | '/models' + | '/agent/cockpit' | '/agent/hub' | '/agent/skills' | '/agent/tools' @@ -188,6 +199,7 @@ export interface FileRouteTypes { | '/launcher-setup' | '/logs' | '/models' + | '/agent/cockpit' | '/agent/hub' | '/agent/skills' | '/agent/tools' @@ -307,6 +319,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AgentHubRouteImport parentRoute: typeof AgentRoute } + '/agent/cockpit': { + id: '/agent/cockpit' + path: '/cockpit' + fullPath: '/agent/cockpit' + preLoaderRoute: typeof AgentCockpitRouteImport + parentRoute: typeof AgentRoute + } } } @@ -323,12 +342,14 @@ const ChannelsRouteRouteWithChildren = ChannelsRouteRoute._addFileChildren( ) interface AgentRouteChildren { + AgentCockpitRoute: typeof AgentCockpitRoute AgentHubRoute: typeof AgentHubRoute AgentSkillsRoute: typeof AgentSkillsRoute AgentToolsRoute: typeof AgentToolsRoute } const AgentRouteChildren: AgentRouteChildren = { + AgentCockpitRoute: AgentCockpitRoute, AgentHubRoute: AgentHubRoute, AgentSkillsRoute: AgentSkillsRoute, AgentToolsRoute: AgentToolsRoute, diff --git a/web/frontend/src/routes/agent/cockpit.tsx b/web/frontend/src/routes/agent/cockpit.tsx new file mode 100644 index 000000000..dfd6d6b74 --- /dev/null +++ b/web/frontend/src/routes/agent/cockpit.tsx @@ -0,0 +1,11 @@ +import { createFileRoute } from "@tanstack/react-router" + +import { CockpitPage } from "@/components/agent/cockpit/cockpit-page" + +export const Route = createFileRoute("/agent/cockpit")({ + component: AgentCockpitRoute, +}) + +function AgentCockpitRoute() { + return +} From f270a878940c80d51b9140bdc1308b1b5a7aea79 Mon Sep 17 00:00:00 2001 From: anthrodjear Date: Wed, 6 May 2026 19:35:35 +0300 Subject: [PATCH 3/3] updated the UI in chat adding voice and image. In cockpit redesigned it. --- .../components/agent/cockpit/cockpit-page.tsx | 719 ++++-------------- .../src/components/chat/chat-composer.tsx | 108 ++- 2 files changed, 229 insertions(+), 598 deletions(-) diff --git a/web/frontend/src/components/agent/cockpit/cockpit-page.tsx b/web/frontend/src/components/agent/cockpit/cockpit-page.tsx index 7e49c911f..28cff97dc 100644 --- a/web/frontend/src/components/agent/cockpit/cockpit-page.tsx +++ b/web/frontend/src/components/agent/cockpit/cockpit-page.tsx @@ -1,84 +1,15 @@ -import { - IconArrowRight, - IconBrain, - IconMicrophone, - IconMicrophoneOff, - IconPhoto, - IconSearch, - IconSettings, - IconUpload, -} from "@tabler/icons-react" -import { Link } from "@tanstack/react-router" import dayjs from "dayjs" -import { type ChangeEvent, useEffect, useMemo, useRef, useState } from "react" -import { toast } from "sonner" +import { useMemo } from "react" +import { IconArrowRight } from "@tabler/icons-react" -import type { ChatAttachment } from "@/store/chat" import { usePicoChat } from "@/hooks/use-pico-chat" -import { PageHeader } from "@/components/page-header" import { Badge } from "@/components/ui/badge" -import { Button } from "@/components/ui/button" -import { - Card, - CardContent, - CardDescription, - CardHeader, - CardTitle, -} from "@/components/ui/card" -import { Input } from "@/components/ui/input" -import { Skeleton } from "@/components/ui/skeleton" import { Switch } from "@/components/ui/switch" import { cn } from "@/lib/utils" import { MemoryGraph } from "./memory-graph" import { useAgentCockpit } from "./use-agent-cockpit" -const MAX_IMAGE_SIZE_BYTES = 7 * 1024 * 1024 -const ALLOWED_IMAGE_TYPES = new Set([ - "image/jpeg", - "image/png", - "image/gif", - "image/webp", - "image/bmp", -]) - -declare global { - interface Window { - SpeechRecognition?: new () => SpeechRecognitionLike - webkitSpeechRecognition?: new () => SpeechRecognitionLike - } -} - -interface SpeechRecognitionLike { - continuous: boolean - interimResults: boolean - lang: string - onresult: ((event: SpeechRecognitionEventLike) => void) | null - onend: (() => void) | null - onerror: ((event: { error: string }) => void) | null - start(): void - stop(): void -} - -interface SpeechRecognitionEventLike { - results: ArrayLike> -} - -function statusBadgeVariant(status: string) { - switch (status) { - case "enabled": - case "completed": - return "default" as const - case "blocked": - case "failed": - return "destructive" as const - case "running": - return "secondary" as const - default: - return "outline" as const - } -} - function reasonLabel(reasonCode?: string) { switch (reasonCode) { case "requires_subagent": @@ -96,556 +27,182 @@ function reasonLabel(reasonCode?: string) { } } -function readFileAsDataUrl(file: File): Promise { - return new Promise((resolve, reject) => { - const reader = new FileReader() - reader.onload = () => { - if (typeof reader.result === "string") { - resolve(reader.result) - return - } - reject(new Error("Failed to read file")) - } - reader.onerror = () => - reject(reader.error || new Error("Failed to read file")) - reader.readAsDataURL(file) - }) -} - export function CockpitPage() { - const { activeSessionId, connectionState, sendMessage } = usePicoChat() + const { activeSessionId } = usePicoChat() const { - categoryCounts, groupedTools, pendingToolName, - searchQuery, sessionSubagents, sessionMemoryGraph, - statusCounts, - statusFilter, - webSearchConfig, - hasMemoryGraphError, - hasSubagentsError, - hasToolsError, - isMemoryGraphLoading, - isSubagentsLoading, - isToolsLoading, - isWebSearchLoading, - setSearchQuery, - setStatusFilter, toggleTool, } = useAgentCockpit(activeSessionId) - const [prompt, setPrompt] = useState("") - const [attachments, setAttachments] = useState([]) - const [isListening, setIsListening] = useState(false) - const fileInputRef = useRef(null) - const recognitionRef = useRef(null) - - useEffect(() => { - const Recognition = - window.SpeechRecognition ?? window.webkitSpeechRecognition - if (!Recognition) { - return - } - const recognition = new Recognition() - recognition.continuous = false - recognition.interimResults = false - recognition.lang = "en-US" - recognition.onresult = (event) => { - const transcript = event.results[0]?.[0]?.transcript?.trim() ?? "" - setPrompt(transcript) - if (!transcript) { - return - } - const sent = sendMessage({ content: transcript }) - if (!sent) { - toast.error("Voice capture worked, but chat is not ready to send.") - } - } - recognition.onend = () => setIsListening(false) - recognition.onerror = (event) => { - setIsListening(false) - toast.error(`Voice capture error: ${event.error}`) - } - recognitionRef.current = recognition - }, [sendMessage]) - const filteredToolCount = useMemo( () => groupedTools.reduce((total, [, items]) => total + items.length, 0), [groupedTools], ) - const currentProviderLabel = useMemo(() => { - const current = webSearchConfig?.providers.find((provider) => provider.current) - return current?.label ?? webSearchConfig?.provider ?? "Auto" - }, [webSearchConfig]) - - const handleImageSelection = async (event: ChangeEvent) => { - const files = Array.from(event.target.files ?? []) - event.target.value = "" - if (files.length === 0) { - return - } - - const nextAttachments: ChatAttachment[] = [] - for (const file of files) { - if (!ALLOWED_IMAGE_TYPES.has(file.type)) { - toast.error(`Unsupported image type: ${file.name}`) - continue - } - if (file.size > MAX_IMAGE_SIZE_BYTES) { - toast.error(`${file.name} exceeds 7 MB.`) - continue - } - try { - const url = await readFileAsDataUrl(file) - nextAttachments.push({ - type: "image", - url, - filename: file.name, - contentType: file.type, - }) - } catch (error) { - toast.error( - error instanceof Error ? error.message : `Failed to read ${file.name}`, - ) - } - } - - setAttachments((current) => [...current, ...nextAttachments]) - } - - const handleSendBridgeMessage = () => { - const sent = sendMessage({ content: prompt, attachments }) - if (!sent) { - toast.error("Chat connection is not ready yet.") - return - } - setPrompt("") - setAttachments([]) - } - - const toggleVoice = () => { - if (!recognitionRef.current) { - toast.error("Voice capture is not supported in this browser.") - return - } - if (isListening) { - recognitionRef.current.stop() - setIsListening(false) - return - } - try { - recognitionRef.current.start() - setIsListening(true) - } catch (error) { - toast.error( - error instanceof Error ? error.message : "Unable to start voice capture.", - ) - setIsListening(false) - } - } - return ( -
- +
+ {/* Ghost Background Typography */} +
+ COCKPIT + SYSTEM +
-
-
- - -
- - -
-
- - Tool Grid - - - Real launcher tools, live from PicoClaw. - -
-
- - {filteredToolCount} visible - - + {/* Memory Network */} +
+
+
+ Relational Map +

Memory Network

- - - {hasToolsError ? ( -
- Failed to load tools. -
- ) : isToolsLoading ? ( -
- {Array.from({ length: 6 }).map((_, index) => ( - - ))} -
- ) : ( -
- {groupedTools.map(([category, items]) => ( -
-
-

- {category} -

- - {items.length} - -
-
- {items.map((tool) => ( - - -
-
- - {tool.name} - - - {tool.status} - -
- - toggleTool(tool.name, checked) - } - /> -
- - {tool.description} - - - -
- {tool.category} - {tool.config_key} -
- {tool.reason_code ? ( -
- {reasonLabel(tool.reason_code)} -
- ) : null} - - - ))} -
-
- ))} -
- )} - - - - - - - Memory Network - - - Obsidian-style graph built from PicoClaw workspace memory and the active session trail. - - - - {hasMemoryGraphError ? ( -
- Failed to load memory graph. -
- ) : isMemoryGraphLoading ? ( - - ) : sessionMemoryGraph && sessionMemoryGraph.nodes.length > 0 ? ( +
- ) : ( -
- Memory graph will appear when the session and workspace memory have visible context. -
- )} - - -
- -
+ +
- setPrompt(event.target.value)} - placeholder="Send a message to the active agent session" - className="border-[#1f4f31] bg-[#050d08] text-[#d7f9df] placeholder:text-[#5c8168]" - /> - - {attachments.length > 0 ? ( -
- {attachments.map((attachment, index) => ( -
- {attachment.filename ?? "Image"} - -
- ))} -
- ) : null} - -
- - -
- - - -
- - -
- - - - - - - - - Main Agent Subagents - - - Live status for the current Pico session only. - - - - {hasSubagentsError ? ( -
- Failed to load subagent status. -
- ) : isSubagentsLoading ? ( - Array.from({ length: 3 }).map((_, index) => ( - - )) - ) : sessionSubagents.length === 0 ? ( -
+ {/* Right Sidebar */} +
+ + {/* Footer */} +
) } diff --git a/web/frontend/src/components/chat/chat-composer.tsx b/web/frontend/src/components/chat/chat-composer.tsx index b3354cc33..f722e51f2 100644 --- a/web/frontend/src/components/chat/chat-composer.tsx +++ b/web/frontend/src/components/chat/chat-composer.tsx @@ -1,5 +1,6 @@ -import { IconArrowUp, IconPhotoPlus, IconX } from "@tabler/icons-react" -import type { KeyboardEvent } from "react" +import { IconArrowUp, IconMicrophone, IconMicrophoneOff, IconPhotoPlus, IconX } from "@tabler/icons-react" +import { type KeyboardEvent } from "react" +import { useEffect, useRef, useState } from "react" import { useTranslation } from "react-i18next" import TextareaAutosize from "react-textarea-autosize" @@ -13,6 +14,28 @@ import { import { cn } from "@/lib/utils" import type { ChatAttachment, ContextUsage } from "@/store/chat" +declare global { + interface Window { + SpeechRecognition?: new () => SpeechRecognitionLike + webkitSpeechRecognition?: new () => SpeechRecognitionLike + } +} + +interface SpeechRecognitionLike { + continuous: boolean + interimResults: boolean + lang: string + onresult: ((event: SpeechRecognitionEventLike) => void) | null + onend: (() => void) | null + onerror: ((event: { error: string }) => void) | null + start(): void + stop(): void +} + +interface SpeechRecognitionEventLike { + results: ArrayLike> +} + export type ChatInputDisabledReason = | "gatewayUnknown" | "gatewayStarting" @@ -51,6 +74,41 @@ export function ChatComposer({ contextUsage, }: ChatComposerProps) { const { t } = useTranslation() + const [isListening, setIsListening] = useState(false) + const recognitionRef = useRef(null) + + useEffect(() => { + const Recognition = + window.SpeechRecognition ?? window.webkitSpeechRecognition + if (!Recognition) return + const recognition = new Recognition() + recognition.continuous = false + recognition.interimResults = false + recognition.lang = "en-US" + recognition.onresult = (event) => { + const transcript = event.results[0]?.[0]?.transcript?.trim() ?? "" + if (transcript) onInputChange(transcript) + } + recognition.onend = () => setIsListening(false) + recognition.onerror = () => setIsListening(false) + recognitionRef.current = recognition + }, [onInputChange]) + + const toggleVoice = () => { + if (!recognitionRef.current) return + if (isListening) { + recognitionRef.current.stop() + setIsListening(false) + } else { + try { + recognitionRef.current.start() + setIsListening(true) + } catch { + setIsListening(false) + } + } + } + const canInput = inputDisabledReason === null const disabledMessage = inputDisabledReason === null @@ -110,21 +168,37 @@ export function ChatComposer({ maxRows={8} /> -
-
- -
+
+
+ + +
{contextUsage && (