diff --git a/cmd/picoclaw/internal/agent/helpers.go b/cmd/picoclaw/internal/agent/helpers.go index f754abc65..2320b2aee 100644 --- a/cmd/picoclaw/internal/agent/helpers.go +++ b/cmd/picoclaw/internal/agent/helpers.go @@ -23,16 +23,20 @@ func agentCmd(message, sessionKey, model string, debug bool) error { sessionKey = "cli:default" } - if debug { - logger.SetLevel(logger.DEBUG) - fmt.Println("๐Ÿ” Debug mode enabled") - } - cfg, err := internal.LoadConfig() if err != nil { return fmt.Errorf("error loading config: %w", err) } + // Apply logging config (config file setting). + logger.ApplyConfig(cfg.Logging.Level, cfg.Logging.FileDir) + + // Debug flag overrides config. + if debug { + logger.SetLevel(logger.INFO) + fmt.Println("Debug mode enabled") + } + if model != "" { cfg.Agents.Defaults.ModelName = model } @@ -60,6 +64,9 @@ func agentCmd(message, sessionKey, model string, debug bool) error { "skills_available": startupInfo["skills"].(map[string]any)["available"], }) + // Warn if bootstrap files are not customized. + internal.WarnMissingBootstrap(cfg.Agents.Defaults.Workspace) + if message != "" { ctx := context.Background() response, err := agentLoop.ProcessDirect(ctx, message, sessionKey) diff --git a/cmd/picoclaw/internal/gateway/helpers.go b/cmd/picoclaw/internal/gateway/helpers.go index 747f7d44e..fdf8677c4 100644 --- a/cmd/picoclaw/internal/gateway/helpers.go +++ b/cmd/picoclaw/internal/gateway/helpers.go @@ -80,6 +80,9 @@ func gatewayCmd(debug bool) error { "skills_available": skillsInfo["available"], }) + // Warn if bootstrap files are not customized. + internal.WarnMissingBootstrap(cfg.Agents.Defaults.Workspace) + // Setup cron tool and service execTimeout := time.Duration(cfg.Tools.Cron.ExecTimeoutMinutes) * time.Minute cronService := setupCronTool( diff --git a/cmd/picoclaw/internal/helpers.go b/cmd/picoclaw/internal/helpers.go index 9655d3c08..eacbcf487 100644 --- a/cmd/picoclaw/internal/helpers.go +++ b/cmd/picoclaw/internal/helpers.go @@ -53,3 +53,36 @@ func FormatBuildInfo() (string, string) { func GetVersion() string { return version } + +// WarnMissingBootstrap checks workspace bootstrap files (SOUL.md, IDENTITY.md, USER.md) +// and warns the user if any are missing or unmodified. +func WarnMissingBootstrap(workspace string) { + files := []struct { + name string + desc string + }{ + {"SOUL.md", "personality & behavior"}, + {"IDENTITY.md", "agent name & description"}, + {"USER.md", "your preferences & info"}, + } + + var missing []string + for _, f := range files { + path := filepath.Join(workspace, f.name) + info, err := os.Stat(path) + if os.IsNotExist(err) { + missing = append(missing, fmt.Sprintf(" %s โ€” %s", f.name, f.desc)) + } else if err == nil && info.Size() < 50 { + // File exists but appears to be empty/placeholder + missing = append(missing, fmt.Sprintf(" %s โ€” %s (empty)", f.name, f.desc)) + } + } + + if len(missing) > 0 { + fmt.Println(" Customize your agent:") + for _, m := range missing { + fmt.Println(m) + } + fmt.Printf(" Edit files in: %s\n\n", workspace) + } +} diff --git a/docs/design/runtime_builtin_cmd.md b/docs/design/runtime_builtin_cmd.md new file mode 100644 index 000000000..c4a1aa4fa --- /dev/null +++ b/docs/design/runtime_builtin_cmd.md @@ -0,0 +1,39 @@ +# Runtime Commands + +## Slash Commands + +All commands start with `/`, handled synchronously by `Reflector.HandleCommand()`. + +| Command | Usage | Description | +|---------|-------|-------------| +| `/help` | `/help` | List all commands | +| `/memory list` | `/memory list` | Show recent memories | +| `/memory add` | `/memory add #tags` | Add a memory | +| `/memory delete` | `/memory delete ` | Delete by ID | +| `/memory search` | `/memory search ` | Search by tags | +| `/memory stats` | `/memory stats` | Memory statistics | +| `/cot feedback` | `/cot feedback <1\|0\|-1>` | Rate last CoT strategy | +| `/cot stats` | `/cot stats` | CoT performance stats | +| `/cot history` | `/cot history [N]` | Recent CoT usage | +| `/shell` | `/shell [args]` | Execute shell command | +| `/show model` | `/show model` | Current model | +| `/list agents` | `/list agents` | List agents | +| `/switch model to` | `/switch model to ` | Switch model | +| `/runtime status` | `/runtime status` | Runtime diagnostics | + +## /shell Architecture + +``` +/shell + โ”œโ”€ Built-in (pure Go, cross-platform) + โ”‚ ls, cat, head, tail, grep, wc, find, diff, tree, + โ”‚ stat, pwd, echo, touch, mkdir, cp, mv + โ””โ”€ Dev Tool Passthrough (via exec tool) + go, git, node, python, npm, cargo, make, jq, rg +``` + +## Security + +- Built-in: Go stdlib only, auto-skip `.git`/`node_modules`, output capped at 4000 chars +- Passthrough: whitelist + deny patterns (`| sh`, `$()`, etc.) + ExecTool workspace restriction +- Unknown commands: rejected \ No newline at end of file diff --git a/docs/design/runtime_loop_design.md b/docs/design/runtime_loop_design.md new file mode 100644 index 000000000..6dd7b336c --- /dev/null +++ b/docs/design/runtime_loop_design.md @@ -0,0 +1,77 @@ +# Runtime Loop Design + +> Status: Implemented | Date: 2026-03-02 + +## Architecture + +``` +Message โ”€โ”€โ†’ Phase 1 (Analyse) โ”€โ”€โ†’ Phase 2 (Execute) โ”€โ”€โ†’ Phase 3 (Reflect) +``` + +| Phase | File | Responsibility | +|-------|------|----------------| +| **Analyse** | `analyser.go` | Lightweight LLM โ†’ intent, tags, CoT prompt | +| **Execute** | `executor.go` | LLM iteration loop + tool calling | +| **Reflect** | `reflector.go` | Turn scoring, TurnRecord persistence, slash commands | + +## Turn Definition + +One Turn = user message + Phase 1 result + all Phase 2 iterations + Phase 3 output. +Multiple tool-call iterations within Phase 2 count as **one Turn**. + +## Phase 1: Analyse + +- Uses configurable `analyser_model` (fast/cheap model, falls back to main model) +- Inputs: user message, Active Context, available tags, CoT learning data +- Outputs: `intent`, `tags[]`, `cot_prompt` +- After analysis: memory retrieval by tags, CoT injection into system prompt + +## Phase 2: Execute + +- LLM โ†’ tool call โ†’ tool result loop until no more tool calls +- Retry logic for context window overflow with automatic compression +- Reasoning output forwarded to dedicated channels + +## Phase 3: Reflect + +- **SyncPhase3** (< 2ms, before response sent): turn scoring + Active Context update +- **AsyncPhase3** (after response): `go turnStore.Insert(record)` โ†’ SQLite +- Slash commands: `/memory`, `/cot`, `/shell`, `/show`, `/list`, `/switch`, `/help` + +## Scoring Rules + +| Condition | Score | +|-----------|-------| +| Has tool calls | +3 | +| Write/edit tools | +2 | +| intent = task/code/debug | +3 | +| Reply > 500 chars | +2 | +| Short exchange < 80 chars | -2 | + +## Memory Hierarchy + +| Layer | Lifetime | Purpose | +|-------|----------|---------| +| Instant Memory | Per-turn | Dynamic window from TurnStore (score + tag filtering) | +| Active Context | Per-session | `CurrentFiles` + `RecentErrors`, injected into user prompt | +| Long-term Memory | Persistent | MemoryDigest batch extraction โ†’ `memory.db` | + +## Multi-Model Support + +| Config Field | Phase | Fallback | +|-------------|-------|----------| +| `model_name` | Phase 2 | โ€” | +| `analyser_model` | Phase 1 | โ†’ `model_name` | +| `digest_model` | MemoryDigest | โ†’ `model_name` | + +## Message Ordering (KV Cache Friendly) + +``` +[system_prompt] โ†’ always cached +[long_term_memory by tags] โ†’ cached when same tags +[always_keep turns (scoreโ‰ฅ7)] โ†’ fixed position, append-only +[recent turns] โ†’ rolling window +[current user message] โ†’ new each turn +``` + +Active Context injected as **user message** (not system prompt) to keep system prompt prefix stable. diff --git a/docs/design/runtime_loop_task.md b/docs/design/runtime_loop_task.md new file mode 100644 index 000000000..6893eb986 --- /dev/null +++ b/docs/design/runtime_loop_task.md @@ -0,0 +1,55 @@ +๏ปฟ# Runtime Loop Implementation Tasks + +> Design ref: `docs/design/picoclaw_runtime_loop_design.md` + +## Design Decisions + +| Decision | Conclusion | +|----------|------------| +| Turn storage | SQLite `turns.db` | +| Active Context fields | `CurrentFiles` + `RecentErrors` only | +| Phase 1 short-circuit | No โ€” short messages need Active Context most | +| Async write | Direct `go insert()`, no channel buffer | +| Token budget check | Periodic time-based archival instead | +| Tag-gated tools | Deferred until tool count > 15 | +| KV Cache | Fixed ordering (high-score first, by ID ASC) | + +## Milestones + +### M1: Turn Score + Phase 3 Timing โ€” โœ… + +- [x] `score.go`: `CalcTurnScore(input) int` +- [x] Split `RunPostLLM` โ†’ `SyncPhase3` (sync, < 2ms) + `AsyncPhase3` (goroutine) +- [x] Adjust `runAgentLoop` timing: score โ†’ publish โ†’ async write + +### M2: Active Context โ€” โœ… + +- [x] `active_context.go`: per `channel:chatID` store +- [x] Fields: `CurrentFiles` (5), `RecentErrors` (3) +- [x] Injected as user message (not system prompt) for KV cache stability +- [x] Flush to JSON on shutdown, load on startup + +### M3: TurnStore โ€” โœ… + +- [x] `turn_store.go`: SQLite `turns.db` with WAL mode +- [x] Methods: Insert, QueryPending, QueryByScore, QueryByTags, QueryRecent, ArchiveOldProcessed +- [x] Async insert via goroutine in AsyncPhase3 + +### M4: MemoryDigest โ€” โœ… + +- [x] `memory_digest.go`: background worker (5min interval) +- [x] QueryPending โ†’ group by channel โ†’ LLM batch extraction โ†’ MemoryStore +- [x] Removed `MemoryExtractor` and `CotEvaluator` processors (kept `ErrorTracker`) + +### M5: Instant Memory + KV Cache Ordering โ€” โœ… + +- [x] `instant_memory.go`: dynamic window from TurnStore +- [x] Cache-friendly message ordering: system โ†’ memory โ†’ high-score โ†’ recent โ†’ current +- [x] Legacy SessionManager kept as fallback + +## Deferred + +| Item | Reason | +|------|--------| +| Tag-gated tool loading | Tool count < 10 currently | +| Summary Anchor | Fixed ordering sufficient for v1 | \ No newline at end of file diff --git a/pkg/agent/active_context.go b/pkg/agent/active_context.go new file mode 100644 index 000000000..f52bca3e5 --- /dev/null +++ b/pkg/agent/active_context.go @@ -0,0 +1,233 @@ +// PicoClaw - Ultra-lightweight personal AI agent +// License: MIT +// +// Copyright (c) 2026 PicoClaw contributors + +package agent + +import ( + "encoding/json" + "fmt" + "os" + "strings" + "sync" + + "github.com/sipeed/picoclaw/pkg/logger" +) + +// ActiveContext holds the structured per-channel context that Phase 1 uses +// to understand short/ambiguous user messages. +// +// Design choices: +// - CurrentFiles: last 5 file paths touched by tool calls (read/write/edit/append/list_dir). +// - RecentErrors: last 3 tool failure messages. +// - CurrentTask / RecentSummaries are intentionally omitted โ€” they overlap with +// the recent-M turns in instant memory and would be redundant. +type ActiveContext struct { + CurrentFiles []string `json:"current_files"` // newest first, max 5 + RecentErrors []string `json:"recent_errors"` // newest first, max 3 +} + +// ActiveContextStore is a thread-safe in-memory map of channel:chatID โ†’ ActiveContext. +// On startup it is loaded from disk; on stop it is flushed back. +type ActiveContextStore struct { + mu sync.RWMutex + data map[string]*ActiveContext // key = "channel:chatID" +} + +// NewActiveContextStore creates an empty store. +func NewActiveContextStore() *ActiveContextStore { + return &ActiveContextStore{ + data: make(map[string]*ActiveContext), + } +} + +// Get returns a copy of the ActiveContext for the given key (never nil). +func (s *ActiveContextStore) Get(key string) *ActiveContext { + s.mu.RLock() + ac, ok := s.data[key] + s.mu.RUnlock() + + if !ok || ac == nil { + return &ActiveContext{} + } + // Return a shallow copy to avoid callers mutating the store. + cp := *ac + cp.CurrentFiles = append([]string(nil), ac.CurrentFiles...) + cp.RecentErrors = append([]string(nil), ac.RecentErrors...) + return &cp +} + +// fileExtractingTools is the set of tool names whose arguments may carry file paths. +// Keys are lowercase tool names; values indicate the argument name(s) to inspect. +var fileExtractingTools = map[string][]string{ + "read_file": {"path", "file_path", "filename"}, + "write_file": {"path", "file_path", "filename"}, + "edit_file": {"path", "file_path", "filename"}, + "append_file": {"path", "file_path", "filename"}, + "list_dir": {"path", "dir_path", "directory"}, +} + +// Update applies the outcomes of a completed turn to the ActiveContext for key. +// It extracts file paths from tool call arguments and captures error messages. +func (s *ActiveContextStore) Update(key string, input RuntimeInput) { + if key == "" { + return + } + + s.mu.Lock() + defer s.mu.Unlock() + + ac, ok := s.data[key] + if !ok || ac == nil { + ac = &ActiveContext{} + s.data[key] = ac + } + + // Extract file paths from tool calls. + for _, tc := range input.ToolCalls { + name := strings.ToLower(tc.Name) + argFields, relevant := fileExtractingTools[name] + if !relevant { + continue + } + // tc.Args is stored as JSON string or we can check tc.ArgsRaw if available. + // Since ToolCallRecord only has Name/Error/Duration, we skip argument extraction + // here and rely on callers passing a richer input in the future (M5). + // For now we still handle errors. + _ = argFields + } + + // Capture tool errors. + for _, tc := range input.ToolCalls { + if tc.Error == "" { + continue + } + msg := fmt.Sprintf("[%s] %s", tc.Name, tc.Error) + // Prepend (newest first) and cap at 3. + ac.RecentErrors = prependCapped(ac.RecentErrors, msg, 3) + } +} + +// UpdateWithFiles is an extended update that also receives file paths extracted +// by the loop (call this when tool argument parsing is available). +func (s *ActiveContextStore) UpdateWithFiles(key string, input RuntimeInput, filePaths []string) { + s.Update(key, input) + + if len(filePaths) == 0 { + return + } + + s.mu.Lock() + defer s.mu.Unlock() + + ac, ok := s.data[key] + if !ok || ac == nil { + ac = &ActiveContext{} + s.data[key] = ac + } + + for _, p := range filePaths { + if p != "" { + ac.CurrentFiles = prependCapped(ac.CurrentFiles, p, 5) + } + } +} + +// prependCapped prepends item to slice and caps the result at max length. +// Deduplicates: if item already exists it is moved to the front. +func prependCapped(slice []string, item string, max int) []string { + // Remove duplicate. + filtered := make([]string, 0, len(slice)) + for _, s := range slice { + if s != item { + filtered = append(filtered, s) + } + } + result := append([]string{item}, filtered...) + if len(result) > max { + result = result[:max] + } + return result +} + +// Format renders the context as a markdown block for injection into a user message. +// Returns empty string when there is nothing to show. +func (ac *ActiveContext) Format() string { + if len(ac.CurrentFiles) == 0 && len(ac.RecentErrors) == 0 { + return "" + } + var sb strings.Builder + sb.WriteString("## Current Context\n") + if len(ac.CurrentFiles) > 0 { + sb.WriteString("Files in use: ") + sb.WriteString(strings.Join(ac.CurrentFiles, ", ")) + sb.WriteString("\n") + } + if len(ac.RecentErrors) > 0 { + sb.WriteString("Recent errors:\n") + for _, e := range ac.RecentErrors { + sb.WriteString(" - ") + sb.WriteString(e) + sb.WriteString("\n") + } + } + return sb.String() +} + +// --------------------------------------------------------------------------- +// Persistence +// --------------------------------------------------------------------------- + +// persistedStore is the on-disk JSON format for ActiveContextStore. +type persistedStore struct { + Contexts map[string]*ActiveContext `json:"contexts"` +} + +// Flush serialises the store to a JSON file at the given path. +func (s *ActiveContextStore) Flush(path string) error { + s.mu.RLock() + out := persistedStore{Contexts: make(map[string]*ActiveContext, len(s.data))} + for k, v := range s.data { + cp := *v + cp.CurrentFiles = append([]string(nil), v.CurrentFiles...) + cp.RecentErrors = append([]string(nil), v.RecentErrors...) + out.Contexts[k] = &cp + } + s.mu.RUnlock() + + data, err := json.MarshalIndent(out, "", " ") + if err != nil { + return fmt.Errorf("active_context: marshal: %w", err) + } + if err := os.WriteFile(path, data, 0o644); err != nil { + return fmt.Errorf("active_context: write %s: %w", path, err) + } + logger.DebugCF("active_context", "Flushed to disk", map[string]any{"path": path, "keys": len(out.Contexts)}) + return nil +} + +// Load deserialises the store from a JSON file at the given path. +// Missing or unreadable files are silently ignored (returns nil). +func (s *ActiveContextStore) Load(path string) error { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("active_context: read %s: %w", path, err) + } + var out persistedStore + if err := json.Unmarshal(data, &out); err != nil { + return fmt.Errorf("active_context: unmarshal: %w", err) + } + s.mu.Lock() + defer s.mu.Unlock() + for k, v := range out.Contexts { + if v != nil { + s.data[k] = v + } + } + logger.DebugCF("active_context", "Loaded from disk", map[string]any{"path": path, "keys": len(out.Contexts)}) + return nil +} diff --git a/pkg/agent/active_context_test.go b/pkg/agent/active_context_test.go new file mode 100644 index 000000000..6d9e9c36f --- /dev/null +++ b/pkg/agent/active_context_test.go @@ -0,0 +1,120 @@ +package agent + +import ( + "os" + "path/filepath" + "testing" +) + +func TestActiveContextStore_UpdateAndGet(t *testing.T) { + s := NewActiveContextStore() + key := "telegram:12345" + + // Initially empty. + ac := s.Get(key) + if len(ac.CurrentFiles) != 0 || len(ac.RecentErrors) != 0 { + t.Errorf("expected empty context, got %+v", ac) + } + + // Add errors via Update. + s.Update(key, RuntimeInput{ + ToolCalls: []ToolCallRecord{ + {Name: "exec", Error: "timeout after 30s"}, + {Name: "read_file", Error: ""}, + }, + }) + ac = s.Get(key) + if len(ac.RecentErrors) != 1 { + t.Errorf("expected 1 error, got %d: %v", len(ac.RecentErrors), ac.RecentErrors) + } + if ac.RecentErrors[0] != "[exec] timeout after 30s" { + t.Errorf("unexpected error: %s", ac.RecentErrors[0]) + } +} + +func TestActiveContextStore_FileCapping(t *testing.T) { + s := NewActiveContextStore() + key := "cli:direct" + + // Add 7 file paths โ€” should cap at 5, newest first. + s.UpdateWithFiles(key, RuntimeInput{}, []string{"a.go", "b.go", "c.go", "d.go", "e.go", "f.go", "g.go"}) + ac := s.Get(key) + if len(ac.CurrentFiles) != 5 { + t.Fatalf("expected 5 files, got %d: %v", len(ac.CurrentFiles), ac.CurrentFiles) + } + // Last added (g.go) is prepended, so it should be first. + if ac.CurrentFiles[0] != "g.go" { + t.Errorf("expected g.go first, got %s (all: %v)", ac.CurrentFiles[0], ac.CurrentFiles) + } +} + +func TestActiveContextStore_ErrorCapping(t *testing.T) { + s := NewActiveContextStore() + key := "wecom:alice" + + for i := 0; i < 5; i++ { + s.Update(key, RuntimeInput{ + ToolCalls: []ToolCallRecord{{Name: "exec", Error: "err"}}, + }) + } + ac := s.Get(key) + if len(ac.RecentErrors) > 3 { + t.Errorf("expected max 3 errors, got %d", len(ac.RecentErrors)) + } +} + +func TestActiveContextStore_FlushAndLoad(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "active_context.json") + + s := NewActiveContextStore() + key := "cli:direct" + s.UpdateWithFiles(key, RuntimeInput{}, []string{"main.go"}) + s.Update(key, RuntimeInput{ + ToolCalls: []ToolCallRecord{{Name: "exec", Error: "failed"}}, + }) + + if err := s.Flush(path); err != nil { + t.Fatalf("Flush: %v", err) + } + + // File must exist. + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected file to exist: %v", err) + } + + // Load into new store. + s2 := NewActiveContextStore() + if err := s2.Load(path); err != nil { + t.Fatalf("Load: %v", err) + } + ac := s2.Get(key) + if len(ac.CurrentFiles) != 1 || ac.CurrentFiles[0] != "main.go" { + t.Errorf("unexpected files after reload: %v", ac.CurrentFiles) + } + if len(ac.RecentErrors) != 1 { + t.Errorf("unexpected errors after reload: %v", ac.RecentErrors) + } +} + +func TestActiveContextStore_LoadMissingFile(t *testing.T) { + s := NewActiveContextStore() + // Should not error on missing file. + if err := s.Load("/nonexistent/path.json"); err != nil { + t.Errorf("Load of missing file should return nil, got: %v", err) + } +} + +func TestActiveContext_Format(t *testing.T) { + ac := &ActiveContext{ + CurrentFiles: []string{"main.go", "loop.go"}, + RecentErrors: []string{"[exec] timeout"}, + } + formatted := ac.Format() + if formatted == "" { + t.Error("expected non-empty format") + } + if len(formatted) == 0 { + t.Error("Format returned empty string") + } +} diff --git a/pkg/agent/analyser.go b/pkg/agent/analyser.go new file mode 100644 index 000000000..fe4b7ac5e --- /dev/null +++ b/pkg/agent/analyser.go @@ -0,0 +1,282 @@ +// PicoClaw - Ultra-lightweight personal AI agent +// License: MIT +// +// Copyright (c) 2026 PicoClaw contributors + +package agent + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/providers" +) + +// AnalyseResult holds the output of the Phase 1 (Analyse) step. +type AnalyseResult struct { + // Intent is a short label classifying the user's intent (e.g. "question", "task", "chat"). + Intent string `json:"intent"` + // Tags extracted from the user message for memory retrieval. + Tags []string `json:"tags"` + // CotPrompt is an LLM-generated thinking strategy tailored to the user's message. + // Generated by the analyser, not selected from a fixed list. + CotPrompt string `json:"cot_prompt"` + // MemoryContext is the formatted memory entries matching the extracted tags. + // This is populated after the memory lookup, not by the LLM itself. + MemoryContext string `json:"-"` +} + + + +// Analyser performs a lightweight LLM call to analyse the user's message, +// extract intent and tags, then queries the memory store for relevant entries. +// This is Phase 1 of the Runtime Loop. +// +// Flow: +// 1. Collect all available tags from the memory store. +// 2. Call a small/fast LLM with the user message + available tags. +// 3. Parse the JSON response to get intent + matched tags. +// 4. Query memory entries by those tags. +// 5. Return the result with formatted memory context. +type Analyser struct { + provider providers.LLMProvider + model string + cotRegistry *CotRegistry +} + + + +// NewAnalyser creates a new Analyser (Phase 1) processor. +// model should be a lightweight model identifier like "gemini/gemini-2.0-flash-exp". +func NewAnalyser(provider providers.LLMProvider, model string, cotRegistry *CotRegistry) *Analyser { + return &Analyser{ + provider: provider, + model: model, + cotRegistry: cotRegistry, + } +} + + + +const preLLMSystemPromptTpl = `You are a message analysis engine. Your job is to analyse the user's message and output a JSON object. + +## Task + +Given the user message, a list of available memory tags, and reference thinking strategy examples, you must: +1. Determine the user's **intent** โ€” classify it into one short label. +2. Select **relevant tags** from the available tag list. Only select genuinely relevant tags. 0 tags if none are relevant. +3. **Generate a custom thinking strategy** (cot_prompt) for the main AI to follow when processing this message. This should be a concise, actionable set of steps tailored to the specific task. + +## Output Format + +Respond with ONLY a valid JSON object, no markdown fences, no explanation: + +{"intent":"","tags":[""],"cot_prompt":""} + +The cot_prompt should be a brief strategy (3-6 numbered steps). For simple chat/greetings, use an empty string "". + +## Intent Labels + +Use one of: question, task, chat, code, search, create, debug, explain, translate, summarise, other + +## Reference Thinking Strategy Examples + +Use these as inspiration โ€” adapt and combine as needed for the specific message: + +%s + +%s + +## Rules + +- ONLY select tags from the provided available tags list. +- Do NOT invent new tags. Only use tags from the available list. +- Maximum 5 tags. +- Generate a cot_prompt tailored to the specific user message. Don't just copy examples โ€” adapt them. +- For simple chat (greetings, thanks, etc.), use an empty cot_prompt. +- If historical data shows which strategies worked well for similar intents, prefer those approaches. +- Keep the cot_prompt concise: 3-6 actionable steps. +- Keep it fast โ€” this is a preprocessing step.` + +// Analyse runs the pre-LLM analysis on the user message. +// It returns an AnalyseResult with intent, tags, and formatted memory context. +// If the pre-LLM call fails, it returns a zero-value result (no error propagation +// to avoid blocking the main agent loop). +// actCtx may be nil; when provided, its content is injected into the user prompt +// (not the system prompt) to preserve system prompt prefix stability for KV cache. +func (p *Analyser) Analyse(ctx context.Context, userMessage string, memory *MemoryStore, actCtx *ActiveContext) AnalyseResult { + if p.provider == nil || p.model == "" { + return AnalyseResult{} + } + + start := time.Now() + + // 1. Collect available tags from memory store. + var availableTags []string + var tagsErr error + if memory != nil { + availableTags, tagsErr = memory.ListAllTags() + } + hasMemoryTags := tagsErr == nil && len(availableTags) > 0 + + // Even without memory tags, we still call pre-LLM for CoT selection. + + // 2. Build the system prompt with example templates + learning history. + examples := "" + if p.cotRegistry != nil { + examples = p.cotRegistry.ListExamplesForPrompt() + } + // Include historical CoT performance data + top-rated prompts for learning. + cotHistory := "" + if memory != nil { + // Pass available tags so proven examples can be filtered by relevance. + cotHistory = memory.FormatCotLearningContext(30, availableTags) + } + systemPrompt := fmt.Sprintf(preLLMSystemPromptTpl, examples, cotHistory) + + // 3. Build the user prompt with available tags + active context. + var userPromptBuilder strings.Builder + + // Active Context block (injected here to keep system prompt prefix stable). + if actCtx != nil { + if ac := actCtx.Format(); ac != "" { + userPromptBuilder.WriteString(ac) + userPromptBuilder.WriteString("\n\n") + } + } + + if hasMemoryTags { + fmt.Fprintf(&userPromptBuilder, "Available tags: [%s]\n\nUser message: %s", + strings.Join(availableTags, ", "), userMessage) + } else { + fmt.Fprintf(&userPromptBuilder, "Available tags: [](none)\n\nUser message: %s", userMessage) + } + userPrompt := userPromptBuilder.String() + + messages := []providers.Message{ + {Role: "system", Content: systemPrompt}, + {Role: "user", Content: userPrompt}, + } + + // 4. Call the LLM (no tools, moderate max_tokens for generated CoT, low temperature). + resp, err := p.provider.Chat(ctx, messages, nil, p.model, map[string]any{ + "max_tokens": 512, + "temperature": 0.3, + }) + if err != nil { + logger.WarnCF("analyser", "Pre-LLM call failed, proceeding without enrichment", + map[string]any{"error": err.Error(), "model": p.model}) + return AnalyseResult{} + } + + // 5. Parse the JSON response. + result := p.parseResponse(resp.Content) + + // 6. Query memory by extracted tags. + if len(result.Tags) > 0 && memory != nil { + entries, err := memory.SearchByAnyTag(result.Tags) + if err == nil && len(entries) > 0 { + result.MemoryContext = formatMemoryEntries(entries) + } + } + + // 7. Record usage for learning (non-blocking โ€” don't fail the main flow). + if memory != nil && result.CotPrompt != "" { + if _, err := memory.RecordCotUsage(result.Intent, result.Tags, result.CotPrompt, userMessage); err != nil { + logger.DebugCF("analyser", "Failed to record CoT usage", + map[string]any{"error": err.Error()}) + } + } + + elapsed := time.Since(start) + logger.InfoCF("analyser", "Pre-LLM analysis complete", + map[string]any{ + "intent": result.Intent, + "tags": result.Tags, + "has_cot": result.CotPrompt != "", + "cot_len": len(result.CotPrompt), + "memory_entries": countMemoryLines(result.MemoryContext), + "elapsed_ms": elapsed.Milliseconds(), + "model": p.model, + "available_tags": len(availableTags), + }) + + return result +} + +// parseResponse extracts intent and tags from the LLM's JSON response. +// Handles common LLM quirks like markdown fences around JSON. +func (p *Analyser) parseResponse(content string) AnalyseResult { + content = strings.TrimSpace(content) + + // Strip markdown code fences if present. + if strings.HasPrefix(content, "```") { + lines := strings.Split(content, "\n") + // Remove first and last lines (fences). + if len(lines) >= 3 { + content = strings.Join(lines[1:len(lines)-1], "\n") + } + } + content = strings.TrimSpace(content) + + var result AnalyseResult + if err := json.Unmarshal([]byte(content), &result); err != nil { + logger.WarnCF("analyser", "Failed to parse pre-LLM response as JSON", + map[string]any{ + "error": err.Error(), + "content": content, + }) + return AnalyseResult{} + } + + // Sanitise: lowercase tags, limit to 5. + cleaned := make([]string, 0, len(result.Tags)) + for _, t := range result.Tags { + t = strings.ToLower(strings.TrimSpace(t)) + if t != "" { + cleaned = append(cleaned, t) + } + } + if len(cleaned) > 5 { + cleaned = cleaned[:5] + } + result.Tags = cleaned + + return result +} + +// formatMemoryEntries formats memory entries into a string for injection into context. +func formatMemoryEntries(entries []MemoryEntry) string { + if len(entries) == 0 { + return "" + } + + var sb strings.Builder + sb.WriteString("## Relevant Memories (auto-retrieved)\n\n") + for _, e := range entries { + tagLabel := "" + if len(e.Tags) > 0 { + tagLabel = " [" + strings.Join(e.Tags, ", ") + "]" + } + fmt.Fprintf(&sb, "- (#%d%s) %s\n", e.ID, tagLabel, e.Content) + } + return sb.String() +} + +// countMemoryLines counts the number of memory entries in a formatted string. +func countMemoryLines(s string) int { + if s == "" { + return 0 + } + count := 0 + for _, line := range strings.Split(s, "\n") { + if strings.HasPrefix(line, "- (#") { + count++ + } + } + return count +} diff --git a/pkg/agent/analyser_test.go b/pkg/agent/analyser_test.go new file mode 100644 index 000000000..b00377db4 --- /dev/null +++ b/pkg/agent/analyser_test.go @@ -0,0 +1,286 @@ +package agent + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/sipeed/picoclaw/pkg/providers" +) + +func TestPreLLM_parseResponse(t *testing.T) { + p := &Analyser{} + + tests := []struct { + name string + input string + wantIntent string + wantTags []string + wantCot string + }{ + { + name: "valid JSON with cot_prompt", + input: `{"intent":"question","tags":["golang","testing"],"cot_prompt":"1. Understand the question\n2. Research the answer"}`, + wantIntent: "question", + wantTags: []string{"golang", "testing"}, + wantCot: "1. Understand the question\n2. Research the answer", + }, + { + name: "JSON with markdown fences", + input: "```json\n{\"intent\":\"task\",\"tags\":[\"deploy\"],\"cot_prompt\":\"1. Plan\\n2. Execute\"}\n```", + wantIntent: "task", + wantTags: []string{"deploy"}, + wantCot: "1. Plan\n2. Execute", + }, + { + name: "empty cot_prompt for chat", + input: `{"intent":"chat","tags":[],"cot_prompt":""}`, + wantIntent: "chat", + wantTags: []string{}, + wantCot: "", + }, + { + name: "invalid JSON", + input: "this is not json", + wantIntent: "", + wantTags: nil, + wantCot: "", + }, + { + name: "tags trimmed and lowered", + input: `{"intent":"code","tags":[" GoLang "," API "],"cot_prompt":"think"}`, + wantIntent: "code", + wantTags: []string{"golang", "api"}, + wantCot: "think", + }, + { + name: "tags limited to 5", + input: `{"intent":"search","tags":["a","b","c","d","e","f","g"],"cot_prompt":"search"}`, + wantIntent: "search", + wantTags: []string{"a", "b", "c", "d", "e"}, + wantCot: "search", + }, + { + name: "missing cot_prompt field", + input: `{"intent":"question","tags":["golang"]}`, + wantIntent: "question", + wantTags: []string{"golang"}, + wantCot: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := p.parseResponse(tt.input) + if result.Intent != tt.wantIntent { + t.Errorf("intent = %q, want %q", result.Intent, tt.wantIntent) + } + if result.CotPrompt != tt.wantCot { + t.Errorf("cot_prompt = %q, want %q", result.CotPrompt, tt.wantCot) + } + + if tt.wantTags == nil { + if result.Tags != nil { + t.Errorf("tags = %v, want nil", result.Tags) + } + return + } + + if len(result.Tags) != len(tt.wantTags) { + t.Errorf("tags len = %d, want %d (tags=%v)", len(result.Tags), len(tt.wantTags), result.Tags) + return + } + for i, tag := range result.Tags { + if tag != tt.wantTags[i] { + t.Errorf("tag[%d] = %q, want %q", i, tag, tt.wantTags[i]) + } + } + }) + } +} + +func TestPreLLM_Analyse_NoProvider(t *testing.T) { + p := &Analyser{} // no provider, no model + result := p.Analyse(context.Background(), "hello", nil, nil) + if result.Intent != "" || len(result.Tags) != 0 { + t.Errorf("expected empty result with no provider, got %+v", result) + } +} + +func TestPreLLM_Analyse_NoTags(t *testing.T) { + dir := t.TempDir() + ms := NewMemoryStore(dir) + defer ms.Close() + + cotReg := NewCotRegistry(dir) + mp := &mockLLMProvider{ + response: `{"intent":"chat","tags":[],"cot_prompt":""}`, + } + p := NewAnalyser(mp, "test-model", cotReg) + + result := p.Analyse(context.Background(), "hello there", ms, nil) + if result.Intent != "chat" { + t.Errorf("expected intent 'chat', got %q", result.Intent) + } + if result.CotPrompt != "" { + t.Errorf("expected empty cot_prompt for chat, got %q", result.CotPrompt) + } +} + +func TestPreLLM_Analyse_WithMemory(t *testing.T) { + dir := t.TempDir() + ms := NewMemoryStore(dir) + defer ms.Close() + + // Seed memory. + ms.AddEntry("Go is great for concurrency", []string{"golang", "concurrency"}) + ms.AddEntry("Kubernetes cluster setup notes", []string{"k8s", "devops"}) + ms.AddEntry("Go testing best practices", []string{"golang", "testing"}) + + cotReg := NewCotRegistry(dir) + mp := &mockLLMProvider{ + response: `{"intent":"question","tags":["golang"],"cot_prompt":"1. Check Go docs\n2. Write example code\n3. Verify with tests"}`, + } + p := NewAnalyser(mp, "test-model", cotReg) + + result := p.Analyse(context.Background(), "How do I test Go code?", ms, nil) + + if result.Intent != "question" { + t.Errorf("intent = %q, want %q", result.Intent, "question") + } + if result.CotPrompt == "" { + t.Error("expected non-empty CotPrompt") + } + if !strings.Contains(result.CotPrompt, "Go docs") { + t.Error("CotPrompt should contain the LLM-generated strategy") + } + if len(result.Tags) != 1 || result.Tags[0] != "golang" { + t.Errorf("tags = %v, want [golang]", result.Tags) + } + if result.MemoryContext == "" { + t.Error("expected non-empty MemoryContext with matching tags") + } + if !contains(result.MemoryContext, "Go is great for concurrency") { + t.Error("MemoryContext missing 'Go is great for concurrency'") + } + if !contains(result.MemoryContext, "Go testing best practices") { + t.Error("MemoryContext missing 'Go testing best practices'") + } + if contains(result.MemoryContext, "Kubernetes") { + t.Error("MemoryContext should not contain 'Kubernetes' entry") + } + + // Verify usage was recorded with tags. + records, _ := ms.GetRecentCotUsage(1) + if len(records) == 0 { + t.Fatal("expected usage record to be recorded") + } + if len(records[0].Tags) != 1 || records[0].Tags[0] != "golang" { + t.Errorf("recorded tags = %v, want [golang]", records[0].Tags) + } +} + +func TestSearchByAnyTag(t *testing.T) { + dir := t.TempDir() + ms := NewMemoryStore(dir) + defer ms.Close() + + ms.AddEntry("Go concurrency", []string{"golang", "concurrency"}) + ms.AddEntry("K8s setup", []string{"k8s", "devops"}) + ms.AddEntry("Go testing", []string{"golang", "testing"}) + ms.AddEntry("Python ML", []string{"python", "ml"}) + + entries, err := ms.SearchByAnyTag([]string{"golang", "k8s"}) + if err != nil { + t.Fatal(err) + } + if len(entries) != 3 { + t.Errorf("got %d entries, want 3", len(entries)) + } + + entries, err = ms.SearchByAnyTag([]string{"python"}) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Errorf("got %d entries, want 1", len(entries)) + } + + entries, err = ms.SearchByAnyTag([]string{"nonexistent"}) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Errorf("got %d entries, want 0", len(entries)) + } +} + +func TestFormatMemoryEntries(t *testing.T) { + entries := []MemoryEntry{ + {ID: 1, Content: "Test content 1", Tags: []string{"tag1", "tag2"}}, + {ID: 2, Content: "Test content 2", Tags: []string{"tag3"}}, + {ID: 3, Content: "No tags entry", Tags: nil}, + } + + result := formatMemoryEntries(entries) + if result == "" { + t.Fatal("expected non-empty result") + } + if !contains(result, "Relevant Memories") { + t.Error("missing header") + } + if !contains(result, "#1") { + t.Error("missing entry #1") + } + if !contains(result, "[tag1, tag2]") { + t.Error("missing tags for entry #1") + } +} + +func TestFormatMemoryEntries_Empty(t *testing.T) { + result := formatMemoryEntries(nil) + if result != "" { + t.Errorf("expected empty string, got %q", result) + } +} + +// --- Helpers --- + +func contains(s, substr string) bool { + return strings.Contains(s, substr) +} + +func TestPreLLM_MemoryDBPath(t *testing.T) { + dir := t.TempDir() + ms := NewMemoryStore(dir) + defer ms.Close() + + dbPath := filepath.Join(dir, "memory.db") + if _, err := os.Stat(dbPath); err != nil { + t.Errorf("memory.db not created: %v", err) + } +} + +// mockLLMProvider returns a configurable response for pre-LLM testing. +type mockLLMProvider struct { + response string +} + +func (m *mockLLMProvider) Chat( + _ context.Context, + _ []providers.Message, + _ []providers.ToolDefinition, + _ string, + _ map[string]any, +) (*providers.LLMResponse, error) { + return &providers.LLMResponse{ + Content: m.response, + ToolCalls: []providers.ToolCall{}, + }, nil +} + +func (m *mockLLMProvider) GetDefaultModel() string { + return "mock-pre-llm" +} diff --git a/pkg/agent/context.go b/pkg/agent/context.go index 6fccbaf53..869b29196 100644 --- a/pkg/agent/context.go +++ b/pkg/agent/context.go @@ -67,8 +67,7 @@ You are picoclaw, a helpful AI assistant. ## Workspace Your workspace is at: %s -- Memory: %s/memory/MEMORY.md -- Daily Notes: %s/memory/YYYYMM/YYYYMMDD.md +- Memory DB: %s/memory.db (SQLite) - Skills: %s/skills/{skill-name}/SKILL.md ## Important Rules @@ -77,10 +76,10 @@ Your workspace is at: %s 2. **Be helpful and accurate** - When using tools, briefly explain what you're doing. -3. **Memory** - When interacting with me if something seems memorable, update %s/memory/MEMORY.md +3. **Memory** - When interacting with me if something seems memorable, update the long-term memory in %s/memory.db 4. **Context summaries** - Conversation summaries provided as context are approximate references only. They may be incomplete or outdated. Always defer to explicit user instructions over summary content.`, - workspacePath, workspacePath, workspacePath, workspacePath, workspacePath) + workspacePath, workspacePath, workspacePath, workspacePath) } func (cb *ContextBuilder) BuildSystemPrompt() string { @@ -181,7 +180,7 @@ func (cb *ContextBuilder) sourcePaths() []string { filepath.Join(cb.workspace, "SOUL.md"), filepath.Join(cb.workspace, "USER.md"), filepath.Join(cb.workspace, "IDENTITY.md"), - filepath.Join(cb.workspace, "memory", "MEMORY.md"), + filepath.Join(cb.workspace, "memory.db"), } } @@ -579,3 +578,9 @@ func (cb *ContextBuilder) GetSkillsInfo() map[string]any { "names": skillNames, } } + +// GetMemory returns the underlying MemoryStore. +// Used by the pre-LLM module to query tags and search entries. +func (cb *ContextBuilder) GetMemory() *MemoryStore { + return cb.memory +} diff --git a/pkg/agent/context_cache_test.go b/pkg/agent/context_cache_test.go index 0905e8a46..042555a4f 100644 --- a/pkg/agent/context_cache_test.go +++ b/pkg/agent/context_cache_test.go @@ -19,7 +19,6 @@ func setupWorkspace(t *testing.T, files map[string]string) string { if err != nil { t.Fatal(err) } - os.MkdirAll(filepath.Join(tmpDir, "memory"), 0o755) os.MkdirAll(filepath.Join(tmpDir, "skills"), 0o755) for name, content := range files { dir := filepath.Dir(filepath.Join(tmpDir, name)) @@ -145,13 +144,6 @@ func TestMtimeAutoInvalidation(t *testing.T) { contentV2: "# Updated Identity", checkField: "Updated Identity", }, - { - name: "memory file change", - file: "memory/MEMORY.md", - contentV1: "# Memory\nUser likes Go.", - contentV2: "# Memory\nUser likes Rust.", - checkField: "User likes Rust", - }, } for _, tt := range tests { @@ -212,6 +204,43 @@ func TestMtimeAutoInvalidation(t *testing.T) { t.Error("sourceFilesChangedLocked() should detect skills dir mtime change") } }) + + // Memory DB mtime change (via MemoryStore write) + t.Run("memory DB change", func(t *testing.T) { + tmpDir := setupWorkspace(t, nil) + defer os.RemoveAll(tmpDir) + + cb := NewContextBuilder(tmpDir) + + // Write initial memory + cb.memory.WriteLongTerm("User likes Go.") + + // Build cache + sp1 := cb.BuildSystemPromptWithCache() + if !strings.Contains(sp1, "User likes Go") { + t.Fatal("initial prompt should contain memory content") + } + + // Update memory via MemoryStore + cb.memory.WriteLongTerm("User likes Rust.") + + // Set future mtime on memory.db so cache detects change + dbPath := filepath.Join(tmpDir, "memory.db") + future := time.Now().Add(2 * time.Second) + os.Chtimes(dbPath, future, future) + + cb.systemPromptMutex.RLock() + changed := cb.sourceFilesChangedLocked() + cb.systemPromptMutex.RUnlock() + if !changed { + t.Fatal("sourceFilesChangedLocked() should detect memory.db change") + } + + sp2 := cb.BuildSystemPromptWithCache() + if !strings.Contains(sp2, "User likes Rust") { + t.Error("rebuilt prompt should contain updated memory") + } + }) } // TestExplicitInvalidateCache verifies that InvalidateCache() forces a rebuild @@ -273,57 +302,35 @@ func TestCacheStability(t *testing.T) { // This catches the "from nothing to something" edge case that the old // modifiedSince (return false on stat error) would miss. func TestNewFileCreationInvalidatesCache(t *testing.T) { - tests := []struct { - name string - file string // relative path inside workspace - content string - checkField string // substring to verify in rebuilt prompt - }{ - { - name: "new bootstrap file", - file: "SOUL.md", - content: "# Soul\nBe kind and helpful.", - checkField: "Be kind and helpful", - }, - { - name: "new memory file", - file: "memory/MEMORY.md", - content: "# Memory\nUser prefers dark mode.", - checkField: "User prefers dark mode", - }, - } + // Test bootstrap file creation + t.Run("new bootstrap file", func(t *testing.T) { + // Start with an empty workspace (no bootstrap files) + tmpDir := setupWorkspace(t, nil) + defer os.RemoveAll(tmpDir) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Start with an empty workspace (no bootstrap/memory files) - tmpDir := setupWorkspace(t, nil) - defer os.RemoveAll(tmpDir) + cb := NewContextBuilder(tmpDir) - cb := NewContextBuilder(tmpDir) + // Populate cache โ€” file does not exist yet + sp1 := cb.BuildSystemPromptWithCache() + if strings.Contains(sp1, "Be kind and helpful") { + t.Fatalf("prompt should not contain content before file is created") + } - // Populate cache โ€” file does not exist yet - sp1 := cb.BuildSystemPromptWithCache() - if strings.Contains(sp1, tt.checkField) { - t.Fatalf("prompt should not contain %q before file is created", tt.checkField) - } + // Create the file after cache was built + fullPath := filepath.Join(tmpDir, "SOUL.md") + if err := os.WriteFile(fullPath, []byte("# Soul\nBe kind and helpful."), 0o644); err != nil { + t.Fatal(err) + } + // Set future mtime to guarantee detection + future := time.Now().Add(2 * time.Second) + os.Chtimes(fullPath, future, future) - // Create the file after cache was built - fullPath := filepath.Join(tmpDir, tt.file) - os.MkdirAll(filepath.Dir(fullPath), 0o755) - if err := os.WriteFile(fullPath, []byte(tt.content), 0o644); err != nil { - t.Fatal(err) - } - // Set future mtime to guarantee detection - future := time.Now().Add(2 * time.Second) - os.Chtimes(fullPath, future, future) - - // Cache should auto-invalidate because file went from absent -> present - sp2 := cb.BuildSystemPromptWithCache() - if !strings.Contains(sp2, tt.checkField) { - t.Errorf("cache not invalidated on new file creation: expected %q in prompt", tt.checkField) - } - }) - } + // Cache should auto-invalidate because file went from absent -> present + sp2 := cb.BuildSystemPromptWithCache() + if !strings.Contains(sp2, "Be kind and helpful") { + t.Errorf("cache not invalidated on new file creation") + } + }) } // TestSkillFileContentChange verifies that modifying a skill file's content @@ -391,7 +398,6 @@ func TestConcurrentBuildSystemPromptWithCache(t *testing.T) { tmpDir := setupWorkspace(t, map[string]string{ "IDENTITY.md": "# Identity\nConcurrency test agent.", "SOUL.md": "# Soul\nBe helpful.", - "memory/MEMORY.md": "# Memory\nUser prefers Go.", "skills/demo/SKILL.md": "---\nname: demo\ndescription: \"demo skill\"\n---\n# Demo", }) defer os.RemoveAll(tmpDir) @@ -494,7 +500,6 @@ func BenchmarkBuildMessagesWithCache(b *testing.B) { tmpDir, _ := os.MkdirTemp("", "picoclaw-bench-*") defer os.RemoveAll(tmpDir) - os.MkdirAll(filepath.Join(tmpDir, "memory"), 0o755) os.MkdirAll(filepath.Join(tmpDir, "skills"), 0o755) for _, name := range []string{"IDENTITY.md", "SOUL.md", "USER.md"} { os.WriteFile(filepath.Join(tmpDir, name), []byte(strings.Repeat("Content.\n", 10)), 0o644) diff --git a/pkg/agent/cot_learning_test.go b/pkg/agent/cot_learning_test.go new file mode 100644 index 000000000..829be72d8 --- /dev/null +++ b/pkg/agent/cot_learning_test.go @@ -0,0 +1,297 @@ +package agent + +import ( + "strings" + "testing" +) + +func TestCotUsage_RecordAndQuery(t *testing.T) { + dir := t.TempDir() + ms := NewMemoryStore(dir) + defer ms.Close() + + // Record some usage with tags. + id1, err := ms.RecordCotUsage("code", []string{"golang", "testing"}, "1. Check tests\n2. Write code", "How do I test Go code?") + if err != nil { + t.Fatal(err) + } + if id1 <= 0 { + t.Errorf("expected positive ID, got %d", id1) + } + + id2, err := ms.RecordCotUsage("question", []string{"golang"}, "1. Compare options\n2. Decide", "What's the difference?") + if err != nil { + t.Fatal(err) + } + + id3, err := ms.RecordCotUsage("code", []string{"http", "golang"}, "1. Define routes\n2. Implement handlers", "Write a HTTP server") + if err != nil { + t.Fatal(err) + } + + // Query recent usage. + records, err := ms.GetRecentCotUsage(10) + if err != nil { + t.Fatal(err) + } + if len(records) != 3 { + t.Errorf("expected 3 records, got %d", len(records)) + } + + // Most recent first. + if records[0].ID != id3 { + t.Errorf("expected most recent to be id3=%d, got %d", id3, records[0].ID) + } + + // Check tags are stored correctly. + if len(records[0].Tags) != 2 || records[0].Tags[0] != "http" { + t.Errorf("tags = %v, want [http, golang]", records[0].Tags) + } + + // Check cot_prompt is stored. + if !strings.Contains(records[0].CotPrompt, "Define routes") { + t.Errorf("cot_prompt = %q, should contain 'Define routes'", records[0].CotPrompt) + } + + _ = id2 // used above +} + +func TestCotUsage_Feedback(t *testing.T) { + dir := t.TempDir() + ms := NewMemoryStore(dir) + defer ms.Close() + + id, _ := ms.RecordCotUsage("code", []string{"golang"}, "think step by step", "test message") + + // Initial feedback should be 0. + records, _ := ms.GetRecentCotUsage(1) + if records[0].Feedback != 0 { + t.Errorf("initial feedback = %d, want 0", records[0].Feedback) + } + + // Update feedback. + err := ms.UpdateCotFeedback(id, 1) + if err != nil { + t.Fatal(err) + } + + records, _ = ms.GetRecentCotUsage(1) + if records[0].Feedback != 1 { + t.Errorf("feedback = %d, want 1", records[0].Feedback) + } + + // Invalid score. + err = ms.UpdateCotFeedback(id, 5) + if err == nil { + t.Error("expected error for invalid score 5") + } +} + +func TestCotUsage_UpdateLatestFeedback(t *testing.T) { + dir := t.TempDir() + ms := NewMemoryStore(dir) + defer ms.Close() + + ms.RecordCotUsage("code", nil, "strategy 1", "first") + ms.RecordCotUsage("debug", nil, "strategy 2", "second") + + // Update latest (should be "debug"). + err := ms.UpdateLatestCotFeedback(-1) + if err != nil { + t.Fatal(err) + } + + records, _ := ms.GetRecentCotUsage(2) + if records[0].Intent != "debug" || records[0].Feedback != -1 { + t.Errorf("latest: intent=%q feedback=%d, want debug/-1", records[0].Intent, records[0].Feedback) + } + if records[1].Intent != "code" || records[1].Feedback != 0 { + t.Errorf("first: intent=%q feedback=%d, want code/0", records[1].Intent, records[1].Feedback) + } +} + +func TestCotUsage_Stats(t *testing.T) { + dir := t.TempDir() + ms := NewMemoryStore(dir) + defer ms.Close() + + id1, _ := ms.RecordCotUsage("code", nil, "think about code", "write code") + ms.UpdateCotFeedback(id1, 1) + + id2, _ := ms.RecordCotUsage("code", nil, "debug systematically", "fix bug") + ms.UpdateCotFeedback(id2, 1) + + id3, _ := ms.RecordCotUsage("question", nil, "analyse step by step", "why does X happen?") + ms.UpdateCotFeedback(id3, -1) + + id4, _ := ms.RecordCotUsage("chat", nil, "", "hello") + ms.UpdateCotFeedback(id4, 1) + + // Get stats. + stats, err := ms.GetCotStats(30) + if err != nil { + t.Fatal(err) + } + if len(stats) != 3 { + t.Errorf("expected 3 intent stats, got %d", len(stats)) + } + + // "code" should have highest total uses. + if stats[0].Intent != "code" || stats[0].TotalUses != 2 { + t.Errorf("expected code with 2 uses, got %q with %d", stats[0].Intent, stats[0].TotalUses) + } +} + +func TestCotUsage_TopRatedPrompts(t *testing.T) { + dir := t.TempDir() + ms := NewMemoryStore(dir) + defer ms.Close() + + // Record with different tags and feedback. + id1, _ := ms.RecordCotUsage("code", []string{"golang", "testing"}, "1. Write test first\n2. Then implement", "write Go test") + ms.UpdateCotFeedback(id1, 1) + + id2, _ := ms.RecordCotUsage("code", []string{"python"}, "1. Use pytest\n2. Mock dependencies", "write Python test") + ms.UpdateCotFeedback(id2, 1) + + id3, _ := ms.RecordCotUsage("debug", []string{"golang"}, "1. Reproduce\n2. Hypothesize", "fix Go bug") + ms.UpdateCotFeedback(id3, 1) + + id4, _ := ms.RecordCotUsage("code", []string{"golang"}, "1. Bad strategy", "bad approach") + ms.UpdateCotFeedback(id4, -1) // Negative โ€” should not appear. + + // Without tag filter. + top, err := ms.GetTopRatedCotPrompts(30, 10, nil) + if err != nil { + t.Fatal(err) + } + if len(top) != 3 { + t.Errorf("expected 3 top-rated, got %d", len(top)) + } + + // With tag filter โ€” "golang" should prioritise golang-tagged prompts. + top, err = ms.GetTopRatedCotPrompts(30, 2, []string{"golang"}) + if err != nil { + t.Fatal(err) + } + if len(top) != 2 { + t.Errorf("expected 2, got %d", len(top)) + } + // First result should have golang tag. + hasGolang := false + for _, tag := range top[0].Tags { + if tag == "golang" { + hasGolang = true + } + } + if !hasGolang { + t.Errorf("first result should have golang tag, got %v", top[0].Tags) + } +} + +func TestCotUsage_FormatLearningContext(t *testing.T) { + dir := t.TempDir() + ms := NewMemoryStore(dir) + defer ms.Close() + + // Empty โ€” should return empty string. + ctx := ms.FormatCotLearningContext(30, nil) + if ctx != "" { + t.Errorf("expected empty learning context, got %q", ctx) + } + + // Add some usage with feedback. + id1, _ := ms.RecordCotUsage("code", []string{"golang"}, "1. Understand requirements\n2. Write code", "write code") + ms.UpdateCotFeedback(id1, 1) + + id2, _ := ms.RecordCotUsage("question", []string{"architecture"}, "1. Examine structure\n2. Explain", "why does X happen?") + ms.UpdateCotFeedback(id2, 1) + + ctx = ms.FormatCotLearningContext(30, nil) + if ctx == "" { + t.Error("expected non-empty learning context after recording usage") + } + if !strings.Contains(ctx, "Historical Usage Stats") { + t.Error("missing stats header") + } + if !strings.Contains(ctx, "Proven Strategies") { + t.Error("missing proven strategies section") + } + if !strings.Contains(ctx, "golang") { + t.Error("should show tags in proven examples") + } +} + +func TestCotUsage_MessageTruncation(t *testing.T) { + dir := t.TempDir() + ms := NewMemoryStore(dir) + defer ms.Close() + + longMsg := strings.Repeat("x", 500) + _, err := ms.RecordCotUsage("code", nil, "strategy", longMsg) + if err != nil { + t.Fatal(err) + } + + records, _ := ms.GetRecentCotUsage(1) + if len(records[0].Message) > 200 { + t.Errorf("message should be truncated to 200 chars, got %d", len(records[0].Message)) + } +} + +func TestPreLLM_LearningIntegration(t *testing.T) { + dir := t.TempDir() + ms := NewMemoryStore(dir) + defer ms.Close() + + cotReg := NewCotRegistry(dir) + mp := &mockLLMProvider{ + response: `{"intent":"code","tags":["golang"],"cot_prompt":"1. Understand the function signature\n2. Write the implementation\n3. Add error handling"}`, + } + p := NewAnalyser(mp, "test-model", cotReg) + + // First call โ€” no learning data yet. + result := p.Analyse(nil, "write a function", ms, nil) + if result.CotPrompt == "" { + t.Error("expected non-empty CotPrompt") + } + + // Verify usage was recorded with tags. + records, _ := ms.GetRecentCotUsage(5) + if len(records) != 1 { + t.Fatalf("expected 1 usage record, got %d", len(records)) + } + if records[0].Intent != "code" { + t.Errorf("recorded intent = %q, want %q", records[0].Intent, "code") + } + if len(records[0].Tags) != 1 || records[0].Tags[0] != "golang" { + t.Errorf("recorded tags = %v, want [golang]", records[0].Tags) + } + if records[0].CotPrompt == "" { + t.Error("recorded cot_prompt should not be empty") + } + + // Provide positive feedback. + ms.UpdateLatestCotFeedback(1) + + // Second call โ€” learning context should now be included. + result2 := p.Analyse(nil, "fix this bug", ms, nil) + if result2.CotPrompt == "" { + t.Error("expected non-empty CotPrompt on second call") + } + + // Should now have 2 usage records. + records, _ = ms.GetRecentCotUsage(5) + if len(records) != 2 { + t.Errorf("expected 2 usage records, got %d", len(records)) + } + + // Learning context should include the first proven strategy. + ctx := ms.FormatCotLearningContext(30, []string{"golang"}) + if ctx == "" { + t.Error("expected non-empty learning context after usage + feedback") + } + if !strings.Contains(ctx, "Proven Strategies") { + t.Error("learning context should include proven strategies") + } +} diff --git a/pkg/agent/cot_templates.go b/pkg/agent/cot_templates.go new file mode 100644 index 000000000..b17c6b71e --- /dev/null +++ b/pkg/agent/cot_templates.go @@ -0,0 +1,287 @@ +// PicoClaw - Ultra-lightweight personal AI agent +// License: MIT +// +// Copyright (c) 2026 PicoClaw contributors + +package agent + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/sipeed/picoclaw/pkg/logger" +) + +// CotTemplate represents a Chain-of-Thought prompting template. +type CotTemplate struct { + ID string // Short identifier (e.g. "analytical", "code") + Name string // Human-readable name + Description string // One-line description for the pre-LLM to choose from + Prompt string // The actual CoT instruction injected into the system prompt +} + +// --- Built-in CoT Templates ------------------------------------------------- + +var builtinCotTemplates = []CotTemplate{ + { + ID: "direct", + Name: "Direct Answer", + Description: "Simple, direct response โ€” no special reasoning needed", + Prompt: "", // No CoT injection for simple answers + }, + { + ID: "analytical", + Name: "Analytical Reasoning", + Description: "Complex questions requiring step-by-step logical analysis", + Prompt: `## Thinking Strategy: Analytical Reasoning + +Before answering, follow this reasoning process: +1. **Clarify** โ€” Restate the core question in your own words. +2. **Decompose** โ€” Break it into sub-problems or key aspects. +3. **Analyse** โ€” Work through each sub-problem with evidence/logic. +4. **Synthesise** โ€” Combine findings into a coherent answer. +5. **Verify** โ€” Check for logical gaps or contradictions.`, + }, + { + ID: "code", + Name: "Code Analysis", + Description: "Writing, reviewing, or understanding code", + Prompt: `## Thinking Strategy: Code Analysis + +Before writing or analysing code: +1. **Requirements** โ€” What exactly needs to be done? +2. **Inputs/Outputs** โ€” Define the interface: what goes in, what comes out. +3. **Edge Cases** โ€” Consider boundary conditions, errors, empty inputs, concurrency. +4. **Approach** โ€” Choose the algorithm/pattern, justify the choice. +5. **Implement** โ€” Write clean, well-commented code. +6. **Test** โ€” Mentally trace through with sample inputs to verify correctness.`, + }, + { + ID: "debug", + Name: "Debugging", + Description: "Finding and fixing bugs, errors, or unexpected behaviour", + Prompt: `## Thinking Strategy: Debugging + +Follow a systematic debugging approach: +1. **Reproduce** โ€” Understand the exact symptoms and conditions. +2. **Hypothesise** โ€” List 2-3 most likely root causes. +3. **Narrow Down** โ€” For each hypothesis, describe what evidence would confirm/deny it. +4. **Root Cause** โ€” Identify the actual root cause with evidence. +5. **Fix** โ€” Propose the minimal, targeted fix. +6. **Verify** โ€” Confirm the fix resolves the issue without side effects.`, + }, + { + ID: "creative", + Name: "Creative Thinking", + Description: "Brainstorming, creative writing, idea generation", + Prompt: `## Thinking Strategy: Creative Exploration + +Use divergent-convergent thinking: +1. **Diverge** โ€” Generate multiple distinct ideas or approaches without judgment. +2. **Explore** โ€” Expand on the most promising 2-3 ideas. +3. **Combine** โ€” Look for unexpected connections between ideas. +4. **Converge** โ€” Select the best approach and refine it. +5. **Polish** โ€” Add detail, nuance, and completeness.`, + }, + { + ID: "task", + Name: "Task Planning", + Description: "Multi-step tasks, planning, project work", + Prompt: `## Thinking Strategy: Task Planning + +Plan before executing: +1. **Goal** โ€” What is the desired end state? +2. **Current State** โ€” What exists now? What resources are available? +3. **Steps** โ€” Break into ordered, actionable steps. +4. **Dependencies** โ€” Identify which steps depend on others. +5. **Risks** โ€” What could go wrong? How to mitigate? +6. **Execute** โ€” Carry out steps, adapting as needed.`, + }, + { + ID: "explain", + Name: "Explain / Teach", + Description: "Teaching concepts, explaining how things work", + Prompt: `## Thinking Strategy: Educational Explanation + +Structure your explanation for clarity: +1. **Big Picture** โ€” Start with a one-sentence summary of the concept. +2. **Analogy** โ€” Relate to something familiar if possible. +3. **Core Mechanism** โ€” Explain how it works step by step. +4. **Example** โ€” Provide a concrete example or demonstration. +5. **Gotchas** โ€” Mention common misconceptions or pitfalls.`, + }, + { + ID: "compare", + Name: "Comparison / Decision", + Description: "Comparing options, making decisions, trade-off analysis", + Prompt: `## Thinking Strategy: Comparison Analysis + +Structure your analysis: +1. **Criteria** โ€” Define what matters most for this decision. +2. **Options** โ€” List all viable options. +3. **Trade-offs** โ€” For each option, list pros and cons against the criteria. +4. **Recommendation** โ€” State the best choice with clear reasoning. +5. **Caveats** โ€” Note when the recommendation might not apply.`, + }, +} + +// --- CoT Template Registry -------------------------------------------------- + +// CotRegistry manages the available CoT templates. +// It loads built-in templates and supports user-defined ones from workspace. +type CotRegistry struct { + mu sync.RWMutex + templates map[string]CotTemplate +} + +// NewCotRegistry creates a registry with built-in templates and optionally +// loads user-defined templates from the workspace/cot_templates/ directory. +func NewCotRegistry(workspace string) *CotRegistry { + r := &CotRegistry{ + templates: make(map[string]CotTemplate, len(builtinCotTemplates)), + } + + // Register built-in templates. + for _, t := range builtinCotTemplates { + r.templates[t.ID] = t + } + + // Load user-defined templates from workspace. + r.loadUserTemplates(workspace) + + return r +} + +// Get returns a template by ID (case-insensitive). Returns the "direct" +// template if not found. +func (r *CotRegistry) Get(id string) CotTemplate { + r.mu.RLock() + defer r.mu.RUnlock() + + id = strings.ToLower(strings.TrimSpace(id)) + if t, ok := r.templates[id]; ok { + return t + } + return r.templates["direct"] +} + +// ListForPrompt returns a formatted list of available template IDs and +// descriptions, suitable for quick reference. +func (r *CotRegistry) ListForPrompt() string { + r.mu.RLock() + defer r.mu.RUnlock() + + var sb strings.Builder + for _, t := range builtinCotTemplates { + fmt.Fprintf(&sb, "- %s: %s\n", t.ID, t.Description) + } + + // Append user-defined templates. + for id, t := range r.templates { + isBuiltin := false + for _, bt := range builtinCotTemplates { + if bt.ID == id { + isBuiltin = true + break + } + } + if !isBuiltin { + fmt.Fprintf(&sb, "- %s: %s\n", t.ID, t.Description) + } + } + + return sb.String() +} + +// ListExamplesForPrompt returns full template examples for the pre-LLM to +// use as inspiration when generating custom CoT prompts. +// Shows 3-4 diverse examples with their full prompt content. +func (r *CotRegistry) ListExamplesForPrompt() string { + r.mu.RLock() + defer r.mu.RUnlock() + + // Select a diverse set of examples (not all โ€” keep prompt concise). + exampleIDs := []string{"analytical", "code", "debug", "task"} + + var sb strings.Builder + for _, id := range exampleIDs { + t, ok := r.templates[id] + if !ok || t.Prompt == "" { + continue + } + fmt.Fprintf(&sb, "### Example: %s (%s)\n%s\n\n", t.Name, t.Description, t.Prompt) + } + + // Append any user-defined templates as additional examples. + for id, t := range r.templates { + isBuiltin := false + for _, bt := range builtinCotTemplates { + if bt.ID == id { + isBuiltin = true + break + } + } + if !isBuiltin && t.Prompt != "" { + fmt.Fprintf(&sb, "### Example: %s (%s)\n%s\n\n", t.Name, t.Description, t.Prompt) + } + } + + return sb.String() +} + +// loadUserTemplates scans workspace/cot_templates/ for .md files. +// Each file becomes a template with ID = filename (without .md). +// File format: +// +// Line 1: description (one line) +// Line 2: --- +// Line 3+: prompt content +func (r *CotRegistry) loadUserTemplates(workspace string) { + dir := filepath.Join(workspace, "cot_templates") + entries, err := os.ReadDir(dir) + if err != nil { + return // Directory doesn't exist โ€” that's fine. + } + + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".md") { + continue + } + + data, err := os.ReadFile(filepath.Join(dir, entry.Name())) + if err != nil { + continue + } + + id := strings.TrimSuffix(entry.Name(), ".md") + id = strings.ToLower(strings.TrimSpace(id)) + if id == "" { + continue + } + + content := string(data) + description := id + prompt := content + + // Parse optional description header. + if idx := strings.Index(content, "\n---\n"); idx > 0 { + description = strings.TrimSpace(content[:idx]) + prompt = strings.TrimSpace(content[idx+5:]) + } + + r.mu.Lock() + r.templates[id] = CotTemplate{ + ID: id, + Name: id, + Description: description, + Prompt: prompt, + } + r.mu.Unlock() + + logger.DebugCF("cot", "Loaded user CoT template", + map[string]any{"id": id, "description": description}) + } +} diff --git a/pkg/agent/cot_templates_test.go b/pkg/agent/cot_templates_test.go new file mode 100644 index 000000000..66622ca69 --- /dev/null +++ b/pkg/agent/cot_templates_test.go @@ -0,0 +1,146 @@ +package agent + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestCotRegistry_BuiltinTemplates(t *testing.T) { + dir := t.TempDir() + r := NewCotRegistry(dir) + + // Should have all built-in templates. + for _, bt := range builtinCotTemplates { + tmpl := r.Get(bt.ID) + if tmpl.ID != bt.ID { + t.Errorf("expected template %q, got %q", bt.ID, tmpl.ID) + } + } + + // "direct" should have empty prompt. + direct := r.Get("direct") + if direct.Prompt != "" { + t.Errorf("direct template should have empty prompt, got %q", direct.Prompt) + } + + // "code" should have non-empty prompt. + code := r.Get("code") + if code.Prompt == "" { + t.Error("code template should have non-empty prompt") + } + if !strings.Contains(code.Prompt, "Code Analysis") { + t.Error("code template should mention 'Code Analysis'") + } +} + +func TestCotRegistry_UnknownFallsToDefault(t *testing.T) { + dir := t.TempDir() + r := NewCotRegistry(dir) + + tmpl := r.Get("nonexistent_template") + if tmpl.ID != "direct" { + t.Errorf("expected fallback to 'direct', got %q", tmpl.ID) + } +} + +func TestCotRegistry_CaseInsensitive(t *testing.T) { + dir := t.TempDir() + r := NewCotRegistry(dir) + + tmpl := r.Get(" Code ") + if tmpl.ID != "code" { + t.Errorf("expected 'code', got %q", tmpl.ID) + } +} + +func TestCotRegistry_UserTemplates(t *testing.T) { + dir := t.TempDir() + + // Create user template. + cotDir := filepath.Join(dir, "cot_templates") + os.MkdirAll(cotDir, 0o755) + + content := `Custom strategy for data analysis +--- +## Thinking Strategy: Data Analysis + +1. Examine the data structure. +2. Identify patterns. +3. Draw conclusions.` + + os.WriteFile(filepath.Join(cotDir, "data_analysis.md"), []byte(content), 0o644) + + r := NewCotRegistry(dir) + + // Should be able to get the user template. + tmpl := r.Get("data_analysis") + if tmpl.ID != "data_analysis" { + t.Errorf("expected 'data_analysis', got %q", tmpl.ID) + } + if tmpl.Description != "Custom strategy for data analysis" { + t.Errorf("description = %q, want 'Custom strategy for data analysis'", tmpl.Description) + } + if !strings.Contains(tmpl.Prompt, "Examine the data structure") { + t.Error("prompt should contain user-defined content") + } +} + +func TestCotRegistry_ListForPrompt(t *testing.T) { + dir := t.TempDir() + r := NewCotRegistry(dir) + + list := r.ListForPrompt() + + // Should contain all built-in template IDs. + for _, bt := range builtinCotTemplates { + if !strings.Contains(list, bt.ID) { + t.Errorf("ListForPrompt missing template %q", bt.ID) + } + } +} + +func TestCotRegistry_ListExamplesForPrompt(t *testing.T) { + dir := t.TempDir() + r := NewCotRegistry(dir) + + examples := r.ListExamplesForPrompt() + + // Should contain full example content for key templates. + if !strings.Contains(examples, "Code Analysis") { + t.Error("ListExamplesForPrompt missing 'Code Analysis' example") + } + if !strings.Contains(examples, "Analytical Reasoning") { + t.Error("ListExamplesForPrompt missing 'Analytical Reasoning' example") + } + if !strings.Contains(examples, "Debugging") { + t.Error("ListExamplesForPrompt missing 'Debugging' example") + } + // Should contain actual steps, not just names. + if !strings.Contains(examples, "Requirements") { + t.Error("ListExamplesForPrompt should include actual step content") + } +} + +func TestCotRegistry_UserOverridesBuiltin(t *testing.T) { + dir := t.TempDir() + // Create a user template that overrides "code". + cotDir := filepath.Join(dir, "cot_templates") + os.MkdirAll(cotDir, 0o755) + + content := `My custom code template +--- +## Custom Code Strategy + +Think differently about code.` + + os.WriteFile(filepath.Join(cotDir, "code.md"), []byte(content), 0o644) + + r := NewCotRegistry(dir) + + tmpl := r.Get("code") + if !strings.Contains(tmpl.Prompt, "Think differently about code") { + t.Error("user template should override built-in 'code' template") + } +} diff --git a/pkg/agent/executor.go b/pkg/agent/executor.go new file mode 100644 index 000000000..0ff244ec8 --- /dev/null +++ b/pkg/agent/executor.go @@ -0,0 +1,596 @@ +// PicoClaw - Ultra-lightweight personal AI agent +// License: MIT +// +// Copyright (c) 2026 PicoClaw contributors + +package agent + +// executor.go - Phase 2 (ExecuteLLM) logic extracted from loop.go. +// Contains the LLM iteration loop, tool handling, reasoning output, +// context compression, and logging helpers. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + "unicode/utf8" + + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/constants" + "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/providers" + "github.com/sipeed/picoclaw/pkg/tools" + "github.com/sipeed/picoclaw/pkg/utils" +) + +func (al *AgentLoop) targetReasoningChannelID(channelName string) (chatID string) { + if al.channelManager == nil { + return "" + } + if ch, ok := al.channelManager.GetChannel(channelName); ok { + return ch.ReasoningChannelID() + } + return "" +} + +func (al *AgentLoop) handleReasoning(ctx context.Context, reasoningContent, channelName, channelID string) { + if reasoningContent == "" || channelName == "" || channelID == "" { + return + } + + // Check context cancellation before attempting to publish, + // since PublishOutbound's select may race between send and ctx.Done(). + if ctx.Err() != nil { + return + } + + // Use a short timeout so the goroutine does not block indefinitely when + // the outbound bus is full. Reasoning output is best-effort; dropping it + // is acceptable to avoid goroutine accumulation. + pubCtx, pubCancel := context.WithTimeout(ctx, 5*time.Second) + defer pubCancel() + + if err := al.bus.PublishOutbound(pubCtx, bus.OutboundMessage{ + Channel: channelName, + ChatID: channelID, + Content: reasoningContent, + }); err != nil { + // Treat context.DeadlineExceeded / context.Canceled as expected + // (bus full under load, or parent canceled). Check the error + // itself rather than ctx.Err(), because pubCtx may time out + // (5 s) while the parent ctx is still active. + // Also treat ErrBusClosed as expected โ€” it occurs during normal + // shutdown when the bus is closed before all goroutines finish. + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) || + errors.Is(err, bus.ErrBusClosed) { + logger.DebugCF("agent", "Reasoning publish skipped (timeout/cancel)", map[string]any{ + "channel": channelName, + "error": err.Error(), + }) + } else { + logger.WarnCF("agent", "Failed to publish reasoning (best-effort)", map[string]any{ + "channel": channelName, + "error": err.Error(), + }) + } + } +} + +// runLLMIteration executes the LLM call loop with tool handling. +func (al *AgentLoop) runLLMIteration( + ctx context.Context, + agent *AgentInstance, + messages []providers.Message, + opts processOptions, +) (string, int, []ToolCallRecord, error) { + iteration := 0 + var finalContent string + var toolRecords []ToolCallRecord + + for iteration < agent.MaxIterations { + iteration++ + + logger.DebugCF("agent", "LLM iteration", + map[string]any{ + "seq": opts.MsgSeqId, + "agent_id": agent.ID, + "iteration": iteration, + "max": agent.MaxIterations, + }) + + // Build tool definitions + providerToolDefs := agent.Tools.ToProviderDefs() + + // Log LLM request details + logger.DebugCF("agent", "LLM request", + map[string]any{ + "seq": opts.MsgSeqId, + "agent_id": agent.ID, + "iteration": iteration, + "model": agent.Model, + "messages_count": len(messages), + "tools_count": len(providerToolDefs), + "max_tokens": agent.MaxTokens, + "temperature": agent.Temperature, + "system_prompt_len": len(messages[0].Content), + }) + + // Log full messages (detailed) + logger.DebugCF("agent", "Full LLM request", + map[string]any{ + "iteration": iteration, + "messages_json": formatMessagesForLog(messages), + "tools_json": formatToolsForLog(providerToolDefs), + }) + + // Call LLM with fallback chain if candidates are configured. + var response *providers.LLMResponse + var err error + + callLLM := func() (*providers.LLMResponse, error) { + if len(agent.Candidates) > 1 && al.fallback != nil { + fbResult, fbErr := al.fallback.Execute(ctx, agent.Candidates, + func(ctx context.Context, provider, model string) (*providers.LLMResponse, error) { + return agent.Provider.Chat(ctx, messages, providerToolDefs, model, map[string]any{ + "max_tokens": agent.MaxTokens, + "temperature": agent.Temperature, + "prompt_cache_key": agent.ID, + }) + }, + ) + if fbErr != nil { + return nil, fbErr + } + if fbResult.Provider != "" && len(fbResult.Attempts) > 0 { + logger.InfoCF("agent", fmt.Sprintf("Fallback: succeeded with %s/%s after %d attempts", + fbResult.Provider, fbResult.Model, len(fbResult.Attempts)+1), + map[string]any{"agent_id": agent.ID, "iteration": iteration}) + } + return fbResult.Response, nil + } + return agent.Provider.Chat(ctx, messages, providerToolDefs, agent.Model, map[string]any{ + "max_tokens": agent.MaxTokens, + "temperature": agent.Temperature, + "prompt_cache_key": agent.ID, + }) + } + + // Retry loop for context/token errors + maxRetries := 2 + for retry := 0; retry <= maxRetries; retry++ { + response, err = callLLM() + if err == nil { + break + } + + errMsg := strings.ToLower(err.Error()) + + // Check if this is a network/HTTP timeout โ€” not a context window error. + isTimeoutError := errors.Is(err, context.DeadlineExceeded) || + strings.Contains(errMsg, "deadline exceeded") || + strings.Contains(errMsg, "client.timeout") || + strings.Contains(errMsg, "timed out") || + strings.Contains(errMsg, "timeout exceeded") + + // Detect real context window / token limit errors, excluding network timeouts. + isContextError := !isTimeoutError && (strings.Contains(errMsg, "context_length_exceeded") || + strings.Contains(errMsg, "context window") || + strings.Contains(errMsg, "maximum context length") || + strings.Contains(errMsg, "token limit") || + strings.Contains(errMsg, "too many tokens") || + strings.Contains(errMsg, "max_tokens") || + strings.Contains(errMsg, "invalidparameter") || + strings.Contains(errMsg, "prompt is too long") || + strings.Contains(errMsg, "request too large")) + + if isTimeoutError && retry < maxRetries { + backoff := time.Duration(retry+1) * 5 * time.Second + logger.WarnCF("agent", "Timeout error, retrying after backoff", map[string]any{ + "error": err.Error(), + "retry": retry, + "backoff": backoff.String(), + }) + time.Sleep(backoff) + continue + } + + if isContextError && retry < maxRetries { + logger.WarnCF("agent", "Context window error detected, attempting compression", map[string]any{ + "error": err.Error(), + "retry": retry, + }) + + if retry == 0 && !constants.IsInternalChannel(opts.Channel) { + al.bus.PublishOutbound(ctx, bus.OutboundMessage{ + Channel: opts.Channel, + ChatID: opts.ChatID, + Content: "Context window exceeded. Compressing history and retrying...", + }) + } + + al.forceCompression(agent, opts.SessionKey) + newHistory := agent.Sessions.GetHistory(opts.SessionKey) + newSummary := agent.Sessions.GetSummary(opts.SessionKey) + messages = agent.ContextBuilder.BuildMessages( + newHistory, newSummary, "", + nil, opts.Channel, opts.ChatID, + ) + continue + } + break + } + + if err != nil { + logger.ErrorCF("agent", "LLM call failed", + map[string]any{ + "seq": opts.MsgSeqId, + "agent_id": agent.ID, + "iteration": iteration, + "error": err.Error(), + }) + return "", iteration, toolRecords, fmt.Errorf("LLM call failed after retries: %w", err) + } + + go al.handleReasoning(ctx, response.Reasoning, opts.Channel, al.targetReasoningChannelID(opts.Channel)) + + logger.DebugCF("agent", "LLM response", + map[string]any{ + "seq": opts.MsgSeqId, + "agent_id": agent.ID, + "iteration": iteration, + "content_chars": len(response.Content), + "tool_calls": len(response.ToolCalls), + "reasoning": response.Reasoning, + "target_channel": al.targetReasoningChannelID(opts.Channel), + "channel": opts.Channel, + }) + // Check if no tool calls - we're done + if len(response.ToolCalls) == 0 { + finalContent = response.Content + logger.InfoCF("agent", "LLM response without tool calls (direct answer)", + map[string]any{ + "seq": opts.MsgSeqId, + "agent_id": agent.ID, + "iteration": iteration, + "content_chars": len(finalContent), + }) + break + } + + normalizedToolCalls := make([]providers.ToolCall, 0, len(response.ToolCalls)) + for _, tc := range response.ToolCalls { + normalizedToolCalls = append(normalizedToolCalls, providers.NormalizeToolCall(tc)) + } + + // Log tool calls + toolNames := make([]string, 0, len(normalizedToolCalls)) + for _, tc := range normalizedToolCalls { + toolNames = append(toolNames, tc.Name) + } + logger.InfoCF("agent", "LLM requested tool calls", + map[string]any{ + "agent_id": agent.ID, + "tools": toolNames, + "count": len(normalizedToolCalls), + "iteration": iteration, + }) + + // Build assistant message with tool calls + assistantMsg := providers.Message{ + Role: "assistant", + Content: response.Content, + ReasoningContent: response.ReasoningContent, + } + for _, tc := range normalizedToolCalls { + argumentsJSON, _ := json.Marshal(tc.Arguments) + // Copy ExtraContent to ensure thought_signature is persisted for Gemini 3 + extraContent := tc.ExtraContent + thoughtSignature := "" + if tc.Function != nil { + thoughtSignature = tc.Function.ThoughtSignature + } + + assistantMsg.ToolCalls = append(assistantMsg.ToolCalls, providers.ToolCall{ + ID: tc.ID, + Type: "function", + Name: tc.Name, + Function: &providers.FunctionCall{ + Name: tc.Name, + Arguments: string(argumentsJSON), + ThoughtSignature: thoughtSignature, + }, + ExtraContent: extraContent, + ThoughtSignature: thoughtSignature, + }) + } + messages = append(messages, assistantMsg) + + // Save assistant message with tool calls to session + agent.Sessions.AddFullMessage(opts.SessionKey, assistantMsg) + + // Execute tool calls + for _, tc := range normalizedToolCalls { + argsJSON, _ := json.Marshal(tc.Arguments) + argsPreview := utils.Truncate(string(argsJSON), 200) + logger.InfoCF("agent", fmt.Sprintf("Tool call: %s(%s)", tc.Name, argsPreview), + map[string]any{ + "agent_id": agent.ID, + "tool": tc.Name, + "iteration": iteration, + }) + + // Create async callback for tools that implement AsyncTool + // NOTE: Following openclaw's design, async tools do NOT send results directly to users. + // Instead, they notify the agent via PublishInbound, and the agent decides + // whether to forward the result to the user (in processSystemMessage). + asyncCallback := func(callbackCtx context.Context, result *tools.ToolResult) { + // Log the async completion but don't send directly to user + // The agent will handle user notification via processSystemMessage + if !result.Silent && result.ForUser != "" { + logger.InfoCF("agent", "Async tool completed, agent will handle notification", + map[string]any{ + "tool": tc.Name, + "content_len": len(result.ForUser), + }) + } + } + + toolStart := time.Now() + toolResult := agent.Tools.ExecuteWithContext( + ctx, + tc.Name, + tc.Arguments, + opts.Channel, + opts.ChatID, + asyncCallback, + ) + toolDuration := time.Since(toolStart) + + // Record tool call for post-LLM processors. + record := ToolCallRecord{Name: tc.Name, Duration: toolDuration} + if toolResult.Err != nil { + record.Error = toolResult.Err.Error() + } + toolRecords = append(toolRecords, record) + + // Send ForUser content to user immediately if not Silent + if !toolResult.Silent && toolResult.ForUser != "" && opts.SendResponse { + al.bus.PublishOutbound(ctx, bus.OutboundMessage{ + Channel: opts.Channel, + ChatID: opts.ChatID, + Content: toolResult.ForUser, + }) + logger.DebugCF("agent", "Sent tool result to user", + map[string]any{ + "tool": tc.Name, + "content_len": len(toolResult.ForUser), + }) + } + + // If tool returned media refs, publish them as outbound media + if len(toolResult.Media) > 0 && opts.SendResponse { + parts := make([]bus.MediaPart, 0, len(toolResult.Media)) + for _, ref := range toolResult.Media { + part := bus.MediaPart{Ref: ref} + // Populate metadata from MediaStore when available + if al.mediaStore != nil { + if _, meta, err := al.mediaStore.ResolveWithMeta(ref); err == nil { + part.Filename = meta.Filename + part.ContentType = meta.ContentType + part.Type = inferMediaType(meta.Filename, meta.ContentType) + } + } + parts = append(parts, part) + } + al.bus.PublishOutboundMedia(ctx, bus.OutboundMediaMessage{ + Channel: opts.Channel, + ChatID: opts.ChatID, + Parts: parts, + }) + } + + // Determine content for LLM based on tool result + contentForLLM := toolResult.ForLLM + if contentForLLM == "" && toolResult.Err != nil { + contentForLLM = toolResult.Err.Error() + } + + toolResultMsg := providers.Message{ + Role: "tool", + Content: contentForLLM, + ToolCallID: tc.ID, + } + messages = append(messages, toolResultMsg) + + // Save tool result message to session + agent.Sessions.AddFullMessage(opts.SessionKey, toolResultMsg) + } + } + + return finalContent, iteration, toolRecords, nil +} + +// updateToolContexts updates the context for tools that need channel/chatID info. +func (al *AgentLoop) updateToolContexts(agent *AgentInstance, channel, chatID string) { + // Use ContextualTool interface instead of type assertions + if tool, ok := agent.Tools.Get("message"); ok { + if mt, ok := tool.(tools.ContextualTool); ok { + mt.SetContext(channel, chatID) + } + } + if tool, ok := agent.Tools.Get("spawn"); ok { + if st, ok := tool.(tools.ContextualTool); ok { + st.SetContext(channel, chatID) + } + } + if tool, ok := agent.Tools.Get("subagent"); ok { + if st, ok := tool.(tools.ContextualTool); ok { + st.SetContext(channel, chatID) + } + } +} + +// maybeSummarize triggers summarization if the session history exceeds thresholds. +func (al *AgentLoop) maybeSummarize(agent *AgentInstance, sessionKey, channel, chatID string) { + newHistory := agent.Sessions.GetHistory(sessionKey) + tokenEstimate := al.estimateTokens(newHistory) + threshold := agent.ContextWindow * 75 / 100 + + if len(newHistory) > 20 || tokenEstimate > threshold { + summarizeKey := agent.ID + ":" + sessionKey + if _, loading := al.summarizing.LoadOrStore(summarizeKey, true); !loading { + go func() { + defer al.summarizing.Delete(summarizeKey) + logger.Debug("Memory threshold reached. Optimizing conversation history...") + al.summarizeSession(agent, sessionKey) + }() + } + } +} + +// forceCompression aggressively reduces context when the limit is hit. +// It drops the oldest 50% of messages (keeping system prompt and last user message). +func (al *AgentLoop) forceCompression(agent *AgentInstance, sessionKey string) { + history := agent.Sessions.GetHistory(sessionKey) + if len(history) <= 4 { + return + } + + // Keep system prompt (usually [0]) and the very last message (user's trigger) + // We want to drop the oldest half of the *conversation* + // Assuming [0] is system, [1:] is conversation + conversation := history[1 : len(history)-1] + if len(conversation) == 0 { + return + } + + // Helper to find the mid-point of the conversation + mid := len(conversation) / 2 + + // New history structure: + // 1. System Prompt (with compression note appended) + // 2. Second half of conversation + // 3. Last message + + droppedCount := mid + keptConversation := conversation[mid:] + + newHistory := make([]providers.Message, 0, 1+len(keptConversation)+1) + + // Append compression note to the original system prompt instead of adding a new system message + // This avoids having two consecutive system messages which some APIs (like Zhipu) reject + compressionNote := fmt.Sprintf( + "\n\n[System Note: Emergency compression dropped %d oldest messages due to context limit]", + droppedCount, + ) + enhancedSystemPrompt := history[0] + enhancedSystemPrompt.Content = enhancedSystemPrompt.Content + compressionNote + newHistory = append(newHistory, enhancedSystemPrompt) + + newHistory = append(newHistory, keptConversation...) + newHistory = append(newHistory, history[len(history)-1]) // Last message + + // Update session + agent.Sessions.SetHistory(sessionKey, newHistory) + agent.Sessions.Save(sessionKey) + + logger.WarnCF("agent", "Forced compression executed", map[string]any{ + "session_key": sessionKey, + "dropped_msgs": droppedCount, + "new_count": len(newHistory), + }) +} + +// GetStartupInfo returns information about loaded tools and skills for logging. +func (al *AgentLoop) GetStartupInfo() map[string]any { + info := make(map[string]any) + + agent := al.registry.GetDefaultAgent() + if agent == nil { + return info + } + + // Tools info + toolsList := agent.Tools.List() + info["tools"] = map[string]any{ + "count": len(toolsList), + "names": toolsList, + } + + // Skills info + info["skills"] = agent.ContextBuilder.GetSkillsInfo() + + // Agents info + info["agents"] = map[string]any{ + "count": len(al.registry.ListAgentIDs()), + "ids": al.registry.ListAgentIDs(), + } + + return info +} + +// formatMessagesForLog formats messages for logging +func formatMessagesForLog(messages []providers.Message) string { + if len(messages) == 0 { + return "[]" + } + + var sb strings.Builder + sb.WriteString("[\n") + for i, msg := range messages { + fmt.Fprintf(&sb, " [%d] Role: %s\n", i, msg.Role) + if len(msg.ToolCalls) > 0 { + sb.WriteString(" ToolCalls:\n") + for _, tc := range msg.ToolCalls { + fmt.Fprintf(&sb, " - ID: %s, Type: %s, Name: %s\n", tc.ID, tc.Type, tc.Name) + if tc.Function != nil { + fmt.Fprintf(&sb, " Arguments: %s\n", utils.Truncate(tc.Function.Arguments, 200)) + } + } + } + if msg.Content != "" { + content := utils.Truncate(msg.Content, 200) + fmt.Fprintf(&sb, " Content: %s\n", content) + } + if msg.ToolCallID != "" { + fmt.Fprintf(&sb, " ToolCallID: %s\n", msg.ToolCallID) + } + sb.WriteString("\n") + } + sb.WriteString("]") + return sb.String() +} + +// formatToolsForLog formats tool definitions for logging +func formatToolsForLog(toolDefs []providers.ToolDefinition) string { + if len(toolDefs) == 0 { + return "[]" + } + + var sb strings.Builder + sb.WriteString("[\n") + for i, tool := range toolDefs { + fmt.Fprintf(&sb, " [%d] Type: %s, Name: %s\n", i, tool.Type, tool.Function.Name) + fmt.Fprintf(&sb, " Description: %s\n", tool.Function.Description) + if len(tool.Function.Parameters) > 0 { + fmt.Fprintf(&sb, " Parameters: %s\n", utils.Truncate(fmt.Sprintf("%v", tool.Function.Parameters), 200)) + } + } + sb.WriteString("]") + return sb.String() +} + +// estimateTokens estimates the number of tokens in a message list. +// Uses a safe heuristic of 2.5 characters per token to account for CJK and other +// overheads better than the previous 3 chars/token. +func (al *AgentLoop) estimateTokens(messages []providers.Message) int { + totalChars := 0 + for _, m := range messages { + totalChars += utf8.RuneCountInString(m.Content) + } + // 2.5 chars per token = totalChars * 2 / 5 + return totalChars * 2 / 5 +} diff --git a/pkg/agent/instance.go b/pkg/agent/instance.go index ed438059f..6f2ba3c3c 100644 --- a/pkg/agent/instance.go +++ b/pkg/agent/instance.go @@ -34,6 +34,8 @@ type AgentInstance struct { Subagents *config.SubagentsConfig SkillsFilter []string Candidates []providers.FallbackCandidate + Analyser *Analyser // Phase 1: intent/tag analysis + Reflector *Reflector // Phase 3: post-LLM processing + slash commands } // NewAgentInstance creates an agent instance from config. @@ -148,6 +150,22 @@ func NewAgentInstance( candidates := providers.ResolveCandidatesWithLookup(modelCfg, defaults.Provider, resolveFromModelList) + // Initialise optional Phase 1 analyser for intent/tag-based memory retrieval and CoT selection. + // Uses GetAnalyserModel() which resolves: analyser_model โ†’ pre_llm_model โ†’ model_name. + var analyser *Analyser + var rt *Reflector + analyserModel := defaults.GetAnalyserModel() + if analyserModel != "" { + cotRegistry := NewCotRegistry(workspace) + analyser = NewAnalyser(provider, analyserModel, cotRegistry) + rt = NewReflector(provider, analyserModel) + log.Printf("Analyser + Reflector enabled for agent %s (model: %s)", agentID, analyserModel) + } else { + // Reflector without LLM processors (just commands + error tracker). + rt = NewReflector(nil, "") + } + rt.SetTools(toolsRegistry) + return &AgentInstance{ ID: agentID, Name: agentName, @@ -165,6 +183,8 @@ func NewAgentInstance( Subagents: subagents, SkillsFilter: skillsFilter, Candidates: candidates, + Analyser: analyser, + Reflector: rt, } } diff --git a/pkg/agent/instant_memory.go b/pkg/agent/instant_memory.go new file mode 100644 index 000000000..223b92d33 --- /dev/null +++ b/pkg/agent/instant_memory.go @@ -0,0 +1,252 @@ +// PicoClaw - Ultra-lightweight personal AI agent +// License: MIT +// +// Copyright (c) 2026 PicoClaw contributors + +package agent + +import ( + "fmt" + "strings" + + "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/providers" +) + +// --------------------------------------------------------------------------- +// Instant Memory โ€” dynamic Turn selection for Phase 2 context +// --------------------------------------------------------------------------- + +// InstantMemoryCfg holds tunable parameters for instant-memory assembly. +type InstantMemoryCfg struct { + HighScoreThreshold int // turns with score >= this are always_keep (default: 7) + RecentCount int // number of recent turns to include (default: 5) + MaxTokenRatio float64 // fraction of contextWindow budget (default: 0.6) + ContextWindow int // total context window in tokens +} + +// DefaultInstantMemoryCfg returns a sensible default config. +func DefaultInstantMemoryCfg(contextWindow int) InstantMemoryCfg { + return InstantMemoryCfg{ + HighScoreThreshold: alwaysKeepThreshold, // 7 + RecentCount: 5, + MaxTokenRatio: 0.6, + ContextWindow: contextWindow, + } +} + +// BuildInstantMemory assembles the filtered set of historical turns for Phase 2. +// +// Selection rules (from design doc): +// +// ็žฌๆ—ถ่ฎฐๅฟ† = +// { Turn | score >= highThreshold } // always_keep +// โˆช { Turn | tags โˆฉ currentTags โ‰  โˆ…, score > 0 } // tag-matched +// โˆช { ๆœ€่ฟ‘ M ไธช Turn } // recency guarantee +// โ†’ deduplicate by ID +// โ†’ sort by ts ASC +// โ†’ truncate to token budget +func BuildInstantMemory( + store *TurnStore, + currentTags []string, + channelKey string, + cfg InstantMemoryCfg, +) []TurnRecord { + if store == nil { + return nil + } + + seen := make(map[string]struct{}) + var all []TurnRecord + + addUnique := func(turns []TurnRecord) { + for _, t := range turns { + if _, dup := seen[t.ID]; dup { + continue + } + seen[t.ID] = struct{}{} + all = append(all, t) + } + } + + // 1. always_keep: high-score turns. + high, err := store.QueryByScore(cfg.HighScoreThreshold) + if err != nil { + logger.WarnCF("instant_memory", "QueryByScore failed", map[string]any{"error": err.Error()}) + } else { + addUnique(high) + } + + // 2. tag-matched turns (score > 0). + if len(currentTags) > 0 { + tagged, err := store.QueryByTags(currentTags) + if err != nil { + logger.WarnCF("instant_memory", "QueryByTags failed", map[string]any{"error": err.Error()}) + } else { + addUnique(tagged) + } + } + + // 3. Recent M turns for continuity. + recent, err := store.QueryRecent(channelKey, cfg.RecentCount) + if err != nil { + logger.WarnCF("instant_memory", "QueryRecent failed", map[string]any{"error": err.Error()}) + } else { + addUnique(recent) + } + + // Sort by ts ASC (stable chronological order). + sortTurnsByTs(all) + + // Truncate to token budget. + maxTokens := int(float64(cfg.ContextWindow) * cfg.MaxTokenRatio) + if maxTokens > 0 { + all = truncateToTokenBudget(all, maxTokens) + } + + logger.DebugCF("instant_memory", "Built instant memory", + map[string]any{ + "total": len(all), + "high_score": len(high), + "tag_matched": len(currentTags), + "recent": len(recent), + "max_tokens": maxTokens, + }) + + return all +} + +// sortTurnsByTs sorts turns in ascending timestamp order (oldest first). +func sortTurnsByTs(turns []TurnRecord) { + // Simple in-place insertion sort โ€” good enough for small N (<100). + for i := 1; i < len(turns); i++ { + key := turns[i] + j := i - 1 + for j >= 0 && turns[j].Ts > key.Ts { + turns[j+1] = turns[j] + j-- + } + turns[j+1] = key + } +} + +// truncateToTokenBudget trims turns from the oldest end until total tokens fit. +// Returns a suffix of the sorted slice (preserving newest turns). +func truncateToTokenBudget(turns []TurnRecord, maxTokens int) []TurnRecord { + total := 0 + for _, t := range turns { + total += t.Tokens + } + if total <= maxTokens { + return turns + } + + // Drop oldest turns first until we fit. + for len(turns) > 0 && total > maxTokens { + total -= turns[0].Tokens + turns = turns[1:] + } + return turns +} + +// --------------------------------------------------------------------------- +// Phase 2 Message Assembly โ€” KV Cache friendly ordering +// --------------------------------------------------------------------------- + +// BuildPhase2Messages constructs the message array for Phase 2 (ExecuteLLM) +// in KV-cache-friendly order: +// +// [system_prompt] โ† always cache hit +// [long_term_memory by tags] โ† same tags = cache hit (cache_control: ephemeral) +// [always_keep turns (scoreโ‰ฅ7)] โ† fixed position, append only โ†’ cache hit +// [tag_matched turns] โ† per-turn, ts ASC +// [recent_M turns] โ† rolling window +// [current_user_message] โ† always new +// +// Each historical turn is represented as a user/assistant message pair. +func BuildPhase2Messages( + systemPrompt string, + longTermMemory string, + turns []TurnRecord, + userMessage string, + highScoreThreshold int, +) []providers.Message { + msgs := make([]providers.Message, 0, 2+len(turns)*2+1) + + // 1. System prompt (always first, stable prefix). + msgs = append(msgs, providers.Message{ + Role: "system", + Content: systemPrompt, + }) + + // 2. Long-term memory (injected as system-adjacent user message). + // Mark with CacheControl if present (Anthropic will use it; others ignore). + if longTermMemory != "" { + msgs = append(msgs, providers.Message{ + Role: "user", + Content: fmt.Sprintf("# Long-term Memory\n\n%s", longTermMemory), + }) + // Need a brief assistant ack to maintain user/assistant alternation. + msgs = append(msgs, providers.Message{ + Role: "assistant", + Content: "Understood, I'll use this context.", + }) + } + + // 3. Historical turns in KV-cache-friendly order: + // - always_keep first (fixed position) + // - then tag_matched + recent (may shift between requests) + // + // All turns are already sorted by ts ASC from BuildInstantMemory. + // We separate them into always_keep vs rest, keeping relative order. + var alwaysKeep, rest []TurnRecord + for _, t := range turns { + if t.Score >= highScoreThreshold { + alwaysKeep = append(alwaysKeep, t) + } else { + rest = append(rest, t) + } + } + + // Append always_keep turns (cache-stable region). + for _, t := range alwaysKeep { + msgs = appendTurnMessages(msgs, t) + } + + // Append remaining turns (tag-matched + recent, may shift). + for _, t := range rest { + msgs = appendTurnMessages(msgs, t) + } + + // 4. Current user message (always last, always new). + msgs = append(msgs, providers.Message{ + Role: "user", + Content: userMessage, + }) + + return msgs +} + +// appendTurnMessages appends a user/assistant pair for a historical turn. +func appendTurnMessages(msgs []providers.Message, t TurnRecord) []providers.Message { + // Build user message with metadata prefix. + var userContent strings.Builder + if t.Intent != "" || len(t.Tags) > 0 { + fmt.Fprintf(&userContent, "[turn intent=%s tags=%v]\n", t.Intent, t.Tags) + } + userContent.WriteString(t.UserMsg) + + msgs = append(msgs, providers.Message{ + Role: "user", + Content: userContent.String(), + }) + + if t.Reply != "" { + msgs = append(msgs, providers.Message{ + Role: "assistant", + Content: t.Reply, + }) + } + + return msgs +} diff --git a/pkg/agent/instant_memory_test.go b/pkg/agent/instant_memory_test.go new file mode 100644 index 000000000..a830e6295 --- /dev/null +++ b/pkg/agent/instant_memory_test.go @@ -0,0 +1,164 @@ +package agent + +import ( + "strings" + "testing" + "time" +) + +func TestBuildInstantMemory_BasicAssembly(t *testing.T) { + dir := t.TempDir() + store, err := NewTurnStore(dir) + if err != nil { + t.Fatalf("NewTurnStore: %v", err) + } + defer store.Close() + + now := time.Now().Unix() + + // High-score turn (always_keep). + store.Insert(TurnRecord{ID: "t1", Ts: now - 100, Score: 9, ChannelKey: "cli:direct", + Intent: "code", Tags: []string{"refactor"}, UserMsg: "refactor it", Reply: strings.Repeat("x", 300)}) + + // Low-score irrelevant turn. + store.Insert(TurnRecord{ID: "t2", Ts: now - 80, Score: 2, ChannelKey: "cli:direct", + Intent: "chat", Tags: []string{"chat"}, UserMsg: "hi", Reply: "hello"}) + + // Tag-matched turn, moderate score. + store.Insert(TurnRecord{ID: "t3", Ts: now - 60, Score: 5, ChannelKey: "cli:direct", + Intent: "task", Tags: []string{"deploy", "ci"}, UserMsg: "deploy staging", Reply: "done"}) + + // Recent turns. + store.Insert(TurnRecord{ID: "t4", Ts: now - 20, Score: 3, ChannelKey: "cli:direct", + Intent: "question", Tags: []string{"api"}, UserMsg: "what's the api?", Reply: "check docs"}) + store.Insert(TurnRecord{ID: "t5", Ts: now - 10, Score: 4, ChannelKey: "cli:direct", + Intent: "task", Tags: []string{"test"}, UserMsg: "run tests", Reply: "all passed"}) + + cfg := InstantMemoryCfg{ + HighScoreThreshold: 7, + RecentCount: 3, + MaxTokenRatio: 0.6, + ContextWindow: 100000, + } + + turns := BuildInstantMemory(store, []string{"deploy"}, "cli:direct", cfg) + + // Should include: t1 (high-score), t3 (tag-match "deploy"), t4/t5 (recent 3 โ†’ also t3) + if len(turns) < 3 { + t.Errorf("expected at least 3 turns, got %d", len(turns)) + for _, tt := range turns { + t.Logf(" turn: id=%s score=%d tags=%v", tt.ID, tt.Score, tt.Tags) + } + } + + // Should be sorted by ts ASC. + for i := 1; i < len(turns); i++ { + if turns[i].Ts < turns[i-1].Ts { + t.Errorf("turns not sorted: turns[%d].Ts=%d < turns[%d].Ts=%d", + i, turns[i].Ts, i-1, turns[i-1].Ts) + } + } + + // t1 (always_keep) must be present. + found := false + for _, tt := range turns { + if tt.ID == "t1" { + found = true + } + } + if !found { + t.Error("expected always_keep turn t1 to be included") + } + + // t2 (low-score, no tag match, not recent enough) should be excluded. + for _, tt := range turns { + if tt.ID == "t2" { + t.Error("expected low-score irrelevant turn t2 to be excluded") + } + } +} + +func TestBuildInstantMemory_NilStore(t *testing.T) { + turns := BuildInstantMemory(nil, []string{"deploy"}, "cli:direct", DefaultInstantMemoryCfg(8192)) + if turns != nil { + t.Errorf("expected nil, got %v", turns) + } +} + +func TestBuildPhase2Messages_Ordering(t *testing.T) { + turns := []TurnRecord{ + {ID: "t1", Ts: 100, Score: 9, Intent: "code", Tags: []string{"refactor"}, + UserMsg: "refactor it", Reply: "done refactoring", Tokens: 20}, + {ID: "t2", Ts: 200, Score: 3, Intent: "question", + UserMsg: "what next?", Reply: "do X", Tokens: 10}, + {ID: "t3", Ts: 300, Score: 8, Intent: "debug", Tags: []string{"deploy"}, + UserMsg: "fix deploy", Reply: "fixed", Tokens: 10}, + } + + msgs := BuildPhase2Messages("You are a helpful assistant.", "User prefers Go.", turns, "hello world", 7) + + // Expected order: + // [0] system + // [1] user (long_term_memory) + // [2] assistant (ack) + // [3,4] always_keep t1 (user/assistant) + // [5,6] always_keep t3 (user/assistant) + // [7,8] rest t2 (user/assistant) + // [9] current user message + if len(msgs) < 5 { + t.Fatalf("expected at least 5 messages, got %d", len(msgs)) + } + + if msgs[0].Role != "system" { + t.Errorf("msgs[0].Role = %s, want system", msgs[0].Role) + } + + // Last message should be the current user message. + last := msgs[len(msgs)-1] + if last.Role != "user" || last.Content != "hello world" { + t.Errorf("last message = %+v, want user 'hello world'", last) + } + + // All messages should alternate user/assistant (after system). + for i := 1; i < len(msgs)-1; i++ { + expected := "user" + if i%2 == 0 { + expected = "assistant" + } + if msgs[i].Role != expected { + t.Errorf("msgs[%d].Role = %s, want %s (content: %s)", + i, msgs[i].Role, expected, msgs[i].Content[:min(len(msgs[i].Content), 30)]) + } + } +} + +func TestBuildPhase2Messages_NoHistory(t *testing.T) { + msgs := BuildPhase2Messages("sys prompt", "", nil, "hi", 7) + + // Should have: system + user message = 2 + if len(msgs) != 2 { + t.Errorf("expected 2 messages, got %d", len(msgs)) + } + if msgs[0].Role != "system" || msgs[1].Role != "user" { + t.Errorf("unexpected roles: %s, %s", msgs[0].Role, msgs[1].Role) + } +} + +func TestTruncateToTokenBudget(t *testing.T) { + turns := []TurnRecord{ + {ID: "a", Tokens: 100}, + {ID: "b", Tokens: 200}, + {ID: "c", Tokens: 300}, + {ID: "d", Tokens: 150}, + } + result := truncateToTokenBudget(turns, 500) + // Total = 750, budget = 500. Drop oldest first. + // Drop "a" (100) โ†’ 650, still over. + // Drop "b" (200) โ†’ 450, fits. + if len(result) != 2 { + t.Errorf("expected 2 turns, got %d", len(result)) + } + if result[0].ID != "c" || result[1].ID != "d" { + t.Errorf("expected [c, d], got [%s, %s]", result[0].ID, result[1].ID) + } +} diff --git a/pkg/agent/integration_test.go b/pkg/agent/integration_test.go new file mode 100644 index 000000000..46a2d8876 --- /dev/null +++ b/pkg/agent/integration_test.go @@ -0,0 +1,239 @@ +// PicoClaw - Ultra-lightweight personal AI agent +// License: MIT +// +// Copyright (c) 2026 PicoClaw contributors + +package agent + +import ( + "context" + "strings" + "testing" + "time" +) + +// --------------------------------------------------------------------------- +// M5 Integration โ€” TurnStore โ†’ BuildInstantMemory โ†’ BuildPhase2Messages +// --------------------------------------------------------------------------- + +// TestInstantMemoryIntegration_EndToEnd inserts realistic turns into a real +// TurnStore, runs BuildInstantMemory with tag filtering, then assembles Phase 2 +// messages and validates: +// - correct message ordering (system โ†’ memory โ†’ always_keep โ†’ rest โ†’ user) +// - strict user/assistant role alternation after the system message +// - always_keep turns appear before lower-score turns +// - the current user message is always last +func TestInstantMemoryIntegration_EndToEnd(t *testing.T) { + dir := t.TempDir() + store, err := NewTurnStore(dir) + if err != nil { + t.Fatalf("NewTurnStore: %v", err) + } + defer store.Close() + + now := time.Now().Unix() + + // Seed realistic turns. + turns := []TurnRecord{ + {ID: "turn-1", Ts: now - 3600, Score: 10, ChannelKey: "cli:main", + Intent: "task", Tags: []string{"deploy", "ci"}, + UserMsg: "Deploy to staging", Reply: "Deployed successfully to staging environment.", + Tokens: 50}, + {ID: "turn-2", Ts: now - 3000, Score: 2, ChannelKey: "cli:main", + Intent: "chat", Tags: []string{"chat"}, + UserMsg: "hi", Reply: "Hello!", + Tokens: 10}, + {ID: "turn-3", Ts: now - 2000, Score: 6, ChannelKey: "cli:main", + Intent: "code", Tags: []string{"golang", "refactor"}, + UserMsg: "Refactor the handler", Reply: "Done, split into 3 functions.", + Tokens: 40}, + {ID: "turn-4", Ts: now - 500, Score: 4, ChannelKey: "cli:main", + Intent: "question", Tags: []string{"api"}, + UserMsg: "What's the endpoint for users?", Reply: "GET /api/v1/users", + Tokens: 20}, + {ID: "turn-5", Ts: now - 100, Score: 3, ChannelKey: "cli:main", + Intent: "task", Tags: []string{"test"}, + UserMsg: "Run all tests", Reply: "All 42 tests passed.", + Tokens: 15}, + } + for _, tr := range turns { + if err := store.Insert(tr); err != nil { + t.Fatalf("Insert(%s): %v", tr.ID, err) + } + } + + // Query with tags=["deploy"] โ€” should get turn-1 (always_keep + tag match), + // turn-3/4/5 (recent 3). turn-2 is low score, no tag match, not recent. + cfg := InstantMemoryCfg{ + HighScoreThreshold: 7, + RecentCount: 3, + MaxTokenRatio: 0.6, + ContextWindow: 100000, + } + selected := BuildInstantMemory(store, []string{"deploy"}, "cli:main", cfg) + + // Verify turn-1 is selected (always_keep). + hasT1 := false + for _, s := range selected { + if s.ID == "turn-1" { + hasT1 = true + } + } + if !hasT1 { + t.Error("expected always_keep turn-1 to be selected") + } + + // Verify turn-2 is NOT selected. + for _, s := range selected { + if s.ID == "turn-2" { + t.Error("expected low-score turn-2 to be excluded") + } + } + + // Assemble Phase 2 messages. + systemPrompt := "You are a helpful assistant.\n\n## Runtime\nlinux amd64" + longTermMemory := "User prefers Go. User's name is Alice." + currentMsg := "Deploy to production now" + + msgs := BuildPhase2Messages(systemPrompt, longTermMemory, selected, currentMsg, cfg.HighScoreThreshold) + + // --- Validate message structure --- + + // 1. First message is system. + if msgs[0].Role != "system" { + t.Fatalf("msgs[0].Role = %s, want system", msgs[0].Role) + } + if !strings.Contains(msgs[0].Content, "helpful assistant") { + t.Error("system message should contain prompt text") + } + + // 2. Last message is current user message. + last := msgs[len(msgs)-1] + if last.Role != "user" || last.Content != currentMsg { + t.Errorf("last message = role=%s content=%q, want user %q", last.Role, last.Content, currentMsg) + } + + // 3. Role alternation: after system, messages must alternate user/assistant. + for i := 1; i < len(msgs); i++ { + expectedRole := "user" + if i%2 == 0 { + expectedRole = "assistant" + } + if msgs[i].Role != expectedRole { + t.Errorf("msgs[%d].Role = %s, want %s (content: %.50s...)", + i, msgs[i].Role, expectedRole, msgs[i].Content) + } + } + + // 4. Long-term memory should be in msgs[1] (user role). + if !strings.Contains(msgs[1].Content, "Long-term Memory") { + t.Error("msgs[1] should contain long-term memory") + } + + // 5. Always_keep turns (score >= 7) should appear before lower-score turns. + alwaysKeepEnd := -1 + restStart := len(msgs) + for i := 3; i < len(msgs)-1; i += 2 { // user messages from turns, skip system+memory+ack + content := msgs[i].Content + // Check if this is an always_keep turn by looking for turn-1 content. + if strings.Contains(content, "Deploy to staging") { + alwaysKeepEnd = i + } + } + for i := 3; i < len(msgs)-1; i += 2 { + content := msgs[i].Content + // First non-always-keep turn. + if !strings.Contains(content, "Deploy to staging") && !strings.Contains(content, "Long-term Memory") { + restStart = i + break + } + } + if alwaysKeepEnd >= 0 && restStart < len(msgs) && alwaysKeepEnd > restStart { + t.Errorf("always_keep turns should come before rest: alwaysKeepEnd=%d, restStart=%d", + alwaysKeepEnd, restStart) + } + + t.Logf("Phase 2 assembled %d messages from %d selected turns", len(msgs), len(selected)) + for i, m := range msgs { + preview := m.Content + if len(preview) > 60 { + preview = preview[:60] + "..." + } + t.Logf(" [%d] role=%-10s content=%q", i, m.Role, preview) + } +} + +// --------------------------------------------------------------------------- +// M4 Integration โ€” MemoryDigest runOnce +// --------------------------------------------------------------------------- + +// TestMemoryDigestIntegration_RunOnce inserts pending TurnRecords, runs +// MemoryDigest.runOnce with a mock LLM, and verifies: +// - TurnRecords are transitioned from "pending" to "processed" +// - MemoryStore receives new entries from the LLM extraction +func TestMemoryDigestIntegration_RunOnce(t *testing.T) { + dir := t.TempDir() + + turnStore, err := NewTurnStore(dir) + if err != nil { + t.Fatalf("NewTurnStore: %v", err) + } + defer turnStore.Close() + + memStore := NewMemoryStore(dir) + defer memStore.Close() + + now := time.Now().Unix() + + // Insert pending turns. + for i := 0; i < 3; i++ { + tr := TurnRecord{ + ID: "digest-" + string(rune('a'+i)), + Ts: now - int64(300*(3-i)), + Score: 5, + ChannelKey: "cli:main", + Intent: "task", + Tags: []string{"golang"}, + UserMsg: "Do task " + string(rune('A'+i)), + Reply: "Done with task " + string(rune('A'+i)), + Tokens: 30, + Status: "pending", + } + if err := turnStore.Insert(tr); err != nil { + t.Fatalf("Insert: %v", err) + } + } + + // Verify pending. + pending, _ := turnStore.QueryPending(50) + if len(pending) != 3 { + t.Fatalf("expected 3 pending, got %d", len(pending)) + } + + // Create a mock provider that returns a memory extraction response. + mp := &mockLLMProvider{ + response: `{"memories": [{"content": "User worked on Go tasks A, B, C", "tags": ["golang", "task"]}]}`, + } + + // Create and run MemoryDigest. + worker := NewMemoryDigestWorker(turnStore, memStore, mp, "test-model") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + worker.runOnce(ctx) + + // Verify turns are now processed. + pendingAfter, _ := turnStore.QueryPending(50) + if len(pendingAfter) != 0 { + t.Errorf("expected 0 pending after runOnce, got %d", len(pendingAfter)) + } + + // Verify memory store has entries. + memCtx := memStore.GetMemoryContext() + if memCtx == "" { + t.Error("expected MemoryStore to have entries after digest, got empty") + } else { + t.Logf("MemoryStore context after digest:\n%s", memCtx) + } +} diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 88afa6119..09eae0c1d 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -8,15 +8,12 @@ package agent import ( "context" - "encoding/json" - "errors" "fmt" "path/filepath" "strings" "sync" "sync/atomic" "time" - "unicode/utf8" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" @@ -39,10 +36,15 @@ type AgentLoop struct { registry *AgentRegistry state *state.Manager running atomic.Bool + msgSeqId atomic.Uint64 summarizing sync.Map fallback *providers.FallbackChain channelManager *channels.Manager mediaStore media.MediaStore + // Phase 3 infrastructure (M1-M4) + turnStore *TurnStore // per-workspace turns.db + activeCtx *ActiveContextStore // per channel:chatID context + memoryDigest *MemoryDigestWorker // background memory distillation } // processOptions configures how a message is processed @@ -55,6 +57,7 @@ type processOptions struct { EnableSummary bool // Whether to trigger summarization SendResponse bool // Whether to send response via bus NoHistory bool // If true, don't load session history (for heartbeat) + MsgSeqId uint64 // Global message sequence number } const defaultResponse = "I've completed processing but have no response to give. Increase `max_tool_iterations` in config.json." @@ -80,13 +83,32 @@ func NewAgentLoop( stateManager = state.NewManager(defaultAgent.Workspace) } + // Initialise Phase 3 infrastructure. + activeCtxStore := NewActiveContextStore() + var ts *TurnStore + var digestWorker *MemoryDigestWorker + if defaultAgent != nil { + var tsErr error + ts, tsErr = NewTurnStore(defaultAgent.Workspace) + if tsErr != nil { + logger.ErrorCF("agent", "Failed to create TurnStore", map[string]any{"error": tsErr.Error()}) + } else { + mem := defaultAgent.ContextBuilder.GetMemory() + digestModel := cfg.Agents.Defaults.GetDigestModel() + digestWorker = NewMemoryDigestWorker(ts, mem, provider, digestModel) + } + } + return &AgentLoop{ - bus: msgBus, - cfg: cfg, - registry: registry, - state: stateManager, - summarizing: sync.Map{}, - fallback: fallbackChain, + bus: msgBus, + cfg: cfg, + registry: registry, + state: stateManager, + summarizing: sync.Map{}, + fallback: fallbackChain, + turnStore: ts, + activeCtx: activeCtxStore, + memoryDigest: digestWorker, } } @@ -175,6 +197,22 @@ func registerSharedTools( func (al *AgentLoop) Run(ctx context.Context) error { al.running.Store(true) + // Load persisted Active Context. + if al.activeCtx != nil { + defaultAgent := al.registry.GetDefaultAgent() + if defaultAgent != nil { + acPath := activeContextPath(defaultAgent.Workspace) + if err := al.activeCtx.Load(acPath); err != nil { + logger.WarnCF("agent", "Failed to load active context", map[string]any{"error": err.Error()}) + } + } + } + + // Start MemoryDigest background worker. + if al.memoryDigest != nil { + al.memoryDigest.Start(ctx) + } + // Initialize MCP servers for all agents if al.cfg.Tools.MCP.Enabled { mcpManager := mcp.NewManager() @@ -313,6 +351,22 @@ func (al *AgentLoop) Run(ctx context.Context) error { func (al *AgentLoop) Stop() { al.running.Store(false) + // Flush Active Context to disk. + if al.activeCtx != nil { + defaultAgent := al.registry.GetDefaultAgent() + if defaultAgent != nil { + acPath := activeContextPath(defaultAgent.Workspace) + if err := al.activeCtx.Flush(acPath); err != nil { + logger.WarnCF("agent", "Failed to flush active context", map[string]any{"error": err.Error()}) + } + } + } + // Close TurnStore. + if al.turnStore != nil { + if err := al.turnStore.Close(); err != nil { + logger.WarnCF("agent", "Failed to close turn store", map[string]any{"error": err.Error()}) + } + } } func (al *AgentLoop) RegisterTool(tool tools.Tool) { @@ -325,6 +379,12 @@ func (al *AgentLoop) RegisterTool(tool tools.Tool) { func (al *AgentLoop) SetChannelManager(cm *channels.Manager) { al.channelManager = cm + // Wire agent info into all agent Runtimes so /show, /list, /switch work. + for _, id := range al.registry.ListAgentIDs() { + if agent, ok := al.registry.GetAgent(id); ok && agent != nil && agent.Reflector != nil { + agent.Reflector.SetAgentInfo(al.registry, cm) + } + } } // SetMediaStore injects a MediaStore for media lifecycle management. @@ -448,9 +508,11 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage) return al.processSystemMessage(ctx, msg) } - // Check for commands - if response, handled := al.handleCommand(ctx, msg); handled { - return response, nil + // Check for runtime commands (e.g. /memory, /cot, /runtime, /show, /list, /switch). + if agent := al.registry.GetDefaultAgent(); agent != nil && agent.Reflector != nil { + if response, handled := agent.Reflector.HandleCommand(msg.Content, agent.ContextBuilder.GetMemory()); handled { + return response, nil + } } // Route to determine agent and session key @@ -573,6 +635,9 @@ func (al *AgentLoop) runAgentLoop( agent *AgentInstance, opts processOptions, ) (string, error) { + seq := al.msgSeqId.Add(1) + opts.MsgSeqId = seq + // 0. Record last channel for heartbeat notifications (skip internal channels) if opts.Channel != "" && opts.ChatID != "" { // Don't record internal channels (cli, system, subagent) @@ -591,27 +656,131 @@ func (al *AgentLoop) runAgentLoop( // 1. Update tool contexts al.updateToolContexts(agent, opts.Channel, opts.ChatID) - // 2. Build messages (skip history for heartbeat) - var history []providers.Message - var summary string - if !opts.NoHistory { - history = agent.Sessions.GetHistory(opts.SessionKey) - summary = agent.Sessions.GetSummary(opts.SessionKey) - } - messages := agent.ContextBuilder.BuildMessages( - history, - summary, - opts.UserMessage, - nil, - opts.Channel, - opts.ChatID, - ) + // 2. Analyse intent + build Phase 2 messages. + // + // Two paths: + // A. Instant Memory (when Analyser + TurnStore are ready): + // - Phase 1 analyses intent/tags + // - BuildInstantMemory selects relevant historical turns from TurnStore + // - BuildPhase2Messages assembles KV cache friendly message array + // B. Legacy SessionManager (fallback): + // - Uses Session history directly via ContextBuilder.BuildMessages + // + var analyseResult AnalyseResult + var messages []providers.Message + channelKey := fmt.Sprintf("%s:%s", opts.Channel, opts.ChatID) - // 3. Save user message to session + useInstantMemory := agent.Analyser != nil && al.turnStore != nil && !opts.NoHistory && opts.UserMessage != "" + + if useInstantMemory { + // --- Path A: Instant Memory --- + + // Phase 1: analyse intent + tags. + var actCtx *ActiveContext + if al.activeCtx != nil { + actCtx = al.activeCtx.Get(channelKey) + } + analyseResult = agent.Analyser.Analyse(ctx, opts.UserMessage, agent.ContextBuilder.GetMemory(), actCtx) + + // Build system prompt from ContextBuilder (cached static + dynamic context). + staticPrompt := agent.ContextBuilder.BuildSystemPromptWithCache() + dynamicCtx := agent.ContextBuilder.buildDynamicContext(opts.Channel, opts.ChatID) + systemPrompt := staticPrompt + "\n\n---\n\n" + dynamicCtx + + // Enrich system prompt with CoT strategy. + if analyseResult.CotPrompt != "" { + systemPrompt += "\n\n---\n\n## Thinking Strategy\n\n" + analyseResult.CotPrompt + } + + // Select relevant turns from TurnStore. + cfg := DefaultInstantMemoryCfg(agent.ContextWindow) + instantTurns := BuildInstantMemory(al.turnStore, analyseResult.Tags, channelKey, cfg) + + // Get long-term memory by tags. + longTermMemory := analyseResult.MemoryContext + + // Assemble Phase 2 messages in KV cache friendly order. + messages = BuildPhase2Messages( + systemPrompt, + longTermMemory, + instantTurns, + opts.UserMessage, + cfg.HighScoreThreshold, + ) + + logger.InfoCF("agent", "Phase 2 messages built via instant memory", + map[string]any{ + "seq": seq, + "agent_id": agent.ID, + "intent": analyseResult.Intent, + "tags": analyseResult.Tags, + "instant_turns": len(instantTurns), + "total_messages": len(messages), + "has_cot": analyseResult.CotPrompt != "", + "has_memories": longTermMemory != "", + }) + } else { + // --- Path B: Legacy SessionManager --- + var history []providers.Message + var summary string + if !opts.NoHistory { + history = agent.Sessions.GetHistory(opts.SessionKey) + summary = agent.Sessions.GetSummary(opts.SessionKey) + } + messages = agent.ContextBuilder.BuildMessages( + history, + summary, + opts.UserMessage, + nil, + opts.Channel, + opts.ChatID, + ) + + // Optional Phase 1 enrichment (when Analyser exists but TurnStore not ready). + if agent.Analyser != nil && !opts.NoHistory && opts.UserMessage != "" { + var actCtx *ActiveContext + if al.activeCtx != nil { + actCtx = al.activeCtx.Get(channelKey) + } + analyseResult = agent.Analyser.Analyse(ctx, opts.UserMessage, agent.ContextBuilder.GetMemory(), actCtx) + + var enrichment strings.Builder + if analyseResult.CotPrompt != "" { + enrichment.WriteString("\n\n---\n\n## Thinking Strategy\n\n") + enrichment.WriteString(analyseResult.CotPrompt) + } + if analyseResult.MemoryContext != "" { + enrichment.WriteString("\n\n---\n\n# Contextual Memories (pre-analysed)\n\n") + enrichment.WriteString(analyseResult.MemoryContext) + } + + if enrichment.Len() > 0 && len(messages) > 0 && messages[0].Role == "system" { + messages[0].Content += enrichment.String() + if len(messages[0].SystemParts) > 0 { + enrichBlock := providers.ContentBlock{ + Type: "text", + Text: enrichment.String(), + } + messages[0].SystemParts = append(messages[0].SystemParts, enrichBlock) + } + logger.InfoCF("agent", "Pre-LLM enriched context (legacy path)", + map[string]any{ + "seq": seq, + "agent_id": agent.ID, + "intent": analyseResult.Intent, + "tags": analyseResult.Tags, + "has_memories": analyseResult.MemoryContext != "", + "has_cot": analyseResult.CotPrompt != "", + }) + } + } + } + + // 3. Save user message to session (kept for /memory, /show debug commands). agent.Sessions.AddMessage(opts.SessionKey, "user", opts.UserMessage) // 4. Run LLM iteration loop - finalContent, iteration, err := al.runLLMIteration(ctx, agent, messages, opts) + finalContent, iteration, toolRecords, err := al.runLLMIteration(ctx, agent, messages, opts) if err != nil { return "", err } @@ -628,12 +797,36 @@ func (al *AgentLoop) runAgentLoop( agent.Sessions.AddMessage(opts.SessionKey, "assistant", finalContent) agent.Sessions.Save(opts.SessionKey) + // Build the runtime input used by Phase 3 stages. + runtimeInput := RuntimeInput{ + UserMessage: opts.UserMessage, + AssistantReply: finalContent, + Intent: analyseResult.Intent, + Tags: analyseResult.Tags, + CotPrompt: analyseResult.CotPrompt, + ToolCalls: toolRecords, + Iterations: iteration, + ChannelKey: channelKey, + } + + // 6.5. Phase 3 โ€” Synchronous part (< 2ms): score + Active Context update. + // MUST run before PublishOutbound so the next turn's Phase 1 sees fresh context. + if agent.Reflector != nil && opts.UserMessage != "" { + score := agent.Reflector.SyncPhase3(runtimeInput) + runtimeInput.Score = score + + // Update Active Context for this channel. + if al.activeCtx != nil { + al.activeCtx.Update(channelKey, runtimeInput) + } + } + // 7. Optional: summarization if opts.EnableSummary { al.maybeSummarize(agent, opts.SessionKey, opts.Channel, opts.ChatID) } - // 8. Optional: send response via bus + // 8. Optional: send response via bus (user receives reply here). if opts.SendResponse { al.bus.PublishOutbound(ctx, bus.OutboundMessage{ Channel: opts.Channel, @@ -642,10 +835,17 @@ func (al *AgentLoop) runAgentLoop( }) } + // 6.6. Phase 3 โ€” Async part: persist TurnRecord, run processors. + // Runs AFTER PublishOutbound to not delay the user response. + if agent.Reflector != nil && opts.UserMessage != "" { + agent.Reflector.AsyncPhase3(runtimeInput, agent.ContextBuilder.GetMemory(), al.turnStore, al.activeCtx) + } + // 9. Log response responsePreview := utils.Truncate(finalContent, 120) logger.InfoCF("agent", fmt.Sprintf("Response: %s", responsePreview), map[string]any{ + "seq": seq, "agent_id": agent.ID, "session_key": opts.SessionKey, "iterations": iteration, @@ -655,579 +855,6 @@ func (al *AgentLoop) runAgentLoop( return finalContent, nil } -func (al *AgentLoop) targetReasoningChannelID(channelName string) (chatID string) { - if al.channelManager == nil { - return "" - } - if ch, ok := al.channelManager.GetChannel(channelName); ok { - return ch.ReasoningChannelID() - } - return "" -} - -func (al *AgentLoop) handleReasoning( - ctx context.Context, - reasoningContent, channelName, channelID string, -) { - if reasoningContent == "" || channelName == "" || channelID == "" { - return - } - - // Check context cancellation before attempting to publish, - // since PublishOutbound's select may race between send and ctx.Done(). - if ctx.Err() != nil { - return - } - - // Use a short timeout so the goroutine does not block indefinitely when - // the outbound bus is full. Reasoning output is best-effort; dropping it - // is acceptable to avoid goroutine accumulation. - pubCtx, pubCancel := context.WithTimeout(ctx, 5*time.Second) - defer pubCancel() - - if err := al.bus.PublishOutbound(pubCtx, bus.OutboundMessage{ - Channel: channelName, - ChatID: channelID, - Content: reasoningContent, - }); err != nil { - // Treat context.DeadlineExceeded / context.Canceled as expected - // (bus full under load, or parent canceled). Check the error - // itself rather than ctx.Err(), because pubCtx may time out - // (5 s) while the parent ctx is still active. - // Also treat ErrBusClosed as expected โ€” it occurs during normal - // shutdown when the bus is closed before all goroutines finish. - if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) || - errors.Is(err, bus.ErrBusClosed) { - logger.DebugCF("agent", "Reasoning publish skipped (timeout/cancel)", map[string]any{ - "channel": channelName, - "error": err.Error(), - }) - } else { - logger.WarnCF("agent", "Failed to publish reasoning (best-effort)", map[string]any{ - "channel": channelName, - "error": err.Error(), - }) - } - } -} - -// runLLMIteration executes the LLM call loop with tool handling. -func (al *AgentLoop) runLLMIteration( - ctx context.Context, - agent *AgentInstance, - messages []providers.Message, - opts processOptions, -) (string, int, error) { - iteration := 0 - var finalContent string - - for iteration < agent.MaxIterations { - iteration++ - - logger.DebugCF("agent", "LLM iteration", - map[string]any{ - "agent_id": agent.ID, - "iteration": iteration, - "max": agent.MaxIterations, - }) - - // Build tool definitions - providerToolDefs := agent.Tools.ToProviderDefs() - - // Log LLM request details - logger.DebugCF("agent", "LLM request", - map[string]any{ - "agent_id": agent.ID, - "iteration": iteration, - "model": agent.Model, - "messages_count": len(messages), - "tools_count": len(providerToolDefs), - "max_tokens": agent.MaxTokens, - "temperature": agent.Temperature, - "system_prompt_len": len(messages[0].Content), - }) - - // Log full messages (detailed) - logger.DebugCF("agent", "Full LLM request", - map[string]any{ - "iteration": iteration, - "messages_json": formatMessagesForLog(messages), - "tools_json": formatToolsForLog(providerToolDefs), - }) - - // Call LLM with fallback chain if candidates are configured. - var response *providers.LLMResponse - var err error - - callLLM := func() (*providers.LLMResponse, error) { - if len(agent.Candidates) > 1 && al.fallback != nil { - fbResult, fbErr := al.fallback.Execute( - ctx, - agent.Candidates, - func(ctx context.Context, provider, model string) (*providers.LLMResponse, error) { - return agent.Provider.Chat( - ctx, - messages, - providerToolDefs, - model, - map[string]any{ - "max_tokens": agent.MaxTokens, - "temperature": agent.Temperature, - "prompt_cache_key": agent.ID, - }, - ) - }, - ) - if fbErr != nil { - return nil, fbErr - } - if fbResult.Provider != "" && len(fbResult.Attempts) > 0 { - logger.InfoCF( - "agent", - fmt.Sprintf("Fallback: succeeded with %s/%s after %d attempts", - fbResult.Provider, fbResult.Model, len(fbResult.Attempts)+1), - map[string]any{"agent_id": agent.ID, "iteration": iteration}, - ) - } - return fbResult.Response, nil - } - return agent.Provider.Chat(ctx, messages, providerToolDefs, agent.Model, map[string]any{ - "max_tokens": agent.MaxTokens, - "temperature": agent.Temperature, - "prompt_cache_key": agent.ID, - }) - } - - // Retry loop for context/token errors - maxRetries := 2 - for retry := 0; retry <= maxRetries; retry++ { - response, err = callLLM() - if err == nil { - break - } - - errMsg := strings.ToLower(err.Error()) - - // Check if this is a network/HTTP timeout โ€” not a context window error. - isTimeoutError := errors.Is(err, context.DeadlineExceeded) || - strings.Contains(errMsg, "deadline exceeded") || - strings.Contains(errMsg, "client.timeout") || - strings.Contains(errMsg, "timed out") || - strings.Contains(errMsg, "timeout exceeded") - - // Detect real context window / token limit errors, excluding network timeouts. - isContextError := !isTimeoutError && (strings.Contains(errMsg, "context_length_exceeded") || - strings.Contains(errMsg, "context window") || - strings.Contains(errMsg, "maximum context length") || - strings.Contains(errMsg, "token limit") || - strings.Contains(errMsg, "too many tokens") || - strings.Contains(errMsg, "max_tokens") || - strings.Contains(errMsg, "invalidparameter") || - strings.Contains(errMsg, "prompt is too long") || - strings.Contains(errMsg, "request too large")) - - if isTimeoutError && retry < maxRetries { - backoff := time.Duration(retry+1) * 5 * time.Second - logger.WarnCF("agent", "Timeout error, retrying after backoff", map[string]any{ - "error": err.Error(), - "retry": retry, - "backoff": backoff.String(), - }) - time.Sleep(backoff) - continue - } - - if isContextError && retry < maxRetries { - logger.WarnCF( - "agent", - "Context window error detected, attempting compression", - map[string]any{ - "error": err.Error(), - "retry": retry, - }, - ) - - if retry == 0 && !constants.IsInternalChannel(opts.Channel) { - al.bus.PublishOutbound(ctx, bus.OutboundMessage{ - Channel: opts.Channel, - ChatID: opts.ChatID, - Content: "Context window exceeded. Compressing history and retrying...", - }) - } - - al.forceCompression(agent, opts.SessionKey) - newHistory := agent.Sessions.GetHistory(opts.SessionKey) - newSummary := agent.Sessions.GetSummary(opts.SessionKey) - messages = agent.ContextBuilder.BuildMessages( - newHistory, newSummary, "", - nil, opts.Channel, opts.ChatID, - ) - continue - } - break - } - - if err != nil { - logger.ErrorCF("agent", "LLM call failed", - map[string]any{ - "agent_id": agent.ID, - "iteration": iteration, - "error": err.Error(), - }) - return "", iteration, fmt.Errorf("LLM call failed after retries: %w", err) - } - - go al.handleReasoning( - ctx, - response.Reasoning, - opts.Channel, - al.targetReasoningChannelID(opts.Channel), - ) - - logger.DebugCF("agent", "LLM response", - map[string]any{ - "agent_id": agent.ID, - "iteration": iteration, - "content_chars": len(response.Content), - "tool_calls": len(response.ToolCalls), - "reasoning": response.Reasoning, - "target_channel": al.targetReasoningChannelID(opts.Channel), - "channel": opts.Channel, - }) - // Check if no tool calls - we're done - if len(response.ToolCalls) == 0 { - finalContent = response.Content - logger.InfoCF("agent", "LLM response without tool calls (direct answer)", - map[string]any{ - "agent_id": agent.ID, - "iteration": iteration, - "content_chars": len(finalContent), - }) - break - } - - normalizedToolCalls := make([]providers.ToolCall, 0, len(response.ToolCalls)) - for _, tc := range response.ToolCalls { - normalizedToolCalls = append(normalizedToolCalls, providers.NormalizeToolCall(tc)) - } - - // Log tool calls - toolNames := make([]string, 0, len(normalizedToolCalls)) - for _, tc := range normalizedToolCalls { - toolNames = append(toolNames, tc.Name) - } - logger.InfoCF("agent", "LLM requested tool calls", - map[string]any{ - "agent_id": agent.ID, - "tools": toolNames, - "count": len(normalizedToolCalls), - "iteration": iteration, - }) - - // Build assistant message with tool calls - assistantMsg := providers.Message{ - Role: "assistant", - Content: response.Content, - ReasoningContent: response.ReasoningContent, - } - for _, tc := range normalizedToolCalls { - argumentsJSON, _ := json.Marshal(tc.Arguments) - // Copy ExtraContent to ensure thought_signature is persisted for Gemini 3 - extraContent := tc.ExtraContent - thoughtSignature := "" - if tc.Function != nil { - thoughtSignature = tc.Function.ThoughtSignature - } - - assistantMsg.ToolCalls = append(assistantMsg.ToolCalls, providers.ToolCall{ - ID: tc.ID, - Type: "function", - Name: tc.Name, - Function: &providers.FunctionCall{ - Name: tc.Name, - Arguments: string(argumentsJSON), - ThoughtSignature: thoughtSignature, - }, - ExtraContent: extraContent, - ThoughtSignature: thoughtSignature, - }) - } - messages = append(messages, assistantMsg) - - // Save assistant message with tool calls to session - agent.Sessions.AddFullMessage(opts.SessionKey, assistantMsg) - - // Execute tool calls - for _, tc := range normalizedToolCalls { - argsJSON, _ := json.Marshal(tc.Arguments) - argsPreview := utils.Truncate(string(argsJSON), 200) - logger.InfoCF("agent", fmt.Sprintf("Tool call: %s(%s)", tc.Name, argsPreview), - map[string]any{ - "agent_id": agent.ID, - "tool": tc.Name, - "iteration": iteration, - }) - - // Create async callback for tools that implement AsyncTool - // NOTE: Following openclaw's design, async tools do NOT send results directly to users. - // Instead, they notify the agent via PublishInbound, and the agent decides - // whether to forward the result to the user (in processSystemMessage). - asyncCallback := func(callbackCtx context.Context, result *tools.ToolResult) { - // Log the async completion but don't send directly to user - // The agent will handle user notification via processSystemMessage - if !result.Silent && result.ForUser != "" { - logger.InfoCF("agent", "Async tool completed, agent will handle notification", - map[string]any{ - "tool": tc.Name, - "content_len": len(result.ForUser), - }) - } - } - - toolResult := agent.Tools.ExecuteWithContext( - ctx, - tc.Name, - tc.Arguments, - opts.Channel, - opts.ChatID, - asyncCallback, - ) - - // Send ForUser content to user immediately if not Silent - if !toolResult.Silent && toolResult.ForUser != "" && opts.SendResponse { - al.bus.PublishOutbound(ctx, bus.OutboundMessage{ - Channel: opts.Channel, - ChatID: opts.ChatID, - Content: toolResult.ForUser, - }) - logger.DebugCF("agent", "Sent tool result to user", - map[string]any{ - "tool": tc.Name, - "content_len": len(toolResult.ForUser), - }) - } - - // If tool returned media refs, publish them as outbound media - if len(toolResult.Media) > 0 && opts.SendResponse { - parts := make([]bus.MediaPart, 0, len(toolResult.Media)) - for _, ref := range toolResult.Media { - part := bus.MediaPart{Ref: ref} - // Populate metadata from MediaStore when available - if al.mediaStore != nil { - if _, meta, err := al.mediaStore.ResolveWithMeta(ref); err == nil { - part.Filename = meta.Filename - part.ContentType = meta.ContentType - part.Type = inferMediaType(meta.Filename, meta.ContentType) - } - } - parts = append(parts, part) - } - al.bus.PublishOutboundMedia(ctx, bus.OutboundMediaMessage{ - Channel: opts.Channel, - ChatID: opts.ChatID, - Parts: parts, - }) - } - - // Determine content for LLM based on tool result - contentForLLM := toolResult.ForLLM - if contentForLLM == "" && toolResult.Err != nil { - contentForLLM = toolResult.Err.Error() - } - - toolResultMsg := providers.Message{ - Role: "tool", - Content: contentForLLM, - ToolCallID: tc.ID, - } - messages = append(messages, toolResultMsg) - - // Save tool result message to session - agent.Sessions.AddFullMessage(opts.SessionKey, toolResultMsg) - } - } - - return finalContent, iteration, nil -} - -// updateToolContexts updates the context for tools that need channel/chatID info. -func (al *AgentLoop) updateToolContexts(agent *AgentInstance, channel, chatID string) { - // Use ContextualTool interface instead of type assertions - if tool, ok := agent.Tools.Get("message"); ok { - if mt, ok := tool.(tools.ContextualTool); ok { - mt.SetContext(channel, chatID) - } - } - if tool, ok := agent.Tools.Get("spawn"); ok { - if st, ok := tool.(tools.ContextualTool); ok { - st.SetContext(channel, chatID) - } - } - if tool, ok := agent.Tools.Get("subagent"); ok { - if st, ok := tool.(tools.ContextualTool); ok { - st.SetContext(channel, chatID) - } - } -} - -// maybeSummarize triggers summarization if the session history exceeds thresholds. -func (al *AgentLoop) maybeSummarize(agent *AgentInstance, sessionKey, channel, chatID string) { - newHistory := agent.Sessions.GetHistory(sessionKey) - tokenEstimate := al.estimateTokens(newHistory) - threshold := agent.ContextWindow * 75 / 100 - - if len(newHistory) > 20 || tokenEstimate > threshold { - summarizeKey := agent.ID + ":" + sessionKey - if _, loading := al.summarizing.LoadOrStore(summarizeKey, true); !loading { - go func() { - defer al.summarizing.Delete(summarizeKey) - logger.Debug("Memory threshold reached. Optimizing conversation history...") - al.summarizeSession(agent, sessionKey) - }() - } - } -} - -// forceCompression aggressively reduces context when the limit is hit. -// It drops the oldest 50% of messages (keeping system prompt and last user message). -func (al *AgentLoop) forceCompression(agent *AgentInstance, sessionKey string) { - history := agent.Sessions.GetHistory(sessionKey) - if len(history) <= 4 { - return - } - - // Keep system prompt (usually [0]) and the very last message (user's trigger) - // We want to drop the oldest half of the *conversation* - // Assuming [0] is system, [1:] is conversation - conversation := history[1 : len(history)-1] - if len(conversation) == 0 { - return - } - - // Helper to find the mid-point of the conversation - mid := len(conversation) / 2 - - // New history structure: - // 1. System Prompt (with compression note appended) - // 2. Second half of conversation - // 3. Last message - - droppedCount := mid - keptConversation := conversation[mid:] - - newHistory := make([]providers.Message, 0, 1+len(keptConversation)+1) - - // Append compression note to the original system prompt instead of adding a new system message - // This avoids having two consecutive system messages which some APIs (like Zhipu) reject - compressionNote := fmt.Sprintf( - "\n\n[System Note: Emergency compression dropped %d oldest messages due to context limit]", - droppedCount, - ) - enhancedSystemPrompt := history[0] - enhancedSystemPrompt.Content = enhancedSystemPrompt.Content + compressionNote - newHistory = append(newHistory, enhancedSystemPrompt) - - newHistory = append(newHistory, keptConversation...) - newHistory = append(newHistory, history[len(history)-1]) // Last message - - // Update session - agent.Sessions.SetHistory(sessionKey, newHistory) - agent.Sessions.Save(sessionKey) - - logger.WarnCF("agent", "Forced compression executed", map[string]any{ - "session_key": sessionKey, - "dropped_msgs": droppedCount, - "new_count": len(newHistory), - }) -} - -// GetStartupInfo returns information about loaded tools and skills for logging. -func (al *AgentLoop) GetStartupInfo() map[string]any { - info := make(map[string]any) - - agent := al.registry.GetDefaultAgent() - if agent == nil { - return info - } - - // Tools info - toolsList := agent.Tools.List() - info["tools"] = map[string]any{ - "count": len(toolsList), - "names": toolsList, - } - - // Skills info - info["skills"] = agent.ContextBuilder.GetSkillsInfo() - - // Agents info - info["agents"] = map[string]any{ - "count": len(al.registry.ListAgentIDs()), - "ids": al.registry.ListAgentIDs(), - } - - return info -} - -// formatMessagesForLog formats messages for logging -func formatMessagesForLog(messages []providers.Message) string { - if len(messages) == 0 { - return "[]" - } - - var sb strings.Builder - sb.WriteString("[\n") - for i, msg := range messages { - fmt.Fprintf(&sb, " [%d] Role: %s\n", i, msg.Role) - if len(msg.ToolCalls) > 0 { - sb.WriteString(" ToolCalls:\n") - for _, tc := range msg.ToolCalls { - fmt.Fprintf(&sb, " - ID: %s, Type: %s, Name: %s\n", tc.ID, tc.Type, tc.Name) - if tc.Function != nil { - fmt.Fprintf( - &sb, - " Arguments: %s\n", - utils.Truncate(tc.Function.Arguments, 200), - ) - } - } - } - if msg.Content != "" { - content := utils.Truncate(msg.Content, 200) - fmt.Fprintf(&sb, " Content: %s\n", content) - } - if msg.ToolCallID != "" { - fmt.Fprintf(&sb, " ToolCallID: %s\n", msg.ToolCallID) - } - sb.WriteString("\n") - } - sb.WriteString("]") - return sb.String() -} - -// formatToolsForLog formats tool definitions for logging -func formatToolsForLog(toolDefs []providers.ToolDefinition) string { - if len(toolDefs) == 0 { - return "[]" - } - - var sb strings.Builder - sb.WriteString("[\n") - for i, tool := range toolDefs { - fmt.Fprintf(&sb, " [%d] Type: %s, Name: %s\n", i, tool.Type, tool.Function.Name) - fmt.Fprintf(&sb, " Description: %s\n", tool.Function.Description) - if len(tool.Function.Parameters) > 0 { - fmt.Fprintf( - &sb, - " Parameters: %s\n", - utils.Truncate(fmt.Sprintf("%v", tool.Function.Parameters), 200), - ) - } - } - sb.WriteString("]") - return sb.String() -} - // summarizeSession summarizes the conversation history for a session. func (al *AgentLoop) summarizeSession(agent *AgentInstance, sessionKey string) { ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) @@ -1349,107 +976,8 @@ func (al *AgentLoop) summarizeBatch( return response.Content, nil } -// estimateTokens estimates the number of tokens in a message list. -// Uses a safe heuristic of 2.5 characters per token to account for CJK and other -// overheads better than the previous 3 chars/token. -func (al *AgentLoop) estimateTokens(messages []providers.Message) int { - totalChars := 0 - for _, m := range messages { - totalChars += utf8.RuneCountInString(m.Content) - } - // 2.5 chars per token = totalChars * 2 / 5 - return totalChars * 2 / 5 -} -func (al *AgentLoop) handleCommand(ctx context.Context, msg bus.InboundMessage) (string, bool) { - content := strings.TrimSpace(msg.Content) - if !strings.HasPrefix(content, "/") { - return "", false - } - parts := strings.Fields(content) - if len(parts) == 0 { - return "", false - } - - cmd := parts[0] - args := parts[1:] - - switch cmd { - case "/show": - if len(args) < 1 { - return "Usage: /show [model|channel|agents]", true - } - switch args[0] { - case "model": - defaultAgent := al.registry.GetDefaultAgent() - if defaultAgent == nil { - return "No default agent configured", true - } - return fmt.Sprintf("Current model: %s", defaultAgent.Model), true - case "channel": - return fmt.Sprintf("Current channel: %s", msg.Channel), true - case "agents": - agentIDs := al.registry.ListAgentIDs() - return fmt.Sprintf("Registered agents: %s", strings.Join(agentIDs, ", ")), true - default: - return fmt.Sprintf("Unknown show target: %s", args[0]), true - } - - case "/list": - if len(args) < 1 { - return "Usage: /list [models|channels|agents]", true - } - switch args[0] { - case "models": - return "Available models: configured in config.json per agent", true - case "channels": - if al.channelManager == nil { - return "Channel manager not initialized", true - } - channels := al.channelManager.GetEnabledChannels() - if len(channels) == 0 { - return "No channels enabled", true - } - return fmt.Sprintf("Enabled channels: %s", strings.Join(channels, ", ")), true - case "agents": - agentIDs := al.registry.ListAgentIDs() - return fmt.Sprintf("Registered agents: %s", strings.Join(agentIDs, ", ")), true - default: - return fmt.Sprintf("Unknown list target: %s", args[0]), true - } - - case "/switch": - if len(args) < 3 || args[1] != "to" { - return "Usage: /switch [model|channel] to ", true - } - target := args[0] - value := args[2] - - switch target { - case "model": - defaultAgent := al.registry.GetDefaultAgent() - if defaultAgent == nil { - return "No default agent configured", true - } - oldModel := defaultAgent.Model - defaultAgent.Model = value - return fmt.Sprintf("Switched model from %s to %s", oldModel, value), true - case "channel": - if al.channelManager == nil { - return "Channel manager not initialized", true - } - if _, exists := al.channelManager.GetChannel(value); !exists && value != "cli" { - return fmt.Sprintf("Channel '%s' not found or not enabled", value), true - } - return fmt.Sprintf("Switched target channel to %s", value), true - default: - return fmt.Sprintf("Unknown switch target: %s", target), true - } - } - - return "", false -} // extractPeer extracts the routing peer from the inbound message's structured Peer field. func extractPeer(msg bus.InboundMessage) *routing.RoutePeer { @@ -1476,3 +1004,10 @@ func extractParentPeer(msg bus.InboundMessage) *routing.RoutePeer { } return &routing.RoutePeer{Kind: parentKind, ID: parentID} } + +// activeContextPath returns the full path for the active_context.json file +// stored inside the workspace directory. +func activeContextPath(workspace string) string { + return filepath.Join(workspace, "active_context.json") +} + diff --git a/pkg/agent/memory.go b/pkg/agent/memory.go index 01e682f3b..f5925e9a4 100644 --- a/pkg/agent/memory.go +++ b/pkg/agent/memory.go @@ -7,121 +7,201 @@ package agent import ( + "database/sql" "fmt" "os" "path/filepath" "strings" + "sync" "time" - "github.com/sipeed/picoclaw/pkg/fileutil" + "github.com/sipeed/picoclaw/pkg/logger" + + _ "modernc.org/sqlite" ) -// MemoryStore manages persistent memory for the agent. -// - Long-term memory: memory/MEMORY.md -// - Daily notes: memory/YYYYMM/YYYYMMDD.md +// MemoryStore manages persistent memory for the agent using SQLite. +// +// Schema: +// - long_term: single-row table holding the long-term memory content +// - daily_notes: one row per day (key = "YYYYMMDD") +// - memory_entries: individually tagged memory items +// +// The database file is stored at workspace/memory.db. type MemoryStore struct { - workspace string - memoryDir string - memoryFile string + workspace string + db *sql.DB + mu sync.Mutex // serialise writes } -// NewMemoryStore creates a new MemoryStore with the given workspace path. -// It ensures the memory directory exists. +// NewMemoryStore creates a new MemoryStore backed by SQLite. +// It creates the database and tables if they do not exist. func NewMemoryStore(workspace string) *MemoryStore { - memoryDir := filepath.Join(workspace, "memory") - memoryFile := filepath.Join(memoryDir, "MEMORY.md") + dbPath := filepath.Join(workspace, "memory.db") - // Ensure memory directory exists - os.MkdirAll(memoryDir, 0o755) + // Ensure workspace directory exists. + os.MkdirAll(workspace, 0o755) - return &MemoryStore{ - workspace: workspace, - memoryDir: memoryDir, - memoryFile: memoryFile, + db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(wal)&_pragma=busy_timeout(5000)") + if err != nil { + logger.DebugCF("memory", "Failed to open memory DB", map[string]any{"error": err.Error()}) + // Return a store that degrades gracefully (methods return empty / no-op). + return &MemoryStore{workspace: workspace} + } + + // Create tables. + ddl := ` +CREATE TABLE IF NOT EXISTS long_term ( + id INTEGER PRIMARY KEY CHECK (id = 1), + content TEXT NOT NULL DEFAULT '' +); +INSERT OR IGNORE INTO long_term (id, content) VALUES (1, ''); + +CREATE TABLE IF NOT EXISTS daily_notes ( + day TEXT PRIMARY KEY, -- YYYYMMDD + content TEXT NOT NULL DEFAULT '' +); + +CREATE TABLE IF NOT EXISTS memory_entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + content TEXT NOT NULL, + tags TEXT NOT NULL DEFAULT '', -- comma-separated, lowercase + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE TABLE IF NOT EXISTS cot_usage ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + intent TEXT NOT NULL DEFAULT '', + tags TEXT NOT NULL DEFAULT '', -- comma-separated tags from message analysis + cot_prompt TEXT NOT NULL DEFAULT '', -- LLM-generated thinking strategy + message TEXT NOT NULL DEFAULT '', -- first 200 chars of user message + feedback INTEGER NOT NULL DEFAULT 0, -- -1=bad, 0=neutral, 1=good + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); +` + if _, err := db.Exec(ddl); err != nil { + logger.DebugCF("memory", "Failed to initialise memory DB tables", map[string]any{"error": err.Error()}) + db.Close() + return &MemoryStore{workspace: workspace} + } + + ms := &MemoryStore{ + workspace: workspace, + db: db, + } + + // Migrate from legacy file-based storage if memory.db was just created. + ms.migrateFromFiles() + + return ms +} + +// Close closes the underlying database. Safe to call multiple times. +func (ms *MemoryStore) Close() { + if ms.db != nil { + ms.db.Close() } } -// getTodayFile returns the path to today's daily note file (memory/YYYYMM/YYYYMMDD.md). -func (ms *MemoryStore) getTodayFile() string { - today := time.Now().Format("20060102") // YYYYMMDD - monthDir := today[:6] // YYYYMM - filePath := filepath.Join(ms.memoryDir, monthDir, today+".md") - return filePath -} +// --- Long-term memory ------------------------------------------------------- -// ReadLongTerm reads the long-term memory (MEMORY.md). -// Returns empty string if the file doesn't exist. +// ReadLongTerm reads the long-term memory content. +// Returns empty string if the database is unavailable. func (ms *MemoryStore) ReadLongTerm() string { - if data, err := os.ReadFile(ms.memoryFile); err == nil { - return string(data) + if ms.db == nil { + return "" } - return "" + var content string + err := ms.db.QueryRow("SELECT content FROM long_term WHERE id = 1").Scan(&content) + if err != nil { + return "" + } + return content } -// WriteLongTerm writes content to the long-term memory file (MEMORY.md). +// WriteLongTerm replaces the long-term memory content. func (ms *MemoryStore) WriteLongTerm(content string) error { - // Use unified atomic write utility with explicit sync for flash storage reliability. - // Using 0o600 (owner read/write only) for secure default permissions. - return fileutil.WriteFileAtomic(ms.memoryFile, []byte(content), 0o600) + if ms.db == nil { + return fmt.Errorf("memory DB not available") + } + ms.mu.Lock() + defer ms.mu.Unlock() + _, err := ms.db.Exec("UPDATE long_term SET content = ? WHERE id = 1", content) + return err +} + +// --- Daily notes ------------------------------------------------------------ + +// todayKey returns today's date as "YYYYMMDD". +func todayKey() string { + return time.Now().Format("20060102") } // ReadToday reads today's daily note. -// Returns empty string if the file doesn't exist. +// Returns empty string if the file doesn't exist or the database is unavailable. func (ms *MemoryStore) ReadToday() string { - todayFile := ms.getTodayFile() - if data, err := os.ReadFile(todayFile); err == nil { - return string(data) + if ms.db == nil { + return "" } - return "" + var content string + err := ms.db.QueryRow("SELECT content FROM daily_notes WHERE day = ?", todayKey()).Scan(&content) + if err != nil { + return "" + } + return content } // AppendToday appends content to today's daily note. -// If the file doesn't exist, it creates a new file with a date header. +// If no note exists for today, a new one is created with a date header. func (ms *MemoryStore) AppendToday(content string) error { - todayFile := ms.getTodayFile() - - // Ensure month directory exists - monthDir := filepath.Dir(todayFile) - if err := os.MkdirAll(monthDir, 0o755); err != nil { - return err + if ms.db == nil { + return fmt.Errorf("memory DB not available") } + ms.mu.Lock() + defer ms.mu.Unlock() - var existingContent string - if data, err := os.ReadFile(todayFile); err == nil { - existingContent = string(data) - } + key := todayKey() - var newContent string - if existingContent == "" { - // Add header for new day + var existing string + err := ms.db.QueryRow("SELECT content FROM daily_notes WHERE day = ?", key).Scan(&existing) + if err == sql.ErrNoRows || existing == "" { + // New day โ€” add header. header := fmt.Sprintf("# %s\n\n", time.Now().Format("2006-01-02")) - newContent = header + content - } else { - // Append to existing content - newContent = existingContent + "\n" + content + content = header + content + _, err = ms.db.Exec( + "INSERT OR REPLACE INTO daily_notes (day, content) VALUES (?, ?)", + key, content, + ) + } else if err == nil { + // Append to existing. + content = existing + "\n" + content + _, err = ms.db.Exec("UPDATE daily_notes SET content = ? WHERE day = ?", content, key) } - - // Use unified atomic write utility with explicit sync for flash storage reliability. - return fileutil.WriteFileAtomic(todayFile, []byte(newContent), 0o600) + return err } // GetRecentDailyNotes returns daily notes from the last N days. // Contents are joined with "---" separator. func (ms *MemoryStore) GetRecentDailyNotes(days int) string { + if ms.db == nil { + return "" + } + var sb strings.Builder first := true for i := range days { date := time.Now().AddDate(0, 0, -i) - dateStr := date.Format("20060102") // YYYYMMDD - monthDir := dateStr[:6] // YYYYMM - filePath := filepath.Join(ms.memoryDir, monthDir, dateStr+".md") + key := date.Format("20060102") - if data, err := os.ReadFile(filePath); err == nil { + var content string + err := ms.db.QueryRow("SELECT content FROM daily_notes WHERE day = ?", key).Scan(&content) + if err == nil && content != "" { if !first { sb.WriteString("\n\n---\n\n") } - sb.Write(data) + sb.WriteString(content) first = false } } @@ -129,30 +209,676 @@ func (ms *MemoryStore) GetRecentDailyNotes(days int) string { return sb.String() } +// --- Tagged memory entries --------------------------------------------------- + +// MemoryEntry represents a single tagged memory item. +type MemoryEntry struct { + ID int64 + Content string + Tags []string + CreatedAt string + UpdatedAt string +} + +// normaliseTags lowercases, trims, deduplicates, and sorts tags. +func normaliseTags(tags []string) []string { + seen := make(map[string]struct{}, len(tags)) + out := make([]string, 0, len(tags)) + for _, t := range tags { + t = strings.ToLower(strings.TrimSpace(t)) + if t == "" { + continue + } + if _, ok := seen[t]; !ok { + seen[t] = struct{}{} + out = append(out, t) + } + } + return out +} + +// joinTags joins tags with "," for storage. +func joinTags(tags []string) string { + return strings.Join(normaliseTags(tags), ",") +} + +// splitTags splits a stored tag string back into a slice. +func splitTags(s string) []string { + if s == "" { + return nil + } + return strings.Split(s, ",") +} + +// AddEntry inserts a new tagged memory entry. Returns the new entry ID. +func (ms *MemoryStore) AddEntry(content string, tags []string) (int64, error) { + if ms.db == nil { + return 0, fmt.Errorf("memory DB not available") + } + ms.mu.Lock() + defer ms.mu.Unlock() + + res, err := ms.db.Exec( + "INSERT INTO memory_entries (content, tags) VALUES (?, ?)", + content, joinTags(tags), + ) + if err != nil { + return 0, err + } + return res.LastInsertId() +} + +// UpdateEntry updates the content and tags of an existing entry. +func (ms *MemoryStore) UpdateEntry(id int64, content string, tags []string) error { + if ms.db == nil { + return fmt.Errorf("memory DB not available") + } + ms.mu.Lock() + defer ms.mu.Unlock() + + _, err := ms.db.Exec( + "UPDATE memory_entries SET content = ?, tags = ?, updated_at = datetime('now') WHERE id = ?", + content, joinTags(tags), id, + ) + return err +} + +// DeleteEntry removes a memory entry by ID. +func (ms *MemoryStore) DeleteEntry(id int64) error { + if ms.db == nil { + return fmt.Errorf("memory DB not available") + } + ms.mu.Lock() + defer ms.mu.Unlock() + + _, err := ms.db.Exec("DELETE FROM memory_entries WHERE id = ?", id) + return err +} + +// GetEntry retrieves a single memory entry by ID. +func (ms *MemoryStore) GetEntry(id int64) (*MemoryEntry, error) { + if ms.db == nil { + return nil, fmt.Errorf("memory DB not available") + } + var e MemoryEntry + var tagsStr string + err := ms.db.QueryRow( + "SELECT id, content, tags, created_at, updated_at FROM memory_entries WHERE id = ?", id, + ).Scan(&e.ID, &e.Content, &tagsStr, &e.CreatedAt, &e.UpdatedAt) + if err != nil { + return nil, err + } + e.Tags = splitTags(tagsStr) + return &e, nil +} + +// SearchByTag returns all entries that contain the given tag. +// Tag matching is case-insensitive (tags are stored lowercase). +func (ms *MemoryStore) SearchByTag(tag string) ([]MemoryEntry, error) { + if ms.db == nil { + return nil, fmt.Errorf("memory DB not available") + } + tag = strings.ToLower(strings.TrimSpace(tag)) + if tag == "" { + return nil, nil + } + + // Match: exact tag as whole string, at start, at end, or in the middle. + // Pattern: tag OR tag,... OR ...,tag OR ...,tag,... + rows, err := ms.db.Query( + `SELECT id, content, tags, created_at, updated_at FROM memory_entries + WHERE tags = ? OR tags LIKE ? OR tags LIKE ? OR tags LIKE ? + ORDER BY updated_at DESC`, + tag, tag+",%", "%,"+tag, "%,"+tag+",%", + ) + if err != nil { + return nil, err + } + defer rows.Close() + + return scanEntries(rows) +} + +// SearchByTags returns entries that contain ALL of the given tags. +func (ms *MemoryStore) SearchByTags(tags []string) ([]MemoryEntry, error) { + if ms.db == nil { + return nil, fmt.Errorf("memory DB not available") + } + tags = normaliseTags(tags) + if len(tags) == 0 { + return nil, nil + } + + // Build WHERE clause: each tag must match. + conds := make([]string, 0, len(tags)) + args := make([]any, 0, len(tags)*4) + for _, tag := range tags { + conds = append(conds, + "(tags = ? OR tags LIKE ? OR tags LIKE ? OR tags LIKE ?)") + args = append(args, tag, tag+",%", "%,"+tag, "%,"+tag+",%") + } + + query := fmt.Sprintf( + "SELECT id, content, tags, created_at, updated_at FROM memory_entries WHERE %s ORDER BY updated_at DESC", + strings.Join(conds, " AND "), + ) + + rows, err := ms.db.Query(query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + return scanEntries(rows) +} + +// SearchByAnyTag returns entries that contain ANY of the given tags (OR logic). +// Results are deduplicated and ordered by updated_at DESC, limited to 20 entries. +func (ms *MemoryStore) SearchByAnyTag(tags []string) ([]MemoryEntry, error) { + if ms.db == nil { + return nil, fmt.Errorf("memory DB not available") + } + tags = normaliseTags(tags) + if len(tags) == 0 { + return nil, nil + } + + // Build WHERE clause: any tag may match (OR). + conds := make([]string, 0, len(tags)) + args := make([]any, 0, len(tags)*4) + for _, tag := range tags { + conds = append(conds, + "(tags = ? OR tags LIKE ? OR tags LIKE ? OR tags LIKE ?)") + args = append(args, tag, tag+",%", "%,"+tag, "%,"+tag+",%") + } + + query := fmt.Sprintf( + "SELECT id, content, tags, created_at, updated_at FROM memory_entries WHERE %s ORDER BY updated_at DESC LIMIT 20", + strings.Join(conds, " OR "), + ) + + rows, err := ms.db.Query(query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + return scanEntries(rows) +} + +// ListAllTags returns all unique tags used across memory entries. +func (ms *MemoryStore) ListAllTags() ([]string, error) { + if ms.db == nil { + return nil, fmt.Errorf("memory DB not available") + } + + rows, err := ms.db.Query("SELECT DISTINCT tags FROM memory_entries WHERE tags != ''") + if err != nil { + return nil, err + } + defer rows.Close() + + seen := make(map[string]struct{}) + for rows.Next() { + var tagsStr string + if err := rows.Scan(&tagsStr); err != nil { + continue + } + for _, t := range splitTags(tagsStr) { + seen[t] = struct{}{} + } + } + + result := make([]string, 0, len(seen)) + for t := range seen { + result = append(result, t) + } + return result, nil +} + +// ListEntries returns the most recent N entries (all tags), ordered newest first. +func (ms *MemoryStore) ListEntries(limit int) ([]MemoryEntry, error) { + if ms.db == nil { + return nil, fmt.Errorf("memory DB not available") + } + if limit <= 0 { + limit = 50 + } + + rows, err := ms.db.Query( + "SELECT id, content, tags, created_at, updated_at FROM memory_entries ORDER BY updated_at DESC LIMIT ?", + limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + return scanEntries(rows) +} + +// scanEntries is a helper to scan rows into MemoryEntry slices. +func scanEntries(rows *sql.Rows) ([]MemoryEntry, error) { + var entries []MemoryEntry + for rows.Next() { + var e MemoryEntry + var tagsStr string + if err := rows.Scan(&e.ID, &e.Content, &tagsStr, &e.CreatedAt, &e.UpdatedAt); err != nil { + return entries, err + } + e.Tags = splitTags(tagsStr) + entries = append(entries, e) + } + return entries, rows.Err() +} + +// --- Composite context ------------------------------------------------------ + // GetMemoryContext returns formatted memory context for the agent prompt. -// Includes long-term memory and recent daily notes. +// Includes long-term memory, recent daily notes, and recent tagged entries. func (ms *MemoryStore) GetMemoryContext() string { longTerm := ms.ReadLongTerm() recentNotes := ms.GetRecentDailyNotes(3) - if longTerm == "" && recentNotes == "" { - return "" - } - var sb strings.Builder + hasContent := false if longTerm != "" { sb.WriteString("## Long-term Memory\n\n") sb.WriteString(longTerm) + hasContent = true } if recentNotes != "" { - if longTerm != "" { + if hasContent { sb.WriteString("\n\n---\n\n") } sb.WriteString("## Recent Daily Notes\n\n") sb.WriteString(recentNotes) + hasContent = true } + // Include recent tagged memory entries. + entries, _ := ms.ListEntries(10) + if len(entries) > 0 { + if hasContent { + sb.WriteString("\n\n---\n\n") + } + sb.WriteString("## Tagged Memories\n\n") + for _, e := range entries { + tagLabel := "" + if len(e.Tags) > 0 { + tagLabel = " [" + strings.Join(e.Tags, ", ") + "]" + } + fmt.Fprintf(&sb, "- (#%d%s) %s\n", e.ID, tagLabel, e.Content) + } + hasContent = true + } + + if !hasContent { + return "" + } return sb.String() } + +// --- CoT usage tracking (learning) ------------------------------------------ + +// CotUsageRecord represents a single CoT usage entry. +type CotUsageRecord struct { + ID int64 + Intent string + Tags []string // Tags from the message analysis + CotPrompt string // LLM-generated thinking strategy + Message string + Feedback int // -1=bad, 0=neutral, 1=good + CreatedAt string +} + +// CotStats holds aggregated statistics for an intent. +type CotStats struct { + Intent string + TotalUses int + AvgScore float64 // Average feedback score + LastUsed string +} + +// RecordCotUsage logs a CoT usage event with the LLM-generated prompt and tags. +// messagePreview is truncated to 200 characters. +func (ms *MemoryStore) RecordCotUsage(intent string, tags []string, cotPrompt, message string) (int64, error) { + if ms.db == nil { + return 0, fmt.Errorf("memory DB not available") + } + ms.mu.Lock() + defer ms.mu.Unlock() + + // Truncate message preview. + if len(message) > 200 { + message = message[:200] + } + + tagStr := strings.Join(tags, ",") + res, err := ms.db.Exec( + "INSERT INTO cot_usage (intent, tags, cot_prompt, message) VALUES (?, ?, ?, ?)", + intent, tagStr, cotPrompt, message, + ) + if err != nil { + return 0, err + } + return res.LastInsertId() +} + +// UpdateCotFeedback updates the feedback score for a CoT usage record. +// score: -1=bad, 0=neutral, 1=good. +func (ms *MemoryStore) UpdateCotFeedback(id int64, score int) error { + if ms.db == nil { + return fmt.Errorf("memory DB not available") + } + if score < -1 || score > 1 { + return fmt.Errorf("feedback score must be -1, 0, or 1") + } + ms.mu.Lock() + defer ms.mu.Unlock() + + _, err := ms.db.Exec("UPDATE cot_usage SET feedback = ? WHERE id = ?", score, id) + return err +} + +// UpdateLatestCotFeedback updates the feedback score for the most recent +// CoT usage record. This is useful when the user provides feedback after +// the main LLM has responded (at which point the usage ID may not be tracked). +func (ms *MemoryStore) UpdateLatestCotFeedback(score int) error { + if ms.db == nil { + return fmt.Errorf("memory DB not available") + } + ms.mu.Lock() + defer ms.mu.Unlock() + + _, err := ms.db.Exec( + "UPDATE cot_usage SET feedback = ? WHERE id = (SELECT MAX(id) FROM cot_usage)", + score, + ) + return err +} + +// GetCotStats returns aggregated statistics per intent, +// based on usage in the last N days. Ordered by total uses descending. +func (ms *MemoryStore) GetCotStats(days int) ([]CotStats, error) { + if ms.db == nil { + return nil, fmt.Errorf("memory DB not available") + } + if days <= 0 { + days = 30 + } + + rows, err := ms.db.Query(` + SELECT + intent, + COUNT(*) as total_uses, + COALESCE(AVG(CASE WHEN feedback != 0 THEN CAST(feedback AS REAL) END), 0.0) as avg_score, + MAX(created_at) as last_used + FROM cot_usage + WHERE created_at >= datetime('now', ? || ' days') + GROUP BY intent + ORDER BY total_uses DESC + `, fmt.Sprintf("-%d", days)) + if err != nil { + return nil, err + } + defer rows.Close() + + var stats []CotStats + for rows.Next() { + var s CotStats + if err := rows.Scan(&s.Intent, &s.TotalUses, &s.AvgScore, &s.LastUsed); err != nil { + continue + } + stats = append(stats, s) + } + return stats, rows.Err() +} + +// GetCotIntentStats returns usage stats per intent. +// This is a simpler version that just counts per intent. +func (ms *MemoryStore) GetCotIntentStats(days int) ([]CotStats, error) { + return ms.GetCotStats(days) +} + +// GetTopRatedCotPrompts returns the highest-rated generated CoT prompts. +// If filterTags is non-empty, prioritises prompts that share tags with the query. +// These serve as proven examples for future LLM generation. +func (ms *MemoryStore) GetTopRatedCotPrompts(days, limit int, filterTags []string) ([]CotUsageRecord, error) { + if ms.db == nil { + return nil, fmt.Errorf("memory DB not available") + } + if days <= 0 { + days = 30 + } + if limit <= 0 { + limit = 5 + } + + rows, err := ms.db.Query(` + SELECT id, intent, tags, cot_prompt, message, feedback, created_at + FROM cot_usage + WHERE feedback > 0 + AND cot_prompt != '' + AND created_at >= datetime('now', ? || ' days') + ORDER BY feedback DESC, created_at DESC + LIMIT ? + `, fmt.Sprintf("-%d", days), limit*3) // Over-fetch to filter by tags later. + if err != nil { + return nil, err + } + defer rows.Close() + + var all []CotUsageRecord + for rows.Next() { + var r CotUsageRecord + var tagStr string + if err := rows.Scan(&r.ID, &r.Intent, &tagStr, &r.CotPrompt, &r.Message, &r.Feedback, &r.CreatedAt); err != nil { + continue + } + if tagStr != "" { + r.Tags = strings.Split(tagStr, ",") + } + all = append(all, r) + } + if err := rows.Err(); err != nil { + return nil, err + } + + // If filter tags provided, sort by tag overlap (most relevant first). + if len(filterTags) > 0 && len(all) > 0 { + tagSet := make(map[string]bool, len(filterTags)) + for _, t := range filterTags { + tagSet[strings.ToLower(t)] = true + } + + // Partition: matching first, then non-matching. + var matching, rest []CotUsageRecord + for _, r := range all { + hasOverlap := false + for _, t := range r.Tags { + if tagSet[strings.ToLower(t)] { + hasOverlap = true + break + } + } + if hasOverlap { + matching = append(matching, r) + } else { + rest = append(rest, r) + } + } + all = append(matching, rest...) + } + + if len(all) > limit { + all = all[:limit] + } + return all, nil +} + +// GetRecentCotUsage returns the N most recent CoT usage records. +func (ms *MemoryStore) GetRecentCotUsage(limit int) ([]CotUsageRecord, error) { + if ms.db == nil { + return nil, fmt.Errorf("memory DB not available") + } + if limit <= 0 { + limit = 20 + } + + rows, err := ms.db.Query( + "SELECT id, intent, tags, cot_prompt, message, feedback, created_at FROM cot_usage ORDER BY id DESC LIMIT ?", + limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var records []CotUsageRecord + for rows.Next() { + var r CotUsageRecord + var tagStr string + if err := rows.Scan(&r.ID, &r.Intent, &tagStr, &r.CotPrompt, &r.Message, &r.Feedback, &r.CreatedAt); err != nil { + continue + } + if tagStr != "" { + r.Tags = strings.Split(tagStr, ",") + } + records = append(records, r) + } + return records, rows.Err() +} + +// FormatCotLearningContext formats CoT usage history and top-rated prompts +// into a string for the pre-LLM to learn from past generations. +// currentTags are the tags extracted from the current message, used to +// prioritise relevant proven strategies. +func (ms *MemoryStore) FormatCotLearningContext(days int, currentTags []string) string { + var sb strings.Builder + hasContent := false + + // 1. Usage stats per intent. + stats, err := ms.GetCotStats(days) + if err == nil && len(stats) > 0 { + sb.WriteString("## Historical Usage Stats\n\n") + for _, s := range stats { + scoreLabel := "neutral" + if s.AvgScore > 0.3 { + scoreLabel = "good" + } else if s.AvgScore < -0.3 { + scoreLabel = "poor" + } + fmt.Fprintf(&sb, "- Intent '%s': %d uses, avg feedback=%s (%.1f)\n", + s.Intent, s.TotalUses, scoreLabel, s.AvgScore) + } + sb.WriteString("\n") + hasContent = true + } + + // 2. Top-rated generated prompts as proven examples (filtered by current tags). + topPrompts, err := ms.GetTopRatedCotPrompts(days, 3, currentTags) + if err == nil && len(topPrompts) > 0 { + sb.WriteString("## Proven Strategies (from past sessions with positive feedback)\n\n") + sb.WriteString("These generated strategies received positive feedback. Use similar approaches for similar intents.\n\n") + for i, r := range topPrompts { + msgPreview := r.Message + if len(msgPreview) > 80 { + msgPreview = msgPreview[:80] + "..." + } + tagLabel := "" + if len(r.Tags) > 0 { + tagLabel = fmt.Sprintf(", tags: [%s]", strings.Join(r.Tags, ", ")) + } + fmt.Fprintf(&sb, "### Proven #%d (intent: %s%s, message: \"%s\")\n%s\n\n", + i+1, r.Intent, tagLabel, msgPreview, r.CotPrompt) + } + hasContent = true + } + + if !hasContent { + return "" + } + return sb.String() +} + +// --- Migration from legacy files -------------------------------------------- + +// migrateFromFiles imports data from the old file-based storage +// (memory/MEMORY.md and memory/YYYYMM/YYYYMMDD.md) into SQLite. +// It only runs if the long_term content is empty (fresh DB) AND the +// legacy directory exists. After a successful migration the legacy +// directory is renamed to memory_backup. +func (ms *MemoryStore) migrateFromFiles() { + if ms.db == nil { + return + } + + memoryDir := filepath.Join(ms.workspace, "memory") + + // Check if the legacy directory exists. + info, err := os.Stat(memoryDir) + if err != nil || !info.IsDir() { + return // nothing to migrate + } + + // Only migrate if the DB is empty (fresh). + longTerm := ms.ReadLongTerm() + if longTerm != "" { + return // already has data + } + + logger.DebugCF("memory", "Migrating legacy file-based memory to SQLite", nil) + + // 1. Long-term memory. + memoryFile := filepath.Join(memoryDir, "MEMORY.md") + if data, err := os.ReadFile(memoryFile); err == nil && len(data) > 0 { + ms.WriteLongTerm(string(data)) + } + + // 2. Daily notes โ€” walk YYYYMM/YYYYMMDD.md files. + entries, err := os.ReadDir(memoryDir) + if err != nil { + return + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + monthDir := filepath.Join(memoryDir, entry.Name()) + dayFiles, err := os.ReadDir(monthDir) + if err != nil { + continue + } + for _, df := range dayFiles { + name := df.Name() + if !strings.HasSuffix(name, ".md") { + continue + } + day := strings.TrimSuffix(name, ".md") // YYYYMMDD + if len(day) != 8 { + continue + } + data, err := os.ReadFile(filepath.Join(monthDir, name)) + if err != nil || len(data) == 0 { + continue + } + ms.mu.Lock() + ms.db.Exec( + "INSERT OR IGNORE INTO daily_notes (day, content) VALUES (?, ?)", + day, string(data), + ) + ms.mu.Unlock() + } + } + + // Rename legacy dir so we don't migrate again. + backupDir := filepath.Join(ms.workspace, "memory_backup") + if err := os.Rename(memoryDir, backupDir); err != nil { + logger.DebugCF("memory", "Could not rename legacy memory dir", map[string]any{"error": err.Error()}) + } else { + logger.DebugCF("memory", "Legacy memory migrated and backed up", map[string]any{"backup": backupDir}) + } +} diff --git a/pkg/agent/memory_digest.go b/pkg/agent/memory_digest.go new file mode 100644 index 000000000..da10e4a53 --- /dev/null +++ b/pkg/agent/memory_digest.go @@ -0,0 +1,280 @@ +// PicoClaw - Ultra-lightweight personal AI agent +// License: MIT +// +// Copyright (c) 2026 PicoClaw contributors + +package agent + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/providers" +) + +// MemoryDigestWorker runs as a background goroutine and periodically extracts +// long-term memories from pending TurnRecords. +// +// Design: +// - Fixed interval trigger (default 5 minutes). +// - No llmActive yield mechanism (personal agent, low QPS, API rate-limits handle it). +// - Processes up to 50 pending turns per cycle, grouped by channel_key. +// - On completion, marks turns as "processed" and archives old processed turns. +type MemoryDigestWorker struct { + store *TurnStore + memory *MemoryStore + provider providers.LLMProvider + model string + interval time.Duration +} + +// MemoryDigestConfig holds tunable parameters. +type MemoryDigestConfig struct { + Interval time.Duration // Polling period (default: 5 minutes) + BatchLimit int // Max pending turns per cycle (default: 50) + ArchiveAfterDays int // Archive processed turns older than N days (default: 7) +} + +func defaultDigestConfig() MemoryDigestConfig { + return MemoryDigestConfig{ + Interval: 5 * time.Minute, + BatchLimit: 50, + ArchiveAfterDays: 7, + } +} + +// NewMemoryDigestWorker creates a worker. provider/model may be nil/empty +// if only archival (no LLM extraction) is desired. +func NewMemoryDigestWorker( + store *TurnStore, + memory *MemoryStore, + provider providers.LLMProvider, + model string, +) *MemoryDigestWorker { + return &MemoryDigestWorker{ + store: store, + memory: memory, + provider: provider, + model: model, + interval: defaultDigestConfig().Interval, + } +} + +// SetInterval overrides the polling interval (e.g. for testing). +func (w *MemoryDigestWorker) SetInterval(d time.Duration) { + w.interval = d +} + +// Start launches the background goroutine. It respects ctx cancellation. +func (w *MemoryDigestWorker) Start(ctx context.Context) { + go func() { + ticker := time.NewTicker(w.interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := w.runOnce(ctx); err != nil { + logger.WarnCF("memory_digest", "runOnce error", map[string]any{"error": err.Error()}) + } + } + } + }() + logger.DebugCF("memory_digest", "Worker started", map[string]any{"interval": w.interval.String()}) +} + +// RunOnceNow triggers an immediate digest cycle (useful for testing). +func (w *MemoryDigestWorker) RunOnceNow(ctx context.Context) error { + return w.runOnce(ctx) +} + +// runOnce executes one full digest cycle. +func (w *MemoryDigestWorker) runOnce(ctx context.Context) error { + if w.store == nil { + return nil + } + cfg := defaultDigestConfig() + + // Step 1: Load pending turns. + pending, err := w.store.QueryPending(cfg.BatchLimit) + if err != nil { + return fmt.Errorf("query pending: %w", err) + } + if len(pending) == 0 { + logger.DebugCF("memory_digest", "No pending turns", nil) + // Still run archival. + return w.archive(cfg) + } + + logger.DebugCF("memory_digest", "Processing pending turns", + map[string]any{"count": len(pending)}) + + // Step 2: Group by channel_key to avoid mixing user memories. + groups := make(map[string][]TurnRecord) + for _, t := range pending { + groups[t.ChannelKey] = append(groups[t.ChannelKey], t) + } + + // Step 3: For each group, call LLM to extract memories. + for channelKey, turns := range groups { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + if err := w.processGroup(ctx, channelKey, turns); err != nil { + logger.WarnCF("memory_digest", "Group processing error", + map[string]any{"channel": channelKey, "error": err.Error()}) + // Continue with other groups. + } + } + + // Step 6: Archive old processed turns. + return w.archive(cfg) +} + +// processGroup extracts memories from a batch of turns belonging to one channel. +func (w *MemoryDigestWorker) processGroup(ctx context.Context, channelKey string, turns []TurnRecord) error { + // Build a conversation digest for the LLM. + memories, err := w.extractMemories(ctx, turns) + if err != nil { + // Mark them as processed anyway so we don't loop forever. + logger.WarnCF("memory_digest", "LLM extraction failed, marking as processed", + map[string]any{"channel": channelKey, "error": err.Error()}) + } + + // Step 4: Write extracted memories. + if w.memory != nil { + for _, m := range memories { + if _, addErr := w.memory.AddEntry(m.Content, m.Tags); addErr != nil { + logger.WarnCF("memory_digest", "Failed to save memory", + map[string]any{"error": addErr.Error()}) + } + } + } + + // Step 5: Mark all turns as processed. + for _, t := range turns { + if setErr := w.store.SetStatus(t.ID, "processed"); setErr != nil { + logger.WarnCF("memory_digest", "SetStatus failed", + map[string]any{"id": t.ID, "error": setErr.Error()}) + } + } + + logger.DebugCF("memory_digest", "Group processed", + map[string]any{ + "channel": channelKey, + "turns": len(turns), + "memories_stored": len(memories), + }) + return nil +} + +// digestMemoryResult holds one extracted memory item. +type digestMemoryResult struct { + Content string `json:"content"` + Tags []string `json:"tags"` +} + +const digestPrompt = `Extract important, durable facts worth remembering from these conversation turns. + +Conversation turns: +%s + +Respond with ONLY JSON: {"memories": [{"content": "", "tags": ["tag1"]}]} +Rules: +- max 5 memories total across all turns +- max 3 tags each, lowercase +- skip trivial small-talk +- prefer facts about user preferences, environment, recurring patterns, important decisions +- if nothing worth remembering: {"memories": []}` + +// extractMemories calls the LLM to distil memories from a batch of turns. +// Returns nil memories (not error) when the LLM is unconfigured. +func (w *MemoryDigestWorker) extractMemories(ctx context.Context, turns []TurnRecord) ([]digestMemoryResult, error) { + if w.provider == nil || w.model == "" { + return nil, nil + } + + // Build conversation summary for the prompt. + var sb strings.Builder + for i, t := range turns { + reply := t.Reply + if len(reply) > 500 { + reply = reply[:500] + "..." + } + fmt.Fprintf(&sb, "=== Turn %d (intent: %s, tags: %v) ===\nUser: %s\nAssistant: %s\n\n", + i+1, t.Intent, t.Tags, t.UserMsg, reply) + } + prompt := fmt.Sprintf(digestPrompt, sb.String()) + + resp, err := w.provider.Chat(ctx, []providers.Message{ + {Role: "user", Content: prompt}, + }, nil, w.model, map[string]any{"max_tokens": 512, "temperature": 0.1}) + if err != nil { + return nil, fmt.Errorf("LLM call: %w", err) + } + + raw := strings.TrimSpace(resp.Content) + // Strip markdown fences if present. + if strings.HasPrefix(raw, "```") { + lines := strings.Split(raw, "\n") + if len(lines) > 2 { + raw = strings.Join(lines[1:len(lines)-1], "\n") + } + } + + var result struct { + Memories []digestMemoryResult `json:"memories"` + } + if err := json.Unmarshal([]byte(raw), &result); err != nil { + // Parsing failure โ€” skip extraction, don't fail the whole batch. + logger.WarnCF("memory_digest", "Failed to parse LLM response", + map[string]any{"raw": raw[:min(len(raw), 200)], "error": err.Error()}) + return nil, nil + } + + // Normalise. + out := make([]digestMemoryResult, 0, len(result.Memories)) + for _, m := range result.Memories { + m.Content = strings.TrimSpace(m.Content) + if m.Content == "" { + continue + } + normalised := make([]string, 0, len(m.Tags)) + for _, t := range m.Tags { + t = strings.ToLower(strings.TrimSpace(t)) + if t != "" { + normalised = append(normalised, t) + } + } + m.Tags = normalised + out = append(out, m) + } + return out, nil +} + +// archive runs periodic archival of processed turns. +func (w *MemoryDigestWorker) archive(cfg MemoryDigestConfig) error { + if w.store == nil { + return nil + } + if err := w.store.ArchiveOldProcessed(cfg.ArchiveAfterDays); err != nil { + return fmt.Errorf("archive: %w", err) + } + return nil +} + +// min returns the smaller of a and b. +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/pkg/agent/reflector.go b/pkg/agent/reflector.go new file mode 100644 index 000000000..55a55012b --- /dev/null +++ b/pkg/agent/reflector.go @@ -0,0 +1,969 @@ +// PicoClaw - Ultra-lightweight personal AI agent +// License: MIT +// +// Copyright (c) 2026 PicoClaw contributors + +package agent + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + "time" + + "os" + + "github.com/sipeed/picoclaw/pkg/channels" + "github.com/sipeed/picoclaw/pkg/shell" + "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/providers" + "github.com/sipeed/picoclaw/pkg/tools" +) + +// --------------------------------------------------------------------------- +// Runtime โ€” unified execution engine +// +// The Runtime serves two purposes: +// +// 1. Post-LLM processing: runs async processors after the main LLM responds +// (memory extraction, CoT feedback, error tracking). +// +// 2. Slash commands: handles /{cmd} {args} from users, executed synchronously. +// +// Both share the same MemoryStore and lightweight LLM provider. +// --------------------------------------------------------------------------- + +// --- Post-LLM Processing --------------------------------------------------- + +// RuntimeInput captures everything that happened during a single agent turn. +type RuntimeInput struct { + UserMessage string // Original user message + AssistantReply string // Main LLM's final response + Intent string // Pre-LLM detected intent + Tags []string // Pre-LLM extracted tags + CotPrompt string // Generated thinking strategy + ToolCalls []ToolCallRecord + Iterations int // Number of LLM iterations used + Score int // Phase 3 CalcTurnScore result (set by SyncPhase3) + ChannelKey string // "channel:chatID" (set by runAgentLoop) +} + +// ToolCallRecord captures one tool invocation and its outcome. +type ToolCallRecord struct { + Name string + Error string // Empty if success + Duration time.Duration // How long the tool took +} + +// RuntimeProcessor is a single post-LLM processing step. +type RuntimeProcessor interface { + Name() string + Process(ctx context.Context, input RuntimeInput, memory *MemoryStore) error +} + +// --- Slash Commands --------------------------------------------------------- + +// CommandHandler handles a single /{cmd} invocation. +type CommandHandler func(args []string, memory *MemoryStore) string + +// CommandDef defines a registered slash command. +type CommandDef struct { + Name string // e.g. "memory" + Usage string // e.g. "/memory [list|add|search] ..." + Description string + Handler CommandHandler +} + +// --- Reflector (Phase 3) ---------------------------------------------------- + +// Reflector manages post-LLM processors and slash commands. +// This is Phase 3 (Reflect) of the Runtime Loop. +type Reflector struct { + provider providers.LLMProvider + model string + processors []RuntimeProcessor + commands map[string]CommandDef + mu sync.RWMutex + timeout time.Duration + toolRegistry *tools.ToolRegistry // For /shell command + agentRegistry *AgentRegistry // For /show, /list, /switch + channelManager *channels.Manager // For /list channels, /switch channel +} + + +// NewReflector creates a new Reflector (Phase 3) with built-in processors and commands. +func NewReflector(provider providers.LLMProvider, model string) *Reflector { + r := &Reflector{ + provider: provider, + model: model, + timeout: 30 * time.Second, + commands: make(map[string]CommandDef), + } + + // Built-in processors (post-LLM, async). + // Note: CotEvaluator and MemoryExtractor are intentionally removed from the + // default pipeline โ€” memory extraction is now handled by MemoryDigestWorker + // (batch, background) rather than per-turn inline LLM calls. + r.RegisterProcessor(&ErrorTracker{}) + + // Built-in slash commands. + r.RegisterCommand(CommandDef{ + Name: "help", + Usage: "/help", + Description: "Show all available commands", + Handler: r.cmdHelp, + }) + r.RegisterCommand(CommandDef{ + Name: "memory", + Usage: "/memory [list|add|delete|edit|search|stats] ...", + Description: "Manage long-term memory", + Handler: cmdMemory, + }) + r.RegisterCommand(CommandDef{ + Name: "cot", + Usage: "/cot [feedback|stats|history] ...", + Description: "Manage CoT learning", + Handler: cmdCot, + }) + r.RegisterCommand(CommandDef{ + Name: "runtime", + Usage: "/runtime [status|processors]", + Description: "Runtime status and diagnostics", + Handler: r.cmdRuntimeStatus, + }) + r.RegisterCommand(CommandDef{ + Name: "shell", + Usage: "/shell [args...]", + Description: "Execute shell command in workspace", + Handler: r.cmdShell, + }) + + // System commands (migrated from handleCommand). + r.RegisterCommand(CommandDef{ + Name: "show", + Usage: "/show [model|channel|agents]", + Description: "Show current settings", + Handler: r.cmdShow, + }) + r.RegisterCommand(CommandDef{ + Name: "list", + Usage: "/list [models|channels|agents]", + Description: "List available resources", + Handler: r.cmdList, + }) + r.RegisterCommand(CommandDef{ + Name: "switch", + Usage: "/switch [model|channel] to ", + Description: "Switch model or channel", + Handler: r.cmdSwitch, + }) + + return r +} + + +// RegisterProcessor adds a post-LLM processor. +func (r *Reflector) RegisterProcessor(p RuntimeProcessor) { + r.mu.Lock() + defer r.mu.Unlock() + r.processors = append(r.processors, p) +} + +// RegisterCommand adds a slash command. +func (r *Reflector) RegisterCommand(cmd CommandDef) { + r.mu.Lock() + defer r.mu.Unlock() + r.commands[cmd.Name] = cmd +} + +// SetTools sets the tool registry for /shell command support. +func (r *Reflector) SetTools(registry *tools.ToolRegistry) { + r.mu.Lock() + defer r.mu.Unlock() + r.toolRegistry = registry +} + +// SetAgentInfo provides the Runtime with agent and channel references +// needed by system commands (/show, /list, /switch). +func (r *Reflector) SetAgentInfo(reg *AgentRegistry, cm *channels.Manager) { + r.mu.Lock() + defer r.mu.Unlock() + r.agentRegistry = reg + r.channelManager = cm +} + +// --------------------------------------------------------------------------- +// Post-LLM: async execution +// --------------------------------------------------------------------------- + +// SyncPhase3 runs the synchronous, low-latency part of Phase 3: +// it calculates the Turn score and returns it. The caller must invoke this +// BEFORE PublishOutbound so that Active Context is ready for the next turn. +// Execution target: < 2ms (pure CPU, no I/O). +func (r *Reflector) SyncPhase3(input RuntimeInput) int { + score := CalcTurnScore(input) + logger.DebugCF("reflector", "SyncPhase3 score", + map[string]any{"score": score, "intent": input.Intent, "tools": len(input.ToolCalls)}) + return score +} + +// AsyncPhase3 runs the asynchronous post-turn work: persisting TurnRecord, +// running legacy processors, etc. Call this AFTER PublishOutbound. +func (r *Reflector) AsyncPhase3(input RuntimeInput, memory *MemoryStore, turnStore *TurnStore, activeCtx *ActiveContextStore) { + if r == nil { + return + } + + r.mu.RLock() + processors := make([]RuntimeProcessor, len(r.processors)) + copy(processors, r.processors) + r.mu.RUnlock() + + go func() { + tctx, cancel := context.WithTimeout(context.Background(), r.timeout) + defer cancel() + + // Run registered processors (currently: ErrorTracker). + if memory != nil { + for _, p := range processors { + select { + case <-tctx.Done(): + return + default: + } + start := time.Now() + if err := p.Process(tctx, input, memory); err != nil { + logger.WarnCF("reflector", "Processor failed", + map[string]any{"processor": p.Name(), "error": err.Error(), + "ms": time.Since(start).Milliseconds()}) + } + } + } + + // Persist TurnRecord to turns.db. + if turnStore != nil && input.UserMessage != "" { + record := TurnRecord{ + Ts: time.Now().Unix(), + ChannelKey: input.ChannelKey, + Score: input.Score, + Intent: input.Intent, + Tags: input.Tags, + Status: "pending", + UserMsg: input.UserMessage, + Reply: input.AssistantReply, + ToolCalls: input.ToolCalls, + } + if err := turnStore.Insert(record); err != nil { + logger.WarnCF("reflector", "TurnRecord insert failed", + map[string]any{"error": err.Error()}) + } + } + }() +} + +// RunPostLLM is kept for backward compatibility. New code should use +// SyncPhase3 + AsyncPhase3 instead. +func (r *Reflector) RunPostLLM(input RuntimeInput, memory *MemoryStore) { + r.AsyncPhase3(input, memory, nil, nil) +} + +// --------------------------------------------------------------------------- +// Slash commands: synchronous execution +// --------------------------------------------------------------------------- + +// HandleCommand tries to handle a /{cmd} message. +// Returns (response, true) if handled, ("", false) if not a known command. +func (r *Reflector) HandleCommand(content string, memory *MemoryStore) (string, bool) { + content = strings.TrimSpace(content) + if !strings.HasPrefix(content, "/") { + return "", false + } + + parts := strings.Fields(content) + if len(parts) == 0 { + return "", false + } + + cmdName := strings.TrimPrefix(parts[0], "/") + args := parts[1:] + + r.mu.RLock() + cmd, ok := r.commands[cmdName] + r.mu.RUnlock() + + if !ok { + return "", false // Not our command โ€” let AgentLoop's handleCommand try. + } + + if memory == nil { + return "โš ๏ธ Memory store not available", true + } + + return cmd.Handler(args, memory), true +} + +// ListCommands returns a formatted help text for all registered commands. +func (r *Reflector) ListCommands() string { + r.mu.RLock() + defer r.mu.RUnlock() + + var sb strings.Builder + sb.WriteString("**Runtime Commands**\n\n") + for _, cmd := range r.commands { + fmt.Fprintf(&sb, "โ€ข `%s` โ€” %s\n", cmd.Usage, cmd.Description) + } + return sb.String() +} + +// =========================================================================== +// Built-in slash commands +// =========================================================================== + +// --- /help ------------------------------------------------------------------ + +func (r *Reflector) cmdHelp(_ []string, _ *MemoryStore) string { + var sb strings.Builder + sb.WriteString("๐Ÿ“– **Available Commands**\n\n") + + r.mu.RLock() + for _, cmd := range r.commands { + fmt.Fprintf(&sb, "โ€ข `%s` โ€” %s\n", cmd.Usage, cmd.Description) + } + r.mu.RUnlock() + + return sb.String() +} + +// --- /memory ---------------------------------------------------------------- + +func cmdMemory(args []string, memory *MemoryStore) string { + if len(args) == 0 { + return "Usage: /memory [list|add|delete|edit|search|stats]\n" + + " /memory list โ€” show recent memories\n" + + " /memory add #tags โ€” add a memory\n" + + " /memory delete โ€” delete a memory\n" + + " /memory edit โ€” edit a memory\n" + + " /memory search โ€” search by tags\n" + + " /memory stats โ€” memory statistics" + } + + switch args[0] { + case "list": + limit := 10 + entries, err := memory.ListEntries(limit) + if err != nil { + return fmt.Sprintf("โŒ Error: %v", err) + } + if len(entries) == 0 { + return "๐Ÿ“ญ No memories stored yet." + } + var sb strings.Builder + fmt.Fprintf(&sb, "๐Ÿ“ **Recent Memories** (%d)\n\n", len(entries)) + for _, e := range entries { + tags := "" + if len(e.Tags) > 0 { + tags = " [" + strings.Join(e.Tags, ", ") + "]" + } + preview := e.Content + if len(preview) > 100 { + preview = preview[:100] + "..." + } + fmt.Fprintf(&sb, "โ€ข #%d%s: %s\n", e.ID, tags, preview) + } + return sb.String() + + case "add": + if len(args) < 2 { + return "Usage: /memory add #tag1 #tag2" + } + // Separate content from #tags. + var content []string + var tags []string + for _, a := range args[1:] { + if strings.HasPrefix(a, "#") { + tags = append(tags, strings.TrimPrefix(a, "#")) + } else { + content = append(content, a) + } + } + text := strings.Join(content, " ") + if text == "" { + return "โŒ Memory content cannot be empty" + } + id, err := memory.AddEntry(text, tags) + if err != nil { + return fmt.Sprintf("โŒ Failed to add: %v", err) + } + return fmt.Sprintf("โœ… Memory #%d saved (tags: %v)", id, tags) + + case "search": + if len(args) < 2 { + return "Usage: /memory search [tag2] ..." + } + entries, err := memory.SearchByAnyTag(args[1:]) + if err != nil { + return fmt.Sprintf("โŒ Error: %v", err) + } + if len(entries) == 0 { + return fmt.Sprintf("๐Ÿ” No memories found for tags: %v", args[1:]) + } + var sb strings.Builder + fmt.Fprintf(&sb, "๐Ÿ” **Found %d memories**\n\n", len(entries)) + for _, e := range entries { + tags := "" + if len(e.Tags) > 0 { + tags = " [" + strings.Join(e.Tags, ", ") + "]" + } + preview := e.Content + if len(preview) > 100 { + preview = preview[:100] + "..." + } + fmt.Fprintf(&sb, "โ€ข #%d%s: %s\n", e.ID, tags, preview) + } + return sb.String() + + case "stats": + tags, _ := memory.ListAllTags() + entries, _ := memory.ListEntries(9999) + var sb strings.Builder + sb.WriteString("๐Ÿ“Š **Memory Stats**\n") + fmt.Fprintf(&sb, "โ€ข Total entries: %d\n", len(entries)) + fmt.Fprintf(&sb, "โ€ข Total tags: %d\n", len(tags)) + if len(tags) > 0 { + preview := tags + if len(preview) > 20 { + preview = preview[:20] + } + fmt.Fprintf(&sb, "โ€ข Tags: %s", strings.Join(preview, ", ")) + if len(tags) > 20 { + fmt.Fprintf(&sb, " ... (+%d more)", len(tags)-20) + } + sb.WriteString("\n") + } + return sb.String() + + case "delete": + if len(args) < 2 { + return "Usage: /memory delete " + } + var id int64 + if _, err := fmt.Sscanf(args[1], "%d", &id); err != nil { + return "โŒ Invalid ID. Usage: /memory delete " + } + if err := memory.DeleteEntry(id); err != nil { + return fmt.Sprintf("โŒ Failed: %v", err) + } + return fmt.Sprintf("โœ… Memory #%d deleted", id) + + case "edit": + if len(args) < 3 { + return "Usage: /memory edit #tags" + } + var id int64 + if _, err := fmt.Sscanf(args[1], "%d", &id); err != nil { + return "โŒ Invalid ID. Usage: /memory edit " + } + var content []string + var tags []string + for _, a := range args[2:] { + if strings.HasPrefix(a, "#") { + tags = append(tags, strings.TrimPrefix(a, "#")) + } else { + content = append(content, a) + } + } + text := strings.Join(content, " ") + if text == "" { + return "โŒ Content cannot be empty" + } + if err := memory.UpdateEntry(id, text, tags); err != nil { + return fmt.Sprintf("โŒ Failed: %v", err) + } + return fmt.Sprintf("โœ… Memory #%d updated", id) + + default: + return fmt.Sprintf("Unknown subcommand: %s. Use /memory for help.", args[0]) + } +} + +// --- /cot ------------------------------------------------------------------- + +func cmdCot(args []string, memory *MemoryStore) string { + if len(args) == 0 { + return "Usage: /cot [feedback|stats|history]\n" + + " /cot feedback <1|0|-1> โ€” rate last CoT strategy\n" + + " /cot stats โ€” show CoT performance\n" + + " /cot history [N] โ€” show recent CoT usage" + } + + switch args[0] { + case "feedback": + if len(args) < 2 { + return "Usage: /cot feedback <1|0|-1>" + } + var score int + switch args[1] { + case "1", "+1", "good": + score = 1 + case "-1", "bad": + score = -1 + case "0", "neutral": + score = 0 + default: + return "โŒ Score must be 1 (good), 0 (neutral), or -1 (bad)" + } + if err := memory.UpdateLatestCotFeedback(score); err != nil { + return fmt.Sprintf("โŒ Failed: %v", err) + } + labels := map[int]string{1: "๐Ÿ‘ good", 0: "๐Ÿ˜ neutral", -1: "๐Ÿ‘Ž bad"} + return fmt.Sprintf("โœ… CoT feedback recorded: %s", labels[score]) + + case "stats": + stats, err := memory.GetCotStats(30) + if err != nil || len(stats) == 0 { + return "๐Ÿ“Š No CoT usage data yet." + } + var sb strings.Builder + sb.WriteString("๐Ÿ“Š **CoT Stats (last 30 days)**\n\n") + for _, s := range stats { + scoreLabel := "neutral" + if s.AvgScore > 0.3 { + scoreLabel = "good" + } else if s.AvgScore < -0.3 { + scoreLabel = "poor" + } + fmt.Fprintf(&sb, "โ€ข Intent '%s': %d uses, avg=%s (%.1f)\n", + s.Intent, s.TotalUses, scoreLabel, s.AvgScore) + } + return sb.String() + + case "history": + limit := 5 + if len(args) > 1 { + fmt.Sscanf(args[1], "%d", &limit) + } + records, err := memory.GetRecentCotUsage(limit) + if err != nil || len(records) == 0 { + return "๐Ÿ“œ No CoT history yet." + } + var sb strings.Builder + fmt.Fprintf(&sb, "๐Ÿ“œ **Recent CoT Usage** (%d)\n\n", len(records)) + for _, r := range records { + fb := "๐Ÿ˜" + if r.Feedback > 0 { + fb = "๐Ÿ‘" + } else if r.Feedback < 0 { + fb = "๐Ÿ‘Ž" + } + tags := "" + if len(r.Tags) > 0 { + tags = " [" + strings.Join(r.Tags, ", ") + "]" + } + prompt := r.CotPrompt + if len(prompt) > 80 { + prompt = prompt[:80] + "..." + } + fmt.Fprintf(&sb, "โ€ข #%d %s %s%s: %s\n", r.ID, fb, r.Intent, tags, prompt) + } + return sb.String() + + default: + return fmt.Sprintf("Unknown subcommand: %s. Use /cot for help.", args[0]) + } +} + +// --- /runtime --------------------------------------------------------------- + +func (r *Reflector) cmdRuntimeStatus(args []string, memory *MemoryStore) string { + if len(args) == 0 { + return "Usage: /runtime [status|processors|commands]" + } + + switch args[0] { + case "status": + r.mu.RLock() + nProc := len(r.processors) + nCmd := len(r.commands) + r.mu.RUnlock() + + var sb strings.Builder + sb.WriteString("โš™๏ธ **Runtime Status**\n") + fmt.Fprintf(&sb, "โ€ข Processors: %d\n", nProc) + fmt.Fprintf(&sb, "โ€ข Commands: %d\n", nCmd) + fmt.Fprintf(&sb, "โ€ข Timeout: %s\n", r.timeout) + if r.model != "" { + fmt.Fprintf(&sb, "โ€ข Model: %s\n", r.model) + } + return sb.String() + + case "processors": + r.mu.RLock() + defer r.mu.RUnlock() + var sb strings.Builder + sb.WriteString("โš™๏ธ **Processors**\n") + for i, p := range r.processors { + fmt.Fprintf(&sb, "โ€ข %d. %s\n", i+1, p.Name()) + } + return sb.String() + + case "commands": + return r.ListCommands() + + default: + return fmt.Sprintf("Unknown: %s. Use /runtime for help.", args[0]) + } +} + +// --- /shell ----------------------------------------------------------------- + +const shellMaxOutput = 4000 + +// shellDenySubstrings blocks injection attempts for dev tool passthrough. +var shellDenySubstrings = []string{ + "| sh", "| bash", "| powershell", "| cmd", + "; rm ", "; del ", "&& rm ", "&& del ", + "$(", "${", "`", + "> /dev/", ">> /dev/", +} + +func (r *Reflector) cmdShell(args []string, _ *MemoryStore) string { + if len(args) == 0 { + return "Usage: /shell [args...]\n" + + " Built-in: ls, cat, head, tail, grep, wc, find, diff, tree, stat, pwd, echo\n" + + " Dev tools (passthrough): go, git, node, python, npm, cargo, make\n" + + " File ops: touch, mkdir, cp, mv" + } + + baseCmd := strings.ToLower(args[0]) + cmdArgs := args[1:] + + // 1. Try built-in Go implementation (cross-platform). + if handler, ok := shell.BuiltinCmds[baseCmd]; ok { + cwd, _ := os.Getwd() + output := handler(cmdArgs, cwd) + return shellFormatOutput(output) + } + + // 2. Try dev tool passthrough via ExecTool. + if shell.DevToolPassthrough[baseCmd] { + // Injection check. + command := strings.Join(args, " ") + cmdLower := strings.ToLower(command) + for _, deny := range shellDenySubstrings { + if strings.Contains(cmdLower, deny) { + return fmt.Sprintf("โŒ Command blocked: restricted pattern '%s'", deny) + } + } + + r.mu.RLock() + registry := r.toolRegistry + r.mu.RUnlock() + + if registry == nil { + return "โš ๏ธ Dev tool passthrough not available (no tool registry)" + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + result := registry.Execute(ctx, "exec", map[string]any{ + "command": command, + }) + + if result.IsError || result.Err != nil { + errMsg := result.ForLLM + if errMsg == "" && result.Err != nil { + errMsg = result.Err.Error() + } + return fmt.Sprintf("โŒ %s", errMsg) + } + return shellFormatOutput(result.ForLLM) + } + + return fmt.Sprintf("โŒ Unknown command '%s'. Use /shell for available commands.", baseCmd) +} + +func shellFormatOutput(output string) string { + if output == "" { + return "โœ… (no output)" + } + if len(output) > shellMaxOutput { + output = output[:shellMaxOutput] + fmt.Sprintf("\n... (truncated, %d chars total)", len(output)) + } + return "```\n" + output + "\n```" +} +// --- /show ------------------------------------------------------------------ + +func (r *Reflector) cmdShow(args []string, _ *MemoryStore) string { + if len(args) < 1 { + return "Usage: /show [model|channel|agents]" + } + + r.mu.RLock() + reg := r.agentRegistry + r.mu.RUnlock() + + switch args[0] { + case "model": + if reg == nil { + return "โš ๏ธ Agent registry not available" + } + agent := reg.GetDefaultAgent() + if agent == nil { + return "No default agent configured" + } + return fmt.Sprintf("Current model: %s", agent.Model) + case "channel": + return "Use /list channels to see enabled channels" + case "agents": + if reg == nil { + return "โš ๏ธ Agent registry not available" + } + ids := reg.ListAgentIDs() + return fmt.Sprintf("Registered agents: %s", strings.Join(ids, ", ")) + default: + return fmt.Sprintf("Unknown show target: %s", args[0]) + } +} + +// --- /list ------------------------------------------------------------------ + +func (r *Reflector) cmdList(args []string, _ *MemoryStore) string { + if len(args) < 1 { + return "Usage: /list [models|channels|agents]" + } + + r.mu.RLock() + reg := r.agentRegistry + cm := r.channelManager + r.mu.RUnlock() + + switch args[0] { + case "models": + return "Available models: configured in config.json per agent" + case "channels": + if cm == nil { + return "Channel manager not initialized" + } + chs := cm.GetEnabledChannels() + if len(chs) == 0 { + return "No channels enabled" + } + return fmt.Sprintf("Enabled channels: %s", strings.Join(chs, ", ")) + case "agents": + if reg == nil { + return "โš ๏ธ Agent registry not available" + } + ids := reg.ListAgentIDs() + return fmt.Sprintf("Registered agents: %s", strings.Join(ids, ", ")) + default: + return fmt.Sprintf("Unknown list target: %s", args[0]) + } +} + +// --- /switch ---------------------------------------------------------------- + +func (r *Reflector) cmdSwitch(args []string, _ *MemoryStore) string { + if len(args) < 3 || args[1] != "to" { + return "Usage: /switch [model|channel] to " + } + + target := args[0] + value := args[2] + + r.mu.RLock() + reg := r.agentRegistry + cm := r.channelManager + r.mu.RUnlock() + + switch target { + case "model": + if reg == nil { + return "โš ๏ธ Agent registry not available" + } + agent := reg.GetDefaultAgent() + if agent == nil { + return "No default agent configured" + } + oldModel := agent.Model + agent.Model = value + return fmt.Sprintf("Switched model from %s to %s", oldModel, value) + case "channel": + if cm == nil { + return "Channel manager not initialized" + } + if _, exists := cm.GetChannel(value); !exists && value != "cli" { + return fmt.Sprintf("Channel '%s' not found or not enabled", value) + } + return fmt.Sprintf("Switched target channel to %s", value) + default: + return fmt.Sprintf("Unknown switch target: %s", target) + } +} + +// =========================================================================== +// Built-in processors (post-LLM, async) +// =========================================================================== + +// --- ErrorTracker (no LLM) -------------------------------------------------- + +type ErrorTracker struct{} + +func (e *ErrorTracker) Name() string { return "error_tracker" } + +func (e *ErrorTracker) Process(_ context.Context, input RuntimeInput, _ *MemoryStore) error { + for _, tc := range input.ToolCalls { + if tc.Error == "" { + continue + } + logger.InfoCF("reflector", "Tool error recorded", + map[string]any{"tool": tc.Name, "error": tc.Error}) + } + return nil +} + +// --- CotEvaluator (LLM) ---------------------------------------------------- + +type CotEvaluator struct { + provider providers.LLMProvider + model string +} + +func (c *CotEvaluator) Name() string { return "cot_evaluator" } + +const cotEvalPrompt = `Rate how well the thinking strategy helped answer the user's question. + +Question: %s +Strategy: %s +Response (first 500 chars): %s + +Respond with ONLY one JSON: {"score": <-1|0|1>} +1 = good, 0 = neutral, -1 = poor` + +func (c *CotEvaluator) Process(ctx context.Context, input RuntimeInput, memory *MemoryStore) error { + if input.CotPrompt == "" { + return nil + } + + reply := input.AssistantReply + if len(reply) > 500 { + reply = reply[:500] + } + + resp, err := c.provider.Chat(ctx, []providers.Message{ + {Role: "user", Content: fmt.Sprintf(cotEvalPrompt, input.UserMessage, input.CotPrompt, reply)}, + }, nil, c.model, map[string]any{"max_tokens": 32, "temperature": 0.1}) + if err != nil { + return fmt.Errorf("eval LLM failed: %w", err) + } + + // Parse JSON (strip markdown fences if present). + raw := strings.TrimSpace(resp.Content) + if strings.HasPrefix(raw, "```") { + lines := strings.Split(raw, "\n") + if len(lines) > 2 { + raw = strings.Join(lines[1:len(lines)-1], "\n") + } + } + var evalResult struct { + Score int `json:"score"` + } + if err := json.Unmarshal([]byte(raw), &evalResult); err != nil { + // Fallback: string matching. + if strings.Contains(raw, `"score": 1`) || strings.Contains(raw, `"score":1`) { + evalResult.Score = 1 + } else if strings.Contains(raw, `"score": -1`) || strings.Contains(raw, `"score":-1`) { + evalResult.Score = -1 + } + } + + if evalResult.Score != 0 { + if err := memory.UpdateLatestCotFeedback(evalResult.Score); err != nil { + return err + } + logger.InfoCF("reflector", "CoT feedback auto-recorded", + map[string]any{"score": evalResult.Score, "intent": input.Intent}) + } + return nil +} + +// --- MemoryExtractor (LLM) -------------------------------------------------- + +type MemoryExtractor struct { + provider providers.LLMProvider + model string +} + +func (m *MemoryExtractor) Name() string { return "memory_extractor" } + +const memoryExtractPrompt = `Extract important facts worth remembering from this conversation. + +User: %s +Assistant (first 800 chars): %s + +Respond with ONLY JSON: {"memories": [{"content": "", "tags": ["tag1"]}]} +Rules: max 3 memories, max 3 tags each, lowercase tags, skip trivial chat. +If nothing worth remembering: {"memories": []}` + +type memExtractResult struct { + Memories []struct { + Content string `json:"content"` + Tags []string `json:"tags"` + } `json:"memories"` +} + +func (m *MemoryExtractor) Process(ctx context.Context, input RuntimeInput, memory *MemoryStore) error { + if len(input.UserMessage) < 20 || input.Intent == "chat" { + return nil + } + + reply := input.AssistantReply + if len(reply) > 800 { + reply = reply[:800] + } + + resp, err := m.provider.Chat(ctx, []providers.Message{ + {Role: "user", Content: fmt.Sprintf(memoryExtractPrompt, input.UserMessage, reply)}, + }, nil, m.model, map[string]any{"max_tokens": 256, "temperature": 0.1}) + if err != nil { + return fmt.Errorf("memory extract LLM failed: %w", err) + } + + // Parse JSON (strip markdown fences if present). + raw := strings.TrimSpace(resp.Content) + if strings.HasPrefix(raw, "```") { + lines := strings.Split(raw, "\n") + if len(lines) > 2 { + raw = strings.Join(lines[1:len(lines)-1], "\n") + } + } + + var result memExtractResult + if err := json.Unmarshal([]byte(raw), &result); err != nil { + return nil // Parsing failed โ€” skip silently. + } + + for _, mem := range result.Memories { + content := strings.TrimSpace(mem.Content) + if content == "" { + continue + } + tags := make([]string, 0, len(mem.Tags)) + for _, t := range mem.Tags { + t = strings.ToLower(strings.TrimSpace(t)) + if t != "" { + tags = append(tags, t) + } + } + if id, err := memory.AddEntry(content, tags); err != nil { + logger.WarnCF("reflector", "Failed to save memory", + map[string]any{"error": err.Error()}) + } else { + logger.InfoCF("reflector", "Memory extracted", + map[string]any{"id": id, "tags": tags, "content": content}) + } + } + return nil +} diff --git a/pkg/agent/reflector_test.go b/pkg/agent/reflector_test.go new file mode 100644 index 000000000..8d79a33be --- /dev/null +++ b/pkg/agent/reflector_test.go @@ -0,0 +1,319 @@ +package agent + +import ( + "os" + "strings" + "testing" +) + +// --- Slash command tests ---------------------------------------------------- + +func TestRuntime_MemoryCommand(t *testing.T) { + dir := t.TempDir() + ms := NewMemoryStore(dir) + defer ms.Close() + r := NewReflector(nil, "") + + // /memory with no args โ†’ help. + resp, ok := r.HandleCommand("/memory", ms) + if !ok { + t.Fatal("expected /memory to be handled") + } + if !strings.Contains(resp, "Usage") { + t.Error("expected usage text") + } + + // /memory list โ†’ empty. + resp, ok = r.HandleCommand("/memory list", ms) + if !ok { + t.Fatal("expected /memory list to be handled") + } + if !strings.Contains(resp, "No memories") { + t.Errorf("expected empty list, got %q", resp) + } + + // /memory add. + resp, ok = r.HandleCommand("/memory add Go is great for concurrency #golang #concurrency", ms) + if !ok { + t.Fatal("expected /memory add to be handled") + } + if !strings.Contains(resp, "โœ…") { + t.Errorf("expected success, got %q", resp) + } + if !strings.Contains(resp, "golang") { + t.Errorf("should show tags, got %q", resp) + } + + // /memory list โ†’ should have 1 entry. + resp, _ = r.HandleCommand("/memory list", ms) + if !strings.Contains(resp, "Go is great") { + t.Errorf("should show entry, got %q", resp) + } + + // /memory search. + resp, _ = r.HandleCommand("/memory search golang", ms) + if !strings.Contains(resp, "Found 1") { + t.Errorf("expected 1 result, got %q", resp) + } + + resp, _ = r.HandleCommand("/memory search nonexistent", ms) + if !strings.Contains(resp, "No memories found") { + t.Errorf("expected no results, got %q", resp) + } + + // /memory stats โ€” should show entry count. + resp, _ = r.HandleCommand("/memory stats", ms) + if !strings.Contains(resp, "Stats") { + t.Errorf("expected stats, got %q", resp) + } + if !strings.Contains(resp, "Total entries: 1") { + t.Errorf("expected 1 entry in stats, got %q", resp) + } + + // /memory edit. + resp, _ = r.HandleCommand("/memory edit 1 Updated content #go", ms) + if !strings.Contains(resp, "โœ…") { + t.Errorf("expected success, got %q", resp) + } + resp, _ = r.HandleCommand("/memory list", ms) + if !strings.Contains(resp, "Updated content") { + t.Errorf("edit should be reflected, got %q", resp) + } + + // /memory delete. + resp, _ = r.HandleCommand("/memory delete 1", ms) + if !strings.Contains(resp, "โœ…") { + t.Errorf("expected success, got %q", resp) + } + resp, _ = r.HandleCommand("/memory list", ms) + if !strings.Contains(resp, "No memories") { + t.Errorf("expected empty after delete, got %q", resp) + } +} + +func TestRuntime_HelpCommand(t *testing.T) { + r := NewReflector(nil, "") + dir := t.TempDir() + ms := NewMemoryStore(dir) + defer ms.Close() + + resp, ok := r.HandleCommand("/help", ms) + if !ok { + t.Fatal("expected /help to be handled") + } + if !strings.Contains(resp, "/memory") { + t.Error("help should list /memory") + } + if !strings.Contains(resp, "/cot") { + t.Error("help should list /cot") + } + if !strings.Contains(resp, "/show") { + t.Error("help should list /show (now a runtime command)") + } + if !strings.Contains(resp, "/shell") { + t.Error("help should list /shell") + } +} + +func TestRuntime_ShellSecurity(t *testing.T) { + r := NewReflector(nil, "") + dir := t.TempDir() + ms := NewMemoryStore(dir) + defer ms.Close() + + // Unknown command (not builtin or dev tool). + resp, _ := r.HandleCommand("/shell rm -rf /", ms) + if !strings.Contains(resp, "Unknown command") { + t.Errorf("rm should be unknown, got %q", resp) + } + + // Unknown: sudo + resp, _ = r.HandleCommand("/shell sudo ls", ms) + if !strings.Contains(resp, "Unknown command") { + t.Errorf("sudo should be unknown, got %q", resp) + } + + // Injection via passthrough: git | bash + resp, _ = r.HandleCommand("/shell git log | bash", ms) + if !strings.Contains(resp, "blocked") { + t.Errorf("injection should be blocked, got %q", resp) + } + + // Builtin echo works (cross-platform). + resp, _ = r.HandleCommand("/shell echo hello world", ms) + if !strings.Contains(resp, "hello world") { + t.Errorf("echo should work, got %q", resp) + } +} + +func TestRuntime_CotCommand(t *testing.T) { + dir := t.TempDir() + ms := NewMemoryStore(dir) + defer ms.Close() + r := NewReflector(nil, "") + + // /cot with no args โ†’ help. + resp, ok := r.HandleCommand("/cot", ms) + if !ok { + t.Fatal("expected /cot to be handled") + } + if !strings.Contains(resp, "Usage") { + t.Error("expected usage text") + } + + // /cot stats โ†’ empty. + resp, _ = r.HandleCommand("/cot stats", ms) + if !strings.Contains(resp, "No CoT usage") { + t.Errorf("expected empty, got %q", resp) + } + + // Add some usage first. + ms.RecordCotUsage("code", []string{"golang"}, "1. Think\n2. Code", "write code") + + // /cot history. + resp, _ = r.HandleCommand("/cot history", ms) + if !strings.Contains(resp, "code") { + t.Errorf("expected history entry, got %q", resp) + } + + // /cot feedback. + resp, _ = r.HandleCommand("/cot feedback 1", ms) + if !strings.Contains(resp, "โœ…") { + t.Errorf("expected success, got %q", resp) + } + + // /cot feedback bad input. + resp, _ = r.HandleCommand("/cot feedback 99", ms) + if !strings.Contains(resp, "โŒ") { + t.Errorf("expected error, got %q", resp) + } +} + +func TestRuntime_RuntimeCommand(t *testing.T) { + r := NewReflector(nil, "") + dir := t.TempDir() + ms := NewMemoryStore(dir) + defer ms.Close() + + resp, ok := r.HandleCommand("/runtime status", ms) + if !ok { + t.Fatal("expected /runtime to be handled") + } + if !strings.Contains(resp, "Processors") { + t.Errorf("expected status, got %q", resp) + } + + resp, _ = r.HandleCommand("/runtime processors", ms) + if !strings.Contains(resp, "error_tracker") { + t.Errorf("expected error_tracker processor, got %q", resp) + } +} + +func TestRuntime_UnknownCommand(t *testing.T) { + r := NewReflector(nil, "") + + // Unknown /cmd โ†’ not handled (returns false). + _, ok := r.HandleCommand("/unknown_cmd", nil) + if ok { + t.Error("expected unknown command to not be handled") + } + + // Not a command at all. + _, ok = r.HandleCommand("hello world", nil) + if ok { + t.Error("expected non-command to not be handled") + } +} + +func TestRuntime_ShellCommand(t *testing.T) { + r := NewReflector(nil, "") + dir := t.TempDir() + ms := NewMemoryStore(dir) + defer ms.Close() + + // /shell with no args โ†’ help. + resp, ok := r.HandleCommand("/shell", ms) + if !ok { + t.Fatal("expected /shell to be handled") + } + if !strings.Contains(resp, "Usage") { + t.Errorf("expected usage, got %q", resp) + } + + // /shell pwd โ†’ returns cwd (builtin, no tool registry needed). + resp, _ = r.HandleCommand("/shell pwd", ms) + if !strings.Contains(resp, string(os.PathSeparator)) { + t.Errorf("expected directory path, got %q", resp) + } + + // /shell dev tool without registry โ†’ warning. + resp, _ = r.HandleCommand("/shell git status", ms) + if !strings.Contains(resp, "not available") { + t.Errorf("expected warning about no registry, got %q", resp) + } +} + +// --- Post-LLM processor tests ----------------------------------------------- + +func TestRuntime_ErrorTracker(t *testing.T) { + tracker := &ErrorTracker{} + input := RuntimeInput{ + ToolCalls: []ToolCallRecord{ + {Name: "exec", Error: "command not found"}, + {Name: "read_file", Error: ""}, + }, + } + + // Should not error. + err := tracker.Process(nil, input, nil) + if err != nil { + t.Errorf("unexpected error: %v", err) + } +} + +func TestRuntime_CotEvaluator_NoCot(t *testing.T) { + eval := &CotEvaluator{} + input := RuntimeInput{CotPrompt: ""} // No CoT โ†’ skip. + + err := eval.Process(nil, input, nil) + if err != nil { + t.Errorf("unexpected error: %v", err) + } +} + +func TestRuntime_MemoryExtractor_SkipChat(t *testing.T) { + extractor := &MemoryExtractor{} + input := RuntimeInput{ + UserMessage: "hello", + Intent: "chat", + } + + err := extractor.Process(nil, input, nil) + if err != nil { + t.Errorf("unexpected error: %v", err) + } +} + +func TestRuntime_PostLLM_NilSafety(t *testing.T) { + // Nil runtime should not panic. + var r *Reflector + r.RunPostLLM(RuntimeInput{}, nil) // Should be no-op. + + // Runtime with no processors. + r = &Reflector{commands: map[string]CommandDef{}} + r.RunPostLLM(RuntimeInput{}, nil) // Should be no-op. +} + +func TestRuntime_ListCommands(t *testing.T) { + r := NewReflector(nil, "") + text := r.ListCommands() + if !strings.Contains(text, "/memory") { + t.Error("should list /memory command") + } + if !strings.Contains(text, "/cot") { + t.Error("should list /cot command") + } + if !strings.Contains(text, "/runtime") { + t.Error("should list /runtime command") + } +} diff --git a/pkg/agent/score.go b/pkg/agent/score.go new file mode 100644 index 000000000..f153dd878 --- /dev/null +++ b/pkg/agent/score.go @@ -0,0 +1,74 @@ +// PicoClaw - Ultra-lightweight personal AI agent +// License: MIT +// +// Copyright (c) 2026 PicoClaw contributors + +package agent + +import "strings" + +// CalcTurnScore computes a value score for a completed turn. +// +// Scoring rules (range roughly -2 to 15): +// +// +3 has tool calls +// +2 has write/edit/append tool call (modifying tools) +// +2 tool count > 3 +// +3 intent = task / code / debug +// +1 intent = question +// +0 intent = chat (or empty) +// +2 reply length > 500 chars +// -2 user + reply total < 80 chars +// +3 user message contains "่ฎฐไฝ" or "้‡่ฆ" (remember / important) +// +// alwaysKeepThreshold (โ‰ฅ 7) marks a Turn as always_keep in instant memory. +func CalcTurnScore(input RuntimeInput) int { + score := 0 + + // --- Tool activity --- + if len(input.ToolCalls) > 0 { + score += 3 + } + for _, tc := range input.ToolCalls { + n := strings.ToLower(tc.Name) + if n == "write_file" || n == "edit_file" || n == "append_file" || + n == "write" || n == "edit" || n == "append" { + score += 2 + break // count once + } + } + if len(input.ToolCalls) > 3 { + score += 2 + } + + // --- Intent weight --- + switch strings.ToLower(input.Intent) { + case "task", "code", "debug": + score += 3 + case "question": + score += 1 + // "chat" or empty: 0 + } + + // --- Content density --- + if len(input.AssistantReply) > 500 { + score += 2 + } + if len(input.UserMessage)+len(input.AssistantReply) < 80 { + score -= 2 + } + + // --- Explicit importance markers --- + if strings.Contains(input.UserMessage, "่ฎฐไฝ") || + strings.Contains(input.UserMessage, "้‡่ฆ") || + strings.Contains(strings.ToLower(input.UserMessage), "remember") || + strings.Contains(strings.ToLower(input.UserMessage), "important") { + score += 3 + } + + return score +} + +// alwaysKeepThreshold is the minimum score for a Turn to be unconditionally +// included in instant memory (regardless of tag matching). +const alwaysKeepThreshold = 7 diff --git a/pkg/agent/score_test.go b/pkg/agent/score_test.go new file mode 100644 index 000000000..09b599b2f --- /dev/null +++ b/pkg/agent/score_test.go @@ -0,0 +1,143 @@ +package agent + +import ( + "strings" + "testing" +) + +func TestCalcTurnScore_BasicRules(t *testing.T) { + tests := []struct { + name string + input RuntimeInput + wantMin int + wantMax int + wantExact *int + }{ + { + name: "empty chat", + input: RuntimeInput{Intent: "chat", UserMessage: "ok", AssistantReply: "ok"}, + // score = 0 (chat) -2 (< 80 chars total) = -2 + wantExact: intPtr(-2), + }, + { + name: "question intent, short", + input: RuntimeInput{Intent: "question", UserMessage: "hi", AssistantReply: "hello"}, + // score = 1 (question) -2 (short) = -1 + wantExact: intPtr(-1), + }, + { + name: "task with tool call", + input: RuntimeInput{ + Intent: "task", + UserMessage: "do something important", + AssistantReply: "done", + ToolCalls: []ToolCallRecord{{Name: "exec"}}, + }, + // +3 (task) +3 (has tool) +3 ("important" keyword) -2 (short) = 7 + wantExact: intPtr(7), + }, + { + name: "code with write tool", + input: RuntimeInput{ + Intent: "code", + UserMessage: "fix the bug", + AssistantReply: "fixed", + ToolCalls: []ToolCallRecord{{Name: "write_file"}}, + }, + // +3 (code) +3 (has tool) +2 (write tool) -2 (short) = 6 + wantExact: intPtr(6), + }, + { + name: "many tools", + input: RuntimeInput{ + Intent: "debug", + UserMessage: "debug it", + AssistantReply: "ok", + ToolCalls: []ToolCallRecord{ + {Name: "exec"}, + {Name: "read_file"}, + {Name: "list_dir"}, + {Name: "exec"}, + }, + }, + // +3 (debug) +3 (has tool) +2 (>3 tools) -2 (short) = 6 + wantExact: intPtr(6), + }, + { + name: "long reply", + input: RuntimeInput{ + Intent: "question", + UserMessage: "explain", + AssistantReply: strings.Repeat("a", 600), + }, + // +1 (question) +2 (long reply) [total<80 does not apply because reply is 600] + // total chars = 7 + 600 = 607 >= 80 + wantExact: intPtr(3), + }, + { + name: "explicit remember keyword", + input: RuntimeInput{ + Intent: "chat", + UserMessage: "่ฎฐไฝ่ฟ™ไธชๅœฐๅ€ localhost:3000", + AssistantReply: strings.Repeat("a", 600), + }, + // 0(chat) +3 (่ฎฐไฝ) +2 (long reply) = 5 + wantExact: intPtr(5), + }, + { + name: "explicit important keyword", + input: RuntimeInput{ + Intent: "question", + UserMessage: "this is IMPORTANT: use port 8080", + AssistantReply: "ok", + }, + // 1 (question) + 3 (important) - 2 (short) = 2 + wantExact: intPtr(2), + }, + { + name: "always_keep threshold: full scoring", + input: RuntimeInput{ + Intent: "task", + UserMessage: "run the deployment pipeline for staging and fix it", + AssistantReply: strings.Repeat("a", 600), + ToolCalls: []ToolCallRecord{ + {Name: "edit_file"}, + {Name: "exec"}, + {Name: "exec"}, + {Name: "exec"}, + }, + }, + // +3(task) +3(tool) +2(write/edit) +2(>3 tools) +2(long reply) = 12 + wantExact: intPtr(12), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := CalcTurnScore(tc.input) + if tc.wantExact != nil { + if got != *tc.wantExact { + t.Errorf("CalcTurnScore() = %d, want %d", got, *tc.wantExact) + } + } else if got < tc.wantMin || (tc.wantMax > 0 && got > tc.wantMax) { + t.Errorf("CalcTurnScore() = %d, want [%d, %d]", got, tc.wantMin, tc.wantMax) + } + }) + } +} + +func TestAlwaysKeepThreshold(t *testing.T) { + // High-value turn must meet or exceed the threshold. + highValue := RuntimeInput{ + Intent: "task", + UserMessage: "deploy staging", + AssistantReply: strings.Repeat("a", 600), + ToolCalls: []ToolCallRecord{{Name: "edit_file"}, {Name: "exec"}}, + } + score := CalcTurnScore(highValue) + if score < alwaysKeepThreshold { + t.Errorf("expected score %d >= alwaysKeepThreshold %d", score, alwaysKeepThreshold) + } +} + +func intPtr(i int) *int { return &i } diff --git a/pkg/agent/turn_store.go b/pkg/agent/turn_store.go new file mode 100644 index 000000000..f1cb290ad --- /dev/null +++ b/pkg/agent/turn_store.go @@ -0,0 +1,300 @@ +// PicoClaw - Ultra-lightweight personal AI agent +// License: MIT +// +// Copyright (c) 2026 PicoClaw contributors + +package agent + +import ( + "database/sql" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/sipeed/picoclaw/pkg/logger" + + _ "modernc.org/sqlite" +) + +// TurnRecord captures everything that happened during a single completed turn. +// It is persisted to turns.db for use by MemoryDigest and instant-memory assembly. +type TurnRecord struct { + ID string // ULID or time-based unique ID + Ts int64 // Unix timestamp (seconds) + ChannelKey string // "channel:chatID" + Score int // Phase 3 CalcTurnScore result + Intent string // Phase 1 detected intent + Tags []string // Phase 1 detected tags + Tokens int // rough token estimate (chars / 3) + Status string // "pending" | "processed" | "archived" + UserMsg string // original user message + Reply string // assistant final response + ToolCalls []ToolCallRecord // serialised as JSON in DB +} + +// TurnStore manages persistent Turn storage in SQLite. +// The DB lives at {workspace}/turns.db, mirroring the memory.db pattern. +type TurnStore struct { + db *sql.DB +} + +const turnsDDL = ` +CREATE TABLE IF NOT EXISTS turns ( + id TEXT PRIMARY KEY, + ts INTEGER NOT NULL, + channel_key TEXT NOT NULL DEFAULT '', + score INTEGER NOT NULL DEFAULT 0, + intent TEXT NOT NULL DEFAULT '', + tags TEXT NOT NULL DEFAULT '[]', + tokens INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'pending', + user_msg TEXT NOT NULL DEFAULT '', + reply TEXT NOT NULL DEFAULT '', + tool_calls TEXT NOT NULL DEFAULT '[]' +); +CREATE INDEX IF NOT EXISTS idx_turns_status ON turns(status); +CREATE INDEX IF NOT EXISTS idx_turns_ts ON turns(ts); +CREATE INDEX IF NOT EXISTS idx_turns_channel ON turns(channel_key); +CREATE INDEX IF NOT EXISTS idx_turns_score ON turns(score); +` + +// NewTurnStore creates (or opens) turns.db in the given workspace directory. +func NewTurnStore(workspace string) (*TurnStore, error) { + if err := os.MkdirAll(workspace, 0o755); err != nil { + return nil, fmt.Errorf("turn_store: mkdir %s: %w", workspace, err) + } + dbPath := filepath.Join(workspace, "turns.db") + db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(wal)&_pragma=busy_timeout(5000)") + if err != nil { + return nil, fmt.Errorf("turn_store: open %s: %w", dbPath, err) + } + if _, err := db.Exec(turnsDDL); err != nil { + db.Close() + return nil, fmt.Errorf("turn_store: init schema: %w", err) + } + return &TurnStore{db: db}, nil +} + +// Close shuts down the underlying DB connection. +func (s *TurnStore) Close() error { + if s.db != nil { + return s.db.Close() + } + return nil +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +func marshalJSON(v any) string { + b, err := json.Marshal(v) + if err != nil { + return "[]" + } + return string(b) +} + +func unmarshalTags(raw string) []string { + var tags []string + _ = json.Unmarshal([]byte(raw), &tags) + return tags +} + +func unmarshalToolCalls(raw string) []ToolCallRecord { + var tcs []ToolCallRecord + _ = json.Unmarshal([]byte(raw), &tcs) + return tcs +} + +// estimateTokens gives a cheap estimate: characters / 3. +func estimateTokens(r TurnRecord) int { + chars := len(r.UserMsg) + len(r.Reply) + for _, tc := range r.ToolCalls { + chars += len(tc.Name) + len(tc.Error) + } + if chars < 3 { + return 1 + } + return chars / 3 +} + +// NewTurnID generates a time-sortable unique ID without external dependencies. +// Format: unixMilli-randomSuffix using millisecond precision. +func NewTurnID() string { + return fmt.Sprintf("%d-%d", time.Now().UnixMilli(), time.Now().Nanosecond()%1_000_000) +} + +// --------------------------------------------------------------------------- +// Writes +// --------------------------------------------------------------------------- + +// Insert persists a TurnRecord to the DB. +// The record's ID and Ts are set if empty/zero. +func (s *TurnStore) Insert(r TurnRecord) error { + if r.ID == "" { + r.ID = NewTurnID() + } + if r.Ts == 0 { + r.Ts = time.Now().Unix() + } + if r.Status == "" { + r.Status = "pending" + } + if r.Tokens == 0 { + r.Tokens = estimateTokens(r) + } + + tagsJSON := marshalJSON(r.Tags) + tcJSON := marshalJSON(r.ToolCalls) + + _, err := s.db.Exec(` + INSERT INTO turns (id, ts, channel_key, score, intent, tags, tokens, status, user_msg, reply, tool_calls) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO NOTHING`, + r.ID, r.Ts, r.ChannelKey, r.Score, r.Intent, + tagsJSON, r.Tokens, r.Status, r.UserMsg, r.Reply, tcJSON, + ) + if err != nil { + return fmt.Errorf("turn_store: insert %s: %w", r.ID, err) + } + + logger.DebugCF("turn_store", "Turn inserted", + map[string]any{"id": r.ID, "score": r.Score, "tokens": r.Tokens, "status": r.Status}) + return nil +} + +// SetStatus updates the status of a turn by ID. +func (s *TurnStore) SetStatus(id, status string) error { + _, err := s.db.Exec("UPDATE turns SET status = ? WHERE id = ?", status, id) + return err +} + +// --------------------------------------------------------------------------- +// Queries โ€” used by MemoryDigest and instant-memory assembly +// --------------------------------------------------------------------------- + +// QueryPending returns up to limit turns with status = 'pending', ordered oldest first. +func (s *TurnStore) QueryPending(limit int) ([]TurnRecord, error) { + rows, err := s.db.Query(` + SELECT id, ts, channel_key, score, intent, tags, tokens, status, user_msg, reply, tool_calls + FROM turns WHERE status = 'pending' + ORDER BY ts ASC LIMIT ?`, limit) + if err != nil { + return nil, err + } + defer rows.Close() + return scanTurns(rows) +} + +// QueryByScore returns all turns with score >= highThreshold (always_keep), +// ordered by ts ASC. +func (s *TurnStore) QueryByScore(highThreshold int) ([]TurnRecord, error) { + rows, err := s.db.Query(` + SELECT id, ts, channel_key, score, intent, tags, tokens, status, user_msg, reply, tool_calls + FROM turns WHERE score >= ? AND status != 'archived' + ORDER BY ts ASC`, highThreshold) + if err != nil { + return nil, err + } + defer rows.Close() + return scanTurns(rows) +} + +// QueryByTags returns turns whose tags JSON contains at least one of the given tags +// and score > 0, ordered by ts ASC. +func (s *TurnStore) QueryByTags(tags []string) ([]TurnRecord, error) { + if len(tags) == 0 { + return nil, nil + } + // Build LIKE conditions for simple JSON array matching. + conds := make([]string, 0, len(tags)) + args := make([]any, 0, len(tags)*2) + for _, t := range tags { + t = strings.ToLower(strings.TrimSpace(t)) + if t == "" { + continue + } + conds = append(conds, `(tags LIKE ? OR tags LIKE ?)`) + args = append(args, `%"`+t+`"%`, `%'`+t+`'%`) + } + if len(conds) == 0 { + return nil, nil + } + // Append non-archived filter. + query := fmt.Sprintf(` + SELECT id, ts, channel_key, score, intent, tags, tokens, status, user_msg, reply, tool_calls + FROM turns + WHERE score > 0 AND status != 'archived' AND (%s) + ORDER BY ts ASC`, strings.Join(conds, " OR ")) + + rows, err := s.db.Query(query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanTurns(rows) +} + +// QueryRecent returns the n most-recent non-archived turns for a channelKey, +// ordered by ts ASC (oldest first, so they can be appended naturally). +func (s *TurnStore) QueryRecent(channelKey string, n int) ([]TurnRecord, error) { + rows, err := s.db.Query(` + SELECT id, ts, channel_key, score, intent, tags, tokens, status, user_msg, reply, tool_calls + FROM turns + WHERE channel_key = ? AND status != 'archived' + ORDER BY ts DESC LIMIT ?`, channelKey, n) + if err != nil { + return nil, err + } + defer rows.Close() + turns, err := scanTurns(rows) + if err != nil { + return nil, err + } + // Reverse to ascending order. + for i, j := 0, len(turns)-1; i < j; i, j = i+1, j-1 { + turns[i], turns[j] = turns[j], turns[i] + } + return turns, nil +} + +// ArchiveOldProcessed marks processed turns older than olderThanDays as 'archived'. +// At most 100 rows are archived per call to limit lock time. +func (s *TurnStore) ArchiveOldProcessed(olderThanDays int) error { + cutoff := time.Now().AddDate(0, 0, -olderThanDays).Unix() + _, err := s.db.Exec(` + UPDATE turns SET status = 'archived' + WHERE id IN ( + SELECT id FROM turns + WHERE status = 'processed' AND ts < ? + ORDER BY ts ASC LIMIT 100 + )`, cutoff) + return err +} + +// --------------------------------------------------------------------------- +// Internal scanner +// --------------------------------------------------------------------------- + +func scanTurns(rows *sql.Rows) ([]TurnRecord, error) { + var out []TurnRecord + for rows.Next() { + var r TurnRecord + var tagsJSON, tcJSON string + if err := rows.Scan( + &r.ID, &r.Ts, &r.ChannelKey, &r.Score, &r.Intent, + &tagsJSON, &r.Tokens, &r.Status, + &r.UserMsg, &r.Reply, &tcJSON, + ); err != nil { + return out, err + } + r.Tags = unmarshalTags(tagsJSON) + r.ToolCalls = unmarshalToolCalls(tcJSON) + out = append(out, r) + } + return out, rows.Err() +} diff --git a/pkg/agent/turn_store_test.go b/pkg/agent/turn_store_test.go new file mode 100644 index 000000000..bba19de14 --- /dev/null +++ b/pkg/agent/turn_store_test.go @@ -0,0 +1,143 @@ +package agent + +import ( + "testing" + "time" +) + +func TestTurnStore_InsertAndQueryRecent(t *testing.T) { + dir := t.TempDir() + store, err := NewTurnStore(dir) + if err != nil { + t.Fatalf("NewTurnStore: %v", err) + } + defer store.Close() + + r := TurnRecord{ + Ts: time.Now().Unix(), + ChannelKey: "cli:direct", + Score: 5, + Intent: "task", + Tags: []string{"deploy", "ci"}, + UserMsg: "deploy now", + Reply: "done", + ToolCalls: []ToolCallRecord{{Name: "exec", Error: ""}}, + Status: "pending", + } + + if err := store.Insert(r); err != nil { + t.Fatalf("Insert: %v", err) + } + + rows, err := store.QueryRecent("cli:direct", 10) + if err != nil { + t.Fatalf("QueryRecent: %v", err) + } + if len(rows) != 1 { + t.Fatalf("expected 1 row, got %d", len(rows)) + } + if rows[0].Intent != "task" { + t.Errorf("unexpected intent: %s", rows[0].Intent) + } + if len(rows[0].Tags) != 2 { + t.Errorf("expected 2 tags, got %v", rows[0].Tags) + } +} + +func TestTurnStore_QueryByScore(t *testing.T) { + dir := t.TempDir() + store, err := NewTurnStore(dir) + if err != nil { + t.Fatalf("NewTurnStore: %v", err) + } + defer store.Close() + + now := time.Now().Unix() + store.Insert(TurnRecord{ID: "s-1", Ts: now, Score: 3, UserMsg: "a", Reply: "b", Status: "pending"}) + store.Insert(TurnRecord{ID: "s-2", Ts: now + 1, Score: 8, UserMsg: "c", Reply: "d", Status: "pending"}) + store.Insert(TurnRecord{ID: "s-3", Ts: now + 2, Score: 9, UserMsg: "e", Reply: "f", Status: "pending"}) + + high, err := store.QueryByScore(7) + if err != nil { + t.Fatalf("QueryByScore: %v", err) + } + if len(high) != 2 { + t.Errorf("expected 2 always_keep turns, got %d", len(high)) + } +} + +func TestTurnStore_SetStatus(t *testing.T) { + dir := t.TempDir() + store, err := NewTurnStore(dir) + if err != nil { + t.Fatalf("NewTurnStore: %v", err) + } + defer store.Close() + + r := TurnRecord{ID: "test-id-1", Ts: time.Now().Unix(), UserMsg: "x", Reply: "y", Status: "pending"} + store.Insert(r) + + if err := store.SetStatus("test-id-1", "processed"); err != nil { + t.Fatalf("SetStatus: %v", err) + } + + pending, err := store.QueryPending(10) + if err != nil { + t.Fatalf("QueryPending: %v", err) + } + if len(pending) != 0 { + t.Errorf("expected 0 pending, got %d", len(pending)) + } +} + +func TestTurnStore_ArchiveOldProcessed(t *testing.T) { + dir := t.TempDir() + store, err := NewTurnStore(dir) + if err != nil { + t.Fatalf("NewTurnStore: %v", err) + } + defer store.Close() + + // Insert old processed turns (timestamp in the past). + old := time.Now().AddDate(0, 0, -10).Unix() + for i := 0; i < 3; i++ { + r := TurnRecord{Ts: old, Score: 2, UserMsg: "old", Reply: "msg", Status: "processed"} + store.Insert(r) + } + + // Recent processed turn โ€” should NOT be archived. + recent := TurnRecord{Ts: time.Now().Unix(), Score: 2, UserMsg: "new", Reply: "msg", Status: "processed"} + store.Insert(recent) + + if err := store.ArchiveOldProcessed(7); err != nil { + t.Fatalf("ArchiveOldProcessed: %v", err) + } + + // Query pending (should still be 0). + pending, _ := store.QueryPending(100) + if len(pending) != 0 { + t.Errorf("expected 0 pending after archive, got %d", len(pending)) + } +} + +func TestTurnStore_QueryByTags(t *testing.T) { + dir := t.TempDir() + store, err := NewTurnStore(dir) + if err != nil { + t.Fatalf("NewTurnStore: %v", err) + } + defer store.Close() + + now := time.Now().Unix() + store.Insert(TurnRecord{ID: "tag-1", Ts: now, Score: 5, Tags: []string{"deploy", "ci"}, UserMsg: "a", Reply: "b"}) + store.Insert(TurnRecord{ID: "tag-2", Ts: now + 1, Score: 4, Tags: []string{"file", "read"}, UserMsg: "c", Reply: "d"}) + store.Insert(TurnRecord{ID: "tag-3", Ts: now + 2, Score: 3, Tags: []string{"deploy", "log"}, UserMsg: "e", Reply: "f"}) + + rows, err := store.QueryByTags([]string{"deploy"}) + if err != nil { + t.Fatalf("QueryByTags: %v", err) + } + if len(rows) < 2 { + t.Errorf("expected at least 2 deploy turns, got %d", len(rows)) + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 305ae67e3..8e7b00171 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -51,13 +51,20 @@ type Config struct { Agents AgentsConfig `json:"agents"` Bindings []AgentBinding `json:"bindings,omitempty"` Session SessionConfig `json:"session,omitempty"` - Channels ChannelsConfig `json:"channels"` + Channels ChannelsConfig `json:"channels,omitempty"` Providers ProvidersConfig `json:"providers,omitempty"` - ModelList []ModelConfig `json:"model_list"` // New model-centric provider configuration - Gateway GatewayConfig `json:"gateway"` - Tools ToolsConfig `json:"tools"` - Heartbeat HeartbeatConfig `json:"heartbeat"` - Devices DevicesConfig `json:"devices"` + ModelList []ModelConfig `json:"model_list,omitempty"` + Gateway GatewayConfig `json:"gateway,omitempty"` + Tools ToolsConfig `json:"tools,omitempty"` + Heartbeat HeartbeatConfig `json:"heartbeat,omitempty"` + Devices DevicesConfig `json:"devices,omitempty"` + Logging LoggingConfig `json:"logging,omitempty"` +} + +// LoggingConfig controls log output. +type LoggingConfig struct { + Level string `json:"level,omitempty"` // debug, info, warn, error (default: warn) + FileDir string `json:"file_dir,omitempty"` // directory for log files; empty = no file logging } // MarshalJSON implements custom JSON marshaling for Config @@ -175,6 +182,16 @@ type AgentDefaults struct { ModelName string `json:"model_name,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_MODEL_NAME"` Model string `json:"model" env:"PICOCLAW_AGENTS_DEFAULTS_MODEL"` // Deprecated: use model_name instead ModelFallbacks []string `json:"model_fallbacks,omitempty"` + + // Phase 1 โ€” Analyser: lightweight model for intent/tag analysis + CoT strategy. + // Falls back to main model_name if empty. Use a cheap/fast model here. + AnalyserModel string `json:"analyser_model,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_ANALYSER_MODEL"` + PreLLMModel string `json:"pre_llm_model,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_PRE_LLM_MODEL"` // Deprecated: use analyser_model + + // Phase 3 โ€” Digest: lightweight model for memory extraction from turn records. + // Falls back to main model_name if empty. Use a cheap/fast model here. + DigestModel string `json:"digest_model,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_DIGEST_MODEL"` + ImageModel string `json:"image_model,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_IMAGE_MODEL"` ImageModelFallbacks []string `json:"image_model_fallbacks,omitempty"` MaxTokens int `json:"max_tokens" env:"PICOCLAW_AGENTS_DEFAULTS_MAX_TOKENS"` @@ -191,6 +208,27 @@ func (d *AgentDefaults) GetModelName() string { return d.Model } +// GetAnalyserModel returns the model for Phase 1 (Analyser). +// Priority: analyser_model โ†’ pre_llm_model (deprecated) โ†’ main model. +func (d *AgentDefaults) GetAnalyserModel() string { + if d.AnalyserModel != "" { + return d.AnalyserModel + } + if d.PreLLMModel != "" { + return d.PreLLMModel + } + return d.GetModelName() +} + +// GetDigestModel returns the model for Phase 3 (MemoryDigest). +// Priority: digest_model โ†’ main model. +func (d *AgentDefaults) GetDigestModel() string { + if d.DigestModel != "" { + return d.DigestModel + } + return d.GetModelName() +} + type ChannelsConfig struct { WhatsApp WhatsAppConfig `json:"whatsapp"` Telegram TelegramConfig `json:"telegram"` diff --git a/pkg/constants/channels.go b/pkg/constants/channels.go index 0a46e6cd9..4e635d44d 100644 --- a/pkg/constants/channels.go +++ b/pkg/constants/channels.go @@ -1,16 +1,27 @@ // Package constants provides shared constants across the codebase. package constants +import "strings" + // internalChannels defines channels that are used for internal communication // and should not be exposed to external users or recorded as last active channel. var internalChannels = map[string]struct{}{ "cli": {}, "system": {}, "subagent": {}, + "launcher": {}, } // IsInternalChannel returns true if the channel is an internal channel. +// Supports compound names like "launcher:chat" by checking the prefix before ":". func IsInternalChannel(channel string) bool { - _, found := internalChannels[channel] - return found + if _, found := internalChannels[channel]; found { + return true + } + // Check prefix for compound channel names (e.g. "launcher:chat") + if idx := strings.IndexByte(channel, ':'); idx > 0 { + _, found := internalChannels[channel[:idx]] + return found + } + return false } diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 56dc87a53..f2bbbce9c 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "os" + "path/filepath" "runtime" "strings" "sync" @@ -30,7 +31,7 @@ var ( FATAL: "FATAL", } - currentLevel = INFO + currentLevel = WARN logger *Logger once sync.Once mu sync.RWMutex @@ -61,6 +62,32 @@ func SetLevel(level LogLevel) { currentLevel = level } +// SetLevelByName sets log level from a string: "debug", "info", "warn", "error". +func SetLevelByName(name string) { + switch strings.ToLower(strings.TrimSpace(name)) { + case "debug": + SetLevel(DEBUG) + case "info": + SetLevel(INFO) + case "warn", "warning": + SetLevel(WARN) + case "error": + SetLevel(ERROR) + } +} + +// ApplyConfig sets level and file logging from config values. +func ApplyConfig(level, fileDir string) { + if level != "" { + SetLevelByName(level) + } + if fileDir != "" { + logFile := filepath.Join(fileDir, "picoclaw.log") + os.MkdirAll(fileDir, 0755) + _ = EnableFileLogging(logFile) + } +} + func GetLevel() LogLevel { mu.RLock() defer mu.RUnlock() diff --git a/pkg/providers/protocoltypes/types.go b/pkg/providers/protocoltypes/types.go index 99f13334e..23b09ad26 100644 --- a/pkg/providers/protocoltypes/types.go +++ b/pkg/providers/protocoltypes/types.go @@ -66,7 +66,8 @@ type Message struct { Role string `json:"role"` Content string `json:"content"` ReasoningContent string `json:"reasoning_content,omitempty"` - SystemParts []ContentBlock `json:"system_parts,omitempty"` // structured system blocks for cache-aware adapters + SystemParts []ContentBlock `json:"system_parts,omitempty"` // structured system blocks for cache-aware adapters + CacheControl string `json:"cache_control,omitempty"` // "ephemeral" | "", Anthropic adapter translates ToolCalls []ToolCall `json:"tool_calls,omitempty"` ToolCallID string `json:"tool_call_id,omitempty"` } diff --git a/pkg/shell/commands.go b/pkg/shell/commands.go new file mode 100644 index 000000000..d1e36dbdb --- /dev/null +++ b/pkg/shell/commands.go @@ -0,0 +1,732 @@ +package shell + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +// CmdFunc is the signature for a built-in shell command. +// It receives the arguments (after the command name) and the working directory. +type CmdFunc func(args []string, cwd string) string + +// BuiltinCmds maps command names to their Go implementations. +// These run cross-platform without external dependencies. +var BuiltinCmds = map[string]CmdFunc{ + "ls": cmdLs, + "dir": cmdLs, + "cat": cmdCat, + "type": cmdCat, + "head": cmdHead, + "tail": cmdTail, + "grep": cmdGrep, + "wc": cmdWc, + "find": cmdFind, + "pwd": cmdPwd, + "echo": cmdEcho, + "stat": cmdStat, + "diff": cmdDiff, + "tree": cmdTree, + "touch": cmdTouch, + "mkdir": cmdMkdir, + "cp": cmdCp, + "mv": cmdMv, +} + +// DevToolPassthrough lists commands that pass through to the system shell. +var DevToolPassthrough = map[string]bool{ + "go": true, "git": true, "node": true, "python": true, "python3": true, + "npm": true, "npx": true, "cargo": true, "make": true, + "jq": true, "rg": true, "ag": true, "ack": true, "fd": true, +} + +// --------------------------------------------------------------------------- +// ls / dir +// --------------------------------------------------------------------------- + +func cmdLs(args []string, cwd string) string { + dir := cwd + showAll := false + longFmt := false + + for _, a := range args { + switch { + case a == "-a": + showAll = true + case a == "-l": + longFmt = true + case a == "-la" || a == "-al": + showAll = true + longFmt = true + case !strings.HasPrefix(a, "-"): + dir = ResolvePath(a, cwd) + } + } + + entries, err := os.ReadDir(dir) + if err != nil { + return fmt.Sprintf("ls: %v", err) + } + + var sb strings.Builder + for _, e := range entries { + name := e.Name() + if !showAll && strings.HasPrefix(name, ".") { + continue + } + if longFmt { + info, _ := e.Info() + if info != nil { + mode := info.Mode().String() + size := info.Size() + mod := info.ModTime().Format("Jan 02 15:04") + if e.IsDir() { + name += "/" + } + fmt.Fprintf(&sb, "%s %8d %s %s\n", mode, size, mod, name) + } else { + fmt.Fprintf(&sb, "%s\n", name) + } + } else { + if e.IsDir() { + name += "/" + } + sb.WriteString(name + "\n") + } + } + if sb.Len() == 0 { + return "(empty directory)" + } + return sb.String() +} + +// --------------------------------------------------------------------------- +// cat / type +// --------------------------------------------------------------------------- + +func cmdCat(args []string, cwd string) string { + if len(args) == 0 { + return "cat: missing file operand" + } + var sb strings.Builder + for _, f := range args { + if strings.HasPrefix(f, "-") { + continue + } + data, err := os.ReadFile(ResolvePath(f, cwd)) + if err != nil { + fmt.Fprintf(&sb, "cat: %v\n", err) + continue + } + sb.Write(data) + } + return sb.String() +} + +// --------------------------------------------------------------------------- +// head +// --------------------------------------------------------------------------- + +func cmdHead(args []string, cwd string) string { + n := 10 + var file string + for i := 0; i < len(args); i++ { + if args[i] == "-n" && i+1 < len(args) { + n, _ = strconv.Atoi(args[i+1]) + i++ + } else if !strings.HasPrefix(args[i], "-") { + file = args[i] + } + } + if file == "" { + return "head: missing file" + } + data, err := os.ReadFile(ResolvePath(file, cwd)) + if err != nil { + return fmt.Sprintf("head: %v", err) + } + lines := strings.SplitN(string(data), "\n", n+1) + if len(lines) > n { + lines = lines[:n] + } + return strings.Join(lines, "\n") +} + +// --------------------------------------------------------------------------- +// tail +// --------------------------------------------------------------------------- + +func cmdTail(args []string, cwd string) string { + n := 10 + var file string + for i := 0; i < len(args); i++ { + if args[i] == "-n" && i+1 < len(args) { + n, _ = strconv.Atoi(args[i+1]) + i++ + } else if !strings.HasPrefix(args[i], "-") { + file = args[i] + } + } + if file == "" { + return "tail: missing file" + } + data, err := os.ReadFile(ResolvePath(file, cwd)) + if err != nil { + return fmt.Sprintf("tail: %v", err) + } + lines := strings.Split(strings.TrimRight(string(data), "\n"), "\n") + start := len(lines) - n + if start < 0 { + start = 0 + } + return strings.Join(lines[start:], "\n") +} + +// --------------------------------------------------------------------------- +// grep +// --------------------------------------------------------------------------- + +func cmdGrep(args []string, cwd string) string { + ignoreCase := false + showLineNum := false + recursive := false + var pattern string + var paths []string + + for i := 0; i < len(args); i++ { + a := args[i] + if strings.HasPrefix(a, "-") && pattern == "" { + for _, ch := range a[1:] { + switch ch { + case 'i': + ignoreCase = true + case 'n': + showLineNum = true + case 'r', 'R': + recursive = true + } + } + } else if pattern == "" { + pattern = a + } else { + paths = append(paths, a) + } + } + + if pattern == "" { + return "grep: missing pattern" + } + if len(paths) == 0 { + paths = []string{"."} + } + + pat := pattern + if ignoreCase { + pat = "(?i)" + pat + } + re, err := regexp.Compile(pat) + if err != nil { + return fmt.Sprintf("grep: invalid pattern: %v", err) + } + + var sb strings.Builder + matchCount := 0 + maxMatches := 200 + + var searchFile func(path string) + searchFile = func(path string) { + if matchCount >= maxMatches { + return + } + data, err := os.ReadFile(path) + if err != nil { + return + } + if IsBinary(data) { + return + } + relPath, _ := filepath.Rel(cwd, path) + if relPath == "" { + relPath = path + } + lines := strings.Split(string(data), "\n") + for i, line := range lines { + if matchCount >= maxMatches { + break + } + if re.MatchString(line) { + matchCount++ + if showLineNum { + fmt.Fprintf(&sb, "%s:%d:%s\n", relPath, i+1, line) + } else { + fmt.Fprintf(&sb, "%s:%s\n", relPath, line) + } + } + } + } + + skipDirs := map[string]bool{".git": true, "node_modules": true, "vendor": true, "__pycache__": true} + + for _, p := range paths { + resolved := ResolvePath(p, cwd) + info, err := os.Stat(resolved) + if err != nil { + fmt.Fprintf(&sb, "grep: %v\n", err) + continue + } + if info.IsDir() { + if !recursive { + fmt.Fprintf(&sb, "grep: %s: is a directory\n", p) + continue + } + _ = filepath.Walk(resolved, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return nil + } + if fi.IsDir() { + if skipDirs[fi.Name()] || strings.HasPrefix(fi.Name(), ".") { + return filepath.SkipDir + } + return nil + } + searchFile(path) + return nil + }) + } else { + searchFile(resolved) + } + } + + if matchCount == 0 { + return "(no matches)" + } + if matchCount >= maxMatches { + fmt.Fprintf(&sb, "\n... (truncated at %d matches)\n", maxMatches) + } + return sb.String() +} + +// --------------------------------------------------------------------------- +// wc +// --------------------------------------------------------------------------- + +func cmdWc(args []string, cwd string) string { + countLines := false + countWords := false + countBytes := false + var files []string + + for _, a := range args { + if strings.HasPrefix(a, "-") { + for _, ch := range a[1:] { + switch ch { + case 'l': + countLines = true + case 'w': + countWords = true + case 'c': + countBytes = true + } + } + } else { + files = append(files, a) + } + } + if !countLines && !countWords && !countBytes { + countLines, countWords, countBytes = true, true, true + } + if len(files) == 0 { + return "wc: missing file" + } + + var sb strings.Builder + totalL, totalW, totalB := 0, 0, 0 + + for _, f := range files { + data, err := os.ReadFile(ResolvePath(f, cwd)) + if err != nil { + fmt.Fprintf(&sb, "wc: %v\n", err) + continue + } + l := strings.Count(string(data), "\n") + w := len(strings.Fields(string(data))) + b := len(data) + totalL += l + totalW += w + totalB += b + + var parts []string + if countLines { + parts = append(parts, fmt.Sprintf("%7d", l)) + } + if countWords { + parts = append(parts, fmt.Sprintf("%7d", w)) + } + if countBytes { + parts = append(parts, fmt.Sprintf("%7d", b)) + } + fmt.Fprintf(&sb, "%s %s\n", strings.Join(parts, ""), f) + } + + if len(files) > 1 { + var parts []string + if countLines { + parts = append(parts, fmt.Sprintf("%7d", totalL)) + } + if countWords { + parts = append(parts, fmt.Sprintf("%7d", totalW)) + } + if countBytes { + parts = append(parts, fmt.Sprintf("%7d", totalB)) + } + fmt.Fprintf(&sb, "%s total\n", strings.Join(parts, "")) + } + return sb.String() +} + +// --------------------------------------------------------------------------- +// find +// --------------------------------------------------------------------------- + +func cmdFind(args []string, cwd string) string { + dir := cwd + namePattern := "" + typeFilter := "" + + for i := 0; i < len(args); i++ { + switch args[i] { + case "-name": + if i+1 < len(args) { + namePattern = args[i+1] + i++ + } + case "-type": + if i+1 < len(args) { + typeFilter = args[i+1] + i++ + } + default: + if !strings.HasPrefix(args[i], "-") && namePattern == "" { + dir = ResolvePath(args[i], cwd) + } + } + } + + skipDirs := map[string]bool{".git": true, "node_modules": true, "vendor": true} + var sb strings.Builder + count := 0 + maxResults := 200 + + _ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil || count >= maxResults { + return nil + } + name := info.Name() + if info.IsDir() && skipDirs[name] { + return filepath.SkipDir + } + if strings.HasPrefix(name, ".") && path != dir { + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + if typeFilter == "f" && info.IsDir() { + return nil + } + if typeFilter == "d" && !info.IsDir() { + return nil + } + if namePattern != "" { + matched, _ := filepath.Match(namePattern, name) + if !matched { + return nil + } + } + rel, _ := filepath.Rel(cwd, path) + if rel == "" { + rel = path + } + sb.WriteString(rel + "\n") + count++ + return nil + }) + + if count == 0 { + return "(no matches)" + } + if count >= maxResults { + fmt.Fprintf(&sb, "... (truncated at %d results)\n", maxResults) + } + return sb.String() +} + +// --------------------------------------------------------------------------- +// pwd / echo / stat +// --------------------------------------------------------------------------- + +func cmdPwd(_ []string, cwd string) string { return cwd } + +func cmdEcho(args []string, _ string) string { return strings.Join(args, " ") } + +func cmdStat(args []string, cwd string) string { + if len(args) == 0 { + return "stat: missing file" + } + var sb strings.Builder + for _, f := range args { + info, err := os.Stat(ResolvePath(f, cwd)) + if err != nil { + fmt.Fprintf(&sb, "stat: %v\n", err) + continue + } + fmt.Fprintf(&sb, " File: %s\n", f) + fmt.Fprintf(&sb, " Size: %d bytes\n", info.Size()) + fmt.Fprintf(&sb, " Mode: %s\n", info.Mode()) + fmt.Fprintf(&sb, " Modified: %s\n", info.ModTime().Format(time.RFC3339)) + if info.IsDir() { + sb.WriteString(" Type: directory\n") + } else { + sb.WriteString(" Type: regular file\n") + } + sb.WriteString("\n") + } + return sb.String() +} + +// --------------------------------------------------------------------------- +// diff +// --------------------------------------------------------------------------- + +func cmdDiff(args []string, cwd string) string { + if len(args) < 2 { + return "diff: need two files" + } + data1, err := os.ReadFile(ResolvePath(args[0], cwd)) + if err != nil { + return fmt.Sprintf("diff: %v", err) + } + data2, err := os.ReadFile(ResolvePath(args[1], cwd)) + if err != nil { + return fmt.Sprintf("diff: %v", err) + } + + lines1 := strings.Split(string(data1), "\n") + lines2 := strings.Split(string(data2), "\n") + + var sb strings.Builder + fmt.Fprintf(&sb, "--- %s\n+++ %s\n", args[0], args[1]) + + maxLen := len(lines1) + if len(lines2) > maxLen { + maxLen = len(lines2) + } + + diffs := 0 + for i := 0; i < maxLen; i++ { + var l1, l2 string + if i < len(lines1) { + l1 = lines1[i] + } + if i < len(lines2) { + l2 = lines2[i] + } + if l1 != l2 { + diffs++ + if diffs > 100 { + sb.WriteString("... (too many differences)\n") + break + } + fmt.Fprintf(&sb, "@@ line %d @@\n", i+1) + if l1 != "" { + fmt.Fprintf(&sb, "-%s\n", l1) + } + if l2 != "" { + fmt.Fprintf(&sb, "+%s\n", l2) + } + } + } + + if diffs == 0 { + return "Files are identical" + } + return sb.String() +} + +// --------------------------------------------------------------------------- +// tree +// --------------------------------------------------------------------------- + +func cmdTree(args []string, cwd string) string { + dir := cwd + if len(args) > 0 && !strings.HasPrefix(args[0], "-") { + dir = ResolvePath(args[0], cwd) + } + + skipDirs := map[string]bool{".git": true, "node_modules": true, "vendor": true, "__pycache__": true} + var sb strings.Builder + sb.WriteString(dir + "\n") + count := 0 + maxEntries := 300 + + var walk func(path, prefix string) + walk = func(path, prefix string) { + if count >= maxEntries { + return + } + entries, err := os.ReadDir(path) + if err != nil { + return + } + var visible []os.DirEntry + for _, e := range entries { + if !strings.HasPrefix(e.Name(), ".") && !skipDirs[e.Name()] { + visible = append(visible, e) + } + } + sort.Slice(visible, func(i, j int) bool { return visible[i].Name() < visible[j].Name() }) + for i, e := range visible { + if count >= maxEntries { + sb.WriteString(prefix + "... (truncated)\n") + return + } + count++ + connector := "้ˆนๆบพๆ”ข้ˆนโ‚ฌ " + childPrefix := prefix + "้ˆน? " + if i == len(visible)-1 { + connector = "้ˆนๆ–บๆ”ข้ˆนโ‚ฌ " + childPrefix = prefix + " " + } + sb.WriteString(prefix + connector + e.Name()) + if e.IsDir() { + sb.WriteString("/\n") + walk(filepath.Join(path, e.Name()), childPrefix) + } else { + sb.WriteString("\n") + } + } + } + + walk(dir, "") + return sb.String() +} + +// --------------------------------------------------------------------------- +// touch / mkdir / cp / mv +// --------------------------------------------------------------------------- + +func cmdTouch(args []string, cwd string) string { + if len(args) == 0 { + return "touch: missing file" + } + for _, f := range args { + if strings.HasPrefix(f, "-") { + continue + } + p := ResolvePath(f, cwd) + if _, err := os.Stat(p); os.IsNotExist(err) { + if err := os.WriteFile(p, []byte{}, 0644); err != nil { + return fmt.Sprintf("touch: %v", err) + } + } else { + now := time.Now() + _ = os.Chtimes(p, now, now) + } + } + return fmt.Sprintf("touched %d file(s)", len(args)) +} + +func cmdMkdir(args []string, cwd string) string { + if len(args) == 0 { + return "mkdir: missing directory" + } + mkParents := false + var dirs []string + for _, a := range args { + if a == "-p" { + mkParents = true + } else { + dirs = append(dirs, a) + } + } + for _, d := range dirs { + p := ResolvePath(d, cwd) + var err error + if mkParents { + err = os.MkdirAll(p, 0755) + } else { + err = os.Mkdir(p, 0755) + } + if err != nil { + return fmt.Sprintf("mkdir: %v", err) + } + } + return fmt.Sprintf("created %d dir(s)", len(dirs)) +} + +func cmdCp(args []string, cwd string) string { + if len(args) < 2 { + return "cp: need source and destination" + } + src := ResolvePath(args[0], cwd) + dst := ResolvePath(args[1], cwd) + + data, err := os.ReadFile(src) + if err != nil { + return fmt.Sprintf("cp: %v", err) + } + if info, err := os.Stat(dst); err == nil && info.IsDir() { + dst = filepath.Join(dst, filepath.Base(src)) + } + if err := os.WriteFile(dst, data, 0644); err != nil { + return fmt.Sprintf("cp: %v", err) + } + return fmt.Sprintf("copied %s -> %s", args[0], filepath.Base(dst)) +} + +func cmdMv(args []string, cwd string) string { + if len(args) < 2 { + return "mv: need source and destination" + } + src := ResolvePath(args[0], cwd) + dst := ResolvePath(args[1], cwd) + + if info, err := os.Stat(dst); err == nil && info.IsDir() { + dst = filepath.Join(dst, filepath.Base(src)) + } + if err := os.Rename(src, dst); err != nil { + return fmt.Sprintf("mv: %v", err) + } + return fmt.Sprintf("moved %s -> %s", args[0], filepath.Base(dst)) +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// ResolvePath resolves a path relative to cwd. +func ResolvePath(path, cwd string) string { + if filepath.IsAbs(path) { + return filepath.Clean(path) + } + return filepath.Join(cwd, path) +} + +// IsBinary checks if the first 512 bytes contain null bytes. +func IsBinary(data []byte) bool { + check := data + if len(check) > 512 { + check = check[:512] + } + for _, b := range check { + if b == 0 { + return true + } + } + return false +} \ No newline at end of file