refactor: 3-phase runtime loop with CoT system and memory hierarchy

Architecture:
- Phase 1 (Analyse): lightweight pre-LLM for intent/tag extraction + CoT generation
- Phase 2 (Execute): LLM iteration loop with tool handling (from loop.go)
- Phase 3 (Reflect): post-LLM scoring, async processors, slash commands

Features:
- CoT template registry (8 built-in) with self-learning feedback loop
- SQLite-backed TurnStore for turn persistence
- Active Context: per-session file/error tracking
- Instant Memory: dynamic context window from TurnStore
- Memory Digest: batch background worker for long-term memory extraction
- Cross-platform shell commands for /shell
- Turn quality scoring for context prioritization

Design docs: runtime_loop_design.md, runtime_loop_task.md, runtime_builtin_cmd.md
This commit is contained in:
mingzhi1 2026-03-03 13:09:01 +08:00
parent 946af6b53d
commit b29d13dcbb
34 changed files with 7293 additions and 849 deletions

View file

@ -23,16 +23,20 @@ func agentCmd(message, sessionKey, model string, debug bool) error {
sessionKey = "cli:default" sessionKey = "cli:default"
} }
if debug {
logger.SetLevel(logger.DEBUG)
fmt.Println("🔍 Debug mode enabled")
}
cfg, err := internal.LoadConfig() cfg, err := internal.LoadConfig()
if err != nil { if err != nil {
return fmt.Errorf("error loading config: %w", err) return fmt.Errorf("error loading config: %w", err)
} }
// Apply logging config (config file setting).
logger.ApplyConfig(cfg.Logging.Level, cfg.Logging.FileDir)
// Debug flag overrides config.
if debug {
logger.SetLevel(logger.INFO)
fmt.Println("Debug mode enabled")
}
if model != "" { if model != "" {
cfg.Agents.Defaults.ModelName = model cfg.Agents.Defaults.ModelName = model
} }
@ -60,6 +64,9 @@ func agentCmd(message, sessionKey, model string, debug bool) error {
"skills_available": startupInfo["skills"].(map[string]any)["available"], "skills_available": startupInfo["skills"].(map[string]any)["available"],
}) })
// Warn if bootstrap files are not customized.
internal.WarnMissingBootstrap(cfg.Agents.Defaults.Workspace)
if message != "" { if message != "" {
ctx := context.Background() ctx := context.Background()
response, err := agentLoop.ProcessDirect(ctx, message, sessionKey) response, err := agentLoop.ProcessDirect(ctx, message, sessionKey)

View file

@ -80,6 +80,9 @@ func gatewayCmd(debug bool) error {
"skills_available": skillsInfo["available"], "skills_available": skillsInfo["available"],
}) })
// Warn if bootstrap files are not customized.
internal.WarnMissingBootstrap(cfg.Agents.Defaults.Workspace)
// Setup cron tool and service // Setup cron tool and service
execTimeout := time.Duration(cfg.Tools.Cron.ExecTimeoutMinutes) * time.Minute execTimeout := time.Duration(cfg.Tools.Cron.ExecTimeoutMinutes) * time.Minute
cronService := setupCronTool( cronService := setupCronTool(

View file

@ -53,3 +53,36 @@ func FormatBuildInfo() (string, string) {
func GetVersion() string { func GetVersion() string {
return version return version
} }
// WarnMissingBootstrap checks workspace bootstrap files (SOUL.md, IDENTITY.md, USER.md)
// and warns the user if any are missing or unmodified.
func WarnMissingBootstrap(workspace string) {
files := []struct {
name string
desc string
}{
{"SOUL.md", "personality & behavior"},
{"IDENTITY.md", "agent name & description"},
{"USER.md", "your preferences & info"},
}
var missing []string
for _, f := range files {
path := filepath.Join(workspace, f.name)
info, err := os.Stat(path)
if os.IsNotExist(err) {
missing = append(missing, fmt.Sprintf(" %s — %s", f.name, f.desc))
} else if err == nil && info.Size() < 50 {
// File exists but appears to be empty/placeholder
missing = append(missing, fmt.Sprintf(" %s — %s (empty)", f.name, f.desc))
}
}
if len(missing) > 0 {
fmt.Println(" Customize your agent:")
for _, m := range missing {
fmt.Println(m)
}
fmt.Printf(" Edit files in: %s\n\n", workspace)
}
}

View file

@ -0,0 +1,39 @@
# Runtime Commands
## Slash Commands
All commands start with `/`, handled synchronously by `Reflector.HandleCommand()`.
| Command | Usage | Description |
|---------|-------|-------------|
| `/help` | `/help` | List all commands |
| `/memory list` | `/memory list` | Show recent memories |
| `/memory add` | `/memory add <text> #tags` | Add a memory |
| `/memory delete` | `/memory delete <id>` | Delete by ID |
| `/memory search` | `/memory search <tags>` | Search by tags |
| `/memory stats` | `/memory stats` | Memory statistics |
| `/cot feedback` | `/cot feedback <1\|0\|-1>` | Rate last CoT strategy |
| `/cot stats` | `/cot stats` | CoT performance stats |
| `/cot history` | `/cot history [N]` | Recent CoT usage |
| `/shell` | `/shell <cmd> [args]` | Execute shell command |
| `/show model` | `/show model` | Current model |
| `/list agents` | `/list agents` | List agents |
| `/switch model to` | `/switch model to <name>` | Switch model |
| `/runtime status` | `/runtime status` | Runtime diagnostics |
## /shell Architecture
```
/shell <cmd> <args>
├─ Built-in (pure Go, cross-platform)
│ ls, cat, head, tail, grep, wc, find, diff, tree,
│ stat, pwd, echo, touch, mkdir, cp, mv
└─ Dev Tool Passthrough (via exec tool)
go, git, node, python, npm, cargo, make, jq, rg
```
## Security
- Built-in: Go stdlib only, auto-skip `.git`/`node_modules`, output capped at 4000 chars
- Passthrough: whitelist + deny patterns (`| sh`, `$()`, etc.) + ExecTool workspace restriction
- Unknown commands: rejected

View file

@ -0,0 +1,77 @@
# Runtime Loop Design
> Status: Implemented | Date: 2026-03-02
## Architecture
```
Message ──→ Phase 1 (Analyse) ──→ Phase 2 (Execute) ──→ Phase 3 (Reflect)
```
| Phase | File | Responsibility |
|-------|------|----------------|
| **Analyse** | `analyser.go` | Lightweight LLM → intent, tags, CoT prompt |
| **Execute** | `executor.go` | LLM iteration loop + tool calling |
| **Reflect** | `reflector.go` | Turn scoring, TurnRecord persistence, slash commands |
## Turn Definition
One Turn = user message + Phase 1 result + all Phase 2 iterations + Phase 3 output.
Multiple tool-call iterations within Phase 2 count as **one Turn**.
## Phase 1: Analyse
- Uses configurable `analyser_model` (fast/cheap model, falls back to main model)
- Inputs: user message, Active Context, available tags, CoT learning data
- Outputs: `intent`, `tags[]`, `cot_prompt`
- After analysis: memory retrieval by tags, CoT injection into system prompt
## Phase 2: Execute
- LLM → tool call → tool result loop until no more tool calls
- Retry logic for context window overflow with automatic compression
- Reasoning output forwarded to dedicated channels
## Phase 3: Reflect
- **SyncPhase3** (< 2ms, before response sent): turn scoring + Active Context update
- **AsyncPhase3** (after response): `go turnStore.Insert(record)` → SQLite
- Slash commands: `/memory`, `/cot`, `/shell`, `/show`, `/list`, `/switch`, `/help`
## Scoring Rules
| Condition | Score |
|-----------|-------|
| Has tool calls | +3 |
| Write/edit tools | +2 |
| intent = task/code/debug | +3 |
| Reply > 500 chars | +2 |
| Short exchange < 80 chars | -2 |
## Memory Hierarchy
| Layer | Lifetime | Purpose |
|-------|----------|---------|
| Instant Memory | Per-turn | Dynamic window from TurnStore (score + tag filtering) |
| Active Context | Per-session | `CurrentFiles` + `RecentErrors`, injected into user prompt |
| Long-term Memory | Persistent | MemoryDigest batch extraction → `memory.db` |
## Multi-Model Support
| Config Field | Phase | Fallback |
|-------------|-------|----------|
| `model_name` | Phase 2 | — |
| `analyser_model` | Phase 1 | → `model_name` |
| `digest_model` | MemoryDigest | → `model_name` |
## Message Ordering (KV Cache Friendly)
```
[system_prompt] → always cached
[long_term_memory by tags] → cached when same tags
[always_keep turns (score≥7)] → fixed position, append-only
[recent turns] → rolling window
[current user message] → new each turn
```
Active Context injected as **user message** (not system prompt) to keep system prompt prefix stable.

View file

@ -0,0 +1,55 @@
# Runtime Loop Implementation Tasks
> Design ref: `docs/design/picoclaw_runtime_loop_design.md`
## Design Decisions
| Decision | Conclusion |
|----------|------------|
| Turn storage | SQLite `turns.db` |
| Active Context fields | `CurrentFiles` + `RecentErrors` only |
| Phase 1 short-circuit | No — short messages need Active Context most |
| Async write | Direct `go insert()`, no channel buffer |
| Token budget check | Periodic time-based archival instead |
| Tag-gated tools | Deferred until tool count > 15 |
| KV Cache | Fixed ordering (high-score first, by ID ASC) |
## Milestones
### M1: Turn Score + Phase 3 Timing — ✅
- [x] `score.go`: `CalcTurnScore(input) int`
- [x] Split `RunPostLLM``SyncPhase3` (sync, < 2ms) + `AsyncPhase3` (goroutine)
- [x] Adjust `runAgentLoop` timing: score → publish → async write
### M2: Active Context — ✅
- [x] `active_context.go`: per `channel:chatID` store
- [x] Fields: `CurrentFiles` (5), `RecentErrors` (3)
- [x] Injected as user message (not system prompt) for KV cache stability
- [x] Flush to JSON on shutdown, load on startup
### M3: TurnStore — ✅
- [x] `turn_store.go`: SQLite `turns.db` with WAL mode
- [x] Methods: Insert, QueryPending, QueryByScore, QueryByTags, QueryRecent, ArchiveOldProcessed
- [x] Async insert via goroutine in AsyncPhase3
### M4: MemoryDigest — ✅
- [x] `memory_digest.go`: background worker (5min interval)
- [x] QueryPending → group by channel → LLM batch extraction → MemoryStore
- [x] Removed `MemoryExtractor` and `CotEvaluator` processors (kept `ErrorTracker`)
### M5: Instant Memory + KV Cache Ordering — ✅
- [x] `instant_memory.go`: dynamic window from TurnStore
- [x] Cache-friendly message ordering: system → memory → high-score → recent → current
- [x] Legacy SessionManager kept as fallback
## Deferred
| Item | Reason |
|------|--------|
| Tag-gated tool loading | Tool count < 10 currently |
| Summary Anchor | Fixed ordering sufficient for v1 |

233
pkg/agent/active_context.go Normal file
View file

@ -0,0 +1,233 @@
// PicoClaw - Ultra-lightweight personal AI agent
// License: MIT
//
// Copyright (c) 2026 PicoClaw contributors
package agent
import (
"encoding/json"
"fmt"
"os"
"strings"
"sync"
"github.com/sipeed/picoclaw/pkg/logger"
)
// ActiveContext holds the structured per-channel context that Phase 1 uses
// to understand short/ambiguous user messages.
//
// Design choices:
// - CurrentFiles: last 5 file paths touched by tool calls (read/write/edit/append/list_dir).
// - RecentErrors: last 3 tool failure messages.
// - CurrentTask / RecentSummaries are intentionally omitted — they overlap with
// the recent-M turns in instant memory and would be redundant.
type ActiveContext struct {
CurrentFiles []string `json:"current_files"` // newest first, max 5
RecentErrors []string `json:"recent_errors"` // newest first, max 3
}
// ActiveContextStore is a thread-safe in-memory map of channel:chatID → ActiveContext.
// On startup it is loaded from disk; on stop it is flushed back.
type ActiveContextStore struct {
mu sync.RWMutex
data map[string]*ActiveContext // key = "channel:chatID"
}
// NewActiveContextStore creates an empty store.
func NewActiveContextStore() *ActiveContextStore {
return &ActiveContextStore{
data: make(map[string]*ActiveContext),
}
}
// Get returns a copy of the ActiveContext for the given key (never nil).
func (s *ActiveContextStore) Get(key string) *ActiveContext {
s.mu.RLock()
ac, ok := s.data[key]
s.mu.RUnlock()
if !ok || ac == nil {
return &ActiveContext{}
}
// Return a shallow copy to avoid callers mutating the store.
cp := *ac
cp.CurrentFiles = append([]string(nil), ac.CurrentFiles...)
cp.RecentErrors = append([]string(nil), ac.RecentErrors...)
return &cp
}
// fileExtractingTools is the set of tool names whose arguments may carry file paths.
// Keys are lowercase tool names; values indicate the argument name(s) to inspect.
var fileExtractingTools = map[string][]string{
"read_file": {"path", "file_path", "filename"},
"write_file": {"path", "file_path", "filename"},
"edit_file": {"path", "file_path", "filename"},
"append_file": {"path", "file_path", "filename"},
"list_dir": {"path", "dir_path", "directory"},
}
// Update applies the outcomes of a completed turn to the ActiveContext for key.
// It extracts file paths from tool call arguments and captures error messages.
func (s *ActiveContextStore) Update(key string, input RuntimeInput) {
if key == "" {
return
}
s.mu.Lock()
defer s.mu.Unlock()
ac, ok := s.data[key]
if !ok || ac == nil {
ac = &ActiveContext{}
s.data[key] = ac
}
// Extract file paths from tool calls.
for _, tc := range input.ToolCalls {
name := strings.ToLower(tc.Name)
argFields, relevant := fileExtractingTools[name]
if !relevant {
continue
}
// tc.Args is stored as JSON string or we can check tc.ArgsRaw if available.
// Since ToolCallRecord only has Name/Error/Duration, we skip argument extraction
// here and rely on callers passing a richer input in the future (M5).
// For now we still handle errors.
_ = argFields
}
// Capture tool errors.
for _, tc := range input.ToolCalls {
if tc.Error == "" {
continue
}
msg := fmt.Sprintf("[%s] %s", tc.Name, tc.Error)
// Prepend (newest first) and cap at 3.
ac.RecentErrors = prependCapped(ac.RecentErrors, msg, 3)
}
}
// UpdateWithFiles is an extended update that also receives file paths extracted
// by the loop (call this when tool argument parsing is available).
func (s *ActiveContextStore) UpdateWithFiles(key string, input RuntimeInput, filePaths []string) {
s.Update(key, input)
if len(filePaths) == 0 {
return
}
s.mu.Lock()
defer s.mu.Unlock()
ac, ok := s.data[key]
if !ok || ac == nil {
ac = &ActiveContext{}
s.data[key] = ac
}
for _, p := range filePaths {
if p != "" {
ac.CurrentFiles = prependCapped(ac.CurrentFiles, p, 5)
}
}
}
// prependCapped prepends item to slice and caps the result at max length.
// Deduplicates: if item already exists it is moved to the front.
func prependCapped(slice []string, item string, max int) []string {
// Remove duplicate.
filtered := make([]string, 0, len(slice))
for _, s := range slice {
if s != item {
filtered = append(filtered, s)
}
}
result := append([]string{item}, filtered...)
if len(result) > max {
result = result[:max]
}
return result
}
// Format renders the context as a markdown block for injection into a user message.
// Returns empty string when there is nothing to show.
func (ac *ActiveContext) Format() string {
if len(ac.CurrentFiles) == 0 && len(ac.RecentErrors) == 0 {
return ""
}
var sb strings.Builder
sb.WriteString("## Current Context\n")
if len(ac.CurrentFiles) > 0 {
sb.WriteString("Files in use: ")
sb.WriteString(strings.Join(ac.CurrentFiles, ", "))
sb.WriteString("\n")
}
if len(ac.RecentErrors) > 0 {
sb.WriteString("Recent errors:\n")
for _, e := range ac.RecentErrors {
sb.WriteString(" - ")
sb.WriteString(e)
sb.WriteString("\n")
}
}
return sb.String()
}
// ---------------------------------------------------------------------------
// Persistence
// ---------------------------------------------------------------------------
// persistedStore is the on-disk JSON format for ActiveContextStore.
type persistedStore struct {
Contexts map[string]*ActiveContext `json:"contexts"`
}
// Flush serialises the store to a JSON file at the given path.
func (s *ActiveContextStore) Flush(path string) error {
s.mu.RLock()
out := persistedStore{Contexts: make(map[string]*ActiveContext, len(s.data))}
for k, v := range s.data {
cp := *v
cp.CurrentFiles = append([]string(nil), v.CurrentFiles...)
cp.RecentErrors = append([]string(nil), v.RecentErrors...)
out.Contexts[k] = &cp
}
s.mu.RUnlock()
data, err := json.MarshalIndent(out, "", " ")
if err != nil {
return fmt.Errorf("active_context: marshal: %w", err)
}
if err := os.WriteFile(path, data, 0o644); err != nil {
return fmt.Errorf("active_context: write %s: %w", path, err)
}
logger.DebugCF("active_context", "Flushed to disk", map[string]any{"path": path, "keys": len(out.Contexts)})
return nil
}
// Load deserialises the store from a JSON file at the given path.
// Missing or unreadable files are silently ignored (returns nil).
func (s *ActiveContextStore) Load(path string) error {
data, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("active_context: read %s: %w", path, err)
}
var out persistedStore
if err := json.Unmarshal(data, &out); err != nil {
return fmt.Errorf("active_context: unmarshal: %w", err)
}
s.mu.Lock()
defer s.mu.Unlock()
for k, v := range out.Contexts {
if v != nil {
s.data[k] = v
}
}
logger.DebugCF("active_context", "Loaded from disk", map[string]any{"path": path, "keys": len(out.Contexts)})
return nil
}

View file

@ -0,0 +1,120 @@
package agent
import (
"os"
"path/filepath"
"testing"
)
func TestActiveContextStore_UpdateAndGet(t *testing.T) {
s := NewActiveContextStore()
key := "telegram:12345"
// Initially empty.
ac := s.Get(key)
if len(ac.CurrentFiles) != 0 || len(ac.RecentErrors) != 0 {
t.Errorf("expected empty context, got %+v", ac)
}
// Add errors via Update.
s.Update(key, RuntimeInput{
ToolCalls: []ToolCallRecord{
{Name: "exec", Error: "timeout after 30s"},
{Name: "read_file", Error: ""},
},
})
ac = s.Get(key)
if len(ac.RecentErrors) != 1 {
t.Errorf("expected 1 error, got %d: %v", len(ac.RecentErrors), ac.RecentErrors)
}
if ac.RecentErrors[0] != "[exec] timeout after 30s" {
t.Errorf("unexpected error: %s", ac.RecentErrors[0])
}
}
func TestActiveContextStore_FileCapping(t *testing.T) {
s := NewActiveContextStore()
key := "cli:direct"
// Add 7 file paths — should cap at 5, newest first.
s.UpdateWithFiles(key, RuntimeInput{}, []string{"a.go", "b.go", "c.go", "d.go", "e.go", "f.go", "g.go"})
ac := s.Get(key)
if len(ac.CurrentFiles) != 5 {
t.Fatalf("expected 5 files, got %d: %v", len(ac.CurrentFiles), ac.CurrentFiles)
}
// Last added (g.go) is prepended, so it should be first.
if ac.CurrentFiles[0] != "g.go" {
t.Errorf("expected g.go first, got %s (all: %v)", ac.CurrentFiles[0], ac.CurrentFiles)
}
}
func TestActiveContextStore_ErrorCapping(t *testing.T) {
s := NewActiveContextStore()
key := "wecom:alice"
for i := 0; i < 5; i++ {
s.Update(key, RuntimeInput{
ToolCalls: []ToolCallRecord{{Name: "exec", Error: "err"}},
})
}
ac := s.Get(key)
if len(ac.RecentErrors) > 3 {
t.Errorf("expected max 3 errors, got %d", len(ac.RecentErrors))
}
}
func TestActiveContextStore_FlushAndLoad(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "active_context.json")
s := NewActiveContextStore()
key := "cli:direct"
s.UpdateWithFiles(key, RuntimeInput{}, []string{"main.go"})
s.Update(key, RuntimeInput{
ToolCalls: []ToolCallRecord{{Name: "exec", Error: "failed"}},
})
if err := s.Flush(path); err != nil {
t.Fatalf("Flush: %v", err)
}
// File must exist.
if _, err := os.Stat(path); err != nil {
t.Fatalf("expected file to exist: %v", err)
}
// Load into new store.
s2 := NewActiveContextStore()
if err := s2.Load(path); err != nil {
t.Fatalf("Load: %v", err)
}
ac := s2.Get(key)
if len(ac.CurrentFiles) != 1 || ac.CurrentFiles[0] != "main.go" {
t.Errorf("unexpected files after reload: %v", ac.CurrentFiles)
}
if len(ac.RecentErrors) != 1 {
t.Errorf("unexpected errors after reload: %v", ac.RecentErrors)
}
}
func TestActiveContextStore_LoadMissingFile(t *testing.T) {
s := NewActiveContextStore()
// Should not error on missing file.
if err := s.Load("/nonexistent/path.json"); err != nil {
t.Errorf("Load of missing file should return nil, got: %v", err)
}
}
func TestActiveContext_Format(t *testing.T) {
ac := &ActiveContext{
CurrentFiles: []string{"main.go", "loop.go"},
RecentErrors: []string{"[exec] timeout"},
}
formatted := ac.Format()
if formatted == "" {
t.Error("expected non-empty format")
}
if len(formatted) == 0 {
t.Error("Format returned empty string")
}
}

282
pkg/agent/analyser.go Normal file
View file

@ -0,0 +1,282 @@
// PicoClaw - Ultra-lightweight personal AI agent
// License: MIT
//
// Copyright (c) 2026 PicoClaw contributors
package agent
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/sipeed/picoclaw/pkg/logger"
"github.com/sipeed/picoclaw/pkg/providers"
)
// AnalyseResult holds the output of the Phase 1 (Analyse) step.
type AnalyseResult struct {
// Intent is a short label classifying the user's intent (e.g. "question", "task", "chat").
Intent string `json:"intent"`
// Tags extracted from the user message for memory retrieval.
Tags []string `json:"tags"`
// CotPrompt is an LLM-generated thinking strategy tailored to the user's message.
// Generated by the analyser, not selected from a fixed list.
CotPrompt string `json:"cot_prompt"`
// MemoryContext is the formatted memory entries matching the extracted tags.
// This is populated after the memory lookup, not by the LLM itself.
MemoryContext string `json:"-"`
}
// Analyser performs a lightweight LLM call to analyse the user's message,
// extract intent and tags, then queries the memory store for relevant entries.
// This is Phase 1 of the Runtime Loop.
//
// Flow:
// 1. Collect all available tags from the memory store.
// 2. Call a small/fast LLM with the user message + available tags.
// 3. Parse the JSON response to get intent + matched tags.
// 4. Query memory entries by those tags.
// 5. Return the result with formatted memory context.
type Analyser struct {
provider providers.LLMProvider
model string
cotRegistry *CotRegistry
}
// NewAnalyser creates a new Analyser (Phase 1) processor.
// model should be a lightweight model identifier like "gemini/gemini-2.0-flash-exp".
func NewAnalyser(provider providers.LLMProvider, model string, cotRegistry *CotRegistry) *Analyser {
return &Analyser{
provider: provider,
model: model,
cotRegistry: cotRegistry,
}
}
const preLLMSystemPromptTpl = `You are a message analysis engine. Your job is to analyse the user's message and output a JSON object.
## Task
Given the user message, a list of available memory tags, and reference thinking strategy examples, you must:
1. Determine the user's **intent** classify it into one short label.
2. Select **relevant tags** from the available tag list. Only select genuinely relevant tags. 0 tags if none are relevant.
3. **Generate a custom thinking strategy** (cot_prompt) for the main AI to follow when processing this message. This should be a concise, actionable set of steps tailored to the specific task.
## Output Format
Respond with ONLY a valid JSON object, no markdown fences, no explanation:
{"intent":"<intent_label>","tags":["<tag1>"],"cot_prompt":"<thinking_strategy_text>"}
The cot_prompt should be a brief strategy (3-6 numbered steps). For simple chat/greetings, use an empty string "".
## Intent Labels
Use one of: question, task, chat, code, search, create, debug, explain, translate, summarise, other
## Reference Thinking Strategy Examples
Use these as inspiration adapt and combine as needed for the specific message:
%s
%s
## Rules
- ONLY select tags from the provided available tags list.
- Do NOT invent new tags. Only use tags from the available list.
- Maximum 5 tags.
- Generate a cot_prompt tailored to the specific user message. Don't just copy examples adapt them.
- For simple chat (greetings, thanks, etc.), use an empty cot_prompt.
- If historical data shows which strategies worked well for similar intents, prefer those approaches.
- Keep the cot_prompt concise: 3-6 actionable steps.
- Keep it fast this is a preprocessing step.`
// Analyse runs the pre-LLM analysis on the user message.
// It returns an AnalyseResult with intent, tags, and formatted memory context.
// If the pre-LLM call fails, it returns a zero-value result (no error propagation
// to avoid blocking the main agent loop).
// actCtx may be nil; when provided, its content is injected into the user prompt
// (not the system prompt) to preserve system prompt prefix stability for KV cache.
func (p *Analyser) Analyse(ctx context.Context, userMessage string, memory *MemoryStore, actCtx *ActiveContext) AnalyseResult {
if p.provider == nil || p.model == "" {
return AnalyseResult{}
}
start := time.Now()
// 1. Collect available tags from memory store.
var availableTags []string
var tagsErr error
if memory != nil {
availableTags, tagsErr = memory.ListAllTags()
}
hasMemoryTags := tagsErr == nil && len(availableTags) > 0
// Even without memory tags, we still call pre-LLM for CoT selection.
// 2. Build the system prompt with example templates + learning history.
examples := ""
if p.cotRegistry != nil {
examples = p.cotRegistry.ListExamplesForPrompt()
}
// Include historical CoT performance data + top-rated prompts for learning.
cotHistory := ""
if memory != nil {
// Pass available tags so proven examples can be filtered by relevance.
cotHistory = memory.FormatCotLearningContext(30, availableTags)
}
systemPrompt := fmt.Sprintf(preLLMSystemPromptTpl, examples, cotHistory)
// 3. Build the user prompt with available tags + active context.
var userPromptBuilder strings.Builder
// Active Context block (injected here to keep system prompt prefix stable).
if actCtx != nil {
if ac := actCtx.Format(); ac != "" {
userPromptBuilder.WriteString(ac)
userPromptBuilder.WriteString("\n\n")
}
}
if hasMemoryTags {
fmt.Fprintf(&userPromptBuilder, "Available tags: [%s]\n\nUser message: %s",
strings.Join(availableTags, ", "), userMessage)
} else {
fmt.Fprintf(&userPromptBuilder, "Available tags: [](none)\n\nUser message: %s", userMessage)
}
userPrompt := userPromptBuilder.String()
messages := []providers.Message{
{Role: "system", Content: systemPrompt},
{Role: "user", Content: userPrompt},
}
// 4. Call the LLM (no tools, moderate max_tokens for generated CoT, low temperature).
resp, err := p.provider.Chat(ctx, messages, nil, p.model, map[string]any{
"max_tokens": 512,
"temperature": 0.3,
})
if err != nil {
logger.WarnCF("analyser", "Pre-LLM call failed, proceeding without enrichment",
map[string]any{"error": err.Error(), "model": p.model})
return AnalyseResult{}
}
// 5. Parse the JSON response.
result := p.parseResponse(resp.Content)
// 6. Query memory by extracted tags.
if len(result.Tags) > 0 && memory != nil {
entries, err := memory.SearchByAnyTag(result.Tags)
if err == nil && len(entries) > 0 {
result.MemoryContext = formatMemoryEntries(entries)
}
}
// 7. Record usage for learning (non-blocking — don't fail the main flow).
if memory != nil && result.CotPrompt != "" {
if _, err := memory.RecordCotUsage(result.Intent, result.Tags, result.CotPrompt, userMessage); err != nil {
logger.DebugCF("analyser", "Failed to record CoT usage",
map[string]any{"error": err.Error()})
}
}
elapsed := time.Since(start)
logger.InfoCF("analyser", "Pre-LLM analysis complete",
map[string]any{
"intent": result.Intent,
"tags": result.Tags,
"has_cot": result.CotPrompt != "",
"cot_len": len(result.CotPrompt),
"memory_entries": countMemoryLines(result.MemoryContext),
"elapsed_ms": elapsed.Milliseconds(),
"model": p.model,
"available_tags": len(availableTags),
})
return result
}
// parseResponse extracts intent and tags from the LLM's JSON response.
// Handles common LLM quirks like markdown fences around JSON.
func (p *Analyser) parseResponse(content string) AnalyseResult {
content = strings.TrimSpace(content)
// Strip markdown code fences if present.
if strings.HasPrefix(content, "```") {
lines := strings.Split(content, "\n")
// Remove first and last lines (fences).
if len(lines) >= 3 {
content = strings.Join(lines[1:len(lines)-1], "\n")
}
}
content = strings.TrimSpace(content)
var result AnalyseResult
if err := json.Unmarshal([]byte(content), &result); err != nil {
logger.WarnCF("analyser", "Failed to parse pre-LLM response as JSON",
map[string]any{
"error": err.Error(),
"content": content,
})
return AnalyseResult{}
}
// Sanitise: lowercase tags, limit to 5.
cleaned := make([]string, 0, len(result.Tags))
for _, t := range result.Tags {
t = strings.ToLower(strings.TrimSpace(t))
if t != "" {
cleaned = append(cleaned, t)
}
}
if len(cleaned) > 5 {
cleaned = cleaned[:5]
}
result.Tags = cleaned
return result
}
// formatMemoryEntries formats memory entries into a string for injection into context.
func formatMemoryEntries(entries []MemoryEntry) string {
if len(entries) == 0 {
return ""
}
var sb strings.Builder
sb.WriteString("## Relevant Memories (auto-retrieved)\n\n")
for _, e := range entries {
tagLabel := ""
if len(e.Tags) > 0 {
tagLabel = " [" + strings.Join(e.Tags, ", ") + "]"
}
fmt.Fprintf(&sb, "- (#%d%s) %s\n", e.ID, tagLabel, e.Content)
}
return sb.String()
}
// countMemoryLines counts the number of memory entries in a formatted string.
func countMemoryLines(s string) int {
if s == "" {
return 0
}
count := 0
for _, line := range strings.Split(s, "\n") {
if strings.HasPrefix(line, "- (#") {
count++
}
}
return count
}

286
pkg/agent/analyser_test.go Normal file
View file

@ -0,0 +1,286 @@
package agent
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"github.com/sipeed/picoclaw/pkg/providers"
)
func TestPreLLM_parseResponse(t *testing.T) {
p := &Analyser{}
tests := []struct {
name string
input string
wantIntent string
wantTags []string
wantCot string
}{
{
name: "valid JSON with cot_prompt",
input: `{"intent":"question","tags":["golang","testing"],"cot_prompt":"1. Understand the question\n2. Research the answer"}`,
wantIntent: "question",
wantTags: []string{"golang", "testing"},
wantCot: "1. Understand the question\n2. Research the answer",
},
{
name: "JSON with markdown fences",
input: "```json\n{\"intent\":\"task\",\"tags\":[\"deploy\"],\"cot_prompt\":\"1. Plan\\n2. Execute\"}\n```",
wantIntent: "task",
wantTags: []string{"deploy"},
wantCot: "1. Plan\n2. Execute",
},
{
name: "empty cot_prompt for chat",
input: `{"intent":"chat","tags":[],"cot_prompt":""}`,
wantIntent: "chat",
wantTags: []string{},
wantCot: "",
},
{
name: "invalid JSON",
input: "this is not json",
wantIntent: "",
wantTags: nil,
wantCot: "",
},
{
name: "tags trimmed and lowered",
input: `{"intent":"code","tags":[" GoLang "," API "],"cot_prompt":"think"}`,
wantIntent: "code",
wantTags: []string{"golang", "api"},
wantCot: "think",
},
{
name: "tags limited to 5",
input: `{"intent":"search","tags":["a","b","c","d","e","f","g"],"cot_prompt":"search"}`,
wantIntent: "search",
wantTags: []string{"a", "b", "c", "d", "e"},
wantCot: "search",
},
{
name: "missing cot_prompt field",
input: `{"intent":"question","tags":["golang"]}`,
wantIntent: "question",
wantTags: []string{"golang"},
wantCot: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := p.parseResponse(tt.input)
if result.Intent != tt.wantIntent {
t.Errorf("intent = %q, want %q", result.Intent, tt.wantIntent)
}
if result.CotPrompt != tt.wantCot {
t.Errorf("cot_prompt = %q, want %q", result.CotPrompt, tt.wantCot)
}
if tt.wantTags == nil {
if result.Tags != nil {
t.Errorf("tags = %v, want nil", result.Tags)
}
return
}
if len(result.Tags) != len(tt.wantTags) {
t.Errorf("tags len = %d, want %d (tags=%v)", len(result.Tags), len(tt.wantTags), result.Tags)
return
}
for i, tag := range result.Tags {
if tag != tt.wantTags[i] {
t.Errorf("tag[%d] = %q, want %q", i, tag, tt.wantTags[i])
}
}
})
}
}
func TestPreLLM_Analyse_NoProvider(t *testing.T) {
p := &Analyser{} // no provider, no model
result := p.Analyse(context.Background(), "hello", nil, nil)
if result.Intent != "" || len(result.Tags) != 0 {
t.Errorf("expected empty result with no provider, got %+v", result)
}
}
func TestPreLLM_Analyse_NoTags(t *testing.T) {
dir := t.TempDir()
ms := NewMemoryStore(dir)
defer ms.Close()
cotReg := NewCotRegistry(dir)
mp := &mockLLMProvider{
response: `{"intent":"chat","tags":[],"cot_prompt":""}`,
}
p := NewAnalyser(mp, "test-model", cotReg)
result := p.Analyse(context.Background(), "hello there", ms, nil)
if result.Intent != "chat" {
t.Errorf("expected intent 'chat', got %q", result.Intent)
}
if result.CotPrompt != "" {
t.Errorf("expected empty cot_prompt for chat, got %q", result.CotPrompt)
}
}
func TestPreLLM_Analyse_WithMemory(t *testing.T) {
dir := t.TempDir()
ms := NewMemoryStore(dir)
defer ms.Close()
// Seed memory.
ms.AddEntry("Go is great for concurrency", []string{"golang", "concurrency"})
ms.AddEntry("Kubernetes cluster setup notes", []string{"k8s", "devops"})
ms.AddEntry("Go testing best practices", []string{"golang", "testing"})
cotReg := NewCotRegistry(dir)
mp := &mockLLMProvider{
response: `{"intent":"question","tags":["golang"],"cot_prompt":"1. Check Go docs\n2. Write example code\n3. Verify with tests"}`,
}
p := NewAnalyser(mp, "test-model", cotReg)
result := p.Analyse(context.Background(), "How do I test Go code?", ms, nil)
if result.Intent != "question" {
t.Errorf("intent = %q, want %q", result.Intent, "question")
}
if result.CotPrompt == "" {
t.Error("expected non-empty CotPrompt")
}
if !strings.Contains(result.CotPrompt, "Go docs") {
t.Error("CotPrompt should contain the LLM-generated strategy")
}
if len(result.Tags) != 1 || result.Tags[0] != "golang" {
t.Errorf("tags = %v, want [golang]", result.Tags)
}
if result.MemoryContext == "" {
t.Error("expected non-empty MemoryContext with matching tags")
}
if !contains(result.MemoryContext, "Go is great for concurrency") {
t.Error("MemoryContext missing 'Go is great for concurrency'")
}
if !contains(result.MemoryContext, "Go testing best practices") {
t.Error("MemoryContext missing 'Go testing best practices'")
}
if contains(result.MemoryContext, "Kubernetes") {
t.Error("MemoryContext should not contain 'Kubernetes' entry")
}
// Verify usage was recorded with tags.
records, _ := ms.GetRecentCotUsage(1)
if len(records) == 0 {
t.Fatal("expected usage record to be recorded")
}
if len(records[0].Tags) != 1 || records[0].Tags[0] != "golang" {
t.Errorf("recorded tags = %v, want [golang]", records[0].Tags)
}
}
func TestSearchByAnyTag(t *testing.T) {
dir := t.TempDir()
ms := NewMemoryStore(dir)
defer ms.Close()
ms.AddEntry("Go concurrency", []string{"golang", "concurrency"})
ms.AddEntry("K8s setup", []string{"k8s", "devops"})
ms.AddEntry("Go testing", []string{"golang", "testing"})
ms.AddEntry("Python ML", []string{"python", "ml"})
entries, err := ms.SearchByAnyTag([]string{"golang", "k8s"})
if err != nil {
t.Fatal(err)
}
if len(entries) != 3 {
t.Errorf("got %d entries, want 3", len(entries))
}
entries, err = ms.SearchByAnyTag([]string{"python"})
if err != nil {
t.Fatal(err)
}
if len(entries) != 1 {
t.Errorf("got %d entries, want 1", len(entries))
}
entries, err = ms.SearchByAnyTag([]string{"nonexistent"})
if err != nil {
t.Fatal(err)
}
if len(entries) != 0 {
t.Errorf("got %d entries, want 0", len(entries))
}
}
func TestFormatMemoryEntries(t *testing.T) {
entries := []MemoryEntry{
{ID: 1, Content: "Test content 1", Tags: []string{"tag1", "tag2"}},
{ID: 2, Content: "Test content 2", Tags: []string{"tag3"}},
{ID: 3, Content: "No tags entry", Tags: nil},
}
result := formatMemoryEntries(entries)
if result == "" {
t.Fatal("expected non-empty result")
}
if !contains(result, "Relevant Memories") {
t.Error("missing header")
}
if !contains(result, "#1") {
t.Error("missing entry #1")
}
if !contains(result, "[tag1, tag2]") {
t.Error("missing tags for entry #1")
}
}
func TestFormatMemoryEntries_Empty(t *testing.T) {
result := formatMemoryEntries(nil)
if result != "" {
t.Errorf("expected empty string, got %q", result)
}
}
// --- Helpers ---
func contains(s, substr string) bool {
return strings.Contains(s, substr)
}
func TestPreLLM_MemoryDBPath(t *testing.T) {
dir := t.TempDir()
ms := NewMemoryStore(dir)
defer ms.Close()
dbPath := filepath.Join(dir, "memory.db")
if _, err := os.Stat(dbPath); err != nil {
t.Errorf("memory.db not created: %v", err)
}
}
// mockLLMProvider returns a configurable response for pre-LLM testing.
type mockLLMProvider struct {
response string
}
func (m *mockLLMProvider) Chat(
_ context.Context,
_ []providers.Message,
_ []providers.ToolDefinition,
_ string,
_ map[string]any,
) (*providers.LLMResponse, error) {
return &providers.LLMResponse{
Content: m.response,
ToolCalls: []providers.ToolCall{},
}, nil
}
func (m *mockLLMProvider) GetDefaultModel() string {
return "mock-pre-llm"
}

View file

@ -67,8 +67,7 @@ You are picoclaw, a helpful AI assistant.
## Workspace ## Workspace
Your workspace is at: %s Your workspace is at: %s
- Memory: %s/memory/MEMORY.md - Memory DB: %s/memory.db (SQLite)
- Daily Notes: %s/memory/YYYYMM/YYYYMMDD.md
- Skills: %s/skills/{skill-name}/SKILL.md - Skills: %s/skills/{skill-name}/SKILL.md
## Important Rules ## Important Rules
@ -77,10 +76,10 @@ Your workspace is at: %s
2. **Be helpful and accurate** - When using tools, briefly explain what you're doing. 2. **Be helpful and accurate** - When using tools, briefly explain what you're doing.
3. **Memory** - When interacting with me if something seems memorable, update %s/memory/MEMORY.md 3. **Memory** - When interacting with me if something seems memorable, update the long-term memory in %s/memory.db
4. **Context summaries** - Conversation summaries provided as context are approximate references only. They may be incomplete or outdated. Always defer to explicit user instructions over summary content.`, 4. **Context summaries** - Conversation summaries provided as context are approximate references only. They may be incomplete or outdated. Always defer to explicit user instructions over summary content.`,
workspacePath, workspacePath, workspacePath, workspacePath, workspacePath) workspacePath, workspacePath, workspacePath, workspacePath)
} }
func (cb *ContextBuilder) BuildSystemPrompt() string { func (cb *ContextBuilder) BuildSystemPrompt() string {
@ -181,7 +180,7 @@ func (cb *ContextBuilder) sourcePaths() []string {
filepath.Join(cb.workspace, "SOUL.md"), filepath.Join(cb.workspace, "SOUL.md"),
filepath.Join(cb.workspace, "USER.md"), filepath.Join(cb.workspace, "USER.md"),
filepath.Join(cb.workspace, "IDENTITY.md"), filepath.Join(cb.workspace, "IDENTITY.md"),
filepath.Join(cb.workspace, "memory", "MEMORY.md"), filepath.Join(cb.workspace, "memory.db"),
} }
} }
@ -579,3 +578,9 @@ func (cb *ContextBuilder) GetSkillsInfo() map[string]any {
"names": skillNames, "names": skillNames,
} }
} }
// GetMemory returns the underlying MemoryStore.
// Used by the pre-LLM module to query tags and search entries.
func (cb *ContextBuilder) GetMemory() *MemoryStore {
return cb.memory
}

View file

@ -19,7 +19,6 @@ func setupWorkspace(t *testing.T, files map[string]string) string {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
os.MkdirAll(filepath.Join(tmpDir, "memory"), 0o755)
os.MkdirAll(filepath.Join(tmpDir, "skills"), 0o755) os.MkdirAll(filepath.Join(tmpDir, "skills"), 0o755)
for name, content := range files { for name, content := range files {
dir := filepath.Dir(filepath.Join(tmpDir, name)) dir := filepath.Dir(filepath.Join(tmpDir, name))
@ -145,13 +144,6 @@ func TestMtimeAutoInvalidation(t *testing.T) {
contentV2: "# Updated Identity", contentV2: "# Updated Identity",
checkField: "Updated Identity", checkField: "Updated Identity",
}, },
{
name: "memory file change",
file: "memory/MEMORY.md",
contentV1: "# Memory\nUser likes Go.",
contentV2: "# Memory\nUser likes Rust.",
checkField: "User likes Rust",
},
} }
for _, tt := range tests { for _, tt := range tests {
@ -212,6 +204,43 @@ func TestMtimeAutoInvalidation(t *testing.T) {
t.Error("sourceFilesChangedLocked() should detect skills dir mtime change") t.Error("sourceFilesChangedLocked() should detect skills dir mtime change")
} }
}) })
// Memory DB mtime change (via MemoryStore write)
t.Run("memory DB change", func(t *testing.T) {
tmpDir := setupWorkspace(t, nil)
defer os.RemoveAll(tmpDir)
cb := NewContextBuilder(tmpDir)
// Write initial memory
cb.memory.WriteLongTerm("User likes Go.")
// Build cache
sp1 := cb.BuildSystemPromptWithCache()
if !strings.Contains(sp1, "User likes Go") {
t.Fatal("initial prompt should contain memory content")
}
// Update memory via MemoryStore
cb.memory.WriteLongTerm("User likes Rust.")
// Set future mtime on memory.db so cache detects change
dbPath := filepath.Join(tmpDir, "memory.db")
future := time.Now().Add(2 * time.Second)
os.Chtimes(dbPath, future, future)
cb.systemPromptMutex.RLock()
changed := cb.sourceFilesChangedLocked()
cb.systemPromptMutex.RUnlock()
if !changed {
t.Fatal("sourceFilesChangedLocked() should detect memory.db change")
}
sp2 := cb.BuildSystemPromptWithCache()
if !strings.Contains(sp2, "User likes Rust") {
t.Error("rebuilt prompt should contain updated memory")
}
})
} }
// TestExplicitInvalidateCache verifies that InvalidateCache() forces a rebuild // TestExplicitInvalidateCache verifies that InvalidateCache() forces a rebuild
@ -273,57 +302,35 @@ func TestCacheStability(t *testing.T) {
// This catches the "from nothing to something" edge case that the old // This catches the "from nothing to something" edge case that the old
// modifiedSince (return false on stat error) would miss. // modifiedSince (return false on stat error) would miss.
func TestNewFileCreationInvalidatesCache(t *testing.T) { func TestNewFileCreationInvalidatesCache(t *testing.T) {
tests := []struct { // Test bootstrap file creation
name string t.Run("new bootstrap file", func(t *testing.T) {
file string // relative path inside workspace // Start with an empty workspace (no bootstrap files)
content string tmpDir := setupWorkspace(t, nil)
checkField string // substring to verify in rebuilt prompt defer os.RemoveAll(tmpDir)
}{
{
name: "new bootstrap file",
file: "SOUL.md",
content: "# Soul\nBe kind and helpful.",
checkField: "Be kind and helpful",
},
{
name: "new memory file",
file: "memory/MEMORY.md",
content: "# Memory\nUser prefers dark mode.",
checkField: "User prefers dark mode",
},
}
for _, tt := range tests { cb := NewContextBuilder(tmpDir)
t.Run(tt.name, func(t *testing.T) {
// Start with an empty workspace (no bootstrap/memory files)
tmpDir := setupWorkspace(t, nil)
defer os.RemoveAll(tmpDir)
cb := NewContextBuilder(tmpDir) // Populate cache — file does not exist yet
sp1 := cb.BuildSystemPromptWithCache()
if strings.Contains(sp1, "Be kind and helpful") {
t.Fatalf("prompt should not contain content before file is created")
}
// Populate cache — file does not exist yet // Create the file after cache was built
sp1 := cb.BuildSystemPromptWithCache() fullPath := filepath.Join(tmpDir, "SOUL.md")
if strings.Contains(sp1, tt.checkField) { if err := os.WriteFile(fullPath, []byte("# Soul\nBe kind and helpful."), 0o644); err != nil {
t.Fatalf("prompt should not contain %q before file is created", tt.checkField) t.Fatal(err)
} }
// Set future mtime to guarantee detection
future := time.Now().Add(2 * time.Second)
os.Chtimes(fullPath, future, future)
// Create the file after cache was built // Cache should auto-invalidate because file went from absent -> present
fullPath := filepath.Join(tmpDir, tt.file) sp2 := cb.BuildSystemPromptWithCache()
os.MkdirAll(filepath.Dir(fullPath), 0o755) if !strings.Contains(sp2, "Be kind and helpful") {
if err := os.WriteFile(fullPath, []byte(tt.content), 0o644); err != nil { t.Errorf("cache not invalidated on new file creation")
t.Fatal(err) }
} })
// Set future mtime to guarantee detection
future := time.Now().Add(2 * time.Second)
os.Chtimes(fullPath, future, future)
// Cache should auto-invalidate because file went from absent -> present
sp2 := cb.BuildSystemPromptWithCache()
if !strings.Contains(sp2, tt.checkField) {
t.Errorf("cache not invalidated on new file creation: expected %q in prompt", tt.checkField)
}
})
}
} }
// TestSkillFileContentChange verifies that modifying a skill file's content // TestSkillFileContentChange verifies that modifying a skill file's content
@ -391,7 +398,6 @@ func TestConcurrentBuildSystemPromptWithCache(t *testing.T) {
tmpDir := setupWorkspace(t, map[string]string{ tmpDir := setupWorkspace(t, map[string]string{
"IDENTITY.md": "# Identity\nConcurrency test agent.", "IDENTITY.md": "# Identity\nConcurrency test agent.",
"SOUL.md": "# Soul\nBe helpful.", "SOUL.md": "# Soul\nBe helpful.",
"memory/MEMORY.md": "# Memory\nUser prefers Go.",
"skills/demo/SKILL.md": "---\nname: demo\ndescription: \"demo skill\"\n---\n# Demo", "skills/demo/SKILL.md": "---\nname: demo\ndescription: \"demo skill\"\n---\n# Demo",
}) })
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
@ -494,7 +500,6 @@ func BenchmarkBuildMessagesWithCache(b *testing.B) {
tmpDir, _ := os.MkdirTemp("", "picoclaw-bench-*") tmpDir, _ := os.MkdirTemp("", "picoclaw-bench-*")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
os.MkdirAll(filepath.Join(tmpDir, "memory"), 0o755)
os.MkdirAll(filepath.Join(tmpDir, "skills"), 0o755) os.MkdirAll(filepath.Join(tmpDir, "skills"), 0o755)
for _, name := range []string{"IDENTITY.md", "SOUL.md", "USER.md"} { for _, name := range []string{"IDENTITY.md", "SOUL.md", "USER.md"} {
os.WriteFile(filepath.Join(tmpDir, name), []byte(strings.Repeat("Content.\n", 10)), 0o644) os.WriteFile(filepath.Join(tmpDir, name), []byte(strings.Repeat("Content.\n", 10)), 0o644)

View file

@ -0,0 +1,297 @@
package agent
import (
"strings"
"testing"
)
func TestCotUsage_RecordAndQuery(t *testing.T) {
dir := t.TempDir()
ms := NewMemoryStore(dir)
defer ms.Close()
// Record some usage with tags.
id1, err := ms.RecordCotUsage("code", []string{"golang", "testing"}, "1. Check tests\n2. Write code", "How do I test Go code?")
if err != nil {
t.Fatal(err)
}
if id1 <= 0 {
t.Errorf("expected positive ID, got %d", id1)
}
id2, err := ms.RecordCotUsage("question", []string{"golang"}, "1. Compare options\n2. Decide", "What's the difference?")
if err != nil {
t.Fatal(err)
}
id3, err := ms.RecordCotUsage("code", []string{"http", "golang"}, "1. Define routes\n2. Implement handlers", "Write a HTTP server")
if err != nil {
t.Fatal(err)
}
// Query recent usage.
records, err := ms.GetRecentCotUsage(10)
if err != nil {
t.Fatal(err)
}
if len(records) != 3 {
t.Errorf("expected 3 records, got %d", len(records))
}
// Most recent first.
if records[0].ID != id3 {
t.Errorf("expected most recent to be id3=%d, got %d", id3, records[0].ID)
}
// Check tags are stored correctly.
if len(records[0].Tags) != 2 || records[0].Tags[0] != "http" {
t.Errorf("tags = %v, want [http, golang]", records[0].Tags)
}
// Check cot_prompt is stored.
if !strings.Contains(records[0].CotPrompt, "Define routes") {
t.Errorf("cot_prompt = %q, should contain 'Define routes'", records[0].CotPrompt)
}
_ = id2 // used above
}
func TestCotUsage_Feedback(t *testing.T) {
dir := t.TempDir()
ms := NewMemoryStore(dir)
defer ms.Close()
id, _ := ms.RecordCotUsage("code", []string{"golang"}, "think step by step", "test message")
// Initial feedback should be 0.
records, _ := ms.GetRecentCotUsage(1)
if records[0].Feedback != 0 {
t.Errorf("initial feedback = %d, want 0", records[0].Feedback)
}
// Update feedback.
err := ms.UpdateCotFeedback(id, 1)
if err != nil {
t.Fatal(err)
}
records, _ = ms.GetRecentCotUsage(1)
if records[0].Feedback != 1 {
t.Errorf("feedback = %d, want 1", records[0].Feedback)
}
// Invalid score.
err = ms.UpdateCotFeedback(id, 5)
if err == nil {
t.Error("expected error for invalid score 5")
}
}
func TestCotUsage_UpdateLatestFeedback(t *testing.T) {
dir := t.TempDir()
ms := NewMemoryStore(dir)
defer ms.Close()
ms.RecordCotUsage("code", nil, "strategy 1", "first")
ms.RecordCotUsage("debug", nil, "strategy 2", "second")
// Update latest (should be "debug").
err := ms.UpdateLatestCotFeedback(-1)
if err != nil {
t.Fatal(err)
}
records, _ := ms.GetRecentCotUsage(2)
if records[0].Intent != "debug" || records[0].Feedback != -1 {
t.Errorf("latest: intent=%q feedback=%d, want debug/-1", records[0].Intent, records[0].Feedback)
}
if records[1].Intent != "code" || records[1].Feedback != 0 {
t.Errorf("first: intent=%q feedback=%d, want code/0", records[1].Intent, records[1].Feedback)
}
}
func TestCotUsage_Stats(t *testing.T) {
dir := t.TempDir()
ms := NewMemoryStore(dir)
defer ms.Close()
id1, _ := ms.RecordCotUsage("code", nil, "think about code", "write code")
ms.UpdateCotFeedback(id1, 1)
id2, _ := ms.RecordCotUsage("code", nil, "debug systematically", "fix bug")
ms.UpdateCotFeedback(id2, 1)
id3, _ := ms.RecordCotUsage("question", nil, "analyse step by step", "why does X happen?")
ms.UpdateCotFeedback(id3, -1)
id4, _ := ms.RecordCotUsage("chat", nil, "", "hello")
ms.UpdateCotFeedback(id4, 1)
// Get stats.
stats, err := ms.GetCotStats(30)
if err != nil {
t.Fatal(err)
}
if len(stats) != 3 {
t.Errorf("expected 3 intent stats, got %d", len(stats))
}
// "code" should have highest total uses.
if stats[0].Intent != "code" || stats[0].TotalUses != 2 {
t.Errorf("expected code with 2 uses, got %q with %d", stats[0].Intent, stats[0].TotalUses)
}
}
func TestCotUsage_TopRatedPrompts(t *testing.T) {
dir := t.TempDir()
ms := NewMemoryStore(dir)
defer ms.Close()
// Record with different tags and feedback.
id1, _ := ms.RecordCotUsage("code", []string{"golang", "testing"}, "1. Write test first\n2. Then implement", "write Go test")
ms.UpdateCotFeedback(id1, 1)
id2, _ := ms.RecordCotUsage("code", []string{"python"}, "1. Use pytest\n2. Mock dependencies", "write Python test")
ms.UpdateCotFeedback(id2, 1)
id3, _ := ms.RecordCotUsage("debug", []string{"golang"}, "1. Reproduce\n2. Hypothesize", "fix Go bug")
ms.UpdateCotFeedback(id3, 1)
id4, _ := ms.RecordCotUsage("code", []string{"golang"}, "1. Bad strategy", "bad approach")
ms.UpdateCotFeedback(id4, -1) // Negative — should not appear.
// Without tag filter.
top, err := ms.GetTopRatedCotPrompts(30, 10, nil)
if err != nil {
t.Fatal(err)
}
if len(top) != 3 {
t.Errorf("expected 3 top-rated, got %d", len(top))
}
// With tag filter — "golang" should prioritise golang-tagged prompts.
top, err = ms.GetTopRatedCotPrompts(30, 2, []string{"golang"})
if err != nil {
t.Fatal(err)
}
if len(top) != 2 {
t.Errorf("expected 2, got %d", len(top))
}
// First result should have golang tag.
hasGolang := false
for _, tag := range top[0].Tags {
if tag == "golang" {
hasGolang = true
}
}
if !hasGolang {
t.Errorf("first result should have golang tag, got %v", top[0].Tags)
}
}
func TestCotUsage_FormatLearningContext(t *testing.T) {
dir := t.TempDir()
ms := NewMemoryStore(dir)
defer ms.Close()
// Empty — should return empty string.
ctx := ms.FormatCotLearningContext(30, nil)
if ctx != "" {
t.Errorf("expected empty learning context, got %q", ctx)
}
// Add some usage with feedback.
id1, _ := ms.RecordCotUsage("code", []string{"golang"}, "1. Understand requirements\n2. Write code", "write code")
ms.UpdateCotFeedback(id1, 1)
id2, _ := ms.RecordCotUsage("question", []string{"architecture"}, "1. Examine structure\n2. Explain", "why does X happen?")
ms.UpdateCotFeedback(id2, 1)
ctx = ms.FormatCotLearningContext(30, nil)
if ctx == "" {
t.Error("expected non-empty learning context after recording usage")
}
if !strings.Contains(ctx, "Historical Usage Stats") {
t.Error("missing stats header")
}
if !strings.Contains(ctx, "Proven Strategies") {
t.Error("missing proven strategies section")
}
if !strings.Contains(ctx, "golang") {
t.Error("should show tags in proven examples")
}
}
func TestCotUsage_MessageTruncation(t *testing.T) {
dir := t.TempDir()
ms := NewMemoryStore(dir)
defer ms.Close()
longMsg := strings.Repeat("x", 500)
_, err := ms.RecordCotUsage("code", nil, "strategy", longMsg)
if err != nil {
t.Fatal(err)
}
records, _ := ms.GetRecentCotUsage(1)
if len(records[0].Message) > 200 {
t.Errorf("message should be truncated to 200 chars, got %d", len(records[0].Message))
}
}
func TestPreLLM_LearningIntegration(t *testing.T) {
dir := t.TempDir()
ms := NewMemoryStore(dir)
defer ms.Close()
cotReg := NewCotRegistry(dir)
mp := &mockLLMProvider{
response: `{"intent":"code","tags":["golang"],"cot_prompt":"1. Understand the function signature\n2. Write the implementation\n3. Add error handling"}`,
}
p := NewAnalyser(mp, "test-model", cotReg)
// First call — no learning data yet.
result := p.Analyse(nil, "write a function", ms, nil)
if result.CotPrompt == "" {
t.Error("expected non-empty CotPrompt")
}
// Verify usage was recorded with tags.
records, _ := ms.GetRecentCotUsage(5)
if len(records) != 1 {
t.Fatalf("expected 1 usage record, got %d", len(records))
}
if records[0].Intent != "code" {
t.Errorf("recorded intent = %q, want %q", records[0].Intent, "code")
}
if len(records[0].Tags) != 1 || records[0].Tags[0] != "golang" {
t.Errorf("recorded tags = %v, want [golang]", records[0].Tags)
}
if records[0].CotPrompt == "" {
t.Error("recorded cot_prompt should not be empty")
}
// Provide positive feedback.
ms.UpdateLatestCotFeedback(1)
// Second call — learning context should now be included.
result2 := p.Analyse(nil, "fix this bug", ms, nil)
if result2.CotPrompt == "" {
t.Error("expected non-empty CotPrompt on second call")
}
// Should now have 2 usage records.
records, _ = ms.GetRecentCotUsage(5)
if len(records) != 2 {
t.Errorf("expected 2 usage records, got %d", len(records))
}
// Learning context should include the first proven strategy.
ctx := ms.FormatCotLearningContext(30, []string{"golang"})
if ctx == "" {
t.Error("expected non-empty learning context after usage + feedback")
}
if !strings.Contains(ctx, "Proven Strategies") {
t.Error("learning context should include proven strategies")
}
}

287
pkg/agent/cot_templates.go Normal file
View file

@ -0,0 +1,287 @@
// PicoClaw - Ultra-lightweight personal AI agent
// License: MIT
//
// Copyright (c) 2026 PicoClaw contributors
package agent
import (
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"github.com/sipeed/picoclaw/pkg/logger"
)
// CotTemplate represents a Chain-of-Thought prompting template.
type CotTemplate struct {
ID string // Short identifier (e.g. "analytical", "code")
Name string // Human-readable name
Description string // One-line description for the pre-LLM to choose from
Prompt string // The actual CoT instruction injected into the system prompt
}
// --- Built-in CoT Templates -------------------------------------------------
var builtinCotTemplates = []CotTemplate{
{
ID: "direct",
Name: "Direct Answer",
Description: "Simple, direct response — no special reasoning needed",
Prompt: "", // No CoT injection for simple answers
},
{
ID: "analytical",
Name: "Analytical Reasoning",
Description: "Complex questions requiring step-by-step logical analysis",
Prompt: `## Thinking Strategy: Analytical Reasoning
Before answering, follow this reasoning process:
1. **Clarify** Restate the core question in your own words.
2. **Decompose** Break it into sub-problems or key aspects.
3. **Analyse** Work through each sub-problem with evidence/logic.
4. **Synthesise** Combine findings into a coherent answer.
5. **Verify** Check for logical gaps or contradictions.`,
},
{
ID: "code",
Name: "Code Analysis",
Description: "Writing, reviewing, or understanding code",
Prompt: `## Thinking Strategy: Code Analysis
Before writing or analysing code:
1. **Requirements** What exactly needs to be done?
2. **Inputs/Outputs** Define the interface: what goes in, what comes out.
3. **Edge Cases** Consider boundary conditions, errors, empty inputs, concurrency.
4. **Approach** Choose the algorithm/pattern, justify the choice.
5. **Implement** Write clean, well-commented code.
6. **Test** Mentally trace through with sample inputs to verify correctness.`,
},
{
ID: "debug",
Name: "Debugging",
Description: "Finding and fixing bugs, errors, or unexpected behaviour",
Prompt: `## Thinking Strategy: Debugging
Follow a systematic debugging approach:
1. **Reproduce** Understand the exact symptoms and conditions.
2. **Hypothesise** List 2-3 most likely root causes.
3. **Narrow Down** For each hypothesis, describe what evidence would confirm/deny it.
4. **Root Cause** Identify the actual root cause with evidence.
5. **Fix** Propose the minimal, targeted fix.
6. **Verify** Confirm the fix resolves the issue without side effects.`,
},
{
ID: "creative",
Name: "Creative Thinking",
Description: "Brainstorming, creative writing, idea generation",
Prompt: `## Thinking Strategy: Creative Exploration
Use divergent-convergent thinking:
1. **Diverge** Generate multiple distinct ideas or approaches without judgment.
2. **Explore** Expand on the most promising 2-3 ideas.
3. **Combine** Look for unexpected connections between ideas.
4. **Converge** Select the best approach and refine it.
5. **Polish** Add detail, nuance, and completeness.`,
},
{
ID: "task",
Name: "Task Planning",
Description: "Multi-step tasks, planning, project work",
Prompt: `## Thinking Strategy: Task Planning
Plan before executing:
1. **Goal** What is the desired end state?
2. **Current State** What exists now? What resources are available?
3. **Steps** Break into ordered, actionable steps.
4. **Dependencies** Identify which steps depend on others.
5. **Risks** What could go wrong? How to mitigate?
6. **Execute** Carry out steps, adapting as needed.`,
},
{
ID: "explain",
Name: "Explain / Teach",
Description: "Teaching concepts, explaining how things work",
Prompt: `## Thinking Strategy: Educational Explanation
Structure your explanation for clarity:
1. **Big Picture** Start with a one-sentence summary of the concept.
2. **Analogy** Relate to something familiar if possible.
3. **Core Mechanism** Explain how it works step by step.
4. **Example** Provide a concrete example or demonstration.
5. **Gotchas** Mention common misconceptions or pitfalls.`,
},
{
ID: "compare",
Name: "Comparison / Decision",
Description: "Comparing options, making decisions, trade-off analysis",
Prompt: `## Thinking Strategy: Comparison Analysis
Structure your analysis:
1. **Criteria** Define what matters most for this decision.
2. **Options** List all viable options.
3. **Trade-offs** For each option, list pros and cons against the criteria.
4. **Recommendation** State the best choice with clear reasoning.
5. **Caveats** Note when the recommendation might not apply.`,
},
}
// --- CoT Template Registry --------------------------------------------------
// CotRegistry manages the available CoT templates.
// It loads built-in templates and supports user-defined ones from workspace.
type CotRegistry struct {
mu sync.RWMutex
templates map[string]CotTemplate
}
// NewCotRegistry creates a registry with built-in templates and optionally
// loads user-defined templates from the workspace/cot_templates/ directory.
func NewCotRegistry(workspace string) *CotRegistry {
r := &CotRegistry{
templates: make(map[string]CotTemplate, len(builtinCotTemplates)),
}
// Register built-in templates.
for _, t := range builtinCotTemplates {
r.templates[t.ID] = t
}
// Load user-defined templates from workspace.
r.loadUserTemplates(workspace)
return r
}
// Get returns a template by ID (case-insensitive). Returns the "direct"
// template if not found.
func (r *CotRegistry) Get(id string) CotTemplate {
r.mu.RLock()
defer r.mu.RUnlock()
id = strings.ToLower(strings.TrimSpace(id))
if t, ok := r.templates[id]; ok {
return t
}
return r.templates["direct"]
}
// ListForPrompt returns a formatted list of available template IDs and
// descriptions, suitable for quick reference.
func (r *CotRegistry) ListForPrompt() string {
r.mu.RLock()
defer r.mu.RUnlock()
var sb strings.Builder
for _, t := range builtinCotTemplates {
fmt.Fprintf(&sb, "- %s: %s\n", t.ID, t.Description)
}
// Append user-defined templates.
for id, t := range r.templates {
isBuiltin := false
for _, bt := range builtinCotTemplates {
if bt.ID == id {
isBuiltin = true
break
}
}
if !isBuiltin {
fmt.Fprintf(&sb, "- %s: %s\n", t.ID, t.Description)
}
}
return sb.String()
}
// ListExamplesForPrompt returns full template examples for the pre-LLM to
// use as inspiration when generating custom CoT prompts.
// Shows 3-4 diverse examples with their full prompt content.
func (r *CotRegistry) ListExamplesForPrompt() string {
r.mu.RLock()
defer r.mu.RUnlock()
// Select a diverse set of examples (not all — keep prompt concise).
exampleIDs := []string{"analytical", "code", "debug", "task"}
var sb strings.Builder
for _, id := range exampleIDs {
t, ok := r.templates[id]
if !ok || t.Prompt == "" {
continue
}
fmt.Fprintf(&sb, "### Example: %s (%s)\n%s\n\n", t.Name, t.Description, t.Prompt)
}
// Append any user-defined templates as additional examples.
for id, t := range r.templates {
isBuiltin := false
for _, bt := range builtinCotTemplates {
if bt.ID == id {
isBuiltin = true
break
}
}
if !isBuiltin && t.Prompt != "" {
fmt.Fprintf(&sb, "### Example: %s (%s)\n%s\n\n", t.Name, t.Description, t.Prompt)
}
}
return sb.String()
}
// loadUserTemplates scans workspace/cot_templates/ for .md files.
// Each file becomes a template with ID = filename (without .md).
// File format:
//
// Line 1: description (one line)
// Line 2: ---
// Line 3+: prompt content
func (r *CotRegistry) loadUserTemplates(workspace string) {
dir := filepath.Join(workspace, "cot_templates")
entries, err := os.ReadDir(dir)
if err != nil {
return // Directory doesn't exist — that's fine.
}
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".md") {
continue
}
data, err := os.ReadFile(filepath.Join(dir, entry.Name()))
if err != nil {
continue
}
id := strings.TrimSuffix(entry.Name(), ".md")
id = strings.ToLower(strings.TrimSpace(id))
if id == "" {
continue
}
content := string(data)
description := id
prompt := content
// Parse optional description header.
if idx := strings.Index(content, "\n---\n"); idx > 0 {
description = strings.TrimSpace(content[:idx])
prompt = strings.TrimSpace(content[idx+5:])
}
r.mu.Lock()
r.templates[id] = CotTemplate{
ID: id,
Name: id,
Description: description,
Prompt: prompt,
}
r.mu.Unlock()
logger.DebugCF("cot", "Loaded user CoT template",
map[string]any{"id": id, "description": description})
}
}

View file

@ -0,0 +1,146 @@
package agent
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestCotRegistry_BuiltinTemplates(t *testing.T) {
dir := t.TempDir()
r := NewCotRegistry(dir)
// Should have all built-in templates.
for _, bt := range builtinCotTemplates {
tmpl := r.Get(bt.ID)
if tmpl.ID != bt.ID {
t.Errorf("expected template %q, got %q", bt.ID, tmpl.ID)
}
}
// "direct" should have empty prompt.
direct := r.Get("direct")
if direct.Prompt != "" {
t.Errorf("direct template should have empty prompt, got %q", direct.Prompt)
}
// "code" should have non-empty prompt.
code := r.Get("code")
if code.Prompt == "" {
t.Error("code template should have non-empty prompt")
}
if !strings.Contains(code.Prompt, "Code Analysis") {
t.Error("code template should mention 'Code Analysis'")
}
}
func TestCotRegistry_UnknownFallsToDefault(t *testing.T) {
dir := t.TempDir()
r := NewCotRegistry(dir)
tmpl := r.Get("nonexistent_template")
if tmpl.ID != "direct" {
t.Errorf("expected fallback to 'direct', got %q", tmpl.ID)
}
}
func TestCotRegistry_CaseInsensitive(t *testing.T) {
dir := t.TempDir()
r := NewCotRegistry(dir)
tmpl := r.Get(" Code ")
if tmpl.ID != "code" {
t.Errorf("expected 'code', got %q", tmpl.ID)
}
}
func TestCotRegistry_UserTemplates(t *testing.T) {
dir := t.TempDir()
// Create user template.
cotDir := filepath.Join(dir, "cot_templates")
os.MkdirAll(cotDir, 0o755)
content := `Custom strategy for data analysis
---
## Thinking Strategy: Data Analysis
1. Examine the data structure.
2. Identify patterns.
3. Draw conclusions.`
os.WriteFile(filepath.Join(cotDir, "data_analysis.md"), []byte(content), 0o644)
r := NewCotRegistry(dir)
// Should be able to get the user template.
tmpl := r.Get("data_analysis")
if tmpl.ID != "data_analysis" {
t.Errorf("expected 'data_analysis', got %q", tmpl.ID)
}
if tmpl.Description != "Custom strategy for data analysis" {
t.Errorf("description = %q, want 'Custom strategy for data analysis'", tmpl.Description)
}
if !strings.Contains(tmpl.Prompt, "Examine the data structure") {
t.Error("prompt should contain user-defined content")
}
}
func TestCotRegistry_ListForPrompt(t *testing.T) {
dir := t.TempDir()
r := NewCotRegistry(dir)
list := r.ListForPrompt()
// Should contain all built-in template IDs.
for _, bt := range builtinCotTemplates {
if !strings.Contains(list, bt.ID) {
t.Errorf("ListForPrompt missing template %q", bt.ID)
}
}
}
func TestCotRegistry_ListExamplesForPrompt(t *testing.T) {
dir := t.TempDir()
r := NewCotRegistry(dir)
examples := r.ListExamplesForPrompt()
// Should contain full example content for key templates.
if !strings.Contains(examples, "Code Analysis") {
t.Error("ListExamplesForPrompt missing 'Code Analysis' example")
}
if !strings.Contains(examples, "Analytical Reasoning") {
t.Error("ListExamplesForPrompt missing 'Analytical Reasoning' example")
}
if !strings.Contains(examples, "Debugging") {
t.Error("ListExamplesForPrompt missing 'Debugging' example")
}
// Should contain actual steps, not just names.
if !strings.Contains(examples, "Requirements") {
t.Error("ListExamplesForPrompt should include actual step content")
}
}
func TestCotRegistry_UserOverridesBuiltin(t *testing.T) {
dir := t.TempDir()
// Create a user template that overrides "code".
cotDir := filepath.Join(dir, "cot_templates")
os.MkdirAll(cotDir, 0o755)
content := `My custom code template
---
## Custom Code Strategy
Think differently about code.`
os.WriteFile(filepath.Join(cotDir, "code.md"), []byte(content), 0o644)
r := NewCotRegistry(dir)
tmpl := r.Get("code")
if !strings.Contains(tmpl.Prompt, "Think differently about code") {
t.Error("user template should override built-in 'code' template")
}
}

596
pkg/agent/executor.go Normal file
View file

@ -0,0 +1,596 @@
// PicoClaw - Ultra-lightweight personal AI agent
// License: MIT
//
// Copyright (c) 2026 PicoClaw contributors
package agent
// executor.go - Phase 2 (ExecuteLLM) logic extracted from loop.go.
// Contains the LLM iteration loop, tool handling, reasoning output,
// context compression, and logging helpers.
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"unicode/utf8"
"github.com/sipeed/picoclaw/pkg/bus"
"github.com/sipeed/picoclaw/pkg/constants"
"github.com/sipeed/picoclaw/pkg/logger"
"github.com/sipeed/picoclaw/pkg/providers"
"github.com/sipeed/picoclaw/pkg/tools"
"github.com/sipeed/picoclaw/pkg/utils"
)
func (al *AgentLoop) targetReasoningChannelID(channelName string) (chatID string) {
if al.channelManager == nil {
return ""
}
if ch, ok := al.channelManager.GetChannel(channelName); ok {
return ch.ReasoningChannelID()
}
return ""
}
func (al *AgentLoop) handleReasoning(ctx context.Context, reasoningContent, channelName, channelID string) {
if reasoningContent == "" || channelName == "" || channelID == "" {
return
}
// Check context cancellation before attempting to publish,
// since PublishOutbound's select may race between send and ctx.Done().
if ctx.Err() != nil {
return
}
// Use a short timeout so the goroutine does not block indefinitely when
// the outbound bus is full. Reasoning output is best-effort; dropping it
// is acceptable to avoid goroutine accumulation.
pubCtx, pubCancel := context.WithTimeout(ctx, 5*time.Second)
defer pubCancel()
if err := al.bus.PublishOutbound(pubCtx, bus.OutboundMessage{
Channel: channelName,
ChatID: channelID,
Content: reasoningContent,
}); err != nil {
// Treat context.DeadlineExceeded / context.Canceled as expected
// (bus full under load, or parent canceled). Check the error
// itself rather than ctx.Err(), because pubCtx may time out
// (5 s) while the parent ctx is still active.
// Also treat ErrBusClosed as expected — it occurs during normal
// shutdown when the bus is closed before all goroutines finish.
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) ||
errors.Is(err, bus.ErrBusClosed) {
logger.DebugCF("agent", "Reasoning publish skipped (timeout/cancel)", map[string]any{
"channel": channelName,
"error": err.Error(),
})
} else {
logger.WarnCF("agent", "Failed to publish reasoning (best-effort)", map[string]any{
"channel": channelName,
"error": err.Error(),
})
}
}
}
// runLLMIteration executes the LLM call loop with tool handling.
func (al *AgentLoop) runLLMIteration(
ctx context.Context,
agent *AgentInstance,
messages []providers.Message,
opts processOptions,
) (string, int, []ToolCallRecord, error) {
iteration := 0
var finalContent string
var toolRecords []ToolCallRecord
for iteration < agent.MaxIterations {
iteration++
logger.DebugCF("agent", "LLM iteration",
map[string]any{
"seq": opts.MsgSeqId,
"agent_id": agent.ID,
"iteration": iteration,
"max": agent.MaxIterations,
})
// Build tool definitions
providerToolDefs := agent.Tools.ToProviderDefs()
// Log LLM request details
logger.DebugCF("agent", "LLM request",
map[string]any{
"seq": opts.MsgSeqId,
"agent_id": agent.ID,
"iteration": iteration,
"model": agent.Model,
"messages_count": len(messages),
"tools_count": len(providerToolDefs),
"max_tokens": agent.MaxTokens,
"temperature": agent.Temperature,
"system_prompt_len": len(messages[0].Content),
})
// Log full messages (detailed)
logger.DebugCF("agent", "Full LLM request",
map[string]any{
"iteration": iteration,
"messages_json": formatMessagesForLog(messages),
"tools_json": formatToolsForLog(providerToolDefs),
})
// Call LLM with fallback chain if candidates are configured.
var response *providers.LLMResponse
var err error
callLLM := func() (*providers.LLMResponse, error) {
if len(agent.Candidates) > 1 && al.fallback != nil {
fbResult, fbErr := al.fallback.Execute(ctx, agent.Candidates,
func(ctx context.Context, provider, model string) (*providers.LLMResponse, error) {
return agent.Provider.Chat(ctx, messages, providerToolDefs, model, map[string]any{
"max_tokens": agent.MaxTokens,
"temperature": agent.Temperature,
"prompt_cache_key": agent.ID,
})
},
)
if fbErr != nil {
return nil, fbErr
}
if fbResult.Provider != "" && len(fbResult.Attempts) > 0 {
logger.InfoCF("agent", fmt.Sprintf("Fallback: succeeded with %s/%s after %d attempts",
fbResult.Provider, fbResult.Model, len(fbResult.Attempts)+1),
map[string]any{"agent_id": agent.ID, "iteration": iteration})
}
return fbResult.Response, nil
}
return agent.Provider.Chat(ctx, messages, providerToolDefs, agent.Model, map[string]any{
"max_tokens": agent.MaxTokens,
"temperature": agent.Temperature,
"prompt_cache_key": agent.ID,
})
}
// Retry loop for context/token errors
maxRetries := 2
for retry := 0; retry <= maxRetries; retry++ {
response, err = callLLM()
if err == nil {
break
}
errMsg := strings.ToLower(err.Error())
// Check if this is a network/HTTP timeout — not a context window error.
isTimeoutError := errors.Is(err, context.DeadlineExceeded) ||
strings.Contains(errMsg, "deadline exceeded") ||
strings.Contains(errMsg, "client.timeout") ||
strings.Contains(errMsg, "timed out") ||
strings.Contains(errMsg, "timeout exceeded")
// Detect real context window / token limit errors, excluding network timeouts.
isContextError := !isTimeoutError && (strings.Contains(errMsg, "context_length_exceeded") ||
strings.Contains(errMsg, "context window") ||
strings.Contains(errMsg, "maximum context length") ||
strings.Contains(errMsg, "token limit") ||
strings.Contains(errMsg, "too many tokens") ||
strings.Contains(errMsg, "max_tokens") ||
strings.Contains(errMsg, "invalidparameter") ||
strings.Contains(errMsg, "prompt is too long") ||
strings.Contains(errMsg, "request too large"))
if isTimeoutError && retry < maxRetries {
backoff := time.Duration(retry+1) * 5 * time.Second
logger.WarnCF("agent", "Timeout error, retrying after backoff", map[string]any{
"error": err.Error(),
"retry": retry,
"backoff": backoff.String(),
})
time.Sleep(backoff)
continue
}
if isContextError && retry < maxRetries {
logger.WarnCF("agent", "Context window error detected, attempting compression", map[string]any{
"error": err.Error(),
"retry": retry,
})
if retry == 0 && !constants.IsInternalChannel(opts.Channel) {
al.bus.PublishOutbound(ctx, bus.OutboundMessage{
Channel: opts.Channel,
ChatID: opts.ChatID,
Content: "Context window exceeded. Compressing history and retrying...",
})
}
al.forceCompression(agent, opts.SessionKey)
newHistory := agent.Sessions.GetHistory(opts.SessionKey)
newSummary := agent.Sessions.GetSummary(opts.SessionKey)
messages = agent.ContextBuilder.BuildMessages(
newHistory, newSummary, "",
nil, opts.Channel, opts.ChatID,
)
continue
}
break
}
if err != nil {
logger.ErrorCF("agent", "LLM call failed",
map[string]any{
"seq": opts.MsgSeqId,
"agent_id": agent.ID,
"iteration": iteration,
"error": err.Error(),
})
return "", iteration, toolRecords, fmt.Errorf("LLM call failed after retries: %w", err)
}
go al.handleReasoning(ctx, response.Reasoning, opts.Channel, al.targetReasoningChannelID(opts.Channel))
logger.DebugCF("agent", "LLM response",
map[string]any{
"seq": opts.MsgSeqId,
"agent_id": agent.ID,
"iteration": iteration,
"content_chars": len(response.Content),
"tool_calls": len(response.ToolCalls),
"reasoning": response.Reasoning,
"target_channel": al.targetReasoningChannelID(opts.Channel),
"channel": opts.Channel,
})
// Check if no tool calls - we're done
if len(response.ToolCalls) == 0 {
finalContent = response.Content
logger.InfoCF("agent", "LLM response without tool calls (direct answer)",
map[string]any{
"seq": opts.MsgSeqId,
"agent_id": agent.ID,
"iteration": iteration,
"content_chars": len(finalContent),
})
break
}
normalizedToolCalls := make([]providers.ToolCall, 0, len(response.ToolCalls))
for _, tc := range response.ToolCalls {
normalizedToolCalls = append(normalizedToolCalls, providers.NormalizeToolCall(tc))
}
// Log tool calls
toolNames := make([]string, 0, len(normalizedToolCalls))
for _, tc := range normalizedToolCalls {
toolNames = append(toolNames, tc.Name)
}
logger.InfoCF("agent", "LLM requested tool calls",
map[string]any{
"agent_id": agent.ID,
"tools": toolNames,
"count": len(normalizedToolCalls),
"iteration": iteration,
})
// Build assistant message with tool calls
assistantMsg := providers.Message{
Role: "assistant",
Content: response.Content,
ReasoningContent: response.ReasoningContent,
}
for _, tc := range normalizedToolCalls {
argumentsJSON, _ := json.Marshal(tc.Arguments)
// Copy ExtraContent to ensure thought_signature is persisted for Gemini 3
extraContent := tc.ExtraContent
thoughtSignature := ""
if tc.Function != nil {
thoughtSignature = tc.Function.ThoughtSignature
}
assistantMsg.ToolCalls = append(assistantMsg.ToolCalls, providers.ToolCall{
ID: tc.ID,
Type: "function",
Name: tc.Name,
Function: &providers.FunctionCall{
Name: tc.Name,
Arguments: string(argumentsJSON),
ThoughtSignature: thoughtSignature,
},
ExtraContent: extraContent,
ThoughtSignature: thoughtSignature,
})
}
messages = append(messages, assistantMsg)
// Save assistant message with tool calls to session
agent.Sessions.AddFullMessage(opts.SessionKey, assistantMsg)
// Execute tool calls
for _, tc := range normalizedToolCalls {
argsJSON, _ := json.Marshal(tc.Arguments)
argsPreview := utils.Truncate(string(argsJSON), 200)
logger.InfoCF("agent", fmt.Sprintf("Tool call: %s(%s)", tc.Name, argsPreview),
map[string]any{
"agent_id": agent.ID,
"tool": tc.Name,
"iteration": iteration,
})
// Create async callback for tools that implement AsyncTool
// NOTE: Following openclaw's design, async tools do NOT send results directly to users.
// Instead, they notify the agent via PublishInbound, and the agent decides
// whether to forward the result to the user (in processSystemMessage).
asyncCallback := func(callbackCtx context.Context, result *tools.ToolResult) {
// Log the async completion but don't send directly to user
// The agent will handle user notification via processSystemMessage
if !result.Silent && result.ForUser != "" {
logger.InfoCF("agent", "Async tool completed, agent will handle notification",
map[string]any{
"tool": tc.Name,
"content_len": len(result.ForUser),
})
}
}
toolStart := time.Now()
toolResult := agent.Tools.ExecuteWithContext(
ctx,
tc.Name,
tc.Arguments,
opts.Channel,
opts.ChatID,
asyncCallback,
)
toolDuration := time.Since(toolStart)
// Record tool call for post-LLM processors.
record := ToolCallRecord{Name: tc.Name, Duration: toolDuration}
if toolResult.Err != nil {
record.Error = toolResult.Err.Error()
}
toolRecords = append(toolRecords, record)
// Send ForUser content to user immediately if not Silent
if !toolResult.Silent && toolResult.ForUser != "" && opts.SendResponse {
al.bus.PublishOutbound(ctx, bus.OutboundMessage{
Channel: opts.Channel,
ChatID: opts.ChatID,
Content: toolResult.ForUser,
})
logger.DebugCF("agent", "Sent tool result to user",
map[string]any{
"tool": tc.Name,
"content_len": len(toolResult.ForUser),
})
}
// If tool returned media refs, publish them as outbound media
if len(toolResult.Media) > 0 && opts.SendResponse {
parts := make([]bus.MediaPart, 0, len(toolResult.Media))
for _, ref := range toolResult.Media {
part := bus.MediaPart{Ref: ref}
// Populate metadata from MediaStore when available
if al.mediaStore != nil {
if _, meta, err := al.mediaStore.ResolveWithMeta(ref); err == nil {
part.Filename = meta.Filename
part.ContentType = meta.ContentType
part.Type = inferMediaType(meta.Filename, meta.ContentType)
}
}
parts = append(parts, part)
}
al.bus.PublishOutboundMedia(ctx, bus.OutboundMediaMessage{
Channel: opts.Channel,
ChatID: opts.ChatID,
Parts: parts,
})
}
// Determine content for LLM based on tool result
contentForLLM := toolResult.ForLLM
if contentForLLM == "" && toolResult.Err != nil {
contentForLLM = toolResult.Err.Error()
}
toolResultMsg := providers.Message{
Role: "tool",
Content: contentForLLM,
ToolCallID: tc.ID,
}
messages = append(messages, toolResultMsg)
// Save tool result message to session
agent.Sessions.AddFullMessage(opts.SessionKey, toolResultMsg)
}
}
return finalContent, iteration, toolRecords, nil
}
// updateToolContexts updates the context for tools that need channel/chatID info.
func (al *AgentLoop) updateToolContexts(agent *AgentInstance, channel, chatID string) {
// Use ContextualTool interface instead of type assertions
if tool, ok := agent.Tools.Get("message"); ok {
if mt, ok := tool.(tools.ContextualTool); ok {
mt.SetContext(channel, chatID)
}
}
if tool, ok := agent.Tools.Get("spawn"); ok {
if st, ok := tool.(tools.ContextualTool); ok {
st.SetContext(channel, chatID)
}
}
if tool, ok := agent.Tools.Get("subagent"); ok {
if st, ok := tool.(tools.ContextualTool); ok {
st.SetContext(channel, chatID)
}
}
}
// maybeSummarize triggers summarization if the session history exceeds thresholds.
func (al *AgentLoop) maybeSummarize(agent *AgentInstance, sessionKey, channel, chatID string) {
newHistory := agent.Sessions.GetHistory(sessionKey)
tokenEstimate := al.estimateTokens(newHistory)
threshold := agent.ContextWindow * 75 / 100
if len(newHistory) > 20 || tokenEstimate > threshold {
summarizeKey := agent.ID + ":" + sessionKey
if _, loading := al.summarizing.LoadOrStore(summarizeKey, true); !loading {
go func() {
defer al.summarizing.Delete(summarizeKey)
logger.Debug("Memory threshold reached. Optimizing conversation history...")
al.summarizeSession(agent, sessionKey)
}()
}
}
}
// forceCompression aggressively reduces context when the limit is hit.
// It drops the oldest 50% of messages (keeping system prompt and last user message).
func (al *AgentLoop) forceCompression(agent *AgentInstance, sessionKey string) {
history := agent.Sessions.GetHistory(sessionKey)
if len(history) <= 4 {
return
}
// Keep system prompt (usually [0]) and the very last message (user's trigger)
// We want to drop the oldest half of the *conversation*
// Assuming [0] is system, [1:] is conversation
conversation := history[1 : len(history)-1]
if len(conversation) == 0 {
return
}
// Helper to find the mid-point of the conversation
mid := len(conversation) / 2
// New history structure:
// 1. System Prompt (with compression note appended)
// 2. Second half of conversation
// 3. Last message
droppedCount := mid
keptConversation := conversation[mid:]
newHistory := make([]providers.Message, 0, 1+len(keptConversation)+1)
// Append compression note to the original system prompt instead of adding a new system message
// This avoids having two consecutive system messages which some APIs (like Zhipu) reject
compressionNote := fmt.Sprintf(
"\n\n[System Note: Emergency compression dropped %d oldest messages due to context limit]",
droppedCount,
)
enhancedSystemPrompt := history[0]
enhancedSystemPrompt.Content = enhancedSystemPrompt.Content + compressionNote
newHistory = append(newHistory, enhancedSystemPrompt)
newHistory = append(newHistory, keptConversation...)
newHistory = append(newHistory, history[len(history)-1]) // Last message
// Update session
agent.Sessions.SetHistory(sessionKey, newHistory)
agent.Sessions.Save(sessionKey)
logger.WarnCF("agent", "Forced compression executed", map[string]any{
"session_key": sessionKey,
"dropped_msgs": droppedCount,
"new_count": len(newHistory),
})
}
// GetStartupInfo returns information about loaded tools and skills for logging.
func (al *AgentLoop) GetStartupInfo() map[string]any {
info := make(map[string]any)
agent := al.registry.GetDefaultAgent()
if agent == nil {
return info
}
// Tools info
toolsList := agent.Tools.List()
info["tools"] = map[string]any{
"count": len(toolsList),
"names": toolsList,
}
// Skills info
info["skills"] = agent.ContextBuilder.GetSkillsInfo()
// Agents info
info["agents"] = map[string]any{
"count": len(al.registry.ListAgentIDs()),
"ids": al.registry.ListAgentIDs(),
}
return info
}
// formatMessagesForLog formats messages for logging
func formatMessagesForLog(messages []providers.Message) string {
if len(messages) == 0 {
return "[]"
}
var sb strings.Builder
sb.WriteString("[\n")
for i, msg := range messages {
fmt.Fprintf(&sb, " [%d] Role: %s\n", i, msg.Role)
if len(msg.ToolCalls) > 0 {
sb.WriteString(" ToolCalls:\n")
for _, tc := range msg.ToolCalls {
fmt.Fprintf(&sb, " - ID: %s, Type: %s, Name: %s\n", tc.ID, tc.Type, tc.Name)
if tc.Function != nil {
fmt.Fprintf(&sb, " Arguments: %s\n", utils.Truncate(tc.Function.Arguments, 200))
}
}
}
if msg.Content != "" {
content := utils.Truncate(msg.Content, 200)
fmt.Fprintf(&sb, " Content: %s\n", content)
}
if msg.ToolCallID != "" {
fmt.Fprintf(&sb, " ToolCallID: %s\n", msg.ToolCallID)
}
sb.WriteString("\n")
}
sb.WriteString("]")
return sb.String()
}
// formatToolsForLog formats tool definitions for logging
func formatToolsForLog(toolDefs []providers.ToolDefinition) string {
if len(toolDefs) == 0 {
return "[]"
}
var sb strings.Builder
sb.WriteString("[\n")
for i, tool := range toolDefs {
fmt.Fprintf(&sb, " [%d] Type: %s, Name: %s\n", i, tool.Type, tool.Function.Name)
fmt.Fprintf(&sb, " Description: %s\n", tool.Function.Description)
if len(tool.Function.Parameters) > 0 {
fmt.Fprintf(&sb, " Parameters: %s\n", utils.Truncate(fmt.Sprintf("%v", tool.Function.Parameters), 200))
}
}
sb.WriteString("]")
return sb.String()
}
// estimateTokens estimates the number of tokens in a message list.
// Uses a safe heuristic of 2.5 characters per token to account for CJK and other
// overheads better than the previous 3 chars/token.
func (al *AgentLoop) estimateTokens(messages []providers.Message) int {
totalChars := 0
for _, m := range messages {
totalChars += utf8.RuneCountInString(m.Content)
}
// 2.5 chars per token = totalChars * 2 / 5
return totalChars * 2 / 5
}

View file

@ -34,6 +34,8 @@ type AgentInstance struct {
Subagents *config.SubagentsConfig Subagents *config.SubagentsConfig
SkillsFilter []string SkillsFilter []string
Candidates []providers.FallbackCandidate Candidates []providers.FallbackCandidate
Analyser *Analyser // Phase 1: intent/tag analysis
Reflector *Reflector // Phase 3: post-LLM processing + slash commands
} }
// NewAgentInstance creates an agent instance from config. // NewAgentInstance creates an agent instance from config.
@ -148,6 +150,22 @@ func NewAgentInstance(
candidates := providers.ResolveCandidatesWithLookup(modelCfg, defaults.Provider, resolveFromModelList) candidates := providers.ResolveCandidatesWithLookup(modelCfg, defaults.Provider, resolveFromModelList)
// Initialise optional Phase 1 analyser for intent/tag-based memory retrieval and CoT selection.
// Uses GetAnalyserModel() which resolves: analyser_model → pre_llm_model → model_name.
var analyser *Analyser
var rt *Reflector
analyserModel := defaults.GetAnalyserModel()
if analyserModel != "" {
cotRegistry := NewCotRegistry(workspace)
analyser = NewAnalyser(provider, analyserModel, cotRegistry)
rt = NewReflector(provider, analyserModel)
log.Printf("Analyser + Reflector enabled for agent %s (model: %s)", agentID, analyserModel)
} else {
// Reflector without LLM processors (just commands + error tracker).
rt = NewReflector(nil, "")
}
rt.SetTools(toolsRegistry)
return &AgentInstance{ return &AgentInstance{
ID: agentID, ID: agentID,
Name: agentName, Name: agentName,
@ -165,6 +183,8 @@ func NewAgentInstance(
Subagents: subagents, Subagents: subagents,
SkillsFilter: skillsFilter, SkillsFilter: skillsFilter,
Candidates: candidates, Candidates: candidates,
Analyser: analyser,
Reflector: rt,
} }
} }

252
pkg/agent/instant_memory.go Normal file
View file

@ -0,0 +1,252 @@
// PicoClaw - Ultra-lightweight personal AI agent
// License: MIT
//
// Copyright (c) 2026 PicoClaw contributors
package agent
import (
"fmt"
"strings"
"github.com/sipeed/picoclaw/pkg/logger"
"github.com/sipeed/picoclaw/pkg/providers"
)
// ---------------------------------------------------------------------------
// Instant Memory — dynamic Turn selection for Phase 2 context
// ---------------------------------------------------------------------------
// InstantMemoryCfg holds tunable parameters for instant-memory assembly.
type InstantMemoryCfg struct {
HighScoreThreshold int // turns with score >= this are always_keep (default: 7)
RecentCount int // number of recent turns to include (default: 5)
MaxTokenRatio float64 // fraction of contextWindow budget (default: 0.6)
ContextWindow int // total context window in tokens
}
// DefaultInstantMemoryCfg returns a sensible default config.
func DefaultInstantMemoryCfg(contextWindow int) InstantMemoryCfg {
return InstantMemoryCfg{
HighScoreThreshold: alwaysKeepThreshold, // 7
RecentCount: 5,
MaxTokenRatio: 0.6,
ContextWindow: contextWindow,
}
}
// BuildInstantMemory assembles the filtered set of historical turns for Phase 2.
//
// Selection rules (from design doc):
//
// 瞬时记忆 =
// { Turn | score >= highThreshold } // always_keep
// { Turn | tags ∩ currentTags ≠ ∅, score > 0 } // tag-matched
// { 最近 M 个 Turn } // recency guarantee
// → deduplicate by ID
// → sort by ts ASC
// → truncate to token budget
func BuildInstantMemory(
store *TurnStore,
currentTags []string,
channelKey string,
cfg InstantMemoryCfg,
) []TurnRecord {
if store == nil {
return nil
}
seen := make(map[string]struct{})
var all []TurnRecord
addUnique := func(turns []TurnRecord) {
for _, t := range turns {
if _, dup := seen[t.ID]; dup {
continue
}
seen[t.ID] = struct{}{}
all = append(all, t)
}
}
// 1. always_keep: high-score turns.
high, err := store.QueryByScore(cfg.HighScoreThreshold)
if err != nil {
logger.WarnCF("instant_memory", "QueryByScore failed", map[string]any{"error": err.Error()})
} else {
addUnique(high)
}
// 2. tag-matched turns (score > 0).
if len(currentTags) > 0 {
tagged, err := store.QueryByTags(currentTags)
if err != nil {
logger.WarnCF("instant_memory", "QueryByTags failed", map[string]any{"error": err.Error()})
} else {
addUnique(tagged)
}
}
// 3. Recent M turns for continuity.
recent, err := store.QueryRecent(channelKey, cfg.RecentCount)
if err != nil {
logger.WarnCF("instant_memory", "QueryRecent failed", map[string]any{"error": err.Error()})
} else {
addUnique(recent)
}
// Sort by ts ASC (stable chronological order).
sortTurnsByTs(all)
// Truncate to token budget.
maxTokens := int(float64(cfg.ContextWindow) * cfg.MaxTokenRatio)
if maxTokens > 0 {
all = truncateToTokenBudget(all, maxTokens)
}
logger.DebugCF("instant_memory", "Built instant memory",
map[string]any{
"total": len(all),
"high_score": len(high),
"tag_matched": len(currentTags),
"recent": len(recent),
"max_tokens": maxTokens,
})
return all
}
// sortTurnsByTs sorts turns in ascending timestamp order (oldest first).
func sortTurnsByTs(turns []TurnRecord) {
// Simple in-place insertion sort — good enough for small N (<100).
for i := 1; i < len(turns); i++ {
key := turns[i]
j := i - 1
for j >= 0 && turns[j].Ts > key.Ts {
turns[j+1] = turns[j]
j--
}
turns[j+1] = key
}
}
// truncateToTokenBudget trims turns from the oldest end until total tokens fit.
// Returns a suffix of the sorted slice (preserving newest turns).
func truncateToTokenBudget(turns []TurnRecord, maxTokens int) []TurnRecord {
total := 0
for _, t := range turns {
total += t.Tokens
}
if total <= maxTokens {
return turns
}
// Drop oldest turns first until we fit.
for len(turns) > 0 && total > maxTokens {
total -= turns[0].Tokens
turns = turns[1:]
}
return turns
}
// ---------------------------------------------------------------------------
// Phase 2 Message Assembly — KV Cache friendly ordering
// ---------------------------------------------------------------------------
// BuildPhase2Messages constructs the message array for Phase 2 (ExecuteLLM)
// in KV-cache-friendly order:
//
// [system_prompt] ← always cache hit
// [long_term_memory by tags] ← same tags = cache hit (cache_control: ephemeral)
// [always_keep turns (score≥7)] ← fixed position, append only → cache hit
// [tag_matched turns] ← per-turn, ts ASC
// [recent_M turns] ← rolling window
// [current_user_message] ← always new
//
// Each historical turn is represented as a user/assistant message pair.
func BuildPhase2Messages(
systemPrompt string,
longTermMemory string,
turns []TurnRecord,
userMessage string,
highScoreThreshold int,
) []providers.Message {
msgs := make([]providers.Message, 0, 2+len(turns)*2+1)
// 1. System prompt (always first, stable prefix).
msgs = append(msgs, providers.Message{
Role: "system",
Content: systemPrompt,
})
// 2. Long-term memory (injected as system-adjacent user message).
// Mark with CacheControl if present (Anthropic will use it; others ignore).
if longTermMemory != "" {
msgs = append(msgs, providers.Message{
Role: "user",
Content: fmt.Sprintf("# Long-term Memory\n\n%s", longTermMemory),
})
// Need a brief assistant ack to maintain user/assistant alternation.
msgs = append(msgs, providers.Message{
Role: "assistant",
Content: "Understood, I'll use this context.",
})
}
// 3. Historical turns in KV-cache-friendly order:
// - always_keep first (fixed position)
// - then tag_matched + recent (may shift between requests)
//
// All turns are already sorted by ts ASC from BuildInstantMemory.
// We separate them into always_keep vs rest, keeping relative order.
var alwaysKeep, rest []TurnRecord
for _, t := range turns {
if t.Score >= highScoreThreshold {
alwaysKeep = append(alwaysKeep, t)
} else {
rest = append(rest, t)
}
}
// Append always_keep turns (cache-stable region).
for _, t := range alwaysKeep {
msgs = appendTurnMessages(msgs, t)
}
// Append remaining turns (tag-matched + recent, may shift).
for _, t := range rest {
msgs = appendTurnMessages(msgs, t)
}
// 4. Current user message (always last, always new).
msgs = append(msgs, providers.Message{
Role: "user",
Content: userMessage,
})
return msgs
}
// appendTurnMessages appends a user/assistant pair for a historical turn.
func appendTurnMessages(msgs []providers.Message, t TurnRecord) []providers.Message {
// Build user message with metadata prefix.
var userContent strings.Builder
if t.Intent != "" || len(t.Tags) > 0 {
fmt.Fprintf(&userContent, "[turn intent=%s tags=%v]\n", t.Intent, t.Tags)
}
userContent.WriteString(t.UserMsg)
msgs = append(msgs, providers.Message{
Role: "user",
Content: userContent.String(),
})
if t.Reply != "" {
msgs = append(msgs, providers.Message{
Role: "assistant",
Content: t.Reply,
})
}
return msgs
}

View file

@ -0,0 +1,164 @@
package agent
import (
"strings"
"testing"
"time"
)
func TestBuildInstantMemory_BasicAssembly(t *testing.T) {
dir := t.TempDir()
store, err := NewTurnStore(dir)
if err != nil {
t.Fatalf("NewTurnStore: %v", err)
}
defer store.Close()
now := time.Now().Unix()
// High-score turn (always_keep).
store.Insert(TurnRecord{ID: "t1", Ts: now - 100, Score: 9, ChannelKey: "cli:direct",
Intent: "code", Tags: []string{"refactor"}, UserMsg: "refactor it", Reply: strings.Repeat("x", 300)})
// Low-score irrelevant turn.
store.Insert(TurnRecord{ID: "t2", Ts: now - 80, Score: 2, ChannelKey: "cli:direct",
Intent: "chat", Tags: []string{"chat"}, UserMsg: "hi", Reply: "hello"})
// Tag-matched turn, moderate score.
store.Insert(TurnRecord{ID: "t3", Ts: now - 60, Score: 5, ChannelKey: "cli:direct",
Intent: "task", Tags: []string{"deploy", "ci"}, UserMsg: "deploy staging", Reply: "done"})
// Recent turns.
store.Insert(TurnRecord{ID: "t4", Ts: now - 20, Score: 3, ChannelKey: "cli:direct",
Intent: "question", Tags: []string{"api"}, UserMsg: "what's the api?", Reply: "check docs"})
store.Insert(TurnRecord{ID: "t5", Ts: now - 10, Score: 4, ChannelKey: "cli:direct",
Intent: "task", Tags: []string{"test"}, UserMsg: "run tests", Reply: "all passed"})
cfg := InstantMemoryCfg{
HighScoreThreshold: 7,
RecentCount: 3,
MaxTokenRatio: 0.6,
ContextWindow: 100000,
}
turns := BuildInstantMemory(store, []string{"deploy"}, "cli:direct", cfg)
// Should include: t1 (high-score), t3 (tag-match "deploy"), t4/t5 (recent 3 → also t3)
if len(turns) < 3 {
t.Errorf("expected at least 3 turns, got %d", len(turns))
for _, tt := range turns {
t.Logf(" turn: id=%s score=%d tags=%v", tt.ID, tt.Score, tt.Tags)
}
}
// Should be sorted by ts ASC.
for i := 1; i < len(turns); i++ {
if turns[i].Ts < turns[i-1].Ts {
t.Errorf("turns not sorted: turns[%d].Ts=%d < turns[%d].Ts=%d",
i, turns[i].Ts, i-1, turns[i-1].Ts)
}
}
// t1 (always_keep) must be present.
found := false
for _, tt := range turns {
if tt.ID == "t1" {
found = true
}
}
if !found {
t.Error("expected always_keep turn t1 to be included")
}
// t2 (low-score, no tag match, not recent enough) should be excluded.
for _, tt := range turns {
if tt.ID == "t2" {
t.Error("expected low-score irrelevant turn t2 to be excluded")
}
}
}
func TestBuildInstantMemory_NilStore(t *testing.T) {
turns := BuildInstantMemory(nil, []string{"deploy"}, "cli:direct", DefaultInstantMemoryCfg(8192))
if turns != nil {
t.Errorf("expected nil, got %v", turns)
}
}
func TestBuildPhase2Messages_Ordering(t *testing.T) {
turns := []TurnRecord{
{ID: "t1", Ts: 100, Score: 9, Intent: "code", Tags: []string{"refactor"},
UserMsg: "refactor it", Reply: "done refactoring", Tokens: 20},
{ID: "t2", Ts: 200, Score: 3, Intent: "question",
UserMsg: "what next?", Reply: "do X", Tokens: 10},
{ID: "t3", Ts: 300, Score: 8, Intent: "debug", Tags: []string{"deploy"},
UserMsg: "fix deploy", Reply: "fixed", Tokens: 10},
}
msgs := BuildPhase2Messages("You are a helpful assistant.", "User prefers Go.", turns, "hello world", 7)
// Expected order:
// [0] system
// [1] user (long_term_memory)
// [2] assistant (ack)
// [3,4] always_keep t1 (user/assistant)
// [5,6] always_keep t3 (user/assistant)
// [7,8] rest t2 (user/assistant)
// [9] current user message
if len(msgs) < 5 {
t.Fatalf("expected at least 5 messages, got %d", len(msgs))
}
if msgs[0].Role != "system" {
t.Errorf("msgs[0].Role = %s, want system", msgs[0].Role)
}
// Last message should be the current user message.
last := msgs[len(msgs)-1]
if last.Role != "user" || last.Content != "hello world" {
t.Errorf("last message = %+v, want user 'hello world'", last)
}
// All messages should alternate user/assistant (after system).
for i := 1; i < len(msgs)-1; i++ {
expected := "user"
if i%2 == 0 {
expected = "assistant"
}
if msgs[i].Role != expected {
t.Errorf("msgs[%d].Role = %s, want %s (content: %s)",
i, msgs[i].Role, expected, msgs[i].Content[:min(len(msgs[i].Content), 30)])
}
}
}
func TestBuildPhase2Messages_NoHistory(t *testing.T) {
msgs := BuildPhase2Messages("sys prompt", "", nil, "hi", 7)
// Should have: system + user message = 2
if len(msgs) != 2 {
t.Errorf("expected 2 messages, got %d", len(msgs))
}
if msgs[0].Role != "system" || msgs[1].Role != "user" {
t.Errorf("unexpected roles: %s, %s", msgs[0].Role, msgs[1].Role)
}
}
func TestTruncateToTokenBudget(t *testing.T) {
turns := []TurnRecord{
{ID: "a", Tokens: 100},
{ID: "b", Tokens: 200},
{ID: "c", Tokens: 300},
{ID: "d", Tokens: 150},
}
result := truncateToTokenBudget(turns, 500)
// Total = 750, budget = 500. Drop oldest first.
// Drop "a" (100) → 650, still over.
// Drop "b" (200) → 450, fits.
if len(result) != 2 {
t.Errorf("expected 2 turns, got %d", len(result))
}
if result[0].ID != "c" || result[1].ID != "d" {
t.Errorf("expected [c, d], got [%s, %s]", result[0].ID, result[1].ID)
}
}

View file

@ -0,0 +1,239 @@
// PicoClaw - Ultra-lightweight personal AI agent
// License: MIT
//
// Copyright (c) 2026 PicoClaw contributors
package agent
import (
"context"
"strings"
"testing"
"time"
)
// ---------------------------------------------------------------------------
// M5 Integration — TurnStore → BuildInstantMemory → BuildPhase2Messages
// ---------------------------------------------------------------------------
// TestInstantMemoryIntegration_EndToEnd inserts realistic turns into a real
// TurnStore, runs BuildInstantMemory with tag filtering, then assembles Phase 2
// messages and validates:
// - correct message ordering (system → memory → always_keep → rest → user)
// - strict user/assistant role alternation after the system message
// - always_keep turns appear before lower-score turns
// - the current user message is always last
func TestInstantMemoryIntegration_EndToEnd(t *testing.T) {
dir := t.TempDir()
store, err := NewTurnStore(dir)
if err != nil {
t.Fatalf("NewTurnStore: %v", err)
}
defer store.Close()
now := time.Now().Unix()
// Seed realistic turns.
turns := []TurnRecord{
{ID: "turn-1", Ts: now - 3600, Score: 10, ChannelKey: "cli:main",
Intent: "task", Tags: []string{"deploy", "ci"},
UserMsg: "Deploy to staging", Reply: "Deployed successfully to staging environment.",
Tokens: 50},
{ID: "turn-2", Ts: now - 3000, Score: 2, ChannelKey: "cli:main",
Intent: "chat", Tags: []string{"chat"},
UserMsg: "hi", Reply: "Hello!",
Tokens: 10},
{ID: "turn-3", Ts: now - 2000, Score: 6, ChannelKey: "cli:main",
Intent: "code", Tags: []string{"golang", "refactor"},
UserMsg: "Refactor the handler", Reply: "Done, split into 3 functions.",
Tokens: 40},
{ID: "turn-4", Ts: now - 500, Score: 4, ChannelKey: "cli:main",
Intent: "question", Tags: []string{"api"},
UserMsg: "What's the endpoint for users?", Reply: "GET /api/v1/users",
Tokens: 20},
{ID: "turn-5", Ts: now - 100, Score: 3, ChannelKey: "cli:main",
Intent: "task", Tags: []string{"test"},
UserMsg: "Run all tests", Reply: "All 42 tests passed.",
Tokens: 15},
}
for _, tr := range turns {
if err := store.Insert(tr); err != nil {
t.Fatalf("Insert(%s): %v", tr.ID, err)
}
}
// Query with tags=["deploy"] — should get turn-1 (always_keep + tag match),
// turn-3/4/5 (recent 3). turn-2 is low score, no tag match, not recent.
cfg := InstantMemoryCfg{
HighScoreThreshold: 7,
RecentCount: 3,
MaxTokenRatio: 0.6,
ContextWindow: 100000,
}
selected := BuildInstantMemory(store, []string{"deploy"}, "cli:main", cfg)
// Verify turn-1 is selected (always_keep).
hasT1 := false
for _, s := range selected {
if s.ID == "turn-1" {
hasT1 = true
}
}
if !hasT1 {
t.Error("expected always_keep turn-1 to be selected")
}
// Verify turn-2 is NOT selected.
for _, s := range selected {
if s.ID == "turn-2" {
t.Error("expected low-score turn-2 to be excluded")
}
}
// Assemble Phase 2 messages.
systemPrompt := "You are a helpful assistant.\n\n## Runtime\nlinux amd64"
longTermMemory := "User prefers Go. User's name is Alice."
currentMsg := "Deploy to production now"
msgs := BuildPhase2Messages(systemPrompt, longTermMemory, selected, currentMsg, cfg.HighScoreThreshold)
// --- Validate message structure ---
// 1. First message is system.
if msgs[0].Role != "system" {
t.Fatalf("msgs[0].Role = %s, want system", msgs[0].Role)
}
if !strings.Contains(msgs[0].Content, "helpful assistant") {
t.Error("system message should contain prompt text")
}
// 2. Last message is current user message.
last := msgs[len(msgs)-1]
if last.Role != "user" || last.Content != currentMsg {
t.Errorf("last message = role=%s content=%q, want user %q", last.Role, last.Content, currentMsg)
}
// 3. Role alternation: after system, messages must alternate user/assistant.
for i := 1; i < len(msgs); i++ {
expectedRole := "user"
if i%2 == 0 {
expectedRole = "assistant"
}
if msgs[i].Role != expectedRole {
t.Errorf("msgs[%d].Role = %s, want %s (content: %.50s...)",
i, msgs[i].Role, expectedRole, msgs[i].Content)
}
}
// 4. Long-term memory should be in msgs[1] (user role).
if !strings.Contains(msgs[1].Content, "Long-term Memory") {
t.Error("msgs[1] should contain long-term memory")
}
// 5. Always_keep turns (score >= 7) should appear before lower-score turns.
alwaysKeepEnd := -1
restStart := len(msgs)
for i := 3; i < len(msgs)-1; i += 2 { // user messages from turns, skip system+memory+ack
content := msgs[i].Content
// Check if this is an always_keep turn by looking for turn-1 content.
if strings.Contains(content, "Deploy to staging") {
alwaysKeepEnd = i
}
}
for i := 3; i < len(msgs)-1; i += 2 {
content := msgs[i].Content
// First non-always-keep turn.
if !strings.Contains(content, "Deploy to staging") && !strings.Contains(content, "Long-term Memory") {
restStart = i
break
}
}
if alwaysKeepEnd >= 0 && restStart < len(msgs) && alwaysKeepEnd > restStart {
t.Errorf("always_keep turns should come before rest: alwaysKeepEnd=%d, restStart=%d",
alwaysKeepEnd, restStart)
}
t.Logf("Phase 2 assembled %d messages from %d selected turns", len(msgs), len(selected))
for i, m := range msgs {
preview := m.Content
if len(preview) > 60 {
preview = preview[:60] + "..."
}
t.Logf(" [%d] role=%-10s content=%q", i, m.Role, preview)
}
}
// ---------------------------------------------------------------------------
// M4 Integration — MemoryDigest runOnce
// ---------------------------------------------------------------------------
// TestMemoryDigestIntegration_RunOnce inserts pending TurnRecords, runs
// MemoryDigest.runOnce with a mock LLM, and verifies:
// - TurnRecords are transitioned from "pending" to "processed"
// - MemoryStore receives new entries from the LLM extraction
func TestMemoryDigestIntegration_RunOnce(t *testing.T) {
dir := t.TempDir()
turnStore, err := NewTurnStore(dir)
if err != nil {
t.Fatalf("NewTurnStore: %v", err)
}
defer turnStore.Close()
memStore := NewMemoryStore(dir)
defer memStore.Close()
now := time.Now().Unix()
// Insert pending turns.
for i := 0; i < 3; i++ {
tr := TurnRecord{
ID: "digest-" + string(rune('a'+i)),
Ts: now - int64(300*(3-i)),
Score: 5,
ChannelKey: "cli:main",
Intent: "task",
Tags: []string{"golang"},
UserMsg: "Do task " + string(rune('A'+i)),
Reply: "Done with task " + string(rune('A'+i)),
Tokens: 30,
Status: "pending",
}
if err := turnStore.Insert(tr); err != nil {
t.Fatalf("Insert: %v", err)
}
}
// Verify pending.
pending, _ := turnStore.QueryPending(50)
if len(pending) != 3 {
t.Fatalf("expected 3 pending, got %d", len(pending))
}
// Create a mock provider that returns a memory extraction response.
mp := &mockLLMProvider{
response: `{"memories": [{"content": "User worked on Go tasks A, B, C", "tags": ["golang", "task"]}]}`,
}
// Create and run MemoryDigest.
worker := NewMemoryDigestWorker(turnStore, memStore, mp, "test-model")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
worker.runOnce(ctx)
// Verify turns are now processed.
pendingAfter, _ := turnStore.QueryPending(50)
if len(pendingAfter) != 0 {
t.Errorf("expected 0 pending after runOnce, got %d", len(pendingAfter))
}
// Verify memory store has entries.
memCtx := memStore.GetMemoryContext()
if memCtx == "" {
t.Error("expected MemoryStore to have entries after digest, got empty")
} else {
t.Logf("MemoryStore context after digest:\n%s", memCtx)
}
}

File diff suppressed because it is too large Load diff

View file

@ -7,121 +7,201 @@
package agent package agent
import ( import (
"database/sql"
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"sync"
"time" "time"
"github.com/sipeed/picoclaw/pkg/fileutil" "github.com/sipeed/picoclaw/pkg/logger"
_ "modernc.org/sqlite"
) )
// MemoryStore manages persistent memory for the agent. // MemoryStore manages persistent memory for the agent using SQLite.
// - Long-term memory: memory/MEMORY.md //
// - Daily notes: memory/YYYYMM/YYYYMMDD.md // Schema:
// - long_term: single-row table holding the long-term memory content
// - daily_notes: one row per day (key = "YYYYMMDD")
// - memory_entries: individually tagged memory items
//
// The database file is stored at workspace/memory.db.
type MemoryStore struct { type MemoryStore struct {
workspace string workspace string
memoryDir string db *sql.DB
memoryFile string mu sync.Mutex // serialise writes
} }
// NewMemoryStore creates a new MemoryStore with the given workspace path. // NewMemoryStore creates a new MemoryStore backed by SQLite.
// It ensures the memory directory exists. // It creates the database and tables if they do not exist.
func NewMemoryStore(workspace string) *MemoryStore { func NewMemoryStore(workspace string) *MemoryStore {
memoryDir := filepath.Join(workspace, "memory") dbPath := filepath.Join(workspace, "memory.db")
memoryFile := filepath.Join(memoryDir, "MEMORY.md")
// Ensure memory directory exists // Ensure workspace directory exists.
os.MkdirAll(memoryDir, 0o755) os.MkdirAll(workspace, 0o755)
return &MemoryStore{ db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(wal)&_pragma=busy_timeout(5000)")
workspace: workspace, if err != nil {
memoryDir: memoryDir, logger.DebugCF("memory", "Failed to open memory DB", map[string]any{"error": err.Error()})
memoryFile: memoryFile, // Return a store that degrades gracefully (methods return empty / no-op).
return &MemoryStore{workspace: workspace}
}
// Create tables.
ddl := `
CREATE TABLE IF NOT EXISTS long_term (
id INTEGER PRIMARY KEY CHECK (id = 1),
content TEXT NOT NULL DEFAULT ''
);
INSERT OR IGNORE INTO long_term (id, content) VALUES (1, '');
CREATE TABLE IF NOT EXISTS daily_notes (
day TEXT PRIMARY KEY, -- YYYYMMDD
content TEXT NOT NULL DEFAULT ''
);
CREATE TABLE IF NOT EXISTS memory_entries (
id INTEGER PRIMARY KEY AUTOINCREMENT,
content TEXT NOT NULL,
tags TEXT NOT NULL DEFAULT '', -- comma-separated, lowercase
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS cot_usage (
id INTEGER PRIMARY KEY AUTOINCREMENT,
intent TEXT NOT NULL DEFAULT '',
tags TEXT NOT NULL DEFAULT '', -- comma-separated tags from message analysis
cot_prompt TEXT NOT NULL DEFAULT '', -- LLM-generated thinking strategy
message TEXT NOT NULL DEFAULT '', -- first 200 chars of user message
feedback INTEGER NOT NULL DEFAULT 0, -- -1=bad, 0=neutral, 1=good
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
`
if _, err := db.Exec(ddl); err != nil {
logger.DebugCF("memory", "Failed to initialise memory DB tables", map[string]any{"error": err.Error()})
db.Close()
return &MemoryStore{workspace: workspace}
}
ms := &MemoryStore{
workspace: workspace,
db: db,
}
// Migrate from legacy file-based storage if memory.db was just created.
ms.migrateFromFiles()
return ms
}
// Close closes the underlying database. Safe to call multiple times.
func (ms *MemoryStore) Close() {
if ms.db != nil {
ms.db.Close()
} }
} }
// getTodayFile returns the path to today's daily note file (memory/YYYYMM/YYYYMMDD.md). // --- Long-term memory -------------------------------------------------------
func (ms *MemoryStore) getTodayFile() string {
today := time.Now().Format("20060102") // YYYYMMDD
monthDir := today[:6] // YYYYMM
filePath := filepath.Join(ms.memoryDir, monthDir, today+".md")
return filePath
}
// ReadLongTerm reads the long-term memory (MEMORY.md). // ReadLongTerm reads the long-term memory content.
// Returns empty string if the file doesn't exist. // Returns empty string if the database is unavailable.
func (ms *MemoryStore) ReadLongTerm() string { func (ms *MemoryStore) ReadLongTerm() string {
if data, err := os.ReadFile(ms.memoryFile); err == nil { if ms.db == nil {
return string(data) return ""
} }
return "" var content string
err := ms.db.QueryRow("SELECT content FROM long_term WHERE id = 1").Scan(&content)
if err != nil {
return ""
}
return content
} }
// WriteLongTerm writes content to the long-term memory file (MEMORY.md). // WriteLongTerm replaces the long-term memory content.
func (ms *MemoryStore) WriteLongTerm(content string) error { func (ms *MemoryStore) WriteLongTerm(content string) error {
// Use unified atomic write utility with explicit sync for flash storage reliability. if ms.db == nil {
// Using 0o600 (owner read/write only) for secure default permissions. return fmt.Errorf("memory DB not available")
return fileutil.WriteFileAtomic(ms.memoryFile, []byte(content), 0o600) }
ms.mu.Lock()
defer ms.mu.Unlock()
_, err := ms.db.Exec("UPDATE long_term SET content = ? WHERE id = 1", content)
return err
}
// --- Daily notes ------------------------------------------------------------
// todayKey returns today's date as "YYYYMMDD".
func todayKey() string {
return time.Now().Format("20060102")
} }
// ReadToday reads today's daily note. // ReadToday reads today's daily note.
// Returns empty string if the file doesn't exist. // Returns empty string if the file doesn't exist or the database is unavailable.
func (ms *MemoryStore) ReadToday() string { func (ms *MemoryStore) ReadToday() string {
todayFile := ms.getTodayFile() if ms.db == nil {
if data, err := os.ReadFile(todayFile); err == nil { return ""
return string(data)
} }
return "" var content string
err := ms.db.QueryRow("SELECT content FROM daily_notes WHERE day = ?", todayKey()).Scan(&content)
if err != nil {
return ""
}
return content
} }
// AppendToday appends content to today's daily note. // AppendToday appends content to today's daily note.
// If the file doesn't exist, it creates a new file with a date header. // If no note exists for today, a new one is created with a date header.
func (ms *MemoryStore) AppendToday(content string) error { func (ms *MemoryStore) AppendToday(content string) error {
todayFile := ms.getTodayFile() if ms.db == nil {
return fmt.Errorf("memory DB not available")
// Ensure month directory exists
monthDir := filepath.Dir(todayFile)
if err := os.MkdirAll(monthDir, 0o755); err != nil {
return err
} }
ms.mu.Lock()
defer ms.mu.Unlock()
var existingContent string key := todayKey()
if data, err := os.ReadFile(todayFile); err == nil {
existingContent = string(data)
}
var newContent string var existing string
if existingContent == "" { err := ms.db.QueryRow("SELECT content FROM daily_notes WHERE day = ?", key).Scan(&existing)
// Add header for new day if err == sql.ErrNoRows || existing == "" {
// New day — add header.
header := fmt.Sprintf("# %s\n\n", time.Now().Format("2006-01-02")) header := fmt.Sprintf("# %s\n\n", time.Now().Format("2006-01-02"))
newContent = header + content content = header + content
} else { _, err = ms.db.Exec(
// Append to existing content "INSERT OR REPLACE INTO daily_notes (day, content) VALUES (?, ?)",
newContent = existingContent + "\n" + content key, content,
)
} else if err == nil {
// Append to existing.
content = existing + "\n" + content
_, err = ms.db.Exec("UPDATE daily_notes SET content = ? WHERE day = ?", content, key)
} }
return err
// Use unified atomic write utility with explicit sync for flash storage reliability.
return fileutil.WriteFileAtomic(todayFile, []byte(newContent), 0o600)
} }
// GetRecentDailyNotes returns daily notes from the last N days. // GetRecentDailyNotes returns daily notes from the last N days.
// Contents are joined with "---" separator. // Contents are joined with "---" separator.
func (ms *MemoryStore) GetRecentDailyNotes(days int) string { func (ms *MemoryStore) GetRecentDailyNotes(days int) string {
if ms.db == nil {
return ""
}
var sb strings.Builder var sb strings.Builder
first := true first := true
for i := range days { for i := range days {
date := time.Now().AddDate(0, 0, -i) date := time.Now().AddDate(0, 0, -i)
dateStr := date.Format("20060102") // YYYYMMDD key := date.Format("20060102")
monthDir := dateStr[:6] // YYYYMM
filePath := filepath.Join(ms.memoryDir, monthDir, dateStr+".md")
if data, err := os.ReadFile(filePath); err == nil { var content string
err := ms.db.QueryRow("SELECT content FROM daily_notes WHERE day = ?", key).Scan(&content)
if err == nil && content != "" {
if !first { if !first {
sb.WriteString("\n\n---\n\n") sb.WriteString("\n\n---\n\n")
} }
sb.Write(data) sb.WriteString(content)
first = false first = false
} }
} }
@ -129,30 +209,676 @@ func (ms *MemoryStore) GetRecentDailyNotes(days int) string {
return sb.String() return sb.String()
} }
// --- Tagged memory entries ---------------------------------------------------
// MemoryEntry represents a single tagged memory item.
type MemoryEntry struct {
ID int64
Content string
Tags []string
CreatedAt string
UpdatedAt string
}
// normaliseTags lowercases, trims, deduplicates, and sorts tags.
func normaliseTags(tags []string) []string {
seen := make(map[string]struct{}, len(tags))
out := make([]string, 0, len(tags))
for _, t := range tags {
t = strings.ToLower(strings.TrimSpace(t))
if t == "" {
continue
}
if _, ok := seen[t]; !ok {
seen[t] = struct{}{}
out = append(out, t)
}
}
return out
}
// joinTags joins tags with "," for storage.
func joinTags(tags []string) string {
return strings.Join(normaliseTags(tags), ",")
}
// splitTags splits a stored tag string back into a slice.
func splitTags(s string) []string {
if s == "" {
return nil
}
return strings.Split(s, ",")
}
// AddEntry inserts a new tagged memory entry. Returns the new entry ID.
func (ms *MemoryStore) AddEntry(content string, tags []string) (int64, error) {
if ms.db == nil {
return 0, fmt.Errorf("memory DB not available")
}
ms.mu.Lock()
defer ms.mu.Unlock()
res, err := ms.db.Exec(
"INSERT INTO memory_entries (content, tags) VALUES (?, ?)",
content, joinTags(tags),
)
if err != nil {
return 0, err
}
return res.LastInsertId()
}
// UpdateEntry updates the content and tags of an existing entry.
func (ms *MemoryStore) UpdateEntry(id int64, content string, tags []string) error {
if ms.db == nil {
return fmt.Errorf("memory DB not available")
}
ms.mu.Lock()
defer ms.mu.Unlock()
_, err := ms.db.Exec(
"UPDATE memory_entries SET content = ?, tags = ?, updated_at = datetime('now') WHERE id = ?",
content, joinTags(tags), id,
)
return err
}
// DeleteEntry removes a memory entry by ID.
func (ms *MemoryStore) DeleteEntry(id int64) error {
if ms.db == nil {
return fmt.Errorf("memory DB not available")
}
ms.mu.Lock()
defer ms.mu.Unlock()
_, err := ms.db.Exec("DELETE FROM memory_entries WHERE id = ?", id)
return err
}
// GetEntry retrieves a single memory entry by ID.
func (ms *MemoryStore) GetEntry(id int64) (*MemoryEntry, error) {
if ms.db == nil {
return nil, fmt.Errorf("memory DB not available")
}
var e MemoryEntry
var tagsStr string
err := ms.db.QueryRow(
"SELECT id, content, tags, created_at, updated_at FROM memory_entries WHERE id = ?", id,
).Scan(&e.ID, &e.Content, &tagsStr, &e.CreatedAt, &e.UpdatedAt)
if err != nil {
return nil, err
}
e.Tags = splitTags(tagsStr)
return &e, nil
}
// SearchByTag returns all entries that contain the given tag.
// Tag matching is case-insensitive (tags are stored lowercase).
func (ms *MemoryStore) SearchByTag(tag string) ([]MemoryEntry, error) {
if ms.db == nil {
return nil, fmt.Errorf("memory DB not available")
}
tag = strings.ToLower(strings.TrimSpace(tag))
if tag == "" {
return nil, nil
}
// Match: exact tag as whole string, at start, at end, or in the middle.
// Pattern: tag OR tag,... OR ...,tag OR ...,tag,...
rows, err := ms.db.Query(
`SELECT id, content, tags, created_at, updated_at FROM memory_entries
WHERE tags = ? OR tags LIKE ? OR tags LIKE ? OR tags LIKE ?
ORDER BY updated_at DESC`,
tag, tag+",%", "%,"+tag, "%,"+tag+",%",
)
if err != nil {
return nil, err
}
defer rows.Close()
return scanEntries(rows)
}
// SearchByTags returns entries that contain ALL of the given tags.
func (ms *MemoryStore) SearchByTags(tags []string) ([]MemoryEntry, error) {
if ms.db == nil {
return nil, fmt.Errorf("memory DB not available")
}
tags = normaliseTags(tags)
if len(tags) == 0 {
return nil, nil
}
// Build WHERE clause: each tag must match.
conds := make([]string, 0, len(tags))
args := make([]any, 0, len(tags)*4)
for _, tag := range tags {
conds = append(conds,
"(tags = ? OR tags LIKE ? OR tags LIKE ? OR tags LIKE ?)")
args = append(args, tag, tag+",%", "%,"+tag, "%,"+tag+",%")
}
query := fmt.Sprintf(
"SELECT id, content, tags, created_at, updated_at FROM memory_entries WHERE %s ORDER BY updated_at DESC",
strings.Join(conds, " AND "),
)
rows, err := ms.db.Query(query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
return scanEntries(rows)
}
// SearchByAnyTag returns entries that contain ANY of the given tags (OR logic).
// Results are deduplicated and ordered by updated_at DESC, limited to 20 entries.
func (ms *MemoryStore) SearchByAnyTag(tags []string) ([]MemoryEntry, error) {
if ms.db == nil {
return nil, fmt.Errorf("memory DB not available")
}
tags = normaliseTags(tags)
if len(tags) == 0 {
return nil, nil
}
// Build WHERE clause: any tag may match (OR).
conds := make([]string, 0, len(tags))
args := make([]any, 0, len(tags)*4)
for _, tag := range tags {
conds = append(conds,
"(tags = ? OR tags LIKE ? OR tags LIKE ? OR tags LIKE ?)")
args = append(args, tag, tag+",%", "%,"+tag, "%,"+tag+",%")
}
query := fmt.Sprintf(
"SELECT id, content, tags, created_at, updated_at FROM memory_entries WHERE %s ORDER BY updated_at DESC LIMIT 20",
strings.Join(conds, " OR "),
)
rows, err := ms.db.Query(query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
return scanEntries(rows)
}
// ListAllTags returns all unique tags used across memory entries.
func (ms *MemoryStore) ListAllTags() ([]string, error) {
if ms.db == nil {
return nil, fmt.Errorf("memory DB not available")
}
rows, err := ms.db.Query("SELECT DISTINCT tags FROM memory_entries WHERE tags != ''")
if err != nil {
return nil, err
}
defer rows.Close()
seen := make(map[string]struct{})
for rows.Next() {
var tagsStr string
if err := rows.Scan(&tagsStr); err != nil {
continue
}
for _, t := range splitTags(tagsStr) {
seen[t] = struct{}{}
}
}
result := make([]string, 0, len(seen))
for t := range seen {
result = append(result, t)
}
return result, nil
}
// ListEntries returns the most recent N entries (all tags), ordered newest first.
func (ms *MemoryStore) ListEntries(limit int) ([]MemoryEntry, error) {
if ms.db == nil {
return nil, fmt.Errorf("memory DB not available")
}
if limit <= 0 {
limit = 50
}
rows, err := ms.db.Query(
"SELECT id, content, tags, created_at, updated_at FROM memory_entries ORDER BY updated_at DESC LIMIT ?",
limit,
)
if err != nil {
return nil, err
}
defer rows.Close()
return scanEntries(rows)
}
// scanEntries is a helper to scan rows into MemoryEntry slices.
func scanEntries(rows *sql.Rows) ([]MemoryEntry, error) {
var entries []MemoryEntry
for rows.Next() {
var e MemoryEntry
var tagsStr string
if err := rows.Scan(&e.ID, &e.Content, &tagsStr, &e.CreatedAt, &e.UpdatedAt); err != nil {
return entries, err
}
e.Tags = splitTags(tagsStr)
entries = append(entries, e)
}
return entries, rows.Err()
}
// --- Composite context ------------------------------------------------------
// GetMemoryContext returns formatted memory context for the agent prompt. // GetMemoryContext returns formatted memory context for the agent prompt.
// Includes long-term memory and recent daily notes. // Includes long-term memory, recent daily notes, and recent tagged entries.
func (ms *MemoryStore) GetMemoryContext() string { func (ms *MemoryStore) GetMemoryContext() string {
longTerm := ms.ReadLongTerm() longTerm := ms.ReadLongTerm()
recentNotes := ms.GetRecentDailyNotes(3) recentNotes := ms.GetRecentDailyNotes(3)
if longTerm == "" && recentNotes == "" {
return ""
}
var sb strings.Builder var sb strings.Builder
hasContent := false
if longTerm != "" { if longTerm != "" {
sb.WriteString("## Long-term Memory\n\n") sb.WriteString("## Long-term Memory\n\n")
sb.WriteString(longTerm) sb.WriteString(longTerm)
hasContent = true
} }
if recentNotes != "" { if recentNotes != "" {
if longTerm != "" { if hasContent {
sb.WriteString("\n\n---\n\n") sb.WriteString("\n\n---\n\n")
} }
sb.WriteString("## Recent Daily Notes\n\n") sb.WriteString("## Recent Daily Notes\n\n")
sb.WriteString(recentNotes) sb.WriteString(recentNotes)
hasContent = true
} }
// Include recent tagged memory entries.
entries, _ := ms.ListEntries(10)
if len(entries) > 0 {
if hasContent {
sb.WriteString("\n\n---\n\n")
}
sb.WriteString("## Tagged Memories\n\n")
for _, e := range entries {
tagLabel := ""
if len(e.Tags) > 0 {
tagLabel = " [" + strings.Join(e.Tags, ", ") + "]"
}
fmt.Fprintf(&sb, "- (#%d%s) %s\n", e.ID, tagLabel, e.Content)
}
hasContent = true
}
if !hasContent {
return ""
}
return sb.String() return sb.String()
} }
// --- CoT usage tracking (learning) ------------------------------------------
// CotUsageRecord represents a single CoT usage entry.
type CotUsageRecord struct {
ID int64
Intent string
Tags []string // Tags from the message analysis
CotPrompt string // LLM-generated thinking strategy
Message string
Feedback int // -1=bad, 0=neutral, 1=good
CreatedAt string
}
// CotStats holds aggregated statistics for an intent.
type CotStats struct {
Intent string
TotalUses int
AvgScore float64 // Average feedback score
LastUsed string
}
// RecordCotUsage logs a CoT usage event with the LLM-generated prompt and tags.
// messagePreview is truncated to 200 characters.
func (ms *MemoryStore) RecordCotUsage(intent string, tags []string, cotPrompt, message string) (int64, error) {
if ms.db == nil {
return 0, fmt.Errorf("memory DB not available")
}
ms.mu.Lock()
defer ms.mu.Unlock()
// Truncate message preview.
if len(message) > 200 {
message = message[:200]
}
tagStr := strings.Join(tags, ",")
res, err := ms.db.Exec(
"INSERT INTO cot_usage (intent, tags, cot_prompt, message) VALUES (?, ?, ?, ?)",
intent, tagStr, cotPrompt, message,
)
if err != nil {
return 0, err
}
return res.LastInsertId()
}
// UpdateCotFeedback updates the feedback score for a CoT usage record.
// score: -1=bad, 0=neutral, 1=good.
func (ms *MemoryStore) UpdateCotFeedback(id int64, score int) error {
if ms.db == nil {
return fmt.Errorf("memory DB not available")
}
if score < -1 || score > 1 {
return fmt.Errorf("feedback score must be -1, 0, or 1")
}
ms.mu.Lock()
defer ms.mu.Unlock()
_, err := ms.db.Exec("UPDATE cot_usage SET feedback = ? WHERE id = ?", score, id)
return err
}
// UpdateLatestCotFeedback updates the feedback score for the most recent
// CoT usage record. This is useful when the user provides feedback after
// the main LLM has responded (at which point the usage ID may not be tracked).
func (ms *MemoryStore) UpdateLatestCotFeedback(score int) error {
if ms.db == nil {
return fmt.Errorf("memory DB not available")
}
ms.mu.Lock()
defer ms.mu.Unlock()
_, err := ms.db.Exec(
"UPDATE cot_usage SET feedback = ? WHERE id = (SELECT MAX(id) FROM cot_usage)",
score,
)
return err
}
// GetCotStats returns aggregated statistics per intent,
// based on usage in the last N days. Ordered by total uses descending.
func (ms *MemoryStore) GetCotStats(days int) ([]CotStats, error) {
if ms.db == nil {
return nil, fmt.Errorf("memory DB not available")
}
if days <= 0 {
days = 30
}
rows, err := ms.db.Query(`
SELECT
intent,
COUNT(*) as total_uses,
COALESCE(AVG(CASE WHEN feedback != 0 THEN CAST(feedback AS REAL) END), 0.0) as avg_score,
MAX(created_at) as last_used
FROM cot_usage
WHERE created_at >= datetime('now', ? || ' days')
GROUP BY intent
ORDER BY total_uses DESC
`, fmt.Sprintf("-%d", days))
if err != nil {
return nil, err
}
defer rows.Close()
var stats []CotStats
for rows.Next() {
var s CotStats
if err := rows.Scan(&s.Intent, &s.TotalUses, &s.AvgScore, &s.LastUsed); err != nil {
continue
}
stats = append(stats, s)
}
return stats, rows.Err()
}
// GetCotIntentStats returns usage stats per intent.
// This is a simpler version that just counts per intent.
func (ms *MemoryStore) GetCotIntentStats(days int) ([]CotStats, error) {
return ms.GetCotStats(days)
}
// GetTopRatedCotPrompts returns the highest-rated generated CoT prompts.
// If filterTags is non-empty, prioritises prompts that share tags with the query.
// These serve as proven examples for future LLM generation.
func (ms *MemoryStore) GetTopRatedCotPrompts(days, limit int, filterTags []string) ([]CotUsageRecord, error) {
if ms.db == nil {
return nil, fmt.Errorf("memory DB not available")
}
if days <= 0 {
days = 30
}
if limit <= 0 {
limit = 5
}
rows, err := ms.db.Query(`
SELECT id, intent, tags, cot_prompt, message, feedback, created_at
FROM cot_usage
WHERE feedback > 0
AND cot_prompt != ''
AND created_at >= datetime('now', ? || ' days')
ORDER BY feedback DESC, created_at DESC
LIMIT ?
`, fmt.Sprintf("-%d", days), limit*3) // Over-fetch to filter by tags later.
if err != nil {
return nil, err
}
defer rows.Close()
var all []CotUsageRecord
for rows.Next() {
var r CotUsageRecord
var tagStr string
if err := rows.Scan(&r.ID, &r.Intent, &tagStr, &r.CotPrompt, &r.Message, &r.Feedback, &r.CreatedAt); err != nil {
continue
}
if tagStr != "" {
r.Tags = strings.Split(tagStr, ",")
}
all = append(all, r)
}
if err := rows.Err(); err != nil {
return nil, err
}
// If filter tags provided, sort by tag overlap (most relevant first).
if len(filterTags) > 0 && len(all) > 0 {
tagSet := make(map[string]bool, len(filterTags))
for _, t := range filterTags {
tagSet[strings.ToLower(t)] = true
}
// Partition: matching first, then non-matching.
var matching, rest []CotUsageRecord
for _, r := range all {
hasOverlap := false
for _, t := range r.Tags {
if tagSet[strings.ToLower(t)] {
hasOverlap = true
break
}
}
if hasOverlap {
matching = append(matching, r)
} else {
rest = append(rest, r)
}
}
all = append(matching, rest...)
}
if len(all) > limit {
all = all[:limit]
}
return all, nil
}
// GetRecentCotUsage returns the N most recent CoT usage records.
func (ms *MemoryStore) GetRecentCotUsage(limit int) ([]CotUsageRecord, error) {
if ms.db == nil {
return nil, fmt.Errorf("memory DB not available")
}
if limit <= 0 {
limit = 20
}
rows, err := ms.db.Query(
"SELECT id, intent, tags, cot_prompt, message, feedback, created_at FROM cot_usage ORDER BY id DESC LIMIT ?",
limit,
)
if err != nil {
return nil, err
}
defer rows.Close()
var records []CotUsageRecord
for rows.Next() {
var r CotUsageRecord
var tagStr string
if err := rows.Scan(&r.ID, &r.Intent, &tagStr, &r.CotPrompt, &r.Message, &r.Feedback, &r.CreatedAt); err != nil {
continue
}
if tagStr != "" {
r.Tags = strings.Split(tagStr, ",")
}
records = append(records, r)
}
return records, rows.Err()
}
// FormatCotLearningContext formats CoT usage history and top-rated prompts
// into a string for the pre-LLM to learn from past generations.
// currentTags are the tags extracted from the current message, used to
// prioritise relevant proven strategies.
func (ms *MemoryStore) FormatCotLearningContext(days int, currentTags []string) string {
var sb strings.Builder
hasContent := false
// 1. Usage stats per intent.
stats, err := ms.GetCotStats(days)
if err == nil && len(stats) > 0 {
sb.WriteString("## Historical Usage Stats\n\n")
for _, s := range stats {
scoreLabel := "neutral"
if s.AvgScore > 0.3 {
scoreLabel = "good"
} else if s.AvgScore < -0.3 {
scoreLabel = "poor"
}
fmt.Fprintf(&sb, "- Intent '%s': %d uses, avg feedback=%s (%.1f)\n",
s.Intent, s.TotalUses, scoreLabel, s.AvgScore)
}
sb.WriteString("\n")
hasContent = true
}
// 2. Top-rated generated prompts as proven examples (filtered by current tags).
topPrompts, err := ms.GetTopRatedCotPrompts(days, 3, currentTags)
if err == nil && len(topPrompts) > 0 {
sb.WriteString("## Proven Strategies (from past sessions with positive feedback)\n\n")
sb.WriteString("These generated strategies received positive feedback. Use similar approaches for similar intents.\n\n")
for i, r := range topPrompts {
msgPreview := r.Message
if len(msgPreview) > 80 {
msgPreview = msgPreview[:80] + "..."
}
tagLabel := ""
if len(r.Tags) > 0 {
tagLabel = fmt.Sprintf(", tags: [%s]", strings.Join(r.Tags, ", "))
}
fmt.Fprintf(&sb, "### Proven #%d (intent: %s%s, message: \"%s\")\n%s\n\n",
i+1, r.Intent, tagLabel, msgPreview, r.CotPrompt)
}
hasContent = true
}
if !hasContent {
return ""
}
return sb.String()
}
// --- Migration from legacy files --------------------------------------------
// migrateFromFiles imports data from the old file-based storage
// (memory/MEMORY.md and memory/YYYYMM/YYYYMMDD.md) into SQLite.
// It only runs if the long_term content is empty (fresh DB) AND the
// legacy directory exists. After a successful migration the legacy
// directory is renamed to memory_backup.
func (ms *MemoryStore) migrateFromFiles() {
if ms.db == nil {
return
}
memoryDir := filepath.Join(ms.workspace, "memory")
// Check if the legacy directory exists.
info, err := os.Stat(memoryDir)
if err != nil || !info.IsDir() {
return // nothing to migrate
}
// Only migrate if the DB is empty (fresh).
longTerm := ms.ReadLongTerm()
if longTerm != "" {
return // already has data
}
logger.DebugCF("memory", "Migrating legacy file-based memory to SQLite", nil)
// 1. Long-term memory.
memoryFile := filepath.Join(memoryDir, "MEMORY.md")
if data, err := os.ReadFile(memoryFile); err == nil && len(data) > 0 {
ms.WriteLongTerm(string(data))
}
// 2. Daily notes — walk YYYYMM/YYYYMMDD.md files.
entries, err := os.ReadDir(memoryDir)
if err != nil {
return
}
for _, entry := range entries {
if !entry.IsDir() {
continue
}
monthDir := filepath.Join(memoryDir, entry.Name())
dayFiles, err := os.ReadDir(monthDir)
if err != nil {
continue
}
for _, df := range dayFiles {
name := df.Name()
if !strings.HasSuffix(name, ".md") {
continue
}
day := strings.TrimSuffix(name, ".md") // YYYYMMDD
if len(day) != 8 {
continue
}
data, err := os.ReadFile(filepath.Join(monthDir, name))
if err != nil || len(data) == 0 {
continue
}
ms.mu.Lock()
ms.db.Exec(
"INSERT OR IGNORE INTO daily_notes (day, content) VALUES (?, ?)",
day, string(data),
)
ms.mu.Unlock()
}
}
// Rename legacy dir so we don't migrate again.
backupDir := filepath.Join(ms.workspace, "memory_backup")
if err := os.Rename(memoryDir, backupDir); err != nil {
logger.DebugCF("memory", "Could not rename legacy memory dir", map[string]any{"error": err.Error()})
} else {
logger.DebugCF("memory", "Legacy memory migrated and backed up", map[string]any{"backup": backupDir})
}
}

280
pkg/agent/memory_digest.go Normal file
View file

@ -0,0 +1,280 @@
// PicoClaw - Ultra-lightweight personal AI agent
// License: MIT
//
// Copyright (c) 2026 PicoClaw contributors
package agent
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/sipeed/picoclaw/pkg/logger"
"github.com/sipeed/picoclaw/pkg/providers"
)
// MemoryDigestWorker runs as a background goroutine and periodically extracts
// long-term memories from pending TurnRecords.
//
// Design:
// - Fixed interval trigger (default 5 minutes).
// - No llmActive yield mechanism (personal agent, low QPS, API rate-limits handle it).
// - Processes up to 50 pending turns per cycle, grouped by channel_key.
// - On completion, marks turns as "processed" and archives old processed turns.
type MemoryDigestWorker struct {
store *TurnStore
memory *MemoryStore
provider providers.LLMProvider
model string
interval time.Duration
}
// MemoryDigestConfig holds tunable parameters.
type MemoryDigestConfig struct {
Interval time.Duration // Polling period (default: 5 minutes)
BatchLimit int // Max pending turns per cycle (default: 50)
ArchiveAfterDays int // Archive processed turns older than N days (default: 7)
}
func defaultDigestConfig() MemoryDigestConfig {
return MemoryDigestConfig{
Interval: 5 * time.Minute,
BatchLimit: 50,
ArchiveAfterDays: 7,
}
}
// NewMemoryDigestWorker creates a worker. provider/model may be nil/empty
// if only archival (no LLM extraction) is desired.
func NewMemoryDigestWorker(
store *TurnStore,
memory *MemoryStore,
provider providers.LLMProvider,
model string,
) *MemoryDigestWorker {
return &MemoryDigestWorker{
store: store,
memory: memory,
provider: provider,
model: model,
interval: defaultDigestConfig().Interval,
}
}
// SetInterval overrides the polling interval (e.g. for testing).
func (w *MemoryDigestWorker) SetInterval(d time.Duration) {
w.interval = d
}
// Start launches the background goroutine. It respects ctx cancellation.
func (w *MemoryDigestWorker) Start(ctx context.Context) {
go func() {
ticker := time.NewTicker(w.interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if err := w.runOnce(ctx); err != nil {
logger.WarnCF("memory_digest", "runOnce error", map[string]any{"error": err.Error()})
}
}
}
}()
logger.DebugCF("memory_digest", "Worker started", map[string]any{"interval": w.interval.String()})
}
// RunOnceNow triggers an immediate digest cycle (useful for testing).
func (w *MemoryDigestWorker) RunOnceNow(ctx context.Context) error {
return w.runOnce(ctx)
}
// runOnce executes one full digest cycle.
func (w *MemoryDigestWorker) runOnce(ctx context.Context) error {
if w.store == nil {
return nil
}
cfg := defaultDigestConfig()
// Step 1: Load pending turns.
pending, err := w.store.QueryPending(cfg.BatchLimit)
if err != nil {
return fmt.Errorf("query pending: %w", err)
}
if len(pending) == 0 {
logger.DebugCF("memory_digest", "No pending turns", nil)
// Still run archival.
return w.archive(cfg)
}
logger.DebugCF("memory_digest", "Processing pending turns",
map[string]any{"count": len(pending)})
// Step 2: Group by channel_key to avoid mixing user memories.
groups := make(map[string][]TurnRecord)
for _, t := range pending {
groups[t.ChannelKey] = append(groups[t.ChannelKey], t)
}
// Step 3: For each group, call LLM to extract memories.
for channelKey, turns := range groups {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if err := w.processGroup(ctx, channelKey, turns); err != nil {
logger.WarnCF("memory_digest", "Group processing error",
map[string]any{"channel": channelKey, "error": err.Error()})
// Continue with other groups.
}
}
// Step 6: Archive old processed turns.
return w.archive(cfg)
}
// processGroup extracts memories from a batch of turns belonging to one channel.
func (w *MemoryDigestWorker) processGroup(ctx context.Context, channelKey string, turns []TurnRecord) error {
// Build a conversation digest for the LLM.
memories, err := w.extractMemories(ctx, turns)
if err != nil {
// Mark them as processed anyway so we don't loop forever.
logger.WarnCF("memory_digest", "LLM extraction failed, marking as processed",
map[string]any{"channel": channelKey, "error": err.Error()})
}
// Step 4: Write extracted memories.
if w.memory != nil {
for _, m := range memories {
if _, addErr := w.memory.AddEntry(m.Content, m.Tags); addErr != nil {
logger.WarnCF("memory_digest", "Failed to save memory",
map[string]any{"error": addErr.Error()})
}
}
}
// Step 5: Mark all turns as processed.
for _, t := range turns {
if setErr := w.store.SetStatus(t.ID, "processed"); setErr != nil {
logger.WarnCF("memory_digest", "SetStatus failed",
map[string]any{"id": t.ID, "error": setErr.Error()})
}
}
logger.DebugCF("memory_digest", "Group processed",
map[string]any{
"channel": channelKey,
"turns": len(turns),
"memories_stored": len(memories),
})
return nil
}
// digestMemoryResult holds one extracted memory item.
type digestMemoryResult struct {
Content string `json:"content"`
Tags []string `json:"tags"`
}
const digestPrompt = `Extract important, durable facts worth remembering from these conversation turns.
Conversation turns:
%s
Respond with ONLY JSON: {"memories": [{"content": "<fact>", "tags": ["tag1"]}]}
Rules:
- max 5 memories total across all turns
- max 3 tags each, lowercase
- skip trivial small-talk
- prefer facts about user preferences, environment, recurring patterns, important decisions
- if nothing worth remembering: {"memories": []}`
// extractMemories calls the LLM to distil memories from a batch of turns.
// Returns nil memories (not error) when the LLM is unconfigured.
func (w *MemoryDigestWorker) extractMemories(ctx context.Context, turns []TurnRecord) ([]digestMemoryResult, error) {
if w.provider == nil || w.model == "" {
return nil, nil
}
// Build conversation summary for the prompt.
var sb strings.Builder
for i, t := range turns {
reply := t.Reply
if len(reply) > 500 {
reply = reply[:500] + "..."
}
fmt.Fprintf(&sb, "=== Turn %d (intent: %s, tags: %v) ===\nUser: %s\nAssistant: %s\n\n",
i+1, t.Intent, t.Tags, t.UserMsg, reply)
}
prompt := fmt.Sprintf(digestPrompt, sb.String())
resp, err := w.provider.Chat(ctx, []providers.Message{
{Role: "user", Content: prompt},
}, nil, w.model, map[string]any{"max_tokens": 512, "temperature": 0.1})
if err != nil {
return nil, fmt.Errorf("LLM call: %w", err)
}
raw := strings.TrimSpace(resp.Content)
// Strip markdown fences if present.
if strings.HasPrefix(raw, "```") {
lines := strings.Split(raw, "\n")
if len(lines) > 2 {
raw = strings.Join(lines[1:len(lines)-1], "\n")
}
}
var result struct {
Memories []digestMemoryResult `json:"memories"`
}
if err := json.Unmarshal([]byte(raw), &result); err != nil {
// Parsing failure — skip extraction, don't fail the whole batch.
logger.WarnCF("memory_digest", "Failed to parse LLM response",
map[string]any{"raw": raw[:min(len(raw), 200)], "error": err.Error()})
return nil, nil
}
// Normalise.
out := make([]digestMemoryResult, 0, len(result.Memories))
for _, m := range result.Memories {
m.Content = strings.TrimSpace(m.Content)
if m.Content == "" {
continue
}
normalised := make([]string, 0, len(m.Tags))
for _, t := range m.Tags {
t = strings.ToLower(strings.TrimSpace(t))
if t != "" {
normalised = append(normalised, t)
}
}
m.Tags = normalised
out = append(out, m)
}
return out, nil
}
// archive runs periodic archival of processed turns.
func (w *MemoryDigestWorker) archive(cfg MemoryDigestConfig) error {
if w.store == nil {
return nil
}
if err := w.store.ArchiveOldProcessed(cfg.ArchiveAfterDays); err != nil {
return fmt.Errorf("archive: %w", err)
}
return nil
}
// min returns the smaller of a and b.
func min(a, b int) int {
if a < b {
return a
}
return b
}

969
pkg/agent/reflector.go Normal file
View file

@ -0,0 +1,969 @@
// PicoClaw - Ultra-lightweight personal AI agent
// License: MIT
//
// Copyright (c) 2026 PicoClaw contributors
package agent
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"time"
"os"
"github.com/sipeed/picoclaw/pkg/channels"
"github.com/sipeed/picoclaw/pkg/shell"
"github.com/sipeed/picoclaw/pkg/logger"
"github.com/sipeed/picoclaw/pkg/providers"
"github.com/sipeed/picoclaw/pkg/tools"
)
// ---------------------------------------------------------------------------
// Runtime — unified execution engine
//
// The Runtime serves two purposes:
//
// 1. Post-LLM processing: runs async processors after the main LLM responds
// (memory extraction, CoT feedback, error tracking).
//
// 2. Slash commands: handles /{cmd} {args} from users, executed synchronously.
//
// Both share the same MemoryStore and lightweight LLM provider.
// ---------------------------------------------------------------------------
// --- Post-LLM Processing ---------------------------------------------------
// RuntimeInput captures everything that happened during a single agent turn.
type RuntimeInput struct {
UserMessage string // Original user message
AssistantReply string // Main LLM's final response
Intent string // Pre-LLM detected intent
Tags []string // Pre-LLM extracted tags
CotPrompt string // Generated thinking strategy
ToolCalls []ToolCallRecord
Iterations int // Number of LLM iterations used
Score int // Phase 3 CalcTurnScore result (set by SyncPhase3)
ChannelKey string // "channel:chatID" (set by runAgentLoop)
}
// ToolCallRecord captures one tool invocation and its outcome.
type ToolCallRecord struct {
Name string
Error string // Empty if success
Duration time.Duration // How long the tool took
}
// RuntimeProcessor is a single post-LLM processing step.
type RuntimeProcessor interface {
Name() string
Process(ctx context.Context, input RuntimeInput, memory *MemoryStore) error
}
// --- Slash Commands ---------------------------------------------------------
// CommandHandler handles a single /{cmd} invocation.
type CommandHandler func(args []string, memory *MemoryStore) string
// CommandDef defines a registered slash command.
type CommandDef struct {
Name string // e.g. "memory"
Usage string // e.g. "/memory [list|add|search] ..."
Description string
Handler CommandHandler
}
// --- Reflector (Phase 3) ----------------------------------------------------
// Reflector manages post-LLM processors and slash commands.
// This is Phase 3 (Reflect) of the Runtime Loop.
type Reflector struct {
provider providers.LLMProvider
model string
processors []RuntimeProcessor
commands map[string]CommandDef
mu sync.RWMutex
timeout time.Duration
toolRegistry *tools.ToolRegistry // For /shell command
agentRegistry *AgentRegistry // For /show, /list, /switch
channelManager *channels.Manager // For /list channels, /switch channel
}
// NewReflector creates a new Reflector (Phase 3) with built-in processors and commands.
func NewReflector(provider providers.LLMProvider, model string) *Reflector {
r := &Reflector{
provider: provider,
model: model,
timeout: 30 * time.Second,
commands: make(map[string]CommandDef),
}
// Built-in processors (post-LLM, async).
// Note: CotEvaluator and MemoryExtractor are intentionally removed from the
// default pipeline — memory extraction is now handled by MemoryDigestWorker
// (batch, background) rather than per-turn inline LLM calls.
r.RegisterProcessor(&ErrorTracker{})
// Built-in slash commands.
r.RegisterCommand(CommandDef{
Name: "help",
Usage: "/help",
Description: "Show all available commands",
Handler: r.cmdHelp,
})
r.RegisterCommand(CommandDef{
Name: "memory",
Usage: "/memory [list|add|delete|edit|search|stats] ...",
Description: "Manage long-term memory",
Handler: cmdMemory,
})
r.RegisterCommand(CommandDef{
Name: "cot",
Usage: "/cot [feedback|stats|history] ...",
Description: "Manage CoT learning",
Handler: cmdCot,
})
r.RegisterCommand(CommandDef{
Name: "runtime",
Usage: "/runtime [status|processors]",
Description: "Runtime status and diagnostics",
Handler: r.cmdRuntimeStatus,
})
r.RegisterCommand(CommandDef{
Name: "shell",
Usage: "/shell <cmd> [args...]",
Description: "Execute shell command in workspace",
Handler: r.cmdShell,
})
// System commands (migrated from handleCommand).
r.RegisterCommand(CommandDef{
Name: "show",
Usage: "/show [model|channel|agents]",
Description: "Show current settings",
Handler: r.cmdShow,
})
r.RegisterCommand(CommandDef{
Name: "list",
Usage: "/list [models|channels|agents]",
Description: "List available resources",
Handler: r.cmdList,
})
r.RegisterCommand(CommandDef{
Name: "switch",
Usage: "/switch [model|channel] to <name>",
Description: "Switch model or channel",
Handler: r.cmdSwitch,
})
return r
}
// RegisterProcessor adds a post-LLM processor.
func (r *Reflector) RegisterProcessor(p RuntimeProcessor) {
r.mu.Lock()
defer r.mu.Unlock()
r.processors = append(r.processors, p)
}
// RegisterCommand adds a slash command.
func (r *Reflector) RegisterCommand(cmd CommandDef) {
r.mu.Lock()
defer r.mu.Unlock()
r.commands[cmd.Name] = cmd
}
// SetTools sets the tool registry for /shell command support.
func (r *Reflector) SetTools(registry *tools.ToolRegistry) {
r.mu.Lock()
defer r.mu.Unlock()
r.toolRegistry = registry
}
// SetAgentInfo provides the Runtime with agent and channel references
// needed by system commands (/show, /list, /switch).
func (r *Reflector) SetAgentInfo(reg *AgentRegistry, cm *channels.Manager) {
r.mu.Lock()
defer r.mu.Unlock()
r.agentRegistry = reg
r.channelManager = cm
}
// ---------------------------------------------------------------------------
// Post-LLM: async execution
// ---------------------------------------------------------------------------
// SyncPhase3 runs the synchronous, low-latency part of Phase 3:
// it calculates the Turn score and returns it. The caller must invoke this
// BEFORE PublishOutbound so that Active Context is ready for the next turn.
// Execution target: < 2ms (pure CPU, no I/O).
func (r *Reflector) SyncPhase3(input RuntimeInput) int {
score := CalcTurnScore(input)
logger.DebugCF("reflector", "SyncPhase3 score",
map[string]any{"score": score, "intent": input.Intent, "tools": len(input.ToolCalls)})
return score
}
// AsyncPhase3 runs the asynchronous post-turn work: persisting TurnRecord,
// running legacy processors, etc. Call this AFTER PublishOutbound.
func (r *Reflector) AsyncPhase3(input RuntimeInput, memory *MemoryStore, turnStore *TurnStore, activeCtx *ActiveContextStore) {
if r == nil {
return
}
r.mu.RLock()
processors := make([]RuntimeProcessor, len(r.processors))
copy(processors, r.processors)
r.mu.RUnlock()
go func() {
tctx, cancel := context.WithTimeout(context.Background(), r.timeout)
defer cancel()
// Run registered processors (currently: ErrorTracker).
if memory != nil {
for _, p := range processors {
select {
case <-tctx.Done():
return
default:
}
start := time.Now()
if err := p.Process(tctx, input, memory); err != nil {
logger.WarnCF("reflector", "Processor failed",
map[string]any{"processor": p.Name(), "error": err.Error(),
"ms": time.Since(start).Milliseconds()})
}
}
}
// Persist TurnRecord to turns.db.
if turnStore != nil && input.UserMessage != "" {
record := TurnRecord{
Ts: time.Now().Unix(),
ChannelKey: input.ChannelKey,
Score: input.Score,
Intent: input.Intent,
Tags: input.Tags,
Status: "pending",
UserMsg: input.UserMessage,
Reply: input.AssistantReply,
ToolCalls: input.ToolCalls,
}
if err := turnStore.Insert(record); err != nil {
logger.WarnCF("reflector", "TurnRecord insert failed",
map[string]any{"error": err.Error()})
}
}
}()
}
// RunPostLLM is kept for backward compatibility. New code should use
// SyncPhase3 + AsyncPhase3 instead.
func (r *Reflector) RunPostLLM(input RuntimeInput, memory *MemoryStore) {
r.AsyncPhase3(input, memory, nil, nil)
}
// ---------------------------------------------------------------------------
// Slash commands: synchronous execution
// ---------------------------------------------------------------------------
// HandleCommand tries to handle a /{cmd} message.
// Returns (response, true) if handled, ("", false) if not a known command.
func (r *Reflector) HandleCommand(content string, memory *MemoryStore) (string, bool) {
content = strings.TrimSpace(content)
if !strings.HasPrefix(content, "/") {
return "", false
}
parts := strings.Fields(content)
if len(parts) == 0 {
return "", false
}
cmdName := strings.TrimPrefix(parts[0], "/")
args := parts[1:]
r.mu.RLock()
cmd, ok := r.commands[cmdName]
r.mu.RUnlock()
if !ok {
return "", false // Not our command — let AgentLoop's handleCommand try.
}
if memory == nil {
return "⚠️ Memory store not available", true
}
return cmd.Handler(args, memory), true
}
// ListCommands returns a formatted help text for all registered commands.
func (r *Reflector) ListCommands() string {
r.mu.RLock()
defer r.mu.RUnlock()
var sb strings.Builder
sb.WriteString("**Runtime Commands**\n\n")
for _, cmd := range r.commands {
fmt.Fprintf(&sb, "• `%s` — %s\n", cmd.Usage, cmd.Description)
}
return sb.String()
}
// ===========================================================================
// Built-in slash commands
// ===========================================================================
// --- /help ------------------------------------------------------------------
func (r *Reflector) cmdHelp(_ []string, _ *MemoryStore) string {
var sb strings.Builder
sb.WriteString("📖 **Available Commands**\n\n")
r.mu.RLock()
for _, cmd := range r.commands {
fmt.Fprintf(&sb, "• `%s` — %s\n", cmd.Usage, cmd.Description)
}
r.mu.RUnlock()
return sb.String()
}
// --- /memory ----------------------------------------------------------------
func cmdMemory(args []string, memory *MemoryStore) string {
if len(args) == 0 {
return "Usage: /memory [list|add|delete|edit|search|stats]\n" +
" /memory list — show recent memories\n" +
" /memory add <text> #tags — add a memory\n" +
" /memory delete <id> — delete a memory\n" +
" /memory edit <id> <text> — edit a memory\n" +
" /memory search <query> — search by tags\n" +
" /memory stats — memory statistics"
}
switch args[0] {
case "list":
limit := 10
entries, err := memory.ListEntries(limit)
if err != nil {
return fmt.Sprintf("❌ Error: %v", err)
}
if len(entries) == 0 {
return "📭 No memories stored yet."
}
var sb strings.Builder
fmt.Fprintf(&sb, "📝 **Recent Memories** (%d)\n\n", len(entries))
for _, e := range entries {
tags := ""
if len(e.Tags) > 0 {
tags = " [" + strings.Join(e.Tags, ", ") + "]"
}
preview := e.Content
if len(preview) > 100 {
preview = preview[:100] + "..."
}
fmt.Fprintf(&sb, "• #%d%s: %s\n", e.ID, tags, preview)
}
return sb.String()
case "add":
if len(args) < 2 {
return "Usage: /memory add <text> #tag1 #tag2"
}
// Separate content from #tags.
var content []string
var tags []string
for _, a := range args[1:] {
if strings.HasPrefix(a, "#") {
tags = append(tags, strings.TrimPrefix(a, "#"))
} else {
content = append(content, a)
}
}
text := strings.Join(content, " ")
if text == "" {
return "❌ Memory content cannot be empty"
}
id, err := memory.AddEntry(text, tags)
if err != nil {
return fmt.Sprintf("❌ Failed to add: %v", err)
}
return fmt.Sprintf("✅ Memory #%d saved (tags: %v)", id, tags)
case "search":
if len(args) < 2 {
return "Usage: /memory search <tag1> [tag2] ..."
}
entries, err := memory.SearchByAnyTag(args[1:])
if err != nil {
return fmt.Sprintf("❌ Error: %v", err)
}
if len(entries) == 0 {
return fmt.Sprintf("🔍 No memories found for tags: %v", args[1:])
}
var sb strings.Builder
fmt.Fprintf(&sb, "🔍 **Found %d memories**\n\n", len(entries))
for _, e := range entries {
tags := ""
if len(e.Tags) > 0 {
tags = " [" + strings.Join(e.Tags, ", ") + "]"
}
preview := e.Content
if len(preview) > 100 {
preview = preview[:100] + "..."
}
fmt.Fprintf(&sb, "• #%d%s: %s\n", e.ID, tags, preview)
}
return sb.String()
case "stats":
tags, _ := memory.ListAllTags()
entries, _ := memory.ListEntries(9999)
var sb strings.Builder
sb.WriteString("📊 **Memory Stats**\n")
fmt.Fprintf(&sb, "• Total entries: %d\n", len(entries))
fmt.Fprintf(&sb, "• Total tags: %d\n", len(tags))
if len(tags) > 0 {
preview := tags
if len(preview) > 20 {
preview = preview[:20]
}
fmt.Fprintf(&sb, "• Tags: %s", strings.Join(preview, ", "))
if len(tags) > 20 {
fmt.Fprintf(&sb, " ... (+%d more)", len(tags)-20)
}
sb.WriteString("\n")
}
return sb.String()
case "delete":
if len(args) < 2 {
return "Usage: /memory delete <id>"
}
var id int64
if _, err := fmt.Sscanf(args[1], "%d", &id); err != nil {
return "❌ Invalid ID. Usage: /memory delete <id>"
}
if err := memory.DeleteEntry(id); err != nil {
return fmt.Sprintf("❌ Failed: %v", err)
}
return fmt.Sprintf("✅ Memory #%d deleted", id)
case "edit":
if len(args) < 3 {
return "Usage: /memory edit <id> <new content> #tags"
}
var id int64
if _, err := fmt.Sscanf(args[1], "%d", &id); err != nil {
return "❌ Invalid ID. Usage: /memory edit <id> <text>"
}
var content []string
var tags []string
for _, a := range args[2:] {
if strings.HasPrefix(a, "#") {
tags = append(tags, strings.TrimPrefix(a, "#"))
} else {
content = append(content, a)
}
}
text := strings.Join(content, " ")
if text == "" {
return "❌ Content cannot be empty"
}
if err := memory.UpdateEntry(id, text, tags); err != nil {
return fmt.Sprintf("❌ Failed: %v", err)
}
return fmt.Sprintf("✅ Memory #%d updated", id)
default:
return fmt.Sprintf("Unknown subcommand: %s. Use /memory for help.", args[0])
}
}
// --- /cot -------------------------------------------------------------------
func cmdCot(args []string, memory *MemoryStore) string {
if len(args) == 0 {
return "Usage: /cot [feedback|stats|history]\n" +
" /cot feedback <1|0|-1> — rate last CoT strategy\n" +
" /cot stats — show CoT performance\n" +
" /cot history [N] — show recent CoT usage"
}
switch args[0] {
case "feedback":
if len(args) < 2 {
return "Usage: /cot feedback <1|0|-1>"
}
var score int
switch args[1] {
case "1", "+1", "good":
score = 1
case "-1", "bad":
score = -1
case "0", "neutral":
score = 0
default:
return "❌ Score must be 1 (good), 0 (neutral), or -1 (bad)"
}
if err := memory.UpdateLatestCotFeedback(score); err != nil {
return fmt.Sprintf("❌ Failed: %v", err)
}
labels := map[int]string{1: "👍 good", 0: "😐 neutral", -1: "👎 bad"}
return fmt.Sprintf("✅ CoT feedback recorded: %s", labels[score])
case "stats":
stats, err := memory.GetCotStats(30)
if err != nil || len(stats) == 0 {
return "📊 No CoT usage data yet."
}
var sb strings.Builder
sb.WriteString("📊 **CoT Stats (last 30 days)**\n\n")
for _, s := range stats {
scoreLabel := "neutral"
if s.AvgScore > 0.3 {
scoreLabel = "good"
} else if s.AvgScore < -0.3 {
scoreLabel = "poor"
}
fmt.Fprintf(&sb, "• Intent '%s': %d uses, avg=%s (%.1f)\n",
s.Intent, s.TotalUses, scoreLabel, s.AvgScore)
}
return sb.String()
case "history":
limit := 5
if len(args) > 1 {
fmt.Sscanf(args[1], "%d", &limit)
}
records, err := memory.GetRecentCotUsage(limit)
if err != nil || len(records) == 0 {
return "📜 No CoT history yet."
}
var sb strings.Builder
fmt.Fprintf(&sb, "📜 **Recent CoT Usage** (%d)\n\n", len(records))
for _, r := range records {
fb := "😐"
if r.Feedback > 0 {
fb = "👍"
} else if r.Feedback < 0 {
fb = "👎"
}
tags := ""
if len(r.Tags) > 0 {
tags = " [" + strings.Join(r.Tags, ", ") + "]"
}
prompt := r.CotPrompt
if len(prompt) > 80 {
prompt = prompt[:80] + "..."
}
fmt.Fprintf(&sb, "• #%d %s %s%s: %s\n", r.ID, fb, r.Intent, tags, prompt)
}
return sb.String()
default:
return fmt.Sprintf("Unknown subcommand: %s. Use /cot for help.", args[0])
}
}
// --- /runtime ---------------------------------------------------------------
func (r *Reflector) cmdRuntimeStatus(args []string, memory *MemoryStore) string {
if len(args) == 0 {
return "Usage: /runtime [status|processors|commands]"
}
switch args[0] {
case "status":
r.mu.RLock()
nProc := len(r.processors)
nCmd := len(r.commands)
r.mu.RUnlock()
var sb strings.Builder
sb.WriteString("⚙️ **Runtime Status**\n")
fmt.Fprintf(&sb, "• Processors: %d\n", nProc)
fmt.Fprintf(&sb, "• Commands: %d\n", nCmd)
fmt.Fprintf(&sb, "• Timeout: %s\n", r.timeout)
if r.model != "" {
fmt.Fprintf(&sb, "• Model: %s\n", r.model)
}
return sb.String()
case "processors":
r.mu.RLock()
defer r.mu.RUnlock()
var sb strings.Builder
sb.WriteString("⚙️ **Processors**\n")
for i, p := range r.processors {
fmt.Fprintf(&sb, "• %d. %s\n", i+1, p.Name())
}
return sb.String()
case "commands":
return r.ListCommands()
default:
return fmt.Sprintf("Unknown: %s. Use /runtime for help.", args[0])
}
}
// --- /shell -----------------------------------------------------------------
const shellMaxOutput = 4000
// shellDenySubstrings blocks injection attempts for dev tool passthrough.
var shellDenySubstrings = []string{
"| sh", "| bash", "| powershell", "| cmd",
"; rm ", "; del ", "&& rm ", "&& del ",
"$(", "${", "`",
"> /dev/", ">> /dev/",
}
func (r *Reflector) cmdShell(args []string, _ *MemoryStore) string {
if len(args) == 0 {
return "Usage: /shell <command> [args...]\n" +
" Built-in: ls, cat, head, tail, grep, wc, find, diff, tree, stat, pwd, echo\n" +
" Dev tools (passthrough): go, git, node, python, npm, cargo, make\n" +
" File ops: touch, mkdir, cp, mv"
}
baseCmd := strings.ToLower(args[0])
cmdArgs := args[1:]
// 1. Try built-in Go implementation (cross-platform).
if handler, ok := shell.BuiltinCmds[baseCmd]; ok {
cwd, _ := os.Getwd()
output := handler(cmdArgs, cwd)
return shellFormatOutput(output)
}
// 2. Try dev tool passthrough via ExecTool.
if shell.DevToolPassthrough[baseCmd] {
// Injection check.
command := strings.Join(args, " ")
cmdLower := strings.ToLower(command)
for _, deny := range shellDenySubstrings {
if strings.Contains(cmdLower, deny) {
return fmt.Sprintf("❌ Command blocked: restricted pattern '%s'", deny)
}
}
r.mu.RLock()
registry := r.toolRegistry
r.mu.RUnlock()
if registry == nil {
return "⚠️ Dev tool passthrough not available (no tool registry)"
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
result := registry.Execute(ctx, "exec", map[string]any{
"command": command,
})
if result.IsError || result.Err != nil {
errMsg := result.ForLLM
if errMsg == "" && result.Err != nil {
errMsg = result.Err.Error()
}
return fmt.Sprintf("❌ %s", errMsg)
}
return shellFormatOutput(result.ForLLM)
}
return fmt.Sprintf("❌ Unknown command '%s'. Use /shell for available commands.", baseCmd)
}
func shellFormatOutput(output string) string {
if output == "" {
return "✅ (no output)"
}
if len(output) > shellMaxOutput {
output = output[:shellMaxOutput] + fmt.Sprintf("\n... (truncated, %d chars total)", len(output))
}
return "```\n" + output + "\n```"
}
// --- /show ------------------------------------------------------------------
func (r *Reflector) cmdShow(args []string, _ *MemoryStore) string {
if len(args) < 1 {
return "Usage: /show [model|channel|agents]"
}
r.mu.RLock()
reg := r.agentRegistry
r.mu.RUnlock()
switch args[0] {
case "model":
if reg == nil {
return "⚠️ Agent registry not available"
}
agent := reg.GetDefaultAgent()
if agent == nil {
return "No default agent configured"
}
return fmt.Sprintf("Current model: %s", agent.Model)
case "channel":
return "Use /list channels to see enabled channels"
case "agents":
if reg == nil {
return "⚠️ Agent registry not available"
}
ids := reg.ListAgentIDs()
return fmt.Sprintf("Registered agents: %s", strings.Join(ids, ", "))
default:
return fmt.Sprintf("Unknown show target: %s", args[0])
}
}
// --- /list ------------------------------------------------------------------
func (r *Reflector) cmdList(args []string, _ *MemoryStore) string {
if len(args) < 1 {
return "Usage: /list [models|channels|agents]"
}
r.mu.RLock()
reg := r.agentRegistry
cm := r.channelManager
r.mu.RUnlock()
switch args[0] {
case "models":
return "Available models: configured in config.json per agent"
case "channels":
if cm == nil {
return "Channel manager not initialized"
}
chs := cm.GetEnabledChannels()
if len(chs) == 0 {
return "No channels enabled"
}
return fmt.Sprintf("Enabled channels: %s", strings.Join(chs, ", "))
case "agents":
if reg == nil {
return "⚠️ Agent registry not available"
}
ids := reg.ListAgentIDs()
return fmt.Sprintf("Registered agents: %s", strings.Join(ids, ", "))
default:
return fmt.Sprintf("Unknown list target: %s", args[0])
}
}
// --- /switch ----------------------------------------------------------------
func (r *Reflector) cmdSwitch(args []string, _ *MemoryStore) string {
if len(args) < 3 || args[1] != "to" {
return "Usage: /switch [model|channel] to <name>"
}
target := args[0]
value := args[2]
r.mu.RLock()
reg := r.agentRegistry
cm := r.channelManager
r.mu.RUnlock()
switch target {
case "model":
if reg == nil {
return "⚠️ Agent registry not available"
}
agent := reg.GetDefaultAgent()
if agent == nil {
return "No default agent configured"
}
oldModel := agent.Model
agent.Model = value
return fmt.Sprintf("Switched model from %s to %s", oldModel, value)
case "channel":
if cm == nil {
return "Channel manager not initialized"
}
if _, exists := cm.GetChannel(value); !exists && value != "cli" {
return fmt.Sprintf("Channel '%s' not found or not enabled", value)
}
return fmt.Sprintf("Switched target channel to %s", value)
default:
return fmt.Sprintf("Unknown switch target: %s", target)
}
}
// ===========================================================================
// Built-in processors (post-LLM, async)
// ===========================================================================
// --- ErrorTracker (no LLM) --------------------------------------------------
type ErrorTracker struct{}
func (e *ErrorTracker) Name() string { return "error_tracker" }
func (e *ErrorTracker) Process(_ context.Context, input RuntimeInput, _ *MemoryStore) error {
for _, tc := range input.ToolCalls {
if tc.Error == "" {
continue
}
logger.InfoCF("reflector", "Tool error recorded",
map[string]any{"tool": tc.Name, "error": tc.Error})
}
return nil
}
// --- CotEvaluator (LLM) ----------------------------------------------------
type CotEvaluator struct {
provider providers.LLMProvider
model string
}
func (c *CotEvaluator) Name() string { return "cot_evaluator" }
const cotEvalPrompt = `Rate how well the thinking strategy helped answer the user's question.
Question: %s
Strategy: %s
Response (first 500 chars): %s
Respond with ONLY one JSON: {"score": <-1|0|1>}
1 = good, 0 = neutral, -1 = poor`
func (c *CotEvaluator) Process(ctx context.Context, input RuntimeInput, memory *MemoryStore) error {
if input.CotPrompt == "" {
return nil
}
reply := input.AssistantReply
if len(reply) > 500 {
reply = reply[:500]
}
resp, err := c.provider.Chat(ctx, []providers.Message{
{Role: "user", Content: fmt.Sprintf(cotEvalPrompt, input.UserMessage, input.CotPrompt, reply)},
}, nil, c.model, map[string]any{"max_tokens": 32, "temperature": 0.1})
if err != nil {
return fmt.Errorf("eval LLM failed: %w", err)
}
// Parse JSON (strip markdown fences if present).
raw := strings.TrimSpace(resp.Content)
if strings.HasPrefix(raw, "```") {
lines := strings.Split(raw, "\n")
if len(lines) > 2 {
raw = strings.Join(lines[1:len(lines)-1], "\n")
}
}
var evalResult struct {
Score int `json:"score"`
}
if err := json.Unmarshal([]byte(raw), &evalResult); err != nil {
// Fallback: string matching.
if strings.Contains(raw, `"score": 1`) || strings.Contains(raw, `"score":1`) {
evalResult.Score = 1
} else if strings.Contains(raw, `"score": -1`) || strings.Contains(raw, `"score":-1`) {
evalResult.Score = -1
}
}
if evalResult.Score != 0 {
if err := memory.UpdateLatestCotFeedback(evalResult.Score); err != nil {
return err
}
logger.InfoCF("reflector", "CoT feedback auto-recorded",
map[string]any{"score": evalResult.Score, "intent": input.Intent})
}
return nil
}
// --- MemoryExtractor (LLM) --------------------------------------------------
type MemoryExtractor struct {
provider providers.LLMProvider
model string
}
func (m *MemoryExtractor) Name() string { return "memory_extractor" }
const memoryExtractPrompt = `Extract important facts worth remembering from this conversation.
User: %s
Assistant (first 800 chars): %s
Respond with ONLY JSON: {"memories": [{"content": "<fact>", "tags": ["tag1"]}]}
Rules: max 3 memories, max 3 tags each, lowercase tags, skip trivial chat.
If nothing worth remembering: {"memories": []}`
type memExtractResult struct {
Memories []struct {
Content string `json:"content"`
Tags []string `json:"tags"`
} `json:"memories"`
}
func (m *MemoryExtractor) Process(ctx context.Context, input RuntimeInput, memory *MemoryStore) error {
if len(input.UserMessage) < 20 || input.Intent == "chat" {
return nil
}
reply := input.AssistantReply
if len(reply) > 800 {
reply = reply[:800]
}
resp, err := m.provider.Chat(ctx, []providers.Message{
{Role: "user", Content: fmt.Sprintf(memoryExtractPrompt, input.UserMessage, reply)},
}, nil, m.model, map[string]any{"max_tokens": 256, "temperature": 0.1})
if err != nil {
return fmt.Errorf("memory extract LLM failed: %w", err)
}
// Parse JSON (strip markdown fences if present).
raw := strings.TrimSpace(resp.Content)
if strings.HasPrefix(raw, "```") {
lines := strings.Split(raw, "\n")
if len(lines) > 2 {
raw = strings.Join(lines[1:len(lines)-1], "\n")
}
}
var result memExtractResult
if err := json.Unmarshal([]byte(raw), &result); err != nil {
return nil // Parsing failed — skip silently.
}
for _, mem := range result.Memories {
content := strings.TrimSpace(mem.Content)
if content == "" {
continue
}
tags := make([]string, 0, len(mem.Tags))
for _, t := range mem.Tags {
t = strings.ToLower(strings.TrimSpace(t))
if t != "" {
tags = append(tags, t)
}
}
if id, err := memory.AddEntry(content, tags); err != nil {
logger.WarnCF("reflector", "Failed to save memory",
map[string]any{"error": err.Error()})
} else {
logger.InfoCF("reflector", "Memory extracted",
map[string]any{"id": id, "tags": tags, "content": content})
}
}
return nil
}

319
pkg/agent/reflector_test.go Normal file
View file

@ -0,0 +1,319 @@
package agent
import (
"os"
"strings"
"testing"
)
// --- Slash command tests ----------------------------------------------------
func TestRuntime_MemoryCommand(t *testing.T) {
dir := t.TempDir()
ms := NewMemoryStore(dir)
defer ms.Close()
r := NewReflector(nil, "")
// /memory with no args → help.
resp, ok := r.HandleCommand("/memory", ms)
if !ok {
t.Fatal("expected /memory to be handled")
}
if !strings.Contains(resp, "Usage") {
t.Error("expected usage text")
}
// /memory list → empty.
resp, ok = r.HandleCommand("/memory list", ms)
if !ok {
t.Fatal("expected /memory list to be handled")
}
if !strings.Contains(resp, "No memories") {
t.Errorf("expected empty list, got %q", resp)
}
// /memory add.
resp, ok = r.HandleCommand("/memory add Go is great for concurrency #golang #concurrency", ms)
if !ok {
t.Fatal("expected /memory add to be handled")
}
if !strings.Contains(resp, "✅") {
t.Errorf("expected success, got %q", resp)
}
if !strings.Contains(resp, "golang") {
t.Errorf("should show tags, got %q", resp)
}
// /memory list → should have 1 entry.
resp, _ = r.HandleCommand("/memory list", ms)
if !strings.Contains(resp, "Go is great") {
t.Errorf("should show entry, got %q", resp)
}
// /memory search.
resp, _ = r.HandleCommand("/memory search golang", ms)
if !strings.Contains(resp, "Found 1") {
t.Errorf("expected 1 result, got %q", resp)
}
resp, _ = r.HandleCommand("/memory search nonexistent", ms)
if !strings.Contains(resp, "No memories found") {
t.Errorf("expected no results, got %q", resp)
}
// /memory stats — should show entry count.
resp, _ = r.HandleCommand("/memory stats", ms)
if !strings.Contains(resp, "Stats") {
t.Errorf("expected stats, got %q", resp)
}
if !strings.Contains(resp, "Total entries: 1") {
t.Errorf("expected 1 entry in stats, got %q", resp)
}
// /memory edit.
resp, _ = r.HandleCommand("/memory edit 1 Updated content #go", ms)
if !strings.Contains(resp, "✅") {
t.Errorf("expected success, got %q", resp)
}
resp, _ = r.HandleCommand("/memory list", ms)
if !strings.Contains(resp, "Updated content") {
t.Errorf("edit should be reflected, got %q", resp)
}
// /memory delete.
resp, _ = r.HandleCommand("/memory delete 1", ms)
if !strings.Contains(resp, "✅") {
t.Errorf("expected success, got %q", resp)
}
resp, _ = r.HandleCommand("/memory list", ms)
if !strings.Contains(resp, "No memories") {
t.Errorf("expected empty after delete, got %q", resp)
}
}
func TestRuntime_HelpCommand(t *testing.T) {
r := NewReflector(nil, "")
dir := t.TempDir()
ms := NewMemoryStore(dir)
defer ms.Close()
resp, ok := r.HandleCommand("/help", ms)
if !ok {
t.Fatal("expected /help to be handled")
}
if !strings.Contains(resp, "/memory") {
t.Error("help should list /memory")
}
if !strings.Contains(resp, "/cot") {
t.Error("help should list /cot")
}
if !strings.Contains(resp, "/show") {
t.Error("help should list /show (now a runtime command)")
}
if !strings.Contains(resp, "/shell") {
t.Error("help should list /shell")
}
}
func TestRuntime_ShellSecurity(t *testing.T) {
r := NewReflector(nil, "")
dir := t.TempDir()
ms := NewMemoryStore(dir)
defer ms.Close()
// Unknown command (not builtin or dev tool).
resp, _ := r.HandleCommand("/shell rm -rf /", ms)
if !strings.Contains(resp, "Unknown command") {
t.Errorf("rm should be unknown, got %q", resp)
}
// Unknown: sudo
resp, _ = r.HandleCommand("/shell sudo ls", ms)
if !strings.Contains(resp, "Unknown command") {
t.Errorf("sudo should be unknown, got %q", resp)
}
// Injection via passthrough: git | bash
resp, _ = r.HandleCommand("/shell git log | bash", ms)
if !strings.Contains(resp, "blocked") {
t.Errorf("injection should be blocked, got %q", resp)
}
// Builtin echo works (cross-platform).
resp, _ = r.HandleCommand("/shell echo hello world", ms)
if !strings.Contains(resp, "hello world") {
t.Errorf("echo should work, got %q", resp)
}
}
func TestRuntime_CotCommand(t *testing.T) {
dir := t.TempDir()
ms := NewMemoryStore(dir)
defer ms.Close()
r := NewReflector(nil, "")
// /cot with no args → help.
resp, ok := r.HandleCommand("/cot", ms)
if !ok {
t.Fatal("expected /cot to be handled")
}
if !strings.Contains(resp, "Usage") {
t.Error("expected usage text")
}
// /cot stats → empty.
resp, _ = r.HandleCommand("/cot stats", ms)
if !strings.Contains(resp, "No CoT usage") {
t.Errorf("expected empty, got %q", resp)
}
// Add some usage first.
ms.RecordCotUsage("code", []string{"golang"}, "1. Think\n2. Code", "write code")
// /cot history.
resp, _ = r.HandleCommand("/cot history", ms)
if !strings.Contains(resp, "code") {
t.Errorf("expected history entry, got %q", resp)
}
// /cot feedback.
resp, _ = r.HandleCommand("/cot feedback 1", ms)
if !strings.Contains(resp, "✅") {
t.Errorf("expected success, got %q", resp)
}
// /cot feedback bad input.
resp, _ = r.HandleCommand("/cot feedback 99", ms)
if !strings.Contains(resp, "❌") {
t.Errorf("expected error, got %q", resp)
}
}
func TestRuntime_RuntimeCommand(t *testing.T) {
r := NewReflector(nil, "")
dir := t.TempDir()
ms := NewMemoryStore(dir)
defer ms.Close()
resp, ok := r.HandleCommand("/runtime status", ms)
if !ok {
t.Fatal("expected /runtime to be handled")
}
if !strings.Contains(resp, "Processors") {
t.Errorf("expected status, got %q", resp)
}
resp, _ = r.HandleCommand("/runtime processors", ms)
if !strings.Contains(resp, "error_tracker") {
t.Errorf("expected error_tracker processor, got %q", resp)
}
}
func TestRuntime_UnknownCommand(t *testing.T) {
r := NewReflector(nil, "")
// Unknown /cmd → not handled (returns false).
_, ok := r.HandleCommand("/unknown_cmd", nil)
if ok {
t.Error("expected unknown command to not be handled")
}
// Not a command at all.
_, ok = r.HandleCommand("hello world", nil)
if ok {
t.Error("expected non-command to not be handled")
}
}
func TestRuntime_ShellCommand(t *testing.T) {
r := NewReflector(nil, "")
dir := t.TempDir()
ms := NewMemoryStore(dir)
defer ms.Close()
// /shell with no args → help.
resp, ok := r.HandleCommand("/shell", ms)
if !ok {
t.Fatal("expected /shell to be handled")
}
if !strings.Contains(resp, "Usage") {
t.Errorf("expected usage, got %q", resp)
}
// /shell pwd → returns cwd (builtin, no tool registry needed).
resp, _ = r.HandleCommand("/shell pwd", ms)
if !strings.Contains(resp, string(os.PathSeparator)) {
t.Errorf("expected directory path, got %q", resp)
}
// /shell dev tool without registry → warning.
resp, _ = r.HandleCommand("/shell git status", ms)
if !strings.Contains(resp, "not available") {
t.Errorf("expected warning about no registry, got %q", resp)
}
}
// --- Post-LLM processor tests -----------------------------------------------
func TestRuntime_ErrorTracker(t *testing.T) {
tracker := &ErrorTracker{}
input := RuntimeInput{
ToolCalls: []ToolCallRecord{
{Name: "exec", Error: "command not found"},
{Name: "read_file", Error: ""},
},
}
// Should not error.
err := tracker.Process(nil, input, nil)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
}
func TestRuntime_CotEvaluator_NoCot(t *testing.T) {
eval := &CotEvaluator{}
input := RuntimeInput{CotPrompt: ""} // No CoT → skip.
err := eval.Process(nil, input, nil)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
}
func TestRuntime_MemoryExtractor_SkipChat(t *testing.T) {
extractor := &MemoryExtractor{}
input := RuntimeInput{
UserMessage: "hello",
Intent: "chat",
}
err := extractor.Process(nil, input, nil)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
}
func TestRuntime_PostLLM_NilSafety(t *testing.T) {
// Nil runtime should not panic.
var r *Reflector
r.RunPostLLM(RuntimeInput{}, nil) // Should be no-op.
// Runtime with no processors.
r = &Reflector{commands: map[string]CommandDef{}}
r.RunPostLLM(RuntimeInput{}, nil) // Should be no-op.
}
func TestRuntime_ListCommands(t *testing.T) {
r := NewReflector(nil, "")
text := r.ListCommands()
if !strings.Contains(text, "/memory") {
t.Error("should list /memory command")
}
if !strings.Contains(text, "/cot") {
t.Error("should list /cot command")
}
if !strings.Contains(text, "/runtime") {
t.Error("should list /runtime command")
}
}

74
pkg/agent/score.go Normal file
View file

@ -0,0 +1,74 @@
// PicoClaw - Ultra-lightweight personal AI agent
// License: MIT
//
// Copyright (c) 2026 PicoClaw contributors
package agent
import "strings"
// CalcTurnScore computes a value score for a completed turn.
//
// Scoring rules (range roughly -2 to 15):
//
// +3 has tool calls
// +2 has write/edit/append tool call (modifying tools)
// +2 tool count > 3
// +3 intent = task / code / debug
// +1 intent = question
// +0 intent = chat (or empty)
// +2 reply length > 500 chars
// -2 user + reply total < 80 chars
// +3 user message contains "记住" or "重要" (remember / important)
//
// alwaysKeepThreshold (≥ 7) marks a Turn as always_keep in instant memory.
func CalcTurnScore(input RuntimeInput) int {
score := 0
// --- Tool activity ---
if len(input.ToolCalls) > 0 {
score += 3
}
for _, tc := range input.ToolCalls {
n := strings.ToLower(tc.Name)
if n == "write_file" || n == "edit_file" || n == "append_file" ||
n == "write" || n == "edit" || n == "append" {
score += 2
break // count once
}
}
if len(input.ToolCalls) > 3 {
score += 2
}
// --- Intent weight ---
switch strings.ToLower(input.Intent) {
case "task", "code", "debug":
score += 3
case "question":
score += 1
// "chat" or empty: 0
}
// --- Content density ---
if len(input.AssistantReply) > 500 {
score += 2
}
if len(input.UserMessage)+len(input.AssistantReply) < 80 {
score -= 2
}
// --- Explicit importance markers ---
if strings.Contains(input.UserMessage, "记住") ||
strings.Contains(input.UserMessage, "重要") ||
strings.Contains(strings.ToLower(input.UserMessage), "remember") ||
strings.Contains(strings.ToLower(input.UserMessage), "important") {
score += 3
}
return score
}
// alwaysKeepThreshold is the minimum score for a Turn to be unconditionally
// included in instant memory (regardless of tag matching).
const alwaysKeepThreshold = 7

143
pkg/agent/score_test.go Normal file
View file

@ -0,0 +1,143 @@
package agent
import (
"strings"
"testing"
)
func TestCalcTurnScore_BasicRules(t *testing.T) {
tests := []struct {
name string
input RuntimeInput
wantMin int
wantMax int
wantExact *int
}{
{
name: "empty chat",
input: RuntimeInput{Intent: "chat", UserMessage: "ok", AssistantReply: "ok"},
// score = 0 (chat) -2 (< 80 chars total) = -2
wantExact: intPtr(-2),
},
{
name: "question intent, short",
input: RuntimeInput{Intent: "question", UserMessage: "hi", AssistantReply: "hello"},
// score = 1 (question) -2 (short) = -1
wantExact: intPtr(-1),
},
{
name: "task with tool call",
input: RuntimeInput{
Intent: "task",
UserMessage: "do something important",
AssistantReply: "done",
ToolCalls: []ToolCallRecord{{Name: "exec"}},
},
// +3 (task) +3 (has tool) +3 ("important" keyword) -2 (short) = 7
wantExact: intPtr(7),
},
{
name: "code with write tool",
input: RuntimeInput{
Intent: "code",
UserMessage: "fix the bug",
AssistantReply: "fixed",
ToolCalls: []ToolCallRecord{{Name: "write_file"}},
},
// +3 (code) +3 (has tool) +2 (write tool) -2 (short) = 6
wantExact: intPtr(6),
},
{
name: "many tools",
input: RuntimeInput{
Intent: "debug",
UserMessage: "debug it",
AssistantReply: "ok",
ToolCalls: []ToolCallRecord{
{Name: "exec"},
{Name: "read_file"},
{Name: "list_dir"},
{Name: "exec"},
},
},
// +3 (debug) +3 (has tool) +2 (>3 tools) -2 (short) = 6
wantExact: intPtr(6),
},
{
name: "long reply",
input: RuntimeInput{
Intent: "question",
UserMessage: "explain",
AssistantReply: strings.Repeat("a", 600),
},
// +1 (question) +2 (long reply) [total<80 does not apply because reply is 600]
// total chars = 7 + 600 = 607 >= 80
wantExact: intPtr(3),
},
{
name: "explicit remember keyword",
input: RuntimeInput{
Intent: "chat",
UserMessage: "记住这个地址 localhost:3000",
AssistantReply: strings.Repeat("a", 600),
},
// 0(chat) +3 (记住) +2 (long reply) = 5
wantExact: intPtr(5),
},
{
name: "explicit important keyword",
input: RuntimeInput{
Intent: "question",
UserMessage: "this is IMPORTANT: use port 8080",
AssistantReply: "ok",
},
// 1 (question) + 3 (important) - 2 (short) = 2
wantExact: intPtr(2),
},
{
name: "always_keep threshold: full scoring",
input: RuntimeInput{
Intent: "task",
UserMessage: "run the deployment pipeline for staging and fix it",
AssistantReply: strings.Repeat("a", 600),
ToolCalls: []ToolCallRecord{
{Name: "edit_file"},
{Name: "exec"},
{Name: "exec"},
{Name: "exec"},
},
},
// +3(task) +3(tool) +2(write/edit) +2(>3 tools) +2(long reply) = 12
wantExact: intPtr(12),
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := CalcTurnScore(tc.input)
if tc.wantExact != nil {
if got != *tc.wantExact {
t.Errorf("CalcTurnScore() = %d, want %d", got, *tc.wantExact)
}
} else if got < tc.wantMin || (tc.wantMax > 0 && got > tc.wantMax) {
t.Errorf("CalcTurnScore() = %d, want [%d, %d]", got, tc.wantMin, tc.wantMax)
}
})
}
}
func TestAlwaysKeepThreshold(t *testing.T) {
// High-value turn must meet or exceed the threshold.
highValue := RuntimeInput{
Intent: "task",
UserMessage: "deploy staging",
AssistantReply: strings.Repeat("a", 600),
ToolCalls: []ToolCallRecord{{Name: "edit_file"}, {Name: "exec"}},
}
score := CalcTurnScore(highValue)
if score < alwaysKeepThreshold {
t.Errorf("expected score %d >= alwaysKeepThreshold %d", score, alwaysKeepThreshold)
}
}
func intPtr(i int) *int { return &i }

300
pkg/agent/turn_store.go Normal file
View file

@ -0,0 +1,300 @@
// PicoClaw - Ultra-lightweight personal AI agent
// License: MIT
//
// Copyright (c) 2026 PicoClaw contributors
package agent
import (
"database/sql"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/sipeed/picoclaw/pkg/logger"
_ "modernc.org/sqlite"
)
// TurnRecord captures everything that happened during a single completed turn.
// It is persisted to turns.db for use by MemoryDigest and instant-memory assembly.
type TurnRecord struct {
ID string // ULID or time-based unique ID
Ts int64 // Unix timestamp (seconds)
ChannelKey string // "channel:chatID"
Score int // Phase 3 CalcTurnScore result
Intent string // Phase 1 detected intent
Tags []string // Phase 1 detected tags
Tokens int // rough token estimate (chars / 3)
Status string // "pending" | "processed" | "archived"
UserMsg string // original user message
Reply string // assistant final response
ToolCalls []ToolCallRecord // serialised as JSON in DB
}
// TurnStore manages persistent Turn storage in SQLite.
// The DB lives at {workspace}/turns.db, mirroring the memory.db pattern.
type TurnStore struct {
db *sql.DB
}
const turnsDDL = `
CREATE TABLE IF NOT EXISTS turns (
id TEXT PRIMARY KEY,
ts INTEGER NOT NULL,
channel_key TEXT NOT NULL DEFAULT '',
score INTEGER NOT NULL DEFAULT 0,
intent TEXT NOT NULL DEFAULT '',
tags TEXT NOT NULL DEFAULT '[]',
tokens INTEGER NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'pending',
user_msg TEXT NOT NULL DEFAULT '',
reply TEXT NOT NULL DEFAULT '',
tool_calls TEXT NOT NULL DEFAULT '[]'
);
CREATE INDEX IF NOT EXISTS idx_turns_status ON turns(status);
CREATE INDEX IF NOT EXISTS idx_turns_ts ON turns(ts);
CREATE INDEX IF NOT EXISTS idx_turns_channel ON turns(channel_key);
CREATE INDEX IF NOT EXISTS idx_turns_score ON turns(score);
`
// NewTurnStore creates (or opens) turns.db in the given workspace directory.
func NewTurnStore(workspace string) (*TurnStore, error) {
if err := os.MkdirAll(workspace, 0o755); err != nil {
return nil, fmt.Errorf("turn_store: mkdir %s: %w", workspace, err)
}
dbPath := filepath.Join(workspace, "turns.db")
db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(wal)&_pragma=busy_timeout(5000)")
if err != nil {
return nil, fmt.Errorf("turn_store: open %s: %w", dbPath, err)
}
if _, err := db.Exec(turnsDDL); err != nil {
db.Close()
return nil, fmt.Errorf("turn_store: init schema: %w", err)
}
return &TurnStore{db: db}, nil
}
// Close shuts down the underlying DB connection.
func (s *TurnStore) Close() error {
if s.db != nil {
return s.db.Close()
}
return nil
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
func marshalJSON(v any) string {
b, err := json.Marshal(v)
if err != nil {
return "[]"
}
return string(b)
}
func unmarshalTags(raw string) []string {
var tags []string
_ = json.Unmarshal([]byte(raw), &tags)
return tags
}
func unmarshalToolCalls(raw string) []ToolCallRecord {
var tcs []ToolCallRecord
_ = json.Unmarshal([]byte(raw), &tcs)
return tcs
}
// estimateTokens gives a cheap estimate: characters / 3.
func estimateTokens(r TurnRecord) int {
chars := len(r.UserMsg) + len(r.Reply)
for _, tc := range r.ToolCalls {
chars += len(tc.Name) + len(tc.Error)
}
if chars < 3 {
return 1
}
return chars / 3
}
// NewTurnID generates a time-sortable unique ID without external dependencies.
// Format: unixMilli-randomSuffix using millisecond precision.
func NewTurnID() string {
return fmt.Sprintf("%d-%d", time.Now().UnixMilli(), time.Now().Nanosecond()%1_000_000)
}
// ---------------------------------------------------------------------------
// Writes
// ---------------------------------------------------------------------------
// Insert persists a TurnRecord to the DB.
// The record's ID and Ts are set if empty/zero.
func (s *TurnStore) Insert(r TurnRecord) error {
if r.ID == "" {
r.ID = NewTurnID()
}
if r.Ts == 0 {
r.Ts = time.Now().Unix()
}
if r.Status == "" {
r.Status = "pending"
}
if r.Tokens == 0 {
r.Tokens = estimateTokens(r)
}
tagsJSON := marshalJSON(r.Tags)
tcJSON := marshalJSON(r.ToolCalls)
_, err := s.db.Exec(`
INSERT INTO turns (id, ts, channel_key, score, intent, tags, tokens, status, user_msg, reply, tool_calls)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO NOTHING`,
r.ID, r.Ts, r.ChannelKey, r.Score, r.Intent,
tagsJSON, r.Tokens, r.Status, r.UserMsg, r.Reply, tcJSON,
)
if err != nil {
return fmt.Errorf("turn_store: insert %s: %w", r.ID, err)
}
logger.DebugCF("turn_store", "Turn inserted",
map[string]any{"id": r.ID, "score": r.Score, "tokens": r.Tokens, "status": r.Status})
return nil
}
// SetStatus updates the status of a turn by ID.
func (s *TurnStore) SetStatus(id, status string) error {
_, err := s.db.Exec("UPDATE turns SET status = ? WHERE id = ?", status, id)
return err
}
// ---------------------------------------------------------------------------
// Queries — used by MemoryDigest and instant-memory assembly
// ---------------------------------------------------------------------------
// QueryPending returns up to limit turns with status = 'pending', ordered oldest first.
func (s *TurnStore) QueryPending(limit int) ([]TurnRecord, error) {
rows, err := s.db.Query(`
SELECT id, ts, channel_key, score, intent, tags, tokens, status, user_msg, reply, tool_calls
FROM turns WHERE status = 'pending'
ORDER BY ts ASC LIMIT ?`, limit)
if err != nil {
return nil, err
}
defer rows.Close()
return scanTurns(rows)
}
// QueryByScore returns all turns with score >= highThreshold (always_keep),
// ordered by ts ASC.
func (s *TurnStore) QueryByScore(highThreshold int) ([]TurnRecord, error) {
rows, err := s.db.Query(`
SELECT id, ts, channel_key, score, intent, tags, tokens, status, user_msg, reply, tool_calls
FROM turns WHERE score >= ? AND status != 'archived'
ORDER BY ts ASC`, highThreshold)
if err != nil {
return nil, err
}
defer rows.Close()
return scanTurns(rows)
}
// QueryByTags returns turns whose tags JSON contains at least one of the given tags
// and score > 0, ordered by ts ASC.
func (s *TurnStore) QueryByTags(tags []string) ([]TurnRecord, error) {
if len(tags) == 0 {
return nil, nil
}
// Build LIKE conditions for simple JSON array matching.
conds := make([]string, 0, len(tags))
args := make([]any, 0, len(tags)*2)
for _, t := range tags {
t = strings.ToLower(strings.TrimSpace(t))
if t == "" {
continue
}
conds = append(conds, `(tags LIKE ? OR tags LIKE ?)`)
args = append(args, `%"`+t+`"%`, `%'`+t+`'%`)
}
if len(conds) == 0 {
return nil, nil
}
// Append non-archived filter.
query := fmt.Sprintf(`
SELECT id, ts, channel_key, score, intent, tags, tokens, status, user_msg, reply, tool_calls
FROM turns
WHERE score > 0 AND status != 'archived' AND (%s)
ORDER BY ts ASC`, strings.Join(conds, " OR "))
rows, err := s.db.Query(query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
return scanTurns(rows)
}
// QueryRecent returns the n most-recent non-archived turns for a channelKey,
// ordered by ts ASC (oldest first, so they can be appended naturally).
func (s *TurnStore) QueryRecent(channelKey string, n int) ([]TurnRecord, error) {
rows, err := s.db.Query(`
SELECT id, ts, channel_key, score, intent, tags, tokens, status, user_msg, reply, tool_calls
FROM turns
WHERE channel_key = ? AND status != 'archived'
ORDER BY ts DESC LIMIT ?`, channelKey, n)
if err != nil {
return nil, err
}
defer rows.Close()
turns, err := scanTurns(rows)
if err != nil {
return nil, err
}
// Reverse to ascending order.
for i, j := 0, len(turns)-1; i < j; i, j = i+1, j-1 {
turns[i], turns[j] = turns[j], turns[i]
}
return turns, nil
}
// ArchiveOldProcessed marks processed turns older than olderThanDays as 'archived'.
// At most 100 rows are archived per call to limit lock time.
func (s *TurnStore) ArchiveOldProcessed(olderThanDays int) error {
cutoff := time.Now().AddDate(0, 0, -olderThanDays).Unix()
_, err := s.db.Exec(`
UPDATE turns SET status = 'archived'
WHERE id IN (
SELECT id FROM turns
WHERE status = 'processed' AND ts < ?
ORDER BY ts ASC LIMIT 100
)`, cutoff)
return err
}
// ---------------------------------------------------------------------------
// Internal scanner
// ---------------------------------------------------------------------------
func scanTurns(rows *sql.Rows) ([]TurnRecord, error) {
var out []TurnRecord
for rows.Next() {
var r TurnRecord
var tagsJSON, tcJSON string
if err := rows.Scan(
&r.ID, &r.Ts, &r.ChannelKey, &r.Score, &r.Intent,
&tagsJSON, &r.Tokens, &r.Status,
&r.UserMsg, &r.Reply, &tcJSON,
); err != nil {
return out, err
}
r.Tags = unmarshalTags(tagsJSON)
r.ToolCalls = unmarshalToolCalls(tcJSON)
out = append(out, r)
}
return out, rows.Err()
}

View file

@ -0,0 +1,143 @@
package agent
import (
"testing"
"time"
)
func TestTurnStore_InsertAndQueryRecent(t *testing.T) {
dir := t.TempDir()
store, err := NewTurnStore(dir)
if err != nil {
t.Fatalf("NewTurnStore: %v", err)
}
defer store.Close()
r := TurnRecord{
Ts: time.Now().Unix(),
ChannelKey: "cli:direct",
Score: 5,
Intent: "task",
Tags: []string{"deploy", "ci"},
UserMsg: "deploy now",
Reply: "done",
ToolCalls: []ToolCallRecord{{Name: "exec", Error: ""}},
Status: "pending",
}
if err := store.Insert(r); err != nil {
t.Fatalf("Insert: %v", err)
}
rows, err := store.QueryRecent("cli:direct", 10)
if err != nil {
t.Fatalf("QueryRecent: %v", err)
}
if len(rows) != 1 {
t.Fatalf("expected 1 row, got %d", len(rows))
}
if rows[0].Intent != "task" {
t.Errorf("unexpected intent: %s", rows[0].Intent)
}
if len(rows[0].Tags) != 2 {
t.Errorf("expected 2 tags, got %v", rows[0].Tags)
}
}
func TestTurnStore_QueryByScore(t *testing.T) {
dir := t.TempDir()
store, err := NewTurnStore(dir)
if err != nil {
t.Fatalf("NewTurnStore: %v", err)
}
defer store.Close()
now := time.Now().Unix()
store.Insert(TurnRecord{ID: "s-1", Ts: now, Score: 3, UserMsg: "a", Reply: "b", Status: "pending"})
store.Insert(TurnRecord{ID: "s-2", Ts: now + 1, Score: 8, UserMsg: "c", Reply: "d", Status: "pending"})
store.Insert(TurnRecord{ID: "s-3", Ts: now + 2, Score: 9, UserMsg: "e", Reply: "f", Status: "pending"})
high, err := store.QueryByScore(7)
if err != nil {
t.Fatalf("QueryByScore: %v", err)
}
if len(high) != 2 {
t.Errorf("expected 2 always_keep turns, got %d", len(high))
}
}
func TestTurnStore_SetStatus(t *testing.T) {
dir := t.TempDir()
store, err := NewTurnStore(dir)
if err != nil {
t.Fatalf("NewTurnStore: %v", err)
}
defer store.Close()
r := TurnRecord{ID: "test-id-1", Ts: time.Now().Unix(), UserMsg: "x", Reply: "y", Status: "pending"}
store.Insert(r)
if err := store.SetStatus("test-id-1", "processed"); err != nil {
t.Fatalf("SetStatus: %v", err)
}
pending, err := store.QueryPending(10)
if err != nil {
t.Fatalf("QueryPending: %v", err)
}
if len(pending) != 0 {
t.Errorf("expected 0 pending, got %d", len(pending))
}
}
func TestTurnStore_ArchiveOldProcessed(t *testing.T) {
dir := t.TempDir()
store, err := NewTurnStore(dir)
if err != nil {
t.Fatalf("NewTurnStore: %v", err)
}
defer store.Close()
// Insert old processed turns (timestamp in the past).
old := time.Now().AddDate(0, 0, -10).Unix()
for i := 0; i < 3; i++ {
r := TurnRecord{Ts: old, Score: 2, UserMsg: "old", Reply: "msg", Status: "processed"}
store.Insert(r)
}
// Recent processed turn — should NOT be archived.
recent := TurnRecord{Ts: time.Now().Unix(), Score: 2, UserMsg: "new", Reply: "msg", Status: "processed"}
store.Insert(recent)
if err := store.ArchiveOldProcessed(7); err != nil {
t.Fatalf("ArchiveOldProcessed: %v", err)
}
// Query pending (should still be 0).
pending, _ := store.QueryPending(100)
if len(pending) != 0 {
t.Errorf("expected 0 pending after archive, got %d", len(pending))
}
}
func TestTurnStore_QueryByTags(t *testing.T) {
dir := t.TempDir()
store, err := NewTurnStore(dir)
if err != nil {
t.Fatalf("NewTurnStore: %v", err)
}
defer store.Close()
now := time.Now().Unix()
store.Insert(TurnRecord{ID: "tag-1", Ts: now, Score: 5, Tags: []string{"deploy", "ci"}, UserMsg: "a", Reply: "b"})
store.Insert(TurnRecord{ID: "tag-2", Ts: now + 1, Score: 4, Tags: []string{"file", "read"}, UserMsg: "c", Reply: "d"})
store.Insert(TurnRecord{ID: "tag-3", Ts: now + 2, Score: 3, Tags: []string{"deploy", "log"}, UserMsg: "e", Reply: "f"})
rows, err := store.QueryByTags([]string{"deploy"})
if err != nil {
t.Fatalf("QueryByTags: %v", err)
}
if len(rows) < 2 {
t.Errorf("expected at least 2 deploy turns, got %d", len(rows))
}
}

View file

@ -51,13 +51,20 @@ type Config struct {
Agents AgentsConfig `json:"agents"` Agents AgentsConfig `json:"agents"`
Bindings []AgentBinding `json:"bindings,omitempty"` Bindings []AgentBinding `json:"bindings,omitempty"`
Session SessionConfig `json:"session,omitempty"` Session SessionConfig `json:"session,omitempty"`
Channels ChannelsConfig `json:"channels"` Channels ChannelsConfig `json:"channels,omitempty"`
Providers ProvidersConfig `json:"providers,omitempty"` Providers ProvidersConfig `json:"providers,omitempty"`
ModelList []ModelConfig `json:"model_list"` // New model-centric provider configuration ModelList []ModelConfig `json:"model_list,omitempty"`
Gateway GatewayConfig `json:"gateway"` Gateway GatewayConfig `json:"gateway,omitempty"`
Tools ToolsConfig `json:"tools"` Tools ToolsConfig `json:"tools,omitempty"`
Heartbeat HeartbeatConfig `json:"heartbeat"` Heartbeat HeartbeatConfig `json:"heartbeat,omitempty"`
Devices DevicesConfig `json:"devices"` Devices DevicesConfig `json:"devices,omitempty"`
Logging LoggingConfig `json:"logging,omitempty"`
}
// LoggingConfig controls log output.
type LoggingConfig struct {
Level string `json:"level,omitempty"` // debug, info, warn, error (default: warn)
FileDir string `json:"file_dir,omitempty"` // directory for log files; empty = no file logging
} }
// MarshalJSON implements custom JSON marshaling for Config // MarshalJSON implements custom JSON marshaling for Config
@ -175,6 +182,16 @@ type AgentDefaults struct {
ModelName string `json:"model_name,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_MODEL_NAME"` ModelName string `json:"model_name,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_MODEL_NAME"`
Model string `json:"model" env:"PICOCLAW_AGENTS_DEFAULTS_MODEL"` // Deprecated: use model_name instead Model string `json:"model" env:"PICOCLAW_AGENTS_DEFAULTS_MODEL"` // Deprecated: use model_name instead
ModelFallbacks []string `json:"model_fallbacks,omitempty"` ModelFallbacks []string `json:"model_fallbacks,omitempty"`
// Phase 1 — Analyser: lightweight model for intent/tag analysis + CoT strategy.
// Falls back to main model_name if empty. Use a cheap/fast model here.
AnalyserModel string `json:"analyser_model,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_ANALYSER_MODEL"`
PreLLMModel string `json:"pre_llm_model,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_PRE_LLM_MODEL"` // Deprecated: use analyser_model
// Phase 3 — Digest: lightweight model for memory extraction from turn records.
// Falls back to main model_name if empty. Use a cheap/fast model here.
DigestModel string `json:"digest_model,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_DIGEST_MODEL"`
ImageModel string `json:"image_model,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_IMAGE_MODEL"` ImageModel string `json:"image_model,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_IMAGE_MODEL"`
ImageModelFallbacks []string `json:"image_model_fallbacks,omitempty"` ImageModelFallbacks []string `json:"image_model_fallbacks,omitempty"`
MaxTokens int `json:"max_tokens" env:"PICOCLAW_AGENTS_DEFAULTS_MAX_TOKENS"` MaxTokens int `json:"max_tokens" env:"PICOCLAW_AGENTS_DEFAULTS_MAX_TOKENS"`
@ -191,6 +208,27 @@ func (d *AgentDefaults) GetModelName() string {
return d.Model return d.Model
} }
// GetAnalyserModel returns the model for Phase 1 (Analyser).
// Priority: analyser_model → pre_llm_model (deprecated) → main model.
func (d *AgentDefaults) GetAnalyserModel() string {
if d.AnalyserModel != "" {
return d.AnalyserModel
}
if d.PreLLMModel != "" {
return d.PreLLMModel
}
return d.GetModelName()
}
// GetDigestModel returns the model for Phase 3 (MemoryDigest).
// Priority: digest_model → main model.
func (d *AgentDefaults) GetDigestModel() string {
if d.DigestModel != "" {
return d.DigestModel
}
return d.GetModelName()
}
type ChannelsConfig struct { type ChannelsConfig struct {
WhatsApp WhatsAppConfig `json:"whatsapp"` WhatsApp WhatsAppConfig `json:"whatsapp"`
Telegram TelegramConfig `json:"telegram"` Telegram TelegramConfig `json:"telegram"`

View file

@ -1,16 +1,27 @@
// Package constants provides shared constants across the codebase. // Package constants provides shared constants across the codebase.
package constants package constants
import "strings"
// internalChannels defines channels that are used for internal communication // internalChannels defines channels that are used for internal communication
// and should not be exposed to external users or recorded as last active channel. // and should not be exposed to external users or recorded as last active channel.
var internalChannels = map[string]struct{}{ var internalChannels = map[string]struct{}{
"cli": {}, "cli": {},
"system": {}, "system": {},
"subagent": {}, "subagent": {},
"launcher": {},
} }
// IsInternalChannel returns true if the channel is an internal channel. // IsInternalChannel returns true if the channel is an internal channel.
// Supports compound names like "launcher:chat" by checking the prefix before ":".
func IsInternalChannel(channel string) bool { func IsInternalChannel(channel string) bool {
_, found := internalChannels[channel] if _, found := internalChannels[channel]; found {
return found return true
}
// Check prefix for compound channel names (e.g. "launcher:chat")
if idx := strings.IndexByte(channel, ':'); idx > 0 {
_, found := internalChannels[channel[:idx]]
return found
}
return false
} }

View file

@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"log" "log"
"os" "os"
"path/filepath"
"runtime" "runtime"
"strings" "strings"
"sync" "sync"
@ -30,7 +31,7 @@ var (
FATAL: "FATAL", FATAL: "FATAL",
} }
currentLevel = INFO currentLevel = WARN
logger *Logger logger *Logger
once sync.Once once sync.Once
mu sync.RWMutex mu sync.RWMutex
@ -61,6 +62,32 @@ func SetLevel(level LogLevel) {
currentLevel = level currentLevel = level
} }
// SetLevelByName sets log level from a string: "debug", "info", "warn", "error".
func SetLevelByName(name string) {
switch strings.ToLower(strings.TrimSpace(name)) {
case "debug":
SetLevel(DEBUG)
case "info":
SetLevel(INFO)
case "warn", "warning":
SetLevel(WARN)
case "error":
SetLevel(ERROR)
}
}
// ApplyConfig sets level and file logging from config values.
func ApplyConfig(level, fileDir string) {
if level != "" {
SetLevelByName(level)
}
if fileDir != "" {
logFile := filepath.Join(fileDir, "picoclaw.log")
os.MkdirAll(fileDir, 0755)
_ = EnableFileLogging(logFile)
}
}
func GetLevel() LogLevel { func GetLevel() LogLevel {
mu.RLock() mu.RLock()
defer mu.RUnlock() defer mu.RUnlock()

View file

@ -66,7 +66,8 @@ type Message struct {
Role string `json:"role"` Role string `json:"role"`
Content string `json:"content"` Content string `json:"content"`
ReasoningContent string `json:"reasoning_content,omitempty"` ReasoningContent string `json:"reasoning_content,omitempty"`
SystemParts []ContentBlock `json:"system_parts,omitempty"` // structured system blocks for cache-aware adapters SystemParts []ContentBlock `json:"system_parts,omitempty"` // structured system blocks for cache-aware adapters
CacheControl string `json:"cache_control,omitempty"` // "ephemeral" | "", Anthropic adapter translates
ToolCalls []ToolCall `json:"tool_calls,omitempty"` ToolCalls []ToolCall `json:"tool_calls,omitempty"`
ToolCallID string `json:"tool_call_id,omitempty"` ToolCallID string `json:"tool_call_id,omitempty"`
} }

732
pkg/shell/commands.go Normal file
View file

@ -0,0 +1,732 @@
package shell
import (
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"time"
)
// CmdFunc is the signature for a built-in shell command.
// It receives the arguments (after the command name) and the working directory.
type CmdFunc func(args []string, cwd string) string
// BuiltinCmds maps command names to their Go implementations.
// These run cross-platform without external dependencies.
var BuiltinCmds = map[string]CmdFunc{
"ls": cmdLs,
"dir": cmdLs,
"cat": cmdCat,
"type": cmdCat,
"head": cmdHead,
"tail": cmdTail,
"grep": cmdGrep,
"wc": cmdWc,
"find": cmdFind,
"pwd": cmdPwd,
"echo": cmdEcho,
"stat": cmdStat,
"diff": cmdDiff,
"tree": cmdTree,
"touch": cmdTouch,
"mkdir": cmdMkdir,
"cp": cmdCp,
"mv": cmdMv,
}
// DevToolPassthrough lists commands that pass through to the system shell.
var DevToolPassthrough = map[string]bool{
"go": true, "git": true, "node": true, "python": true, "python3": true,
"npm": true, "npx": true, "cargo": true, "make": true,
"jq": true, "rg": true, "ag": true, "ack": true, "fd": true,
}
// ---------------------------------------------------------------------------
// ls / dir
// ---------------------------------------------------------------------------
func cmdLs(args []string, cwd string) string {
dir := cwd
showAll := false
longFmt := false
for _, a := range args {
switch {
case a == "-a":
showAll = true
case a == "-l":
longFmt = true
case a == "-la" || a == "-al":
showAll = true
longFmt = true
case !strings.HasPrefix(a, "-"):
dir = ResolvePath(a, cwd)
}
}
entries, err := os.ReadDir(dir)
if err != nil {
return fmt.Sprintf("ls: %v", err)
}
var sb strings.Builder
for _, e := range entries {
name := e.Name()
if !showAll && strings.HasPrefix(name, ".") {
continue
}
if longFmt {
info, _ := e.Info()
if info != nil {
mode := info.Mode().String()
size := info.Size()
mod := info.ModTime().Format("Jan 02 15:04")
if e.IsDir() {
name += "/"
}
fmt.Fprintf(&sb, "%s %8d %s %s\n", mode, size, mod, name)
} else {
fmt.Fprintf(&sb, "%s\n", name)
}
} else {
if e.IsDir() {
name += "/"
}
sb.WriteString(name + "\n")
}
}
if sb.Len() == 0 {
return "(empty directory)"
}
return sb.String()
}
// ---------------------------------------------------------------------------
// cat / type
// ---------------------------------------------------------------------------
func cmdCat(args []string, cwd string) string {
if len(args) == 0 {
return "cat: missing file operand"
}
var sb strings.Builder
for _, f := range args {
if strings.HasPrefix(f, "-") {
continue
}
data, err := os.ReadFile(ResolvePath(f, cwd))
if err != nil {
fmt.Fprintf(&sb, "cat: %v\n", err)
continue
}
sb.Write(data)
}
return sb.String()
}
// ---------------------------------------------------------------------------
// head
// ---------------------------------------------------------------------------
func cmdHead(args []string, cwd string) string {
n := 10
var file string
for i := 0; i < len(args); i++ {
if args[i] == "-n" && i+1 < len(args) {
n, _ = strconv.Atoi(args[i+1])
i++
} else if !strings.HasPrefix(args[i], "-") {
file = args[i]
}
}
if file == "" {
return "head: missing file"
}
data, err := os.ReadFile(ResolvePath(file, cwd))
if err != nil {
return fmt.Sprintf("head: %v", err)
}
lines := strings.SplitN(string(data), "\n", n+1)
if len(lines) > n {
lines = lines[:n]
}
return strings.Join(lines, "\n")
}
// ---------------------------------------------------------------------------
// tail
// ---------------------------------------------------------------------------
func cmdTail(args []string, cwd string) string {
n := 10
var file string
for i := 0; i < len(args); i++ {
if args[i] == "-n" && i+1 < len(args) {
n, _ = strconv.Atoi(args[i+1])
i++
} else if !strings.HasPrefix(args[i], "-") {
file = args[i]
}
}
if file == "" {
return "tail: missing file"
}
data, err := os.ReadFile(ResolvePath(file, cwd))
if err != nil {
return fmt.Sprintf("tail: %v", err)
}
lines := strings.Split(strings.TrimRight(string(data), "\n"), "\n")
start := len(lines) - n
if start < 0 {
start = 0
}
return strings.Join(lines[start:], "\n")
}
// ---------------------------------------------------------------------------
// grep
// ---------------------------------------------------------------------------
func cmdGrep(args []string, cwd string) string {
ignoreCase := false
showLineNum := false
recursive := false
var pattern string
var paths []string
for i := 0; i < len(args); i++ {
a := args[i]
if strings.HasPrefix(a, "-") && pattern == "" {
for _, ch := range a[1:] {
switch ch {
case 'i':
ignoreCase = true
case 'n':
showLineNum = true
case 'r', 'R':
recursive = true
}
}
} else if pattern == "" {
pattern = a
} else {
paths = append(paths, a)
}
}
if pattern == "" {
return "grep: missing pattern"
}
if len(paths) == 0 {
paths = []string{"."}
}
pat := pattern
if ignoreCase {
pat = "(?i)" + pat
}
re, err := regexp.Compile(pat)
if err != nil {
return fmt.Sprintf("grep: invalid pattern: %v", err)
}
var sb strings.Builder
matchCount := 0
maxMatches := 200
var searchFile func(path string)
searchFile = func(path string) {
if matchCount >= maxMatches {
return
}
data, err := os.ReadFile(path)
if err != nil {
return
}
if IsBinary(data) {
return
}
relPath, _ := filepath.Rel(cwd, path)
if relPath == "" {
relPath = path
}
lines := strings.Split(string(data), "\n")
for i, line := range lines {
if matchCount >= maxMatches {
break
}
if re.MatchString(line) {
matchCount++
if showLineNum {
fmt.Fprintf(&sb, "%s:%d:%s\n", relPath, i+1, line)
} else {
fmt.Fprintf(&sb, "%s:%s\n", relPath, line)
}
}
}
}
skipDirs := map[string]bool{".git": true, "node_modules": true, "vendor": true, "__pycache__": true}
for _, p := range paths {
resolved := ResolvePath(p, cwd)
info, err := os.Stat(resolved)
if err != nil {
fmt.Fprintf(&sb, "grep: %v\n", err)
continue
}
if info.IsDir() {
if !recursive {
fmt.Fprintf(&sb, "grep: %s: is a directory\n", p)
continue
}
_ = filepath.Walk(resolved, func(path string, fi os.FileInfo, err error) error {
if err != nil {
return nil
}
if fi.IsDir() {
if skipDirs[fi.Name()] || strings.HasPrefix(fi.Name(), ".") {
return filepath.SkipDir
}
return nil
}
searchFile(path)
return nil
})
} else {
searchFile(resolved)
}
}
if matchCount == 0 {
return "(no matches)"
}
if matchCount >= maxMatches {
fmt.Fprintf(&sb, "\n... (truncated at %d matches)\n", maxMatches)
}
return sb.String()
}
// ---------------------------------------------------------------------------
// wc
// ---------------------------------------------------------------------------
func cmdWc(args []string, cwd string) string {
countLines := false
countWords := false
countBytes := false
var files []string
for _, a := range args {
if strings.HasPrefix(a, "-") {
for _, ch := range a[1:] {
switch ch {
case 'l':
countLines = true
case 'w':
countWords = true
case 'c':
countBytes = true
}
}
} else {
files = append(files, a)
}
}
if !countLines && !countWords && !countBytes {
countLines, countWords, countBytes = true, true, true
}
if len(files) == 0 {
return "wc: missing file"
}
var sb strings.Builder
totalL, totalW, totalB := 0, 0, 0
for _, f := range files {
data, err := os.ReadFile(ResolvePath(f, cwd))
if err != nil {
fmt.Fprintf(&sb, "wc: %v\n", err)
continue
}
l := strings.Count(string(data), "\n")
w := len(strings.Fields(string(data)))
b := len(data)
totalL += l
totalW += w
totalB += b
var parts []string
if countLines {
parts = append(parts, fmt.Sprintf("%7d", l))
}
if countWords {
parts = append(parts, fmt.Sprintf("%7d", w))
}
if countBytes {
parts = append(parts, fmt.Sprintf("%7d", b))
}
fmt.Fprintf(&sb, "%s %s\n", strings.Join(parts, ""), f)
}
if len(files) > 1 {
var parts []string
if countLines {
parts = append(parts, fmt.Sprintf("%7d", totalL))
}
if countWords {
parts = append(parts, fmt.Sprintf("%7d", totalW))
}
if countBytes {
parts = append(parts, fmt.Sprintf("%7d", totalB))
}
fmt.Fprintf(&sb, "%s total\n", strings.Join(parts, ""))
}
return sb.String()
}
// ---------------------------------------------------------------------------
// find
// ---------------------------------------------------------------------------
func cmdFind(args []string, cwd string) string {
dir := cwd
namePattern := ""
typeFilter := ""
for i := 0; i < len(args); i++ {
switch args[i] {
case "-name":
if i+1 < len(args) {
namePattern = args[i+1]
i++
}
case "-type":
if i+1 < len(args) {
typeFilter = args[i+1]
i++
}
default:
if !strings.HasPrefix(args[i], "-") && namePattern == "" {
dir = ResolvePath(args[i], cwd)
}
}
}
skipDirs := map[string]bool{".git": true, "node_modules": true, "vendor": true}
var sb strings.Builder
count := 0
maxResults := 200
_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil || count >= maxResults {
return nil
}
name := info.Name()
if info.IsDir() && skipDirs[name] {
return filepath.SkipDir
}
if strings.HasPrefix(name, ".") && path != dir {
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
if typeFilter == "f" && info.IsDir() {
return nil
}
if typeFilter == "d" && !info.IsDir() {
return nil
}
if namePattern != "" {
matched, _ := filepath.Match(namePattern, name)
if !matched {
return nil
}
}
rel, _ := filepath.Rel(cwd, path)
if rel == "" {
rel = path
}
sb.WriteString(rel + "\n")
count++
return nil
})
if count == 0 {
return "(no matches)"
}
if count >= maxResults {
fmt.Fprintf(&sb, "... (truncated at %d results)\n", maxResults)
}
return sb.String()
}
// ---------------------------------------------------------------------------
// pwd / echo / stat
// ---------------------------------------------------------------------------
func cmdPwd(_ []string, cwd string) string { return cwd }
func cmdEcho(args []string, _ string) string { return strings.Join(args, " ") }
func cmdStat(args []string, cwd string) string {
if len(args) == 0 {
return "stat: missing file"
}
var sb strings.Builder
for _, f := range args {
info, err := os.Stat(ResolvePath(f, cwd))
if err != nil {
fmt.Fprintf(&sb, "stat: %v\n", err)
continue
}
fmt.Fprintf(&sb, " File: %s\n", f)
fmt.Fprintf(&sb, " Size: %d bytes\n", info.Size())
fmt.Fprintf(&sb, " Mode: %s\n", info.Mode())
fmt.Fprintf(&sb, " Modified: %s\n", info.ModTime().Format(time.RFC3339))
if info.IsDir() {
sb.WriteString(" Type: directory\n")
} else {
sb.WriteString(" Type: regular file\n")
}
sb.WriteString("\n")
}
return sb.String()
}
// ---------------------------------------------------------------------------
// diff
// ---------------------------------------------------------------------------
func cmdDiff(args []string, cwd string) string {
if len(args) < 2 {
return "diff: need two files"
}
data1, err := os.ReadFile(ResolvePath(args[0], cwd))
if err != nil {
return fmt.Sprintf("diff: %v", err)
}
data2, err := os.ReadFile(ResolvePath(args[1], cwd))
if err != nil {
return fmt.Sprintf("diff: %v", err)
}
lines1 := strings.Split(string(data1), "\n")
lines2 := strings.Split(string(data2), "\n")
var sb strings.Builder
fmt.Fprintf(&sb, "--- %s\n+++ %s\n", args[0], args[1])
maxLen := len(lines1)
if len(lines2) > maxLen {
maxLen = len(lines2)
}
diffs := 0
for i := 0; i < maxLen; i++ {
var l1, l2 string
if i < len(lines1) {
l1 = lines1[i]
}
if i < len(lines2) {
l2 = lines2[i]
}
if l1 != l2 {
diffs++
if diffs > 100 {
sb.WriteString("... (too many differences)\n")
break
}
fmt.Fprintf(&sb, "@@ line %d @@\n", i+1)
if l1 != "" {
fmt.Fprintf(&sb, "-%s\n", l1)
}
if l2 != "" {
fmt.Fprintf(&sb, "+%s\n", l2)
}
}
}
if diffs == 0 {
return "Files are identical"
}
return sb.String()
}
// ---------------------------------------------------------------------------
// tree
// ---------------------------------------------------------------------------
func cmdTree(args []string, cwd string) string {
dir := cwd
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
dir = ResolvePath(args[0], cwd)
}
skipDirs := map[string]bool{".git": true, "node_modules": true, "vendor": true, "__pycache__": true}
var sb strings.Builder
sb.WriteString(dir + "\n")
count := 0
maxEntries := 300
var walk func(path, prefix string)
walk = func(path, prefix string) {
if count >= maxEntries {
return
}
entries, err := os.ReadDir(path)
if err != nil {
return
}
var visible []os.DirEntry
for _, e := range entries {
if !strings.HasPrefix(e.Name(), ".") && !skipDirs[e.Name()] {
visible = append(visible, e)
}
}
sort.Slice(visible, func(i, j int) bool { return visible[i].Name() < visible[j].Name() })
for i, e := range visible {
if count >= maxEntries {
sb.WriteString(prefix + "... (truncated)\n")
return
}
count++
connector := "鈹溾攢鈹€ "
childPrefix := prefix + "鈹? "
if i == len(visible)-1 {
connector = "鈹斺攢鈹€ "
childPrefix = prefix + " "
}
sb.WriteString(prefix + connector + e.Name())
if e.IsDir() {
sb.WriteString("/\n")
walk(filepath.Join(path, e.Name()), childPrefix)
} else {
sb.WriteString("\n")
}
}
}
walk(dir, "")
return sb.String()
}
// ---------------------------------------------------------------------------
// touch / mkdir / cp / mv
// ---------------------------------------------------------------------------
func cmdTouch(args []string, cwd string) string {
if len(args) == 0 {
return "touch: missing file"
}
for _, f := range args {
if strings.HasPrefix(f, "-") {
continue
}
p := ResolvePath(f, cwd)
if _, err := os.Stat(p); os.IsNotExist(err) {
if err := os.WriteFile(p, []byte{}, 0644); err != nil {
return fmt.Sprintf("touch: %v", err)
}
} else {
now := time.Now()
_ = os.Chtimes(p, now, now)
}
}
return fmt.Sprintf("touched %d file(s)", len(args))
}
func cmdMkdir(args []string, cwd string) string {
if len(args) == 0 {
return "mkdir: missing directory"
}
mkParents := false
var dirs []string
for _, a := range args {
if a == "-p" {
mkParents = true
} else {
dirs = append(dirs, a)
}
}
for _, d := range dirs {
p := ResolvePath(d, cwd)
var err error
if mkParents {
err = os.MkdirAll(p, 0755)
} else {
err = os.Mkdir(p, 0755)
}
if err != nil {
return fmt.Sprintf("mkdir: %v", err)
}
}
return fmt.Sprintf("created %d dir(s)", len(dirs))
}
func cmdCp(args []string, cwd string) string {
if len(args) < 2 {
return "cp: need source and destination"
}
src := ResolvePath(args[0], cwd)
dst := ResolvePath(args[1], cwd)
data, err := os.ReadFile(src)
if err != nil {
return fmt.Sprintf("cp: %v", err)
}
if info, err := os.Stat(dst); err == nil && info.IsDir() {
dst = filepath.Join(dst, filepath.Base(src))
}
if err := os.WriteFile(dst, data, 0644); err != nil {
return fmt.Sprintf("cp: %v", err)
}
return fmt.Sprintf("copied %s -> %s", args[0], filepath.Base(dst))
}
func cmdMv(args []string, cwd string) string {
if len(args) < 2 {
return "mv: need source and destination"
}
src := ResolvePath(args[0], cwd)
dst := ResolvePath(args[1], cwd)
if info, err := os.Stat(dst); err == nil && info.IsDir() {
dst = filepath.Join(dst, filepath.Base(src))
}
if err := os.Rename(src, dst); err != nil {
return fmt.Sprintf("mv: %v", err)
}
return fmt.Sprintf("moved %s -> %s", args[0], filepath.Base(dst))
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
// ResolvePath resolves a path relative to cwd.
func ResolvePath(path, cwd string) string {
if filepath.IsAbs(path) {
return filepath.Clean(path)
}
return filepath.Join(cwd, path)
}
// IsBinary checks if the first 512 bytes contain null bytes.
func IsBinary(data []byte) bool {
check := data
if len(check) > 512 {
check = check[:512]
}
for _, b := range check {
if b == 0 {
return true
}
}
return false
}