diff --git a/pkg/config/config.go b/pkg/config/config.go index 8e8b57d8d..3d4bf47a8 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -49,11 +49,77 @@ type Config struct { Providers ProvidersConfig `json:"providers"` Gateway GatewayConfig `json:"gateway"` Tools ToolsConfig `json:"tools"` + Memory MemoryConfig `json:"memory"` Heartbeat HeartbeatConfig `json:"heartbeat"` Devices DevicesConfig `json:"devices"` mu sync.RWMutex } +// MemoryConfig configures the 3-tier MemGPT memory system. +type MemoryConfig struct { + // Enabled controls whether the memory system is initialized. Default: true. + Enabled bool `json:"enabled" env:"PICOCLAW_MEMORY_ENABLED"` + + // DBPath overrides the default database path (workspace/memory/picoclaw.db). + // Empty string uses the default. + DBPath string `json:"db_path" env:"PICOCLAW_MEMORY_DB_PATH"` + + // EmbeddingDims is the vector dimensionality for archival embeddings. + // Default: 768 (sentence-transformers). Use 1536 for OpenAI ada-002, 384 for MiniLM. + EmbeddingDims int `json:"embedding_dims" env:"PICOCLAW_MEMORY_EMBEDDING_DIMS"` + + // OffloadThresholdTokens is the token count above which tool results + // are automatically offloaded to archival memory. Default: 4000. + OffloadThresholdTokens int `json:"offload_threshold_tokens" env:"PICOCLAW_MEMORY_OFFLOAD_THRESHOLD_TOKENS"` + + // Embedding configures the embedding provider for archival vector search. + Embedding EmbeddingConfig `json:"embedding"` + + // Sync configures Turso embedded replica sync. When SyncURL is set, + // the local DB acts as an embedded replica that syncs with the remote primary. + Sync MemorySyncConfig `json:"sync"` +} + +// EmbeddingConfig selects which embedding provider to use for archival memory. +type EmbeddingConfig struct { + // Provider selects the embedding backend: "ollama", "openai", or "". + // Empty string disables embeddings (FTS5-only search). + Provider string `json:"provider" env:"PICOCLAW_MEMORY_EMBEDDING_PROVIDER"` + + // Model is the embedding model name (e.g., "nomic-embed-text", "text-embedding-3-small"). + // Defaults depend on provider: "nomic-embed-text" for Ollama, "text-embedding-3-small" for OpenAI. + Model string `json:"model" env:"PICOCLAW_MEMORY_EMBEDDING_MODEL"` + + // APIBase overrides the provider's API base URL. + // For Ollama defaults to "http://localhost:11434". + // For OpenAI defaults to "https://api.openai.com/v1". + // Empty string uses the default for the selected provider. + APIBase string `json:"api_base" env:"PICOCLAW_MEMORY_EMBEDDING_API_BASE"` + + // APIKey for the embedding provider. Required for OpenAI, optional for Ollama. + // If empty, falls back to the matching provider's key from providers config. + APIKey string `json:"api_key" env:"PICOCLAW_MEMORY_EMBEDDING_API_KEY"` +} + +// MemorySyncConfig configures Turso embedded replica synchronization. +// When SyncURL is empty, the database operates in local-only mode. +type MemorySyncConfig struct { + // SyncURL is the Turso primary database URL (e.g., "libsql://mydb.turso.io"). + // Empty string disables replication (local-only mode). + SyncURL string `json:"sync_url" env:"PICOCLAW_MEMORY_SYNC_URL"` + + // AuthToken is the Turso authentication token for the remote database. + AuthToken string `json:"auth_token" env:"PICOCLAW_MEMORY_SYNC_AUTH_TOKEN"` + + // SyncIntervalSeconds is how often to sync with the remote primary (in seconds). + // Zero means manual sync only. Default: 60. + SyncIntervalSeconds int `json:"sync_interval_seconds" env:"PICOCLAW_MEMORY_SYNC_INTERVAL_SECONDS"` + + // EncryptionKey enables encryption-at-rest on the local database file. + // Empty string means no encryption. + EncryptionKey string `json:"encryption_key" env:"PICOCLAW_MEMORY_SYNC_ENCRYPTION_KEY"` +} + type AgentsConfig struct { Defaults AgentDefaults `json:"defaults"` } @@ -187,7 +253,7 @@ type ProviderConfig struct { APIBase string `json:"api_base" env:"PICOCLAW_PROVIDERS_{{.Name}}_API_BASE"` Proxy string `json:"proxy,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_PROXY"` AuthMethod string `json:"auth_method,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_AUTH_METHOD"` - Timeout int `json:"timeout,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_TIMEOUT"` // seconds, 0 = default (120s) + Timeout int `json:"timeout,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_TIMEOUT"` // seconds, 0 = default (120s) ConnectMode string `json:"connect_mode,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_CONNECT_MODE"` // only for Github Copilot, `stdio` or `grpc` } @@ -229,8 +295,8 @@ type CronToolsConfig struct { } type ToolsConfig struct { - Web WebToolsConfig `json:"web"` - ProgressiveDisclosure bool `json:"progressive_disclosure" env:"PICOCLAW_TOOLS_PROGRESSIVE_DISCLOSURE"` + Web WebToolsConfig `json:"web"` + ProgressiveDisclosure bool `json:"progressive_disclosure" env:"PICOCLAW_TOOLS_PROGRESSIVE_DISCLOSURE"` Cron CronToolsConfig `json:"cron"` } @@ -354,6 +420,14 @@ func DefaultConfig() *Config { ExecTimeoutMinutes: 5, // default 5 minutes for LLM operations }, }, + Memory: MemoryConfig{ + Enabled: true, + EmbeddingDims: 768, + OffloadThresholdTokens: 4000, + Sync: MemorySyncConfig{ + SyncIntervalSeconds: 60, + }, + }, Heartbeat: HeartbeatConfig{ Enabled: true, Interval: 30, // default 30 minutes diff --git a/pkg/memory/delegate/factory.go b/pkg/memory/delegate/factory.go new file mode 100644 index 000000000..9902df056 --- /dev/null +++ b/pkg/memory/delegate/factory.go @@ -0,0 +1,89 @@ +package delegate + +import ( + "fmt" + "time" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/logger" +) + +// NewFromConfig creates a LibSQLDelegate from the application config. +// It selects local-only or embedded-replica mode based on the sync configuration. +// +// Falls back to local-only mode if replica setup fails (logs a warning). +func NewFromConfig(cfg config.MemoryConfig, defaultDBPath string) (*LibSQLDelegate, error) { + dbPath := cfg.DBPath + if dbPath == "" { + dbPath = defaultDBPath + } + + dims := cfg.EmbeddingDims + if dims <= 0 { + dims = DefaultEmbeddingDims + } + + var d *LibSQLDelegate + var err error + + if cfg.Sync.SyncURL != "" { + d, err = newReplicaDelegate(dbPath, dims, cfg.Sync) + if err != nil { + logger.WarnCF("memory", "Embedded replica setup failed, falling back to local-only mode", + map[string]interface{}{ + "error": err.Error(), + "sync_url": cfg.Sync.SyncURL, + }) + d, err = newLocalDelegate(dbPath, dims) + } + } else { + d, err = newLocalDelegate(dbPath, dims) + } + + if err != nil { + return nil, fmt.Errorf("create memory delegate: %w", err) + } + + if d.IsReplica() { + logger.InfoCF("memory", "Memory delegate initialized in embedded replica mode", + map[string]interface{}{ + "db_path": dbPath, + "sync_url": cfg.Sync.SyncURL, + "sync_interval": cfg.Sync.SyncIntervalSeconds, + }) + } else { + logger.InfoCF("memory", "Memory delegate initialized in local-only mode", + map[string]interface{}{"db_path": dbPath}) + } + + return d, nil +} + +func newLocalDelegate(dbPath string, dims int) (*LibSQLDelegate, error) { + if dims == DefaultEmbeddingDims { + return NewLibSQLDelegate(dbPath) + } + return NewLibSQLDelegateWithDims(dbPath, dims) +} + +func newReplicaDelegate(dbPath string, dims int, syncCfg config.MemorySyncConfig) (*LibSQLDelegate, error) { + var interval time.Duration + if syncCfg.SyncIntervalSeconds > 0 { + interval = time.Duration(syncCfg.SyncIntervalSeconds) * time.Second + } + + d, err := NewLibSQLDelegateWithSync(dbPath, SyncConfig{ + SyncURL: syncCfg.SyncURL, + AuthToken: syncCfg.AuthToken, + SyncInterval: interval, + EncryptionKey: syncCfg.EncryptionKey, + }) + if err != nil { + return nil, err + } + + if dims > 0 && dims != DefaultEmbeddingDims { + d.embeddingDims = dims + } + return d, nil +} diff --git a/pkg/memory/delegate/factory_test.go b/pkg/memory/delegate/factory_test.go new file mode 100644 index 000000000..0736f092c --- /dev/null +++ b/pkg/memory/delegate/factory_test.go @@ -0,0 +1,207 @@ +package delegate + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/ids" + "github.com/sipeed/picoclaw/pkg/memory" +) + +func TestNewFromConfig_LocalOnly_DefaultPath(t *testing.T) { + tmpDir := t.TempDir() + defaultPath := filepath.Join(tmpDir, "test.db") + + cfg := config.MemoryConfig{ + Enabled: true, + } + + d, err := NewFromConfig(cfg, defaultPath) + if err != nil { + t.Fatalf("NewFromConfig: %v", err) + } + defer d.Close() + + if err := d.Init(context.Background()); err != nil { + t.Fatalf("Init: %v", err) + } + + if d.IsReplica() { + t.Error("expected local-only mode, got replica") + } + + // Verify it's functional + err = d.UpsertWorkingContext(context.Background(), "agent", "sess", "test content") + if err != nil { + t.Fatalf("UpsertWorkingContext: %v", err) + } +} + +func TestNewFromConfig_LocalOnly_CustomPath(t *testing.T) { + tmpDir := t.TempDir() + customPath := filepath.Join(tmpDir, "custom.db") + + cfg := config.MemoryConfig{ + Enabled: true, + DBPath: customPath, + } + + d, err := NewFromConfig(cfg, filepath.Join(tmpDir, "default.db")) + if err != nil { + t.Fatalf("NewFromConfig: %v", err) + } + defer d.Close() + + if err := d.Init(context.Background()); err != nil { + t.Fatalf("Init: %v", err) + } + + // The custom path should have been used — check it exists + if _, err := os.Stat(customPath); os.IsNotExist(err) { + t.Error("expected custom DB path to exist") + } +} + +func TestNewFromConfig_CustomDims(t *testing.T) { + tmpDir := t.TempDir() + dbPath := filepath.Join(tmpDir, "test.db") + + cfg := config.MemoryConfig{ + Enabled: true, + EmbeddingDims: 384, + } + + d, err := NewFromConfig(cfg, dbPath) + if err != nil { + t.Fatalf("NewFromConfig: %v", err) + } + defer d.Close() + + if d.EmbeddingDims() != 384 { + t.Errorf("expected 384 dims, got %d", d.EmbeddingDims()) + } +} + +func TestNewFromConfig_DefaultDims(t *testing.T) { + tmpDir := t.TempDir() + dbPath := filepath.Join(tmpDir, "test.db") + + cfg := config.MemoryConfig{ + Enabled: true, + } + + d, err := NewFromConfig(cfg, dbPath) + if err != nil { + t.Fatalf("NewFromConfig: %v", err) + } + defer d.Close() + + if d.EmbeddingDims() != DefaultEmbeddingDims { + t.Errorf("expected %d default dims, got %d", DefaultEmbeddingDims, d.EmbeddingDims()) + } +} + +func TestNewFromConfig_ReplicaFallback(t *testing.T) { + tmpDir := t.TempDir() + dbPath := filepath.Join(tmpDir, "test.db") + + cfg := config.MemoryConfig{ + Enabled: true, + Sync: config.MemorySyncConfig{ + SyncURL: "libsql://nonexistent-db.turso.io", + AuthToken: "invalid-token", + }, + } + + // Should fall back to local-only mode when replica setup fails + d, err := NewFromConfig(cfg, dbPath) + if err != nil { + t.Fatalf("NewFromConfig should fall back, got error: %v", err) + } + defer d.Close() + + if d.IsReplica() { + t.Error("expected fallback to local-only mode") + } + + // Should still be functional in local mode + if err := d.Init(context.Background()); err != nil { + t.Fatalf("Init: %v", err) + } +} + +func TestNewFromConfig_LocalFullRoundTrip(t *testing.T) { + tmpDir := t.TempDir() + dbPath := filepath.Join(tmpDir, "roundtrip.db") + + cfg := config.MemoryConfig{ + Enabled: true, + EmbeddingDims: 768, + } + + d, err := NewFromConfig(cfg, dbPath) + if err != nil { + t.Fatalf("NewFromConfig: %v", err) + } + defer d.Close() + + if err := d.Init(context.Background()); err != nil { + t.Fatalf("Init: %v", err) + } + + ctx := context.Background() + + // Working context round-trip + if err := d.UpsertWorkingContext(ctx, "a1", "s1", "hello"); err != nil { + t.Fatalf("UpsertWorkingContext: %v", err) + } + wc, err := d.GetWorkingContext(ctx, "a1", "s1") + if err != nil { + t.Fatalf("GetWorkingContext: %v", err) + } + if wc.Content != "hello" { + t.Errorf("expected 'hello', got %q", wc.Content) + } + + // Recall item round-trip + item := &memory.RecallItem{ + ID: ids.New(), + AgentID: "a1", + Role: "user", + Sector: memory.SectorEpisodic, + Content: "test recall", + } + if err := d.InsertRecallItem(ctx, item); err != nil { + t.Fatalf("InsertRecallItem: %v", err) + } + + got, err := d.GetRecallItem(ctx, item.ID) + if err != nil { + t.Fatalf("GetRecallItem: %v", err) + } + if got.Content != "test recall" { + t.Errorf("expected 'test recall', got %q", got.Content) + } +} + +func TestSyncConfig_Defaults(t *testing.T) { + cfg := config.DefaultConfig() + if !cfg.Memory.Enabled { + t.Error("memory should be enabled by default") + } + if cfg.Memory.EmbeddingDims != 768 { + t.Errorf("expected 768 default dims, got %d", cfg.Memory.EmbeddingDims) + } + if cfg.Memory.OffloadThresholdTokens != 4000 { + t.Errorf("expected 4000 offload threshold, got %d", cfg.Memory.OffloadThresholdTokens) + } + if cfg.Memory.Sync.SyncIntervalSeconds != 60 { + t.Errorf("expected 60s sync interval, got %d", cfg.Memory.Sync.SyncIntervalSeconds) + } + if cfg.Memory.Sync.SyncURL != "" { + t.Error("sync URL should be empty by default") + } +}