feat(memory): add MemoryConfig and delegate factory for config-driven construction
Introduce MemoryConfig/MemorySyncConfig structs with env-var bindings for embedding dims, offload threshold, and Turso replication settings. Add NewFromConfig factory that selects the correct delegate based on config, preparing for embedded-replica support.
This commit is contained in:
parent
f488d3d8d2
commit
036c1933e1
3 changed files with 373 additions and 3 deletions
|
|
@ -49,11 +49,77 @@ type Config struct {
|
||||||
Providers ProvidersConfig `json:"providers"`
|
Providers ProvidersConfig `json:"providers"`
|
||||||
Gateway GatewayConfig `json:"gateway"`
|
Gateway GatewayConfig `json:"gateway"`
|
||||||
Tools ToolsConfig `json:"tools"`
|
Tools ToolsConfig `json:"tools"`
|
||||||
|
Memory MemoryConfig `json:"memory"`
|
||||||
Heartbeat HeartbeatConfig `json:"heartbeat"`
|
Heartbeat HeartbeatConfig `json:"heartbeat"`
|
||||||
Devices DevicesConfig `json:"devices"`
|
Devices DevicesConfig `json:"devices"`
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MemoryConfig configures the 3-tier MemGPT memory system.
|
||||||
|
type MemoryConfig struct {
|
||||||
|
// Enabled controls whether the memory system is initialized. Default: true.
|
||||||
|
Enabled bool `json:"enabled" env:"PICOCLAW_MEMORY_ENABLED"`
|
||||||
|
|
||||||
|
// DBPath overrides the default database path (workspace/memory/picoclaw.db).
|
||||||
|
// Empty string uses the default.
|
||||||
|
DBPath string `json:"db_path" env:"PICOCLAW_MEMORY_DB_PATH"`
|
||||||
|
|
||||||
|
// EmbeddingDims is the vector dimensionality for archival embeddings.
|
||||||
|
// Default: 768 (sentence-transformers). Use 1536 for OpenAI ada-002, 384 for MiniLM.
|
||||||
|
EmbeddingDims int `json:"embedding_dims" env:"PICOCLAW_MEMORY_EMBEDDING_DIMS"`
|
||||||
|
|
||||||
|
// OffloadThresholdTokens is the token count above which tool results
|
||||||
|
// are automatically offloaded to archival memory. Default: 4000.
|
||||||
|
OffloadThresholdTokens int `json:"offload_threshold_tokens" env:"PICOCLAW_MEMORY_OFFLOAD_THRESHOLD_TOKENS"`
|
||||||
|
|
||||||
|
// Embedding configures the embedding provider for archival vector search.
|
||||||
|
Embedding EmbeddingConfig `json:"embedding"`
|
||||||
|
|
||||||
|
// Sync configures Turso embedded replica sync. When SyncURL is set,
|
||||||
|
// the local DB acts as an embedded replica that syncs with the remote primary.
|
||||||
|
Sync MemorySyncConfig `json:"sync"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// EmbeddingConfig selects which embedding provider to use for archival memory.
|
||||||
|
type EmbeddingConfig struct {
|
||||||
|
// Provider selects the embedding backend: "ollama", "openai", or "".
|
||||||
|
// Empty string disables embeddings (FTS5-only search).
|
||||||
|
Provider string `json:"provider" env:"PICOCLAW_MEMORY_EMBEDDING_PROVIDER"`
|
||||||
|
|
||||||
|
// Model is the embedding model name (e.g., "nomic-embed-text", "text-embedding-3-small").
|
||||||
|
// Defaults depend on provider: "nomic-embed-text" for Ollama, "text-embedding-3-small" for OpenAI.
|
||||||
|
Model string `json:"model" env:"PICOCLAW_MEMORY_EMBEDDING_MODEL"`
|
||||||
|
|
||||||
|
// APIBase overrides the provider's API base URL.
|
||||||
|
// For Ollama defaults to "http://localhost:11434".
|
||||||
|
// For OpenAI defaults to "https://api.openai.com/v1".
|
||||||
|
// Empty string uses the default for the selected provider.
|
||||||
|
APIBase string `json:"api_base" env:"PICOCLAW_MEMORY_EMBEDDING_API_BASE"`
|
||||||
|
|
||||||
|
// APIKey for the embedding provider. Required for OpenAI, optional for Ollama.
|
||||||
|
// If empty, falls back to the matching provider's key from providers config.
|
||||||
|
APIKey string `json:"api_key" env:"PICOCLAW_MEMORY_EMBEDDING_API_KEY"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// MemorySyncConfig configures Turso embedded replica synchronization.
|
||||||
|
// When SyncURL is empty, the database operates in local-only mode.
|
||||||
|
type MemorySyncConfig struct {
|
||||||
|
// SyncURL is the Turso primary database URL (e.g., "libsql://mydb.turso.io").
|
||||||
|
// Empty string disables replication (local-only mode).
|
||||||
|
SyncURL string `json:"sync_url" env:"PICOCLAW_MEMORY_SYNC_URL"`
|
||||||
|
|
||||||
|
// AuthToken is the Turso authentication token for the remote database.
|
||||||
|
AuthToken string `json:"auth_token" env:"PICOCLAW_MEMORY_SYNC_AUTH_TOKEN"`
|
||||||
|
|
||||||
|
// SyncIntervalSeconds is how often to sync with the remote primary (in seconds).
|
||||||
|
// Zero means manual sync only. Default: 60.
|
||||||
|
SyncIntervalSeconds int `json:"sync_interval_seconds" env:"PICOCLAW_MEMORY_SYNC_INTERVAL_SECONDS"`
|
||||||
|
|
||||||
|
// EncryptionKey enables encryption-at-rest on the local database file.
|
||||||
|
// Empty string means no encryption.
|
||||||
|
EncryptionKey string `json:"encryption_key" env:"PICOCLAW_MEMORY_SYNC_ENCRYPTION_KEY"`
|
||||||
|
}
|
||||||
|
|
||||||
type AgentsConfig struct {
|
type AgentsConfig struct {
|
||||||
Defaults AgentDefaults `json:"defaults"`
|
Defaults AgentDefaults `json:"defaults"`
|
||||||
}
|
}
|
||||||
|
|
@ -187,7 +253,7 @@ type ProviderConfig struct {
|
||||||
APIBase string `json:"api_base" env:"PICOCLAW_PROVIDERS_{{.Name}}_API_BASE"`
|
APIBase string `json:"api_base" env:"PICOCLAW_PROVIDERS_{{.Name}}_API_BASE"`
|
||||||
Proxy string `json:"proxy,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_PROXY"`
|
Proxy string `json:"proxy,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_PROXY"`
|
||||||
AuthMethod string `json:"auth_method,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_AUTH_METHOD"`
|
AuthMethod string `json:"auth_method,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_AUTH_METHOD"`
|
||||||
Timeout int `json:"timeout,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_TIMEOUT"` // seconds, 0 = default (120s)
|
Timeout int `json:"timeout,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_TIMEOUT"` // seconds, 0 = default (120s)
|
||||||
ConnectMode string `json:"connect_mode,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_CONNECT_MODE"` // only for Github Copilot, `stdio` or `grpc`
|
ConnectMode string `json:"connect_mode,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_CONNECT_MODE"` // only for Github Copilot, `stdio` or `grpc`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -229,8 +295,8 @@ type CronToolsConfig struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
type ToolsConfig struct {
|
type ToolsConfig struct {
|
||||||
Web WebToolsConfig `json:"web"`
|
Web WebToolsConfig `json:"web"`
|
||||||
ProgressiveDisclosure bool `json:"progressive_disclosure" env:"PICOCLAW_TOOLS_PROGRESSIVE_DISCLOSURE"`
|
ProgressiveDisclosure bool `json:"progressive_disclosure" env:"PICOCLAW_TOOLS_PROGRESSIVE_DISCLOSURE"`
|
||||||
Cron CronToolsConfig `json:"cron"`
|
Cron CronToolsConfig `json:"cron"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -354,6 +420,14 @@ func DefaultConfig() *Config {
|
||||||
ExecTimeoutMinutes: 5, // default 5 minutes for LLM operations
|
ExecTimeoutMinutes: 5, // default 5 minutes for LLM operations
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
Memory: MemoryConfig{
|
||||||
|
Enabled: true,
|
||||||
|
EmbeddingDims: 768,
|
||||||
|
OffloadThresholdTokens: 4000,
|
||||||
|
Sync: MemorySyncConfig{
|
||||||
|
SyncIntervalSeconds: 60,
|
||||||
|
},
|
||||||
|
},
|
||||||
Heartbeat: HeartbeatConfig{
|
Heartbeat: HeartbeatConfig{
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
Interval: 30, // default 30 minutes
|
Interval: 30, // default 30 minutes
|
||||||
|
|
|
||||||
89
pkg/memory/delegate/factory.go
Normal file
89
pkg/memory/delegate/factory.go
Normal file
|
|
@ -0,0 +1,89 @@
|
||||||
|
package delegate
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/sipeed/picoclaw/pkg/config"
|
||||||
|
"github.com/sipeed/picoclaw/pkg/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewFromConfig creates a LibSQLDelegate from the application config.
|
||||||
|
// It selects local-only or embedded-replica mode based on the sync configuration.
|
||||||
|
//
|
||||||
|
// Falls back to local-only mode if replica setup fails (logs a warning).
|
||||||
|
func NewFromConfig(cfg config.MemoryConfig, defaultDBPath string) (*LibSQLDelegate, error) {
|
||||||
|
dbPath := cfg.DBPath
|
||||||
|
if dbPath == "" {
|
||||||
|
dbPath = defaultDBPath
|
||||||
|
}
|
||||||
|
|
||||||
|
dims := cfg.EmbeddingDims
|
||||||
|
if dims <= 0 {
|
||||||
|
dims = DefaultEmbeddingDims
|
||||||
|
}
|
||||||
|
|
||||||
|
var d *LibSQLDelegate
|
||||||
|
var err error
|
||||||
|
|
||||||
|
if cfg.Sync.SyncURL != "" {
|
||||||
|
d, err = newReplicaDelegate(dbPath, dims, cfg.Sync)
|
||||||
|
if err != nil {
|
||||||
|
logger.WarnCF("memory", "Embedded replica setup failed, falling back to local-only mode",
|
||||||
|
map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
"sync_url": cfg.Sync.SyncURL,
|
||||||
|
})
|
||||||
|
d, err = newLocalDelegate(dbPath, dims)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
d, err = newLocalDelegate(dbPath, dims)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("create memory delegate: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if d.IsReplica() {
|
||||||
|
logger.InfoCF("memory", "Memory delegate initialized in embedded replica mode",
|
||||||
|
map[string]interface{}{
|
||||||
|
"db_path": dbPath,
|
||||||
|
"sync_url": cfg.Sync.SyncURL,
|
||||||
|
"sync_interval": cfg.Sync.SyncIntervalSeconds,
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
logger.InfoCF("memory", "Memory delegate initialized in local-only mode",
|
||||||
|
map[string]interface{}{"db_path": dbPath})
|
||||||
|
}
|
||||||
|
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func newLocalDelegate(dbPath string, dims int) (*LibSQLDelegate, error) {
|
||||||
|
if dims == DefaultEmbeddingDims {
|
||||||
|
return NewLibSQLDelegate(dbPath)
|
||||||
|
}
|
||||||
|
return NewLibSQLDelegateWithDims(dbPath, dims)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newReplicaDelegate(dbPath string, dims int, syncCfg config.MemorySyncConfig) (*LibSQLDelegate, error) {
|
||||||
|
var interval time.Duration
|
||||||
|
if syncCfg.SyncIntervalSeconds > 0 {
|
||||||
|
interval = time.Duration(syncCfg.SyncIntervalSeconds) * time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
d, err := NewLibSQLDelegateWithSync(dbPath, SyncConfig{
|
||||||
|
SyncURL: syncCfg.SyncURL,
|
||||||
|
AuthToken: syncCfg.AuthToken,
|
||||||
|
SyncInterval: interval,
|
||||||
|
EncryptionKey: syncCfg.EncryptionKey,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if dims > 0 && dims != DefaultEmbeddingDims {
|
||||||
|
d.embeddingDims = dims
|
||||||
|
}
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
207
pkg/memory/delegate/factory_test.go
Normal file
207
pkg/memory/delegate/factory_test.go
Normal file
|
|
@ -0,0 +1,207 @@
|
||||||
|
package delegate
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/sipeed/picoclaw/pkg/config"
|
||||||
|
"github.com/sipeed/picoclaw/pkg/ids"
|
||||||
|
"github.com/sipeed/picoclaw/pkg/memory"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewFromConfig_LocalOnly_DefaultPath(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
defaultPath := filepath.Join(tmpDir, "test.db")
|
||||||
|
|
||||||
|
cfg := config.MemoryConfig{
|
||||||
|
Enabled: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
d, err := NewFromConfig(cfg, defaultPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewFromConfig: %v", err)
|
||||||
|
}
|
||||||
|
defer d.Close()
|
||||||
|
|
||||||
|
if err := d.Init(context.Background()); err != nil {
|
||||||
|
t.Fatalf("Init: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if d.IsReplica() {
|
||||||
|
t.Error("expected local-only mode, got replica")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify it's functional
|
||||||
|
err = d.UpsertWorkingContext(context.Background(), "agent", "sess", "test content")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("UpsertWorkingContext: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewFromConfig_LocalOnly_CustomPath(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
customPath := filepath.Join(tmpDir, "custom.db")
|
||||||
|
|
||||||
|
cfg := config.MemoryConfig{
|
||||||
|
Enabled: true,
|
||||||
|
DBPath: customPath,
|
||||||
|
}
|
||||||
|
|
||||||
|
d, err := NewFromConfig(cfg, filepath.Join(tmpDir, "default.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewFromConfig: %v", err)
|
||||||
|
}
|
||||||
|
defer d.Close()
|
||||||
|
|
||||||
|
if err := d.Init(context.Background()); err != nil {
|
||||||
|
t.Fatalf("Init: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The custom path should have been used — check it exists
|
||||||
|
if _, err := os.Stat(customPath); os.IsNotExist(err) {
|
||||||
|
t.Error("expected custom DB path to exist")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewFromConfig_CustomDims(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
dbPath := filepath.Join(tmpDir, "test.db")
|
||||||
|
|
||||||
|
cfg := config.MemoryConfig{
|
||||||
|
Enabled: true,
|
||||||
|
EmbeddingDims: 384,
|
||||||
|
}
|
||||||
|
|
||||||
|
d, err := NewFromConfig(cfg, dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewFromConfig: %v", err)
|
||||||
|
}
|
||||||
|
defer d.Close()
|
||||||
|
|
||||||
|
if d.EmbeddingDims() != 384 {
|
||||||
|
t.Errorf("expected 384 dims, got %d", d.EmbeddingDims())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewFromConfig_DefaultDims(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
dbPath := filepath.Join(tmpDir, "test.db")
|
||||||
|
|
||||||
|
cfg := config.MemoryConfig{
|
||||||
|
Enabled: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
d, err := NewFromConfig(cfg, dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewFromConfig: %v", err)
|
||||||
|
}
|
||||||
|
defer d.Close()
|
||||||
|
|
||||||
|
if d.EmbeddingDims() != DefaultEmbeddingDims {
|
||||||
|
t.Errorf("expected %d default dims, got %d", DefaultEmbeddingDims, d.EmbeddingDims())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewFromConfig_ReplicaFallback(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
dbPath := filepath.Join(tmpDir, "test.db")
|
||||||
|
|
||||||
|
cfg := config.MemoryConfig{
|
||||||
|
Enabled: true,
|
||||||
|
Sync: config.MemorySyncConfig{
|
||||||
|
SyncURL: "libsql://nonexistent-db.turso.io",
|
||||||
|
AuthToken: "invalid-token",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should fall back to local-only mode when replica setup fails
|
||||||
|
d, err := NewFromConfig(cfg, dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewFromConfig should fall back, got error: %v", err)
|
||||||
|
}
|
||||||
|
defer d.Close()
|
||||||
|
|
||||||
|
if d.IsReplica() {
|
||||||
|
t.Error("expected fallback to local-only mode")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should still be functional in local mode
|
||||||
|
if err := d.Init(context.Background()); err != nil {
|
||||||
|
t.Fatalf("Init: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewFromConfig_LocalFullRoundTrip(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
dbPath := filepath.Join(tmpDir, "roundtrip.db")
|
||||||
|
|
||||||
|
cfg := config.MemoryConfig{
|
||||||
|
Enabled: true,
|
||||||
|
EmbeddingDims: 768,
|
||||||
|
}
|
||||||
|
|
||||||
|
d, err := NewFromConfig(cfg, dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewFromConfig: %v", err)
|
||||||
|
}
|
||||||
|
defer d.Close()
|
||||||
|
|
||||||
|
if err := d.Init(context.Background()); err != nil {
|
||||||
|
t.Fatalf("Init: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// Working context round-trip
|
||||||
|
if err := d.UpsertWorkingContext(ctx, "a1", "s1", "hello"); err != nil {
|
||||||
|
t.Fatalf("UpsertWorkingContext: %v", err)
|
||||||
|
}
|
||||||
|
wc, err := d.GetWorkingContext(ctx, "a1", "s1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetWorkingContext: %v", err)
|
||||||
|
}
|
||||||
|
if wc.Content != "hello" {
|
||||||
|
t.Errorf("expected 'hello', got %q", wc.Content)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recall item round-trip
|
||||||
|
item := &memory.RecallItem{
|
||||||
|
ID: ids.New(),
|
||||||
|
AgentID: "a1",
|
||||||
|
Role: "user",
|
||||||
|
Sector: memory.SectorEpisodic,
|
||||||
|
Content: "test recall",
|
||||||
|
}
|
||||||
|
if err := d.InsertRecallItem(ctx, item); err != nil {
|
||||||
|
t.Fatalf("InsertRecallItem: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := d.GetRecallItem(ctx, item.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetRecallItem: %v", err)
|
||||||
|
}
|
||||||
|
if got.Content != "test recall" {
|
||||||
|
t.Errorf("expected 'test recall', got %q", got.Content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSyncConfig_Defaults(t *testing.T) {
|
||||||
|
cfg := config.DefaultConfig()
|
||||||
|
if !cfg.Memory.Enabled {
|
||||||
|
t.Error("memory should be enabled by default")
|
||||||
|
}
|
||||||
|
if cfg.Memory.EmbeddingDims != 768 {
|
||||||
|
t.Errorf("expected 768 default dims, got %d", cfg.Memory.EmbeddingDims)
|
||||||
|
}
|
||||||
|
if cfg.Memory.OffloadThresholdTokens != 4000 {
|
||||||
|
t.Errorf("expected 4000 offload threshold, got %d", cfg.Memory.OffloadThresholdTokens)
|
||||||
|
}
|
||||||
|
if cfg.Memory.Sync.SyncIntervalSeconds != 60 {
|
||||||
|
t.Errorf("expected 60s sync interval, got %d", cfg.Memory.Sync.SyncIntervalSeconds)
|
||||||
|
}
|
||||||
|
if cfg.Memory.Sync.SyncURL != "" {
|
||||||
|
t.Error("sync URL should be empty by default")
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Add table
Reference in a new issue