feat(memory): add MemoryConfig and delegate factory for config-driven construction
Introduce MemoryConfig/MemorySyncConfig structs with env-var bindings for embedding dims, offload threshold, and Turso replication settings. Add NewFromConfig factory that selects the correct delegate based on config, preparing for embedded-replica support.
This commit is contained in:
ZanzyTHEbar
parent
f488d3d8d2
commit
036c1933e1
3 changed files with 373 additions and 3 deletions
|
|
@ -49,11 +49,77 @@ type Config struct {
|
|||
Providers ProvidersConfig `json:"providers"`
|
||||
Gateway GatewayConfig `json:"gateway"`
|
||||
Tools ToolsConfig `json:"tools"`
|
||||
Memory MemoryConfig `json:"memory"`
|
||||
Heartbeat HeartbeatConfig `json:"heartbeat"`
|
||||
Devices DevicesConfig `json:"devices"`
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// MemoryConfig configures the 3-tier MemGPT memory system.
|
||||
type MemoryConfig struct {
|
||||
// Enabled controls whether the memory system is initialized. Default: true.
|
||||
Enabled bool `json:"enabled" env:"PICOCLAW_MEMORY_ENABLED"`
|
||||
|
||||
// DBPath overrides the default database path (workspace/memory/picoclaw.db).
|
||||
// Empty string uses the default.
|
||||
DBPath string `json:"db_path" env:"PICOCLAW_MEMORY_DB_PATH"`
|
||||
|
||||
// EmbeddingDims is the vector dimensionality for archival embeddings.
|
||||
// Default: 768 (sentence-transformers). Use 1536 for OpenAI ada-002, 384 for MiniLM.
|
||||
EmbeddingDims int `json:"embedding_dims" env:"PICOCLAW_MEMORY_EMBEDDING_DIMS"`
|
||||
|
||||
// OffloadThresholdTokens is the token count above which tool results
|
||||
// are automatically offloaded to archival memory. Default: 4000.
|
||||
OffloadThresholdTokens int `json:"offload_threshold_tokens" env:"PICOCLAW_MEMORY_OFFLOAD_THRESHOLD_TOKENS"`
|
||||
|
||||
// Embedding configures the embedding provider for archival vector search.
|
||||
Embedding EmbeddingConfig `json:"embedding"`
|
||||
|
||||
// Sync configures Turso embedded replica sync. When SyncURL is set,
|
||||
// the local DB acts as an embedded replica that syncs with the remote primary.
|
||||
Sync MemorySyncConfig `json:"sync"`
|
||||
}
|
||||
|
||||
// EmbeddingConfig selects which embedding provider to use for archival memory.
|
||||
type EmbeddingConfig struct {
|
||||
// Provider selects the embedding backend: "ollama", "openai", or "".
|
||||
// Empty string disables embeddings (FTS5-only search).
|
||||
Provider string `json:"provider" env:"PICOCLAW_MEMORY_EMBEDDING_PROVIDER"`
|
||||
|
||||
// Model is the embedding model name (e.g., "nomic-embed-text", "text-embedding-3-small").
|
||||
// Defaults depend on provider: "nomic-embed-text" for Ollama, "text-embedding-3-small" for OpenAI.
|
||||
Model string `json:"model" env:"PICOCLAW_MEMORY_EMBEDDING_MODEL"`
|
||||
|
||||
// APIBase overrides the provider's API base URL.
|
||||
// For Ollama defaults to "http://localhost:11434".
|
||||
// For OpenAI defaults to "https://api.openai.com/v1".
|
||||
// Empty string uses the default for the selected provider.
|
||||
APIBase string `json:"api_base" env:"PICOCLAW_MEMORY_EMBEDDING_API_BASE"`
|
||||
|
||||
// APIKey for the embedding provider. Required for OpenAI, optional for Ollama.
|
||||
// If empty, falls back to the matching provider's key from providers config.
|
||||
APIKey string `json:"api_key" env:"PICOCLAW_MEMORY_EMBEDDING_API_KEY"`
|
||||
}
|
||||
|
||||
// MemorySyncConfig configures Turso embedded replica synchronization.
|
||||
// When SyncURL is empty, the database operates in local-only mode.
|
||||
type MemorySyncConfig struct {
|
||||
// SyncURL is the Turso primary database URL (e.g., "libsql://mydb.turso.io").
|
||||
// Empty string disables replication (local-only mode).
|
||||
SyncURL string `json:"sync_url" env:"PICOCLAW_MEMORY_SYNC_URL"`
|
||||
|
||||
// AuthToken is the Turso authentication token for the remote database.
|
||||
AuthToken string `json:"auth_token" env:"PICOCLAW_MEMORY_SYNC_AUTH_TOKEN"`
|
||||
|
||||
// SyncIntervalSeconds is how often to sync with the remote primary (in seconds).
|
||||
// Zero means manual sync only. Default: 60.
|
||||
SyncIntervalSeconds int `json:"sync_interval_seconds" env:"PICOCLAW_MEMORY_SYNC_INTERVAL_SECONDS"`
|
||||
|
||||
// EncryptionKey enables encryption-at-rest on the local database file.
|
||||
// Empty string means no encryption.
|
||||
EncryptionKey string `json:"encryption_key" env:"PICOCLAW_MEMORY_SYNC_ENCRYPTION_KEY"`
|
||||
}
|
||||
|
||||
type AgentsConfig struct {
|
||||
Defaults AgentDefaults `json:"defaults"`
|
||||
}
|
||||
|
|
@ -187,7 +253,7 @@ type ProviderConfig struct {
|
|||
APIBase string `json:"api_base" env:"PICOCLAW_PROVIDERS_{{.Name}}_API_BASE"`
|
||||
Proxy string `json:"proxy,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_PROXY"`
|
||||
AuthMethod string `json:"auth_method,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_AUTH_METHOD"`
|
||||
Timeout int `json:"timeout,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_TIMEOUT"` // seconds, 0 = default (120s)
|
||||
Timeout int `json:"timeout,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_TIMEOUT"` // seconds, 0 = default (120s)
|
||||
ConnectMode string `json:"connect_mode,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_CONNECT_MODE"` // only for Github Copilot, `stdio` or `grpc`
|
||||
}
|
||||
|
||||
|
|
@ -229,8 +295,8 @@ type CronToolsConfig struct {
|
|||
}
|
||||
|
||||
type ToolsConfig struct {
|
||||
Web WebToolsConfig `json:"web"`
|
||||
ProgressiveDisclosure bool `json:"progressive_disclosure" env:"PICOCLAW_TOOLS_PROGRESSIVE_DISCLOSURE"`
|
||||
Web WebToolsConfig `json:"web"`
|
||||
ProgressiveDisclosure bool `json:"progressive_disclosure" env:"PICOCLAW_TOOLS_PROGRESSIVE_DISCLOSURE"`
|
||||
Cron CronToolsConfig `json:"cron"`
|
||||
}
|
||||
|
||||
|
|
@ -354,6 +420,14 @@ func DefaultConfig() *Config {
|
|||
ExecTimeoutMinutes: 5, // default 5 minutes for LLM operations
|
||||
},
|
||||
},
|
||||
Memory: MemoryConfig{
|
||||
Enabled: true,
|
||||
EmbeddingDims: 768,
|
||||
OffloadThresholdTokens: 4000,
|
||||
Sync: MemorySyncConfig{
|
||||
SyncIntervalSeconds: 60,
|
||||
},
|
||||
},
|
||||
Heartbeat: HeartbeatConfig{
|
||||
Enabled: true,
|
||||
Interval: 30, // default 30 minutes
|
||||
|
|
|
|||
89
pkg/memory/delegate/factory.go
Normal file
89
pkg/memory/delegate/factory.go
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
package delegate
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/sipeed/picoclaw/pkg/config"
|
||||
"github.com/sipeed/picoclaw/pkg/logger"
|
||||
)
|
||||
|
||||
// NewFromConfig creates a LibSQLDelegate from the application config.
|
||||
// It selects local-only or embedded-replica mode based on the sync configuration.
|
||||
//
|
||||
// Falls back to local-only mode if replica setup fails (logs a warning).
|
||||
func NewFromConfig(cfg config.MemoryConfig, defaultDBPath string) (*LibSQLDelegate, error) {
|
||||
dbPath := cfg.DBPath
|
||||
if dbPath == "" {
|
||||
dbPath = defaultDBPath
|
||||
}
|
||||
|
||||
dims := cfg.EmbeddingDims
|
||||
if dims <= 0 {
|
||||
dims = DefaultEmbeddingDims
|
||||
}
|
||||
|
||||
var d *LibSQLDelegate
|
||||
var err error
|
||||
|
||||
if cfg.Sync.SyncURL != "" {
|
||||
d, err = newReplicaDelegate(dbPath, dims, cfg.Sync)
|
||||
if err != nil {
|
||||
logger.WarnCF("memory", "Embedded replica setup failed, falling back to local-only mode",
|
||||
map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
"sync_url": cfg.Sync.SyncURL,
|
||||
})
|
||||
d, err = newLocalDelegate(dbPath, dims)
|
||||
}
|
||||
} else {
|
||||
d, err = newLocalDelegate(dbPath, dims)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create memory delegate: %w", err)
|
||||
}
|
||||
|
||||
if d.IsReplica() {
|
||||
logger.InfoCF("memory", "Memory delegate initialized in embedded replica mode",
|
||||
map[string]interface{}{
|
||||
"db_path": dbPath,
|
||||
"sync_url": cfg.Sync.SyncURL,
|
||||
"sync_interval": cfg.Sync.SyncIntervalSeconds,
|
||||
})
|
||||
} else {
|
||||
logger.InfoCF("memory", "Memory delegate initialized in local-only mode",
|
||||
map[string]interface{}{"db_path": dbPath})
|
||||
}
|
||||
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func newLocalDelegate(dbPath string, dims int) (*LibSQLDelegate, error) {
|
||||
if dims == DefaultEmbeddingDims {
|
||||
return NewLibSQLDelegate(dbPath)
|
||||
}
|
||||
return NewLibSQLDelegateWithDims(dbPath, dims)
|
||||
}
|
||||
|
||||
func newReplicaDelegate(dbPath string, dims int, syncCfg config.MemorySyncConfig) (*LibSQLDelegate, error) {
|
||||
var interval time.Duration
|
||||
if syncCfg.SyncIntervalSeconds > 0 {
|
||||
interval = time.Duration(syncCfg.SyncIntervalSeconds) * time.Second
|
||||
}
|
||||
|
||||
d, err := NewLibSQLDelegateWithSync(dbPath, SyncConfig{
|
||||
SyncURL: syncCfg.SyncURL,
|
||||
AuthToken: syncCfg.AuthToken,
|
||||
SyncInterval: interval,
|
||||
EncryptionKey: syncCfg.EncryptionKey,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if dims > 0 && dims != DefaultEmbeddingDims {
|
||||
d.embeddingDims = dims
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
207
pkg/memory/delegate/factory_test.go
Normal file
207
pkg/memory/delegate/factory_test.go
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
package delegate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/sipeed/picoclaw/pkg/config"
|
||||
"github.com/sipeed/picoclaw/pkg/ids"
|
||||
"github.com/sipeed/picoclaw/pkg/memory"
|
||||
)
|
||||
|
||||
func TestNewFromConfig_LocalOnly_DefaultPath(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
defaultPath := filepath.Join(tmpDir, "test.db")
|
||||
|
||||
cfg := config.MemoryConfig{
|
||||
Enabled: true,
|
||||
}
|
||||
|
||||
d, err := NewFromConfig(cfg, defaultPath)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFromConfig: %v", err)
|
||||
}
|
||||
defer d.Close()
|
||||
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
|
||||
if d.IsReplica() {
|
||||
t.Error("expected local-only mode, got replica")
|
||||
}
|
||||
|
||||
// Verify it's functional
|
||||
err = d.UpsertWorkingContext(context.Background(), "agent", "sess", "test content")
|
||||
if err != nil {
|
||||
t.Fatalf("UpsertWorkingContext: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewFromConfig_LocalOnly_CustomPath(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
customPath := filepath.Join(tmpDir, "custom.db")
|
||||
|
||||
cfg := config.MemoryConfig{
|
||||
Enabled: true,
|
||||
DBPath: customPath,
|
||||
}
|
||||
|
||||
d, err := NewFromConfig(cfg, filepath.Join(tmpDir, "default.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("NewFromConfig: %v", err)
|
||||
}
|
||||
defer d.Close()
|
||||
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
|
||||
// The custom path should have been used — check it exists
|
||||
if _, err := os.Stat(customPath); os.IsNotExist(err) {
|
||||
t.Error("expected custom DB path to exist")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewFromConfig_CustomDims(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
|
||||
cfg := config.MemoryConfig{
|
||||
Enabled: true,
|
||||
EmbeddingDims: 384,
|
||||
}
|
||||
|
||||
d, err := NewFromConfig(cfg, dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFromConfig: %v", err)
|
||||
}
|
||||
defer d.Close()
|
||||
|
||||
if d.EmbeddingDims() != 384 {
|
||||
t.Errorf("expected 384 dims, got %d", d.EmbeddingDims())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewFromConfig_DefaultDims(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
|
||||
cfg := config.MemoryConfig{
|
||||
Enabled: true,
|
||||
}
|
||||
|
||||
d, err := NewFromConfig(cfg, dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFromConfig: %v", err)
|
||||
}
|
||||
defer d.Close()
|
||||
|
||||
if d.EmbeddingDims() != DefaultEmbeddingDims {
|
||||
t.Errorf("expected %d default dims, got %d", DefaultEmbeddingDims, d.EmbeddingDims())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewFromConfig_ReplicaFallback(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
|
||||
cfg := config.MemoryConfig{
|
||||
Enabled: true,
|
||||
Sync: config.MemorySyncConfig{
|
||||
SyncURL: "libsql://nonexistent-db.turso.io",
|
||||
AuthToken: "invalid-token",
|
||||
},
|
||||
}
|
||||
|
||||
// Should fall back to local-only mode when replica setup fails
|
||||
d, err := NewFromConfig(cfg, dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFromConfig should fall back, got error: %v", err)
|
||||
}
|
||||
defer d.Close()
|
||||
|
||||
if d.IsReplica() {
|
||||
t.Error("expected fallback to local-only mode")
|
||||
}
|
||||
|
||||
// Should still be functional in local mode
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewFromConfig_LocalFullRoundTrip(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
dbPath := filepath.Join(tmpDir, "roundtrip.db")
|
||||
|
||||
cfg := config.MemoryConfig{
|
||||
Enabled: true,
|
||||
EmbeddingDims: 768,
|
||||
}
|
||||
|
||||
d, err := NewFromConfig(cfg, dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFromConfig: %v", err)
|
||||
}
|
||||
defer d.Close()
|
||||
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Working context round-trip
|
||||
if err := d.UpsertWorkingContext(ctx, "a1", "s1", "hello"); err != nil {
|
||||
t.Fatalf("UpsertWorkingContext: %v", err)
|
||||
}
|
||||
wc, err := d.GetWorkingContext(ctx, "a1", "s1")
|
||||
if err != nil {
|
||||
t.Fatalf("GetWorkingContext: %v", err)
|
||||
}
|
||||
if wc.Content != "hello" {
|
||||
t.Errorf("expected 'hello', got %q", wc.Content)
|
||||
}
|
||||
|
||||
// Recall item round-trip
|
||||
item := &memory.RecallItem{
|
||||
ID: ids.New(),
|
||||
AgentID: "a1",
|
||||
Role: "user",
|
||||
Sector: memory.SectorEpisodic,
|
||||
Content: "test recall",
|
||||
}
|
||||
if err := d.InsertRecallItem(ctx, item); err != nil {
|
||||
t.Fatalf("InsertRecallItem: %v", err)
|
||||
}
|
||||
|
||||
got, err := d.GetRecallItem(ctx, item.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetRecallItem: %v", err)
|
||||
}
|
||||
if got.Content != "test recall" {
|
||||
t.Errorf("expected 'test recall', got %q", got.Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncConfig_Defaults(t *testing.T) {
|
||||
cfg := config.DefaultConfig()
|
||||
if !cfg.Memory.Enabled {
|
||||
t.Error("memory should be enabled by default")
|
||||
}
|
||||
if cfg.Memory.EmbeddingDims != 768 {
|
||||
t.Errorf("expected 768 default dims, got %d", cfg.Memory.EmbeddingDims)
|
||||
}
|
||||
if cfg.Memory.OffloadThresholdTokens != 4000 {
|
||||
t.Errorf("expected 4000 offload threshold, got %d", cfg.Memory.OffloadThresholdTokens)
|
||||
}
|
||||
if cfg.Memory.Sync.SyncIntervalSeconds != 60 {
|
||||
t.Errorf("expected 60s sync interval, got %d", cfg.Memory.Sync.SyncIntervalSeconds)
|
||||
}
|
||||
if cfg.Memory.Sync.SyncURL != "" {
|
||||
t.Error("sync URL should be empty by default")
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue