From f488d3d8d291f0ea84686d94cd6239a6eca08e21 Mon Sep 17 00:00:00 2001 From: ZanzyTHEbar Date: Wed, 18 Feb 2026 13:05:38 +0000 Subject: [PATCH] feat(memory): replace embedded DDL with goose versioned migrations Migrate schema management from hardcoded DDL strings to pressly/goose/v3 Go-only migrations. Enables dynamic F32_BLOB(N) dimensions via context, graceful FTS5/vector index degradation, and proper up/down migration paths. - 001_schema: core tables with BLOB PKs and cascade triggers - 002_fts5: full-text search with tokenizer fallback chain - 003_vector_index: libsql_vector_idx with capability detection - LibSQLDelegate.Init() now uses goose.NewProvider with nil filesystem - Removed embedded fts5.sql, vector.sql, schema_init.sql --- go.mod | 4 + go.sum | 28 ++- pkg/memory/delegate/fts5.sql | 32 --- pkg/memory/delegate/schema_init.sql | 55 ----- pkg/memory/delegate/sqlite.go | 286 +++++++--------------- pkg/memory/delegate/vector.sql | 3 - pkg/memory/migrations/001_schema.go | 94 +++++++ pkg/memory/migrations/002_fts5.go | 85 +++++++ pkg/memory/migrations/003_vector_index.go | 34 +++ pkg/memory/migrations/context.go | 20 ++ 10 files changed, 350 insertions(+), 291 deletions(-) delete mode 100644 pkg/memory/delegate/fts5.sql delete mode 100644 pkg/memory/delegate/schema_init.sql delete mode 100644 pkg/memory/delegate/vector.sql create mode 100644 pkg/memory/migrations/001_schema.go create mode 100644 pkg/memory/migrations/002_fts5.go create mode 100644 pkg/memory/migrations/003_vector_index.go create mode 100644 pkg/memory/migrations/context.go diff --git a/go.mod b/go.mod index 93e680083..8eae54033 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/mymmrac/telego v1.6.0 github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1 github.com/openai/openai-go/v2 v2.7.1 + github.com/pressly/goose/v3 v3.26.0 github.com/slack-go/slack v0.17.3 github.com/stretchr/testify v1.11.1 github.com/tencent-connect/botgo v0.2.1 @@ -53,7 +54,9 @@ require ( github.com/klauspost/compress v1.18.4 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06 // indirect + github.com/mfridman/interpolate v0.0.2 // indirect github.com/pkoukk/tiktoken-go v0.1.6 // indirect + github.com/sethvargo/go-retry v0.3.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.2.0 // indirect github.com/tidwall/pretty v1.2.1 // indirect @@ -67,6 +70,7 @@ require ( gitlab.com/golang-commonmark/markdown v0.0.0-20211110145824-bf3e522c626a // indirect gitlab.com/golang-commonmark/mdurl v0.0.0-20191124015652-932350d1cb84 // indirect gitlab.com/golang-commonmark/puny v0.0.0-20191124015043-9f83538fa04f // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.24.0 // indirect golang.org/x/crypto v0.48.0 // indirect golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect diff --git a/go.sum b/go.sum index b1d48dbae..0ce13c6c8 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-json-experiment/json v0.0.0-20251027170946-4849db3c2f7e h1:Lf/gRkoycfOBPa42vU2bbgPurFong6zXeFtPoxholzU= @@ -108,8 +110,14 @@ github.com/larksuite/oapi-sdk-go/v3 v3.5.3 h1:xvf8Dv29kBXC5/DNDCLhHkAFW8l/0LlQJi github.com/larksuite/oapi-sdk-go/v3 v3.5.3/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06 h1:JLvn7D+wXjH9g4Jsjo+VqmzTUpl/LX7vfr6VOfSWTdM= github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06/go.mod h1:FUkZ5OHjlGPjnM2UyGJz9TypXQFgYqw6AFNO1UiROTM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= +github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= github.com/mymmrac/telego v1.6.0 h1:Zc8rgyHozvd/7ZgyrigyHdAF9koHYMfilYfyB6wlFC0= github.com/mymmrac/telego v1.6.0/go.mod h1:xt6ZWA8zi8KmuzryE1ImEdl9JSwjHNpM4yhC7D8hU4Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -132,12 +140,18 @@ github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYde github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pressly/goose/v3 v3.26.0 h1:KJakav68jdH0WDvoAcj8+n61WqOIaPGgH0bJWS6jpmM= +github.com/pressly/goose/v3 v3.26.0/go.mod h1:4hC1KrritdCxtuFsqgs1R4AU5bWtTAf+cnWvfhf2DNY= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/slack-go/slack v0.17.3 h1:zV5qO3Q+WJAQ/XwbGfNFrRMaJ5T/naqaonyPV/1TP4g= github.com/slack-go/slack v0.17.3/go.mod h1:X+UqOufi3LYQHDnMG1vxf0J8asC6+WllXrVrhl8/Prk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -197,6 +211,8 @@ gitlab.com/opennota/wd v0.0.0-20180912061657-c5d65f63c638 h1:uPZaMiz6Sz0PZs3IZJW gitlab.com/opennota/wd v0.0.0-20180912061657-c5d65f63c638/go.mod h1:EGRJaqe2eO9XGmFtQCvV3Lm9NLico3UhFwUpCG/+mVU= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y= golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -313,5 +329,13 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ= +modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.38.2 h1:Aclu7+tgjgcQVShZqim41Bbw9Cho0y/7WzYptXqkEek= +modernc.org/sqlite v1.38.2/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/pkg/memory/delegate/fts5.sql b/pkg/memory/delegate/fts5.sql deleted file mode 100644 index bd42ac624..000000000 --- a/pkg/memory/delegate/fts5.sql +++ /dev/null @@ -1,32 +0,0 @@ --- FTS5 virtual table for keyword search on recall items. --- Standalone FTS5 table (NOT external-content mode) — more reliable with go-libsql. --- Uses unicode61 tokenizer with extended tokenchars for domain-specific identifiers --- and prefix indexes for efficient prefix matching. --- NOTE: tokenchars uses equals-sign syntax (not space+quotes) per go-libsql compatibility. -CREATE VIRTUAL TABLE IF NOT EXISTS recall_items_fts USING fts5( - content, - tags, - tokenize = 'unicode61 tokenchars=:-_@./', - prefix = '2 3 4 5 6 7' -); --- Triggers to keep standalone FTS5 table in sync with recall_items. --- Uses DELETE+INSERT pattern for UPDATE (FTS5 standard approach). -CREATE TRIGGER IF NOT EXISTS recall_items_ai -AFTER -INSERT ON recall_items BEGIN -INSERT INTO recall_items_fts(rowid, content, tags) -VALUES (new.rowid, new.content, new.tags); -END; -CREATE TRIGGER IF NOT EXISTS recall_items_ad -AFTER DELETE ON recall_items BEGIN -DELETE FROM recall_items_fts -WHERE rowid = old.rowid; -END; -CREATE TRIGGER IF NOT EXISTS recall_items_au -AFTER -UPDATE ON recall_items BEGIN -DELETE FROM recall_items_fts -WHERE rowid = old.rowid; -INSERT INTO recall_items_fts(rowid, content, tags) -VALUES (new.rowid, new.content, new.tags); -END; \ No newline at end of file diff --git a/pkg/memory/delegate/schema_init.sql b/pkg/memory/delegate/schema_init.sql deleted file mode 100644 index 438f6f83f..000000000 --- a/pkg/memory/delegate/schema_init.sql +++ /dev/null @@ -1,55 +0,0 @@ --- PicoClaw Memory System Schema (libSQL) --- Managed by delegate, not sqlc, to allow full DDL including F32_BLOB and pragmas. -CREATE TABLE IF NOT EXISTS working_context ( - agent_id TEXT NOT NULL, - session_key TEXT NOT NULL DEFAULT '', - content TEXT NOT NULL DEFAULT '', - updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (agent_id, session_key) -); -CREATE TABLE IF NOT EXISTS recall_items ( - id TEXT PRIMARY KEY, - agent_id TEXT NOT NULL, - session_key TEXT NOT NULL DEFAULT '', - role TEXT NOT NULL DEFAULT 'system', - sector TEXT NOT NULL DEFAULT 'episodic', - importance REAL NOT NULL DEFAULT 0.5, - salience REAL NOT NULL DEFAULT 0.5, - decay_rate REAL NOT NULL DEFAULT 0.01, - content TEXT NOT NULL, - tags TEXT NOT NULL DEFAULT '', - created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP -); -CREATE INDEX IF NOT EXISTS idx_recall_agent_session ON recall_items(agent_id, session_key); -CREATE INDEX IF NOT EXISTS idx_recall_sector ON recall_items(sector); -CREATE INDEX IF NOT EXISTS idx_recall_importance ON recall_items(importance DESC); -CREATE INDEX IF NOT EXISTS idx_recall_created ON recall_items(created_at DESC); -CREATE TABLE IF NOT EXISTS archival_chunks ( - id TEXT PRIMARY KEY, - recall_id TEXT NOT NULL DEFAULT '', - chunk_index INTEGER NOT NULL DEFAULT 0, - content TEXT NOT NULL, - embedding F32_BLOB(768), - source TEXT NOT NULL DEFAULT '', - hash TEXT NOT NULL DEFAULT '', - created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP -); -CREATE INDEX IF NOT EXISTS idx_chunks_recall ON archival_chunks(recall_id); -CREATE INDEX IF NOT EXISTS idx_chunks_source ON archival_chunks(source); -CREATE TABLE IF NOT EXISTS memory_summaries ( - id TEXT PRIMARY KEY, - agent_id TEXT NOT NULL, - session_key TEXT NOT NULL DEFAULT '', - content TEXT NOT NULL, - from_msg_idx INTEGER NOT NULL DEFAULT 0, - to_msg_idx INTEGER NOT NULL DEFAULT 0, - created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP -); -CREATE INDEX IF NOT EXISTS idx_summaries_agent_session ON memory_summaries(agent_id, session_key); --- Cascade: when a recall item is deleted, remove its archival chunks. -CREATE TRIGGER IF NOT EXISTS recall_cascade_delete -AFTER DELETE ON recall_items BEGIN -DELETE FROM archival_chunks -WHERE recall_id = old.id; -END; \ No newline at end of file diff --git a/pkg/memory/delegate/sqlite.go b/pkg/memory/delegate/sqlite.go index 7e9747a16..7a052ce03 100644 --- a/pkg/memory/delegate/sqlite.go +++ b/pkg/memory/delegate/sqlite.go @@ -4,23 +4,18 @@ package delegate import ( "context" "database/sql" - _ "embed" "fmt" - "strings" + "time" + "github.com/pressly/goose/v3" "github.com/sipeed/picoclaw/pkg/ids" "github.com/sipeed/picoclaw/pkg/memory" + "github.com/sipeed/picoclaw/pkg/memory/migrations" memsqlc "github.com/sipeed/picoclaw/pkg/memory/sqlc" - _ "github.com/tursodatabase/go-libsql" // register "libsql" driver + libsql "github.com/tursodatabase/go-libsql" ) -//go:embed fts5.sql -var fts5DDL string - -//go:embed vector.sql -var vectorDDL string - // DefaultEmbeddingDims is the default number of dimensions for embedding vectors. // This matches common models like sentence-transformers (768-dim). const DefaultEmbeddingDims = 768 @@ -30,12 +25,28 @@ const DefaultEmbeddingDims = 768 // Hand-written SQL (FTS5, vector search) uses a prepared statement cache. type LibSQLDelegate struct { db *sql.DB + connector *libsql.Connector // non-nil when using embedded replica mode queries *memsqlc.Queries stmts *stmtCache caps capFlags embeddingDims int } +// SyncConfig configures Turso embedded replica synchronization. +type SyncConfig struct { + // SyncURL is the remote primary database URL (e.g., "libsql://mydb.turso.io"). + SyncURL string + + // AuthToken for the remote database. + AuthToken string + + // SyncInterval is how often to auto-sync. Zero means manual sync only. + SyncInterval time.Duration + + // EncryptionKey enables encryption-at-rest. Empty means no encryption. + EncryptionKey string +} + // NewLibSQLDelegate opens a libSQL database at the given path and returns // a delegate ready for use. Call Init() to create tables. // Uses DefaultEmbeddingDims (768) for the vector column size. @@ -44,11 +55,56 @@ func NewLibSQLDelegate(dbPath string) (*LibSQLDelegate, error) { if err != nil { return nil, fmt.Errorf("open libsql: %w", err) } - // Single writer for WAL mode safety + + return newDelegateFromDB(db, nil) +} + +// NewLibSQLDelegateWithSync opens a libSQL database as an embedded replica +// that syncs with a remote Turso primary. Reads are served from the local file; +// writes propagate to the remote primary and sync back. +func NewLibSQLDelegateWithSync(dbPath string, cfg SyncConfig) (*LibSQLDelegate, error) { + opts := []libsql.Option{ + libsql.WithAuthToken(cfg.AuthToken), + } + if cfg.SyncInterval > 0 { + opts = append(opts, libsql.WithSyncInterval(cfg.SyncInterval)) + } + if cfg.EncryptionKey != "" { + opts = append(opts, libsql.WithEncryption(cfg.EncryptionKey)) + } + + connector, err := libsql.NewEmbeddedReplicaConnector(dbPath, cfg.SyncURL, opts...) + if err != nil { + return nil, fmt.Errorf("create embedded replica connector: %w", err) + } + + db := sql.OpenDB(connector) + d, err := newDelegateFromDB(db, connector) + if err != nil { + connector.Close() + return nil, err + } + return d, nil +} + +// Sync manually syncs the embedded replica with the remote primary. +// Returns nil if not in replica mode. +func (d *LibSQLDelegate) Sync() error { + if d.connector == nil { + return nil + } + _, err := d.connector.Sync() + return err +} + +// IsReplica returns true if this delegate is operating as an embedded replica. +func (d *LibSQLDelegate) IsReplica() bool { + return d.connector != nil +} + +func newDelegateFromDB(db *sql.DB, connector *libsql.Connector) (*LibSQLDelegate, error) { db.SetMaxOpenConns(1) - // Set pragmas — journal_mode returns a row, so use QueryRowContext for it. - // go-libsql doesn't support query-string pragmas. ctx := context.Background() var walMode string if err := db.QueryRowContext(ctx, "PRAGMA journal_mode=WAL").Scan(&walMode); err != nil { @@ -62,6 +118,7 @@ func NewLibSQLDelegate(dbPath string) (*LibSQLDelegate, error) { return &LibSQLDelegate{ db: db, + connector: connector, queries: memsqlc.New(db), stmts: newStmtCache(db), embeddingDims: DefaultEmbeddingDims, @@ -87,194 +144,19 @@ func NewLibSQLInMemory() (*LibSQLDelegate, error) { return NewLibSQLDelegate(":memory:") } -// fts5FallbackDDL is a simplified standalone FTS5 DDL without advanced tokenizer. -// Used when the primary FTS5 DDL fails (e.g., tokenchars not supported). -const fts5FallbackDDL = ` -CREATE VIRTUAL TABLE IF NOT EXISTS recall_items_fts USING fts5( - content, - tags -); - -CREATE TRIGGER IF NOT EXISTS recall_items_ai -AFTER INSERT ON recall_items BEGIN - INSERT INTO recall_items_fts(rowid, content, tags) - VALUES (new.rowid, new.content, new.tags); -END; - -CREATE TRIGGER IF NOT EXISTS recall_items_ad -AFTER DELETE ON recall_items BEGIN - DELETE FROM recall_items_fts WHERE rowid = old.rowid; -END; - -CREATE TRIGGER IF NOT EXISTS recall_items_au -AFTER UPDATE ON recall_items BEGIN - DELETE FROM recall_items_fts WHERE rowid = old.rowid; - INSERT INTO recall_items_fts(rowid, content, tags) - VALUES (new.rowid, new.content, new.tags); -END; -` - -// execMultiStatement splits a SQL string into individual statements and -// executes each one. The go-libsql driver only handles one statement per -// ExecContext call. This function handles triggers with BEGIN...END blocks -// by tracking nesting depth. -func execMultiStatement(ctx context.Context, db *sql.DB, ddl string) error { - stmts := splitSQL(ddl) - for _, s := range stmts { - if _, err := db.ExecContext(ctx, s); err != nil { - return fmt.Errorf("failed to execute query %s\n%w", s, err) - } - } - return nil -} - -// splitSQL splits multi-statement SQL into individual statements, -// correctly handling BEGIN...END blocks (triggers) that contain semicolons. -func splitSQL(ddl string) []string { - var result []string - var current strings.Builder - depth := 0 // tracks BEGIN...END nesting - - for _, line := range strings.Split(ddl, "\n") { - trimmed := strings.TrimSpace(line) - - // Skip comment-only and empty lines - if trimmed == "" || strings.HasPrefix(trimmed, "--") { - current.WriteString(line) - current.WriteByte('\n') - continue - } - - upper := strings.ToUpper(trimmed) - - // Track BEGIN...END nesting for triggers. - // BEGIN can appear at start ("BEGIN") or end of a line ("... BEGIN"). - if upper == "BEGIN" || strings.HasSuffix(upper, " BEGIN") || strings.HasSuffix(upper, "\tBEGIN") { - depth++ - } - if upper == "END;" || strings.HasSuffix(upper, "END;") { - depth-- - current.WriteString(line) - current.WriteByte('\n') - if depth <= 0 { - stmt := strings.TrimSpace(current.String()) - if stmt != "" { - result = append(result, stmt) - } - current.Reset() - depth = 0 - } - continue - } - - current.WriteString(line) - current.WriteByte('\n') - - // If we're outside a BEGIN...END block and the line ends with ';', - // treat it as a statement boundary. - if depth == 0 && strings.HasSuffix(trimmed, ";") { - stmt := strings.TrimSpace(current.String()) - if stmt != "" { - result = append(result, stmt) - } - current.Reset() - } - } - - // Capture any trailing statement without a final semicolon - if s := strings.TrimSpace(current.String()); s != "" { - result = append(result, s) - } - - return result -} - -// schemaDDL generates the core DDL with the configured embedding dimensions. -// Entity IDs use BLOB PRIMARY KEY (16-byte UUIDv7). External identifiers remain TEXT. -func (d *LibSQLDelegate) schemaDDL() string { - return fmt.Sprintf(`-- PicoClaw Memory System Schema (libSQL) --- Entity IDs: BLOB PRIMARY KEY (16-byte UUIDv7 RFC 9562) --- External identifiers (agent_id, session_key): TEXT -CREATE TABLE IF NOT EXISTS working_context ( - agent_id TEXT NOT NULL, - session_key TEXT NOT NULL DEFAULT '', - content TEXT NOT NULL DEFAULT '', - updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (agent_id, session_key) -); -CREATE TABLE IF NOT EXISTS recall_items ( - id BLOB PRIMARY KEY, - agent_id TEXT NOT NULL, - session_key TEXT NOT NULL DEFAULT '', - role TEXT NOT NULL DEFAULT 'system', - sector TEXT NOT NULL DEFAULT 'episodic', - importance REAL NOT NULL DEFAULT 0.5, - salience REAL NOT NULL DEFAULT 0.5, - decay_rate REAL NOT NULL DEFAULT 0.01, - content TEXT NOT NULL, - tags TEXT NOT NULL DEFAULT '', - created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP -); -CREATE INDEX IF NOT EXISTS idx_recall_agent_session ON recall_items(agent_id, session_key); -CREATE INDEX IF NOT EXISTS idx_recall_sector ON recall_items(sector); -CREATE INDEX IF NOT EXISTS idx_recall_importance ON recall_items(importance DESC); -CREATE INDEX IF NOT EXISTS idx_recall_created ON recall_items(created_at DESC); -CREATE TABLE IF NOT EXISTS archival_chunks ( - id BLOB PRIMARY KEY, - recall_id BLOB NOT NULL, - chunk_index INTEGER NOT NULL DEFAULT 0, - content TEXT NOT NULL, - embedding F32_BLOB(%d), - source TEXT NOT NULL DEFAULT '', - hash TEXT NOT NULL DEFAULT '', - created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP -); -CREATE INDEX IF NOT EXISTS idx_chunks_recall ON archival_chunks(recall_id); -CREATE INDEX IF NOT EXISTS idx_chunks_source ON archival_chunks(source); -CREATE TABLE IF NOT EXISTS memory_summaries ( - id BLOB PRIMARY KEY, - agent_id TEXT NOT NULL, - session_key TEXT NOT NULL DEFAULT '', - content TEXT NOT NULL, - from_msg_idx INTEGER NOT NULL DEFAULT 0, - to_msg_idx INTEGER NOT NULL DEFAULT 0, - created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP -); -CREATE INDEX IF NOT EXISTS idx_summaries_agent_session ON memory_summaries(agent_id, session_key); -CREATE TRIGGER IF NOT EXISTS recall_cascade_delete -AFTER DELETE ON recall_items BEGIN - DELETE FROM archival_chunks WHERE recall_id = old.id; -END;`, d.embeddingDims) -} - func (d *LibSQLDelegate) Init(ctx context.Context) error { - if err := execMultiStatement(ctx, d.db, d.schemaDDL()); err != nil { - return fmt.Errorf("create schema: %w", err) + // Pass embedding dimensions into goose migration context so the Go-based + // schema migration can create F32_BLOB(N) with the configured dimension. + mctx := migrations.WithEmbeddingDims(ctx, d.embeddingDims) + + // Go-only migrations registered via init() in the migrations package. + // nil filesystem — no SQL files, all logic is in Go migration functions. + provider, err := goose.NewProvider(goose.DialectSQLite3, d.db, nil) + if err != nil { + return fmt.Errorf("create migration provider: %w", err) } - - // FTS5 virtual tables and triggers — try advanced tokenizer first, - // fall back to basic FTS5, then skip entirely if unavailable. - if err := execMultiStatement(ctx, d.db, fts5DDL); err != nil { - // Advanced tokenizer failed — try simplified FTS5 - if err2 := execMultiStatement(ctx, d.db, fts5FallbackDDL); err2 != nil { - // FTS5 not available at all — LIKE-based search will be used - _ = err2 - } - } - - // Backfill: ensure any existing recall_items are indexed in FTS5. - // This is idempotent — only inserts rows not already present. - _, _ = d.db.ExecContext(ctx, - `INSERT INTO recall_items_fts(rowid, content, tags) - SELECT ri.rowid, ri.content, ri.tags - FROM recall_items ri - WHERE NOT EXISTS (SELECT 1 FROM recall_items_fts f WHERE f.rowid = ri.rowid)`) - - // Vector index -- gracefully skip if libSQL vector extension not available - if err := execMultiStatement(ctx, d.db, vectorDDL); err != nil { - // Not fatal: vector search will fall back to Go-side brute-force - _ = err + if _, err := provider.Up(mctx); err != nil { + return fmt.Errorf("run migrations: %w", err) } // Detect runtime capabilities (FTS5, BM25, vector_top_k) @@ -290,7 +172,13 @@ func (d *LibSQLDelegate) Close() error { if d.stmts != nil { d.stmts.close() } - return d.db.Close() + dbErr := d.db.Close() + if d.connector != nil { + if err := d.connector.Close(); err != nil && dbErr == nil { + dbErr = err + } + } + return dbErr } // --- Working Context --- diff --git a/pkg/memory/delegate/vector.sql b/pkg/memory/delegate/vector.sql deleted file mode 100644 index 4a0bce72f..000000000 --- a/pkg/memory/delegate/vector.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Vector index for ANN search on archival chunk embeddings. --- Uses libSQL's native vector indexing. Gracefully skipped if not supported. -CREATE INDEX IF NOT EXISTS idx_chunks_embedding ON archival_chunks(libsql_vector_idx(embedding)); \ No newline at end of file diff --git a/pkg/memory/migrations/001_schema.go b/pkg/memory/migrations/001_schema.go new file mode 100644 index 000000000..1068d72e9 --- /dev/null +++ b/pkg/memory/migrations/001_schema.go @@ -0,0 +1,94 @@ +package migrations + +import ( + "context" + "database/sql" + "fmt" + + "github.com/pressly/goose/v3" +) + +func init() { + goose.AddMigrationContext(up001Schema, down001Schema) +} + +func up001Schema(ctx context.Context, tx *sql.Tx) error { + dims := embeddingDimsFromContext(ctx) + + stmts := []string{ + `CREATE TABLE IF NOT EXISTS working_context ( + agent_id TEXT NOT NULL, + session_key TEXT NOT NULL DEFAULT '', + content TEXT NOT NULL DEFAULT '', + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (agent_id, session_key) +)`, + `CREATE TABLE IF NOT EXISTS recall_items ( + id BLOB PRIMARY KEY, + agent_id TEXT NOT NULL, + session_key TEXT NOT NULL DEFAULT '', + role TEXT NOT NULL DEFAULT 'system', + sector TEXT NOT NULL DEFAULT 'episodic', + importance REAL NOT NULL DEFAULT 0.5, + salience REAL NOT NULL DEFAULT 0.5, + decay_rate REAL NOT NULL DEFAULT 0.01, + content TEXT NOT NULL, + tags TEXT NOT NULL DEFAULT '', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP +)`, + `CREATE INDEX IF NOT EXISTS idx_recall_agent_session ON recall_items(agent_id, session_key)`, + `CREATE INDEX IF NOT EXISTS idx_recall_sector ON recall_items(sector)`, + `CREATE INDEX IF NOT EXISTS idx_recall_importance ON recall_items(importance DESC)`, + `CREATE INDEX IF NOT EXISTS idx_recall_created ON recall_items(created_at DESC)`, + fmt.Sprintf(`CREATE TABLE IF NOT EXISTS archival_chunks ( + id BLOB PRIMARY KEY, + recall_id BLOB NOT NULL, + chunk_index INTEGER NOT NULL DEFAULT 0, + content TEXT NOT NULL, + embedding F32_BLOB(%d), + source TEXT NOT NULL DEFAULT '', + hash TEXT NOT NULL DEFAULT '', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP +)`, dims), + `CREATE INDEX IF NOT EXISTS idx_chunks_recall ON archival_chunks(recall_id)`, + `CREATE INDEX IF NOT EXISTS idx_chunks_source ON archival_chunks(source)`, + `CREATE TABLE IF NOT EXISTS memory_summaries ( + id BLOB PRIMARY KEY, + agent_id TEXT NOT NULL, + session_key TEXT NOT NULL DEFAULT '', + content TEXT NOT NULL, + from_msg_idx INTEGER NOT NULL DEFAULT 0, + to_msg_idx INTEGER NOT NULL DEFAULT 0, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP +)`, + `CREATE INDEX IF NOT EXISTS idx_summaries_agent_session ON memory_summaries(agent_id, session_key)`, + `CREATE TRIGGER IF NOT EXISTS recall_cascade_delete +AFTER DELETE ON recall_items BEGIN + DELETE FROM archival_chunks WHERE recall_id = old.id; +END`, + } + + for _, s := range stmts { + if _, err := tx.ExecContext(ctx, s); err != nil { + return fmt.Errorf("001_schema up: %w\nSQL: %s", err, s) + } + } + return nil +} + +func down001Schema(ctx context.Context, tx *sql.Tx) error { + stmts := []string{ + `DROP TRIGGER IF EXISTS recall_cascade_delete`, + `DROP TABLE IF EXISTS memory_summaries`, + `DROP TABLE IF EXISTS archival_chunks`, + `DROP TABLE IF EXISTS recall_items`, + `DROP TABLE IF EXISTS working_context`, + } + for _, s := range stmts { + if _, err := tx.ExecContext(ctx, s); err != nil { + return fmt.Errorf("001_schema down: %w\nSQL: %s", err, s) + } + } + return nil +} diff --git a/pkg/memory/migrations/002_fts5.go b/pkg/memory/migrations/002_fts5.go new file mode 100644 index 000000000..f1de1806e --- /dev/null +++ b/pkg/memory/migrations/002_fts5.go @@ -0,0 +1,85 @@ +package migrations + +import ( + "context" + "database/sql" + "fmt" + + "github.com/pressly/goose/v3" +) + +func init() { + goose.AddMigrationContext(up002FTS5, down002FTS5) +} + +const fts5AdvancedDDL = `CREATE VIRTUAL TABLE IF NOT EXISTS recall_items_fts USING fts5( + content, + tags, + tokenize = 'unicode61 tokenchars=:-_@./', + prefix = '2 3 4 5 6 7' +)` + +const fts5BasicDDL = `CREATE VIRTUAL TABLE IF NOT EXISTS recall_items_fts USING fts5( + content, + tags +)` + +var fts5Triggers = []string{ + `CREATE TRIGGER IF NOT EXISTS recall_items_ai +AFTER INSERT ON recall_items BEGIN + INSERT INTO recall_items_fts(rowid, content, tags) + VALUES (new.rowid, new.content, new.tags); +END`, + `CREATE TRIGGER IF NOT EXISTS recall_items_ad +AFTER DELETE ON recall_items BEGIN + DELETE FROM recall_items_fts WHERE rowid = old.rowid; +END`, + `CREATE TRIGGER IF NOT EXISTS recall_items_au +AFTER UPDATE ON recall_items BEGIN + DELETE FROM recall_items_fts WHERE rowid = old.rowid; + INSERT INTO recall_items_fts(rowid, content, tags) + VALUES (new.rowid, new.content, new.tags); +END`, +} + +func up002FTS5(ctx context.Context, tx *sql.Tx) error { + // Try advanced tokenizer first, fall back to basic FTS5. + // If FTS5 is entirely unavailable (e.g., stripped build), skip silently — + // the delegate's capability detection will handle graceful degradation. + if _, err := tx.ExecContext(ctx, fts5AdvancedDDL); err != nil { + if _, err2 := tx.ExecContext(ctx, fts5BasicDDL); err2 != nil { + // FTS5 not available — skip, don't fail the migration + return nil + } + } + + for _, s := range fts5Triggers { + if _, err := tx.ExecContext(ctx, s); err != nil { + return fmt.Errorf("002_fts5 trigger: %w\nSQL: %s", err, s) + } + } + + // Backfill any existing recall_items into the FTS index + _, _ = tx.ExecContext(ctx, + `INSERT INTO recall_items_fts(rowid, content, tags) + SELECT ri.rowid, ri.content, ri.tags + FROM recall_items ri + WHERE NOT EXISTS (SELECT 1 FROM recall_items_fts f WHERE f.rowid = ri.rowid)`) + + return nil +} + +func down002FTS5(ctx context.Context, tx *sql.Tx) error { + stmts := []string{ + `DROP TRIGGER IF EXISTS recall_items_au`, + `DROP TRIGGER IF EXISTS recall_items_ad`, + `DROP TRIGGER IF EXISTS recall_items_ai`, + `DROP TABLE IF EXISTS recall_items_fts`, + } + for _, s := range stmts { + if _, err := tx.ExecContext(ctx, s); err != nil { + return fmt.Errorf("002_fts5 down: %w\nSQL: %s", err, s) + } + } + return nil +} diff --git a/pkg/memory/migrations/003_vector_index.go b/pkg/memory/migrations/003_vector_index.go new file mode 100644 index 000000000..5cfe68c76 --- /dev/null +++ b/pkg/memory/migrations/003_vector_index.go @@ -0,0 +1,34 @@ +package migrations + +import ( + "context" + "database/sql" + "fmt" + + "github.com/pressly/goose/v3" +) + +func init() { + goose.AddMigrationContext(up003VectorIndex, down003VectorIndex) +} + +func up003VectorIndex(ctx context.Context, tx *sql.Tx) error { + // libSQL's native vector index for ANN search. Gracefully skip if the + // vector extension is unavailable — the delegate falls back to Go-side + // brute-force cosine similarity. + _, err := tx.ExecContext(ctx, + `CREATE INDEX IF NOT EXISTS idx_chunks_embedding ON archival_chunks(libsql_vector_idx(embedding))`) + if err != nil { + // Not fatal — vector search degrades to Go-side + return nil + } + return nil +} + +func down003VectorIndex(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, `DROP INDEX IF EXISTS idx_chunks_embedding`) + if err != nil { + return fmt.Errorf("003_vector_index down: %w", err) + } + return nil +} diff --git a/pkg/memory/migrations/context.go b/pkg/memory/migrations/context.go new file mode 100644 index 000000000..0e7eee7d0 --- /dev/null +++ b/pkg/memory/migrations/context.go @@ -0,0 +1,20 @@ +package migrations + +import "context" + +type ctxKey int + +const embeddingDimsKey ctxKey = iota + +// WithEmbeddingDims returns a context carrying the embedding vector dimension +// for use by Go-based migrations that create F32_BLOB columns. +func WithEmbeddingDims(ctx context.Context, dims int) context.Context { + return context.WithValue(ctx, embeddingDimsKey, dims) +} + +func embeddingDimsFromContext(ctx context.Context) int { + if v, ok := ctx.Value(embeddingDimsKey).(int); ok && v > 0 { + return v + } + return 768 // default: sentence-transformers +}