feat(memory): replace embedded DDL with goose versioned migrations
Migrate schema management from hardcoded DDL strings to pressly/goose/v3 Go-only migrations. Enables dynamic F32_BLOB(N) dimensions via context, graceful FTS5/vector index degradation, and proper up/down migration paths. - 001_schema: core tables with BLOB PKs and cascade triggers - 002_fts5: full-text search with tokenizer fallback chain - 003_vector_index: libsql_vector_idx with capability detection - LibSQLDelegate.Init() now uses goose.NewProvider with nil filesystem - Removed embedded fts5.sql, vector.sql, schema_init.sql
This commit is contained in:
ZanzyTHEbar
parent
1340d262ed
commit
f488d3d8d2
10 changed files with 350 additions and 291 deletions
4
go.mod
4
go.mod
|
|
@ -16,6 +16,7 @@ require (
|
|||
github.com/mymmrac/telego v1.6.0
|
||||
github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1
|
||||
github.com/openai/openai-go/v2 v2.7.1
|
||||
github.com/pressly/goose/v3 v3.26.0
|
||||
github.com/slack-go/slack v0.17.3
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/tencent-connect/botgo v0.2.1
|
||||
|
|
@ -53,7 +54,9 @@ require (
|
|||
github.com/klauspost/compress v1.18.4 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06 // indirect
|
||||
github.com/mfridman/interpolate v0.0.2 // indirect
|
||||
github.com/pkoukk/tiktoken-go v0.1.6 // indirect
|
||||
github.com/sethvargo/go-retry v0.3.0 // indirect
|
||||
github.com/tidwall/gjson v1.18.0 // indirect
|
||||
github.com/tidwall/match v1.2.0 // indirect
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
|
|
@ -67,6 +70,7 @@ require (
|
|||
gitlab.com/golang-commonmark/markdown v0.0.0-20211110145824-bf3e522c626a // indirect
|
||||
gitlab.com/golang-commonmark/mdurl v0.0.0-20191124015652-932350d1cb84 // indirect
|
||||
gitlab.com/golang-commonmark/puny v0.0.0-20191124015043-9f83538fa04f // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/arch v0.24.0 // indirect
|
||||
golang.org/x/crypto v0.48.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
|
||||
|
|
|
|||
28
go.sum
28
go.sum
|
|
@ -37,6 +37,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc
|
|||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
|
||||
github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/go-json-experiment/json v0.0.0-20251027170946-4849db3c2f7e h1:Lf/gRkoycfOBPa42vU2bbgPurFong6zXeFtPoxholzU=
|
||||
|
|
@ -108,8 +110,14 @@ github.com/larksuite/oapi-sdk-go/v3 v3.5.3 h1:xvf8Dv29kBXC5/DNDCLhHkAFW8l/0LlQJi
|
|||
github.com/larksuite/oapi-sdk-go/v3 v3.5.3/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI=
|
||||
github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06 h1:JLvn7D+wXjH9g4Jsjo+VqmzTUpl/LX7vfr6VOfSWTdM=
|
||||
github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06/go.mod h1:FUkZ5OHjlGPjnM2UyGJz9TypXQFgYqw6AFNO1UiROTM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
|
||||
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
|
||||
github.com/mymmrac/telego v1.6.0 h1:Zc8rgyHozvd/7ZgyrigyHdAF9koHYMfilYfyB6wlFC0=
|
||||
github.com/mymmrac/telego v1.6.0/go.mod h1:xt6ZWA8zi8KmuzryE1ImEdl9JSwjHNpM4yhC7D8hU4Y=
|
||||
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
|
||||
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
|
|
@ -132,12 +140,18 @@ github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYde
|
|||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pressly/goose/v3 v3.26.0 h1:KJakav68jdH0WDvoAcj8+n61WqOIaPGgH0bJWS6jpmM=
|
||||
github.com/pressly/goose/v3 v3.26.0/go.mod h1:4hC1KrritdCxtuFsqgs1R4AU5bWtTAf+cnWvfhf2DNY=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
|
||||
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE=
|
||||
github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
|
||||
github.com/slack-go/slack v0.17.3 h1:zV5qO3Q+WJAQ/XwbGfNFrRMaJ5T/naqaonyPV/1TP4g=
|
||||
github.com/slack-go/slack v0.17.3/go.mod h1:X+UqOufi3LYQHDnMG1vxf0J8asC6+WllXrVrhl8/Prk=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
|
|
@ -197,6 +211,8 @@ gitlab.com/opennota/wd v0.0.0-20180912061657-c5d65f63c638 h1:uPZaMiz6Sz0PZs3IZJW
|
|||
gitlab.com/opennota/wd v0.0.0-20180912061657-c5d65f63c638/go.mod h1:EGRJaqe2eO9XGmFtQCvV3Lm9NLico3UhFwUpCG/+mVU=
|
||||
go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
|
||||
go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y=
|
||||
golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
|
|
@ -313,5 +329,13 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
|||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
|
||||
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
|
||||
modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ=
|
||||
modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/sqlite v1.38.2 h1:Aclu7+tgjgcQVShZqim41Bbw9Cho0y/7WzYptXqkEek=
|
||||
modernc.org/sqlite v1.38.2/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E=
|
||||
sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
|
||||
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
|
||||
|
|
|
|||
|
|
@ -1,32 +0,0 @@
|
|||
-- FTS5 virtual table for keyword search on recall items.
|
||||
-- Standalone FTS5 table (NOT external-content mode) — more reliable with go-libsql.
|
||||
-- Uses unicode61 tokenizer with extended tokenchars for domain-specific identifiers
|
||||
-- and prefix indexes for efficient prefix matching.
|
||||
-- NOTE: tokenchars uses equals-sign syntax (not space+quotes) per go-libsql compatibility.
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS recall_items_fts USING fts5(
|
||||
content,
|
||||
tags,
|
||||
tokenize = 'unicode61 tokenchars=:-_@./',
|
||||
prefix = '2 3 4 5 6 7'
|
||||
);
|
||||
-- Triggers to keep standalone FTS5 table in sync with recall_items.
|
||||
-- Uses DELETE+INSERT pattern for UPDATE (FTS5 standard approach).
|
||||
CREATE TRIGGER IF NOT EXISTS recall_items_ai
|
||||
AFTER
|
||||
INSERT ON recall_items BEGIN
|
||||
INSERT INTO recall_items_fts(rowid, content, tags)
|
||||
VALUES (new.rowid, new.content, new.tags);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS recall_items_ad
|
||||
AFTER DELETE ON recall_items BEGIN
|
||||
DELETE FROM recall_items_fts
|
||||
WHERE rowid = old.rowid;
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS recall_items_au
|
||||
AFTER
|
||||
UPDATE ON recall_items BEGIN
|
||||
DELETE FROM recall_items_fts
|
||||
WHERE rowid = old.rowid;
|
||||
INSERT INTO recall_items_fts(rowid, content, tags)
|
||||
VALUES (new.rowid, new.content, new.tags);
|
||||
END;
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
-- PicoClaw Memory System Schema (libSQL)
|
||||
-- Managed by delegate, not sqlc, to allow full DDL including F32_BLOB and pragmas.
|
||||
CREATE TABLE IF NOT EXISTS working_context (
|
||||
agent_id TEXT NOT NULL,
|
||||
session_key TEXT NOT NULL DEFAULT '',
|
||||
content TEXT NOT NULL DEFAULT '',
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (agent_id, session_key)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS recall_items (
|
||||
id TEXT PRIMARY KEY,
|
||||
agent_id TEXT NOT NULL,
|
||||
session_key TEXT NOT NULL DEFAULT '',
|
||||
role TEXT NOT NULL DEFAULT 'system',
|
||||
sector TEXT NOT NULL DEFAULT 'episodic',
|
||||
importance REAL NOT NULL DEFAULT 0.5,
|
||||
salience REAL NOT NULL DEFAULT 0.5,
|
||||
decay_rate REAL NOT NULL DEFAULT 0.01,
|
||||
content TEXT NOT NULL,
|
||||
tags TEXT NOT NULL DEFAULT '',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_recall_agent_session ON recall_items(agent_id, session_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_recall_sector ON recall_items(sector);
|
||||
CREATE INDEX IF NOT EXISTS idx_recall_importance ON recall_items(importance DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_recall_created ON recall_items(created_at DESC);
|
||||
CREATE TABLE IF NOT EXISTS archival_chunks (
|
||||
id TEXT PRIMARY KEY,
|
||||
recall_id TEXT NOT NULL DEFAULT '',
|
||||
chunk_index INTEGER NOT NULL DEFAULT 0,
|
||||
content TEXT NOT NULL,
|
||||
embedding F32_BLOB(768),
|
||||
source TEXT NOT NULL DEFAULT '',
|
||||
hash TEXT NOT NULL DEFAULT '',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_chunks_recall ON archival_chunks(recall_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_chunks_source ON archival_chunks(source);
|
||||
CREATE TABLE IF NOT EXISTS memory_summaries (
|
||||
id TEXT PRIMARY KEY,
|
||||
agent_id TEXT NOT NULL,
|
||||
session_key TEXT NOT NULL DEFAULT '',
|
||||
content TEXT NOT NULL,
|
||||
from_msg_idx INTEGER NOT NULL DEFAULT 0,
|
||||
to_msg_idx INTEGER NOT NULL DEFAULT 0,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_summaries_agent_session ON memory_summaries(agent_id, session_key);
|
||||
-- Cascade: when a recall item is deleted, remove its archival chunks.
|
||||
CREATE TRIGGER IF NOT EXISTS recall_cascade_delete
|
||||
AFTER DELETE ON recall_items BEGIN
|
||||
DELETE FROM archival_chunks
|
||||
WHERE recall_id = old.id;
|
||||
END;
|
||||
|
|
@ -4,23 +4,18 @@ package delegate
|
|||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pressly/goose/v3"
|
||||
"github.com/sipeed/picoclaw/pkg/ids"
|
||||
"github.com/sipeed/picoclaw/pkg/memory"
|
||||
"github.com/sipeed/picoclaw/pkg/memory/migrations"
|
||||
memsqlc "github.com/sipeed/picoclaw/pkg/memory/sqlc"
|
||||
|
||||
_ "github.com/tursodatabase/go-libsql" // register "libsql" driver
|
||||
libsql "github.com/tursodatabase/go-libsql"
|
||||
)
|
||||
|
||||
//go:embed fts5.sql
|
||||
var fts5DDL string
|
||||
|
||||
//go:embed vector.sql
|
||||
var vectorDDL string
|
||||
|
||||
// DefaultEmbeddingDims is the default number of dimensions for embedding vectors.
|
||||
// This matches common models like sentence-transformers (768-dim).
|
||||
const DefaultEmbeddingDims = 768
|
||||
|
|
@ -30,12 +25,28 @@ const DefaultEmbeddingDims = 768
|
|||
// Hand-written SQL (FTS5, vector search) uses a prepared statement cache.
|
||||
type LibSQLDelegate struct {
|
||||
db *sql.DB
|
||||
connector *libsql.Connector // non-nil when using embedded replica mode
|
||||
queries *memsqlc.Queries
|
||||
stmts *stmtCache
|
||||
caps capFlags
|
||||
embeddingDims int
|
||||
}
|
||||
|
||||
// SyncConfig configures Turso embedded replica synchronization.
|
||||
type SyncConfig struct {
|
||||
// SyncURL is the remote primary database URL (e.g., "libsql://mydb.turso.io").
|
||||
SyncURL string
|
||||
|
||||
// AuthToken for the remote database.
|
||||
AuthToken string
|
||||
|
||||
// SyncInterval is how often to auto-sync. Zero means manual sync only.
|
||||
SyncInterval time.Duration
|
||||
|
||||
// EncryptionKey enables encryption-at-rest. Empty means no encryption.
|
||||
EncryptionKey string
|
||||
}
|
||||
|
||||
// NewLibSQLDelegate opens a libSQL database at the given path and returns
|
||||
// a delegate ready for use. Call Init() to create tables.
|
||||
// Uses DefaultEmbeddingDims (768) for the vector column size.
|
||||
|
|
@ -44,11 +55,56 @@ func NewLibSQLDelegate(dbPath string) (*LibSQLDelegate, error) {
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("open libsql: %w", err)
|
||||
}
|
||||
// Single writer for WAL mode safety
|
||||
|
||||
return newDelegateFromDB(db, nil)
|
||||
}
|
||||
|
||||
// NewLibSQLDelegateWithSync opens a libSQL database as an embedded replica
|
||||
// that syncs with a remote Turso primary. Reads are served from the local file;
|
||||
// writes propagate to the remote primary and sync back.
|
||||
func NewLibSQLDelegateWithSync(dbPath string, cfg SyncConfig) (*LibSQLDelegate, error) {
|
||||
opts := []libsql.Option{
|
||||
libsql.WithAuthToken(cfg.AuthToken),
|
||||
}
|
||||
if cfg.SyncInterval > 0 {
|
||||
opts = append(opts, libsql.WithSyncInterval(cfg.SyncInterval))
|
||||
}
|
||||
if cfg.EncryptionKey != "" {
|
||||
opts = append(opts, libsql.WithEncryption(cfg.EncryptionKey))
|
||||
}
|
||||
|
||||
connector, err := libsql.NewEmbeddedReplicaConnector(dbPath, cfg.SyncURL, opts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create embedded replica connector: %w", err)
|
||||
}
|
||||
|
||||
db := sql.OpenDB(connector)
|
||||
d, err := newDelegateFromDB(db, connector)
|
||||
if err != nil {
|
||||
connector.Close()
|
||||
return nil, err
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// Sync manually syncs the embedded replica with the remote primary.
|
||||
// Returns nil if not in replica mode.
|
||||
func (d *LibSQLDelegate) Sync() error {
|
||||
if d.connector == nil {
|
||||
return nil
|
||||
}
|
||||
_, err := d.connector.Sync()
|
||||
return err
|
||||
}
|
||||
|
||||
// IsReplica returns true if this delegate is operating as an embedded replica.
|
||||
func (d *LibSQLDelegate) IsReplica() bool {
|
||||
return d.connector != nil
|
||||
}
|
||||
|
||||
func newDelegateFromDB(db *sql.DB, connector *libsql.Connector) (*LibSQLDelegate, error) {
|
||||
db.SetMaxOpenConns(1)
|
||||
|
||||
// Set pragmas — journal_mode returns a row, so use QueryRowContext for it.
|
||||
// go-libsql doesn't support query-string pragmas.
|
||||
ctx := context.Background()
|
||||
var walMode string
|
||||
if err := db.QueryRowContext(ctx, "PRAGMA journal_mode=WAL").Scan(&walMode); err != nil {
|
||||
|
|
@ -62,6 +118,7 @@ func NewLibSQLDelegate(dbPath string) (*LibSQLDelegate, error) {
|
|||
|
||||
return &LibSQLDelegate{
|
||||
db: db,
|
||||
connector: connector,
|
||||
queries: memsqlc.New(db),
|
||||
stmts: newStmtCache(db),
|
||||
embeddingDims: DefaultEmbeddingDims,
|
||||
|
|
@ -87,194 +144,19 @@ func NewLibSQLInMemory() (*LibSQLDelegate, error) {
|
|||
return NewLibSQLDelegate(":memory:")
|
||||
}
|
||||
|
||||
// fts5FallbackDDL is a simplified standalone FTS5 DDL without advanced tokenizer.
|
||||
// Used when the primary FTS5 DDL fails (e.g., tokenchars not supported).
|
||||
const fts5FallbackDDL = `
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS recall_items_fts USING fts5(
|
||||
content,
|
||||
tags
|
||||
);
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS recall_items_ai
|
||||
AFTER INSERT ON recall_items BEGIN
|
||||
INSERT INTO recall_items_fts(rowid, content, tags)
|
||||
VALUES (new.rowid, new.content, new.tags);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS recall_items_ad
|
||||
AFTER DELETE ON recall_items BEGIN
|
||||
DELETE FROM recall_items_fts WHERE rowid = old.rowid;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS recall_items_au
|
||||
AFTER UPDATE ON recall_items BEGIN
|
||||
DELETE FROM recall_items_fts WHERE rowid = old.rowid;
|
||||
INSERT INTO recall_items_fts(rowid, content, tags)
|
||||
VALUES (new.rowid, new.content, new.tags);
|
||||
END;
|
||||
`
|
||||
|
||||
// execMultiStatement splits a SQL string into individual statements and
|
||||
// executes each one. The go-libsql driver only handles one statement per
|
||||
// ExecContext call. This function handles triggers with BEGIN...END blocks
|
||||
// by tracking nesting depth.
|
||||
func execMultiStatement(ctx context.Context, db *sql.DB, ddl string) error {
|
||||
stmts := splitSQL(ddl)
|
||||
for _, s := range stmts {
|
||||
if _, err := db.ExecContext(ctx, s); err != nil {
|
||||
return fmt.Errorf("failed to execute query %s\n%w", s, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// splitSQL splits multi-statement SQL into individual statements,
|
||||
// correctly handling BEGIN...END blocks (triggers) that contain semicolons.
|
||||
func splitSQL(ddl string) []string {
|
||||
var result []string
|
||||
var current strings.Builder
|
||||
depth := 0 // tracks BEGIN...END nesting
|
||||
|
||||
for _, line := range strings.Split(ddl, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
|
||||
// Skip comment-only and empty lines
|
||||
if trimmed == "" || strings.HasPrefix(trimmed, "--") {
|
||||
current.WriteString(line)
|
||||
current.WriteByte('\n')
|
||||
continue
|
||||
}
|
||||
|
||||
upper := strings.ToUpper(trimmed)
|
||||
|
||||
// Track BEGIN...END nesting for triggers.
|
||||
// BEGIN can appear at start ("BEGIN") or end of a line ("... BEGIN").
|
||||
if upper == "BEGIN" || strings.HasSuffix(upper, " BEGIN") || strings.HasSuffix(upper, "\tBEGIN") {
|
||||
depth++
|
||||
}
|
||||
if upper == "END;" || strings.HasSuffix(upper, "END;") {
|
||||
depth--
|
||||
current.WriteString(line)
|
||||
current.WriteByte('\n')
|
||||
if depth <= 0 {
|
||||
stmt := strings.TrimSpace(current.String())
|
||||
if stmt != "" {
|
||||
result = append(result, stmt)
|
||||
}
|
||||
current.Reset()
|
||||
depth = 0
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
current.WriteString(line)
|
||||
current.WriteByte('\n')
|
||||
|
||||
// If we're outside a BEGIN...END block and the line ends with ';',
|
||||
// treat it as a statement boundary.
|
||||
if depth == 0 && strings.HasSuffix(trimmed, ";") {
|
||||
stmt := strings.TrimSpace(current.String())
|
||||
if stmt != "" {
|
||||
result = append(result, stmt)
|
||||
}
|
||||
current.Reset()
|
||||
}
|
||||
}
|
||||
|
||||
// Capture any trailing statement without a final semicolon
|
||||
if s := strings.TrimSpace(current.String()); s != "" {
|
||||
result = append(result, s)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// schemaDDL generates the core DDL with the configured embedding dimensions.
|
||||
// Entity IDs use BLOB PRIMARY KEY (16-byte UUIDv7). External identifiers remain TEXT.
|
||||
func (d *LibSQLDelegate) schemaDDL() string {
|
||||
return fmt.Sprintf(`-- PicoClaw Memory System Schema (libSQL)
|
||||
-- Entity IDs: BLOB PRIMARY KEY (16-byte UUIDv7 RFC 9562)
|
||||
-- External identifiers (agent_id, session_key): TEXT
|
||||
CREATE TABLE IF NOT EXISTS working_context (
|
||||
agent_id TEXT NOT NULL,
|
||||
session_key TEXT NOT NULL DEFAULT '',
|
||||
content TEXT NOT NULL DEFAULT '',
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (agent_id, session_key)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS recall_items (
|
||||
id BLOB PRIMARY KEY,
|
||||
agent_id TEXT NOT NULL,
|
||||
session_key TEXT NOT NULL DEFAULT '',
|
||||
role TEXT NOT NULL DEFAULT 'system',
|
||||
sector TEXT NOT NULL DEFAULT 'episodic',
|
||||
importance REAL NOT NULL DEFAULT 0.5,
|
||||
salience REAL NOT NULL DEFAULT 0.5,
|
||||
decay_rate REAL NOT NULL DEFAULT 0.01,
|
||||
content TEXT NOT NULL,
|
||||
tags TEXT NOT NULL DEFAULT '',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_recall_agent_session ON recall_items(agent_id, session_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_recall_sector ON recall_items(sector);
|
||||
CREATE INDEX IF NOT EXISTS idx_recall_importance ON recall_items(importance DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_recall_created ON recall_items(created_at DESC);
|
||||
CREATE TABLE IF NOT EXISTS archival_chunks (
|
||||
id BLOB PRIMARY KEY,
|
||||
recall_id BLOB NOT NULL,
|
||||
chunk_index INTEGER NOT NULL DEFAULT 0,
|
||||
content TEXT NOT NULL,
|
||||
embedding F32_BLOB(%d),
|
||||
source TEXT NOT NULL DEFAULT '',
|
||||
hash TEXT NOT NULL DEFAULT '',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_chunks_recall ON archival_chunks(recall_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_chunks_source ON archival_chunks(source);
|
||||
CREATE TABLE IF NOT EXISTS memory_summaries (
|
||||
id BLOB PRIMARY KEY,
|
||||
agent_id TEXT NOT NULL,
|
||||
session_key TEXT NOT NULL DEFAULT '',
|
||||
content TEXT NOT NULL,
|
||||
from_msg_idx INTEGER NOT NULL DEFAULT 0,
|
||||
to_msg_idx INTEGER NOT NULL DEFAULT 0,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_summaries_agent_session ON memory_summaries(agent_id, session_key);
|
||||
CREATE TRIGGER IF NOT EXISTS recall_cascade_delete
|
||||
AFTER DELETE ON recall_items BEGIN
|
||||
DELETE FROM archival_chunks WHERE recall_id = old.id;
|
||||
END;`, d.embeddingDims)
|
||||
}
|
||||
|
||||
func (d *LibSQLDelegate) Init(ctx context.Context) error {
|
||||
if err := execMultiStatement(ctx, d.db, d.schemaDDL()); err != nil {
|
||||
return fmt.Errorf("create schema: %w", err)
|
||||
}
|
||||
// Pass embedding dimensions into goose migration context so the Go-based
|
||||
// schema migration can create F32_BLOB(N) with the configured dimension.
|
||||
mctx := migrations.WithEmbeddingDims(ctx, d.embeddingDims)
|
||||
|
||||
// FTS5 virtual tables and triggers — try advanced tokenizer first,
|
||||
// fall back to basic FTS5, then skip entirely if unavailable.
|
||||
if err := execMultiStatement(ctx, d.db, fts5DDL); err != nil {
|
||||
// Advanced tokenizer failed — try simplified FTS5
|
||||
if err2 := execMultiStatement(ctx, d.db, fts5FallbackDDL); err2 != nil {
|
||||
// FTS5 not available at all — LIKE-based search will be used
|
||||
_ = err2
|
||||
// Go-only migrations registered via init() in the migrations package.
|
||||
// nil filesystem — no SQL files, all logic is in Go migration functions.
|
||||
provider, err := goose.NewProvider(goose.DialectSQLite3, d.db, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create migration provider: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Backfill: ensure any existing recall_items are indexed in FTS5.
|
||||
// This is idempotent — only inserts rows not already present.
|
||||
_, _ = d.db.ExecContext(ctx,
|
||||
`INSERT INTO recall_items_fts(rowid, content, tags)
|
||||
SELECT ri.rowid, ri.content, ri.tags
|
||||
FROM recall_items ri
|
||||
WHERE NOT EXISTS (SELECT 1 FROM recall_items_fts f WHERE f.rowid = ri.rowid)`)
|
||||
|
||||
// Vector index -- gracefully skip if libSQL vector extension not available
|
||||
if err := execMultiStatement(ctx, d.db, vectorDDL); err != nil {
|
||||
// Not fatal: vector search will fall back to Go-side brute-force
|
||||
_ = err
|
||||
if _, err := provider.Up(mctx); err != nil {
|
||||
return fmt.Errorf("run migrations: %w", err)
|
||||
}
|
||||
|
||||
// Detect runtime capabilities (FTS5, BM25, vector_top_k)
|
||||
|
|
@ -290,7 +172,13 @@ func (d *LibSQLDelegate) Close() error {
|
|||
if d.stmts != nil {
|
||||
d.stmts.close()
|
||||
}
|
||||
return d.db.Close()
|
||||
dbErr := d.db.Close()
|
||||
if d.connector != nil {
|
||||
if err := d.connector.Close(); err != nil && dbErr == nil {
|
||||
dbErr = err
|
||||
}
|
||||
}
|
||||
return dbErr
|
||||
}
|
||||
|
||||
// --- Working Context ---
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
-- Vector index for ANN search on archival chunk embeddings.
|
||||
-- Uses libSQL's native vector indexing. Gracefully skipped if not supported.
|
||||
CREATE INDEX IF NOT EXISTS idx_chunks_embedding ON archival_chunks(libsql_vector_idx(embedding));
|
||||
94
pkg/memory/migrations/001_schema.go
Normal file
94
pkg/memory/migrations/001_schema.go
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
"github.com/pressly/goose/v3"
|
||||
)
|
||||
|
||||
func init() {
|
||||
goose.AddMigrationContext(up001Schema, down001Schema)
|
||||
}
|
||||
|
||||
func up001Schema(ctx context.Context, tx *sql.Tx) error {
|
||||
dims := embeddingDimsFromContext(ctx)
|
||||
|
||||
stmts := []string{
|
||||
`CREATE TABLE IF NOT EXISTS working_context (
|
||||
agent_id TEXT NOT NULL,
|
||||
session_key TEXT NOT NULL DEFAULT '',
|
||||
content TEXT NOT NULL DEFAULT '',
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (agent_id, session_key)
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS recall_items (
|
||||
id BLOB PRIMARY KEY,
|
||||
agent_id TEXT NOT NULL,
|
||||
session_key TEXT NOT NULL DEFAULT '',
|
||||
role TEXT NOT NULL DEFAULT 'system',
|
||||
sector TEXT NOT NULL DEFAULT 'episodic',
|
||||
importance REAL NOT NULL DEFAULT 0.5,
|
||||
salience REAL NOT NULL DEFAULT 0.5,
|
||||
decay_rate REAL NOT NULL DEFAULT 0.01,
|
||||
content TEXT NOT NULL,
|
||||
tags TEXT NOT NULL DEFAULT '',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_recall_agent_session ON recall_items(agent_id, session_key)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_recall_sector ON recall_items(sector)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_recall_importance ON recall_items(importance DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_recall_created ON recall_items(created_at DESC)`,
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS archival_chunks (
|
||||
id BLOB PRIMARY KEY,
|
||||
recall_id BLOB NOT NULL,
|
||||
chunk_index INTEGER NOT NULL DEFAULT 0,
|
||||
content TEXT NOT NULL,
|
||||
embedding F32_BLOB(%d),
|
||||
source TEXT NOT NULL DEFAULT '',
|
||||
hash TEXT NOT NULL DEFAULT '',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)`, dims),
|
||||
`CREATE INDEX IF NOT EXISTS idx_chunks_recall ON archival_chunks(recall_id)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_chunks_source ON archival_chunks(source)`,
|
||||
`CREATE TABLE IF NOT EXISTS memory_summaries (
|
||||
id BLOB PRIMARY KEY,
|
||||
agent_id TEXT NOT NULL,
|
||||
session_key TEXT NOT NULL DEFAULT '',
|
||||
content TEXT NOT NULL,
|
||||
from_msg_idx INTEGER NOT NULL DEFAULT 0,
|
||||
to_msg_idx INTEGER NOT NULL DEFAULT 0,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_summaries_agent_session ON memory_summaries(agent_id, session_key)`,
|
||||
`CREATE TRIGGER IF NOT EXISTS recall_cascade_delete
|
||||
AFTER DELETE ON recall_items BEGIN
|
||||
DELETE FROM archival_chunks WHERE recall_id = old.id;
|
||||
END`,
|
||||
}
|
||||
|
||||
for _, s := range stmts {
|
||||
if _, err := tx.ExecContext(ctx, s); err != nil {
|
||||
return fmt.Errorf("001_schema up: %w\nSQL: %s", err, s)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func down001Schema(ctx context.Context, tx *sql.Tx) error {
|
||||
stmts := []string{
|
||||
`DROP TRIGGER IF EXISTS recall_cascade_delete`,
|
||||
`DROP TABLE IF EXISTS memory_summaries`,
|
||||
`DROP TABLE IF EXISTS archival_chunks`,
|
||||
`DROP TABLE IF EXISTS recall_items`,
|
||||
`DROP TABLE IF EXISTS working_context`,
|
||||
}
|
||||
for _, s := range stmts {
|
||||
if _, err := tx.ExecContext(ctx, s); err != nil {
|
||||
return fmt.Errorf("001_schema down: %w\nSQL: %s", err, s)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
85
pkg/memory/migrations/002_fts5.go
Normal file
85
pkg/memory/migrations/002_fts5.go
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
"github.com/pressly/goose/v3"
|
||||
)
|
||||
|
||||
func init() {
|
||||
goose.AddMigrationContext(up002FTS5, down002FTS5)
|
||||
}
|
||||
|
||||
const fts5AdvancedDDL = `CREATE VIRTUAL TABLE IF NOT EXISTS recall_items_fts USING fts5(
|
||||
content,
|
||||
tags,
|
||||
tokenize = 'unicode61 tokenchars=:-_@./',
|
||||
prefix = '2 3 4 5 6 7'
|
||||
)`
|
||||
|
||||
const fts5BasicDDL = `CREATE VIRTUAL TABLE IF NOT EXISTS recall_items_fts USING fts5(
|
||||
content,
|
||||
tags
|
||||
)`
|
||||
|
||||
var fts5Triggers = []string{
|
||||
`CREATE TRIGGER IF NOT EXISTS recall_items_ai
|
||||
AFTER INSERT ON recall_items BEGIN
|
||||
INSERT INTO recall_items_fts(rowid, content, tags)
|
||||
VALUES (new.rowid, new.content, new.tags);
|
||||
END`,
|
||||
`CREATE TRIGGER IF NOT EXISTS recall_items_ad
|
||||
AFTER DELETE ON recall_items BEGIN
|
||||
DELETE FROM recall_items_fts WHERE rowid = old.rowid;
|
||||
END`,
|
||||
`CREATE TRIGGER IF NOT EXISTS recall_items_au
|
||||
AFTER UPDATE ON recall_items BEGIN
|
||||
DELETE FROM recall_items_fts WHERE rowid = old.rowid;
|
||||
INSERT INTO recall_items_fts(rowid, content, tags)
|
||||
VALUES (new.rowid, new.content, new.tags);
|
||||
END`,
|
||||
}
|
||||
|
||||
func up002FTS5(ctx context.Context, tx *sql.Tx) error {
|
||||
// Try advanced tokenizer first, fall back to basic FTS5.
|
||||
// If FTS5 is entirely unavailable (e.g., stripped build), skip silently —
|
||||
// the delegate's capability detection will handle graceful degradation.
|
||||
if _, err := tx.ExecContext(ctx, fts5AdvancedDDL); err != nil {
|
||||
if _, err2 := tx.ExecContext(ctx, fts5BasicDDL); err2 != nil {
|
||||
// FTS5 not available — skip, don't fail the migration
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
for _, s := range fts5Triggers {
|
||||
if _, err := tx.ExecContext(ctx, s); err != nil {
|
||||
return fmt.Errorf("002_fts5 trigger: %w\nSQL: %s", err, s)
|
||||
}
|
||||
}
|
||||
|
||||
// Backfill any existing recall_items into the FTS index
|
||||
_, _ = tx.ExecContext(ctx,
|
||||
`INSERT INTO recall_items_fts(rowid, content, tags)
|
||||
SELECT ri.rowid, ri.content, ri.tags
|
||||
FROM recall_items ri
|
||||
WHERE NOT EXISTS (SELECT 1 FROM recall_items_fts f WHERE f.rowid = ri.rowid)`)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func down002FTS5(ctx context.Context, tx *sql.Tx) error {
|
||||
stmts := []string{
|
||||
`DROP TRIGGER IF EXISTS recall_items_au`,
|
||||
`DROP TRIGGER IF EXISTS recall_items_ad`,
|
||||
`DROP TRIGGER IF EXISTS recall_items_ai`,
|
||||
`DROP TABLE IF EXISTS recall_items_fts`,
|
||||
}
|
||||
for _, s := range stmts {
|
||||
if _, err := tx.ExecContext(ctx, s); err != nil {
|
||||
return fmt.Errorf("002_fts5 down: %w\nSQL: %s", err, s)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
34
pkg/memory/migrations/003_vector_index.go
Normal file
34
pkg/memory/migrations/003_vector_index.go
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
"github.com/pressly/goose/v3"
|
||||
)
|
||||
|
||||
func init() {
|
||||
goose.AddMigrationContext(up003VectorIndex, down003VectorIndex)
|
||||
}
|
||||
|
||||
func up003VectorIndex(ctx context.Context, tx *sql.Tx) error {
|
||||
// libSQL's native vector index for ANN search. Gracefully skip if the
|
||||
// vector extension is unavailable — the delegate falls back to Go-side
|
||||
// brute-force cosine similarity.
|
||||
_, err := tx.ExecContext(ctx,
|
||||
`CREATE INDEX IF NOT EXISTS idx_chunks_embedding ON archival_chunks(libsql_vector_idx(embedding))`)
|
||||
if err != nil {
|
||||
// Not fatal — vector search degrades to Go-side
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func down003VectorIndex(ctx context.Context, tx *sql.Tx) error {
|
||||
_, err := tx.ExecContext(ctx, `DROP INDEX IF EXISTS idx_chunks_embedding`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("003_vector_index down: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
20
pkg/memory/migrations/context.go
Normal file
20
pkg/memory/migrations/context.go
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
package migrations
|
||||
|
||||
import "context"
|
||||
|
||||
type ctxKey int
|
||||
|
||||
const embeddingDimsKey ctxKey = iota
|
||||
|
||||
// WithEmbeddingDims returns a context carrying the embedding vector dimension
|
||||
// for use by Go-based migrations that create F32_BLOB columns.
|
||||
func WithEmbeddingDims(ctx context.Context, dims int) context.Context {
|
||||
return context.WithValue(ctx, embeddingDimsKey, dims)
|
||||
}
|
||||
|
||||
func embeddingDimsFromContext(ctx context.Context) int {
|
||||
if v, ok := ctx.Value(embeddingDimsKey).(int); ok && v > 0 {
|
||||
return v
|
||||
}
|
||||
return 768 // default: sentence-transformers
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue