From 8566ff6739c1c07abbade2fe47ad8827841a315f Mon Sep 17 00:00:00 2001 From: Administrator <1280842908@qq.com> Date: Tue, 10 Mar 2026 11:26:48 +0800 Subject: [PATCH] build: remove sqlite vector memory to fix cross-compilation This removes the modernc.org/sqlite dependency from the default build tree, resolving CGO_ENABLED=0 compilation errors on non-amd64 architectures. --- go.sum | 2 - pkg/agent/instance.go | 28 +--- pkg/agent/memory.go | 55 +------- pkg/agent/memory_vector.go | 264 ------------------------------------- pkg/config/config.go | 11 -- 5 files changed, 3 insertions(+), 357 deletions(-) delete mode 100644 pkg/agent/memory_vector.go diff --git a/go.sum b/go.sum index 4060997f8..6d90e6b94 100644 --- a/go.sum +++ b/go.sum @@ -269,8 +269,6 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= -golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= -golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= diff --git a/pkg/agent/instance.go b/pkg/agent/instance.go index 09062746a..96512b929 100644 --- a/pkg/agent/instance.go +++ b/pkg/agent/instance.go @@ -1,7 +1,7 @@ package agent import ( - "context" + "fmt" "log" "os" @@ -11,7 +11,6 @@ import ( "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/providers" - "github.com/sipeed/picoclaw/pkg/providers/openai_compat" "github.com/sipeed/picoclaw/pkg/routing" "github.com/sipeed/picoclaw/pkg/session" "github.com/sipeed/picoclaw/pkg/tools" @@ -117,31 +116,6 @@ func NewAgentInstance( skillsFilter = agentCfg.Skills } - // Initialize vector memory if enabled - if cfg != nil && cfg.Tools.VectorMemory.Enabled { - if cfg.Tools.VectorMemory.APIBase == "" || cfg.Tools.VectorMemory.APIKey == "" { - log.Printf("Warning: vector memory enabled but API base/key not configured") - } else { - // Create a dedicated provider for embeddings (we assume OpenAI-compatible for embeddings) - embedProvider := openai_compat.NewProvider( - cfg.Tools.VectorMemory.APIKey, - cfg.Tools.VectorMemory.APIBase, - "", // no proxy needed by default, could be added later - ) - - dbPath := filepath.Join(workspace, "memory", "memory.sqlite") - vs, err := NewVectorMemoryStore(dbPath, cfg.Tools.VectorMemory.EmbeddingModel, cfg.Tools.VectorMemory.TopK) - if err != nil { - log.Printf("Warning: failed to initialize vector memory store for agent %s: %v", agentName, err) - } else { - embedFn := func(text string) ([]float32, error) { - // Use context.Background() here because this runs asynchronously or during sync - return embedProvider.Embed(context.Background(), text, cfg.Tools.VectorMemory.EmbeddingModel) - } - contextBuilder.memory.SetVectorStore(vs, embedFn) - } - } - } maxIter := defaults.MaxToolIterations if maxIter == 0 { diff --git a/pkg/agent/memory.go b/pkg/agent/memory.go index e3673c154..d5205c228 100644 --- a/pkg/agent/memory.go +++ b/pkg/agent/memory.go @@ -7,9 +7,7 @@ package agent import ( - "context" "fmt" - "io/fs" "os" "path/filepath" "strings" @@ -21,13 +19,10 @@ import ( // MemoryStore manages persistent memory for the agent. // - Long-term memory: memory/MEMORY.md // - Daily notes: memory/YYYYMM/YYYYMMDD.md -// - Optional: SQLite vector store for semantic search type MemoryStore struct { workspace string memoryDir string memoryFile string - vectorStore *VectorMemoryStore - embedFn func(string) ([]float32, error) // nil when vector search disabled lastSyncTime time.Time } @@ -47,13 +42,6 @@ func NewMemoryStore(workspace string) *MemoryStore { } } -// SetVectorStore attaches a VectorMemoryStore and embedding function. -// When set, GetMemoryContext will perform semantic retrieval instead of full-text load. -func (ms *MemoryStore) SetVectorStore(vs *VectorMemoryStore, embedFn func(string) ([]float32, error)) { - ms.vectorStore = vs - ms.embedFn = embedFn -} - // getTodayFile returns the path to today's daily note file (memory/YYYYMM/YYYYMMDD.md). func (ms *MemoryStore) getTodayFile() string { today := time.Now().Format("20060102") // YYYYMMDD @@ -143,48 +131,9 @@ func (ms *MemoryStore) GetRecentDailyNotes(days int) string { } // GetMemoryContext returns formatted memory context for the agent prompt. -// When a vector store is configured, it performs semantic retrieval using the -// query text. Otherwise it falls back to loading the full MEMORY.md. +// It loads the full MEMORY.md. func (ms *MemoryStore) GetMemoryContext(query string) string { - var longTerm string - - if ms.vectorStore != nil && ms.embedFn != nil { - // Auto-sync vector store if *any* file in the memory directory has changed since last sync - var latestModTime time.Time - _ = filepath.WalkDir(ms.memoryDir, func(path string, d fs.DirEntry, err error) error { - if err == nil { - if info, statErr := d.Info(); statErr == nil { - if info.ModTime().After(latestModTime) { - latestModTime = info.ModTime() - } - } - } - return nil - }) - - if latestModTime.After(ms.lastSyncTime) { - // Use context.Background() for the sync operation - ms.vectorStore.SyncFromDirectory(context.Background(), ms.memoryDir, ms.embedFn) - ms.lastSyncTime = latestModTime - } - - if query != "" { - // Semantic path: retrieve top-K relevant memories - vec, err := ms.embedFn(query) - if err == nil { - results, err := ms.vectorStore.Search(vec) - if err == nil && len(results) > 0 { - longTerm = strings.Join(results, "\n\n") - } - } - // On any error, fall through to full-text load - } - } - - if longTerm == "" { - // Full-text fallback (always used when vector store is disabled) - longTerm = ms.ReadLongTerm() - } + longTerm := ms.ReadLongTerm() recentNotes := ms.GetRecentDailyNotes(3) diff --git a/pkg/agent/memory_vector.go b/pkg/agent/memory_vector.go deleted file mode 100644 index b5073ac61..000000000 --- a/pkg/agent/memory_vector.go +++ /dev/null @@ -1,264 +0,0 @@ -// PicoClaw - Ultra-lightweight personal AI agent -// License: MIT -// Copyright (c) 2026 PicoClaw contributors - -package agent - -import ( - "context" - "crypto/md5" - "database/sql" - "encoding/binary" - "encoding/hex" - "fmt" - "io/fs" - "log" - "math" - "os" - "path/filepath" - "sort" - "strings" - - _ "modernc.org/sqlite" // pure-Go SQLite driver -) - -// VectorMemoryStore provides semantic memory search backed by SQLite. -// Embeddings are stored as raw float32 blobs; cosine similarity is computed in Go. -// -// The DB schema is intentionally minimal: -// -// CREATE TABLE memories ( -// id TEXT PRIMARY KEY, -- content hash or sequential key -// content TEXT NOT NULL, -- raw text of the memory entry -// vector BLOB NOT NULL -- float32 little-endian array -// ) -type VectorMemoryStore struct { - db *sql.DB - embeddingModel string - topK int -} - -// NewVectorMemoryStore opens (or creates) the SQLite DB at dbPath. -func NewVectorMemoryStore(dbPath string, embeddingModel string, topK int) (*VectorMemoryStore, error) { - if topK <= 0 { - topK = 5 - } - db, err := sql.Open("sqlite", dbPath) - if err != nil { - return nil, fmt.Errorf("vector memory: open db: %w", err) - } - if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS memories ( - id TEXT PRIMARY KEY, - content TEXT NOT NULL, - vector BLOB NOT NULL - )`); err != nil { - db.Close() - return nil, fmt.Errorf("vector memory: create table: %w", err) - } - return &VectorMemoryStore{ - db: db, - embeddingModel: embeddingModel, - topK: topK, - }, nil -} - -// Close closes the underlying database. -func (vs *VectorMemoryStore) Close() error { - return vs.db.Close() -} - -// Upsert stores a memory entry along with its embedding vector. -func (vs *VectorMemoryStore) Upsert(id, content string, vector []float32) error { - blob := float32SliceToBytes(vector) - _, err := vs.db.Exec( - `INSERT INTO memories (id, content, vector) VALUES (?, ?, ?) - ON CONFLICT(id) DO UPDATE SET content=excluded.content, vector=excluded.vector`, - id, content, blob, - ) - return err -} - -// Search returns the top-K memory entries most semantically similar to queryVec. -func (vs *VectorMemoryStore) Search(queryVec []float32) ([]string, error) { - rows, err := vs.db.Query(`SELECT content, vector FROM memories`) - if err != nil { - return nil, fmt.Errorf("vector memory: query: %w", err) - } - defer rows.Close() - - type scored struct { - content string - score float64 - } - var results []scored - - for rows.Next() { - var content string - var blob []byte - if err := rows.Scan(&content, &blob); err != nil { - continue - } - vec := bytesToFloat32Slice(blob) - if len(vec) == 0 { - continue - } - sim := cosineSimilarity(queryVec, vec) - results = append(results, scored{content: content, score: sim}) - } - - sort.Slice(results, func(i, j int) bool { - return results[i].score > results[j].score - }) - - topK := vs.topK - if topK > len(results) { - topK = len(results) - } - out := make([]string, topK) - for i := range topK { - out[i] = results[i].content - } - return out, nil -} - -// Count returns the number of stored memory entries. -func (vs *VectorMemoryStore) Count() int { - var n int - vs.db.QueryRow(`SELECT COUNT(*) FROM memories`).Scan(&n) //nolint:errcheck - return n -} - -// SyncFromDirectory parses all .md files in the given directory into individual entries -// and upserts any that are not already indexed, using the provided embedder to vectorize them. -// It also deletes entries from the database that are no longer present in any of the files. -func (vs *VectorMemoryStore) SyncFromDirectory(ctx context.Context, dirPath string, embedder func(string) ([]float32, error)) { - // Gather complete content from all .md files in the directory - var allEntries []string - err := filepath.WalkDir(dirPath, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if !d.IsDir() && strings.HasSuffix(strings.ToLower(d.Name()), ".md") { - if data, readErr := os.ReadFile(path); readErr == nil { - allEntries = append(allEntries, splitMemoryEntries(string(data))...) - } - } - return nil - }) - if err != nil && !os.IsNotExist(err) { - log.Printf("vector memory: sync directory walk error: %v", err) - } - - // Map to track current chunks by hash - currentHashes := make(map[string]string) - - for _, entry := range allEntries { - entry = strings.TrimSpace(entry) - if entry == "" { - continue - } - hash := md5.Sum([]byte(entry)) - id := hex.EncodeToString(hash[:]) - currentHashes[id] = entry - } - - // Fetch existing IDs to find what to add/delete - existingIDs := make(map[string]bool) - rows, err := vs.db.Query(`SELECT id FROM memories`) - if err == nil { - for rows.Next() { - var id string - if err := rows.Scan(&id); err == nil { - existingIDs[id] = true - } - } - rows.Close() - } - - // Insert new entries - for id, entry := range currentHashes { - if existingIDs[id] { - continue // Already indexed - } - vec, err := embedder(entry) - if err != nil { - log.Printf("vector memory: sync embed error for %s: %v", id[:8], err) - continue - } - if err := vs.Upsert(id, entry, vec); err != nil { - log.Printf("vector memory: sync upsert error for %s: %v", id[:8], err) - } - } - - // Delete stale entries - for id := range existingIDs { - if _, ok := currentHashes[id]; !ok { - vs.db.Exec(`DELETE FROM memories WHERE id=?`, id) //nolint:errcheck - } - } -} - -// splitMemoryEntries splits a MEMORY.md file into individual memory chunks. -// Splits on markdown headings (##) or blank-line separated paragraphs. -func splitMemoryEntries(content string) []string { - var entries []string - // Split by "##" headings first - if strings.Contains(content, "\n## ") || strings.HasPrefix(content, "## ") { - parts := strings.Split(content, "\n## ") - for i, p := range parts { - if i > 0 { - p = "## " + p - } - if strings.TrimSpace(p) != "" { - entries = append(entries, p) - } - } - return entries - } - // Fallback: split on blank lines - for _, block := range strings.Split(content, "\n\n") { - if strings.TrimSpace(block) != "" { - entries = append(entries, block) - } - } - return entries -} - -// --- Vector math helpers --- - -func cosineSimilarity(a, b []float32) float64 { - n := len(a) - if len(b) < n { - n = len(b) - } - var dot, normA, normB float64 - for i := range n { - ai := float64(a[i]) - bi := float64(b[i]) - dot += ai * bi - normA += ai * ai - normB += bi * bi - } - if normA == 0 || normB == 0 { - return 0 - } - return dot / (math.Sqrt(normA) * math.Sqrt(normB)) -} - -func float32SliceToBytes(v []float32) []byte { - buf := make([]byte, len(v)*4) - for i, f := range v { - binary.LittleEndian.PutUint32(buf[i*4:], math.Float32bits(f)) - } - return buf -} - -func bytesToFloat32Slice(b []byte) []float32 { - n := len(b) / 4 - out := make([]float32, n) - for i := range n { - bits := binary.LittleEndian.Uint32(b[i*4:]) - out[i] = math.Float32frombits(bits) - } - return out -} diff --git a/pkg/config/config.go b/pkg/config/config.go index 421cfa804..668bb9aa7 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -685,7 +685,6 @@ type ToolsConfig struct { Skills SkillsToolsConfig `json:"skills"` MediaCleanup MediaCleanupConfig `json:"media_cleanup"` MCP MCPConfig `json:"mcp"` - VectorMemory VectorMemoryConfig `json:"vector_memory"` AppendFile ToolConfig `json:"append_file" envPrefix:"PICOCLAW_TOOLS_APPEND_FILE_"` EditFile ToolConfig `json:"edit_file" envPrefix:"PICOCLAW_TOOLS_EDIT_FILE_"` FindSkills ToolConfig `json:"find_skills" envPrefix:"PICOCLAW_TOOLS_FIND_SKILLS_"` @@ -702,16 +701,6 @@ type ToolsConfig struct { WriteFile ToolConfig `json:"write_file" envPrefix:"PICOCLAW_TOOLS_WRITE_FILE_"` } -// VectorMemoryConfig configures the optional SQLite-backed semantic memory search. -// When enabled, agent memory retrieval uses embedding-based similarity instead of -// injecting the entire MEMORY.md into every prompt. -type VectorMemoryConfig struct { - Enabled bool `json:"enabled" env:"PICOCLAW_VECTOR_MEMORY_ENABLED"` - APIBase string `json:"api_base" env:"PICOCLAW_VECTOR_MEMORY_API_BASE"` - APIKey string `json:"api_key" env:"PICOCLAW_VECTOR_MEMORY_API_KEY"` - EmbeddingModel string `json:"embedding_model" env:"PICOCLAW_VECTOR_MEMORY_EMBEDDING_MODEL"` // e.g. "text-embedding-3-small" - TopK int `json:"top_k" env:"PICOCLAW_VECTOR_MEMORY_TOP_K"` // Number of memories to retrieve per query (default 5) -} type SearchCacheConfig struct { MaxSize int `json:"max_size" env:"PICOCLAW_SKILLS_SEARCH_CACHE_MAX_SIZE"`