From 8b094b205798bc3d3beb86e9423aea8034e9c476 Mon Sep 17 00:00:00 2001 From: ZanzyTHEbar Date: Wed, 18 Feb 2026 15:53:37 +0000 Subject: [PATCH] feat(memory): add data migration utilities Add migration helpers that import existing file-based data into the structured memory system: daily notes, documents, long-term memory entries, and agent state. Updates session migration tests to cover the expanded migration surface. --- pkg/memory/migrate_dailynotes.go | 93 +++++++++++++++++++++++++++++ pkg/memory/migrate_documents.go | 62 +++++++++++++++++++ pkg/memory/migrate_longterm.go | 50 ++++++++++++++++ pkg/memory/migrate_sessions_test.go | 31 +++++++++- pkg/memory/migrate_state.go | 71 ++++++++++++++++++++++ 5 files changed, 304 insertions(+), 3 deletions(-) create mode 100644 pkg/memory/migrate_dailynotes.go create mode 100644 pkg/memory/migrate_documents.go create mode 100644 pkg/memory/migrate_longterm.go create mode 100644 pkg/memory/migrate_state.go diff --git a/pkg/memory/migrate_dailynotes.go b/pkg/memory/migrate_dailynotes.go new file mode 100644 index 000000000..782605485 --- /dev/null +++ b/pkg/memory/migrate_dailynotes.go @@ -0,0 +1,93 @@ +package memory + +import ( + "context" + "log" + "os" + "path/filepath" + "strings" + "time" + + "github.com/sipeed/picoclaw/pkg/ids" +) + +// MigrateDailyNotes performs a one-time migration of daily note files +// (memory/YYYYMM/YYYYMMDD.md) into recall items. Uses a marker file for idempotency. +func MigrateDailyNotes(ctx context.Context, workspace string, delegate MemoryDelegate, agentID string) error { + if delegate == nil { + return nil + } + + memDir := filepath.Join(workspace, "memory") + markerFile := filepath.Join(memDir, ".dailynotes_migrated") + if _, err := os.Stat(markerFile); err == nil { + return nil + } + + migrated := 0 + err := filepath.WalkDir(memDir, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + if filepath.Ext(path) != ".md" { + return nil + } + + base := strings.TrimSuffix(d.Name(), ".md") + if len(base) != 8 { + return nil + } + + noteDate, parseErr := time.Parse("20060102", base) + if parseErr != nil { + return nil + } + + data, readErr := os.ReadFile(path) + if readErr != nil { + return nil + } + + content := strings.TrimSpace(string(data)) + if content == "" { + return nil + } + + dateTag := noteDate.Format("2006-01-02") + item := &RecallItem{ + ID: ids.New(), + AgentID: agentID, + SessionKey: "", + Role: "system", + Sector: SectorEpisodic, + Importance: 0.4, + Salience: 0.4, + DecayRate: 0.01, + Content: content, + Tags: "daily-note," + dateTag, + CreatedAt: noteDate, + UpdatedAt: noteDate, + } + + if err := delegate.InsertRecallItem(ctx, item); err != nil { + log.Printf("[WARN] migrate_dailynotes: failed to insert note %s: %v", base, err) + return nil + } + migrated++ + return nil + }) + + if err != nil { + return err + } + + if migrated > 0 { + marker := []byte(time.Now().Format(time.RFC3339)) + if err := os.WriteFile(markerFile, marker, 0644); err != nil { + log.Printf("[WARN] migrate_dailynotes: failed to write marker: %v", err) + } + log.Printf("[INFO] migrate_dailynotes: migrated %d daily notes to recall items", migrated) + } + + return nil +} diff --git a/pkg/memory/migrate_documents.go b/pkg/memory/migrate_documents.go new file mode 100644 index 000000000..0d74fe304 --- /dev/null +++ b/pkg/memory/migrate_documents.go @@ -0,0 +1,62 @@ +package memory + +import ( + "context" + "log" + "os" + "path/filepath" + "time" + + "github.com/sipeed/picoclaw/pkg/ids" +) + +var bootstrapDocNames = []string{ + "AGENTS.md", + "SOUL.md", + "USER.md", + "IDENTITY.md", +} + +// MigrateDocuments performs a one-time migration of workspace bootstrap files +// into the agent_documents table. Uses a marker file for idempotency. +func MigrateDocuments(ctx context.Context, workspace string, delegate MemoryDelegate, agentID string) error { + if delegate == nil { + return nil + } + + markerFile := filepath.Join(workspace, ".documents_migrated") + if _, err := os.Stat(markerFile); err == nil { + return nil + } + + migrated := 0 + for _, name := range bootstrapDocNames { + filePath := filepath.Join(workspace, name) + data, err := os.ReadFile(filePath) + if err != nil { + continue + } + + doc := &AgentDocument{ + ID: ids.New(), + AgentID: agentID, + Name: name, + Category: "bootstrap", + Content: string(data), + } + if err := delegate.UpsertDocument(ctx, doc); err != nil { + return err + } + migrated++ + } + + if migrated > 0 { + marker := []byte(time.Now().Format(time.RFC3339)) + if err := os.WriteFile(markerFile, marker, 0644); err != nil { + log.Printf("[WARN] migrate_documents: failed to write marker: %v", err) + } + log.Printf("[INFO] migrate_documents: migrated %d bootstrap files to agent_documents", migrated) + } + + return nil +} diff --git a/pkg/memory/migrate_longterm.go b/pkg/memory/migrate_longterm.go new file mode 100644 index 000000000..a140c1aa1 --- /dev/null +++ b/pkg/memory/migrate_longterm.go @@ -0,0 +1,50 @@ +package memory + +import ( + "context" + "log" + "os" + "path/filepath" + "strings" + "time" +) + +// MigrateLongTermMemory performs a one-time migration of MEMORY.md content +// into the working context tier. Uses a marker file for idempotency. +func MigrateLongTermMemory(ctx context.Context, workspace string, delegate MemoryDelegate, agentID string) error { + if delegate == nil { + return nil + } + + memDir := filepath.Join(workspace, "memory") + markerFile := filepath.Join(memDir, ".longterm_migrated") + if _, err := os.Stat(markerFile); err == nil { + return nil + } + + memFile := filepath.Join(memDir, "MEMORY.md") + data, err := os.ReadFile(memFile) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + content := strings.TrimSpace(string(data)) + if content == "" { + return nil + } + + if err := delegate.UpsertWorkingContext(ctx, agentID, "default", content); err != nil { + return err + } + + marker := []byte(time.Now().Format(time.RFC3339)) + if err := os.WriteFile(markerFile, marker, 0644); err != nil { + log.Printf("[WARN] migrate_longterm: failed to write marker: %v", err) + } + + log.Printf("[INFO] migrate_longterm: migrated MEMORY.md to working context") + return nil +} diff --git a/pkg/memory/migrate_sessions_test.go b/pkg/memory/migrate_sessions_test.go index 18282fd75..12f667fc5 100644 --- a/pkg/memory/migrate_sessions_test.go +++ b/pkg/memory/migrate_sessions_test.go @@ -72,9 +72,34 @@ func (m *mockDelegate) ListSummaries(_ context.Context, _, _ string, _ int) ([]* func (m *mockDelegate) CountRecallItems(_ context.Context, _, _ string) (int, error) { return len(m.recallItems), nil } -func (m *mockDelegate) CountArchivalChunks(_ context.Context) (int, error) { return 0, nil } -func (m *mockDelegate) HasVectorSearch() bool { return false } -func (m *mockDelegate) HasFTS() bool { return false } +func (m *mockDelegate) CountArchivalChunks(_ context.Context) (int, error) { return 0, nil } +func (m *mockDelegate) HasVectorSearch() bool { return false } +func (m *mockDelegate) HasFTS() bool { return false } +func (m *mockDelegate) GetKV(_ context.Context, _, _ string) (string, error) { return "", nil } +func (m *mockDelegate) UpsertKV(_ context.Context, _, _, _ string) error { return nil } +func (m *mockDelegate) DeleteKV(_ context.Context, _, _ string) error { return nil } +func (m *mockDelegate) ListKVByPrefix(_ context.Context, _, _ string, _ int) (map[string]string, error) { + return nil, nil +} +func (m *mockDelegate) GetDocument(_ context.Context, _, _ string) (*AgentDocument, error) { + return nil, nil +} +func (m *mockDelegate) UpsertDocument(_ context.Context, _ *AgentDocument) error { return nil } +func (m *mockDelegate) DeleteDocument(_ context.Context, _, _ string) error { return nil } +func (m *mockDelegate) ListDocumentsByCategory(_ context.Context, _, _ string) ([]*AgentDocument, error) { + return nil, nil +} +func (m *mockDelegate) ListAllDocuments(_ context.Context, _ string) ([]*AgentDocument, error) { + return nil, nil +} +func (m *mockDelegate) InsertAuditEntry(_ context.Context, _ *AuditEntry) error { return nil } +func (m *mockDelegate) ListAuditEntries(_ context.Context, _ string, _ int) ([]*AuditEntry, error) { + return nil, nil +} +func (m *mockDelegate) ListAuditEntriesByAction(_ context.Context, _, _ string, _ int) ([]*AuditEntry, error) { + return nil, nil +} +func (m *mockDelegate) CountAuditEntries(_ context.Context, _ string) (int, error) { return 0, nil } func writeSessionFile(t *testing.T, dir, name string, sess SessionFile) { t.Helper() diff --git a/pkg/memory/migrate_state.go b/pkg/memory/migrate_state.go new file mode 100644 index 000000000..4882d4fe3 --- /dev/null +++ b/pkg/memory/migrate_state.go @@ -0,0 +1,71 @@ +package memory + +import ( + "context" + "encoding/json" + "log" + "os" + "path/filepath" + "time" +) + +type legacyState struct { + LastChannel string `json:"last_channel,omitempty"` + LastChatID string `json:"last_chat_id,omitempty"` + Timestamp time.Time `json:"timestamp"` +} + +// MigrateState performs a one-time migration of workspace/state/state.json +// into agent_kv rows. Uses a marker file to ensure idempotency. +func MigrateState(ctx context.Context, workspace string, delegate MemoryDelegate, agentID string) error { + if delegate == nil { + return nil + } + + stateDir := filepath.Join(workspace, "state") + markerFile := filepath.Join(stateDir, ".state_kv_migrated") + + if _, err := os.Stat(markerFile); err == nil { + return nil + } + + stateFile := filepath.Join(stateDir, "state.json") + data, err := os.ReadFile(stateFile) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + var s legacyState + if err := json.Unmarshal(data, &s); err != nil { + log.Printf("[WARN] migrate_state: failed to parse %s: %v", stateFile, err) + return nil + } + + if s.LastChannel != "" { + if err := delegate.UpsertKV(ctx, agentID, "state:last_channel", s.LastChannel); err != nil { + return err + } + } + if s.LastChatID != "" { + if err := delegate.UpsertKV(ctx, agentID, "state:last_chat_id", s.LastChatID); err != nil { + return err + } + } + if !s.Timestamp.IsZero() { + if err := delegate.UpsertKV(ctx, agentID, "state:timestamp", s.Timestamp.Format(time.RFC3339Nano)); err != nil { + return err + } + } + + os.MkdirAll(stateDir, 0755) + marker := []byte(time.Now().Format(time.RFC3339)) + if err := os.WriteFile(markerFile, marker, 0644); err != nil { + log.Printf("[WARN] migrate_state: failed to write marker: %v", err) + } + + log.Printf("[INFO] migrate_state: migrated state.json to agent_kv") + return nil +}