diff --git a/pkg/migrate/xdg.go b/pkg/migrate/xdg.go new file mode 100644 index 000000000..b25edf3d8 --- /dev/null +++ b/pkg/migrate/xdg.go @@ -0,0 +1,166 @@ +package migrate + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/sipeed/picoclaw/pkg/config" +) + +// MigrateToXDG moves files from the legacy ~/.picoclaw/workspace/ layout to +// XDG-compliant directories. It is idempotent: files are only copied if they +// don't already exist at the destination, and it never deletes the source. +// +// Layout mapping: +// +// ~/.picoclaw/workspace/AGENT.md → $XDG_CONFIG_HOME/picoclaw/identity/AGENT.md +// ~/.picoclaw/workspace/IDENTITY.md → $XDG_CONFIG_HOME/picoclaw/identity/IDENTITY.md +// ~/.picoclaw/workspace/SOUL.md → $XDG_CONFIG_HOME/picoclaw/identity/SOUL.md +// ~/.picoclaw/workspace/USER.md → $XDG_CONFIG_HOME/picoclaw/identity/USER.md +// ~/.picoclaw/workspace/skills/* → $XDG_DATA_HOME/picoclaw/skills/* +// ~/.picoclaw/workspace/memory/*.db → $XDG_DATA_HOME/picoclaw/picoclaw.db +// ~/.picoclaw/workspace/* → $XDG_DATA_HOME/picoclaw/sandbox/* (remaining files) +func MigrateToXDG(legacyWorkspace string) error { + if legacyWorkspace == "" { + home, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("resolve home: %w", err) + } + legacyWorkspace = filepath.Join(home, ".picoclaw", "workspace") + } + + if _, err := os.Stat(legacyWorkspace); os.IsNotExist(err) { + return nil + } + + identityDir, err := config.IdentityDir() + if err != nil { + return fmt.Errorf("resolve identity dir: %w", err) + } + skillsDir, err := config.SkillsDir() + if err != nil { + return fmt.Errorf("resolve skills dir: %w", err) + } + dataDir, err := config.DataDir() + if err != nil { + return fmt.Errorf("resolve data dir: %w", err) + } + sandboxDir, err := config.SandboxDir() + if err != nil { + return fmt.Errorf("resolve sandbox dir: %w", err) + } + + identityFiles := map[string]bool{ + "AGENT.md": true, "IDENTITY.md": true, + "SOUL.md": true, "USER.md": true, + } + + for name := range identityFiles { + src := filepath.Join(legacyWorkspace, name) + dst := filepath.Join(identityDir, name) + if err := copyIfMissing(src, dst); err != nil { + return fmt.Errorf("migrate %s: %w", name, err) + } + } + + legacySkills := filepath.Join(legacyWorkspace, "skills") + if info, err := os.Stat(legacySkills); err == nil && info.IsDir() { + if err := copyDirIfMissing(legacySkills, skillsDir); err != nil { + return fmt.Errorf("migrate skills: %w", err) + } + } + + legacyDB := filepath.Join(legacyWorkspace, "memory", "picoclaw.db") + if _, err := os.Stat(legacyDB); err == nil { + newDB := filepath.Join(dataDir, "picoclaw.db") + if err := copyIfMissing(legacyDB, newDB); err != nil { + return fmt.Errorf("migrate database: %w", err) + } + } + + skipPrefixes := []string{"AGENT.md", "IDENTITY.md", "SOUL.md", "USER.md"} + + entries, err := os.ReadDir(legacyWorkspace) + if err != nil { + return nil + } + for _, e := range entries { + name := e.Name() + + if identityFiles[name] { + continue + } + if name == "skills" || name == "memory" { + continue + } + + skip := false + for _, p := range skipPrefixes { + if strings.EqualFold(name, p) { + skip = true + break + } + } + if skip { + continue + } + + src := filepath.Join(legacyWorkspace, name) + dst := filepath.Join(sandboxDir, name) + if e.IsDir() { + if err := copyDirIfMissing(src, dst); err != nil { + return fmt.Errorf("migrate %s: %w", name, err) + } + } else { + if err := copyIfMissing(src, dst); err != nil { + return fmt.Errorf("migrate %s: %w", name, err) + } + } + } + + return nil +} + +func copyIfMissing(src, dst string) error { + if _, err := os.Stat(src); os.IsNotExist(err) { + return nil + } + if _, err := os.Stat(dst); err == nil { + return nil + } + + os.MkdirAll(filepath.Dir(dst), 0o700) + + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, in) + return err +} + +func copyDirIfMissing(src, dst string) error { + return filepath.WalkDir(src, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + rel, _ := filepath.Rel(src, path) + target := filepath.Join(dst, rel) + + if d.IsDir() { + return os.MkdirAll(target, 0o700) + } + return copyIfMissing(path, target) + }) +} diff --git a/pkg/sync/identity.go b/pkg/sync/identity.go new file mode 100644 index 000000000..e4d711152 --- /dev/null +++ b/pkg/sync/identity.go @@ -0,0 +1,247 @@ +package sync + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/sipeed/picoclaw/pkg/ids" + "github.com/sipeed/picoclaw/pkg/memory" +) + +// IdentityFiles are the canonical set of bootstrap identity documents. +var IdentityFiles = []string{ + "AGENT.md", + "IDENTITY.md", + "SOUL.md", + "USER.md", +} + +const ( + kvPrefix = "sync:hash:" + kvLastSyncKey = "sync:last_sync_ts" + syncCategory = "bootstrap" + debounceDelay = 250 * time.Millisecond + pollInterval = 30 * time.Second +) + +// DocumentStore is the minimal interface the sync engine needs from the memory +// delegate. This avoids importing the full MemoryDelegate interface. +type DocumentStore interface { + GetKV(ctx context.Context, agentID, key string) (string, error) + UpsertKV(ctx context.Context, agentID, key, value string) error + UpsertDocument(ctx context.Context, doc *memory.AgentDocument) error + ListDocumentsByCategory(ctx context.Context, agentID, category string) ([]*memory.AgentDocument, error) +} + +// IdentitySync watches identity files on disk and mirrors their content into +// the database. Disk is the source of truth; the DB is a derived runtime cache. +type IdentitySync struct { + identityDir string + agentID string + store DocumentStore + + watcher *fsnotify.Watcher + cancel context.CancelFunc + wg sync.WaitGroup + lastSync atomic.Int64 // unix nanos of the last successful sync +} + +// New creates a new IdentitySync. identityDir is the path to the directory +// containing the identity markdown files ($XDG_CONFIG_HOME/picoclaw/identity). +func New(identityDir, agentID string, store DocumentStore) *IdentitySync { + s := &IdentitySync{ + identityDir: identityDir, + agentID: agentID, + store: store, + } + s.lastSync.Store(time.Now().UnixNano()) + return s +} + +// SyncAll performs a full reconciliation: reads every identity file from disk, +// compares its content hash against the stored hash in agent_kv, and upserts +// any that differ. This is the startup path and catches all changes made while +// picoclaw was not running. +func (s *IdentitySync) SyncAll(ctx context.Context) error { + for _, name := range IdentityFiles { + if err := s.syncFile(ctx, name); err != nil { + return fmt.Errorf("sync %s: %w", name, err) + } + } + now := time.Now() + s.lastSync.Store(now.UnixNano()) + _ = s.store.UpsertKV(ctx, s.agentID, kvLastSyncKey, now.Format(time.RFC3339Nano)) + return nil +} + +// Watch starts an fsnotify watcher on the identity directory. File change +// events are debounced and trigger a re-sync of the affected file. Call Close +// to stop watching. This is the daemon-mode path for real-time sync. +func (s *IdentitySync) Watch(ctx context.Context) error { + w, err := fsnotify.NewWatcher() + if err != nil { + return fmt.Errorf("create fsnotify watcher: %w", err) + } + if err := w.Add(s.identityDir); err != nil { + w.Close() + return fmt.Errorf("watch %s: %w", s.identityDir, err) + } + + ctx, cancel := context.WithCancel(ctx) + s.watcher = w + s.cancel = cancel + + s.wg.Add(1) + go s.watchLoop(ctx, w) + return nil +} + +// CheckAndSync is a lightweight pre-prompt safety net. It stats every identity +// file and re-syncs any whose mtime is newer than the last sync timestamp. +// Cost: O(n) stat calls where n = len(IdentityFiles). +func (s *IdentitySync) CheckAndSync(ctx context.Context) error { + threshold := time.Unix(0, s.lastSync.Load()) + changed := false + for _, name := range IdentityFiles { + path := filepath.Join(s.identityDir, name) + info, err := os.Stat(path) + if err != nil { + continue + } + if info.ModTime().After(threshold) { + if err := s.syncFile(ctx, name); err != nil { + return err + } + changed = true + } + } + if changed { + now := time.Now() + s.lastSync.Store(now.UnixNano()) + _ = s.store.UpsertKV(ctx, s.agentID, kvLastSyncKey, now.Format(time.RFC3339Nano)) + } + return nil +} + +// Close stops the fsnotify watcher and waits for the watch goroutine to exit. +func (s *IdentitySync) Close() { + if s.cancel != nil { + s.cancel() + } + if s.watcher != nil { + s.watcher.Close() + } + s.wg.Wait() +} + +// syncFile reads a single identity file from disk, computes its SHA-256 hash, +// compares with the stored hash, and upserts to agent_documents if different. +func (s *IdentitySync) syncFile(ctx context.Context, name string) error { + path := filepath.Join(s.identityDir, name) + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + content := strings.TrimSpace(string(data)) + if content == "" { + return nil + } + + hash := contentHash(data) + kvKey := kvPrefix + name + + stored, err := s.store.GetKV(ctx, s.agentID, kvKey) + if err == nil && stored == hash { + return nil + } + + doc := &memory.AgentDocument{ + ID: ids.New(), + AgentID: s.agentID, + Name: name, + Category: syncCategory, + Content: content, + } + if err := s.store.UpsertDocument(ctx, doc); err != nil { + return fmt.Errorf("upsert document %s: %w", name, err) + } + + if err := s.store.UpsertKV(ctx, s.agentID, kvKey, hash); err != nil { + return fmt.Errorf("upsert hash for %s: %w", name, err) + } + + return nil +} + +// watchLoop processes fsnotify events with debouncing. +func (s *IdentitySync) watchLoop(ctx context.Context, w *fsnotify.Watcher) { + defer s.wg.Done() + + pending := make(map[string]time.Time) + ticker := time.NewTicker(debounceDelay) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + + case event, ok := <-w.Events: + if !ok { + return + } + if event.Op&(fsnotify.Write|fsnotify.Create|fsnotify.Rename) == 0 { + continue + } + name := filepath.Base(event.Name) + if !isIdentityFile(name) { + continue + } + pending[name] = time.Now() + + case _, ok := <-w.Errors: + if !ok { + return + } + + case now := <-ticker.C: + for name, queued := range pending { + if now.Sub(queued) < debounceDelay { + continue + } + syncCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + _ = s.syncFile(syncCtx, name) + cancel() + delete(pending, name) + s.lastSync.Store(time.Now().UnixNano()) + } + } + } +} + +func isIdentityFile(name string) bool { + for _, f := range IdentityFiles { + if f == name { + return true + } + } + return false +} + +func contentHash(data []byte) string { + h := sha256.Sum256(data) + return hex.EncodeToString(h[:]) +} diff --git a/pkg/sync/identity_test.go b/pkg/sync/identity_test.go new file mode 100644 index 000000000..790a4f84f --- /dev/null +++ b/pkg/sync/identity_test.go @@ -0,0 +1,314 @@ +package sync + +import ( + "context" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/sipeed/picoclaw/pkg/memory" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mockStore is a minimal in-memory implementation of DocumentStore for tests. +type mockStore struct { + mu sync.Mutex + kv map[string]string + docs map[string]*memory.AgentDocument // keyed by agentID+":"+name +} + +func newMockStore() *mockStore { + return &mockStore{ + kv: make(map[string]string), + docs: make(map[string]*memory.AgentDocument), + } +} + +func (m *mockStore) GetKV(_ context.Context, agentID, key string) (string, error) { + m.mu.Lock() + defer m.mu.Unlock() + v, ok := m.kv[agentID+":"+key] + if !ok { + return "", os.ErrNotExist + } + return v, nil +} + +func (m *mockStore) UpsertKV(_ context.Context, agentID, key, value string) error { + m.mu.Lock() + defer m.mu.Unlock() + m.kv[agentID+":"+key] = value + return nil +} + +func (m *mockStore) UpsertDocument(_ context.Context, doc *memory.AgentDocument) error { + m.mu.Lock() + defer m.mu.Unlock() + m.docs[doc.AgentID+":"+doc.Name] = doc + return nil +} + +func (m *mockStore) ListDocumentsByCategory(_ context.Context, agentID, category string) ([]*memory.AgentDocument, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []*memory.AgentDocument + for _, doc := range m.docs { + if doc.AgentID == agentID && doc.Category == category { + out = append(out, doc) + } + } + return out, nil +} + +func (m *mockStore) getDoc(agentID, name string) *memory.AgentDocument { + m.mu.Lock() + defer m.mu.Unlock() + return m.docs[agentID+":"+name] +} + +func (m *mockStore) getHash(agentID, name string) string { + m.mu.Lock() + defer m.mu.Unlock() + return m.kv[agentID+":"+kvPrefix+name] +} + +func setupIdentityDir(t *testing.T, files map[string]string) string { + t.Helper() + dir := t.TempDir() + for name, content := range files { + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0644)) + } + return dir +} + +func TestSyncAll_InsertsNewFiles(t *testing.T) { + dir := setupIdentityDir(t, map[string]string{ + "AGENT.md": "# Agent\nYou are helpful.", + "SOUL.md": "# Soul\nCurious and kind.", + "USER.md": "# User\nName: Alice", + "IDENTITY.md": "# Identity\nPicoClaw v1", + }) + + store := newMockStore() + s := New(dir, "agent-1", store) + + require.NoError(t, s.SyncAll(context.Background())) + + for _, name := range IdentityFiles { + doc := store.getDoc("agent-1", name) + require.NotNil(t, doc, "expected document for %s", name) + assert.Equal(t, syncCategory, doc.Category) + assert.NotEmpty(t, doc.Content) + + hash := store.getHash("agent-1", name) + assert.NotEmpty(t, hash, "expected hash for %s", name) + } +} + +func TestSyncAll_SkipsUnchangedFiles(t *testing.T) { + dir := setupIdentityDir(t, map[string]string{ + "AGENT.md": "# Agent\nSame content.", + }) + + store := newMockStore() + s := New(dir, "agent-1", store) + + require.NoError(t, s.SyncAll(context.Background())) + firstDoc := store.getDoc("agent-1", "AGENT.md") + require.NotNil(t, firstDoc) + firstID := firstDoc.ID + + require.NoError(t, s.SyncAll(context.Background())) + secondDoc := store.getDoc("agent-1", "AGENT.md") + assert.Equal(t, firstID, secondDoc.ID, "unchanged file should not be re-upserted") +} + +func TestSyncAll_UpsertsModifiedFiles(t *testing.T) { + dir := setupIdentityDir(t, map[string]string{ + "AGENT.md": "# Agent\nVersion 1", + }) + + store := newMockStore() + s := New(dir, "agent-1", store) + require.NoError(t, s.SyncAll(context.Background())) + + v1Hash := store.getHash("agent-1", "AGENT.md") + + require.NoError(t, os.WriteFile(filepath.Join(dir, "AGENT.md"), []byte("# Agent\nVersion 2"), 0644)) + require.NoError(t, s.SyncAll(context.Background())) + + v2Hash := store.getHash("agent-1", "AGENT.md") + assert.NotEqual(t, v1Hash, v2Hash, "hash should change after file modification") + + doc := store.getDoc("agent-1", "AGENT.md") + assert.Contains(t, doc.Content, "Version 2") +} + +func TestSyncAll_SkipsMissingFiles(t *testing.T) { + dir := setupIdentityDir(t, map[string]string{ + "AGENT.md": "# Agent only", + }) + + store := newMockStore() + s := New(dir, "agent-1", store) + + require.NoError(t, s.SyncAll(context.Background())) + + assert.NotNil(t, store.getDoc("agent-1", "AGENT.md")) + assert.Nil(t, store.getDoc("agent-1", "SOUL.md"), "missing file should not create a doc") + assert.Nil(t, store.getDoc("agent-1", "USER.md")) + assert.Nil(t, store.getDoc("agent-1", "IDENTITY.md")) +} + +func TestSyncAll_SkipsEmptyFiles(t *testing.T) { + dir := setupIdentityDir(t, map[string]string{ + "AGENT.md": " \n\t\n ", + }) + + store := newMockStore() + s := New(dir, "agent-1", store) + + require.NoError(t, s.SyncAll(context.Background())) + assert.Nil(t, store.getDoc("agent-1", "AGENT.md"), "empty/whitespace-only file should be skipped") +} + +func TestSyncAll_IsolatesAgents(t *testing.T) { + dir := setupIdentityDir(t, map[string]string{ + "AGENT.md": "# Shared agent file", + }) + + store := newMockStore() + s1 := New(dir, "agent-a", store) + s2 := New(dir, "agent-b", store) + + require.NoError(t, s1.SyncAll(context.Background())) + require.NoError(t, s2.SyncAll(context.Background())) + + docA := store.getDoc("agent-a", "AGENT.md") + docB := store.getDoc("agent-b", "AGENT.md") + require.NotNil(t, docA) + require.NotNil(t, docB) + assert.NotEqual(t, docA.ID, docB.ID, "different agents should get separate doc records") +} + +func TestCheckAndSync_DetectsModifiedFile(t *testing.T) { + dir := setupIdentityDir(t, map[string]string{ + "AGENT.md": "# Original", + }) + + store := newMockStore() + s := New(dir, "agent-1", store) + require.NoError(t, s.SyncAll(context.Background())) + + time.Sleep(50 * time.Millisecond) + require.NoError(t, os.WriteFile(filepath.Join(dir, "AGENT.md"), []byte("# Modified"), 0644)) + + require.NoError(t, s.CheckAndSync(context.Background())) + + doc := store.getDoc("agent-1", "AGENT.md") + assert.Contains(t, doc.Content, "Modified") +} + +func TestCheckAndSync_SkipsUntouchedFiles(t *testing.T) { + dir := setupIdentityDir(t, map[string]string{ + "AGENT.md": "# Stable", + }) + + store := newMockStore() + s := New(dir, "agent-1", store) + require.NoError(t, s.SyncAll(context.Background())) + + hash1 := store.getHash("agent-1", "AGENT.md") + + s.lastSync.Store(time.Now().Add(time.Second).UnixNano()) + require.NoError(t, s.CheckAndSync(context.Background())) + + hash2 := store.getHash("agent-1", "AGENT.md") + assert.Equal(t, hash1, hash2, "untouched file should not trigger re-sync") +} + +func TestWatch_DetectsFileChange(t *testing.T) { + dir := setupIdentityDir(t, map[string]string{ + "AGENT.md": "# Initial", + }) + + store := newMockStore() + s := New(dir, "agent-1", store) + require.NoError(t, s.SyncAll(context.Background())) + + ctx := context.Background() + require.NoError(t, s.Watch(ctx)) + defer s.Close() + + require.NoError(t, os.WriteFile(filepath.Join(dir, "AGENT.md"), []byte("# Updated via watch"), 0644)) + + assert.Eventually(t, func() bool { + doc := store.getDoc("agent-1", "AGENT.md") + return doc != nil && doc.Content == "# Updated via watch" + }, 3*time.Second, 100*time.Millisecond, "watcher should detect and sync the change") +} + +func TestWatch_IgnoresNonIdentityFiles(t *testing.T) { + dir := setupIdentityDir(t, map[string]string{ + "AGENT.md": "# Agent", + }) + + store := newMockStore() + s := New(dir, "agent-1", store) + require.NoError(t, s.SyncAll(context.Background())) + + ctx := context.Background() + require.NoError(t, s.Watch(ctx)) + defer s.Close() + + require.NoError(t, os.WriteFile(filepath.Join(dir, "random.txt"), []byte("not an identity file"), 0644)) + time.Sleep(500 * time.Millisecond) + + assert.Nil(t, store.getDoc("agent-1", "random.txt"), "non-identity file should be ignored") +} + +func TestContentHash_Deterministic(t *testing.T) { + data := []byte("hello world") + h1 := contentHash(data) + h2 := contentHash(data) + assert.Equal(t, h1, h2) + assert.Len(t, h1, 64) +} + +func TestContentHash_DifferentForDifferentContent(t *testing.T) { + h1 := contentHash([]byte("version 1")) + h2 := contentHash([]byte("version 2")) + assert.NotEqual(t, h1, h2) +} + +func TestIsIdentityFile(t *testing.T) { + tests := []struct { + name string + want bool + }{ + {"AGENT.md", true}, + {"IDENTITY.md", true}, + {"SOUL.md", true}, + {"USER.md", true}, + {"README.md", false}, + {"agent.md", false}, + {"AGENT.txt", false}, + {"", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, isIdentityFile(tt.name)) + }) + } +} + +func TestNew_SetsFields(t *testing.T) { + store := newMockStore() + s := New("/tmp/identity", "test-agent", store) + assert.Equal(t, "/tmp/identity", s.identityDir) + assert.Equal(t, "test-agent", s.agentID) + assert.NotNil(t, s.store) +}