feat(sync): add IdentitySync file watcher and XDG migration helper

pkg/sync/IdentitySync: watches identity markdown files (IDENTITY.md,
AGENT.md, SOUL.md, USER.md) in the XDG config directory using fsnotify
and upserts their content into the memory DB on change. Enables live
identity editing without restarting the agent.

pkg/migrate/xdg.go: MigrateToXDG() detects legacy ~/.picoclaw workspace
layouts and migrates them to XDG-compliant paths
($XDG_DATA_HOME/picoclaw, $XDG_CONFIG_HOME/picoclaw). Safe to call
multiple times (idempotent).
This commit is contained in:
ZanzyTHEbar 2026-02-19 18:04:07 +00:00
parent 3778d4f025
commit e8d6e01cea
3 changed files with 727 additions and 0 deletions

166
pkg/migrate/xdg.go Normal file
View file

@ -0,0 +1,166 @@
package migrate
import (
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/sipeed/picoclaw/pkg/config"
)
// MigrateToXDG moves files from the legacy ~/.picoclaw/workspace/ layout to
// XDG-compliant directories. It is idempotent: files are only copied if they
// don't already exist at the destination, and it never deletes the source.
//
// Layout mapping:
//
// ~/.picoclaw/workspace/AGENT.md → $XDG_CONFIG_HOME/picoclaw/identity/AGENT.md
// ~/.picoclaw/workspace/IDENTITY.md → $XDG_CONFIG_HOME/picoclaw/identity/IDENTITY.md
// ~/.picoclaw/workspace/SOUL.md → $XDG_CONFIG_HOME/picoclaw/identity/SOUL.md
// ~/.picoclaw/workspace/USER.md → $XDG_CONFIG_HOME/picoclaw/identity/USER.md
// ~/.picoclaw/workspace/skills/* → $XDG_DATA_HOME/picoclaw/skills/*
// ~/.picoclaw/workspace/memory/*.db → $XDG_DATA_HOME/picoclaw/picoclaw.db
// ~/.picoclaw/workspace/* → $XDG_DATA_HOME/picoclaw/sandbox/* (remaining files)
func MigrateToXDG(legacyWorkspace string) error {
if legacyWorkspace == "" {
home, err := os.UserHomeDir()
if err != nil {
return fmt.Errorf("resolve home: %w", err)
}
legacyWorkspace = filepath.Join(home, ".picoclaw", "workspace")
}
if _, err := os.Stat(legacyWorkspace); os.IsNotExist(err) {
return nil
}
identityDir, err := config.IdentityDir()
if err != nil {
return fmt.Errorf("resolve identity dir: %w", err)
}
skillsDir, err := config.SkillsDir()
if err != nil {
return fmt.Errorf("resolve skills dir: %w", err)
}
dataDir, err := config.DataDir()
if err != nil {
return fmt.Errorf("resolve data dir: %w", err)
}
sandboxDir, err := config.SandboxDir()
if err != nil {
return fmt.Errorf("resolve sandbox dir: %w", err)
}
identityFiles := map[string]bool{
"AGENT.md": true, "IDENTITY.md": true,
"SOUL.md": true, "USER.md": true,
}
for name := range identityFiles {
src := filepath.Join(legacyWorkspace, name)
dst := filepath.Join(identityDir, name)
if err := copyIfMissing(src, dst); err != nil {
return fmt.Errorf("migrate %s: %w", name, err)
}
}
legacySkills := filepath.Join(legacyWorkspace, "skills")
if info, err := os.Stat(legacySkills); err == nil && info.IsDir() {
if err := copyDirIfMissing(legacySkills, skillsDir); err != nil {
return fmt.Errorf("migrate skills: %w", err)
}
}
legacyDB := filepath.Join(legacyWorkspace, "memory", "picoclaw.db")
if _, err := os.Stat(legacyDB); err == nil {
newDB := filepath.Join(dataDir, "picoclaw.db")
if err := copyIfMissing(legacyDB, newDB); err != nil {
return fmt.Errorf("migrate database: %w", err)
}
}
skipPrefixes := []string{"AGENT.md", "IDENTITY.md", "SOUL.md", "USER.md"}
entries, err := os.ReadDir(legacyWorkspace)
if err != nil {
return nil
}
for _, e := range entries {
name := e.Name()
if identityFiles[name] {
continue
}
if name == "skills" || name == "memory" {
continue
}
skip := false
for _, p := range skipPrefixes {
if strings.EqualFold(name, p) {
skip = true
break
}
}
if skip {
continue
}
src := filepath.Join(legacyWorkspace, name)
dst := filepath.Join(sandboxDir, name)
if e.IsDir() {
if err := copyDirIfMissing(src, dst); err != nil {
return fmt.Errorf("migrate %s: %w", name, err)
}
} else {
if err := copyIfMissing(src, dst); err != nil {
return fmt.Errorf("migrate %s: %w", name, err)
}
}
}
return nil
}
func copyIfMissing(src, dst string) error {
if _, err := os.Stat(src); os.IsNotExist(err) {
return nil
}
if _, err := os.Stat(dst); err == nil {
return nil
}
os.MkdirAll(filepath.Dir(dst), 0o700)
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0o600)
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, in)
return err
}
func copyDirIfMissing(src, dst string) error {
return filepath.WalkDir(src, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
rel, _ := filepath.Rel(src, path)
target := filepath.Join(dst, rel)
if d.IsDir() {
return os.MkdirAll(target, 0o700)
}
return copyIfMissing(path, target)
})
}

247
pkg/sync/identity.go Normal file
View file

@ -0,0 +1,247 @@
package sync
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/fsnotify/fsnotify"
"github.com/sipeed/picoclaw/pkg/ids"
"github.com/sipeed/picoclaw/pkg/memory"
)
// IdentityFiles are the canonical set of bootstrap identity documents.
var IdentityFiles = []string{
"AGENT.md",
"IDENTITY.md",
"SOUL.md",
"USER.md",
}
const (
kvPrefix = "sync:hash:"
kvLastSyncKey = "sync:last_sync_ts"
syncCategory = "bootstrap"
debounceDelay = 250 * time.Millisecond
pollInterval = 30 * time.Second
)
// DocumentStore is the minimal interface the sync engine needs from the memory
// delegate. This avoids importing the full MemoryDelegate interface.
type DocumentStore interface {
GetKV(ctx context.Context, agentID, key string) (string, error)
UpsertKV(ctx context.Context, agentID, key, value string) error
UpsertDocument(ctx context.Context, doc *memory.AgentDocument) error
ListDocumentsByCategory(ctx context.Context, agentID, category string) ([]*memory.AgentDocument, error)
}
// IdentitySync watches identity files on disk and mirrors their content into
// the database. Disk is the source of truth; the DB is a derived runtime cache.
type IdentitySync struct {
identityDir string
agentID string
store DocumentStore
watcher *fsnotify.Watcher
cancel context.CancelFunc
wg sync.WaitGroup
lastSync atomic.Int64 // unix nanos of the last successful sync
}
// New creates a new IdentitySync. identityDir is the path to the directory
// containing the identity markdown files ($XDG_CONFIG_HOME/picoclaw/identity).
func New(identityDir, agentID string, store DocumentStore) *IdentitySync {
s := &IdentitySync{
identityDir: identityDir,
agentID: agentID,
store: store,
}
s.lastSync.Store(time.Now().UnixNano())
return s
}
// SyncAll performs a full reconciliation: reads every identity file from disk,
// compares its content hash against the stored hash in agent_kv, and upserts
// any that differ. This is the startup path and catches all changes made while
// picoclaw was not running.
func (s *IdentitySync) SyncAll(ctx context.Context) error {
for _, name := range IdentityFiles {
if err := s.syncFile(ctx, name); err != nil {
return fmt.Errorf("sync %s: %w", name, err)
}
}
now := time.Now()
s.lastSync.Store(now.UnixNano())
_ = s.store.UpsertKV(ctx, s.agentID, kvLastSyncKey, now.Format(time.RFC3339Nano))
return nil
}
// Watch starts an fsnotify watcher on the identity directory. File change
// events are debounced and trigger a re-sync of the affected file. Call Close
// to stop watching. This is the daemon-mode path for real-time sync.
func (s *IdentitySync) Watch(ctx context.Context) error {
w, err := fsnotify.NewWatcher()
if err != nil {
return fmt.Errorf("create fsnotify watcher: %w", err)
}
if err := w.Add(s.identityDir); err != nil {
w.Close()
return fmt.Errorf("watch %s: %w", s.identityDir, err)
}
ctx, cancel := context.WithCancel(ctx)
s.watcher = w
s.cancel = cancel
s.wg.Add(1)
go s.watchLoop(ctx, w)
return nil
}
// CheckAndSync is a lightweight pre-prompt safety net. It stats every identity
// file and re-syncs any whose mtime is newer than the last sync timestamp.
// Cost: O(n) stat calls where n = len(IdentityFiles).
func (s *IdentitySync) CheckAndSync(ctx context.Context) error {
threshold := time.Unix(0, s.lastSync.Load())
changed := false
for _, name := range IdentityFiles {
path := filepath.Join(s.identityDir, name)
info, err := os.Stat(path)
if err != nil {
continue
}
if info.ModTime().After(threshold) {
if err := s.syncFile(ctx, name); err != nil {
return err
}
changed = true
}
}
if changed {
now := time.Now()
s.lastSync.Store(now.UnixNano())
_ = s.store.UpsertKV(ctx, s.agentID, kvLastSyncKey, now.Format(time.RFC3339Nano))
}
return nil
}
// Close stops the fsnotify watcher and waits for the watch goroutine to exit.
func (s *IdentitySync) Close() {
if s.cancel != nil {
s.cancel()
}
if s.watcher != nil {
s.watcher.Close()
}
s.wg.Wait()
}
// syncFile reads a single identity file from disk, computes its SHA-256 hash,
// compares with the stored hash, and upserts to agent_documents if different.
func (s *IdentitySync) syncFile(ctx context.Context, name string) error {
path := filepath.Join(s.identityDir, name)
data, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
content := strings.TrimSpace(string(data))
if content == "" {
return nil
}
hash := contentHash(data)
kvKey := kvPrefix + name
stored, err := s.store.GetKV(ctx, s.agentID, kvKey)
if err == nil && stored == hash {
return nil
}
doc := &memory.AgentDocument{
ID: ids.New(),
AgentID: s.agentID,
Name: name,
Category: syncCategory,
Content: content,
}
if err := s.store.UpsertDocument(ctx, doc); err != nil {
return fmt.Errorf("upsert document %s: %w", name, err)
}
if err := s.store.UpsertKV(ctx, s.agentID, kvKey, hash); err != nil {
return fmt.Errorf("upsert hash for %s: %w", name, err)
}
return nil
}
// watchLoop processes fsnotify events with debouncing.
func (s *IdentitySync) watchLoop(ctx context.Context, w *fsnotify.Watcher) {
defer s.wg.Done()
pending := make(map[string]time.Time)
ticker := time.NewTicker(debounceDelay)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case event, ok := <-w.Events:
if !ok {
return
}
if event.Op&(fsnotify.Write|fsnotify.Create|fsnotify.Rename) == 0 {
continue
}
name := filepath.Base(event.Name)
if !isIdentityFile(name) {
continue
}
pending[name] = time.Now()
case _, ok := <-w.Errors:
if !ok {
return
}
case now := <-ticker.C:
for name, queued := range pending {
if now.Sub(queued) < debounceDelay {
continue
}
syncCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
_ = s.syncFile(syncCtx, name)
cancel()
delete(pending, name)
s.lastSync.Store(time.Now().UnixNano())
}
}
}
}
func isIdentityFile(name string) bool {
for _, f := range IdentityFiles {
if f == name {
return true
}
}
return false
}
func contentHash(data []byte) string {
h := sha256.Sum256(data)
return hex.EncodeToString(h[:])
}

314
pkg/sync/identity_test.go Normal file
View file

@ -0,0 +1,314 @@
package sync
import (
"context"
"os"
"path/filepath"
"sync"
"testing"
"time"
"github.com/sipeed/picoclaw/pkg/memory"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// mockStore is a minimal in-memory implementation of DocumentStore for tests.
type mockStore struct {
mu sync.Mutex
kv map[string]string
docs map[string]*memory.AgentDocument // keyed by agentID+":"+name
}
func newMockStore() *mockStore {
return &mockStore{
kv: make(map[string]string),
docs: make(map[string]*memory.AgentDocument),
}
}
func (m *mockStore) GetKV(_ context.Context, agentID, key string) (string, error) {
m.mu.Lock()
defer m.mu.Unlock()
v, ok := m.kv[agentID+":"+key]
if !ok {
return "", os.ErrNotExist
}
return v, nil
}
func (m *mockStore) UpsertKV(_ context.Context, agentID, key, value string) error {
m.mu.Lock()
defer m.mu.Unlock()
m.kv[agentID+":"+key] = value
return nil
}
func (m *mockStore) UpsertDocument(_ context.Context, doc *memory.AgentDocument) error {
m.mu.Lock()
defer m.mu.Unlock()
m.docs[doc.AgentID+":"+doc.Name] = doc
return nil
}
func (m *mockStore) ListDocumentsByCategory(_ context.Context, agentID, category string) ([]*memory.AgentDocument, error) {
m.mu.Lock()
defer m.mu.Unlock()
var out []*memory.AgentDocument
for _, doc := range m.docs {
if doc.AgentID == agentID && doc.Category == category {
out = append(out, doc)
}
}
return out, nil
}
func (m *mockStore) getDoc(agentID, name string) *memory.AgentDocument {
m.mu.Lock()
defer m.mu.Unlock()
return m.docs[agentID+":"+name]
}
func (m *mockStore) getHash(agentID, name string) string {
m.mu.Lock()
defer m.mu.Unlock()
return m.kv[agentID+":"+kvPrefix+name]
}
func setupIdentityDir(t *testing.T, files map[string]string) string {
t.Helper()
dir := t.TempDir()
for name, content := range files {
require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0644))
}
return dir
}
func TestSyncAll_InsertsNewFiles(t *testing.T) {
dir := setupIdentityDir(t, map[string]string{
"AGENT.md": "# Agent\nYou are helpful.",
"SOUL.md": "# Soul\nCurious and kind.",
"USER.md": "# User\nName: Alice",
"IDENTITY.md": "# Identity\nPicoClaw v1",
})
store := newMockStore()
s := New(dir, "agent-1", store)
require.NoError(t, s.SyncAll(context.Background()))
for _, name := range IdentityFiles {
doc := store.getDoc("agent-1", name)
require.NotNil(t, doc, "expected document for %s", name)
assert.Equal(t, syncCategory, doc.Category)
assert.NotEmpty(t, doc.Content)
hash := store.getHash("agent-1", name)
assert.NotEmpty(t, hash, "expected hash for %s", name)
}
}
func TestSyncAll_SkipsUnchangedFiles(t *testing.T) {
dir := setupIdentityDir(t, map[string]string{
"AGENT.md": "# Agent\nSame content.",
})
store := newMockStore()
s := New(dir, "agent-1", store)
require.NoError(t, s.SyncAll(context.Background()))
firstDoc := store.getDoc("agent-1", "AGENT.md")
require.NotNil(t, firstDoc)
firstID := firstDoc.ID
require.NoError(t, s.SyncAll(context.Background()))
secondDoc := store.getDoc("agent-1", "AGENT.md")
assert.Equal(t, firstID, secondDoc.ID, "unchanged file should not be re-upserted")
}
func TestSyncAll_UpsertsModifiedFiles(t *testing.T) {
dir := setupIdentityDir(t, map[string]string{
"AGENT.md": "# Agent\nVersion 1",
})
store := newMockStore()
s := New(dir, "agent-1", store)
require.NoError(t, s.SyncAll(context.Background()))
v1Hash := store.getHash("agent-1", "AGENT.md")
require.NoError(t, os.WriteFile(filepath.Join(dir, "AGENT.md"), []byte("# Agent\nVersion 2"), 0644))
require.NoError(t, s.SyncAll(context.Background()))
v2Hash := store.getHash("agent-1", "AGENT.md")
assert.NotEqual(t, v1Hash, v2Hash, "hash should change after file modification")
doc := store.getDoc("agent-1", "AGENT.md")
assert.Contains(t, doc.Content, "Version 2")
}
func TestSyncAll_SkipsMissingFiles(t *testing.T) {
dir := setupIdentityDir(t, map[string]string{
"AGENT.md": "# Agent only",
})
store := newMockStore()
s := New(dir, "agent-1", store)
require.NoError(t, s.SyncAll(context.Background()))
assert.NotNil(t, store.getDoc("agent-1", "AGENT.md"))
assert.Nil(t, store.getDoc("agent-1", "SOUL.md"), "missing file should not create a doc")
assert.Nil(t, store.getDoc("agent-1", "USER.md"))
assert.Nil(t, store.getDoc("agent-1", "IDENTITY.md"))
}
func TestSyncAll_SkipsEmptyFiles(t *testing.T) {
dir := setupIdentityDir(t, map[string]string{
"AGENT.md": " \n\t\n ",
})
store := newMockStore()
s := New(dir, "agent-1", store)
require.NoError(t, s.SyncAll(context.Background()))
assert.Nil(t, store.getDoc("agent-1", "AGENT.md"), "empty/whitespace-only file should be skipped")
}
func TestSyncAll_IsolatesAgents(t *testing.T) {
dir := setupIdentityDir(t, map[string]string{
"AGENT.md": "# Shared agent file",
})
store := newMockStore()
s1 := New(dir, "agent-a", store)
s2 := New(dir, "agent-b", store)
require.NoError(t, s1.SyncAll(context.Background()))
require.NoError(t, s2.SyncAll(context.Background()))
docA := store.getDoc("agent-a", "AGENT.md")
docB := store.getDoc("agent-b", "AGENT.md")
require.NotNil(t, docA)
require.NotNil(t, docB)
assert.NotEqual(t, docA.ID, docB.ID, "different agents should get separate doc records")
}
func TestCheckAndSync_DetectsModifiedFile(t *testing.T) {
dir := setupIdentityDir(t, map[string]string{
"AGENT.md": "# Original",
})
store := newMockStore()
s := New(dir, "agent-1", store)
require.NoError(t, s.SyncAll(context.Background()))
time.Sleep(50 * time.Millisecond)
require.NoError(t, os.WriteFile(filepath.Join(dir, "AGENT.md"), []byte("# Modified"), 0644))
require.NoError(t, s.CheckAndSync(context.Background()))
doc := store.getDoc("agent-1", "AGENT.md")
assert.Contains(t, doc.Content, "Modified")
}
func TestCheckAndSync_SkipsUntouchedFiles(t *testing.T) {
dir := setupIdentityDir(t, map[string]string{
"AGENT.md": "# Stable",
})
store := newMockStore()
s := New(dir, "agent-1", store)
require.NoError(t, s.SyncAll(context.Background()))
hash1 := store.getHash("agent-1", "AGENT.md")
s.lastSync.Store(time.Now().Add(time.Second).UnixNano())
require.NoError(t, s.CheckAndSync(context.Background()))
hash2 := store.getHash("agent-1", "AGENT.md")
assert.Equal(t, hash1, hash2, "untouched file should not trigger re-sync")
}
func TestWatch_DetectsFileChange(t *testing.T) {
dir := setupIdentityDir(t, map[string]string{
"AGENT.md": "# Initial",
})
store := newMockStore()
s := New(dir, "agent-1", store)
require.NoError(t, s.SyncAll(context.Background()))
ctx := context.Background()
require.NoError(t, s.Watch(ctx))
defer s.Close()
require.NoError(t, os.WriteFile(filepath.Join(dir, "AGENT.md"), []byte("# Updated via watch"), 0644))
assert.Eventually(t, func() bool {
doc := store.getDoc("agent-1", "AGENT.md")
return doc != nil && doc.Content == "# Updated via watch"
}, 3*time.Second, 100*time.Millisecond, "watcher should detect and sync the change")
}
func TestWatch_IgnoresNonIdentityFiles(t *testing.T) {
dir := setupIdentityDir(t, map[string]string{
"AGENT.md": "# Agent",
})
store := newMockStore()
s := New(dir, "agent-1", store)
require.NoError(t, s.SyncAll(context.Background()))
ctx := context.Background()
require.NoError(t, s.Watch(ctx))
defer s.Close()
require.NoError(t, os.WriteFile(filepath.Join(dir, "random.txt"), []byte("not an identity file"), 0644))
time.Sleep(500 * time.Millisecond)
assert.Nil(t, store.getDoc("agent-1", "random.txt"), "non-identity file should be ignored")
}
func TestContentHash_Deterministic(t *testing.T) {
data := []byte("hello world")
h1 := contentHash(data)
h2 := contentHash(data)
assert.Equal(t, h1, h2)
assert.Len(t, h1, 64)
}
func TestContentHash_DifferentForDifferentContent(t *testing.T) {
h1 := contentHash([]byte("version 1"))
h2 := contentHash([]byte("version 2"))
assert.NotEqual(t, h1, h2)
}
func TestIsIdentityFile(t *testing.T) {
tests := []struct {
name string
want bool
}{
{"AGENT.md", true},
{"IDENTITY.md", true},
{"SOUL.md", true},
{"USER.md", true},
{"README.md", false},
{"agent.md", false},
{"AGENT.txt", false},
{"", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, isIdentityFile(tt.name))
})
}
}
func TestNew_SetsFields(t *testing.T) {
store := newMockStore()
s := New("/tmp/identity", "test-agent", store)
assert.Equal(t, "/tmp/identity", s.identityDir)
assert.Equal(t, "test-agent", s.agentID)
assert.NotNil(t, s.store)
}