feat(membench): add LOCOMO memory benchmark tool (#2353)

Benchmark tool comparing legacy session manager vs seahorse short memory
retrieval on the LOCOMO long-term conversational memory dataset.

- cmd/membench/: CLI with ingest/eval/report/run subcommands
- Mode A (legacy): recency-biased budget truncation baseline
- Mode B (seahorse): per-keyword trigram FTS5 search + expand
- Metrics: Token-Overlap F1 and Recall Hit Rate
- `make mem` builds, downloads data, runs benchmark end-to-end

Makefile

@@ -349,6 +349,25 @@ build-macos-app:build-launcher
 	@./scripts/build-macos-app.sh $(PLATFORM)-$(ARCH)
 	@echo "macOS .app bundle created: $(BUILD_DIR)/PicoClaw.app"
 
+## mem: Build membench, download LOCOMO data (if needed), run benchmark, and show results
+mem:
+	@echo "Building membench..."
+	@mkdir -p $(BUILD_DIR)
+	@$(GO) build -o $(BUILD_DIR)/membench ./cmd/membench
+	@echo "Build complete: $(BUILD_DIR)/membench"
+	@if [ ! -f $(BUILD_DIR)/memdata/locomo10.json ]; then \
+		echo "Downloading LOCOMO dataset..."; \
+		mkdir -p $(BUILD_DIR)/memdata; \
+		curl -sfL "https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json" \
+			-o $(BUILD_DIR)/memdata/locomo10.json && [ -s $(BUILD_DIR)/memdata/locomo10.json ] || { echo "Error: LOCOMO download failed"; exit 1; }; \
+		echo "Download complete"; \
+	else \
+		echo "LOCOMO dataset already exists, skipping download"; \
+	fi
+	@echo "Running benchmark..."
+	@rm -rf $(BUILD_DIR)/memout
+	@$(BUILD_DIR)/membench run --data $(BUILD_DIR)/memdata --out $(BUILD_DIR)/memout --budget 4000
+
 ## help: Show this help message
 help:
 	@echo "picoclaw Makefile"

cmd/membench/eval.go

@@ -0,0 +1,366 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/sipeed/picoclaw/pkg/seahorse"
+)
+
+// EvalResult holds per-sample evaluation results for one mode.
+type EvalResult struct {
+	Mode      string     `json:"mode"`
+	SampleID  string     `json:"sampleId"`
+	QAResults []QAResult `json:"qaResults"`
+	Agg       AggMetrics `json:"aggregated"`
+}
+
+// QAResult holds metrics for a single QA pair.
+type QAResult struct {
+	Question   string  `json:"question"`
+	Category   int     `json:"category"`
+	GoldAnswer string  `json:"goldAnswer"`
+	TokenF1    float64 `json:"tokenF1"`
+	HitRate    float64 `json:"hitRate"`
+}
+
+// AggMetrics holds aggregated evaluation metrics.
+type AggMetrics struct {
+	OverallF1      float64             `json:"overallF1"`
+	OverallHitRate float64             `json:"overallHitRate"`
+	ByCategory     map[int]*CatMetrics `json:"byCategory"`
+	TotalQuestions int                 `json:"totalQuestions"`
+}
+
+// CatMetrics holds metrics for a single category.
+type CatMetrics struct {
+	F1            float64 `json:"f1"`
+	HitRate       float64 `json:"hitRate"`
+	QuestionCount int     `json:"questionCount"`
+}
+
+// EvalLegacy evaluates using legacy session store (raw history + budget truncation).
+func EvalLegacy(
+	ctx context.Context,
+	samples []LocomoSample,
+	legacy *LegacyStore,
+	budgetTokens int,
+) []EvalResult {
+	results := make([]EvalResult, 0, len(samples))
+	for si := range samples {
+		sample := &samples[si]
+		history := legacy.GetHistory(sample.SampleID)
+
+		// Convert messages to content strings
+		allContent := make([]string, 0, len(history))
+		for _, msg := range history {
+			allContent = append(allContent, msg.Content)
+		}
+
+		qaResults := make([]QAResult, 0, len(sample.QA))
+		for qi := range sample.QA {
+			qa := &sample.QA[qi]
+			// Budget truncate the full history
+			truncated, _ := BudgetTruncate(allContent, budgetTokens)
+			context := StringListToContent(truncated)
+
+			f1 := TokenOverlapF1(context, qa.AnswerString())
+			hitRate := RecallHitRate(qa.Evidence, sample, context)
+
+			qaResults = append(qaResults, QAResult{
+				Question:   qa.Question,
+				Category:   qa.Category,
+				GoldAnswer: qa.AnswerString(),
+				TokenF1:    f1,
+				HitRate:    hitRate,
+			})
+		}
+
+		results = append(results, EvalResult{
+			Mode:      "legacy",
+			SampleID:  sample.SampleID,
+			QAResults: qaResults,
+			Agg:       aggregateMetrics(qaResults),
+		})
+	}
+	return results
+}
+
+// EvalSeahorse evaluates using seahorse short memory (per-keyword search + expand).
+func EvalSeahorse(
+	ctx context.Context,
+	samples []LocomoSample,
+	ir *SeahorseIngestResult,
+	budgetTokens int,
+) []EvalResult {
+	store := ir.Engine.GetRetrieval().Store()
+	retrieval := ir.Engine.GetRetrieval()
+
+	results := make([]EvalResult, 0, len(samples))
+	for si := range samples {
+		sample := &samples[si]
+		convID, ok := ir.ConvMap[sample.SampleID]
+		if !ok {
+			log.Printf("WARN: no conversation ID for sample %s", sample.SampleID)
+			continue
+		}
+
+		qaResults := make([]QAResult, 0, len(sample.QA))
+		for qi := range sample.QA {
+			qa := &sample.QA[qi]
+			keywords := ExtractKeywords(qa.Question)
+
+			// Search each keyword individually and union results,
+			// tracking best BM25 rank per message for relevance sorting.
+			bestRank := map[int64]float64{}
+			for _, kw := range keywords {
+				searchResults, err := store.SearchMessages(ctx, seahorse.SearchInput{
+					Pattern:        kw,
+					ConversationID: convID,
+					Limit:          20,
+				})
+				if err != nil {
+					log.Printf("WARN: search failed for keyword %q: %v", kw, err)
+					continue
+				}
+				for _, sr := range searchResults {
+					if sr.MessageID > 0 {
+						if prev, ok := bestRank[sr.MessageID]; !ok || sr.Rank < prev {
+							bestRank[sr.MessageID] = sr.Rank
+						}
+					}
+				}
+			}
+			// Sort messageIDs by rank ascending (best/most-negative first).
+			// BudgetTruncate walks from the front, keeping best-ranked messages.
+			// Note: SQLite FTS5 bm25() returns negative values where more
+			// negative = better match.
+			messageIDs := make([]int64, 0, len(bestRank))
+			for id := range bestRank {
+				messageIDs = append(messageIDs, id)
+			}
+			sort.Slice(messageIDs, func(i, j int) bool {
+				return bestRank[messageIDs[i]] < bestRank[messageIDs[j]]
+			})
+
+			// Expand messages to get full content
+			var contentParts []string
+			if len(messageIDs) > 0 {
+				expandResult, err := retrieval.ExpandMessages(ctx, messageIDs)
+				if err != nil {
+					log.Printf("WARN: expand failed for sample %s: %v", sample.SampleID, err)
+				} else {
+					for _, msg := range expandResult.Messages {
+						contentParts = append(contentParts, msg.Content)
+					}
+				}
+			}
+
+			if len(contentParts) == 0 {
+				qaResults = append(qaResults, QAResult{
+					Question:   qa.Question,
+					Category:   qa.Category,
+					GoldAnswer: qa.AnswerString(),
+					TokenF1:    0.0,
+					HitRate:    0.0,
+				})
+				continue
+			}
+
+			// Budget truncate (drop worst-ranked)
+			truncated, _ := BudgetTruncate(contentParts, budgetTokens)
+			context := StringListToContent(truncated)
+
+			f1 := TokenOverlapF1(context, qa.AnswerString())
+			hitRate := RecallHitRate(qa.Evidence, sample, context)
+
+			qaResults = append(qaResults, QAResult{
+				Question:   qa.Question,
+				Category:   qa.Category,
+				GoldAnswer: qa.AnswerString(),
+				TokenF1:    f1,
+				HitRate:    hitRate,
+			})
+		}
+
+		results = append(results, EvalResult{
+			Mode:      "seahorse",
+			SampleID:  sample.SampleID,
+			QAResults: qaResults,
+			Agg:       aggregateMetrics(qaResults),
+		})
+	}
+	return results
+}
+
+// aggregateMetrics computes overall and per-category metrics.
+func aggregateMetrics(qaResults []QAResult) AggMetrics {
+	byCat := map[int]*CatMetrics{}
+	totalF1 := 0.0
+	totalHitRate := 0.0
+	for _, qr := range qaResults {
+		totalF1 += qr.TokenF1
+		totalHitRate += qr.HitRate
+		cat, ok := byCat[qr.Category]
+		if !ok {
+			cat = &CatMetrics{}
+			byCat[qr.Category] = cat
+		}
+		cat.F1 += qr.TokenF1
+		cat.HitRate += qr.HitRate
+		cat.QuestionCount++
+	}
+	n := len(qaResults)
+	if n == 0 {
+		n = 1
+	}
+	agg := AggMetrics{
+		OverallF1:      totalF1 / float64(n),
+		OverallHitRate: totalHitRate / float64(n),
+		ByCategory:     byCat,
+		TotalQuestions: len(qaResults),
+	}
+	for _, cat := range agg.ByCategory {
+		if cat.QuestionCount > 0 {
+			cat.F1 /= float64(cat.QuestionCount)
+			cat.HitRate /= float64(cat.QuestionCount)
+		}
+	}
+	return agg
+}
+
+// SaveResults writes per-sample eval results to JSON files.
+func SaveResults(results []EvalResult, outDir string) error {
+	if err := os.MkdirAll(outDir, 0o755); err != nil {
+		return fmt.Errorf("create output dir: %w", err)
+	}
+	for _, r := range results {
+		path := filepath.Join(outDir, fmt.Sprintf("eval_%s_%s.json", r.Mode, r.SampleID))
+		data, err := json.MarshalIndent(r, "", "  ")
+		if err != nil {
+			return fmt.Errorf("marshal result: %w", err)
+		}
+		if err := os.WriteFile(path, data, 0o644); err != nil {
+			return fmt.Errorf("write result: %w", err)
+		}
+	}
+	return nil
+}
+
+// SaveAggregated writes a combined results.json with all modes.
+func SaveAggregated(results []EvalResult, outDir string) error {
+	byMode := map[string][]EvalResult{}
+	for _, r := range results {
+		byMode[r.Mode] = append(byMode[r.Mode], r)
+	}
+
+	aggMap := map[string]AggMetrics{}
+	for mode, modeResults := range byMode {
+		aggMap[mode] = computeModeAgg(modeResults)
+	}
+
+	data, err := json.MarshalIndent(aggMap, "", "  ")
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(filepath.Join(outDir, "results.json"), data, 0o644)
+}
+
+// computeModeAgg aggregates results for a single mode using weighted averaging
+// (weighted by question count per sample). All modes must have the same Mode field.
+func computeModeAgg(results []EvalResult) AggMetrics {
+	agg := AggMetrics{ByCategory: map[int]*CatMetrics{}}
+	for _, r := range results {
+		agg.OverallF1 += r.Agg.OverallF1 * float64(r.Agg.TotalQuestions)
+		agg.OverallHitRate += r.Agg.OverallHitRate * float64(r.Agg.TotalQuestions)
+		agg.TotalQuestions += r.Agg.TotalQuestions
+		for cat, cm := range r.Agg.ByCategory {
+			existing, ok := agg.ByCategory[cat]
+			if !ok {
+				existing = &CatMetrics{}
+				agg.ByCategory[cat] = existing
+			}
+			existing.F1 += cm.F1 * float64(cm.QuestionCount)
+			existing.HitRate += cm.HitRate * float64(cm.QuestionCount)
+			existing.QuestionCount += cm.QuestionCount
+		}
+	}
+	if agg.TotalQuestions > 0 {
+		agg.OverallF1 /= float64(agg.TotalQuestions)
+		agg.OverallHitRate /= float64(agg.TotalQuestions)
+	}
+	for _, cat := range agg.ByCategory {
+		if cat.QuestionCount > 0 {
+			cat.F1 /= float64(cat.QuestionCount)
+			cat.HitRate /= float64(cat.QuestionCount)
+		}
+	}
+	return agg
+}
+
+// printSection prints a single comparison table section.
+func printSection(title string, results []EvalResult) {
+	fmt.Printf("\n--- %s ---\n", title)
+	byMode := map[string][]EvalResult{}
+	for _, r := range results {
+		byMode[r.Mode] = append(byMode[r.Mode], r)
+	}
+
+	modes := map[string]AggMetrics{}
+	for mode, modeResults := range byMode {
+		modes[mode] = computeModeAgg(modeResults)
+	}
+
+	modeKeys := make([]string, 0, len(modes))
+	for k := range modes {
+		modeKeys = append(modeKeys, k)
+	}
+	sort.Strings(modeKeys)
+
+	// Collect all category keys across modes
+	catSet := map[int]bool{}
+	for _, agg := range modes {
+		for cat := range agg.ByCategory {
+			catSet[cat] = true
+		}
+	}
+	cats := make([]int, 0, len(catSet))
+	for cat := range catSet {
+		cats = append(cats, cat)
+	}
+	sort.Ints(cats)
+
+	fmt.Printf("%-10s %-8s %-8s", "Mode", "HitRate", "F1")
+	for _, cat := range cats {
+		fmt.Printf(" %-7s", fmt.Sprintf("C%d", cat))
+	}
+	fmt.Println()
+	fmt.Println(strings.Repeat("-", 10+8+8+7*len(cats)+8))
+
+	for _, mode := range modeKeys {
+		agg := modes[mode]
+		fmt.Printf("%-10s %-8.4f %-8.4f", mode, agg.OverallHitRate, agg.OverallF1)
+		for _, cat := range cats {
+			if cm, ok := agg.ByCategory[cat]; ok {
+				fmt.Printf(" %-7.4f", cm.HitRate)
+			} else {
+				fmt.Printf(" %-7s", "N/A")
+			}
+		}
+		fmt.Println()
+	}
+}
+
+// PrintComparison outputs a human-readable comparison table to stdout.
+func PrintComparison(results []EvalResult, llmResults []EvalResult) {
+	printSection("No LLM generation", results)
+	if len(llmResults) > 0 {
+		printSection("With LLM", llmResults)
+	}
+}

cmd/membench/eval_test.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"math"
+	"testing"
+)
+
+func TestComputeModeAggAllCategories(t *testing.T) {
+	results := []EvalResult{
+		{
+			Mode:     "test",
+			SampleID: "s1",
+			QAResults: []QAResult{
+				{Category: 1, TokenF1: 0.5, HitRate: 0.8},
+				{Category: 2, TokenF1: 0.3, HitRate: 0.6},
+				{Category: 3, TokenF1: 0.1, HitRate: 0.4},
+				{Category: 4, TokenF1: 0.7, HitRate: 0.9},
+				{Category: 5, TokenF1: 0.2, HitRate: 0.1},
+			},
+		},
+	}
+	for i := range results {
+		results[i].Agg = aggregateMetrics(results[i].QAResults)
+	}
+
+	got := computeModeAgg(results)
+
+	// Should have all 5 categories
+	for cat := 1; cat <= 5; cat++ {
+		cm, ok := got.ByCategory[cat]
+		if !ok {
+			t.Errorf("ByCategory missing category %d", cat)
+			continue
+		}
+		if cm.QuestionCount != 1 {
+			t.Errorf("ByCategory[%d].QuestionCount = %d, want 1", cat, cm.QuestionCount)
+		}
+	}
+
+	// Verify specific F1 values per category
+	wantF1 := map[int]float64{1: 0.5, 2: 0.3, 3: 0.1, 4: 0.7, 5: 0.2}
+	for cat, want := range wantF1 {
+		if cm, ok := got.ByCategory[cat]; ok {
+			if math.Abs(cm.F1-want) > 1e-9 {
+				t.Errorf("ByCategory[%d].F1 = %.4f, want %.4f", cat, cm.F1, want)
+			}
+		}
+	}
+}
+
+func TestComputeModeAgg(t *testing.T) {
+	// Two samples with different question counts:
+	//   sample-a: 2 questions, F1 = [0.4, 0.6] → avg 0.5
+	//   sample-b: 8 questions, F1 = [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1] → avg 0.1
+	//
+	// Unweighted (PrintComparison bug): (0.5 + 0.1) / 2 = 0.3
+	// Weighted (correct):              (0.4+0.6 + 0.1*8) / 10 = 1.8 / 10 = 0.18
+	results := []EvalResult{
+		{
+			Mode:     "test",
+			SampleID: "sample-a",
+			QAResults: []QAResult{
+				{TokenF1: 0.4, HitRate: 0.5},
+				{TokenF1: 0.6, HitRate: 0.7},
+			},
+		},
+		{
+			Mode:     "test",
+			SampleID: "sample-b",
+			QAResults: []QAResult{
+				{TokenF1: 0.1, HitRate: 0.2},
+				{TokenF1: 0.1, HitRate: 0.2},
+				{TokenF1: 0.1, HitRate: 0.2},
+				{TokenF1: 0.1, HitRate: 0.2},
+				{TokenF1: 0.1, HitRate: 0.2},
+				{TokenF1: 0.1, HitRate: 0.2},
+				{TokenF1: 0.1, HitRate: 0.2},
+				{TokenF1: 0.1, HitRate: 0.2},
+			},
+		},
+	}
+	// Compute per-sample aggregates
+	for i := range results {
+		results[i].Agg = aggregateMetrics(results[i].QAResults)
+	}
+
+	got := computeModeAgg(results)
+
+	// Weighted: (0.4+0.6+0.1*8) / 10 = 1.8/10 = 0.18
+	wantF1 := 0.18
+	if math.Abs(got.OverallF1-wantF1) > 1e-9 {
+		t.Errorf("OverallF1 = %.6f, want %.6f (weighted average)", got.OverallF1, wantF1)
+	}
+
+	// Weighted: (0.5+0.7+0.2*8) / 10 = 2.8/10 = 0.28
+	wantRecall := 0.28
+	if math.Abs(got.OverallHitRate-wantRecall) > 1e-9 {
+		t.Errorf("OverallHitRate = %.6f, want %.6f (weighted average)", got.OverallHitRate, wantRecall)
+	}
+
+	if got.TotalQuestions != 10 {
+		t.Errorf("TotalQuestions = %d, want 10", got.TotalQuestions)
+	}
+}

cmd/membench/ingest.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+
+	"github.com/sipeed/picoclaw/pkg/seahorse"
+)
+
+// ConvMap stores the mapping from sampleID to seahorse ConversationID.
+type ConvMap map[string]int64
+
+// SeahorseIngestResult holds the results of ingesting into seahorse.
+type SeahorseIngestResult struct {
+	Engine  *seahorse.Engine
+	ConvMap ConvMap // sampleID → conversationID
+}
+
+// IngestSeahorse loads all LOCOMO samples into a seahorse Engine.
+// Returns the engine and a mapping from sampleID to conversationID for scoped retrieval.
+func IngestSeahorse(ctx context.Context, samples []LocomoSample, dbPath string) (*SeahorseIngestResult, error) {
+	noopFn := func(ctx context.Context, prompt string, opts seahorse.CompleteOptions) (string, error) {
+		return "", nil
+	}
+
+	engine, err := seahorse.NewEngine(seahorse.Config{
+		DBPath: dbPath,
+	}, noopFn)
+	if err != nil {
+		return nil, fmt.Errorf("create seahorse engine: %w", err)
+	}
+
+	store := engine.GetRetrieval().Store()
+	convMap := make(ConvMap)
+
+	for si := range samples {
+		sample := &samples[si]
+		sessionKey := "locomo-" + sample.SampleID
+
+		// Check if conversation already exists (idempotent)
+		existing, _ := store.GetConversationBySessionKey(ctx, sessionKey)
+		if existing != nil {
+			convMap[sample.SampleID] = existing.ConversationID
+			log.Printf("Skipping existing sample %s: convID=%d", sample.SampleID, existing.ConversationID)
+			continue
+		}
+
+		turns := GetTurns(sample)
+
+		// Convert turns to seahorse messages
+		msgs := make([]seahorse.Message, 0, len(turns))
+		for _, turn := range turns {
+			content := turn.Speaker + ": " + turn.Text
+			msgs = append(msgs, seahorse.Message{
+				Role:       "user",
+				Content:    content,
+				TokenCount: len(turn.Text) / 4,
+			})
+		}
+
+		// Ingest all turns for this sample
+		_, err := engine.Ingest(ctx, sessionKey, msgs)
+		if err != nil {
+			return nil, fmt.Errorf("ingest sample %s: %w", sample.SampleID, err)
+		}
+
+		// Get the conversation ID for scoped retrieval
+		conv, err := store.GetConversationBySessionKey(ctx, sessionKey)
+		if err != nil {
+			return nil, fmt.Errorf("get conversation for %s: %w", sample.SampleID, err)
+		}
+		if conv == nil {
+			return nil, fmt.Errorf("conversation not found for %s after ingest", sample.SampleID)
+		}
+		convMap[sample.SampleID] = conv.ConversationID
+		log.Printf("Ingested sample %s: %d turns, convID=%d", sample.SampleID, len(turns), conv.ConversationID)
+	}
+
+	log.Printf("Seahorse ingestion complete: %d samples, %d conversations", len(samples), len(convMap))
+	return &SeahorseIngestResult{
+		Engine:  engine,
+		ConvMap: convMap,
+	}, nil
+}

cmd/membench/ingest_test.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"path/filepath"
+	"testing"
+
+	"github.com/sipeed/picoclaw/pkg/seahorse"
+)
+
+func TestIngestSeahorseIdempotent(t *testing.T) {
+	ctx := context.Background()
+	tmpDir := t.TempDir()
+	dbPath := filepath.Join(tmpDir, "test.db")
+
+	// Minimal test data
+	samples := []LocomoSample{
+		{
+			SampleID: "test-1",
+			Conversation: map[string]json.RawMessage{
+				"session_1": json.RawMessage(`[
+					{"speaker":"A","dia_id":"D1:1","text":"hello world this is a test message"},
+					{"speaker":"B","dia_id":"D1:2","text":"another message for testing purposes"}
+				]`),
+			},
+		},
+	}
+
+	// First ingestion
+	result1, err := IngestSeahorse(ctx, samples, dbPath)
+	if err != nil {
+		t.Fatalf("first ingest failed: %v", err)
+	}
+	convCount1 := len(result1.ConvMap)
+	result1.Engine.Close()
+
+	// Second ingestion on same DB — should reuse existing data
+	result2, err := IngestSeahorse(ctx, samples, dbPath)
+	if err != nil {
+		t.Fatalf("second ingest failed: %v", err)
+	}
+	defer result2.Engine.Close()
+
+	// ConvMap should have same number of entries (no duplicates)
+	if len(result2.ConvMap) != convCount1 {
+		t.Errorf("second ingest convMap has %d entries, want %d (same as first)",
+			len(result2.ConvMap), convCount1)
+	}
+
+	// Verify conversation IDs are the same (reused, not new ones)
+	for id, cid1 := range result1.ConvMap {
+		cid2, ok := result2.ConvMap[id]
+		if !ok {
+			t.Errorf("sample %s missing from second ConvMap", id)
+			continue
+		}
+		if cid2 != cid1 {
+			t.Errorf("sample %s: second ingest got convID %d, want %d (reused)", id, cid2, cid1)
+		}
+	}
+
+	// Verify no duplicate messages by counting
+	store := result2.Engine.GetRetrieval().Store()
+	for _, convID := range result2.ConvMap {
+		msgs, err := store.SearchMessages(ctx, seahorse.SearchInput{
+			Pattern:        "test",
+			ConversationID: convID,
+			Limit:          100,
+		})
+		if err != nil {
+			t.Fatalf("search failed: %v", err)
+		}
+		// Should find exactly 1 message containing "test" (the first turn)
+		if len(msgs) > 2 {
+			t.Errorf("found %d messages for 'test' in conv %d, expected ≤2 (no duplicates)", len(msgs), convID)
+		}
+	}
+}

cmd/membench/legacy_store.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"github.com/sipeed/picoclaw/pkg/providers"
+	"github.com/sipeed/picoclaw/pkg/session"
+)
+
+// LegacyStore wraps session.SessionManager for legacy baseline.
+type LegacyStore struct {
+	sm *session.SessionManager
+}
+
+// NewLegacyStore creates a new in-memory session manager.
+func NewLegacyStore() *LegacyStore {
+	return &LegacyStore{
+		sm: session.NewSessionManager(""),
+	}
+}
+
+// IngestSample loads all turns from a LOCOMO sample into the legacy session store.
+func (ls *LegacyStore) IngestSample(sample *LocomoSample) {
+	sessionKey := "locomo-" + sample.SampleID
+	turns := GetTurns(sample)
+	for _, turn := range turns {
+		content := turn.Speaker + ": " + turn.Text
+		ls.sm.AddMessage(sessionKey, "user", content)
+	}
+}
+
+// GetHistory returns all messages for a sample's session.
+func (ls *LegacyStore) GetHistory(sampleID string) []providers.Message {
+	sessionKey := "locomo-" + sampleID
+	return ls.sm.GetHistory(sessionKey)
+}

cmd/membench/locomo.go

@@ -0,0 +1,142 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+// LocomoSample represents one conversation sample from the LOCOMO dataset.
+type LocomoSample struct {
+	SampleID     string                     `json:"sample_id"`
+	Conversation map[string]json.RawMessage `json:"conversation"`
+	QA           []LocomoQA                 `json:"qa"`
+}
+
+// LocomoTurn represents a single turn in a conversation.
+type LocomoTurn struct {
+	Speaker string `json:"speaker"`
+	DiaID   string `json:"dia_id"`
+	Text    string `json:"text"`
+}
+
+// LocomoQA represents a question-answer pair with evidence.
+type LocomoQA struct {
+	Question          string          `json:"question"`
+	Answer            json.RawMessage `json:"answer"`             // can be string or int (category 1-4)
+	AdversarialAnswer string          `json:"adversarial_answer"` // category 5 only
+	Evidence          []string        `json:"evidence"`
+	Category          int             `json:"category"` // 1=single-hop, 2=multi-hop, 3=open-ended, 5=adversarial
+}
+
+// AnswerString returns the answer as a string, handling both string and int types.
+func (qa *LocomoQA) AnswerString() string {
+	// Prefer answer field (category 1-4)
+	if len(qa.Answer) > 0 {
+		var s string
+		if err := json.Unmarshal(qa.Answer, &s); err == nil {
+			return s
+		}
+		var n json.Number
+		if err := json.Unmarshal(qa.Answer, &n); err == nil {
+			return n.String()
+		}
+		return strings.Trim(string(qa.Answer), `"`)
+	}
+	// Fallback to adversarial_answer (category 5)
+	return qa.AdversarialAnswer
+}
+
+// LoadDataset reads all JSON files from dataDir and returns parsed samples.
+func LoadDataset(dataDir string) ([]LocomoSample, error) {
+	entries, err := os.ReadDir(dataDir)
+	if err != nil {
+		return nil, fmt.Errorf("read data dir %s: %w", dataDir, err)
+	}
+
+	var samples []LocomoSample
+	for _, entry := range entries {
+		if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".json") {
+			path := filepath.Join(dataDir, entry.Name())
+			data, err := os.ReadFile(path)
+			if err != nil {
+				return nil, fmt.Errorf("read file %s: %w", path, err)
+			}
+			var batch []LocomoSample
+			if err := json.Unmarshal(data, &batch); err != nil {
+				return nil, fmt.Errorf("parse file %s: %w", path, err)
+			}
+			samples = append(samples, batch...)
+		}
+	}
+	return samples, nil
+}
+
+// GetSessionNames returns sorted session keys (session_1, session_2, ...) from conversation.
+func GetSessionNames(conv map[string]json.RawMessage) []string {
+	var names []string
+	for k := range conv {
+		if strings.HasPrefix(k, "session_") && !strings.Contains(k, "_date_time") {
+			names = append(names, k)
+		}
+	}
+	sort.Slice(names, func(i, j int) bool {
+		ni := sessionNum(names[i])
+		nj := sessionNum(names[j])
+		return ni < nj
+	})
+	return names
+}
+
+func sessionNum(key string) int {
+	// "session_1" → 1, "session_10" → 10
+	parts := strings.SplitN(key, "_", 2)
+	if len(parts) < 2 {
+		return 0
+	}
+	n, _ := strconv.Atoi(parts[1])
+	return n
+}
+
+// GetTurns flattens all sessions' turns in chronological order.
+func GetTurns(sample *LocomoSample) []LocomoTurn {
+	names := GetSessionNames(sample.Conversation)
+	var all []LocomoTurn
+	for _, name := range names {
+		raw, ok := sample.Conversation[name]
+		if !ok {
+			continue
+		}
+		var turns []LocomoTurn
+		if err := json.Unmarshal(raw, &turns); err != nil {
+			log.Printf("WARNING: unmarshal failed for session %q in sample %s: %v", name, sample.SampleID, err)
+			continue
+		}
+		all = append(all, turns...)
+	}
+	return all
+}
+
+// GetTurnByDiaID finds a specific turn by dia_id (e.g. "D1:3").
+func GetTurnByDiaID(sample *LocomoSample, diaID string) *LocomoTurn {
+	turns := GetTurns(sample)
+	for i := range turns {
+		if turns[i].DiaID == diaID {
+			return &turns[i]
+		}
+	}
+	return nil
+}
+
+// GetSpeakers returns the two speaker names from conversation metadata.
+func GetSpeakers(conv map[string]json.RawMessage) (string, string) {
+	var a, b string
+	json.Unmarshal(conv["speaker_a"], &a)
+	json.Unmarshal(conv["speaker_b"], &b)
+	return a, b
+}

cmd/membench/locomo_test.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestAnswerString(t *testing.T) {
+	tests := []struct {
+		name string
+		json string
+		want string
+	}{
+		{
+			"string answer",
+			`{"question":"Q","answer":"Paris","evidence":[],"category":1}`,
+			"Paris",
+		},
+		{
+			"int answer",
+			`{"question":"Q","answer":42,"evidence":[],"category":1}`,
+			"42",
+		},
+		{
+			"adversarial answer (category 5)",
+			`{"question":"Q","evidence":[],"category":5,"adversarial_answer":"self-care is important"}`,
+			"self-care is important",
+		},
+		{
+			"both answer and adversarial_answer present",
+			`{"question":"Q","answer":"normal","evidence":[],"category":5,"adversarial_answer":"adversarial"}`,
+			"normal",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var qa LocomoQA
+			if err := json.Unmarshal([]byte(tt.json), &qa); err != nil {
+				t.Fatalf("unmarshal: %v", err)
+			}
+			got := qa.AnswerString()
+			if got != tt.want {
+				t.Errorf("AnswerString() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestGetSessionNames(t *testing.T) {
+	conv := map[string]json.RawMessage{
+		"session_2":           {},
+		"session_1":           {},
+		"session_10":          {},
+		"session_1_date_time": {},
+		"speaker_a":           {},
+	}
+	names := GetSessionNames(conv)
+	want := []string{"session_1", "session_2", "session_10"}
+	if len(names) != len(want) {
+		t.Fatalf("got %v, want %v", names, want)
+	}
+	for i, n := range names {
+		if n != want[i] {
+			t.Errorf("names[%d] = %q, want %q", i, n, want[i])
+		}
+	}
+}

cmd/membench/main.go

@@ -0,0 +1,208 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/cobra"
+
+	"github.com/sipeed/picoclaw/pkg/logger"
+)
+
+var (
+	flagData   string
+	flagOut    string
+	flagMode   string
+	flagBudget int
+)
+
+func main() {
+	// Suppress seahorse INFO logs during benchmark
+	logger.SetLevel(logger.WARN)
+
+	rootCmd := &cobra.Command{
+		Use:   "membench",
+		Short: "Memory benchmark tool for picoclaw",
+	}
+
+	ingestCmd := &cobra.Command{
+		Use:   "ingest",
+		Short: "Load LOCOMO data into storage backends",
+		RunE:  runIngest,
+	}
+	ingestCmd.Flags().StringVar(&flagData, "data", "", "LOCOMO dataset directory (required)")
+	ingestCmd.Flags().StringVar(&flagOut, "out", "./bench-out", "output working directory")
+	ingestCmd.Flags().StringVar(&flagMode, "mode", "all", "modes to ingest: legacy, seahorse, or all")
+
+	evalCmd := &cobra.Command{
+		Use:   "eval",
+		Short: "Run QA evaluation against ingested data",
+		RunE:  runEval,
+	}
+	evalCmd.Flags().StringVar(&flagData, "data", "", "LOCOMO dataset directory (required)")
+	evalCmd.Flags().StringVar(&flagOut, "out", "./bench-out", "output working directory")
+	evalCmd.Flags().StringVar(&flagMode, "mode", "all", "modes to evaluate: legacy, seahorse, or all")
+	evalCmd.Flags().IntVar(&flagBudget, "budget", 4000, "token budget for retrieval")
+
+	reportCmd := &cobra.Command{
+		Use:   "report",
+		Short: "Output comparison results from evaluation",
+		RunE:  runReport,
+	}
+	reportCmd.Flags().StringVar(&flagOut, "out", "./bench-out", "output working directory")
+
+	runCmd := &cobra.Command{
+		Use:   "run",
+		Short: "Convenience: eval + report (ingestion is done inline)",
+		RunE:  runAll,
+	}
+	runCmd.Flags().StringVar(&flagData, "data", "", "LOCOMO dataset directory (required)")
+	runCmd.Flags().StringVar(&flagOut, "out", "./bench-out", "output working directory")
+	runCmd.Flags().StringVar(&flagMode, "mode", "all", "modes to run: legacy, seahorse, or all")
+	runCmd.Flags().IntVar(&flagBudget, "budget", 4000, "token budget for retrieval")
+
+	rootCmd.AddCommand(ingestCmd, evalCmd, reportCmd, runCmd)
+
+	if err := rootCmd.Execute(); err != nil {
+		os.Exit(1)
+	}
+}
+
+func modesFromFlag() []string {
+	switch strings.ToLower(flagMode) {
+	case "all":
+		return []string{"legacy", "seahorse"}
+	default:
+		return []string{strings.ToLower(flagMode)}
+	}
+}
+
+func runIngest(cmd *cobra.Command, args []string) error {
+	if flagData == "" {
+		return fmt.Errorf("--data is required")
+	}
+	modes := modesFromFlag()
+	if len(modes) == 0 {
+		return nil
+	}
+
+	ctx := context.Background()
+	samples, err := LoadDataset(flagData)
+	if err != nil {
+		return fmt.Errorf("load dataset: %w", err)
+	}
+	log.Printf("Loaded %d samples from %s", len(samples), flagData)
+
+	for _, mode := range modes {
+		switch mode {
+		case "legacy":
+			legacy := NewLegacyStore()
+			for i := range samples {
+				legacy.IngestSample(&samples[i])
+			}
+			log.Printf("legacy: ingested %d samples", len(samples))
+		case "seahorse":
+			dbPath := filepath.Join(flagOut, "seahorse.db")
+			if err := os.MkdirAll(flagOut, 0o755); err != nil {
+				return fmt.Errorf("create out dir: %w", err)
+			}
+			_, err := IngestSeahorse(ctx, samples, dbPath)
+			if err != nil {
+				return fmt.Errorf("ingest seahorse: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+func runEval(cmd *cobra.Command, args []string) error {
+	if flagData == "" {
+		return fmt.Errorf("--data is required")
+	}
+	modes := modesFromFlag()
+	if len(modes) == 0 {
+		return nil
+	}
+
+	ctx := context.Background()
+	samples, err := LoadDataset(flagData)
+	if err != nil {
+		return fmt.Errorf("load dataset: %w", err)
+	}
+	log.Printf("Loaded %d samples", len(samples))
+
+	var allResults []EvalResult
+
+	for _, mode := range modes {
+		switch mode {
+		case "legacy":
+			legacy := NewLegacyStore()
+			for i := range samples {
+				legacy.IngestSample(&samples[i])
+			}
+			results := EvalLegacy(ctx, samples, legacy, flagBudget)
+			allResults = append(allResults, results...)
+			log.Printf("legacy: evaluated %d samples", len(results))
+		case "seahorse":
+			dbPath := filepath.Join(flagOut, "seahorse.db")
+			ir, err := IngestSeahorse(ctx, samples, dbPath)
+			if err != nil {
+				return fmt.Errorf("ingest seahorse: %w", err)
+			}
+			results := EvalSeahorse(ctx, samples, ir, flagBudget)
+			allResults = append(allResults, results...)
+			log.Printf("seahorse: evaluated %d samples", len(results))
+		}
+	}
+
+	if err := SaveResults(allResults, flagOut); err != nil {
+		return fmt.Errorf("save results: %w", err)
+	}
+	if err := SaveAggregated(allResults, flagOut); err != nil {
+		return fmt.Errorf("save aggregated: %w", err)
+	}
+
+	PrintComparison(allResults, nil)
+	return nil
+}
+
+func runReport(cmd *cobra.Command, args []string) error {
+	entries, err := os.ReadDir(flagOut)
+	if err != nil {
+		return fmt.Errorf("read out dir: %w", err)
+	}
+
+	var allResults []EvalResult
+	for _, entry := range entries {
+		if !entry.IsDir() && strings.HasPrefix(entry.Name(), "eval_") && strings.HasSuffix(entry.Name(), ".json") {
+			path := filepath.Join(flagOut, entry.Name())
+			var r EvalResult
+			data, err := os.ReadFile(path)
+			if err != nil {
+				log.Printf("WARN: read %s: %v", path, err)
+				continue
+			}
+			if err := json.Unmarshal(data, &r); err != nil {
+				log.Printf("WARN: parse %s: %v", path, err)
+				continue
+			}
+			allResults = append(allResults, r)
+		}
+	}
+
+	if len(allResults) == 0 {
+		return fmt.Errorf("no eval results found in %s", flagOut)
+	}
+
+	PrintComparison(allResults, nil)
+	return nil
+}
+
+func runAll(cmd *cobra.Command, args []string) error {
+	return runEval(cmd, args)
+}

cmd/membench/metrics.go

@@ -0,0 +1,227 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"regexp"
+	"strconv"
+	"strings"
+	"unicode"
+)
+
+// diaIDRe matches valid dia_id patterns like "D1:3", "D30:5".
+var diaIDRe = regexp.MustCompile(`^D(\d+):(\d+)$`)
+
+// SplitEvidenceIDs splits an evidence string that may contain multiple
+// semicolon-separated or space-separated dia_ids. Only returns valid IDs.
+// Example: "D8:6; D9:17" → ["D8:6", "D9:17"]
+// Example: "D9:1 D4:4 D4:6" → ["D9:1", "D4:4", "D4:6"]
+func SplitEvidenceIDs(evidence string) []string {
+	if evidence == "" {
+		return nil
+	}
+	// Split on semicolons first, then spaces
+	parts := strings.Split(evidence, ";")
+	var ids []string
+	for _, part := range parts {
+		for _, token := range strings.Fields(strings.TrimSpace(part)) {
+			token = strings.TrimSpace(token)
+			if diaIDRe.MatchString(token) {
+				ids = append(ids, NormalizeDiaID(token))
+			}
+		}
+	}
+	if len(ids) == 0 {
+		return nil
+	}
+	return ids
+}
+
+// NormalizeDiaID strips leading zeros from the number parts of a dia_id.
+// "D30:05" → "D30:5", "D10:003" → "D10:3"
+func NormalizeDiaID(id string) string {
+	m := diaIDRe.FindStringSubmatch(id)
+	if m == nil {
+		return id
+	}
+	session, _ := strconv.Atoi(m[1])
+	turn, _ := strconv.Atoi(m[2])
+	return fmt.Sprintf("D%d:%d", session, turn)
+}
+
+// stopwords is a fixed English stopword list for deterministic keyword extraction.
+var stopwords = map[string]struct{}{
+	"a": {}, "an": {}, "the": {},
+	"is": {}, "are": {}, "was": {}, "were": {},
+	"did": {}, "does": {}, "do": {},
+	"when": {}, "where": {}, "what": {}, "who": {},
+	"how": {}, "why": {},
+	"to": {}, "of": {}, "in": {}, "on": {}, "at": {},
+	"for": {}, "and": {}, "or": {}, "but": {}, "not": {},
+	"it": {}, "this": {}, "that": {}, "with": {},
+	"from": {}, "by": {}, "as": {},
+	"if": {}, "then": {}, "than": {}, "so": {},
+	"no": {}, "yes": {},
+	"all": {}, "any": {}, "each": {}, "every": {},
+	"some": {}, "such": {},
+	"about": {}, "into": {}, "over": {},
+	"after": {}, "before": {}, "between": {},
+	"through": {}, "during": {}, "until": {},
+	"would": {}, "could": {}, "should": {},
+	"may": {}, "might": {}, "can": {},
+	"will": {}, "shall": {}, "must": {},
+	"have": {}, "has": {}, "had": {},
+	"been": {}, "being": {}, "be": {},
+	"go": {}, "went": {}, "gone": {},
+	"i": {}, "you": {}, "me": {}, "my": {}, "your": {},
+	"we": {}, "they": {}, "them": {}, "our": {},
+	"its": {}, "their": {}, "he": {}, "she": {},
+	"his": {}, "her": {},
+}
+
+// ExtractKeywords removes stopwords and punctuation, returns individual keywords.
+// Deterministic: uses fixed stopword list, no LLM.
+func ExtractKeywords(question string) []string {
+	// Lowercase and split on whitespace/punctuation
+	lower := strings.ToLower(question)
+	words := strings.FieldsFunc(lower, func(r rune) bool {
+		return !unicode.IsLetter(r) && !unicode.IsDigit(r)
+	})
+
+	var keywords []string
+	for _, w := range words {
+		if w == "" || len(w) < 2 {
+			continue
+		}
+		if _, ok := stopwords[w]; ok {
+			continue
+		}
+		keywords = append(keywords, w)
+		if len(keywords) >= 6 {
+			break
+		}
+	}
+	return keywords
+}
+
+// TokenOverlapF1 computes token-level F1 between prediction and reference.
+// Both strings are lowercased and split on whitespace.
+// NOTE: This metric underestimates quality for multi-hop (cat 2) and
+// open-ended (cat 3) questions where the gold answer uses different phrasing
+// than the source text. LLM-Judge scoring is a v2 follow-up.
+func TokenOverlapF1(prediction, reference string) float64 {
+	predTokens := tokenize(prediction)
+	refTokens := tokenize(reference)
+
+	if len(predTokens) == 0 && len(refTokens) == 0 {
+		return 1.0
+	}
+	if len(predTokens) == 0 || len(refTokens) == 0 {
+		return 0.0
+	}
+
+	// Count matches
+	refCount := map[string]int{}
+	for _, t := range refTokens {
+		refCount[t]++
+	}
+
+	predCount := map[string]int{}
+	for _, t := range predTokens {
+		predCount[t]++
+	}
+
+	var matches float64
+	for token, pc := range predCount {
+		if rc, ok := refCount[token]; ok {
+			matches += float64(min(pc, rc))
+		}
+	}
+
+	precision := matches / float64(len(predTokens))
+	recall := matches / float64(len(refTokens))
+
+	if precision+recall == 0 {
+		return 0.0
+	}
+	return 2 * precision * recall / (precision + recall)
+}
+
+func tokenize(s string) []string {
+	lower := strings.ToLower(s)
+	return strings.Fields(lower)
+}
+
+// RecallHitRate computes fraction of evidence IDs found in retrieved content.
+// For each evidence dia_id, looks up the turn text and checks substring match.
+// Logs a warning for turns with text < 20 chars (higher false-positive risk).
+func RecallHitRate(evidenceIDs []string, sample *LocomoSample, retrievedContent string) float64 {
+	if len(evidenceIDs) == 0 {
+		return 1.0 // no evidence required = perfect
+	}
+
+	// Expand any multi-ID evidence entries (e.g. "D8:6; D9:17" or "D9:1 D4:4")
+	var expanded []string
+	for _, id := range evidenceIDs {
+		split := SplitEvidenceIDs(id)
+		if split != nil {
+			expanded = append(expanded, split...)
+		}
+	}
+	if len(expanded) == 0 {
+		log.Printf("WARNING: no valid dia_ids after expanding evidence %v", evidenceIDs)
+		return float64(0) / float64(len(evidenceIDs))
+	}
+
+	// Build turn index once (avoids re-parsing JSON per ID)
+	turns := GetTurns(sample)
+	turnMap := make(map[string]*LocomoTurn, len(turns))
+	for i := range turns {
+		turnMap[turns[i].DiaID] = &turns[i]
+	}
+
+	lowerRetrieved := strings.ToLower(retrievedContent)
+	found := 0
+	resolvable := 0
+	for _, diaID := range expanded {
+		turn, ok := turnMap[diaID]
+		if !ok {
+			log.Printf("WARNING: dia_id %q not found in sample %s", diaID, sample.SampleID)
+			continue
+		}
+		resolvable++
+		if len(turn.Text) < 20 {
+			log.Printf("WARNING: short turn text (%d chars) for dia_id %s: %q",
+				len(turn.Text), diaID, turn.Text)
+		}
+		if strings.Contains(lowerRetrieved, strings.ToLower(turn.Text)) {
+			found++
+		}
+	}
+	if resolvable == 0 {
+		return 0.0 // no resolvable evidence = can't evaluate
+	}
+	return float64(found) / float64(resolvable)
+}
+
+// BudgetTruncate truncates messages to fit within a token budget.
+// Returns the truncated messages and total token count.
+func BudgetTruncate(messages []string, budgetTokens int) ([]string, int) {
+	var result []string
+	total := 0
+	// Walk from the front (best first) and keep until budget exhausted.
+	for i := 0; i < len(messages); i++ {
+		tokens := len(messages[i]) / 4
+		if total+tokens > budgetTokens && len(result) > 0 {
+			break
+		}
+		result = append(result, messages[i])
+		total += tokens
+	}
+	return result, total
+}
+
+// StringListToContent joins a list of strings into a single content string.
+func StringListToContent(parts []string) string {
+	return strings.Join(parts, "\n")
+}

cmd/membench/metrics_test.go

@@ -0,0 +1,239 @@
+package main
+
+import (
+	"encoding/json"
+	"math"
+	"testing"
+)
+
+func TestSplitEvidenceIDs(t *testing.T) {
+	tests := []struct {
+		input string
+		want  []string
+	}{
+		{"D1:3", []string{"D1:3"}},
+		{"D8:6; D9:17", []string{"D8:6", "D9:17"}},
+		{"D9:1 D4:4 D4:6", []string{"D9:1", "D4:4", "D4:6"}},
+		{"D22:1 D22:2 D9:10 D9:11", []string{"D22:1", "D22:2", "D9:10", "D9:11"}},
+		{"D21:18 D21:22 D11:15 D11:19", []string{"D21:18", "D21:22", "D11:15", "D11:19"}},
+		{"D30:05", []string{"D30:5"}},
+		{"D", nil},
+		{"D:", nil},
+		{"", nil},
+	}
+	for _, tt := range tests {
+		t.Run(tt.input, func(t *testing.T) {
+			got := SplitEvidenceIDs(tt.input)
+			if len(got) != len(tt.want) {
+				t.Fatalf("SplitEvidenceIDs(%q) = %v, want %v", tt.input, got, tt.want)
+			}
+			for i := range got {
+				if got[i] != tt.want[i] {
+					t.Errorf("[%d] = %q, want %q", i, got[i], tt.want[i])
+				}
+			}
+		})
+	}
+}
+
+func TestNormalizeDiaID(t *testing.T) {
+	tests := []struct {
+		input string
+		want  string
+	}{
+		{"D1:3", "D1:3"},
+		{"D30:05", "D30:5"},
+		{"D10:003", "D10:3"},
+		{"D1:0", "D1:0"},
+	}
+	for _, tt := range tests {
+		got := NormalizeDiaID(tt.input)
+		if got != tt.want {
+			t.Errorf("NormalizeDiaID(%q) = %q, want %q", tt.input, got, tt.want)
+		}
+	}
+}
+
+func TestTokenOverlapF1(t *testing.T) {
+	tests := []struct {
+		name       string
+		prediction string
+		reference  string
+		want       float64
+	}{
+		{"exact match", "hello world", "hello world", 1.0},
+		{"no overlap", "foo bar", "baz qux", 0.0},
+		{"empty both", "", "", 1.0},
+		{"empty prediction", "", "hello", 0.0},
+		{"empty reference", "hello", "", 0.0},
+		{"partial overlap", "the cat sat on the mat", "the cat on the floor", 8.0 / 11.0},
+		{"case insensitive", "Hello World", "hello world", 1.0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := TokenOverlapF1(tt.prediction, tt.reference)
+			if math.Abs(got-tt.want) > 1e-9 {
+				t.Errorf("TokenOverlapF1(%q, %q) = %.4f, want %.4f",
+					tt.prediction, tt.reference, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestBudgetTruncate(t *testing.T) {
+	t.Run("within budget returns all", func(t *testing.T) {
+		msgs := []string{"short", "message", "here"}
+		result, total := BudgetTruncate(msgs, 1000)
+		if len(result) != 3 {
+			t.Errorf("expected 3 messages, got %d", len(result))
+		}
+		if total == 0 {
+			t.Error("expected non-zero token count")
+		}
+	})
+
+	t.Run("over budget keeps best first", func(t *testing.T) {
+		msgs := []string{
+			"best message that is quite long and takes up tokens",
+			"good message also fairly long content",
+			"worst short",
+		}
+		result, _ := BudgetTruncate(msgs, 5) // very small budget
+		if len(result) == 0 {
+			t.Fatal("expected at least one message")
+		}
+		// Best-ranked (first) should be kept
+		if result[0] != "best message that is quite long and takes up tokens" {
+			t.Errorf("expected best message kept first, got %q", result[0])
+		}
+	})
+
+	t.Run("over budget keeps best ranked first", func(t *testing.T) {
+		// Messages are sorted by bm25 rank ascending (best/most-negative first).
+		// When budget is insufficient, BudgetTruncate must keep the front
+		// (best-ranked) messages, not the tail (worst-ranked).
+		msgs := []string{
+			"best ranked message with some content here",
+			"second best message also has content",
+			"third message here too",
+			"worst ranked short",
+		}
+		// Budget only fits ~1 message (~10 tokens per message, budget=12)
+		result, _ := BudgetTruncate(msgs, 12)
+		if len(result) == 0 {
+			t.Fatal("expected at least one message")
+		}
+		if result[0] != "best ranked message with some content here" {
+			t.Errorf("expected best-ranked (first) message kept, got %q", result[0])
+		}
+		// Worst-ranked (last) must NOT appear
+		for _, m := range result {
+			if m == "worst ranked short" {
+				t.Error("worst-ranked message should have been truncated")
+			}
+		}
+	})
+
+	t.Run("preserves original order", func(t *testing.T) {
+		msgs := []string{"alpha", "beta", "gamma"}
+		result, _ := BudgetTruncate(msgs, 100)
+		for i, got := range result {
+			if got != msgs[i] {
+				t.Errorf("result[%d] = %q, want %q", i, got, msgs[i])
+			}
+		}
+	})
+
+	t.Run("empty input", func(t *testing.T) {
+		result, total := BudgetTruncate(nil, 100)
+		if len(result) != 0 {
+			t.Errorf("expected 0 messages, got %d", len(result))
+		}
+		if total != 0 {
+			t.Errorf("expected 0 tokens, got %d", total)
+		}
+	})
+}
+
+func TestRecallHitRate(t *testing.T) {
+	// Build a sample with known turns
+	sample := &LocomoSample{
+		SampleID: "test-sample",
+		Conversation: map[string]json.RawMessage{
+			"session_1": json.RawMessage(`[
+				{"speaker":"A","dia_id":"D1:1","text":"hello world this is a test message with enough length"},
+				{"speaker":"B","dia_id":"D1:2","text":"another message for testing recall computation purposes here"},
+				{"speaker":"A","dia_id":"D1:3","text":"third turn with some more content to test"}
+			]`),
+		},
+	}
+
+	t.Run("all evidence found", func(t *testing.T) {
+		retrieved := "hello world this is a test message with enough length another message for testing recall computation purposes here"
+		got := RecallHitRate([]string{"D1:1", "D1:2"}, sample, retrieved)
+		if math.Abs(got-1.0) > 1e-9 {
+			t.Errorf("RecallHitRate all found = %.4f, want 1.0", got)
+		}
+	})
+
+	t.Run("partial evidence found", func(t *testing.T) {
+		retrieved := "hello world this is a test message with enough length"
+		got := RecallHitRate([]string{"D1:1", "D1:2"}, sample, retrieved)
+		if math.Abs(got-0.5) > 1e-9 {
+			t.Errorf("RecallHitRate partial = %.4f, want 0.5", got)
+		}
+	})
+
+	t.Run("no evidence required", func(t *testing.T) {
+		got := RecallHitRate(nil, sample, "anything")
+		if got != 1.0 {
+			t.Errorf("RecallHitRate no evidence = %.4f, want 1.0", got)
+		}
+	})
+
+	t.Run("missing turn excluded from denominator", func(t *testing.T) {
+		// D1:1 is found, D99:1 does not exist in sample
+		// Should only count resolvable turns in denominator
+		retrieved := "hello world this is a test message with enough length"
+		got := RecallHitRate([]string{"D1:1", "D99:1"}, sample, retrieved)
+		if math.Abs(got-1.0) > 1e-9 {
+			t.Errorf("RecallHitRate missing turn = %.4f, want 1.0 (unresolvable excluded)", got)
+		}
+	})
+}
+
+func TestExtractKeywords(t *testing.T) {
+	tests := []struct {
+		name  string
+		input string
+		want  []string
+	}{
+		{"simple", "What is the capital of France", []string{"capital", "france"}},
+		{
+			"stops removed",
+			"Who is the president of the United States",
+			[]string{"president", "united", "states"},
+		},
+		{
+			"max 6 keywords",
+			"one two three four five six seven eight nine ten",
+			[]string{"one", "two", "three", "four", "five", "six"},
+		},
+		{"short words filtered", "I am a go to the store", []string{"am", "store"}},
+		{"empty", "", nil},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := ExtractKeywords(tt.input)
+			if len(got) != len(tt.want) {
+				t.Fatalf("ExtractKeywords(%q) = %v (len %d), want %v (len %d)",
+					tt.input, got, len(got), tt.want, len(tt.want))
+			}
+			for i := range got {
+				if got[i] != tt.want[i] {
+					t.Errorf("[%d] = %q, want %q", i, got[i], tt.want[i])
+				}
+			}
+		})
+	}
+}

go.mod

@@ -84,6 +84,7 @@ require (
 	github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 // indirect
 	github.com/pion/randutil v0.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/reiver/go-porterstemmer v1.0.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/segmentio/asm v1.1.3 // indirect

go.sum

@@ -214,6 +214,8 @@ github.com/pion/webrtc/v3 v3.3.6/go.mod h1:zyN7th4mZpV27eXybfR/cnUf3J2DRy8zw/mdj
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/reiver/go-porterstemmer v1.0.1 h1:WyERBkASXgoXrTwq/IQ6wyNj/YG7j/ZURvTuMCoud5w=
+github.com/reiver/go-porterstemmer v1.0.1/go.mod h1:Z8uL/f/7UEwaeAJNwx1sO8kbqXiEuQieNuD735hLrSU=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/tview v0.42.0 h1:b/ftp+RxtDsHSaynXTbJb+/n/BxDEi+W3UfF5jILK6c=