fix: address review P1+P2 — sort alignment, failure sentinel, score parser

- P1: Replace hand-rolled sortByRank with sort.Slice (ascending, best
  first) matching eval.go's EvalSeahorse — ensures BudgetTruncate keeps
  best-ranked messages when truncation occurs
- P2: Use -1.0 sentinel for LLM API failures and parse errors, distinct
  from genuine 0.0 score; aggregateMetrics skips -1.0 entries for F1
  averaging while still counting HitRate
- P2: Use regexp \b([1-5])\b for judge score extraction instead of
  first-digit scan — avoids misparses on '5/5', 'Score: 3' etc.

cmd/membench/eval.go

@@ -201,38 +201,59 @@ func EvalSeahorse(
 
 // aggregateMetrics computes overall and per-category metrics.
 func aggregateMetrics(qaResults []QAResult) AggMetrics {
-	byCat := map[int]*CatMetrics{}
+	type catAccum struct {
+		f1Sum        float64
+		f1Count      int
+		hitRateSum   float64
+		hitRateCount int
+	}
+	byCatAcc := map[int]*catAccum{}
 	totalF1 := 0.0
 	totalHitRate := 0.0
+	validF1Count := 0
 	for _, qr := range qaResults {
-		totalF1 += qr.TokenF1
+		// Skip sentinel -1.0 scores (LLM API/parse failures) from F1 averaging.
-		totalHitRate += qr.HitRate
+		if qr.TokenF1 >= 0 {
-		cat, ok := byCat[qr.Category]
+			totalF1 += qr.TokenF1
-		if !ok {
+			validF1Count++
-			cat = &CatMetrics{}
-			byCat[qr.Category] = cat
 		}
-		cat.F1 += qr.TokenF1
+		totalHitRate += qr.HitRate
-		cat.HitRate += qr.HitRate
+		acc, ok := byCatAcc[qr.Category]
-		cat.QuestionCount++
+		if !ok {
+			acc = &catAccum{}
+			byCatAcc[qr.Category] = acc
+		}
+		if qr.TokenF1 >= 0 {
+			acc.f1Sum += qr.TokenF1
+			acc.f1Count++
+		}
+		acc.hitRateSum += qr.HitRate
+		acc.hitRateCount++
 	}
-	n := len(qaResults)
+	nHit := len(qaResults)
-	if n == 0 {
+	if nHit == 0 {
-		n = 1
+		nHit = 1
 	}
-	agg := AggMetrics{
+	if validF1Count == 0 {
-		OverallF1:      totalF1 / float64(n),
+		validF1Count = 1
-		OverallHitRate: totalHitRate / float64(n),
+	}
+	byCat := map[int]*CatMetrics{}
+	for cat, acc := range byCatAcc {
+		cm := &CatMetrics{QuestionCount: acc.hitRateCount}
+		if acc.f1Count > 0 {
+			cm.F1 = acc.f1Sum / float64(acc.f1Count)
+		}
+		if acc.hitRateCount > 0 {
+			cm.HitRate = acc.hitRateSum / float64(acc.hitRateCount)
+		}
+		byCat[cat] = cm
+	}
+	return AggMetrics{
+		OverallF1:      totalF1 / float64(validF1Count),
+		OverallHitRate: totalHitRate / float64(nHit),
 		ByCategory:     byCat,
 		TotalQuestions: len(qaResults),
 	}
-	for _, cat := range agg.ByCategory {
-		if cat.QuestionCount > 0 {
-			cat.F1 /= float64(cat.QuestionCount)
-			cat.HitRate /= float64(cat.QuestionCount)
-		}
-	}
-	return agg
 }
 
 // SaveResults writes per-sample eval results to JSON files.

cmd/membench/eval_llm.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -37,8 +39,11 @@ func generateAnswer(ctx context.Context, client *LLMClient, contextText, questio
 	return client.Complete(ctx, answerSystemPrompt, userPrompt)
 }
 
+// scoreRe matches the first standalone integer 1-5 in the judge response.
+var scoreRe = regexp.MustCompile(`\b([1-5])\b`)
+
 // judgeAnswer asks the LLM to score the candidate answer vs the gold answer.
-// Returns a score from 0.0 to 1.0.
+// Returns a score from 0.0 to 1.0, or -1.0 on parse failure.
 func judgeAnswer(
 	ctx context.Context,
 	client *LLMClient,
@@ -51,20 +56,16 @@ func judgeAnswer(
 
 	response, err := client.Complete(ctx, judgeSystemPrompt, userPrompt)
 	if err != nil {
-		return 0.0, err
+		return -1.0, err
 	}
 
-	// Parse score from response
 	response = strings.TrimSpace(response)
-	// Extract first digit found
+	if m := scoreRe.FindStringSubmatch(response); len(m) == 2 {
-	for _, ch := range response {
+		score, _ := strconv.Atoi(m[1])
-		if ch >= '1' && ch <= '5' {
+		return float64(score-1) / 4.0, nil // Normalize 1-5 to 0.0-1.0
-			score, _ := strconv.Atoi(string(ch))
-			return float64(score-1) / 4.0, nil // Normalize 1-5 to 0.0-1.0
-		}
 	}
-	log.Printf("WARNING: could not parse judge score from: %q, defaulting to 0.0", response)
+	log.Printf("WARNING: could not parse judge score from: %q, returning -1", response)
-	return 0.0, nil
+	return -1.0, nil
 }
 
 // EvalLegacyLLM evaluates legacy store using LLM generation + LLM-as-Judge.
@@ -101,8 +102,8 @@ func EvalLegacyLLM(
 				llmAnswer = ""
 			}
 
-			// Judge the answer
+			// Judge the answer; -1.0 = API/parse failure.
-			score := 0.0
+			score := -1.0
 			if llmAnswer != "" {
 				score, err = judgeAnswer(ctx, client, qa.Question, qa.AnswerString(), llmAnswer)
 				if err != nil {
@@ -187,7 +188,11 @@ func EvalSeahorseLLM(
 			for id := range bestRank {
 				messageIDs = append(messageIDs, id)
 			}
-			sortByRank(messageIDs, bestRank)
+			// Sort ascending: best (most-negative) rank first.
+			// BudgetTruncate walks front-to-back, so best-ranked messages are kept.
+			sort.Slice(messageIDs, func(i, j int) bool {
+				return bestRank[messageIDs[i]] < bestRank[messageIDs[j]]
+			})
 
 			var contentParts []string
 			if len(messageIDs) > 0 {
@@ -206,10 +211,10 @@ func EvalSeahorseLLM(
 					Question:   qa.Question,
 					Category:   qa.Category,
 					GoldAnswer: qa.AnswerString(),
-					TokenF1:    0.0,
+					TokenF1:    -1.0,
 					HitRate:    0.0,
 				})
-				log.Printf("[seahorse-llm] sample=%s q=%d/%d score=0.00 answer=(no context)",
+				log.Printf("[seahorse-llm] sample=%s q=%d/%d score=-1.00 answer=(no context)",
 					sample.SampleID, total, totalQA)
 				continue
 			}
@@ -219,7 +224,7 @@ func EvalSeahorseLLM(
 
 			// Generate answer with LLM
 			llmAnswer := ""
-			score := 0.0
+			score := -1.0
 			if contextText != "" {
 				var err error
 				llmAnswer, err = generateAnswer(ctx, client, contextText, qa.Question)
@@ -228,7 +233,7 @@ func EvalSeahorseLLM(
 				}
 			}
 
-			// Judge the answer
+			// Judge the answer; -1.0 = API/parse failure.
 			if llmAnswer != "" {
 				var err error
 				score, err = judgeAnswer(ctx, client, qa.Question, qa.AnswerString(), llmAnswer)
@@ -276,16 +281,3 @@ func truncateStr(s string, maxLen int) string {
 	}
 	return s
 }
-
-// sortByRank sorts message IDs by BM25 rank (more negative = better).
-func sortByRank(ids []int64, ranks map[int64]float64) {
-	for i := 1; i < len(ids); i++ {
-		key := ids[i]
-		j := i - 1
-		for j >= 0 && ranks[ids[j]] > ranks[key] {
-			ids[j+1] = ids[j]
-			j--
-		}
-		ids[j+1] = key
-	}
-}