From 90a57642b0ed971aa4b44b1371f444c5f6f145ad Mon Sep 17 00:00:00 2001 From: BeaconCat Date: Mon, 13 Apr 2026 10:25:22 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20address=20review=20P1+P2=20=E2=80=94=20s?= =?UTF-8?q?ort=20alignment,=20failure=20sentinel,=20score=20parser?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - P1: Replace hand-rolled sortByRank with sort.Slice (ascending, best first) matching eval.go's EvalSeahorse — ensures BudgetTruncate keeps best-ranked messages when truncation occurs - P2: Use -1.0 sentinel for LLM API failures and parse errors, distinct from genuine 0.0 score; aggregateMetrics skips -1.0 entries for F1 averaging while still counting HitRate - P2: Use regexp \b([1-5])\b for judge score extraction instead of first-digit scan — avoids misparses on '5/5', 'Score: 3' etc. --- cmd/membench/eval.go | 67 ++++++++++++++++++++++++++-------------- cmd/membench/eval_llm.go | 54 ++++++++++++++------------------ 2 files changed, 67 insertions(+), 54 deletions(-) diff --git a/cmd/membench/eval.go b/cmd/membench/eval.go index bddee76fd..8e7257b97 100644 --- a/cmd/membench/eval.go +++ b/cmd/membench/eval.go @@ -201,38 +201,59 @@ func EvalSeahorse( // aggregateMetrics computes overall and per-category metrics. func aggregateMetrics(qaResults []QAResult) AggMetrics { - byCat := map[int]*CatMetrics{} + type catAccum struct { + f1Sum float64 + f1Count int + hitRateSum float64 + hitRateCount int + } + byCatAcc := map[int]*catAccum{} totalF1 := 0.0 totalHitRate := 0.0 + validF1Count := 0 for _, qr := range qaResults { - totalF1 += qr.TokenF1 - totalHitRate += qr.HitRate - cat, ok := byCat[qr.Category] - if !ok { - cat = &CatMetrics{} - byCat[qr.Category] = cat + // Skip sentinel -1.0 scores (LLM API/parse failures) from F1 averaging. + if qr.TokenF1 >= 0 { + totalF1 += qr.TokenF1 + validF1Count++ } - cat.F1 += qr.TokenF1 - cat.HitRate += qr.HitRate - cat.QuestionCount++ + totalHitRate += qr.HitRate + acc, ok := byCatAcc[qr.Category] + if !ok { + acc = &catAccum{} + byCatAcc[qr.Category] = acc + } + if qr.TokenF1 >= 0 { + acc.f1Sum += qr.TokenF1 + acc.f1Count++ + } + acc.hitRateSum += qr.HitRate + acc.hitRateCount++ } - n := len(qaResults) - if n == 0 { - n = 1 + nHit := len(qaResults) + if nHit == 0 { + nHit = 1 } - agg := AggMetrics{ - OverallF1: totalF1 / float64(n), - OverallHitRate: totalHitRate / float64(n), + if validF1Count == 0 { + validF1Count = 1 + } + byCat := map[int]*CatMetrics{} + for cat, acc := range byCatAcc { + cm := &CatMetrics{QuestionCount: acc.hitRateCount} + if acc.f1Count > 0 { + cm.F1 = acc.f1Sum / float64(acc.f1Count) + } + if acc.hitRateCount > 0 { + cm.HitRate = acc.hitRateSum / float64(acc.hitRateCount) + } + byCat[cat] = cm + } + return AggMetrics{ + OverallF1: totalF1 / float64(validF1Count), + OverallHitRate: totalHitRate / float64(nHit), ByCategory: byCat, TotalQuestions: len(qaResults), } - for _, cat := range agg.ByCategory { - if cat.QuestionCount > 0 { - cat.F1 /= float64(cat.QuestionCount) - cat.HitRate /= float64(cat.QuestionCount) - } - } - return agg } // SaveResults writes per-sample eval results to JSON files. diff --git a/cmd/membench/eval_llm.go b/cmd/membench/eval_llm.go index 7092c2bef..34022830b 100644 --- a/cmd/membench/eval_llm.go +++ b/cmd/membench/eval_llm.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "log" + "regexp" + "sort" "strconv" "strings" @@ -37,8 +39,11 @@ func generateAnswer(ctx context.Context, client *LLMClient, contextText, questio return client.Complete(ctx, answerSystemPrompt, userPrompt) } +// scoreRe matches the first standalone integer 1-5 in the judge response. +var scoreRe = regexp.MustCompile(`\b([1-5])\b`) + // judgeAnswer asks the LLM to score the candidate answer vs the gold answer. -// Returns a score from 0.0 to 1.0. +// Returns a score from 0.0 to 1.0, or -1.0 on parse failure. func judgeAnswer( ctx context.Context, client *LLMClient, @@ -51,20 +56,16 @@ func judgeAnswer( response, err := client.Complete(ctx, judgeSystemPrompt, userPrompt) if err != nil { - return 0.0, err + return -1.0, err } - // Parse score from response response = strings.TrimSpace(response) - // Extract first digit found - for _, ch := range response { - if ch >= '1' && ch <= '5' { - score, _ := strconv.Atoi(string(ch)) - return float64(score-1) / 4.0, nil // Normalize 1-5 to 0.0-1.0 - } + if m := scoreRe.FindStringSubmatch(response); len(m) == 2 { + score, _ := strconv.Atoi(m[1]) + return float64(score-1) / 4.0, nil // Normalize 1-5 to 0.0-1.0 } - log.Printf("WARNING: could not parse judge score from: %q, defaulting to 0.0", response) - return 0.0, nil + log.Printf("WARNING: could not parse judge score from: %q, returning -1", response) + return -1.0, nil } // EvalLegacyLLM evaluates legacy store using LLM generation + LLM-as-Judge. @@ -101,8 +102,8 @@ func EvalLegacyLLM( llmAnswer = "" } - // Judge the answer - score := 0.0 + // Judge the answer; -1.0 = API/parse failure. + score := -1.0 if llmAnswer != "" { score, err = judgeAnswer(ctx, client, qa.Question, qa.AnswerString(), llmAnswer) if err != nil { @@ -187,7 +188,11 @@ func EvalSeahorseLLM( for id := range bestRank { messageIDs = append(messageIDs, id) } - sortByRank(messageIDs, bestRank) + // Sort ascending: best (most-negative) rank first. + // BudgetTruncate walks front-to-back, so best-ranked messages are kept. + sort.Slice(messageIDs, func(i, j int) bool { + return bestRank[messageIDs[i]] < bestRank[messageIDs[j]] + }) var contentParts []string if len(messageIDs) > 0 { @@ -206,10 +211,10 @@ func EvalSeahorseLLM( Question: qa.Question, Category: qa.Category, GoldAnswer: qa.AnswerString(), - TokenF1: 0.0, + TokenF1: -1.0, HitRate: 0.0, }) - log.Printf("[seahorse-llm] sample=%s q=%d/%d score=0.00 answer=(no context)", + log.Printf("[seahorse-llm] sample=%s q=%d/%d score=-1.00 answer=(no context)", sample.SampleID, total, totalQA) continue } @@ -219,7 +224,7 @@ func EvalSeahorseLLM( // Generate answer with LLM llmAnswer := "" - score := 0.0 + score := -1.0 if contextText != "" { var err error llmAnswer, err = generateAnswer(ctx, client, contextText, qa.Question) @@ -228,7 +233,7 @@ func EvalSeahorseLLM( } } - // Judge the answer + // Judge the answer; -1.0 = API/parse failure. if llmAnswer != "" { var err error score, err = judgeAnswer(ctx, client, qa.Question, qa.AnswerString(), llmAnswer) @@ -276,16 +281,3 @@ func truncateStr(s string, maxLen int) string { } return s } - -// sortByRank sorts message IDs by BM25 rank (more negative = better). -func sortByRank(ids []int64, ranks map[int64]float64) { - for i := 1; i < len(ids); i++ { - key := ids[i] - j := i - 1 - for j >= 0 && ranks[ids[j]] > ranks[key] { - ids[j+1] = ids[j] - j-- - } - ids[j+1] = key - } -}