fix: address Copilot review round 3
- ValidF1Count=0 when all scores are sentinel (no forced =1) - Backward compat: old eval JSON without ValidF1Count falls back to TotalQuestions in computeModeAgg - Skip empty section in PrintComparison when tokenResults is empty - Update --api-base flag help to document /v1 default and version path - Add sentinel aggregation unit tests (partial, all, weighted)
This commit is contained in:
BeaconCat
parent
0408bf84b9
commit
baf389786d
3 changed files with 103 additions and 11 deletions
|
|
@ -236,9 +236,6 @@ func aggregateMetrics(qaResults []QAResult) AggMetrics {
|
|||
if nHit == 0 {
|
||||
nHit = 1
|
||||
}
|
||||
if validF1Count == 0 {
|
||||
validF1Count = 1
|
||||
}
|
||||
byCat := map[int]*CatMetrics{}
|
||||
for cat, acc := range byCatAcc {
|
||||
cm := &CatMetrics{
|
||||
|
|
@ -253,8 +250,12 @@ func aggregateMetrics(qaResults []QAResult) AggMetrics {
|
|||
}
|
||||
byCat[cat] = cm
|
||||
}
|
||||
var overallF1 float64
|
||||
if validF1Count > 0 {
|
||||
overallF1 = totalF1 / float64(validF1Count)
|
||||
}
|
||||
return AggMetrics{
|
||||
OverallF1: totalF1 / float64(validF1Count),
|
||||
OverallF1: overallF1,
|
||||
OverallHitRate: totalHitRate / float64(nHit),
|
||||
ByCategory: byCat,
|
||||
TotalQuestions: len(qaResults),
|
||||
|
|
@ -304,20 +305,29 @@ func SaveAggregated(results []EvalResult, outDir string) error {
|
|||
func computeModeAgg(results []EvalResult) AggMetrics {
|
||||
agg := AggMetrics{ByCategory: map[int]*CatMetrics{}}
|
||||
for _, r := range results {
|
||||
agg.OverallF1 += r.Agg.OverallF1 * float64(r.Agg.ValidF1Count)
|
||||
// Backward compat: old eval JSON without ValidF1Count → use TotalQuestions.
|
||||
vf1 := r.Agg.ValidF1Count
|
||||
if vf1 == 0 && r.Agg.TotalQuestions > 0 {
|
||||
vf1 = r.Agg.TotalQuestions
|
||||
}
|
||||
agg.OverallF1 += r.Agg.OverallF1 * float64(vf1)
|
||||
agg.OverallHitRate += r.Agg.OverallHitRate * float64(r.Agg.TotalQuestions)
|
||||
agg.TotalQuestions += r.Agg.TotalQuestions
|
||||
agg.ValidF1Count += r.Agg.ValidF1Count
|
||||
agg.ValidF1Count += vf1
|
||||
for cat, cm := range r.Agg.ByCategory {
|
||||
existing, ok := agg.ByCategory[cat]
|
||||
if !ok {
|
||||
existing = &CatMetrics{}
|
||||
agg.ByCategory[cat] = existing
|
||||
}
|
||||
existing.F1 += cm.F1 * float64(cm.ValidF1Count)
|
||||
cvf1 := cm.ValidF1Count
|
||||
if cvf1 == 0 && cm.QuestionCount > 0 {
|
||||
cvf1 = cm.QuestionCount
|
||||
}
|
||||
existing.F1 += cm.F1 * float64(cvf1)
|
||||
existing.HitRate += cm.HitRate * float64(cm.QuestionCount)
|
||||
existing.QuestionCount += cm.QuestionCount
|
||||
existing.ValidF1Count += cm.ValidF1Count
|
||||
existing.ValidF1Count += cvf1
|
||||
}
|
||||
}
|
||||
if agg.ValidF1Count > 0 {
|
||||
|
|
@ -392,7 +402,9 @@ func printSection(title string, results []EvalResult) {
|
|||
|
||||
// PrintComparison outputs a human-readable comparison table to stdout.
|
||||
func PrintComparison(results []EvalResult, llmResults []EvalResult) {
|
||||
if len(results) > 0 {
|
||||
printSection("No LLM generation", results)
|
||||
}
|
||||
if len(llmResults) > 0 {
|
||||
printSection("With LLM", llmResults)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -102,3 +102,81 @@ func TestComputeModeAgg(t *testing.T) {
|
|||
t.Errorf("TotalQuestions = %d, want 10", got.TotalQuestions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateMetricsSentinel(t *testing.T) {
|
||||
qa := []QAResult{
|
||||
{Category: 1, TokenF1: 0.8, HitRate: 0.5},
|
||||
{Category: 1, TokenF1: -1.0, HitRate: 0.3},
|
||||
{Category: 1, TokenF1: 0.4, HitRate: 0.7},
|
||||
}
|
||||
agg := aggregateMetrics(qa)
|
||||
|
||||
if agg.ValidF1Count != 2 {
|
||||
t.Errorf("ValidF1Count = %d, want 2", agg.ValidF1Count)
|
||||
}
|
||||
if agg.TotalQuestions != 3 {
|
||||
t.Errorf("TotalQuestions = %d, want 3", agg.TotalQuestions)
|
||||
}
|
||||
wantF1 := (0.8 + 0.4) / 2.0
|
||||
if math.Abs(agg.OverallF1-wantF1) > 1e-9 {
|
||||
t.Errorf("OverallF1 = %.6f, want %.6f", agg.OverallF1, wantF1)
|
||||
}
|
||||
wantHR := (0.5 + 0.3 + 0.7) / 3.0
|
||||
if math.Abs(agg.OverallHitRate-wantHR) > 1e-9 {
|
||||
t.Errorf("OverallHitRate = %.6f, want %.6f", agg.OverallHitRate, wantHR)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateMetricsAllSentinel(t *testing.T) {
|
||||
qa := []QAResult{
|
||||
{Category: 1, TokenF1: -1.0, HitRate: 0.5},
|
||||
{Category: 1, TokenF1: -1.0, HitRate: 0.3},
|
||||
}
|
||||
agg := aggregateMetrics(qa)
|
||||
|
||||
if agg.ValidF1Count != 0 {
|
||||
t.Errorf("ValidF1Count = %d, want 0", agg.ValidF1Count)
|
||||
}
|
||||
if agg.OverallF1 != 0 {
|
||||
t.Errorf("OverallF1 = %.6f, want 0", agg.OverallF1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeModeAggSentinelWeighting(t *testing.T) {
|
||||
results := []EvalResult{
|
||||
{
|
||||
Mode: "test",
|
||||
SampleID: "s1",
|
||||
QAResults: []QAResult{
|
||||
{Category: 1, TokenF1: 0.8, HitRate: 0.5},
|
||||
{Category: 1, TokenF1: -1.0, HitRate: 0.3},
|
||||
},
|
||||
},
|
||||
{
|
||||
Mode: "test",
|
||||
SampleID: "s2",
|
||||
QAResults: []QAResult{
|
||||
{Category: 1, TokenF1: 0.4, HitRate: 0.6},
|
||||
{Category: 1, TokenF1: 0.6, HitRate: 0.8},
|
||||
},
|
||||
},
|
||||
}
|
||||
for i := range results {
|
||||
results[i].Agg = aggregateMetrics(results[i].QAResults)
|
||||
}
|
||||
|
||||
got := computeModeAgg(results)
|
||||
|
||||
// s1: ValidF1Count=1, F1=0.8; s2: ValidF1Count=2, F1=0.5
|
||||
// Weighted: (0.8*1 + 0.5*2) / 3 = 1.8/3 = 0.6
|
||||
wantF1 := 0.6
|
||||
if math.Abs(got.OverallF1-wantF1) > 1e-9 {
|
||||
t.Errorf("OverallF1 = %.6f, want %.6f", got.OverallF1, wantF1)
|
||||
}
|
||||
if got.ValidF1Count != 3 {
|
||||
t.Errorf("ValidF1Count = %d, want 3", got.ValidF1Count)
|
||||
}
|
||||
if got.TotalQuestions != 4 {
|
||||
t.Errorf("TotalQuestions = %d, want 4", got.TotalQuestions)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,7 +58,8 @@ func main() {
|
|||
evalCmd.Flags().IntVar(&flagBudget, "budget", 4000, "token budget for retrieval")
|
||||
evalCmd.Flags().
|
||||
StringVar(&flagEvalMode, "eval-mode", "token", "evaluation mode: token (direct match) or llm (LLM-as-Judge)")
|
||||
evalCmd.Flags().StringVar(&flagAPIBase, "api-base", "", "OpenAI-compatible API base URL (env: MEMBENCH_API_BASE)")
|
||||
evalCmd.Flags().
|
||||
StringVar(&flagAPIBase, "api-base", "", "API base URL with version path, e.g. http://host/v1 (default: http://127.0.0.1:8080/v1, env: MEMBENCH_API_BASE)")
|
||||
evalCmd.Flags().StringVar(&flagAPIKey, "api-key", "", "API key for the LLM endpoint (env: MEMBENCH_API_KEY)")
|
||||
evalCmd.Flags().StringVar(&flagModel, "model", "", "model name for LLM eval (env: MEMBENCH_MODEL)")
|
||||
evalCmd.Flags().
|
||||
|
|
@ -84,7 +85,8 @@ func main() {
|
|||
runCmd.Flags().IntVar(&flagBudget, "budget", 4000, "token budget for retrieval")
|
||||
runCmd.Flags().
|
||||
StringVar(&flagEvalMode, "eval-mode", "token", "evaluation mode: token (direct match) or llm (LLM-as-Judge)")
|
||||
runCmd.Flags().StringVar(&flagAPIBase, "api-base", "", "OpenAI-compatible API base URL (env: MEMBENCH_API_BASE)")
|
||||
runCmd.Flags().
|
||||
StringVar(&flagAPIBase, "api-base", "", "API base URL with version path, e.g. http://host/v1 (default: http://127.0.0.1:8080/v1, env: MEMBENCH_API_BASE)")
|
||||
runCmd.Flags().StringVar(&flagAPIKey, "api-key", "", "API key for the LLM endpoint (env: MEMBENCH_API_KEY)")
|
||||
runCmd.Flags().StringVar(&flagModel, "model", "", "model name for LLM eval (env: MEMBENCH_MODEL)")
|
||||
runCmd.Flags().
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue