diff --git a/cmd/membench/eval_llm.go b/cmd/membench/eval_llm.go index 430548003..7318b76bb 100644 --- a/cmd/membench/eval_llm.go +++ b/cmd/membench/eval_llm.go @@ -38,7 +38,11 @@ func generateAnswer(ctx context.Context, client *LLMClient, contextText, questio // judgeAnswer asks the LLM to score the candidate answer vs the gold answer. // Returns a score from 0.0 to 1.0. -func judgeAnswer(ctx context.Context, client *LLMClient, question, goldAnswer, candidateAnswer string) (float64, error) { +func judgeAnswer( + ctx context.Context, + client *LLMClient, + question, goldAnswer, candidateAnswer string, +) (float64, error) { userPrompt := fmt.Sprintf( "Question: %s\n\nReference Answer: %s\n\nCandidate Answer: %s\n\nScore:", question, goldAnswer, candidateAnswer, diff --git a/cmd/membench/llm_client.go b/cmd/membench/llm_client.go index 57734ae5b..cb7b1ee2a 100644 --- a/cmd/membench/llm_client.go +++ b/cmd/membench/llm_client.go @@ -46,11 +46,11 @@ func NewLLMClient(opts LLMClientOptions) *LLMClient { } type chatRequest struct { - Model string `json:"model"` - Messages []chatMessage `json:"messages"` - Temperature float64 `json:"temperature"` - MaxTokens int `json:"max_tokens"` - ChatTemplateKwargs map[string]interface{} `json:"chat_template_kwargs,omitempty"` + Model string `json:"model"` + Messages []chatMessage `json:"messages"` + Temperature float64 `json:"temperature"` + MaxTokens int `json:"max_tokens"` + ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty"` } type chatMessage struct { @@ -81,7 +81,7 @@ func (c *LLMClient) Complete(ctx context.Context, systemPrompt, userPrompt strin MaxTokens: 512, } if c.NoThinking { - body.ChatTemplateKwargs = map[string]interface{}{ + body.ChatTemplateKwargs = map[string]any{ "enable_thinking": false, } } diff --git a/cmd/membench/main.go b/cmd/membench/main.go index ff1970d52..c85105fb2 100644 --- a/cmd/membench/main.go +++ b/cmd/membench/main.go @@ -54,11 +54,13 @@ func main() { evalCmd.Flags().StringVar(&flagOut, "out", "./bench-out", "output working directory") evalCmd.Flags().StringVar(&flagMode, "mode", "all", "modes to evaluate: legacy, seahorse, or all") evalCmd.Flags().IntVar(&flagBudget, "budget", 4000, "token budget for retrieval") - evalCmd.Flags().StringVar(&flagEvalMode, "eval-mode", "token", "evaluation mode: token (direct match) or llm (LLM-as-Judge)") + evalCmd.Flags(). + StringVar(&flagEvalMode, "eval-mode", "token", "evaluation mode: token (direct match) or llm (LLM-as-Judge)") evalCmd.Flags().StringVar(&flagAPIBase, "api-base", "", "OpenAI-compatible API base URL (env: MEMBENCH_API_BASE)") evalCmd.Flags().StringVar(&flagAPIKey, "api-key", "", "API key for the LLM endpoint (env: MEMBENCH_API_KEY)") evalCmd.Flags().StringVar(&flagModel, "model", "", "model name for LLM eval (env: MEMBENCH_MODEL)") - evalCmd.Flags().BoolVar(&flagNoThinking, "no-thinking", false, "disable thinking mode via chat_template_kwargs (llama.cpp + Qwen)") + evalCmd.Flags(). + BoolVar(&flagNoThinking, "no-thinking", false, "disable thinking mode via chat_template_kwargs (llama.cpp + Qwen)") evalCmd.Flags().IntVar(&flagLimit, "limit", 0, "max QA questions per sample (0 = all)") reportCmd := &cobra.Command{ @@ -77,11 +79,13 @@ func main() { runCmd.Flags().StringVar(&flagOut, "out", "./bench-out", "output working directory") runCmd.Flags().StringVar(&flagMode, "mode", "all", "modes to run: legacy, seahorse, or all") runCmd.Flags().IntVar(&flagBudget, "budget", 4000, "token budget for retrieval") - runCmd.Flags().StringVar(&flagEvalMode, "eval-mode", "token", "evaluation mode: token (direct match) or llm (LLM-as-Judge)") + runCmd.Flags(). + StringVar(&flagEvalMode, "eval-mode", "token", "evaluation mode: token (direct match) or llm (LLM-as-Judge)") runCmd.Flags().StringVar(&flagAPIBase, "api-base", "", "OpenAI-compatible API base URL (env: MEMBENCH_API_BASE)") runCmd.Flags().StringVar(&flagAPIKey, "api-key", "", "API key for the LLM endpoint (env: MEMBENCH_API_KEY)") runCmd.Flags().StringVar(&flagModel, "model", "", "model name for LLM eval (env: MEMBENCH_MODEL)") - runCmd.Flags().BoolVar(&flagNoThinking, "no-thinking", false, "disable thinking mode via chat_template_kwargs (llama.cpp + Qwen)") + runCmd.Flags(). + BoolVar(&flagNoThinking, "no-thinking", false, "disable thinking mode via chat_template_kwargs (llama.cpp + Qwen)") runCmd.Flags().IntVar(&flagLimit, "limit", 0, "max QA questions per sample (0 = all)") rootCmd.AddCommand(ingestCmd, evalCmd, reportCmd, runCmd)