Update JSON Extraction Method in Assistants

- Replaced `json.Parse` with `text.ExtractJSON` in the `Next` function of both the `keyword` and `needsearch` assistants for improved fault-tolerant JSON extraction from LLM output.
- Simplified content handling by removing unnecessary markdown code block processing, enhancing clarity and efficiency in keyword extraction.
- Updated comments to reflect the changes in the extraction method, ensuring better understanding of the functionality.

main.go

@@ -3,6 +3,7 @@ package main
 import (
 	_ "github.com/yaoapp/gou/diff"
 	_ "github.com/yaoapp/gou/encoding"
+	_ "github.com/yaoapp/gou/text"
 	_ "github.com/yaoapp/yao/aigc"
 	_ "github.com/yaoapp/yao/crypto"
 	_ "github.com/yaoapp/yao/excel"

yao/assistants/keyword/src/index.ts

@@ -7,7 +7,7 @@
 
 /**
  * Next hook - processes keyword extraction response
- * Uses json.Parse for fault-tolerant JSON parsing
+ * Uses text.ExtractJSON for fault-tolerant JSON extraction from LLM output
  */
 function Next(
   ctx: agent.Context,
@@ -20,24 +20,13 @@ function Next(
     return null;
   }
 
-  // Remove markdown code block if present
+  const content = completion.content;
-  let content = completion.content.trim();
-  if (content.startsWith("```json")) {
-    content = content.slice(7);
-  } else if (content.startsWith("```")) {
-    content = content.slice(3);
-  }
-  if (content.endsWith("```")) {
-    content = content.slice(0, -3);
-  }
-  content = content.trim();
-
-  // Try to parse JSON from completion content
   let keywords: string[] = [];
 
   try {
-    // Use json.Parse for fault-tolerant parsing (handles broken JSON, JSONC, etc.)
+    // Use text.ExtractJSON for fault-tolerant extraction
-    const parsed = Process("json.Parse", content) as {
+    // Handles markdown code blocks, broken JSON, etc.
+    const parsed = Process("text.ExtractJSON", content) as {
       keywords?: string[];
     } | null;
 
@@ -47,7 +36,7 @@ function Next(
       );
     }
   } catch (e) {
-    // If json.Parse fails, try to extract keywords from text
+    // If extraction fails, try to extract keywords from text
     keywords = extractKeywordsFromText(content);
   }
 

yao/assistants/needsearch/src/index.ts

@@ -13,7 +13,7 @@ interface SearchResult {
 
 /**
  * Next hook - processes search intent response
- * Uses json.Parse for fault-tolerant JSON parsing
+ * Uses text.ExtractJSON for fault-tolerant JSON extraction from LLM output
  */
 function Next(
   ctx: agent.Context,
@@ -26,17 +26,7 @@ function Next(
     return null;
   }
 
-  // Remove markdown code block if present
+  const content = completion.content;
-  let content = completion.content.trim();
-  if (content.startsWith("```json")) {
-    content = content.slice(7); // Remove ```json
-  } else if (content.startsWith("```")) {
-    content = content.slice(3); // Remove ```
-  }
-  if (content.endsWith("```")) {
-    content = content.slice(0, -3); // Remove trailing ```
-  }
-  content = content.trim();
 
   // Default result
   let result: SearchResult = {
@@ -46,8 +36,9 @@ function Next(
   };
 
   try {
-    // Use json.Parse for fault-tolerant parsing
+    // Use text.ExtractJSON for fault-tolerant extraction
-    const parsed = Process("json.Parse", content) as {
+    // Handles markdown code blocks, broken JSON, etc.
+    const parsed = Process("text.ExtractJSON", content) as {
       need_search?: boolean;
       search_types?: string[];
       confidence?: number;
@@ -68,7 +59,7 @@ function Next(
           : 0.5;
     }
   } catch (e) {
-    // If json.Parse fails, try to extract from text
+    // If extraction fails, try to extract from text
     result = extractFromText(content);
   }