From 991c884d7857a7c6d7f7fc4df6e621edb6390380 Mon Sep 17 00:00:00 2001
From: Max <shadow.iqka@qq.com>
Date: Mon, 3 Feb 2025 19:24:52 +0800
Subject: [PATCH] Simplify token scanning and streaming logic in chat
 processing

- Removed unnecessary breakpoint and token tracking variables
- Streamlined token scanning and message processing in assistant API
- Simplified text handling and token parsing in contents scanning
- Reduced complexity of stream chat message generation
---
 neo/assistant/api.go    | 12 ------------
 neo/message/contents.go |  4 ----
 2 files changed, 16 deletions(-)

diff --git a/neo/assistant/api.go b/neo/assistant/api.go
index b04ae7e0..1c16d32d 100644
--- a/neo/assistant/api.go
+++ b/neo/assistant/api.go
@@ -321,7 +321,6 @@ func (ast *Assistant) streamChat(
 				msg.AppendTo(contents) // Append content and send message
 
 				// Scan the tokens
-				breakpoint := false
 				contents.ScanTokens(func(token string, begin bool, text string, tails string) {
 					msg.Type = token
 					msg.Text = ""                                    // clear the text
@@ -332,12 +331,6 @@ func (ast *Assistant) streamChat(
 						return
 					}
 
-					// Ignore the end of the token
-					if !begin && tails == "" {
-						breakpoint = true
-						return
-					}
-
 					// New message with the tails
 					newMsg, err := chatMessage.NewString(tails)
 					if err != nil {
@@ -346,11 +339,6 @@ func (ast *Assistant) streamChat(
 					messages = append(messages, *newMsg)
 				})
 
-				// If the breakpoint is true, continue the stream
-				if breakpoint {
-					return 1 // continue
-				}
-
 				// Handle stream
 				res, err := ast.HookStream(c, ctx, messages, msg, contents)
 				if err == nil && res != nil {
diff --git a/neo/message/contents.go b/neo/message/contents.go
index eb0f8788..fff3c847 100644
--- a/neo/message/contents.go
+++ b/neo/message/contents.go
@@ -61,9 +61,6 @@ func (c *Contents) ScanTokens(cb func(token string, begin bool, text string, tai
 			if index > 0 {
 				tails = text[index+len(token[1]):]
 			}
-
-			text = strings.TrimLeft(text[len(token[0]):index], "\n")
-			c.Data[c.Current].Bytes = []byte(text)
 			c.UpdateType(c.token, map[string]interface{}{"text": text})
 			c.NewText([]byte(tails)) // Create new text with the tails
 			cb(c.token, false, text, tails)
@@ -80,7 +77,6 @@ func (c *Contents) ScanTokens(cb func(token string, begin bool, text string, tai
 	for name, token := range tokens {
 		if index := strings.Index(text, token[0]); index >= 0 {
 			c.token = name
-			text = strings.TrimSpace(text[index+len(token[0]):])
 			cb(name, true, text, "") // call the callback
 		}
 	}