Simplify token scanning and streaming logic in chat processing

- Removed unnecessary breakpoint and token tracking variables
- Streamlined token scanning and message processing in assistant API
- Simplified text handling and token parsing in contents scanning
- Reduced complexity of stream chat message generation

neo/assistant/api.go

@@ -321,7 +321,6 @@ func (ast *Assistant) streamChat(
 				msg.AppendTo(contents) // Append content and send message
 
 				// Scan the tokens
-				breakpoint := false
 				contents.ScanTokens(func(token string, begin bool, text string, tails string) {
 					msg.Type = token
 					msg.Text = ""                                    // clear the text
@@ -332,12 +331,6 @@ func (ast *Assistant) streamChat(
 						return
 					}
 
-					// Ignore the end of the token
-					if !begin && tails == "" {
-						breakpoint = true
-						return
-					}
-
 					// New message with the tails
 					newMsg, err := chatMessage.NewString(tails)
 					if err != nil {
@@ -346,11 +339,6 @@ func (ast *Assistant) streamChat(
 					messages = append(messages, *newMsg)
 				})
 
-				// If the breakpoint is true, continue the stream
-				if breakpoint {
-					return 1 // continue
-				}
-
 				// Handle stream
 				res, err := ast.HookStream(c, ctx, messages, msg, contents)
 				if err == nil && res != nil {

neo/message/contents.go

@@ -61,9 +61,6 @@ func (c *Contents) ScanTokens(cb func(token string, begin bool, text string, tai
 			if index > 0 {
 				tails = text[index+len(token[1]):]
 			}
-
-			text = strings.TrimLeft(text[len(token[0]):index], "\n")
-			c.Data[c.Current].Bytes = []byte(text)
 			c.UpdateType(c.token, map[string]interface{}{"text": text})
 			c.NewText([]byte(tails)) // Create new text with the tails
 			cb(c.token, false, text, tails)
@@ -80,7 +77,6 @@ func (c *Contents) ScanTokens(cb func(token string, begin bool, text string, tai
 	for name, token := range tokens {
 		if index := strings.Index(text, token[0]); index >= 0 {
 			c.token = name
-			text = strings.TrimSpace(text[index+len(token[0]):])
 			cb(name, true, text, "") // call the callback
 		}
 	}