From 7fa1ef6f270e88c531b84074403bc2bd9f0b518f Mon Sep 17 00:00:00 2001
From: Diego Fornalha <diegofornalha@gmail.com>
Date: Sun, 5 Apr 2026 16:15:03 -0300
Subject: [PATCH] fix: tolerate whitespace in split marker from LLM output

LLMs sometimes generate <| [SPLIT] |> instead of <|[SPLIT]|>.
Normalize with regex before splitting so messages are divided correctly.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 pkg/channels/marker.go      | 10 +++++++++-
 pkg/channels/marker_test.go | 13 +++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/pkg/channels/marker.go b/pkg/channels/marker.go
index 4801e3d27..3c2ff143b 100644
--- a/pkg/channels/marker.go
+++ b/pkg/channels/marker.go
@@ -7,6 +7,7 @@
 package channels
 
 import (
+	"regexp"
 	"strings"
 )
 
@@ -15,14 +16,21 @@ import (
 // and send each part as a separate message.
 const MessageSplitMarker = "<|[SPLIT]|>"
 
+// splitMarkerRe matches the split marker with optional spaces (LLMs sometimes add them).
+// Matches: <|[SPLIT]|>  <| [SPLIT] |>  <|[ SPLIT ]|>  etc.
+var splitMarkerRe = regexp.MustCompile(`<\|\s*\[\s*SPLIT\s*\]\s*\|>`)
+
 // SplitByMarker splits a message by the MessageSplitMarker and returns the parts.
 // Empty parts (including from consecutive markers) are filtered out.
 // If no marker is found, returns a single-element slice containing the original content.
+// Tolerates whitespace variations in the marker that LLMs may produce.
 func SplitByMarker(content string) []string {
 	if content == "" {
 		return nil
 	}
-	parts := strings.Split(content, MessageSplitMarker)
+	// Normalize any whitespace variations to the canonical marker
+	normalized := splitMarkerRe.ReplaceAllString(content, MessageSplitMarker)
+	parts := strings.Split(normalized, MessageSplitMarker)
 	result := make([]string, 0, len(parts))
 	for _, part := range parts {
 		trimmed := strings.TrimSpace(part)
diff --git a/pkg/channels/marker_test.go b/pkg/channels/marker_test.go
index b7b4ca99e..b8bb6cb65 100644
--- a/pkg/channels/marker_test.go
+++ b/pkg/channels/marker_test.go
@@ -125,6 +125,19 @@ func TestMarkerAndLengthSplitIntegration(t *testing.T) {
 	}
 }
 
+func TestSplitByMarker_SpacedVariant(t *testing.T) {
+	// LLMs sometimes generate the marker with extra spaces
+	content := "Part1 <| [SPLIT] |> Part2 <|[ SPLIT ]|> Part3"
+	chunks := SplitByMarker(content)
+
+	if len(chunks) != 3 {
+		t.Fatalf("Expected 3 chunks, got %d: %q", len(chunks), chunks)
+	}
+	if chunks[0] != "Part1" || chunks[1] != "Part2" || chunks[2] != "Part3" {
+		t.Errorf("Spaced markers not handled: %q", chunks)
+	}
+}
+
 // TestMarkerSplitPreservesCodeBlockIntegrity tests that marker split preserves code block boundaries
 func TestMarkerSplitPreservesCodeBlockIntegrity(t *testing.T) {
 	content := "Hello <|[SPLIT]|>```go\npackage main\n```<|[SPLIT]|>World"