fix: tolerate whitespace in split marker from LLM output

LLMs sometimes generate <| [SPLIT] |> instead of <|[SPLIT]|>.
Normalize with regex before splitting so messages are divided correctly.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Diego Fornalha 2026-04-05 16:15:03 -03:00
parent e96f6d5ea2
commit 7fa1ef6f27
2 changed files with 22 additions and 1 deletions

View file

@ -7,6 +7,7 @@
package channels
import (
"regexp"
"strings"
)
@ -15,14 +16,21 @@ import (
// and send each part as a separate message.
const MessageSplitMarker = "<|[SPLIT]|>"
// splitMarkerRe matches the split marker with optional spaces (LLMs sometimes add them).
// Matches: <|[SPLIT]|> <| [SPLIT] |> <|[ SPLIT ]|> etc.
var splitMarkerRe = regexp.MustCompile(`<\|\s*\[\s*SPLIT\s*\]\s*\|>`)
// SplitByMarker splits a message by the MessageSplitMarker and returns the parts.
// Empty parts (including from consecutive markers) are filtered out.
// If no marker is found, returns a single-element slice containing the original content.
// Tolerates whitespace variations in the marker that LLMs may produce.
func SplitByMarker(content string) []string {
if content == "" {
return nil
}
parts := strings.Split(content, MessageSplitMarker)
// Normalize any whitespace variations to the canonical marker
normalized := splitMarkerRe.ReplaceAllString(content, MessageSplitMarker)
parts := strings.Split(normalized, MessageSplitMarker)
result := make([]string, 0, len(parts))
for _, part := range parts {
trimmed := strings.TrimSpace(part)

View file

@ -125,6 +125,19 @@ func TestMarkerAndLengthSplitIntegration(t *testing.T) {
}
}
func TestSplitByMarker_SpacedVariant(t *testing.T) {
// LLMs sometimes generate the marker with extra spaces
content := "Part1 <| [SPLIT] |> Part2 <|[ SPLIT ]|> Part3"
chunks := SplitByMarker(content)
if len(chunks) != 3 {
t.Fatalf("Expected 3 chunks, got %d: %q", len(chunks), chunks)
}
if chunks[0] != "Part1" || chunks[1] != "Part2" || chunks[2] != "Part3" {
t.Errorf("Spaced markers not handled: %q", chunks)
}
}
// TestMarkerSplitPreservesCodeBlockIntegrity tests that marker split preserves code block boundaries
func TestMarkerSplitPreservesCodeBlockIntegrity(t *testing.T) {
content := "Hello <|[SPLIT]|>```go\npackage main\n```<|[SPLIT]|>World"