fix: tolerate whitespace in split marker from LLM output
LLMs sometimes generate <| [SPLIT] |> instead of <|[SPLIT]|>. Normalize with regex before splitting so messages are divided correctly. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
e96f6d5ea2
commit
7fa1ef6f27
2 changed files with 22 additions and 1 deletions
|
|
@ -7,6 +7,7 @@
|
|||
package channels
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
|
|
@ -15,14 +16,21 @@ import (
|
|||
// and send each part as a separate message.
|
||||
const MessageSplitMarker = "<|[SPLIT]|>"
|
||||
|
||||
// splitMarkerRe matches the split marker with optional spaces (LLMs sometimes add them).
|
||||
// Matches: <|[SPLIT]|> <| [SPLIT] |> <|[ SPLIT ]|> etc.
|
||||
var splitMarkerRe = regexp.MustCompile(`<\|\s*\[\s*SPLIT\s*\]\s*\|>`)
|
||||
|
||||
// SplitByMarker splits a message by the MessageSplitMarker and returns the parts.
|
||||
// Empty parts (including from consecutive markers) are filtered out.
|
||||
// If no marker is found, returns a single-element slice containing the original content.
|
||||
// Tolerates whitespace variations in the marker that LLMs may produce.
|
||||
func SplitByMarker(content string) []string {
|
||||
if content == "" {
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(content, MessageSplitMarker)
|
||||
// Normalize any whitespace variations to the canonical marker
|
||||
normalized := splitMarkerRe.ReplaceAllString(content, MessageSplitMarker)
|
||||
parts := strings.Split(normalized, MessageSplitMarker)
|
||||
result := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
trimmed := strings.TrimSpace(part)
|
||||
|
|
|
|||
|
|
@ -125,6 +125,19 @@ func TestMarkerAndLengthSplitIntegration(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestSplitByMarker_SpacedVariant(t *testing.T) {
|
||||
// LLMs sometimes generate the marker with extra spaces
|
||||
content := "Part1 <| [SPLIT] |> Part2 <|[ SPLIT ]|> Part3"
|
||||
chunks := SplitByMarker(content)
|
||||
|
||||
if len(chunks) != 3 {
|
||||
t.Fatalf("Expected 3 chunks, got %d: %q", len(chunks), chunks)
|
||||
}
|
||||
if chunks[0] != "Part1" || chunks[1] != "Part2" || chunks[2] != "Part3" {
|
||||
t.Errorf("Spaced markers not handled: %q", chunks)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMarkerSplitPreservesCodeBlockIntegrity tests that marker split preserves code block boundaries
|
||||
func TestMarkerSplitPreservesCodeBlockIntegrity(t *testing.T) {
|
||||
content := "Hello <|[SPLIT]|>```go\npackage main\n```<|[SPLIT]|>World"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue