fix: tolerate whitespace in split marker from LLM output
LLMs sometimes generate <| [SPLIT] |> instead of <|[SPLIT]|>. Normalize with regex before splitting so messages are divided correctly. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
e96f6d5ea2
commit
7fa1ef6f27
2 changed files with 22 additions and 1 deletions
|
|
@ -7,6 +7,7 @@
|
||||||
package channels
|
package channels
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -15,14 +16,21 @@ import (
|
||||||
// and send each part as a separate message.
|
// and send each part as a separate message.
|
||||||
const MessageSplitMarker = "<|[SPLIT]|>"
|
const MessageSplitMarker = "<|[SPLIT]|>"
|
||||||
|
|
||||||
|
// splitMarkerRe matches the split marker with optional spaces (LLMs sometimes add them).
|
||||||
|
// Matches: <|[SPLIT]|> <| [SPLIT] |> <|[ SPLIT ]|> etc.
|
||||||
|
var splitMarkerRe = regexp.MustCompile(`<\|\s*\[\s*SPLIT\s*\]\s*\|>`)
|
||||||
|
|
||||||
// SplitByMarker splits a message by the MessageSplitMarker and returns the parts.
|
// SplitByMarker splits a message by the MessageSplitMarker and returns the parts.
|
||||||
// Empty parts (including from consecutive markers) are filtered out.
|
// Empty parts (including from consecutive markers) are filtered out.
|
||||||
// If no marker is found, returns a single-element slice containing the original content.
|
// If no marker is found, returns a single-element slice containing the original content.
|
||||||
|
// Tolerates whitespace variations in the marker that LLMs may produce.
|
||||||
func SplitByMarker(content string) []string {
|
func SplitByMarker(content string) []string {
|
||||||
if content == "" {
|
if content == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
parts := strings.Split(content, MessageSplitMarker)
|
// Normalize any whitespace variations to the canonical marker
|
||||||
|
normalized := splitMarkerRe.ReplaceAllString(content, MessageSplitMarker)
|
||||||
|
parts := strings.Split(normalized, MessageSplitMarker)
|
||||||
result := make([]string, 0, len(parts))
|
result := make([]string, 0, len(parts))
|
||||||
for _, part := range parts {
|
for _, part := range parts {
|
||||||
trimmed := strings.TrimSpace(part)
|
trimmed := strings.TrimSpace(part)
|
||||||
|
|
|
||||||
|
|
@ -125,6 +125,19 @@ func TestMarkerAndLengthSplitIntegration(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSplitByMarker_SpacedVariant(t *testing.T) {
|
||||||
|
// LLMs sometimes generate the marker with extra spaces
|
||||||
|
content := "Part1 <| [SPLIT] |> Part2 <|[ SPLIT ]|> Part3"
|
||||||
|
chunks := SplitByMarker(content)
|
||||||
|
|
||||||
|
if len(chunks) != 3 {
|
||||||
|
t.Fatalf("Expected 3 chunks, got %d: %q", len(chunks), chunks)
|
||||||
|
}
|
||||||
|
if chunks[0] != "Part1" || chunks[1] != "Part2" || chunks[2] != "Part3" {
|
||||||
|
t.Errorf("Spaced markers not handled: %q", chunks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestMarkerSplitPreservesCodeBlockIntegrity tests that marker split preserves code block boundaries
|
// TestMarkerSplitPreservesCodeBlockIntegrity tests that marker split preserves code block boundaries
|
||||||
func TestMarkerSplitPreservesCodeBlockIntegrity(t *testing.T) {
|
func TestMarkerSplitPreservesCodeBlockIntegrity(t *testing.T) {
|
||||||
content := "Hello <|[SPLIT]|>```go\npackage main\n```<|[SPLIT]|>World"
|
content := "Hello <|[SPLIT]|>```go\npackage main\n```<|[SPLIT]|>World"
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue