From 9c4968e2ccec04494c632d9d8f52dadc684b2507 Mon Sep 17 00:00:00 2001 From: dj-oyu <68707227+dj-oyu@users.noreply.github.com> Date: Sat, 21 Feb 2026 03:24:56 +0900 Subject: [PATCH] refactor: unify normalization to alpha-only, simplify XML block detection - normalizeAlpha / NormalizeToolName: keep only lowercase a-z - Replace findToolCallOpenTag + findToolCallCloseTag with single findToolCallBlock using regex + greedy close tag matching - Both XML extraction and stripping use the same findToolCallBlock - Edit distance still used for fuzzy "toolcall" tag identification Co-Authored-By: Claude Opus 4.6 --- pkg/providers/claude_cli_provider_test.go | 37 ++++- pkg/providers/tool_call_extract.go | 185 ++++++++++------------ pkg/tools/registry.go | 18 ++- 3 files changed, 130 insertions(+), 110 deletions(-) diff --git a/pkg/providers/claude_cli_provider_test.go b/pkg/providers/claude_cli_provider_test.go index e4ee3de38..fe7bced16 100644 --- a/pkg/providers/claude_cli_provider_test.go +++ b/pkg/providers/claude_cli_provider_test.go @@ -1161,6 +1161,27 @@ func TestStripXMLToolCalls_NoXML(t *testing.T) { } } +func TestNormalizeAlpha(t *testing.T) { + tests := []struct { + input, want string + }{ + {"toolcall", "toolcall"}, + {"tool_call", "toolcall"}, + {"Tool-Call", "toolcall"}, + {"ReadFile", "readfile"}, + {"read_file", "readfile"}, + {"EXEC", "exec"}, + {"web123search", "websearch"}, + {"", ""}, + } + for _, tt := range tests { + got := normalizeAlpha(tt.input) + if got != tt.want { + t.Errorf("normalizeAlpha(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} + func TestLevenshtein(t *testing.T) { tests := []struct { a, b string @@ -1184,14 +1205,26 @@ func TestLevenshtein(t *testing.T) { } func TestIsToolCallTag(t *testing.T) { - // Should match + // Should match — toolcall variants for _, name := range []string{"toolcall", "tool_call", "tool-call", "ToolCall", "Toolcall", "toolCall", "TOOLCALL"} { if !isToolCallTag(name) { t.Errorf("isToolCallTag(%q) = false, want true", name) } } + // Should match — function_call variants + for _, name := range []string{"function_call", "FunctionCall", "functioncall", "FUNCTION_CALL"} { + if !isToolCallTag(name) { + t.Errorf("isToolCallTag(%q) = false, want true", name) + } + } + // Should match — tool_use variants + for _, name := range []string{"tool_use", "ToolUse", "tooluse", "TOOL_USE"} { + if !isToolCallTag(name) { + t.Errorf("isToolCallTag(%q) = false, want true", name) + } + } // Should NOT match - for _, name := range []string{"invoke", "parameter", "function", "result", "hello"} { + for _, name := range []string{"invoke", "parameter", "function", "result", "hello", "content"} { if isToolCallTag(name) { t.Errorf("isToolCallTag(%q) = true, want false", name) } diff --git a/pkg/providers/tool_call_extract.go b/pkg/providers/tool_call_extract.go index 2011da66d..0f0445d16 100644 --- a/pkg/providers/tool_call_extract.go +++ b/pkg/providers/tool_call_extract.go @@ -3,6 +3,7 @@ package providers import ( "encoding/json" "fmt" + "regexp" "strings" ) @@ -57,17 +58,22 @@ func extractToolCallsFromText(text string) []ToolCall { return result } -// extractXMLToolCalls parses XML tool call blocks (e.g. ) -// into structured ToolCall objects. Used as a fallback when the provider returns -// tool calls as XML in Content but not in the structured tool_calls field. -// -// Expected format: -// -// -// -// value -// -// +// --- Shared helpers for XML tool call extraction --- + +// normalizeAlpha keeps only lowercase ASCII letters. +// "tool_call" → "toolcall", "Tool-Call" → "toolcall", "ReadFile" → "readfile". +func normalizeAlpha(s string) string { + var b strings.Builder + for _, r := range s { + if r >= 'A' && r <= 'Z' { + b.WriteRune(r + 32) + } else if r >= 'a' && r <= 'z' { + b.WriteRune(r) + } + } + return b.String() +} + // levenshtein computes the edit distance between two strings. // O(n*m) where n,m are string lengths — negligible for short tag names. func levenshtein(a, b string) int { @@ -97,97 +103,85 @@ func levenshtein(a, b string) int { return prev[lb] } -// isToolCallTag returns true if name is close enough to "toolcall" by edit -// distance (threshold ≤ 2). Case-insensitive. Catches variants like -// "tool_call", "Tool-Call", "toolCall", "ToolCall", etc. +// Known tool call tag patterns (already alpha-normalized). +// Providers may use different names: tool_call, function_call, tool_use, etc. +var toolCallPatterns = []string{"toolcall", "functioncall", "tooluse"} + +// isToolCallTag returns true if the tag name is close to any known tool call +// pattern after alpha normalization + edit distance (threshold ≤ 2). func isToolCallTag(name string) bool { const threshold = 2 - return levenshtein(strings.ToLower(name), "toolcall") <= threshold + norm := normalizeAlpha(name) + for _, pat := range toolCallPatterns { + if levenshtein(norm, pat) <= threshold { + return true + } + } + return false } -// findToolCallOpenTag finds the next opening toolcall tag using fuzzy -// matching (edit distance). Scans for patterns where name is -// close to "toolcall". Returns the index of '<', the namespace, and the -// full tag length, or idx=-1 if not found. -func findToolCallOpenTag(text string) (idx int, ns string, tagLen int) { - search := text - offset := 0 - for { - lt := strings.Index(search, "<") - if lt == -1 { - return -1, "", 0 - } - // Skip closing tags and comments - if lt+1 < len(search) && (search[lt+1] == '/' || search[lt+1] == '!') { - offset += lt + 2 - search = search[lt+2:] +// tagSuffix returns the part after the last ':' (namespace separator), +// or the whole string if there is no ':'. +func tagSuffix(tag string) string { + if i := strings.LastIndex(tag, ":"); i >= 0 { + return tag[i+1:] + } + return tag +} + +// --- XML block detection via regex --- +// +// Strategy: find … pairs using regex, then check if the tag +// suffix normalizes to something close to "toolcall" (edit distance ≤ 2). +// Uses greedy (longest) match for the closing tag to capture the full block. + +var ( + reOpenTag = regexp.MustCompile(`<([a-zA-Z][\w:.-]*)>`) + reCloseTag = regexp.MustCompile(``) +) + +// findToolCallBlock finds the first XML block whose tag suffix matches +// "toolcall" by edit distance. Returns the block boundaries and the inner +// content, or found=false. +func findToolCallBlock(text string) (blockStart, blockEnd int, content string, found bool) { + for _, om := range reOpenTag.FindAllStringSubmatchIndex(text, -1) { + tagName := text[om[2]:om[3]] + if !isToolCallTag(tagSuffix(tagName)) { continue } - gt := strings.Index(search[lt:], ">") - if gt == -1 { - return -1, "", 0 - } - tagContent := search[lt+1 : lt+gt] // e.g. "minimax:tool_call" - colon := strings.Index(tagContent, ":") - if colon != -1 { - nsCandidate := tagContent[:colon] - nameCandidate := tagContent[colon+1:] - if isToolCallTag(nameCandidate) { - fullTag := "<" + tagContent + ">" - return offset + lt, nsCandidate, len(fullTag) + // Found a toolcall opening tag. Search for the last matching close tag (greedy). + afterOpen := text[om[1]:] + closes := reCloseTag.FindAllStringSubmatchIndex(afterOpen, -1) + for i := len(closes) - 1; i >= 0; i-- { + closeTagName := afterOpen[closes[i][2]:closes[i][3]] + if isToolCallTag(tagSuffix(closeTagName)) { + return om[0], om[1] + closes[i][1], afterOpen[:closes[i][0]], true } } - offset += lt + gt + 1 - search = search[lt+gt+1:] - } -} - -// findToolCallCloseTag finds the close tag for a toolcall block using -// fuzzy matching (edit distance). Returns the index and length of the close tag, or -1. -func findToolCallCloseTag(text, ns string) (idx int, tagLen int) { - // Scan for all ") - if end == -1 { - return -1, 0 - } - tagName := afterPrefix[:end] - if isToolCallTag(tagName) { - fullTag := prefix + tagName + ">" - return offset + i, len(fullTag) - } - offset += i + len(prefix) - search = search[i+len(prefix):] } + return 0, 0, "", false } +// --- XML tool call extraction --- +// +// Expected format: +// +// +// +// value +// +// func extractXMLToolCalls(text string) []ToolCall { var result []ToolCall remaining := text callIdx := 0 for { - // Find next opening toolcall tag using normalized matching - openIdx, ns, openLen := findToolCallOpenTag(remaining) - if openIdx == -1 { + _, blockEnd, block, found := findToolCallBlock(remaining) + if !found { break } - afterOpen := remaining[openIdx+openLen:] - closeIdx, closeLen := findToolCallCloseTag(afterOpen, ns) - if closeIdx == -1 { - break - } - - block := afterOpen[:closeIdx] - remaining = afterOpen[closeIdx+closeLen:] + remaining = remaining[blockEnd:] // Parse elements within the block invokeRemaining := block @@ -234,7 +228,6 @@ func extractXMLToolCalls(text string) []ToolCall { } paramName := paramRemaining[pNameStart : pNameStart+pNameEnd] - // Find closing > of the tag tagClose := strings.Index(paramRemaining[pNameStart:], ">") if tagClose == -1 { break @@ -249,9 +242,7 @@ func extractXMLToolCalls(text string) []ToolCall { paramRemaining = paramRemaining[valueStart+valueEnd+len(""):] } - // Build Arguments JSON string argsJSON, _ := json.Marshal(args) - callIdx++ result = append(result, ToolCall{ ID: fmt.Sprintf("xmltc_%d", callIdx), @@ -269,24 +260,16 @@ func extractXMLToolCalls(text string) []ToolCall { return result } -// stripXMLToolCalls removes XML tool call blocks (e.g. ...) -// from response text. Some providers embed raw XML tool calls in Content alongside -// structured tool_calls; this prevents them from leaking to users. -// Uses normalized tag matching so that , , -// etc. are all recognized and stripped. +// stripXMLToolCalls removes XML tool call blocks from response text. +// Prevents raw XML tool calls from leaking to users. func stripXMLToolCalls(text string) string { - openIdx, ns, openLen := findToolCallOpenTag(text) - if openIdx == -1 { + blockStart, blockEnd, _, found := findToolCallBlock(text) + if !found { return text } - afterOpen := text[openIdx+openLen:] - closeIdx, closeLen := findToolCallCloseTag(afterOpen, ns) - if closeIdx == -1 { - return text - } - cleaned := text[:openIdx] + afterOpen[closeIdx+closeLen:] - // Recursively strip if there are more blocks - if openIdx2, _, _ := findToolCallOpenTag(cleaned); openIdx2 != -1 { + cleaned := text[:blockStart] + text[blockEnd:] + // Recursively strip remaining blocks + if _, _, _, more := findToolCallBlock(cleaned); more { cleaned = stripXMLToolCalls(cleaned) } return strings.TrimSpace(cleaned) diff --git a/pkg/tools/registry.go b/pkg/tools/registry.go index 957226eb3..924985655 100644 --- a/pkg/tools/registry.go +++ b/pkg/tools/registry.go @@ -11,14 +11,18 @@ import ( "github.com/sipeed/picoclaw/pkg/providers" ) -// NormalizeToolName strips underscores and hyphens and lowercases for -// fuzzy tool name matching. LLMs sometimes call "readfile" instead of -// "read_file", etc. +// NormalizeToolName keeps only lowercase ASCII letters. +// "read_file" → "readfile", "ReadFile" → "readfile", "read-file" → "readfile". func NormalizeToolName(s string) string { - s = strings.ToLower(s) - s = strings.ReplaceAll(s, "_", "") - s = strings.ReplaceAll(s, "-", "") - return s + var b strings.Builder + for _, r := range s { + if r >= 'A' && r <= 'Z' { + b.WriteRune(r + 32) + } else if r >= 'a' && r <= 'z' { + b.WriteRune(r) + } + } + return b.String() } type ToolRegistry struct {