From 994ec72d917ff3cf7c3d2a4df39b8b2a66626849 Mon Sep 17 00:00:00 2001 From: harshbansal7 Date: Wed, 18 Feb 2026 16:55:20 +0530 Subject: [PATCH 01/19] Fix parsing of SKILL.md file frontmatter - regex --- pkg/skills/loader.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/skills/loader.go b/pkg/skills/loader.go index 0c63ae067..15e82c31b 100644 --- a/pkg/skills/loader.go +++ b/pkg/skills/loader.go @@ -9,6 +9,8 @@ import ( "path/filepath" "regexp" "strings" + + "github.com/sipeed/picoclaw/pkg/logger" ) var namePattern = regexp.MustCompile(`^[a-zA-Z0-9]+(-[a-zA-Z0-9]+)*$`) @@ -251,6 +253,11 @@ func (sl *SkillsLoader) BuildSkillsSummary() string { func (sl *SkillsLoader) getSkillMetadata(skillPath string) *SkillMetadata { content, err := os.ReadFile(skillPath) if err != nil { + logger.WarnCF("skills", "Failed to read skill metadata", + map[string]interface{}{ + "skill_path": skillPath, + "error": err.Error(), + }) return nil } @@ -306,9 +313,9 @@ func (sl *SkillsLoader) parseSimpleYAML(content string) map[string]string { } func (sl *SkillsLoader) extractFrontmatter(content string) string { - // (?s) enables DOTALL mode so . matches newlines - // Match first ---, capture everything until next --- on its own line - re := regexp.MustCompile(`(?s)^---\n(.*)\n---`) + // Support both Unix (\n) and Windows (\r\n) line endings for frontmatter blocks + // (?s) enables DOTALL so . matches newlines; ^--- at start, then ... --- at start of line + re := regexp.MustCompile(`(?s)^---\r?\n(.*?)\r?\n---`) match := re.FindStringSubmatch(content) if len(match) > 1 { return match[1] From 02b5811b95abf1b6102f1cbce2afc9cace1f3cb4 Mon Sep 17 00:00:00 2001 From: harshbansal7 Date: Wed, 18 Feb 2026 16:58:27 +0530 Subject: [PATCH 02/19] add support for \r as well --- pkg/skills/loader.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/skills/loader.go b/pkg/skills/loader.go index 15e82c31b..c9731b6ae 100644 --- a/pkg/skills/loader.go +++ b/pkg/skills/loader.go @@ -313,9 +313,10 @@ func (sl *SkillsLoader) parseSimpleYAML(content string) map[string]string { } func (sl *SkillsLoader) extractFrontmatter(content string) string { - // Support both Unix (\n) and Windows (\r\n) line endings for frontmatter blocks - // (?s) enables DOTALL so . matches newlines; ^--- at start, then ... --- at start of line - re := regexp.MustCompile(`(?s)^---\r?\n(.*?)\r?\n---`) + // Support \n (Unix), \r\n (Windows), and \r (classic Mac) line endings for frontmatter blocks + // (?s) enables DOTALL so . matches newlines; + // ^--- at start, then ... --- at start of line, honoring all three line ending types + re := regexp.MustCompile(`(?s)^---(?:\r\n|\n|\r)(.*?)(?:\r\n|\n|\r)---`) match := re.FindStringSubmatch(content) if len(match) > 1 { return match[1] From 287100f3030b337ac5f24d7577b0e28e26357616 Mon Sep 17 00:00:00 2001 From: harshbansal7 Date: Wed, 18 Feb 2026 23:13:47 +0530 Subject: [PATCH 03/19] Comments resolved --- pkg/skills/loader.go | 13 ++++- pkg/skills/loader_test.go | 102 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 2 deletions(-) diff --git a/pkg/skills/loader.go b/pkg/skills/loader.go index c9731b6ae..bb0abbdcc 100644 --- a/pkg/skills/loader.go +++ b/pkg/skills/loader.go @@ -290,10 +290,15 @@ func (sl *SkillsLoader) getSkillMetadata(skillPath string) *SkillMetadata { // parseSimpleYAML parses simple key: value YAML format // Example: name: github\n description: "..." +// Normalizes line endings to handle \n (Unix), \r\n (Windows), and \r (classic Mac) func (sl *SkillsLoader) parseSimpleYAML(content string) map[string]string { result := make(map[string]string) - for _, line := range strings.Split(content, "\n") { + // Normalize line endings: convert \r\n and \r to \n + normalized := strings.ReplaceAll(content, "\r\n", "\n") + normalized = strings.ReplaceAll(normalized, "\r", "\n") + + for _, line := range strings.Split(normalized, "\n") { line = strings.TrimSpace(line) if line == "" || strings.HasPrefix(line, "#") { continue @@ -325,7 +330,11 @@ func (sl *SkillsLoader) extractFrontmatter(content string) string { } func (sl *SkillsLoader) stripFrontmatter(content string) string { - re := regexp.MustCompile(`^---\n.*?\n---\n`) + // Support \n (Unix), \r\n (Windows), and \r (classic Mac) line endings for frontmatter blocks + // (?s) enables DOTALL so . matches newlines; + // ^--- at start, then ... --- at start of line, honoring all three line ending types + // Match zero or more trailing line endings after closing --- (handles both with and without blank lines) + re := regexp.MustCompile(`(?s)^---(?:\r\n|\n|\r)(.*?)(?:\r\n|\n|\r)---(?:\r\n|\n|\r)*`) return re.ReplaceAllString(content, "") } diff --git a/pkg/skills/loader_test.go b/pkg/skills/loader_test.go index e0e7109cf..539d24646 100644 --- a/pkg/skills/loader_test.go +++ b/pkg/skills/loader_test.go @@ -75,3 +75,105 @@ func TestSkillsInfoValidate(t *testing.T) { }) } } + +func TestExtractFrontmatter(t *testing.T) { + sl := &SkillsLoader{} + + testcases := []struct { + name string + content string + expectedName string + expectedDesc string + lineEndingType string + }{ + { + name: "unix-line-endings", + lineEndingType: "Unix (\\n)", + content: "---\nname: test-skill\ndescription: A test skill\n---\n\n# Skill Content", + expectedName: "test-skill", + expectedDesc: "A test skill", + }, + { + name: "windows-line-endings", + lineEndingType: "Windows (\\r\\n)", + content: "---\r\nname: test-skill\r\ndescription: A test skill\r\n---\r\n\r\n# Skill Content", + expectedName: "test-skill", + expectedDesc: "A test skill", + }, + { + name: "classic-mac-line-endings", + lineEndingType: "Classic Mac (\\r)", + content: "---\rname: test-skill\rdescription: A test skill\r---\r\r# Skill Content", + expectedName: "test-skill", + expectedDesc: "A test skill", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + // Extract frontmatter + frontmatter := sl.extractFrontmatter(tc.content) + assert.NotEmpty(t, frontmatter, "Frontmatter should be extracted for %s line endings", tc.lineEndingType) + + // Parse YAML to get name and description (parseSimpleYAML now handles all line ending types) + yamlMeta := sl.parseSimpleYAML(frontmatter) + assert.Equal(t, tc.expectedName, yamlMeta["name"], "Name should be correctly parsed from frontmatter with %s line endings", tc.lineEndingType) + assert.Equal(t, tc.expectedDesc, yamlMeta["description"], "Description should be correctly parsed from frontmatter with %s line endings", tc.lineEndingType) + }) + } +} + +func TestStripFrontmatter(t *testing.T) { + sl := &SkillsLoader{} + + testcases := []struct { + name string + content string + expectedContent string + lineEndingType string + }{ + { + name: "unix-line-endings", + lineEndingType: "Unix (\\n)", + content: "---\nname: test-skill\ndescription: A test skill\n---\n\n# Skill Content", + expectedContent: "# Skill Content", + }, + { + name: "windows-line-endings", + lineEndingType: "Windows (\\r\\n)", + content: "---\r\nname: test-skill\r\ndescription: A test skill\r\n---\r\n\r\n# Skill Content", + expectedContent: "# Skill Content", + }, + { + name: "classic-mac-line-endings", + lineEndingType: "Classic Mac (\\r)", + content: "---\rname: test-skill\rdescription: A test skill\r---\r\r# Skill Content", + expectedContent: "# Skill Content", + }, + { + name: "unix-line-endings-without-trailing-newline", + lineEndingType: "Unix (\\n) without trailing newline", + content: "---\nname: test-skill\ndescription: A test skill\n---\n# Skill Content", + expectedContent: "# Skill Content", + }, + { + name: "windows-line-endings-without-trailing-newline", + lineEndingType: "Windows (\\r\\n) without trailing newline", + content: "---\r\nname: test-skill\r\ndescription: A test skill\r\n---\r\n# Skill Content", + expectedContent: "# Skill Content", + }, + { + name: "no-frontmatter", + lineEndingType: "No frontmatter", + content: "# Skill Content\n\nSome content here.", + expectedContent: "# Skill Content\n\nSome content here.", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + result := sl.stripFrontmatter(tc.content) + assert.Equal(t, tc.expectedContent, result, "Frontmatter should be stripped correctly for %s", tc.lineEndingType) + }) + } +} From 94a1b8664ba9637890e93f1864d19d7b78cde1c4 Mon Sep 17 00:00:00 2001 From: Hua Date: Wed, 18 Feb 2026 20:01:53 +0000 Subject: [PATCH 04/19] refactor: extract message splitting logic to shared utils - Move FindLast, findLast, and SplitMessage from discord.go to pkg/utils/message.go - Update discord.go to use utils.SplitMessage() - Makes splitting logic reusable across other channels --- pkg/channels/discord.go | 129 +-------------------------------------- pkg/utils/message.go | 131 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 128 deletions(-) create mode 100644 pkg/utils/message.go diff --git a/pkg/channels/discord.go b/pkg/channels/discord.go index f360c75ef..7dc3f3198 100644 --- a/pkg/channels/discord.go +++ b/pkg/channels/discord.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "os" - "strings" "time" "github.com/bwmarrin/discordgo" @@ -106,7 +105,7 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) erro return nil } - chunks := splitMessage(msg.Content, 1500) // Discord has a limit of 2000 characters per message, leave 500 for natural split e.g. code blocks + chunks := utils.SplitMessage(msg.Content, 1500) // Discord has a limit of 2000 characters per message, leave 500 for natural split e.g. code blocks for _, chunk := range chunks { if err := c.sendChunk(ctx, channelID, chunk); err != nil { @@ -117,132 +116,6 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) erro return nil } -// splitMessage splits long messages into chunks, preserving code block integrity -// Uses natural boundaries (newlines, spaces) and extends messages slightly to avoid breaking code blocks -func splitMessage(content string, limit int) []string { - var messages []string - - for len(content) > 0 { - if len(content) <= limit { - messages = append(messages, content) - break - } - - msgEnd := limit - - // Find natural split point within the limit - msgEnd = findLastNewline(content[:limit], 200) - if msgEnd <= 0 { - msgEnd = findLastSpace(content[:limit], 100) - } - if msgEnd <= 0 { - msgEnd = limit - } - - // Check if this would end with an incomplete code block - candidate := content[:msgEnd] - unclosedIdx := findLastUnclosedCodeBlock(candidate) - - if unclosedIdx >= 0 { - // Message would end with incomplete code block - // Try to extend to include the closing ``` (with some buffer) - extendedLimit := limit + 500 // Allow 500 char buffer for code blocks - if len(content) > extendedLimit { - closingIdx := findNextClosingCodeBlock(content, msgEnd) - if closingIdx > 0 && closingIdx <= extendedLimit { - // Extend to include the closing ``` - msgEnd = closingIdx - } else { - // Can't find closing, split before the code block - msgEnd = findLastNewline(content[:unclosedIdx], 200) - if msgEnd <= 0 { - msgEnd = findLastSpace(content[:unclosedIdx], 100) - } - if msgEnd <= 0 { - msgEnd = unclosedIdx - } - } - } else { - // Remaining content fits within extended limit - msgEnd = len(content) - } - } - - if msgEnd <= 0 { - msgEnd = limit - } - - messages = append(messages, content[:msgEnd]) - content = strings.TrimSpace(content[msgEnd:]) - } - - return messages -} - -// findLastUnclosedCodeBlock finds the last opening ``` that doesn't have a closing ``` -// Returns the position of the opening ``` or -1 if all code blocks are complete -func findLastUnclosedCodeBlock(text string) int { - count := 0 - lastOpenIdx := -1 - - for i := 0; i < len(text); i++ { - if i+2 < len(text) && text[i] == '`' && text[i+1] == '`' && text[i+2] == '`' { - if count == 0 { - lastOpenIdx = i - } - count++ - i += 2 - } - } - - // If odd number of ``` markers, last one is unclosed - if count%2 == 1 { - return lastOpenIdx - } - return -1 -} - -// findNextClosingCodeBlock finds the next closing ``` starting from a position -// Returns the position after the closing ``` or -1 if not found -func findNextClosingCodeBlock(text string, startIdx int) int { - for i := startIdx; i < len(text); i++ { - if i+2 < len(text) && text[i] == '`' && text[i+1] == '`' && text[i+2] == '`' { - return i + 3 - } - } - return -1 -} - -// findLastNewline finds the last newline character within the last N characters -// Returns the position of the newline or -1 if not found -func findLastNewline(s string, searchWindow int) int { - searchStart := len(s) - searchWindow - if searchStart < 0 { - searchStart = 0 - } - for i := len(s) - 1; i >= searchStart; i-- { - if s[i] == '\n' { - return i - } - } - return -1 -} - -// findLastSpace finds the last space character within the last N characters -// Returns the position of the space or -1 if not found -func findLastSpace(s string, searchWindow int) int { - searchStart := len(s) - searchWindow - if searchStart < 0 { - searchStart = 0 - } - for i := len(s) - 1; i >= searchStart; i-- { - if s[i] == ' ' || s[i] == '\t' { - return i - } - } - return -1 -} - func (c *DiscordChannel) sendChunk(ctx context.Context, channelID, content string) error { // 使用传入的 ctx 进行超时控制 sendCtx, cancel := context.WithTimeout(ctx, sendTimeout) diff --git a/pkg/utils/message.go b/pkg/utils/message.go new file mode 100644 index 000000000..3a4cf2ad6 --- /dev/null +++ b/pkg/utils/message.go @@ -0,0 +1,131 @@ +package utils + +import ( + "strings" +) + +// SplitMessage splits long messages into chunks, preserving code block integrity +// Uses natural boundaries (newlines, spaces) and extends messages slightly to avoid breaking code blocks +func SplitMessage(content string, limit int) []string { + var messages []string + + for len(content) > 0 { + if len(content) <= limit { + messages = append(messages, content) + break + } + + msgEnd := limit + + // Find natural split point within the limit + msgEnd = FindLastNewline(content[:limit], 200) + if msgEnd <= 0 { + msgEnd = FindLastSpace(content[:limit], 100) + } + if msgEnd <= 0 { + msgEnd = limit + } + + // Check if this would end with an incomplete code block + candidate := content[:msgEnd] + unclosedIdx := FindLastUnclosedCodeBlock(candidate) + + if unclosedIdx >= 0 { + // Message would end with incomplete code block + // Try to extend to include the closing ``` (with some buffer) + extendedLimit := limit + 500 // Allow 500 char buffer for code blocks + if len(content) > extendedLimit { + closingIdx := FindNextClosingCodeBlock(content, msgEnd) + if closingIdx > 0 && closingIdx <= extendedLimit { + // Extend to include the closing ``` + msgEnd = closingIdx + } else { + // Can't find closing, split before the code block + msgEnd = FindLastNewline(content[:unclosedIdx], 200) + if msgEnd <= 0 { + msgEnd = FindLastSpace(content[:unclosedIdx], 100) + } + if msgEnd <= 0 { + msgEnd = unclosedIdx + } + } + } else { + // Remaining content fits within extended limit + msgEnd = len(content) + } + } + + if msgEnd <= 0 { + msgEnd = limit + } + + messages = append(messages, content[:msgEnd]) + content = strings.TrimSpace(content[msgEnd:]) + } + + return messages +} + +// FindLastUnclosedCodeBlock finds the last opening ``` that doesn't have a closing ``` +// Returns the position of the opening ``` or -1 if all code blocks are complete +func FindLastUnclosedCodeBlock(text string) int { + count := 0 + lastOpenIdx := -1 + + for i := 0; i < len(text); i++ { + if i+2 < len(text) && text[i] == '`' && text[i+1] == '`' && text[i+2] == '`' { + if count == 0 { + lastOpenIdx = i + } + count++ + i += 2 + } + } + + // If odd number of ``` markers, last one is unclosed + if count%2 == 1 { + return lastOpenIdx + } + return -1 +} + +// FindNextClosingCodeBlock finds the next closing ``` starting from a position +// Returns the position after the closing ``` or -1 if not found +func FindNextClosingCodeBlock(text string, startIdx int) int { + for i := startIdx; i < len(text); i++ { + if i+2 < len(text) && text[i] == '`' && text[i+1] == '`' && text[i+2] == '`' { + return i + 3 + } + } + return -1 +} + +// FindLastNewline finds the last newline character within the last N characters +// Returns the position of the newline or -1 if not found +func FindLastNewline(s string, searchWindow int) int { + searchStart := len(s) - searchWindow + if searchStart < 0 { + searchStart = 0 + } + for i := len(s) - 1; i >= searchStart; i-- { + if s[i] == '\n' { + return i + } + } + return -1 +} + +// FindLastSpace finds the last space character within the last N characters +// Returns the position of the space or -1 if not found +func FindLastSpace(s string, searchWindow int) int { + searchStart := len(s) - searchWindow + if searchStart < 0 { + searchStart = 0 + } + for i := len(s) - 1; i >= searchStart; i-- { + if s[i] == ' ' || s[i] == '\t' { + return i + } + } + return -1 +} From e03124dc8a695b36b28eb2798fc914efa4493906 Mon Sep 17 00:00:00 2001 From: Hua Date: Wed, 18 Feb 2026 20:21:51 +0000 Subject: [PATCH 05/19] refactor: improve SplitMessage API clarity - Accept hard upper limit (maxLen) instead of pre-subtracted value - Caller now passes actual platform limit (e.g., 2000 for Discord) - Internal buffer of 500 chars is handled within message.go - Preferred split at maxLen - 500, may extend to maxLen for code blocks - Never exceeds maxLen, no more mental math for callers --- pkg/channels/discord.go | 2 +- pkg/utils/message.go | 41 +++++++++++++++++++++++------------------ 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/pkg/channels/discord.go b/pkg/channels/discord.go index 7dc3f3198..ba02f7598 100644 --- a/pkg/channels/discord.go +++ b/pkg/channels/discord.go @@ -105,7 +105,7 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) erro return nil } - chunks := utils.SplitMessage(msg.Content, 1500) // Discord has a limit of 2000 characters per message, leave 500 for natural split e.g. code blocks + chunks := utils.SplitMessage(msg.Content, 2000) // Discord hard limit: 2000 chars (prefers split at 1500 to leave room for code blocks) for _, chunk := range chunks { if err := c.sendChunk(ctx, channelID, chunk); err != nil { diff --git a/pkg/utils/message.go b/pkg/utils/message.go index 3a4cf2ad6..9ca49ba53 100644 --- a/pkg/utils/message.go +++ b/pkg/utils/message.go @@ -4,26 +4,35 @@ import ( "strings" ) -// SplitMessage splits long messages into chunks, preserving code block integrity -// Uses natural boundaries (newlines, spaces) and extends messages slightly to avoid breaking code blocks -func SplitMessage(content string, limit int) []string { +const defaultCodeBlockBuffer = 500 + +// SplitMessage splits long messages into chunks, preserving code block integrity. +// The maxLen parameter is the hard upper limit - no message will exceed this length. +// The function prefers to split at maxLen - defaultCodeBlockBuffer to leave room for code blocks, +// but may extend up to maxLen when needed to avoid breaking incomplete code blocks. +func SplitMessage(content string, maxLen int) []string { var messages []string + codeBlockBuffer := defaultCodeBlockBuffer for len(content) > 0 { - if len(content) <= limit { + if len(content) <= maxLen { messages = append(messages, content) break } - msgEnd := limit + // Effective split point: maxLen minus buffer, to leave room for code blocks + effectiveLimit := maxLen - codeBlockBuffer + if effectiveLimit < maxLen/2 { + effectiveLimit = maxLen / 2 + } - // Find natural split point within the limit - msgEnd = FindLastNewline(content[:limit], 200) + // Find natural split point within the effective limit + msgEnd := FindLastNewline(content[:effectiveLimit], 200) if msgEnd <= 0 { - msgEnd = FindLastSpace(content[:limit], 100) + msgEnd = FindLastSpace(content[:effectiveLimit], 100) } if msgEnd <= 0 { - msgEnd = limit + msgEnd = effectiveLimit } // Check if this would end with an incomplete code block @@ -32,15 +41,14 @@ func SplitMessage(content string, limit int) []string { if unclosedIdx >= 0 { // Message would end with incomplete code block - // Try to extend to include the closing ``` (with some buffer) - extendedLimit := limit + 500 // Allow 500 char buffer for code blocks - if len(content) > extendedLimit { + // Try to extend up to maxLen (hard limit, never exceed) to include the closing ``` + if len(content) > msgEnd { closingIdx := FindNextClosingCodeBlock(content, msgEnd) - if closingIdx > 0 && closingIdx <= extendedLimit { + if closingIdx > 0 && closingIdx <= maxLen { // Extend to include the closing ``` msgEnd = closingIdx } else { - // Can't find closing, split before the code block + // Can't find closing within maxLen, split before the code block msgEnd = FindLastNewline(content[:unclosedIdx], 200) if msgEnd <= 0 { msgEnd = FindLastSpace(content[:unclosedIdx], 100) @@ -49,14 +57,11 @@ func SplitMessage(content string, limit int) []string { msgEnd = unclosedIdx } } - } else { - // Remaining content fits within extended limit - msgEnd = len(content) } } if msgEnd <= 0 { - msgEnd = limit + msgEnd = effectiveLimit } messages = append(messages, content[:msgEnd]) From e35a82762406cc09df43bbb8d72d1529f317b7fb Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Wed, 18 Feb 2026 21:44:25 +0100 Subject: [PATCH 06/19] update documents --- pkg/channels/discord.go | 2 +- pkg/utils/message.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/channels/discord.go b/pkg/channels/discord.go index ba02f7598..472b51c53 100644 --- a/pkg/channels/discord.go +++ b/pkg/channels/discord.go @@ -105,7 +105,7 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) erro return nil } - chunks := utils.SplitMessage(msg.Content, 2000) // Discord hard limit: 2000 chars (prefers split at 1500 to leave room for code blocks) + chunks := utils.SplitMessage(msg.Content, 2000) // Split messages into chunks, Discord length limit: 2000 chars for _, chunk := range chunks { if err := c.sendChunk(ctx, channelID, chunk); err != nil { diff --git a/pkg/utils/message.go b/pkg/utils/message.go index 9ca49ba53..ed56da95b 100644 --- a/pkg/utils/message.go +++ b/pkg/utils/message.go @@ -7,9 +7,9 @@ import ( const defaultCodeBlockBuffer = 500 // SplitMessage splits long messages into chunks, preserving code block integrity. -// The maxLen parameter is the hard upper limit - no message will exceed this length. // The function prefers to split at maxLen - defaultCodeBlockBuffer to leave room for code blocks, // but may extend up to maxLen when needed to avoid breaking incomplete code blocks. +// Please refer to pkg/channels/discord.go for usage. func SplitMessage(content string, maxLen int) []string { var messages []string codeBlockBuffer := defaultCodeBlockBuffer @@ -41,7 +41,7 @@ func SplitMessage(content string, maxLen int) []string { if unclosedIdx >= 0 { // Message would end with incomplete code block - // Try to extend up to maxLen (hard limit, never exceed) to include the closing ``` + // Try to extend up to maxLen to include the closing ``` if len(content) > msgEnd { closingIdx := FindNextClosingCodeBlock(content, msgEnd) if closingIdx > 0 && closingIdx <= maxLen { From b122abd30f2305631f2dc90f4d894b169ab37451 Mon Sep 17 00:00:00 2001 From: harshbansal7 Date: Thu, 19 Feb 2026 02:28:44 +0530 Subject: [PATCH 07/19] fix --- pkg/skills/loader_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/skills/loader_test.go b/pkg/skills/loader_test.go index 539d24646..efadcdbf2 100644 --- a/pkg/skills/loader_test.go +++ b/pkg/skills/loader_test.go @@ -80,11 +80,11 @@ func TestExtractFrontmatter(t *testing.T) { sl := &SkillsLoader{} testcases := []struct { - name string - content string - expectedName string - expectedDesc string - lineEndingType string + name string + content string + expectedName string + expectedDesc string + lineEndingType string }{ { name: "unix-line-endings", From 4ccee8556179d42ad0c5c3d7cb1f25caed3a49b9 Mon Sep 17 00:00:00 2001 From: Hua Audio <161028864+Huaaudio@users.noreply.github.com> Date: Wed, 18 Feb 2026 22:16:19 +0100 Subject: [PATCH 08/19] Update pkg/utils/message.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/utils/message.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/utils/message.go b/pkg/utils/message.go index ed56da95b..257f2c151 100644 --- a/pkg/utils/message.go +++ b/pkg/utils/message.go @@ -74,21 +74,22 @@ func SplitMessage(content string, maxLen int) []string { // FindLastUnclosedCodeBlock finds the last opening ``` that doesn't have a closing ``` // Returns the position of the opening ``` or -1 if all code blocks are complete func FindLastUnclosedCodeBlock(text string) int { - count := 0 + inCodeBlock := false lastOpenIdx := -1 for i := 0; i < len(text); i++ { if i+2 < len(text) && text[i] == '`' && text[i+1] == '`' && text[i+2] == '`' { - if count == 0 { + // Toggle code block state on each fence + if !inCodeBlock { + // Entering a code block: record this opening fence lastOpenIdx = i } - count++ + inCodeBlock = !inCodeBlock i += 2 } } - // If odd number of ``` markers, last one is unclosed - if count%2 == 1 { + if inCodeBlock { return lastOpenIdx } return -1 From f38ce0d4ac7ce0a7f99dc8b3c9303d0d7a9a69a0 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Wed, 18 Feb 2026 22:31:18 +0100 Subject: [PATCH 09/19] Update to support extra long code blocks --- pkg/utils/message.go | 47 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/pkg/utils/message.go b/pkg/utils/message.go index 257f2c151..6ee57bddb 100644 --- a/pkg/utils/message.go +++ b/pkg/utils/message.go @@ -48,13 +48,48 @@ func SplitMessage(content string, maxLen int) []string { // Extend to include the closing ``` msgEnd = closingIdx } else { - // Can't find closing within maxLen, split before the code block - msgEnd = FindLastNewline(content[:unclosedIdx], 200) - if msgEnd <= 0 { - msgEnd = FindLastSpace(content[:unclosedIdx], 100) + // Code block is too long to fit in one chunk or missing closing fence. + // Try to split inside by injecting closing and reopening fences. + headerEnd := strings.Index(content[unclosedIdx:], "\n") + if headerEnd == -1 { + headerEnd = unclosedIdx + 3 + } else { + headerEnd += unclosedIdx } - if msgEnd <= 0 { - msgEnd = unclosedIdx + header := strings.TrimSpace(content[unclosedIdx:headerEnd]) + + // If we have a reasonable amount of content after the header, split inside + if msgEnd > headerEnd+20 { + // Find a better split point closer to maxLen + innerLimit := maxLen - 5 // Leave room for "\n```" + betterEnd := FindLastNewline(content[:innerLimit], 200) + if betterEnd > headerEnd { + msgEnd = betterEnd + } else { + msgEnd = innerLimit + } + messages = append(messages, strings.TrimRight(content[:msgEnd], " \t\n\r")+"\n```") + content = strings.TrimSpace(header + "\n" + content[msgEnd:]) + continue + } + + // Otherwise, try to split before the code block starts + newEnd := FindLastNewline(content[:unclosedIdx], 200) + if newEnd <= 0 { + newEnd = FindLastSpace(content[:unclosedIdx], 100) + } + if newEnd > 0 { + msgEnd = newEnd + } else { + // If we can't split before, we MUST split inside (last resort) + if unclosedIdx > 20 { + msgEnd = unclosedIdx + } else { + msgEnd = maxLen - 5 + messages = append(messages, strings.TrimRight(content[:msgEnd], " \t\n\r")+"\n```") + content = strings.TrimSpace(header + "\n" + content[msgEnd:]) + continue + } } } } From 82a2faed9d54ba9caaf3f6ec764fd2f92fc6700d Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Wed, 18 Feb 2026 22:37:45 +0100 Subject: [PATCH 10/19] Privated function --- pkg/utils/message.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pkg/utils/message.go b/pkg/utils/message.go index 6ee57bddb..66f637d3d 100644 --- a/pkg/utils/message.go +++ b/pkg/utils/message.go @@ -27,9 +27,9 @@ func SplitMessage(content string, maxLen int) []string { } // Find natural split point within the effective limit - msgEnd := FindLastNewline(content[:effectiveLimit], 200) + msgEnd := findLastNewline(content[:effectiveLimit], 200) if msgEnd <= 0 { - msgEnd = FindLastSpace(content[:effectiveLimit], 100) + msgEnd = findLastSpace(content[:effectiveLimit], 100) } if msgEnd <= 0 { msgEnd = effectiveLimit @@ -37,13 +37,13 @@ func SplitMessage(content string, maxLen int) []string { // Check if this would end with an incomplete code block candidate := content[:msgEnd] - unclosedIdx := FindLastUnclosedCodeBlock(candidate) + unclosedIdx := findLastUnclosedCodeBlock(candidate) if unclosedIdx >= 0 { // Message would end with incomplete code block // Try to extend up to maxLen to include the closing ``` if len(content) > msgEnd { - closingIdx := FindNextClosingCodeBlock(content, msgEnd) + closingIdx := findNextClosingCodeBlock(content, msgEnd) if closingIdx > 0 && closingIdx <= maxLen { // Extend to include the closing ``` msgEnd = closingIdx @@ -62,7 +62,7 @@ func SplitMessage(content string, maxLen int) []string { if msgEnd > headerEnd+20 { // Find a better split point closer to maxLen innerLimit := maxLen - 5 // Leave room for "\n```" - betterEnd := FindLastNewline(content[:innerLimit], 200) + betterEnd := findLastNewline(content[:innerLimit], 200) if betterEnd > headerEnd { msgEnd = betterEnd } else { @@ -74,9 +74,9 @@ func SplitMessage(content string, maxLen int) []string { } // Otherwise, try to split before the code block starts - newEnd := FindLastNewline(content[:unclosedIdx], 200) + newEnd := findLastNewline(content[:unclosedIdx], 200) if newEnd <= 0 { - newEnd = FindLastSpace(content[:unclosedIdx], 100) + newEnd = findLastSpace(content[:unclosedIdx], 100) } if newEnd > 0 { msgEnd = newEnd @@ -106,9 +106,9 @@ func SplitMessage(content string, maxLen int) []string { return messages } -// FindLastUnclosedCodeBlock finds the last opening ``` that doesn't have a closing ``` +// findLastUnclosedCodeBlock finds the last opening ``` that doesn't have a closing ``` // Returns the position of the opening ``` or -1 if all code blocks are complete -func FindLastUnclosedCodeBlock(text string) int { +func findLastUnclosedCodeBlock(text string) int { inCodeBlock := false lastOpenIdx := -1 @@ -130,9 +130,9 @@ func FindLastUnclosedCodeBlock(text string) int { return -1 } -// FindNextClosingCodeBlock finds the next closing ``` starting from a position +// findNextClosingCodeBlock finds the next closing ``` starting from a position // Returns the position after the closing ``` or -1 if not found -func FindNextClosingCodeBlock(text string, startIdx int) int { +func findNextClosingCodeBlock(text string, startIdx int) int { for i := startIdx; i < len(text); i++ { if i+2 < len(text) && text[i] == '`' && text[i+1] == '`' && text[i+2] == '`' { return i + 3 @@ -141,9 +141,9 @@ func FindNextClosingCodeBlock(text string, startIdx int) int { return -1 } -// FindLastNewline finds the last newline character within the last N characters +// findLastNewline finds the last newline character within the last N characters // Returns the position of the newline or -1 if not found -func FindLastNewline(s string, searchWindow int) int { +func findLastNewline(s string, searchWindow int) int { searchStart := len(s) - searchWindow if searchStart < 0 { searchStart = 0 @@ -156,9 +156,9 @@ func FindLastNewline(s string, searchWindow int) int { return -1 } -// FindLastSpace finds the last space character within the last N characters +// findLastSpace finds the last space character within the last N characters // Returns the position of the space or -1 if not found -func FindLastSpace(s string, searchWindow int) int { +func findLastSpace(s string, searchWindow int) int { searchStart := len(s) - searchWindow if searchStart < 0 { searchStart = 0 From dfc3dffd0619530bff2615d48e137dfd531cf1bb Mon Sep 17 00:00:00 2001 From: Hua Audio <161028864+Huaaudio@users.noreply.github.com> Date: Wed, 18 Feb 2026 22:43:49 +0100 Subject: [PATCH 11/19] Update pkg/utils/message.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/utils/message.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/utils/message.go b/pkg/utils/message.go index 66f637d3d..bc648f396 100644 --- a/pkg/utils/message.go +++ b/pkg/utils/message.go @@ -9,7 +9,8 @@ const defaultCodeBlockBuffer = 500 // SplitMessage splits long messages into chunks, preserving code block integrity. // The function prefers to split at maxLen - defaultCodeBlockBuffer to leave room for code blocks, // but may extend up to maxLen when needed to avoid breaking incomplete code blocks. -// Please refer to pkg/channels/discord.go for usage. +// Call SplitMessage with the full text content and the maximum allowed length of a single message; +// it returns a slice of message chunks that each respect maxLen and avoid splitting fenced code blocks. func SplitMessage(content string, maxLen int) []string { var messages []string codeBlockBuffer := defaultCodeBlockBuffer From 7d8894d842e874f1a0e4d413c5931ed8b8185cfa Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Wed, 18 Feb 2026 23:02:16 +0100 Subject: [PATCH 12/19] update message test, change dynamic buffer --- pkg/utils/message.go | 16 ++-- pkg/utils/message_test.go | 151 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 5 deletions(-) create mode 100644 pkg/utils/message_test.go diff --git a/pkg/utils/message.go b/pkg/utils/message.go index bc648f396..1d05950d9 100644 --- a/pkg/utils/message.go +++ b/pkg/utils/message.go @@ -4,16 +4,22 @@ import ( "strings" ) -const defaultCodeBlockBuffer = 500 - // SplitMessage splits long messages into chunks, preserving code block integrity. -// The function prefers to split at maxLen - defaultCodeBlockBuffer to leave room for code blocks, -// but may extend up to maxLen when needed to avoid breaking incomplete code blocks. +// The function reserves a buffer (10% of maxLen, min 50) to leave room for closing code blocks, +// but may extend to maxLen when needed. // Call SplitMessage with the full text content and the maximum allowed length of a single message; // it returns a slice of message chunks that each respect maxLen and avoid splitting fenced code blocks. func SplitMessage(content string, maxLen int) []string { var messages []string - codeBlockBuffer := defaultCodeBlockBuffer + + // Dynamic buffer: 10% of maxLen, but at least 50 chars if possible + codeBlockBuffer := maxLen / 10 + if codeBlockBuffer < 50 { + codeBlockBuffer = 50 + } + if codeBlockBuffer > maxLen/2 { + codeBlockBuffer = maxLen / 2 + } for len(content) > 0 { if len(content) <= maxLen { diff --git a/pkg/utils/message_test.go b/pkg/utils/message_test.go new file mode 100644 index 000000000..33f5e51fc --- /dev/null +++ b/pkg/utils/message_test.go @@ -0,0 +1,151 @@ +package utils + +import ( + "strings" + "testing" +) + +func TestSplitMessage(t *testing.T) { + longText := strings.Repeat("a", 2500) + longCode := "```go\n" + strings.Repeat("fmt.Println(\"hello\")\n", 100) + "```" // ~2100 chars + + tests := []struct { + name string + content string + maxLen int + expectChunks int // Check number of chunks + checkContent func(t *testing.T, chunks []string) // Custom validation + }{ + { + name: "Empty message", + content: "", + maxLen: 2000, + expectChunks: 0, + }, + { + name: "Short message fits in one chunk", + content: "Hello world", + maxLen: 2000, + expectChunks: 1, + }, + { + name: "Simple split regular text", + content: longText, + maxLen: 2000, + expectChunks: 2, + checkContent: func(t *testing.T, chunks []string) { + if len(chunks[0]) > 2000 { + t.Errorf("Chunk 0 too large: %d", len(chunks[0])) + } + if len(chunks[0])+len(chunks[1]) != len(longText) { + t.Errorf("Total length mismatch. Got %d, want %d", len(chunks[0])+len(chunks[1]), len(longText)) + } + }, + }, + { + name: "Split at newline", + // 1750 chars then newline, then more chars. + // Dynamic buffer: 2000 / 10 = 200. + // Effective limit: 2000 - 200 = 1800. + // Split should happen at newline because it's at 1750 (< 1800). + // Total length must > 2000 to trigger split. 1750 + 1 + 300 = 2051. + content: strings.Repeat("a", 1750) + "\n" + strings.Repeat("b", 300), + maxLen: 2000, + expectChunks: 2, + checkContent: func(t *testing.T, chunks []string) { + if len(chunks[0]) != 1750 { + t.Errorf("Expected chunk 0 to be 1750 length (split at newline), got %d", len(chunks[0])) + } + if chunks[1] != strings.Repeat("b", 300) { + t.Errorf("Chunk 1 content mismatch. Len: %d", len(chunks[1])) + } + }, + }, + { + name: "Long code block split", + content: "Prefix\n" + longCode, + maxLen: 2000, + expectChunks: 2, + checkContent: func(t *testing.T, chunks []string) { + // Check that first chunk ends with closing fence + if !strings.HasSuffix(chunks[0], "\n```") { + t.Error("First chunk should end with injected closing fence") + } + // Check that second chunk starts with execution header + if !strings.HasPrefix(chunks[1], "```go") { + t.Error("Second chunk should start with injected code block header") + } + }, + }, + { + name: "Preserve Unicode characters", + content: strings.Repeat("世", 1000), // 3000 bytes + maxLen: 2000, + expectChunks: 2, + checkContent: func(t *testing.T, chunks []string) { + // Just verify we didn't panic and got valid strings. + // Go strings are UTF-8, if we split mid-rune it would be bad, + // but standard slicing might do that. + // Let's assume standard behavior is acceptable or check if it produces invalid rune? + if !strings.Contains(chunks[0], "世") { + t.Error("Chunk should contain unicode characters") + } + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := SplitMessage(tc.content, tc.maxLen) + + if tc.expectChunks == 0 { + if len(got) != 0 { + t.Errorf("Expected 0 chunks, got %d", len(got)) + } + return + } + + if len(got) != tc.expectChunks { + t.Errorf("Expected %d chunks, got %d", tc.expectChunks, len(got)) + // Log sizes for debugging + for i, c := range got { + t.Logf("Chunk %d length: %d", i, len(c)) + } + return // Stop further checks if count assumes specific split + } + + if tc.checkContent != nil { + tc.checkContent(t, got) + } + }) + } +} + +func TestSplitMessage_CodeBlockIntegrity(t *testing.T) { + // Focused test for the core requirement: splitting inside a code block preserves syntax highlighting + + // 60 chars total approximately + content := "```go\npackage main\n\nfunc main() {\n\tprintln(\"Hello\")\n}\n```" + maxLen := 40 + + chunks := SplitMessage(content, maxLen) + + if len(chunks) != 2 { + t.Fatalf("Expected 2 chunks, got %d: %q", len(chunks), chunks) + } + + // First chunk must end with "\n```" + if !strings.HasSuffix(chunks[0], "\n```") { + t.Errorf("First chunk should end with closing fence. Got: %q", chunks[0]) + } + + // Second chunk must start with the header "```go" + if !strings.HasPrefix(chunks[1], "```go") { + t.Errorf("Second chunk should start with code block header. Got: %q", chunks[1]) + } + + // First chunk should contain meaningful content + if len(chunks[0]) > 40 { + t.Errorf("First chunk exceeded maxLen: length %d", len(chunks[0])) + } +} From a46fe140a3c6e10b50d9d9437364865ac528cafb Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Wed, 18 Feb 2026 23:03:57 +0100 Subject: [PATCH 13/19] update dynamic buffer --- pkg/utils/message.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/utils/message.go b/pkg/utils/message.go index 1d05950d9..35914f399 100644 --- a/pkg/utils/message.go +++ b/pkg/utils/message.go @@ -9,6 +9,8 @@ import ( // but may extend to maxLen when needed. // Call SplitMessage with the full text content and the maximum allowed length of a single message; // it returns a slice of message chunks that each respect maxLen and avoid splitting fenced code blocks. +// Call SplitMessage with the full text content and the maximum allowed length of a single message; +// it returns a slice of message chunks that each respect maxLen and avoid splitting fenced code blocks. func SplitMessage(content string, maxLen int) []string { var messages []string From 98afd39913afc07435dcf1e883cb1c447abad786 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Wed, 18 Feb 2026 23:18:17 +0100 Subject: [PATCH 14/19] remove unicode --- pkg/utils/message_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/utils/message_test.go b/pkg/utils/message_test.go index 33f5e51fc..338509437 100644 --- a/pkg/utils/message_test.go +++ b/pkg/utils/message_test.go @@ -79,7 +79,7 @@ func TestSplitMessage(t *testing.T) { }, { name: "Preserve Unicode characters", - content: strings.Repeat("世", 1000), // 3000 bytes + content: strings.Repeat("\u4e16", 1000), // 3000 bytes maxLen: 2000, expectChunks: 2, checkContent: func(t *testing.T, chunks []string) { @@ -87,7 +87,7 @@ func TestSplitMessage(t *testing.T) { // Go strings are UTF-8, if we split mid-rune it would be bad, // but standard slicing might do that. // Let's assume standard behavior is acceptable or check if it produces invalid rune? - if !strings.Contains(chunks[0], "世") { + if !strings.Contains(chunks[0], "\u4e16") { t.Error("Chunk should contain unicode characters") } }, From 0d6b22fb3a8b90a00bc08ba015ec75a95ceb2041 Mon Sep 17 00:00:00 2001 From: Hua Audio <161028864+Huaaudio@users.noreply.github.com> Date: Wed, 18 Feb 2026 23:26:39 +0100 Subject: [PATCH 15/19] Update pkg/utils/message.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/utils/message.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/utils/message.go b/pkg/utils/message.go index 35914f399..1d05950d9 100644 --- a/pkg/utils/message.go +++ b/pkg/utils/message.go @@ -9,8 +9,6 @@ import ( // but may extend to maxLen when needed. // Call SplitMessage with the full text content and the maximum allowed length of a single message; // it returns a slice of message chunks that each respect maxLen and avoid splitting fenced code blocks. -// Call SplitMessage with the full text content and the maximum allowed length of a single message; -// it returns a slice of message chunks that each respect maxLen and avoid splitting fenced code blocks. func SplitMessage(content string, maxLen int) []string { var messages []string From bb0424e1e280c2ccc7861e6b5b431aa05bacca26 Mon Sep 17 00:00:00 2001 From: fipso Date: Thu, 19 Feb 2026 01:29:34 +0100 Subject: [PATCH 16/19] fix: also use max_completion_tokens for gpt5 era models (#445) --- pkg/providers/openai_compat/provider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/providers/openai_compat/provider.go b/pkg/providers/openai_compat/provider.go index 9b404dd77..73fac3435 100644 --- a/pkg/providers/openai_compat/provider.go +++ b/pkg/providers/openai_compat/provider.go @@ -71,7 +71,7 @@ func (p *Provider) Chat(ctx context.Context, messages []Message, tools []ToolDef if maxTokens, ok := asInt(options["max_tokens"]); ok { lowerModel := strings.ToLower(model) - if strings.Contains(lowerModel, "glm") || strings.Contains(lowerModel, "o1") { + if strings.Contains(lowerModel, "glm") || strings.Contains(lowerModel, "o1") || strings.Contains(lowerModel, "gpt-5") { requestBody["max_completion_tokens"] = maxTokens } else { requestBody["max_tokens"] = maxTokens From d167b4743132e2f5e6674bcb9b3d990953e936d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20Xia=28=E5=A4=8F=E6=81=BA=29?= Date: Thu, 19 Feb 2026 11:54:13 +1100 Subject: [PATCH 17/19] dead code cleanup (#210) --- pkg/skills/installer.go | 53 ----------------------------------------- 1 file changed, 53 deletions(-) diff --git a/pkg/skills/installer.go b/pkg/skills/installer.go index a3263c525..0856254e8 100644 --- a/pkg/skills/installer.go +++ b/pkg/skills/installer.go @@ -8,7 +8,6 @@ import ( "net/http" "os" "path/filepath" - "strings" "time" ) @@ -24,12 +23,6 @@ type AvailableSkill struct { Tags []string `json:"tags"` } -type BuiltinSkill struct { - Name string `json:"name"` - Path string `json:"path"` - Enabled bool `json:"enabled"` -} - func NewSkillInstaller(workspace string) *SkillInstaller { return &SkillInstaller{ workspace: workspace, @@ -123,49 +116,3 @@ func (si *SkillInstaller) ListAvailableSkills(ctx context.Context) ([]AvailableS return skills, nil } - -func (si *SkillInstaller) ListBuiltinSkills() []BuiltinSkill { - builtinSkillsDir := filepath.Join(filepath.Dir(si.workspace), "picoclaw", "skills") - - entries, err := os.ReadDir(builtinSkillsDir) - if err != nil { - return nil - } - - var skills []BuiltinSkill - for _, entry := range entries { - if entry.IsDir() { - _ = entry - skillName := entry.Name() - skillFile := filepath.Join(builtinSkillsDir, skillName, "SKILL.md") - - data, err := os.ReadFile(skillFile) - description := "" - if err == nil { - content := string(data) - if idx := strings.Index(content, "\n"); idx > 0 { - firstLine := content[:idx] - if strings.Contains(firstLine, "description:") { - descLine := strings.Index(content[idx:], "\n") - if descLine > 0 { - description = strings.TrimSpace(content[idx+descLine : idx+descLine]) - } - } - } - } - - // skill := BuiltinSkill{ - // Name: skillName, - // Path: description, - // Enabled: true, - // } - - status := "✓" - fmt.Printf(" %s %s\n", status, entry.Name()) - if description != "" { - fmt.Printf(" %s\n", description) - } - } - } - return skills -} From e8afd31b28bf7d0c2e7ebddbc2320bc89f996fe0 Mon Sep 17 00:00:00 2001 From: mattn Date: Thu, 19 Feb 2026 10:02:28 +0900 Subject: [PATCH 18/19] Replace \s+ with [^\S\n]+ to preserve newlines (#299) --- pkg/tools/web.go | 6 ++-- pkg/tools/web_test.go | 74 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/pkg/tools/web.go b/pkg/tools/web.go index 6a6d40ecf..1f5c58ea5 100644 --- a/pkg/tools/web.go +++ b/pkg/tools/web.go @@ -492,8 +492,10 @@ func (t *WebFetchTool) extractText(htmlContent string) string { result = strings.TrimSpace(result) - re = regexp.MustCompile(`\s+`) - result = re.ReplaceAllLiteralString(result, " ") + re = regexp.MustCompile(`[^\S\n]+`) + result = re.ReplaceAllString(result, " ") + re = regexp.MustCompile(`\n{3,}`) + result = re.ReplaceAllString(result, "\n\n") lines := strings.Split(result, "\n") var cleanLines []string diff --git a/pkg/tools/web_test.go b/pkg/tools/web_test.go index a526ea34a..7e6d62213 100644 --- a/pkg/tools/web_test.go +++ b/pkg/tools/web_test.go @@ -234,6 +234,80 @@ func TestWebTool_WebFetch_HTMLExtraction(t *testing.T) { } } +// TestWebFetchTool_extractText verifies text extraction preserves newlines +func TestWebFetchTool_extractText(t *testing.T) { + tool := &WebFetchTool{} + + tests := []struct { + name string + input string + wantFunc func(t *testing.T, got string) + }{ + { + name: "preserves newlines between block elements", + input: "

Title

\n

Paragraph 1

\n

Paragraph 2

", + wantFunc: func(t *testing.T, got string) { + lines := strings.Split(got, "\n") + if len(lines) < 2 { + t.Errorf("Expected multiple lines, got %d: %q", len(lines), got) + } + if !strings.Contains(got, "Title") || !strings.Contains(got, "Paragraph 1") || !strings.Contains(got, "Paragraph 2") { + t.Errorf("Missing expected text: %q", got) + } + }, + }, + { + name: "removes script and style tags", + input: "

Keep this

", + wantFunc: func(t *testing.T, got string) { + if strings.Contains(got, "alert") || strings.Contains(got, "body{}") { + t.Errorf("Expected script/style content removed, got: %q", got) + } + if !strings.Contains(got, "Keep this") { + t.Errorf("Expected 'Keep this' to remain, got: %q", got) + } + }, + }, + { + name: "collapses excessive blank lines", + input: "

A

\n\n\n\n\n

B

", + wantFunc: func(t *testing.T, got string) { + if strings.Contains(got, "\n\n\n") { + t.Errorf("Expected excessive blank lines collapsed, got: %q", got) + } + }, + }, + { + name: "collapses horizontal whitespace", + input: "

hello world

", + wantFunc: func(t *testing.T, got string) { + if strings.Contains(got, " ") { + t.Errorf("Expected spaces collapsed, got: %q", got) + } + if !strings.Contains(got, "hello world") { + t.Errorf("Expected 'hello world', got: %q", got) + } + }, + }, + { + name: "empty input", + input: "", + wantFunc: func(t *testing.T, got string) { + if got != "" { + t.Errorf("Expected empty string, got: %q", got) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tool.extractText(tt.input) + tt.wantFunc(t, got) + }) + } +} + // TestWebTool_WebFetch_MissingDomain verifies error handling for URL without domain func TestWebTool_WebFetch_MissingDomain(t *testing.T) { tool := NewWebFetchTool(50000) From 56a060ff61167c03086a271340574e2f34d28721 Mon Sep 17 00:00:00 2001 From: hsohinna Date: Thu, 19 Feb 2026 14:39:35 +0800 Subject: [PATCH 19/19] feat(onebot): enhance OneBot channel (#192) * fix: change BotStatus type to json.RawMessage and add isAPIResponse function * feat(onebot): add rich media, API callback, keepalive and voice transcription Comprehensive improvements to the OneBot channel for better NapCatQQ compatibility: - Add echo-based API callback mechanism (sendAPIRequest) for request/response correlation via pending map - Add WebSocket ping/pong keepalive (30s ping, 60s read deadline) - Fetch bot self ID via get_login_info on connect/reconnect - Refactor parseMessageContentEx into parseMessageSegments supporting image, record, video, file, reply, face, forward segments - Add voice transcription via Groq transcriber (SetTranscriber) - Switch to message segment array format for sending with auto reply quote via lastMessageID tracking - Add message_sent event handling and detailed notice event processing (recall, poke, group increase/decrease, friend add, etc.) - Use sync/atomic for echoCounter, optimize listen() lock pattern - Clean up pending callbacks on Stop(), defer temp file cleanup - Mount Groq transcriber on OneBot channel in main.go gateway * feat(onebot): add user ID allowlist check for incoming messages - Currently, the agent does not respond to messages sent by users outside the allowlist. * refactor(onebot): simplify channel implementation and add emoji reaction - onebot.go from 1179 to 980 lines (~17%) --- cmd/picoclaw/main.go | 6 + pkg/channels/onebot.go | 707 +++++++++++++++++++++++++++++------------ 2 files changed, 504 insertions(+), 209 deletions(-) diff --git a/cmd/picoclaw/main.go b/cmd/picoclaw/main.go index 128f8c421..36bf2ea83 100644 --- a/cmd/picoclaw/main.go +++ b/cmd/picoclaw/main.go @@ -623,6 +623,12 @@ func gatewayCmd() { logger.InfoC("voice", "Groq transcription attached to Slack channel") } } + if onebotChannel, ok := channelManager.GetChannel("onebot"); ok { + if oc, ok := onebotChannel.(*channels.OneBotChannel); ok { + oc.SetTranscriber(transcriber) + logger.InfoC("voice", "Groq transcription attached to OneBot channel") + } + } } enabledChannels := channelManager.GetEnabledChannels() diff --git a/pkg/channels/onebot.go b/pkg/channels/onebot.go index 5d97fab9c..53e82b44d 100644 --- a/pkg/channels/onebot.go +++ b/pkg/channels/onebot.go @@ -4,9 +4,11 @@ import ( "context" "encoding/json" "fmt" + "os" "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/gorilla/websocket" @@ -14,20 +16,28 @@ import ( "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/utils" + "github.com/sipeed/picoclaw/pkg/voice" ) type OneBotChannel struct { *BaseChannel - config config.OneBotConfig - conn *websocket.Conn - ctx context.Context - cancel context.CancelFunc - dedup map[string]struct{} - dedupRing []string - dedupIdx int - mu sync.Mutex - writeMu sync.Mutex - echoCounter int64 + config config.OneBotConfig + conn *websocket.Conn + ctx context.Context + cancel context.CancelFunc + dedup map[string]struct{} + dedupRing []string + dedupIdx int + mu sync.Mutex + writeMu sync.Mutex + echoCounter int64 + selfID int64 + pending map[string]chan json.RawMessage + pendingMu sync.Mutex + transcriber *voice.GroqTranscriber + lastMessageID sync.Map + pendingEmojiMsg sync.Map } type oneBotRawEvent struct { @@ -43,9 +53,11 @@ type oneBotRawEvent struct { SelfID json.RawMessage `json:"self_id"` Time json.RawMessage `json:"time"` MetaEventType string `json:"meta_event_type"` + NoticeType string `json:"notice_type"` Echo string `json:"echo"` RetCode json.RawMessage `json:"retcode"` - Status BotStatus `json:"status"` + Status json.RawMessage `json:"status"` + Data json.RawMessage `json:"data"` } type BotStatus struct { @@ -53,42 +65,36 @@ type BotStatus struct { Good bool `json:"good"` } +func isAPIResponse(raw json.RawMessage) bool { + if len(raw) == 0 { + return false + } + var s string + if json.Unmarshal(raw, &s) == nil { + return s == "ok" || s == "failed" + } + var bs BotStatus + if json.Unmarshal(raw, &bs) == nil { + return bs.Online || bs.Good + } + return false +} + type oneBotSender struct { UserID json.RawMessage `json:"user_id"` Nickname string `json:"nickname"` Card string `json:"card"` } -type oneBotEvent struct { - PostType string - MessageType string - SubType string - MessageID string - UserID int64 - GroupID int64 - Content string - RawContent string - IsBotMentioned bool - Sender oneBotSender - SelfID int64 - Time int64 - MetaEventType string -} - type oneBotAPIRequest struct { Action string `json:"action"` Params interface{} `json:"params"` Echo string `json:"echo,omitempty"` } -type oneBotSendPrivateMsgParams struct { - UserID int64 `json:"user_id"` - Message string `json:"message"` -} - -type oneBotSendGroupMsgParams struct { - GroupID int64 `json:"group_id"` - Message string `json:"message"` +type oneBotMessageSegment struct { + Type string `json:"type"` + Data map[string]interface{} `json:"data"` } func NewOneBotChannel(cfg config.OneBotConfig, messageBus *bus.MessageBus) (*OneBotChannel, error) { @@ -101,9 +107,30 @@ func NewOneBotChannel(cfg config.OneBotConfig, messageBus *bus.MessageBus) (*One dedup: make(map[string]struct{}, dedupSize), dedupRing: make([]string, dedupSize), dedupIdx: 0, + pending: make(map[string]chan json.RawMessage), }, nil } +func (c *OneBotChannel) SetTranscriber(transcriber *voice.GroqTranscriber) { + c.transcriber = transcriber +} + +func (c *OneBotChannel) setMsgEmojiLike(messageID string, emojiID int, set bool) { + go func() { + _, err := c.sendAPIRequest("set_msg_emoji_like", map[string]interface{}{ + "message_id": messageID, + "emoji_id": emojiID, + "set": set, + }, 5*time.Second) + if err != nil { + logger.DebugCF("onebot", "Failed to set emoji like", map[string]interface{}{ + "message_id": messageID, + "error": err.Error(), + }) + } + }() +} + func (c *OneBotChannel) Start(ctx context.Context) error { if c.config.WSUrl == "" { return fmt.Errorf("OneBot ws_url not configured") @@ -121,12 +148,12 @@ func (c *OneBotChannel) Start(ctx context.Context) error { }) } else { go c.listen() + c.fetchSelfID() } if c.config.ReconnectInterval > 0 { go c.reconnectLoop() } else { - // If reconnect is disabled but initial connection failed, we cannot recover if c.conn == nil { return fmt.Errorf("failed to connect to OneBot and reconnect is disabled") } @@ -152,14 +179,141 @@ func (c *OneBotChannel) connect() error { return err } + conn.SetPongHandler(func(appData string) error { + _ = conn.SetReadDeadline(time.Now().Add(60 * time.Second)) + return nil + }) + _ = conn.SetReadDeadline(time.Now().Add(60 * time.Second)) + c.mu.Lock() c.conn = conn c.mu.Unlock() + go c.pinger(conn) + logger.InfoC("onebot", "WebSocket connected") return nil } +func (c *OneBotChannel) pinger(conn *websocket.Conn) { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + for { + select { + case <-c.ctx.Done(): + return + case <-ticker.C: + c.writeMu.Lock() + err := conn.WriteMessage(websocket.PingMessage, nil) + c.writeMu.Unlock() + if err != nil { + logger.DebugCF("onebot", "Ping write failed, stopping pinger", map[string]interface{}{ + "error": err.Error(), + }) + return + } + } + } +} + +func (c *OneBotChannel) fetchSelfID() { + resp, err := c.sendAPIRequest("get_login_info", nil, 5*time.Second) + if err != nil { + logger.WarnCF("onebot", "Failed to get_login_info", map[string]interface{}{ + "error": err.Error(), + }) + return + } + + type loginInfo struct { + UserID json.RawMessage `json:"user_id"` + Nickname string `json:"nickname"` + } + for _, extract := range []func() (*loginInfo, error){ + func() (*loginInfo, error) { + var w struct { + Data loginInfo `json:"data"` + } + err := json.Unmarshal(resp, &w) + return &w.Data, err + }, + func() (*loginInfo, error) { + var f loginInfo + err := json.Unmarshal(resp, &f) + return &f, err + }, + } { + info, err := extract() + if err != nil || len(info.UserID) == 0 { + continue + } + if uid, err := parseJSONInt64(info.UserID); err == nil && uid > 0 { + atomic.StoreInt64(&c.selfID, uid) + logger.InfoCF("onebot", "Bot self ID retrieved", map[string]interface{}{ + "self_id": uid, + "nickname": info.Nickname, + }) + return + } + } + + logger.WarnCF("onebot", "Could not parse self ID from get_login_info response", map[string]interface{}{ + "response": string(resp), + }) +} + +func (c *OneBotChannel) sendAPIRequest(action string, params interface{}, timeout time.Duration) (json.RawMessage, error) { + c.mu.Lock() + conn := c.conn + c.mu.Unlock() + + if conn == nil { + return nil, fmt.Errorf("WebSocket not connected") + } + + echo := fmt.Sprintf("api_%d_%d", time.Now().UnixNano(), atomic.AddInt64(&c.echoCounter, 1)) + + ch := make(chan json.RawMessage, 1) + c.pendingMu.Lock() + c.pending[echo] = ch + c.pendingMu.Unlock() + + defer func() { + c.pendingMu.Lock() + delete(c.pending, echo) + c.pendingMu.Unlock() + }() + + req := oneBotAPIRequest{ + Action: action, + Params: params, + Echo: echo, + } + + data, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("failed to marshal API request: %w", err) + } + + c.writeMu.Lock() + err = conn.WriteMessage(websocket.TextMessage, data) + c.writeMu.Unlock() + + if err != nil { + return nil, fmt.Errorf("failed to write API request: %w", err) + } + + select { + case resp := <-ch: + return resp, nil + case <-time.After(timeout): + return nil, fmt.Errorf("API request %s timed out after %v", action, timeout) + case <-c.ctx.Done(): + return nil, fmt.Errorf("context cancelled") + } +} + func (c *OneBotChannel) reconnectLoop() { interval := time.Duration(c.config.ReconnectInterval) * time.Second if interval < 5*time.Second { @@ -183,6 +337,7 @@ func (c *OneBotChannel) reconnectLoop() { }) } else { go c.listen() + c.fetchSelfID() } } } @@ -197,6 +352,13 @@ func (c *OneBotChannel) Stop(ctx context.Context) error { c.cancel() } + c.pendingMu.Lock() + for echo, ch := range c.pending { + close(ch) + delete(c.pending, echo) + } + c.pendingMu.Unlock() + c.mu.Lock() if c.conn != nil { c.conn.Close() @@ -225,10 +387,7 @@ func (c *OneBotChannel) Send(ctx context.Context, msg bus.OutboundMessage) error return err } - c.writeMu.Lock() - c.echoCounter++ - echo := fmt.Sprintf("send_%d", c.echoCounter) - c.writeMu.Unlock() + echo := fmt.Sprintf("send_%d", atomic.AddInt64(&c.echoCounter, 1)) req := oneBotAPIRequest{ Action: action, @@ -252,67 +411,78 @@ func (c *OneBotChannel) Send(ctx context.Context, msg bus.OutboundMessage) error return err } + if msgID, ok := c.pendingEmojiMsg.LoadAndDelete(msg.ChatID); ok { + if mid, ok := msgID.(string); ok && mid != "" { + c.setMsgEmojiLike(mid, 289, false) + } + } + return nil } +func (c *OneBotChannel) buildMessageSegments(chatID, content string) []oneBotMessageSegment { + var segments []oneBotMessageSegment + + if lastMsgID, ok := c.lastMessageID.Load(chatID); ok { + if msgID, ok := lastMsgID.(string); ok && msgID != "" { + segments = append(segments, oneBotMessageSegment{ + Type: "reply", + Data: map[string]interface{}{"id": msgID}, + }) + } + } + + segments = append(segments, oneBotMessageSegment{ + Type: "text", + Data: map[string]interface{}{"text": content}, + }) + + return segments +} + func (c *OneBotChannel) buildSendRequest(msg bus.OutboundMessage) (string, interface{}, error) { chatID := msg.ChatID + segments := c.buildMessageSegments(chatID, msg.Content) - if len(chatID) > 6 && chatID[:6] == "group:" { - groupID, err := strconv.ParseInt(chatID[6:], 10, 64) - if err != nil { - return "", nil, fmt.Errorf("invalid group ID in chatID: %s", chatID) - } - return "send_group_msg", oneBotSendGroupMsgParams{ - GroupID: groupID, - Message: msg.Content, - }, nil + var action, idKey string + var rawID string + if rest, ok := strings.CutPrefix(chatID, "group:"); ok { + action, idKey, rawID = "send_group_msg", "group_id", rest + } else if rest, ok := strings.CutPrefix(chatID, "private:"); ok { + action, idKey, rawID = "send_private_msg", "user_id", rest + } else { + action, idKey, rawID = "send_private_msg", "user_id", chatID } - if len(chatID) > 8 && chatID[:8] == "private:" { - userID, err := strconv.ParseInt(chatID[8:], 10, 64) - if err != nil { - return "", nil, fmt.Errorf("invalid user ID in chatID: %s", chatID) - } - return "send_private_msg", oneBotSendPrivateMsgParams{ - UserID: userID, - Message: msg.Content, - }, nil - } - - userID, err := strconv.ParseInt(chatID, 10, 64) + id, err := strconv.ParseInt(rawID, 10, 64) if err != nil { - return "", nil, fmt.Errorf("invalid chatID for OneBot: %s", chatID) + return "", nil, fmt.Errorf("invalid %s in chatID: %s", idKey, chatID) } - - return "send_private_msg", oneBotSendPrivateMsgParams{ - UserID: userID, - Message: msg.Content, - }, nil + return action, map[string]interface{}{idKey: id, "message": segments}, nil } func (c *OneBotChannel) listen() { + c.mu.Lock() + conn := c.conn + c.mu.Unlock() + + if conn == nil { + logger.WarnC("onebot", "WebSocket connection is nil, listener exiting") + return + } + for { select { case <-c.ctx.Done(): return default: - c.mu.Lock() - conn := c.conn - c.mu.Unlock() - - if conn == nil { - logger.WarnC("onebot", "WebSocket connection is nil, listener exiting") - return - } - _, message, err := conn.ReadMessage() if err != nil { logger.ErrorCF("onebot", "WebSocket read error", map[string]interface{}{ "error": err.Error(), }) c.mu.Lock() - if c.conn != nil { + if c.conn == conn { c.conn.Close() c.conn = nil } @@ -320,10 +490,7 @@ func (c *OneBotChannel) listen() { return } - logger.DebugCF("onebot", "Raw WebSocket message received", map[string]interface{}{ - "length": len(message), - "payload": string(message), - }) + _ = conn.SetReadDeadline(time.Now().Add(60 * time.Second)) var raw oneBotRawEvent if err := json.Unmarshal(message, &raw); err != nil { @@ -334,20 +501,37 @@ func (c *OneBotChannel) listen() { continue } - if raw.Echo != "" || raw.Status.Online || raw.Status.Good { - logger.DebugCF("onebot", "Received API response, skipping", map[string]interface{}{ - "echo": raw.Echo, - "status": raw.Status, - }) + logger.DebugCF("onebot", "WebSocket event", map[string]interface{}{ + "length": len(message), + "post_type": raw.PostType, + "sub_type": raw.SubType, + }) + + if raw.Echo != "" { + c.pendingMu.Lock() + ch, ok := c.pending[raw.Echo] + c.pendingMu.Unlock() + + if ok { + select { + case ch <- message: + default: + } + } else { + logger.DebugCF("onebot", "Received API response (no waiter)", map[string]interface{}{ + "echo": raw.Echo, + "status": string(raw.Status), + }) + } continue } - logger.DebugCF("onebot", "Parsed raw event", map[string]interface{}{ - "post_type": raw.PostType, - "message_type": raw.MessageType, - "sub_type": raw.SubType, - "meta_event_type": raw.MetaEventType, - }) + if isAPIResponse(raw.Status) { + logger.DebugCF("onebot", "Received API response without echo, skipping", map[string]interface{}{ + "status": string(raw.Status), + }) + continue + } c.handleRawEvent(&raw) } @@ -386,9 +570,12 @@ func parseJSONString(raw json.RawMessage) string { type parseMessageResult struct { Text string IsBotMentioned bool + Media []string + LocalFiles []string + ReplyTo string } -func parseMessageContentEx(raw json.RawMessage, selfID int64) parseMessageResult { +func (c *OneBotChannel) parseMessageSegments(raw json.RawMessage, selfID int64) parseMessageResult { if len(raw) == 0 { return parseMessageResult{} } @@ -408,60 +595,155 @@ func parseMessageContentEx(raw json.RawMessage, selfID int64) parseMessageResult } var segments []map[string]interface{} - if err := json.Unmarshal(raw, &segments); err == nil { - var text string - mentioned := false - selfIDStr := strconv.FormatInt(selfID, 10) - for _, seg := range segments { - segType, _ := seg["type"].(string) - data, _ := seg["data"].(map[string]interface{}) - switch segType { - case "text": - if data != nil { - if t, ok := data["text"].(string); ok { - text += t - } + if err := json.Unmarshal(raw, &segments); err != nil { + return parseMessageResult{} + } + + var textParts []string + mentioned := false + selfIDStr := strconv.FormatInt(selfID, 10) + var media []string + var localFiles []string + var replyTo string + + for _, seg := range segments { + segType, _ := seg["type"].(string) + data, _ := seg["data"].(map[string]interface{}) + + switch segType { + case "text": + if data != nil { + if t, ok := data["text"].(string); ok { + textParts = append(textParts, t) } - case "at": - if data != nil && selfID > 0 { - qqVal := fmt.Sprintf("%v", data["qq"]) - if qqVal == selfIDStr || qqVal == "all" { - mentioned = true + } + + case "at": + if data != nil && selfID > 0 { + qqVal := fmt.Sprintf("%v", data["qq"]) + if qqVal == selfIDStr || qqVal == "all" { + mentioned = true + } + } + + case "image", "video", "file": + if data != nil { + url, _ := data["url"].(string) + if url != "" { + defaults := map[string]string{"image": "image.jpg", "video": "video.mp4", "file": "file"} + filename := defaults[segType] + if f, ok := data["file"].(string); ok && f != "" { + filename = f + } else if n, ok := data["name"].(string); ok && n != "" { + filename = n + } + localPath := utils.DownloadFile(url, filename, utils.DownloadOptions{ + LoggerPrefix: "onebot", + }) + if localPath != "" { + media = append(media, localPath) + localFiles = append(localFiles, localPath) + textParts = append(textParts, fmt.Sprintf("[%s]", segType)) } } } + + case "record": + if data != nil { + url, _ := data["url"].(string) + if url != "" { + localPath := utils.DownloadFile(url, "voice.amr", utils.DownloadOptions{ + LoggerPrefix: "onebot", + }) + if localPath != "" { + localFiles = append(localFiles, localPath) + if c.transcriber != nil && c.transcriber.IsAvailable() { + tctx, tcancel := context.WithTimeout(c.ctx, 30*time.Second) + result, err := c.transcriber.Transcribe(tctx, localPath) + tcancel() + if err != nil { + logger.WarnCF("onebot", "Voice transcription failed", map[string]interface{}{ + "error": err.Error(), + }) + textParts = append(textParts, "[voice (transcription failed)]") + media = append(media, localPath) + } else { + textParts = append(textParts, fmt.Sprintf("[voice transcription: %s]", result.Text)) + } + } else { + textParts = append(textParts, "[voice]") + media = append(media, localPath) + } + } + } + } + + case "reply": + if data != nil { + if id, ok := data["id"]; ok { + replyTo = fmt.Sprintf("%v", id) + } + } + + case "face": + if data != nil { + faceID, _ := data["id"] + textParts = append(textParts, fmt.Sprintf("[face:%v]", faceID)) + } + + case "forward": + textParts = append(textParts, "[forward message]") + + default: + } - return parseMessageResult{Text: strings.TrimSpace(text), IsBotMentioned: mentioned} } - return parseMessageResult{} + + return parseMessageResult{ + Text: strings.TrimSpace(strings.Join(textParts, "")), + IsBotMentioned: mentioned, + Media: media, + LocalFiles: localFiles, + ReplyTo: replyTo, + } } func (c *OneBotChannel) handleRawEvent(raw *oneBotRawEvent) { switch raw.PostType { case "message": - evt, err := c.normalizeMessageEvent(raw) - if err != nil { - logger.WarnCF("onebot", "Failed to normalize message event", map[string]interface{}{ - "error": err.Error(), - }) - return + if userID, err := parseJSONInt64(raw.UserID); err == nil && userID > 0 { + if !c.IsAllowed(strconv.FormatInt(userID, 10)) { + logger.DebugCF("onebot", "Message rejected by allowlist", map[string]interface{}{ + "user_id": userID, + }) + return + } } - c.handleMessage(evt) + c.handleMessage(raw) + + case "message_sent": + logger.DebugCF("onebot", "Bot sent message event", map[string]interface{}{ + "message_type": raw.MessageType, + "message_id": parseJSONString(raw.MessageID), + }) + case "meta_event": c.handleMetaEvent(raw) + case "notice": - logger.DebugCF("onebot", "Notice event received", map[string]interface{}{ - "sub_type": raw.SubType, - }) + c.handleNoticeEvent(raw) + case "request": logger.DebugCF("onebot", "Request event received", map[string]interface{}{ "sub_type": raw.SubType, }) + case "": logger.DebugCF("onebot", "Event with empty post_type (possibly API response)", map[string]interface{}{ "echo": raw.Echo, "status": raw.Status, }) + default: logger.DebugCF("onebot", "Unknown post_type", map[string]interface{}{ "post_type": raw.PostType, @@ -469,18 +751,51 @@ func (c *OneBotChannel) handleRawEvent(raw *oneBotRawEvent) { } } -func (c *OneBotChannel) normalizeMessageEvent(raw *oneBotRawEvent) (*oneBotEvent, error) { +func (c *OneBotChannel) handleMetaEvent(raw *oneBotRawEvent) { + if raw.MetaEventType == "lifecycle" { + logger.InfoCF("onebot", "Lifecycle event", map[string]interface{}{"sub_type": raw.SubType}) + } else if raw.MetaEventType != "heartbeat" { + logger.DebugCF("onebot", "Meta event: "+raw.MetaEventType, nil) + } +} + +func (c *OneBotChannel) handleNoticeEvent(raw *oneBotRawEvent) { + fields := map[string]interface{}{ + "notice_type": raw.NoticeType, + "sub_type": raw.SubType, + "group_id": parseJSONString(raw.GroupID), + "user_id": parseJSONString(raw.UserID), + "message_id": parseJSONString(raw.MessageID), + } + switch raw.NoticeType { + case "group_recall", "group_increase", "group_decrease", + "friend_add", "group_admin", "group_ban": + logger.InfoCF("onebot", "Notice: "+raw.NoticeType, fields) + default: + logger.DebugCF("onebot", "Notice: "+raw.NoticeType, fields) + } +} + +func (c *OneBotChannel) handleMessage(raw *oneBotRawEvent) { + // Parse fields from raw event userID, err := parseJSONInt64(raw.UserID) if err != nil { - return nil, fmt.Errorf("parse user_id: %w (raw: %s)", err, string(raw.UserID)) + logger.WarnCF("onebot", "Failed to parse user_id", map[string]interface{}{ + "error": err.Error(), + "raw": string(raw.UserID), + }) + return } groupID, _ := parseJSONInt64(raw.GroupID) selfID, _ := parseJSONInt64(raw.SelfID) - ts, _ := parseJSONInt64(raw.Time) messageID := parseJSONString(raw.MessageID) - parsed := parseMessageContentEx(raw.Message, selfID) + if selfID == 0 { + selfID = atomic.LoadInt64(&c.selfID) + } + + parsed := c.parseMessageSegments(raw.Message, selfID) isBotMentioned := parsed.IsBotMentioned content := raw.RawMessage @@ -495,6 +810,10 @@ func (c *OneBotChannel) normalizeMessageEvent(raw *oneBotRawEvent) (*oneBotEvent } } + if parsed.Text != "" && content != parsed.Text && (len(parsed.Media) > 0 || parsed.ReplyTo != "") { + content = parsed.Text + } + var sender oneBotSender if len(raw.Sender) > 0 { if err := json.Unmarshal(raw.Sender, &sender); err != nil { @@ -505,137 +824,107 @@ func (c *OneBotChannel) normalizeMessageEvent(raw *oneBotRawEvent) (*oneBotEvent } } - logger.DebugCF("onebot", "Normalized message event", map[string]interface{}{ - "message_type": raw.MessageType, - "user_id": userID, - "group_id": groupID, - "message_id": messageID, - "content_len": len(content), - "nickname": sender.Nickname, - }) - - return &oneBotEvent{ - PostType: raw.PostType, - MessageType: raw.MessageType, - SubType: raw.SubType, - MessageID: messageID, - UserID: userID, - GroupID: groupID, - Content: content, - RawContent: raw.RawMessage, - IsBotMentioned: isBotMentioned, - Sender: sender, - SelfID: selfID, - Time: ts, - MetaEventType: raw.MetaEventType, - }, nil -} - -func (c *OneBotChannel) handleMetaEvent(raw *oneBotRawEvent) { - switch raw.MetaEventType { - case "lifecycle": - logger.InfoCF("onebot", "Lifecycle event", map[string]interface{}{ - "sub_type": raw.SubType, - }) - case "heartbeat": - logger.DebugC("onebot", "Heartbeat received") - default: - logger.DebugCF("onebot", "Unknown meta_event_type", map[string]interface{}{ - "meta_event_type": raw.MetaEventType, - }) + // Clean up temp files when done + if len(parsed.LocalFiles) > 0 { + defer func() { + for _, f := range parsed.LocalFiles { + if err := os.Remove(f); err != nil { + logger.DebugCF("onebot", "Failed to remove temp file", map[string]interface{}{ + "path": f, + "error": err.Error(), + }) + } + } + }() } -} -func (c *OneBotChannel) handleMessage(evt *oneBotEvent) { - if c.isDuplicate(evt.MessageID) { + if c.isDuplicate(messageID) { logger.DebugCF("onebot", "Duplicate message, skipping", map[string]interface{}{ - "message_id": evt.MessageID, + "message_id": messageID, }) return } - content := evt.Content if content == "" { logger.DebugCF("onebot", "Received empty message, ignoring", map[string]interface{}{ - "message_id": evt.MessageID, + "message_id": messageID, }) return } - senderID := strconv.FormatInt(evt.UserID, 10) + senderID := strconv.FormatInt(userID, 10) var chatID string metadata := map[string]string{ - "message_id": evt.MessageID, + "message_id": messageID, } - switch evt.MessageType { + if parsed.ReplyTo != "" { + metadata["reply_to_message_id"] = parsed.ReplyTo + } + + switch raw.MessageType { case "private": chatID = "private:" + senderID - logger.InfoCF("onebot", "Received private message", map[string]interface{}{ - "sender": senderID, - "message_id": evt.MessageID, - "length": len(content), - "content": truncate(content, 100), - }) case "group": - groupIDStr := strconv.FormatInt(evt.GroupID, 10) + groupIDStr := strconv.FormatInt(groupID, 10) chatID = "group:" + groupIDStr metadata["group_id"] = groupIDStr - senderUserID, _ := parseJSONInt64(evt.Sender.UserID) + senderUserID, _ := parseJSONInt64(sender.UserID) if senderUserID > 0 { metadata["sender_user_id"] = strconv.FormatInt(senderUserID, 10) } - if evt.Sender.Card != "" { - metadata["sender_name"] = evt.Sender.Card - } else if evt.Sender.Nickname != "" { - metadata["sender_name"] = evt.Sender.Nickname + if sender.Card != "" { + metadata["sender_name"] = sender.Card + } else if sender.Nickname != "" { + metadata["sender_name"] = sender.Nickname } - triggered, strippedContent := c.checkGroupTrigger(content, evt.IsBotMentioned) + triggered, strippedContent := c.checkGroupTrigger(content, isBotMentioned) if !triggered { logger.DebugCF("onebot", "Group message ignored (no trigger)", map[string]interface{}{ "sender": senderID, "group": groupIDStr, - "is_mentioned": evt.IsBotMentioned, + "is_mentioned": isBotMentioned, "content": truncate(content, 100), }) return } content = strippedContent - logger.InfoCF("onebot", "Received group message", map[string]interface{}{ - "sender": senderID, - "group": groupIDStr, - "message_id": evt.MessageID, - "is_mentioned": evt.IsBotMentioned, - "length": len(content), - "content": truncate(content, 100), - }) - default: logger.WarnCF("onebot", "Unknown message type, cannot route", map[string]interface{}{ - "type": evt.MessageType, - "message_id": evt.MessageID, - "user_id": evt.UserID, + "type": raw.MessageType, + "message_id": messageID, + "user_id": userID, }) return } - if evt.Sender.Nickname != "" { - metadata["nickname"] = evt.Sender.Nickname - } - - logger.DebugCF("onebot", "Forwarding message to bus", map[string]interface{}{ - "sender_id": senderID, - "chat_id": chatID, - "content": truncate(content, 100), + logger.InfoCF("onebot", "Received "+raw.MessageType+" message", map[string]interface{}{ + "sender": senderID, + "chat_id": chatID, + "message_id": messageID, + "length": len(content), + "content": truncate(content, 100), + "media_count": len(parsed.Media), }) - c.HandleMessage(senderID, chatID, content, []string{}, metadata) + if sender.Nickname != "" { + metadata["nickname"] = sender.Nickname + } + + c.lastMessageID.Store(chatID, messageID) + + if raw.MessageType == "group" && messageID != "" && messageID != "0" { + c.setMsgEmojiLike(messageID, 289, true) + c.pendingEmojiMsg.Store(chatID, messageID) + } + + c.HandleMessage(senderID, chatID, content, parsed.Media, metadata) } func (c *OneBotChannel) isDuplicate(messageID string) bool {