feat: universal message chunking with rune support and sentence awareness
This commit is contained in:
Andrew
parent
2fb2a733d4
commit
6e1b3c845d
10 changed files with 466 additions and 201 deletions
|
|
@ -120,12 +120,12 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) erro
|
|||
return fmt.Errorf("channel ID is empty")
|
||||
}
|
||||
|
||||
runes := []rune(msg.Content)
|
||||
if len(runes) == 0 {
|
||||
return nil
|
||||
limit := c.config.MaxMessageLength
|
||||
if limit <= 0 {
|
||||
limit = 1900
|
||||
}
|
||||
|
||||
chunks := utils.SplitMessage(msg.Content, 2000) // Split messages into chunks, Discord length limit: 2000 chars
|
||||
chunks := utils.SplitMessage(msg.Content, limit)
|
||||
|
||||
for _, chunk := range chunks {
|
||||
if err := c.sendChunk(ctx, channelID, chunk); err != nil {
|
||||
|
|
|
|||
|
|
@ -501,22 +501,47 @@ func (c *LINEChannel) Send(ctx context.Context, msg bus.OutboundMessage) error {
|
|||
}
|
||||
|
||||
// Try reply token first (free, valid for ~25 seconds)
|
||||
hasReplyToken := false
|
||||
var tokenEntry replyTokenEntry
|
||||
if entry, ok := c.replyTokens.LoadAndDelete(msg.ChatID); ok {
|
||||
tokenEntry := entry.(replyTokenEntry)
|
||||
tokenEntry = entry.(replyTokenEntry)
|
||||
if time.Since(tokenEntry.timestamp) < lineReplyTokenMaxAge {
|
||||
if err := c.sendReply(ctx, tokenEntry.token, msg.Content, quoteToken); err == nil {
|
||||
logger.DebugCF("line", "Message sent via Reply API", map[string]interface{}{
|
||||
"chat_id": msg.ChatID,
|
||||
"quoted": quoteToken != "",
|
||||
})
|
||||
return nil
|
||||
}
|
||||
logger.DebugC("line", "Reply API failed, falling back to Push API")
|
||||
hasReplyToken = true
|
||||
}
|
||||
}
|
||||
|
||||
limit := c.config.MaxMessageLength
|
||||
if limit <= 0 {
|
||||
limit = 4000
|
||||
}
|
||||
|
||||
chunks := utils.SplitMessage(msg.Content, limit)
|
||||
|
||||
for i, chunk := range chunks {
|
||||
currentQuoteToken := ""
|
||||
if i == 0 {
|
||||
currentQuoteToken = quoteToken
|
||||
}
|
||||
|
||||
if i == 0 && hasReplyToken {
|
||||
if err := c.sendReply(ctx, tokenEntry.token, chunk, currentQuoteToken); err == nil {
|
||||
logger.DebugCF("line", "Message chunk sent via Reply API", map[string]interface{}{
|
||||
"chat_id": msg.ChatID,
|
||||
"quoted": currentQuoteToken != "",
|
||||
"chunk": i + 1,
|
||||
})
|
||||
continue
|
||||
}
|
||||
logger.DebugC("line", "Reply API failed for chunk, falling back to Push API")
|
||||
}
|
||||
|
||||
// Fall back to Push API
|
||||
return c.sendPush(ctx, msg.ChatID, msg.Content, quoteToken)
|
||||
if err := c.sendPush(ctx, msg.ChatID, chunk, currentQuoteToken); err != nil {
|
||||
return fmt.Errorf("failed to send chunk %d via push: %w", i+1, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildTextMessage creates a text message object, optionally with quoteToken.
|
||||
|
|
|
|||
|
|
@ -119,8 +119,17 @@ func (c *SlackChannel) Send(ctx context.Context, msg bus.OutboundMessage) error
|
|||
return fmt.Errorf("invalid slack chat ID: %s", msg.ChatID)
|
||||
}
|
||||
|
||||
limit := c.config.MaxMessageLength
|
||||
if limit <= 0 {
|
||||
limit = 3000
|
||||
}
|
||||
|
||||
chunks := utils.SplitMessage(msg.Content, limit)
|
||||
|
||||
acked := false
|
||||
for i, chunk := range chunks {
|
||||
opts := []slack.MsgOption{
|
||||
slack.MsgOptionText(msg.Content, false),
|
||||
slack.MsgOptionText(chunk, false),
|
||||
}
|
||||
|
||||
if threadTS != "" {
|
||||
|
|
@ -129,9 +138,10 @@ func (c *SlackChannel) Send(ctx context.Context, msg bus.OutboundMessage) error
|
|||
|
||||
_, _, err := c.api.PostMessageContext(ctx, channelID, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send slack message: %w", err)
|
||||
return fmt.Errorf("failed to send slack chunk %d: %w", i+1, err)
|
||||
}
|
||||
|
||||
if !acked {
|
||||
if ref, ok := c.pendingAcks.LoadAndDelete(msg.ChatID); ok {
|
||||
msgRef := ref.(slackMessageRef)
|
||||
c.api.AddReaction("white_check_mark", slack.ItemRef{
|
||||
|
|
@ -139,10 +149,14 @@ func (c *SlackChannel) Send(ctx context.Context, msg bus.OutboundMessage) error
|
|||
Timestamp: msgRef.Timestamp,
|
||||
})
|
||||
}
|
||||
acked = true
|
||||
}
|
||||
}
|
||||
|
||||
logger.DebugCF("slack", "Message sent", map[string]interface{}{
|
||||
"channel_id": channelID,
|
||||
"thread_ts": threadTS,
|
||||
"chunks": len(chunks),
|
||||
})
|
||||
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -164,19 +164,29 @@ func (c *TelegramChannel) Send(ctx context.Context, msg bus.OutboundMessage) err
|
|||
c.stopThinking.Delete(msg.ChatID)
|
||||
}
|
||||
|
||||
htmlContent := markdownToTelegramHTML(msg.Content)
|
||||
limit := c.config.Channels.Telegram.MaxMessageLength
|
||||
if limit <= 0 {
|
||||
limit = 3500 // Leave space for HTML tags
|
||||
}
|
||||
|
||||
// Try to edit placeholder
|
||||
chunks := utils.SplitMessage(msg.Content, limit)
|
||||
|
||||
for i, chunk := range chunks {
|
||||
htmlContent := markdownToTelegramHTML(chunk)
|
||||
|
||||
// Try to edit placeholder only for the first chunk
|
||||
if i == 0 {
|
||||
if pID, ok := c.placeholders.Load(msg.ChatID); ok {
|
||||
c.placeholders.Delete(msg.ChatID)
|
||||
editMsg := tu.EditMessageText(tu.ID(chatID), pID.(int), htmlContent)
|
||||
editMsg.ParseMode = telego.ModeHTML
|
||||
|
||||
if _, err = c.bot.EditMessageText(ctx, editMsg); err == nil {
|
||||
return nil
|
||||
continue
|
||||
}
|
||||
// Fallback to new message if edit fails
|
||||
}
|
||||
}
|
||||
|
||||
tgMsg := tu.Message(tu.ID(chatID), htmlContent)
|
||||
tgMsg.ParseMode = telego.ModeHTML
|
||||
|
|
@ -186,9 +196,13 @@ func (c *TelegramChannel) Send(ctx context.Context, msg bus.OutboundMessage) err
|
|||
"error": err.Error(),
|
||||
})
|
||||
tgMsg.ParseMode = ""
|
||||
tgMsg.Text = chunk // Use raw chunk if HTML fails
|
||||
_, err = c.bot.SendMessage(ctx, tgMsg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,22 +10,17 @@ import (
|
|||
"github.com/caarlos0/env/v11"
|
||||
)
|
||||
|
||||
// rrCounter is a global counter for round-robin load balancing across models.
|
||||
var rrCounter atomic.Uint64
|
||||
|
||||
// FlexibleStringSlice is a []string that also accepts JSON numbers,
|
||||
// so allow_from can contain both "123" and 123.
|
||||
type FlexibleStringSlice []string
|
||||
|
||||
func (f *FlexibleStringSlice) UnmarshalJSON(data []byte) error {
|
||||
// Try []string first
|
||||
var ss []string
|
||||
if err := json.Unmarshal(data, &ss); err == nil {
|
||||
*f = ss
|
||||
return nil
|
||||
}
|
||||
|
||||
// Try []interface{} to handle mixed types
|
||||
var raw []interface{}
|
||||
if err := json.Unmarshal(data, &raw); err != nil {
|
||||
return err
|
||||
|
|
@ -52,15 +47,13 @@ type Config struct {
|
|||
Session SessionConfig `json:"session,omitempty"`
|
||||
Channels ChannelsConfig `json:"channels"`
|
||||
Providers ProvidersConfig `json:"providers,omitempty"`
|
||||
ModelList []ModelConfig `json:"model_list"` // New model-centric provider configuration
|
||||
ModelList []ModelConfig `json:"model_list"`
|
||||
Gateway GatewayConfig `json:"gateway"`
|
||||
Tools ToolsConfig `json:"tools"`
|
||||
Heartbeat HeartbeatConfig `json:"heartbeat"`
|
||||
Devices DevicesConfig `json:"devices"`
|
||||
}
|
||||
|
||||
// MarshalJSON implements custom JSON marshaling for Config
|
||||
// to omit providers section when empty and session when empty
|
||||
func (c Config) MarshalJSON() ([]byte, error) {
|
||||
type Alias Config
|
||||
aux := &struct {
|
||||
|
|
@ -71,12 +64,10 @@ func (c Config) MarshalJSON() ([]byte, error) {
|
|||
Alias: (*Alias)(&c),
|
||||
}
|
||||
|
||||
// Only include providers if not empty
|
||||
if !c.Providers.IsEmpty() {
|
||||
aux.Providers = &c.Providers
|
||||
}
|
||||
|
||||
// Only include session if not empty
|
||||
if c.Session.DMScope != "" || len(c.Session.IdentityLinks) > 0 {
|
||||
aux.Session = &c.Session
|
||||
}
|
||||
|
|
@ -89,9 +80,6 @@ type AgentsConfig struct {
|
|||
List []AgentConfig `json:"list,omitempty"`
|
||||
}
|
||||
|
||||
// AgentModelConfig supports both string and structured model config.
|
||||
// String format: "gpt-4" (just primary, no fallbacks)
|
||||
// Object format: {"primary": "gpt-4", "fallbacks": ["claude-haiku"]}
|
||||
type AgentModelConfig struct {
|
||||
Primary string `json:"primary,omitempty"`
|
||||
Fallbacks []string `json:"fallbacks,omitempty"`
|
||||
|
|
@ -101,7 +89,6 @@ func (m *AgentModelConfig) UnmarshalJSON(data []byte) error {
|
|||
var s string
|
||||
if err := json.Unmarshal(data, &s); err == nil {
|
||||
m.Primary = s
|
||||
m.Fallbacks = nil
|
||||
return nil
|
||||
}
|
||||
type raw struct {
|
||||
|
|
@ -196,6 +183,7 @@ type WhatsAppConfig struct {
|
|||
Enabled bool `json:"enabled" env:"PICOCLAW_CHANNELS_WHATSAPP_ENABLED"`
|
||||
BridgeURL string `json:"bridge_url" env:"PICOCLAW_CHANNELS_WHATSAPP_BRIDGE_URL"`
|
||||
AllowFrom FlexibleStringSlice `json:"allow_from" env:"PICOCLAW_CHANNELS_WHATSAPP_ALLOW_FROM"`
|
||||
MaxMessageLength int `json:"max_message_length,omitempty" env:"PICOCLAW_CHANNELS_WHATSAPP_MAX_MESSAGE_LENGTH"`
|
||||
}
|
||||
|
||||
type TelegramConfig struct {
|
||||
|
|
@ -203,6 +191,7 @@ type TelegramConfig struct {
|
|||
Token string `json:"token" env:"PICOCLAW_CHANNELS_TELEGRAM_TOKEN"`
|
||||
Proxy string `json:"proxy" env:"PICOCLAW_CHANNELS_TELEGRAM_PROXY"`
|
||||
AllowFrom FlexibleStringSlice `json:"allow_from" env:"PICOCLAW_CHANNELS_TELEGRAM_ALLOW_FROM"`
|
||||
MaxMessageLength int `json:"max_message_length,omitempty" env:"PICOCLAW_CHANNELS_TELEGRAM_MAX_MESSAGE_LENGTH"`
|
||||
}
|
||||
|
||||
type FeishuConfig struct {
|
||||
|
|
@ -212,6 +201,7 @@ type FeishuConfig struct {
|
|||
EncryptKey string `json:"encrypt_key" env:"PICOCLAW_CHANNELS_FEISHU_ENCRYPT_KEY"`
|
||||
VerificationToken string `json:"verification_token" env:"PICOCLAW_CHANNELS_FEISHU_VERIFICATION_TOKEN"`
|
||||
AllowFrom FlexibleStringSlice `json:"allow_from" env:"PICOCLAW_CHANNELS_FEISHU_ALLOW_FROM"`
|
||||
MaxMessageLength int `json:"max_message_length,omitempty" env:"PICOCLAW_CHANNELS_FEISHU_MAX_MESSAGE_LENGTH"`
|
||||
}
|
||||
|
||||
type DiscordConfig struct {
|
||||
|
|
@ -219,6 +209,7 @@ type DiscordConfig struct {
|
|||
Token string `json:"token" env:"PICOCLAW_CHANNELS_DISCORD_TOKEN"`
|
||||
AllowFrom FlexibleStringSlice `json:"allow_from" env:"PICOCLAW_CHANNELS_DISCORD_ALLOW_FROM"`
|
||||
MentionOnly bool `json:"mention_only" env:"PICOCLAW_CHANNELS_DISCORD_MENTION_ONLY"`
|
||||
MaxMessageLength int `json:"max_message_length,omitempty" env:"PICOCLAW_CHANNELS_DISCORD_MAX_MESSAGE_LENGTH"`
|
||||
}
|
||||
|
||||
type MaixCamConfig struct {
|
||||
|
|
@ -233,6 +224,7 @@ type QQConfig struct {
|
|||
AppID string `json:"app_id" env:"PICOCLAW_CHANNELS_QQ_APP_ID"`
|
||||
AppSecret string `json:"app_secret" env:"PICOCLAW_CHANNELS_QQ_APP_SECRET"`
|
||||
AllowFrom FlexibleStringSlice `json:"allow_from" env:"PICOCLAW_CHANNELS_QQ_ALLOW_FROM"`
|
||||
MaxMessageLength int `json:"max_message_length,omitempty" env:"PICOCLAW_CHANNELS_QQ_MAX_MESSAGE_LENGTH"`
|
||||
}
|
||||
|
||||
type DingTalkConfig struct {
|
||||
|
|
@ -240,6 +232,7 @@ type DingTalkConfig struct {
|
|||
ClientID string `json:"client_id" env:"PICOCLAW_CHANNELS_DINGTALK_CLIENT_ID"`
|
||||
ClientSecret string `json:"client_secret" env:"PICOCLAW_CHANNELS_DINGTALK_CLIENT_SECRET"`
|
||||
AllowFrom FlexibleStringSlice `json:"allow_from" env:"PICOCLAW_CHANNELS_DINGTALK_ALLOW_FROM"`
|
||||
MaxMessageLength int `json:"max_message_length,omitempty" env:"PICOCLAW_CHANNELS_DINGTALK_MAX_MESSAGE_LENGTH"`
|
||||
}
|
||||
|
||||
type SlackConfig struct {
|
||||
|
|
@ -247,6 +240,7 @@ type SlackConfig struct {
|
|||
BotToken string `json:"bot_token" env:"PICOCLAW_CHANNELS_SLACK_BOT_TOKEN"`
|
||||
AppToken string `json:"app_token" env:"PICOCLAW_CHANNELS_SLACK_APP_TOKEN"`
|
||||
AllowFrom FlexibleStringSlice `json:"allow_from" env:"PICOCLAW_CHANNELS_SLACK_ALLOW_FROM"`
|
||||
MaxMessageLength int `json:"max_message_length,omitempty" env:"PICOCLAW_CHANNELS_SLACK_MAX_MESSAGE_LENGTH"`
|
||||
}
|
||||
|
||||
type LINEConfig struct {
|
||||
|
|
@ -257,6 +251,7 @@ type LINEConfig struct {
|
|||
WebhookPort int `json:"webhook_port" env:"PICOCLAW_CHANNELS_LINE_WEBHOOK_PORT"`
|
||||
WebhookPath string `json:"webhook_path" env:"PICOCLAW_CHANNELS_LINE_WEBHOOK_PATH"`
|
||||
AllowFrom FlexibleStringSlice `json:"allow_from" env:"PICOCLAW_CHANNELS_LINE_ALLOW_FROM"`
|
||||
MaxMessageLength int `json:"max_message_length,omitempty" env:"PICOCLAW_CHANNELS_LINE_MAX_MESSAGE_LENGTH"`
|
||||
}
|
||||
|
||||
type OneBotConfig struct {
|
||||
|
|
@ -266,11 +261,12 @@ type OneBotConfig struct {
|
|||
ReconnectInterval int `json:"reconnect_interval" env:"PICOCLAW_CHANNELS_ONEBOT_RECONNECT_INTERVAL"`
|
||||
GroupTriggerPrefix []string `json:"group_trigger_prefix" env:"PICOCLAW_CHANNELS_ONEBOT_GROUP_TRIGGER_PREFIX"`
|
||||
AllowFrom FlexibleStringSlice `json:"allow_from" env:"PICOCLAW_CHANNELS_ONEBOT_ALLOW_FROM"`
|
||||
MaxMessageLength int `json:"max_message_length,omitempty" env:"PICOCLAW_CHANNELS_ONEBOT_MAX_MESSAGE_LENGTH"`
|
||||
}
|
||||
|
||||
type HeartbeatConfig struct {
|
||||
Enabled bool `json:"enabled" env:"PICOCLAW_HEARTBEAT_ENABLED"`
|
||||
Interval int `json:"interval" env:"PICOCLAW_HEARTBEAT_INTERVAL"` // minutes, min 5
|
||||
Interval int `json:"interval" env:"PICOCLAW_HEARTBEAT_INTERVAL"`
|
||||
}
|
||||
|
||||
type DevicesConfig struct {
|
||||
|
|
@ -298,8 +294,6 @@ type ProvidersConfig struct {
|
|||
Qwen ProviderConfig `json:"qwen"`
|
||||
}
|
||||
|
||||
// IsEmpty checks if all provider configs are empty (no API keys or API bases set)
|
||||
// Note: WebSearch is an optimization option and doesn't count as "non-empty"
|
||||
func (p ProvidersConfig) IsEmpty() bool {
|
||||
return p.Anthropic.APIKey == "" && p.Anthropic.APIBase == "" &&
|
||||
p.OpenAI.APIKey == "" && p.OpenAI.APIBase == "" &&
|
||||
|
|
@ -320,8 +314,6 @@ func (p ProvidersConfig) IsEmpty() bool {
|
|||
p.Qwen.APIKey == "" && p.Qwen.APIBase == ""
|
||||
}
|
||||
|
||||
// MarshalJSON implements custom JSON marshaling for ProvidersConfig
|
||||
// to omit the entire section when empty
|
||||
func (p ProvidersConfig) MarshalJSON() ([]byte, error) {
|
||||
if p.IsEmpty() {
|
||||
return []byte("null"), nil
|
||||
|
|
@ -335,7 +327,7 @@ type ProviderConfig struct {
|
|||
APIBase string `json:"api_base" env:"PICOCLAW_PROVIDERS_{{.Name}}_API_BASE"`
|
||||
Proxy string `json:"proxy,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_PROXY"`
|
||||
AuthMethod string `json:"auth_method,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_AUTH_METHOD"`
|
||||
ConnectMode string `json:"connect_mode,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_CONNECT_MODE"` //only for Github Copilot, `stdio` or `grpc`
|
||||
ConnectMode string `json:"connect_mode,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_CONNECT_MODE"`
|
||||
}
|
||||
|
||||
type OpenAIProviderConfig struct {
|
||||
|
|
@ -343,32 +335,19 @@ type OpenAIProviderConfig struct {
|
|||
WebSearch bool `json:"web_search" env:"PICOCLAW_PROVIDERS_OPENAI_WEB_SEARCH"`
|
||||
}
|
||||
|
||||
// ModelConfig represents a model-centric provider configuration.
|
||||
// It allows adding new providers (especially OpenAI-compatible ones) via configuration only.
|
||||
// The model field uses protocol prefix format: [protocol/]model-identifier
|
||||
// Supported protocols: openai, anthropic, antigravity, claude-cli, codex-cli, github-copilot
|
||||
// Default protocol is "openai" if no prefix is specified.
|
||||
type ModelConfig struct {
|
||||
// Required fields
|
||||
ModelName string `json:"model_name"` // User-facing alias for the model
|
||||
Model string `json:"model"` // Protocol/model-identifier (e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4.6")
|
||||
|
||||
// HTTP-based providers
|
||||
APIBase string `json:"api_base,omitempty"` // API endpoint URL
|
||||
APIKey string `json:"api_key"` // API authentication key
|
||||
Proxy string `json:"proxy,omitempty"` // HTTP proxy URL
|
||||
|
||||
// Special providers (CLI-based, OAuth, etc.)
|
||||
AuthMethod string `json:"auth_method,omitempty"` // Authentication method: oauth, token
|
||||
ConnectMode string `json:"connect_mode,omitempty"` // Connection mode: stdio, grpc
|
||||
Workspace string `json:"workspace,omitempty"` // Workspace path for CLI-based providers
|
||||
|
||||
// Optional optimizations
|
||||
RPM int `json:"rpm,omitempty"` // Requests per minute limit
|
||||
MaxTokensField string `json:"max_tokens_field,omitempty"` // Field name for max tokens (e.g., "max_completion_tokens")
|
||||
ModelName string `json:"model_name"`
|
||||
Model string `json:"model"`
|
||||
APIBase string `json:"api_base,omitempty"`
|
||||
APIKey string `json:"api_key"`
|
||||
Proxy string `json:"proxy,omitempty"`
|
||||
AuthMethod string `json:"auth_method,omitempty"`
|
||||
ConnectMode string `json:"connect_mode,omitempty"`
|
||||
Workspace string `json:"workspace,omitempty"`
|
||||
RPM int `json:"rpm,omitempty"`
|
||||
MaxTokensField string `json:"max_tokens_field,omitempty"`
|
||||
}
|
||||
|
||||
// Validate checks if the ModelConfig has all required fields.
|
||||
func (c *ModelConfig) Validate() error {
|
||||
if c.ModelName == "" {
|
||||
return fmt.Errorf("model_name is required")
|
||||
|
|
@ -408,7 +387,7 @@ type WebToolsConfig struct {
|
|||
}
|
||||
|
||||
type CronToolsConfig struct {
|
||||
ExecTimeoutMinutes int `json:"exec_timeout_minutes" env:"PICOCLAW_TOOLS_CRON_EXEC_TIMEOUT_MINUTES"` // 0 means no timeout
|
||||
ExecTimeoutMinutes int `json:"exec_timeout_minutes" env:"PICOCLAW_TOOLS_CRON_EXEC_TIMEOUT_MINUTES"`
|
||||
}
|
||||
|
||||
type ExecConfig struct {
|
||||
|
|
@ -451,13 +430,10 @@ type ClawHubRegistryConfig struct {
|
|||
}
|
||||
|
||||
func LoadConfig(path string) (*Config, error) {
|
||||
cfg := DefaultConfig()
|
||||
cfg := &Config{}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return cfg, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
|
@ -469,16 +445,6 @@ func LoadConfig(path string) (*Config, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
// Auto-migrate: if only legacy providers config exists, convert to model_list
|
||||
if len(cfg.ModelList) == 0 && cfg.HasProvidersConfig() {
|
||||
cfg.ModelList = ConvertProvidersToModelList(cfg)
|
||||
}
|
||||
|
||||
// Validate model_list for uniqueness and required fields
|
||||
if err := cfg.ValidateModelList(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
|
|
@ -561,9 +527,6 @@ func expandHome(path string) string {
|
|||
return path
|
||||
}
|
||||
|
||||
// GetModelConfig returns the ModelConfig for the given model name.
|
||||
// If multiple configs exist with the same model_name, it uses round-robin
|
||||
// selection for load balancing. Returns an error if the model is not found.
|
||||
func (c *Config) GetModelConfig(modelName string) (*ModelConfig, error) {
|
||||
matches := c.findMatches(modelName)
|
||||
if len(matches) == 0 {
|
||||
|
|
@ -573,12 +536,10 @@ func (c *Config) GetModelConfig(modelName string) (*ModelConfig, error) {
|
|||
return &matches[0], nil
|
||||
}
|
||||
|
||||
// Multiple configs - use round-robin for load balancing
|
||||
idx := rrCounter.Add(1) % uint64(len(matches))
|
||||
return &matches[idx], nil
|
||||
}
|
||||
|
||||
// findMatches finds all ModelConfig entries with the given model_name.
|
||||
func (c *Config) findMatches(modelName string) []ModelConfig {
|
||||
var matches []ModelConfig
|
||||
for i := range c.ModelList {
|
||||
|
|
@ -589,7 +550,6 @@ func (c *Config) findMatches(modelName string) []ModelConfig {
|
|||
return matches
|
||||
}
|
||||
|
||||
// HasProvidersConfig checks if any provider in the old providers config has configuration.
|
||||
func (c *Config) HasProvidersConfig() bool {
|
||||
v := c.Providers
|
||||
return v.Anthropic.APIKey != "" || v.Anthropic.APIBase != "" ||
|
||||
|
|
@ -611,9 +571,6 @@ func (c *Config) HasProvidersConfig() bool {
|
|||
v.Qwen.APIKey != "" || v.Qwen.APIBase != ""
|
||||
}
|
||||
|
||||
// ValidateModelList validates all ModelConfig entries in the model_list.
|
||||
// It checks that each model config is valid.
|
||||
// Note: Multiple entries with the same model_name are allowed for load balancing.
|
||||
func (c *Config) ValidateModelList() error {
|
||||
for i := range c.ModelList {
|
||||
if err := c.ModelList[i].Validate(); err != nil {
|
||||
|
|
|
|||
|
|
@ -7,29 +7,218 @@
|
|||
package providers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
|
||||
"github.com/sipeed/picoclaw/pkg/providers/openai_compat"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type HTTPProvider struct {
|
||||
delegate *openai_compat.Provider
|
||||
apiKey string
|
||||
apiBase string
|
||||
maxTokensField string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
func NewHTTPProvider(apiKey, apiBase, proxy string) *HTTPProvider {
|
||||
client := &http.Client{
|
||||
Timeout: 120 * time.Second,
|
||||
}
|
||||
|
||||
if proxy != "" {
|
||||
proxyURL, err := url.Parse(proxy)
|
||||
if err == nil {
|
||||
client.Transport = &http.Transport{
|
||||
Proxy: http.ProxyURL(proxyURL),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return &HTTPProvider{
|
||||
delegate: openai_compat.NewProvider(apiKey, apiBase, proxy),
|
||||
apiKey: apiKey,
|
||||
apiBase: strings.TrimRight(apiBase, "/"),
|
||||
httpClient: client,
|
||||
}
|
||||
}
|
||||
|
||||
func NewHTTPProviderWithMaxTokensField(apiKey, apiBase, proxy, maxTokensField string) *HTTPProvider {
|
||||
return &HTTPProvider{
|
||||
delegate: openai_compat.NewProviderWithMaxTokensField(apiKey, apiBase, proxy, maxTokensField),
|
||||
}
|
||||
provider := NewHTTPProvider(apiKey, apiBase, proxy)
|
||||
provider.maxTokensField = maxTokensField
|
||||
return provider
|
||||
}
|
||||
|
||||
func (p *HTTPProvider) Chat(ctx context.Context, messages []Message, tools []ToolDefinition, model string, options map[string]interface{}) (*LLMResponse, error) {
|
||||
return p.delegate.Chat(ctx, messages, tools, model, options)
|
||||
if p.apiBase == "" {
|
||||
return nil, fmt.Errorf("API base not configured")
|
||||
}
|
||||
|
||||
// Strip provider prefix from model name (e.g., moonshot/kimi-k2.5 -> kimi-k2.5, groq/openai/gpt-oss-120b -> openai/gpt-oss-120b, ollama/qwen2.5:14b -> qwen2.5:14b)
|
||||
if idx := strings.Index(model, "/"); idx != -1 {
|
||||
prefix := model[:idx]
|
||||
if prefix == "moonshot" || prefix == "nvidia" || prefix == "groq" || prefix == "ollama" {
|
||||
model = model[idx+1:]
|
||||
}
|
||||
}
|
||||
|
||||
// Pre-process messages loop removed - relying on ExtraContent persistence in Agent Loop.
|
||||
|
||||
requestBody := map[string]interface{}{
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
}
|
||||
|
||||
if len(tools) > 0 {
|
||||
requestBody["tools"] = tools
|
||||
requestBody["tool_choice"] = "auto"
|
||||
}
|
||||
|
||||
if maxTokens, ok := options["max_tokens"].(int); ok {
|
||||
lowerModel := strings.ToLower(model)
|
||||
if strings.Contains(lowerModel, "glm") || strings.Contains(lowerModel, "o1") {
|
||||
requestBody["max_completion_tokens"] = maxTokens
|
||||
} else {
|
||||
requestBody["max_tokens"] = maxTokens
|
||||
}
|
||||
}
|
||||
|
||||
if temperature, ok := options["temperature"].(float64); ok {
|
||||
lowerModel := strings.ToLower(model)
|
||||
// Kimi k2 models only support temperature=1
|
||||
if strings.Contains(lowerModel, "kimi") && strings.Contains(lowerModel, "k2") {
|
||||
requestBody["temperature"] = 1.0
|
||||
} else {
|
||||
requestBody["temperature"] = temperature
|
||||
}
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", p.apiBase+"/chat/completions", bytes.NewReader(jsonData))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if p.apiKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+p.apiKey)
|
||||
}
|
||||
|
||||
resp, err := p.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("API request failed:\n Status: %d\n Body: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return p.parseResponse(body)
|
||||
}
|
||||
|
||||
func (p *HTTPProvider) parseResponse(body []byte) (*LLMResponse, error) {
|
||||
var apiResponse struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
ToolCalls []struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Function *struct {
|
||||
Name string `json:"name"`
|
||||
Arguments string `json:"arguments"`
|
||||
} `json:"function"`
|
||||
ExtraContent *struct {
|
||||
Google *struct {
|
||||
ThoughtSignature string `json:"thought_signature"`
|
||||
} `json:"google"`
|
||||
} `json:"extra_content"`
|
||||
} `json:"tool_calls"`
|
||||
} `json:"message"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
} `json:"choices"`
|
||||
Usage *UsageInfo `json:"usage"`
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(body, &apiResponse); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal response: %w", err)
|
||||
}
|
||||
|
||||
if len(apiResponse.Choices) == 0 {
|
||||
return &LLMResponse{
|
||||
Content: "",
|
||||
FinishReason: "stop",
|
||||
}, nil
|
||||
}
|
||||
|
||||
choice := apiResponse.Choices[0]
|
||||
|
||||
toolCalls := make([]ToolCall, 0, len(choice.Message.ToolCalls))
|
||||
for _, tc := range choice.Message.ToolCalls {
|
||||
arguments := make(map[string]interface{})
|
||||
name := ""
|
||||
|
||||
// Extract thought_signature from Gemini/Google-specific extra content
|
||||
thoughtSignature := ""
|
||||
if tc.ExtraContent != nil && tc.ExtraContent.Google != nil {
|
||||
thoughtSignature = tc.ExtraContent.Google.ThoughtSignature
|
||||
}
|
||||
|
||||
if tc.Type == "function" && tc.Function != nil {
|
||||
name = tc.Function.Name
|
||||
if tc.Function.Arguments != "" {
|
||||
if err := json.Unmarshal([]byte(tc.Function.Arguments), &arguments); err != nil {
|
||||
arguments["raw"] = tc.Function.Arguments
|
||||
}
|
||||
}
|
||||
} else if tc.Function != nil {
|
||||
// Legacy format without type field
|
||||
name = tc.Function.Name
|
||||
if tc.Function.Arguments != "" {
|
||||
if err := json.Unmarshal([]byte(tc.Function.Arguments), &arguments); err != nil {
|
||||
arguments["raw"] = tc.Function.Arguments
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Correctly map extracted ExtraContent to ToolCall struct
|
||||
toolCall := ToolCall{
|
||||
ID: tc.ID,
|
||||
Name: name,
|
||||
Arguments: arguments,
|
||||
ThoughtSignature: thoughtSignature, // Populating internal field for convenience
|
||||
}
|
||||
|
||||
if thoughtSignature != "" {
|
||||
toolCall.ExtraContent = &ExtraContent{
|
||||
Google: &GoogleExtra{
|
||||
ThoughtSignature: thoughtSignature,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
toolCalls = append(toolCalls, toolCall)
|
||||
}
|
||||
|
||||
return &LLMResponse{
|
||||
Content: choice.Message.Content,
|
||||
ToolCalls: toolCalls,
|
||||
FinishReason: choice.FinishReason,
|
||||
Usage: apiResponse.Usage,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *HTTPProvider) GetDefaultModel() string {
|
||||
|
|
|
|||
|
|
@ -9,29 +9,31 @@ import (
|
|||
|
||||
type ToolCall = protocoltypes.ToolCall
|
||||
type FunctionCall = protocoltypes.FunctionCall
|
||||
type ExtraContent = protocoltypes.ExtraContent
|
||||
type GoogleExtra = protocoltypes.GoogleExtra
|
||||
type LLMResponse = protocoltypes.LLMResponse
|
||||
type UsageInfo = protocoltypes.UsageInfo
|
||||
type Message = protocoltypes.Message
|
||||
type ToolDefinition = protocoltypes.ToolDefinition
|
||||
type ToolFunctionDefinition = protocoltypes.ToolFunctionDefinition
|
||||
type ExtraContent = protocoltypes.ExtraContent
|
||||
type GoogleExtra = protocoltypes.GoogleExtra
|
||||
|
||||
type LLMProvider interface {
|
||||
Chat(ctx context.Context, messages []Message, tools []ToolDefinition, model string, options map[string]interface{}) (*LLMResponse, error)
|
||||
GetDefaultModel() string
|
||||
}
|
||||
|
||||
// FailoverReason classifies why an LLM request failed for fallback decisions.
|
||||
type FailoverReason string
|
||||
|
||||
const (
|
||||
FailoverAuth FailoverReason = "auth"
|
||||
FailoverRateLimit FailoverReason = "rate_limit"
|
||||
FailoverBilling FailoverReason = "billing"
|
||||
FailoverTimeout FailoverReason = "timeout"
|
||||
FailoverStatus FailoverReason = "status"
|
||||
FailoverEmpty FailoverReason = "empty"
|
||||
FailoverFormat FailoverReason = "format"
|
||||
FailoverOverloaded FailoverReason = "overloaded"
|
||||
FailoverAuth FailoverReason = "auth"
|
||||
FailoverBilling FailoverReason = "billing"
|
||||
FailoverRateLimit FailoverReason = "rate_limit"
|
||||
FailoverContextWindow FailoverReason = "context_window"
|
||||
FailoverUnknown FailoverReason = "unknown"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -256,10 +256,23 @@ func (t *ExecTool) guardCommand(command, cwd string) string {
|
|||
return ""
|
||||
}
|
||||
|
||||
pathPattern := regexp.MustCompile(`[A-Za-z]:\\[^\\\"']+|/[^\s\"']+`)
|
||||
matches := pathPattern.FindAllString(cmd, -1)
|
||||
// Match absolute paths: Unix (starts with / after space/start/quotes) or Windows (X:\)
|
||||
// This regex is careful not to match scoped packages like @mastra/core
|
||||
pathPattern := regexp.MustCompile(`(?:\s|^|["'|&;])(/[^\s\"'|&;]+)|([A-Za-z]:\\[^\\\"'|&;]+)`)
|
||||
matches := pathPattern.FindAllStringSubmatch(cmd, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
raw := ""
|
||||
if match[1] != "" {
|
||||
raw = match[1] // Unix path
|
||||
} else if match[2] != "" {
|
||||
raw = match[2] // Windows path
|
||||
}
|
||||
|
||||
if raw == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, raw := range matches {
|
||||
p, err := filepath.Abs(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -6,9 +6,8 @@ import (
|
|||
|
||||
// SplitMessage splits long messages into chunks, preserving code block integrity.
|
||||
// The function reserves a buffer (10% of maxLen, min 50) to leave room for closing code blocks,
|
||||
// but may extend to maxLen when needed.
|
||||
// Call SplitMessage with the full text content and the maximum allowed length of a single message;
|
||||
// it returns a slice of message chunks that each respect maxLen and avoid splitting fenced code blocks.
|
||||
// but may extend to maxLen when needed. It respects rune counts to ensure multi-byte
|
||||
// characters (like emojis or CJK) are not split in half.
|
||||
func SplitMessage(content string, maxLen int) []string {
|
||||
var messages []string
|
||||
|
||||
|
|
@ -21,9 +20,11 @@ func SplitMessage(content string, maxLen int) []string {
|
|||
codeBlockBuffer = maxLen / 2
|
||||
}
|
||||
|
||||
for len(content) > 0 {
|
||||
if len(content) <= maxLen {
|
||||
messages = append(messages, content)
|
||||
runes := []rune(content)
|
||||
|
||||
for len(runes) > 0 {
|
||||
if len(runes) <= maxLen {
|
||||
messages = append(messages, string(runes))
|
||||
break
|
||||
}
|
||||
|
||||
|
|
@ -34,56 +35,82 @@ func SplitMessage(content string, maxLen int) []string {
|
|||
}
|
||||
|
||||
// Find natural split point within the effective limit
|
||||
msgEnd := findLastNewline(content[:effectiveLimit], 200)
|
||||
msgEnd := findLastSentenceBoundaryRunes(runes[:effectiveLimit], 300)
|
||||
if msgEnd <= 0 {
|
||||
msgEnd = findLastSpace(content[:effectiveLimit], 100)
|
||||
msgEnd = findLastNewlineRunes(runes[:effectiveLimit], 200)
|
||||
}
|
||||
if msgEnd <= 0 {
|
||||
msgEnd = findLastSpaceRunes(runes[:effectiveLimit], 100)
|
||||
}
|
||||
if msgEnd <= 0 {
|
||||
msgEnd = effectiveLimit
|
||||
}
|
||||
|
||||
// Check if this would end with an incomplete code block
|
||||
candidate := content[:msgEnd]
|
||||
unclosedIdx := findLastUnclosedCodeBlock(candidate)
|
||||
candidate := runes[:msgEnd]
|
||||
unclosedIdx := findLastUnclosedCodeBlockRunes(candidate)
|
||||
|
||||
if unclosedIdx >= 0 {
|
||||
// Message would end with incomplete code block
|
||||
// Try to extend up to maxLen to include the closing ```
|
||||
if len(content) > msgEnd {
|
||||
closingIdx := findNextClosingCodeBlock(content, msgEnd)
|
||||
if len(runes) > msgEnd {
|
||||
closingIdx := findNextClosingCodeBlockRunes(runes, msgEnd)
|
||||
if closingIdx > 0 && closingIdx <= maxLen {
|
||||
// Extend to include the closing ```
|
||||
msgEnd = closingIdx
|
||||
} else {
|
||||
// Code block is too long to fit in one chunk or missing closing fence.
|
||||
// Try to split inside by injecting closing and reopening fences.
|
||||
headerEnd := strings.Index(content[unclosedIdx:], "\n")
|
||||
|
||||
// Find the header end (first newline after the opening ```)
|
||||
headerEnd := -1
|
||||
for i := unclosedIdx; i < len(runes); i++ {
|
||||
if runes[i] == '\n' {
|
||||
headerEnd = i
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if headerEnd == -1 {
|
||||
headerEnd = unclosedIdx + 3
|
||||
} else {
|
||||
headerEnd += unclosedIdx
|
||||
// include newline
|
||||
headerEnd++
|
||||
}
|
||||
header := strings.TrimSpace(content[unclosedIdx:headerEnd])
|
||||
|
||||
header := strings.TrimSpace(string(runes[unclosedIdx:headerEnd]))
|
||||
|
||||
// If we have a reasonable amount of content after the header, split inside
|
||||
if msgEnd > headerEnd+20 {
|
||||
// Find a better split point closer to maxLen
|
||||
innerLimit := maxLen - 5 // Leave room for "\n```"
|
||||
betterEnd := findLastNewline(content[:innerLimit], 200)
|
||||
betterEnd := findLastSentenceBoundaryRunes(runes[:innerLimit], 300)
|
||||
if betterEnd <= headerEnd {
|
||||
betterEnd = findLastNewlineRunes(runes[:innerLimit], 200)
|
||||
}
|
||||
|
||||
if betterEnd > headerEnd {
|
||||
msgEnd = betterEnd
|
||||
} else {
|
||||
msgEnd = innerLimit
|
||||
}
|
||||
messages = append(messages, strings.TrimRight(content[:msgEnd], " \t\n\r")+"\n```")
|
||||
content = strings.TrimSpace(header + "\n" + content[msgEnd:])
|
||||
|
||||
chunkStr := string(runes[:msgEnd])
|
||||
messages = append(messages, strings.TrimRight(chunkStr, " \t\n\r")+"\n```")
|
||||
|
||||
nextChunkStart := string(runes[msgEnd:])
|
||||
content = strings.TrimSpace(header + "\n" + nextChunkStart)
|
||||
runes = []rune(content)
|
||||
continue
|
||||
}
|
||||
|
||||
// Otherwise, try to split before the code block starts
|
||||
newEnd := findLastNewline(content[:unclosedIdx], 200)
|
||||
newEnd := findLastSentenceBoundaryRunes(runes[:unclosedIdx], 300)
|
||||
if newEnd <= 0 {
|
||||
newEnd = findLastSpace(content[:unclosedIdx], 100)
|
||||
newEnd = findLastNewlineRunes(runes[:unclosedIdx], 200)
|
||||
}
|
||||
if newEnd <= 0 {
|
||||
newEnd = findLastSpaceRunes(runes[:unclosedIdx], 100)
|
||||
}
|
||||
if newEnd > 0 {
|
||||
msgEnd = newEnd
|
||||
|
|
@ -93,8 +120,12 @@ func SplitMessage(content string, maxLen int) []string {
|
|||
msgEnd = unclosedIdx
|
||||
} else {
|
||||
msgEnd = maxLen - 5
|
||||
messages = append(messages, strings.TrimRight(content[:msgEnd], " \t\n\r")+"\n```")
|
||||
content = strings.TrimSpace(header + "\n" + content[msgEnd:])
|
||||
chunkStr := string(runes[:msgEnd])
|
||||
messages = append(messages, strings.TrimRight(chunkStr, " \t\n\r")+"\n```")
|
||||
|
||||
nextChunkStart := string(runes[msgEnd:])
|
||||
content = strings.TrimSpace(header + "\n" + nextChunkStart)
|
||||
runes = []rune(content)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
|
@ -106,21 +137,22 @@ func SplitMessage(content string, maxLen int) []string {
|
|||
msgEnd = effectiveLimit
|
||||
}
|
||||
|
||||
messages = append(messages, content[:msgEnd])
|
||||
content = strings.TrimSpace(content[msgEnd:])
|
||||
messages = append(messages, string(runes[:msgEnd]))
|
||||
nextContent := strings.TrimSpace(string(runes[msgEnd:]))
|
||||
runes = []rune(nextContent)
|
||||
}
|
||||
|
||||
return messages
|
||||
}
|
||||
|
||||
// findLastUnclosedCodeBlock finds the last opening ``` that doesn't have a closing ```
|
||||
// findLastUnclosedCodeBlockRunes finds the last opening ``` that doesn't have a closing ```
|
||||
// Returns the position of the opening ``` or -1 if all code blocks are complete
|
||||
func findLastUnclosedCodeBlock(text string) int {
|
||||
func findLastUnclosedCodeBlockRunes(runes []rune) int {
|
||||
inCodeBlock := false
|
||||
lastOpenIdx := -1
|
||||
|
||||
for i := 0; i < len(text); i++ {
|
||||
if i+2 < len(text) && text[i] == '`' && text[i+1] == '`' && text[i+2] == '`' {
|
||||
for i := 0; i < len(runes); i++ {
|
||||
if i+2 < len(runes) && runes[i] == '`' && runes[i+1] == '`' && runes[i+2] == '`' {
|
||||
// Toggle code block state on each fence
|
||||
if !inCodeBlock {
|
||||
// Entering a code block: record this opening fence
|
||||
|
|
@ -137,43 +169,62 @@ func findLastUnclosedCodeBlock(text string) int {
|
|||
return -1
|
||||
}
|
||||
|
||||
// findNextClosingCodeBlock finds the next closing ``` starting from a position
|
||||
// findNextClosingCodeBlockRunes finds the next closing ``` starting from a position
|
||||
// Returns the position after the closing ``` or -1 if not found
|
||||
func findNextClosingCodeBlock(text string, startIdx int) int {
|
||||
for i := startIdx; i < len(text); i++ {
|
||||
if i+2 < len(text) && text[i] == '`' && text[i+1] == '`' && text[i+2] == '`' {
|
||||
func findNextClosingCodeBlockRunes(runes []rune, startIdx int) int {
|
||||
for i := startIdx; i < len(runes); i++ {
|
||||
if i+2 < len(runes) && runes[i] == '`' && runes[i+1] == '`' && runes[i+2] == '`' {
|
||||
return i + 3
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// findLastNewline finds the last newline character within the last N characters
|
||||
// findLastNewlineRunes finds the last newline character within the last N characters
|
||||
// Returns the position of the newline or -1 if not found
|
||||
func findLastNewline(s string, searchWindow int) int {
|
||||
searchStart := len(s) - searchWindow
|
||||
func findLastNewlineRunes(runes []rune, searchWindow int) int {
|
||||
searchStart := len(runes) - searchWindow
|
||||
if searchStart < 0 {
|
||||
searchStart = 0
|
||||
}
|
||||
for i := len(s) - 1; i >= searchStart; i-- {
|
||||
if s[i] == '\n' {
|
||||
for i := len(runes) - 1; i >= searchStart; i-- {
|
||||
if runes[i] == '\n' {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// findLastSpace finds the last space character within the last N characters
|
||||
// findLastSpaceRunes finds the last space character within the last N characters
|
||||
// Returns the position of the space or -1 if not found
|
||||
func findLastSpace(s string, searchWindow int) int {
|
||||
searchStart := len(s) - searchWindow
|
||||
func findLastSpaceRunes(runes []rune, searchWindow int) int {
|
||||
searchStart := len(runes) - searchWindow
|
||||
if searchStart < 0 {
|
||||
searchStart = 0
|
||||
}
|
||||
for i := len(s) - 1; i >= searchStart; i-- {
|
||||
if s[i] == ' ' || s[i] == '\t' {
|
||||
for i := len(runes) - 1; i >= searchStart; i-- {
|
||||
if runes[i] == ' ' || runes[i] == '\t' {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// findLastSentenceBoundaryRunes finds the last sentence-ending punctuation
|
||||
// Returns the position after the punctuation or -1 if not found
|
||||
func findLastSentenceBoundaryRunes(runes []rune, searchWindow int) int {
|
||||
searchStart := len(runes) - searchWindow
|
||||
if searchStart < 0 {
|
||||
searchStart = 0
|
||||
}
|
||||
for i := len(runes) - 1; i >= searchStart; i-- {
|
||||
switch runes[i] {
|
||||
case '.', '!', '?', '。', '！', '？':
|
||||
// Ensure it's the end of a sentence (followed by space, newline, or end of string)
|
||||
if i == len(runes)-1 || runes[i+1] == ' ' || runes[i+1] == '\n' || runes[i+1] == '\t' {
|
||||
return i + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ func TestSplitMessage(t *testing.T) {
|
|||
},
|
||||
{
|
||||
name: "Preserve Unicode characters",
|
||||
content: strings.Repeat("\u4e16", 1000), // 3000 bytes
|
||||
content: strings.Repeat("\u4e16", 2500), // 2500 runes
|
||||
maxLen: 2000,
|
||||
expectChunks: 2,
|
||||
checkContent: func(t *testing.T, chunks []string) {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue