fix: add health monitoring and auto-recovery for dingtalk stream connection
This commit is contained in:
parent
7ea7bb0717
commit
232abb4922
1 changed files with 99 additions and 3 deletions
|
|
@ -7,6 +7,7 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sync"
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/open-dingtalk/dingtalk-stream-sdk-go/chatbot"
|
"github.com/open-dingtalk/dingtalk-stream-sdk-go/chatbot"
|
||||||
"github.com/open-dingtalk/dingtalk-stream-sdk-go/client"
|
"github.com/open-dingtalk/dingtalk-stream-sdk-go/client"
|
||||||
|
|
@ -19,6 +20,14 @@ import (
|
||||||
"github.com/sipeed/picoclaw/pkg/utils"
|
"github.com/sipeed/picoclaw/pkg/utils"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Health check constants
|
||||||
|
const (
|
||||||
|
healthCheckInterval = 5 * time.Minute // Check every 5 minutes
|
||||||
|
maxSilenceDuration = 30 * time.Minute // Max time without messages before recovery
|
||||||
|
recoveryDelay = 2 * time.Second // Delay before reconnecting
|
||||||
|
recoveryRetryDelay = 30 * time.Second // Delay before retrying failed recovery
|
||||||
|
)
|
||||||
|
|
||||||
// DingTalkChannel implements the Channel interface for DingTalk (钉钉)
|
// DingTalkChannel implements the Channel interface for DingTalk (钉钉)
|
||||||
// It uses WebSocket for receiving messages via stream mode and API for sending
|
// It uses WebSocket for receiving messages via stream mode and API for sending
|
||||||
type DingTalkChannel struct {
|
type DingTalkChannel struct {
|
||||||
|
|
@ -31,6 +40,9 @@ type DingTalkChannel struct {
|
||||||
cancel context.CancelFunc
|
cancel context.CancelFunc
|
||||||
// Map to store session webhooks for each chat
|
// Map to store session webhooks for each chat
|
||||||
sessionWebhooks sync.Map // chatID -> sessionWebhook
|
sessionWebhooks sync.Map // chatID -> sessionWebhook
|
||||||
|
// Health monitoring
|
||||||
|
lastMessageTime time.Time
|
||||||
|
mu sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewDingTalkChannel creates a new DingTalk channel instance
|
// NewDingTalkChannel creates a new DingTalk channel instance
|
||||||
|
|
@ -58,7 +70,23 @@ func (c *DingTalkChannel) Start(ctx context.Context) error {
|
||||||
logger.InfoC("dingtalk", "Starting DingTalk channel (Stream Mode)...")
|
logger.InfoC("dingtalk", "Starting DingTalk channel (Stream Mode)...")
|
||||||
|
|
||||||
c.ctx, c.cancel = context.WithCancel(ctx)
|
c.ctx, c.cancel = context.WithCancel(ctx)
|
||||||
|
c.lastMessageTime = time.Now() // Initialize on start
|
||||||
|
|
||||||
|
// Start the stream client
|
||||||
|
if err := c.startStreamClient(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start health monitoring goroutine
|
||||||
|
go c.healthMonitor()
|
||||||
|
|
||||||
|
c.SetRunning(true)
|
||||||
|
logger.InfoC("dingtalk", "DingTalk channel started (Stream Mode)")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// startStreamClient creates and starts the stream client
|
||||||
|
func (c *DingTalkChannel) startStreamClient() error {
|
||||||
// Create credential config
|
// Create credential config
|
||||||
cred := client.NewAppCredentialConfig(c.clientID, c.clientSecret)
|
cred := client.NewAppCredentialConfig(c.clientID, c.clientSecret)
|
||||||
|
|
||||||
|
|
@ -68,7 +96,7 @@ func (c *DingTalkChannel) Start(ctx context.Context) error {
|
||||||
client.WithAutoReconnect(true),
|
client.WithAutoReconnect(true),
|
||||||
)
|
)
|
||||||
|
|
||||||
// Register chatbot callback handler (IChatBotMessageHandler is a function type)
|
// Register chatbot callback handler
|
||||||
c.streamClient.RegisterChatBotCallbackRouter(c.onChatBotMessageReceived)
|
c.streamClient.RegisterChatBotCallbackRouter(c.onChatBotMessageReceived)
|
||||||
|
|
||||||
// Start the stream client
|
// Start the stream client
|
||||||
|
|
@ -76,11 +104,76 @@ func (c *DingTalkChannel) Start(ctx context.Context) error {
|
||||||
return fmt.Errorf("failed to start stream client: %w", err)
|
return fmt.Errorf("failed to start stream client: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
c.SetRunning(true)
|
|
||||||
logger.InfoC("dingtalk", "DingTalk channel started (Stream Mode)")
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// healthMonitor periodically checks connection health and triggers recovery if needed
|
||||||
|
func (c *DingTalkChannel) healthMonitor() {
|
||||||
|
ticker := time.NewTicker(healthCheckInterval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-c.ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
c.checkAndRecover()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkAndRecover checks if connection is stale and triggers recovery
|
||||||
|
func (c *DingTalkChannel) checkAndRecover() {
|
||||||
|
c.mu.RLock()
|
||||||
|
silenceDuration := time.Since(c.lastMessageTime)
|
||||||
|
c.mu.RUnlock()
|
||||||
|
|
||||||
|
logger.DebugCF("dingtalk", "Health check: silence duration %v", silenceDuration)
|
||||||
|
|
||||||
|
if silenceDuration >= maxSilenceDuration {
|
||||||
|
logger.InfoCF("dingtalk", "Connection appears stale (no messages for %v), triggering recovery", silenceDuration)
|
||||||
|
c.recoverConnection()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// recoverConnection attempts to recover the stream connection
|
||||||
|
func (c *DingTalkChannel) recoverConnection() {
|
||||||
|
// Close old client
|
||||||
|
if c.streamClient != nil {
|
||||||
|
c.streamClient.Close()
|
||||||
|
time.Sleep(recoveryDelay)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attempt to reconnect
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-c.ctx.Done():
|
||||||
|
logger.InfoC("dingtalk", "Recovery aborted: context cancelled")
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
err := c.startStreamClient()
|
||||||
|
if err == nil {
|
||||||
|
logger.InfoCF("dingtalk", "Connection recovered successfully")
|
||||||
|
c.mu.Lock()
|
||||||
|
c.lastMessageTime = time.Now()
|
||||||
|
c.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.WarnCF("dingtalk", "Recovery failed: %v, retrying in %v", err, recoveryRetryDelay)
|
||||||
|
time.Sleep(recoveryRetryDelay)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// updateLastMessageTime updates the last message timestamp
|
||||||
|
func (c *DingTalkChannel) updateLastMessageTime() {
|
||||||
|
c.mu.Lock()
|
||||||
|
c.lastMessageTime = time.Now()
|
||||||
|
c.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
// Stop gracefully stops the DingTalk channel
|
// Stop gracefully stops the DingTalk channel
|
||||||
func (c *DingTalkChannel) Stop(ctx context.Context) error {
|
func (c *DingTalkChannel) Stop(ctx context.Context) error {
|
||||||
logger.InfoC("dingtalk", "Stopping DingTalk channel...")
|
logger.InfoC("dingtalk", "Stopping DingTalk channel...")
|
||||||
|
|
@ -131,6 +224,9 @@ func (c *DingTalkChannel) onChatBotMessageReceived(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
data *chatbot.BotCallbackDataModel,
|
data *chatbot.BotCallbackDataModel,
|
||||||
) ([]byte, error) {
|
) ([]byte, error) {
|
||||||
|
// Update last message time for health monitoring
|
||||||
|
c.updateLastMessageTime()
|
||||||
|
|
||||||
// Extract message content from Text field
|
// Extract message content from Text field
|
||||||
content := data.Text.Content
|
content := data.Text.Content
|
||||||
if content == "" {
|
if content == "" {
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue