fix(channels): address email security/reliability review and add unit tests
Security & reliability: - Cap attachment write with io.LimitReader (configurable attachment_max_bytes, default 25MB) - Use sanitized filename for ext in saveAttachmentToLocal to avoid path traversal - Sanitize SMTP header values (strip CR/LF) and use go-message for header construction - Add IMAP reconnection with exponential backoff on IDLE/poll and fetch errors - Cap email body part read with io.LimitReader (10MB) to avoid unbounded io.ReadAll - Log warning when STARTTLS fails so credentials are not sent unencrypted unnoticed Tests: - Add email_test.go: sanitizeHeaderValue, parseFilenameFromDisposition, decodeRFC2047Filename, extractEmailBodyAndAttachments (nil/plain/html/attachment), extractTextFromHTML, saveAttachmentToLocal (size limit and over-limit)
This commit is contained in:
parent
cb19ee997d
commit
8417229012
3 changed files with 475 additions and 126 deletions
|
|
@ -1,6 +1,7 @@
|
||||||
package channels
|
package channels
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
@ -30,6 +31,15 @@ func init() {
|
||||||
charset.RegisterEncoding("gbk", simplifiedchinese.GBK)
|
charset.RegisterEncoding("gbk", simplifiedchinese.GBK)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
// reconnect backoff initial
|
||||||
|
reconnectBackoffInitial = 1 * time.Second
|
||||||
|
// reconnect backoff max
|
||||||
|
reconnectBackoffMax = 10 * time.Minute
|
||||||
|
// default attachment max bytes
|
||||||
|
defaultAttachmentMaxBytes = 25 * 1024 * 1024 // 25MB
|
||||||
|
)
|
||||||
|
|
||||||
type EmailChannel struct {
|
type EmailChannel struct {
|
||||||
*BaseChannel
|
*BaseChannel
|
||||||
config config.EmailConfig
|
config config.EmailConfig
|
||||||
|
|
@ -38,6 +48,10 @@ type EmailChannel struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
cancel context.CancelFunc
|
cancel context.CancelFunc
|
||||||
checkTicker *time.Ticker
|
checkTicker *time.Ticker
|
||||||
|
|
||||||
|
// reconnect control
|
||||||
|
reconnectClientVersion int
|
||||||
|
reconnectMutex sync.Mutex
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewEmailChannel(cfg config.EmailConfig, bus *bus.MessageBus) (*EmailChannel, error) {
|
func NewEmailChannel(cfg config.EmailConfig, bus *bus.MessageBus) (*EmailChannel, error) {
|
||||||
|
|
@ -100,6 +114,12 @@ func (c *EmailChannel) Stop(ctx context.Context) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sanitizeHeaderValue removes CR/LF from s to prevent SMTP header injection.
|
||||||
|
// go-message textproto also rejects \r\n in header values when writing; we sanitize so the send succeeds.
|
||||||
|
func sanitizeHeaderValue(s string) string {
|
||||||
|
return strings.NewReplacer("\r", "", "\n", "").Replace(s)
|
||||||
|
}
|
||||||
|
|
||||||
func (c *EmailChannel) Send(ctx context.Context, msg bus.OutboundMessage) error {
|
func (c *EmailChannel) Send(ctx context.Context, msg bus.OutboundMessage) error {
|
||||||
if !c.IsRunning() {
|
if !c.IsRunning() {
|
||||||
return fmt.Errorf("email channel not running")
|
return fmt.Errorf("email channel not running")
|
||||||
|
|
@ -108,27 +128,39 @@ func (c *EmailChannel) Send(ctx context.Context, msg bus.OutboundMessage) error
|
||||||
return fmt.Errorf("email channel send: SMTP not configured (set smtp_server)")
|
return fmt.Errorf("email channel send: SMTP not configured (set smtp_server)")
|
||||||
}
|
}
|
||||||
|
|
||||||
from := c.config.Username
|
fromRaw := sanitizeHeaderValue(c.config.Username)
|
||||||
to := strings.TrimSpace(msg.ChatID)
|
toRaw := sanitizeHeaderValue(strings.TrimSpace(msg.ChatID))
|
||||||
if to == "" {
|
if toRaw == "" {
|
||||||
return fmt.Errorf("email channel send: missing recipient (chat_id)")
|
return fmt.Errorf("email channel send: missing recipient (chat_id)")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Plain-text message: From / To / Subject / Body (OutboundMessage has no Metadata, use fixed subject)
|
// Build message with go-message/mail: RFC-compliant headers via textproto (folding, encoded-words, address list format).
|
||||||
subject := "Reply from PicoClaw"
|
var h mail.Header
|
||||||
header := map[string]string{
|
if fromAddrs, err := mail.ParseAddressList(fromRaw); err == nil && len(fromAddrs) > 0 {
|
||||||
"From": from,
|
h.SetAddressList("From", fromAddrs)
|
||||||
"To": to,
|
} else {
|
||||||
"Subject": subject,
|
h.Set("From", fromRaw)
|
||||||
"Content-Type": "text/plain; charset=utf-8",
|
|
||||||
}
|
}
|
||||||
var raw strings.Builder
|
if toAddrs, err := mail.ParseAddressList(toRaw); err == nil && len(toAddrs) > 0 {
|
||||||
for k, v := range header {
|
h.SetAddressList("To", toAddrs)
|
||||||
raw.WriteString(k + ": " + v + "\r\n")
|
} else {
|
||||||
|
h.Set("To", toRaw)
|
||||||
}
|
}
|
||||||
raw.WriteString("\r\n")
|
h.SetSubject(sanitizeHeaderValue("Reply from PicoClaw"))
|
||||||
raw.WriteString(msg.Content)
|
h.Set("Content-Type", "text/plain; charset=utf-8")
|
||||||
body := raw.String()
|
var buf bytes.Buffer
|
||||||
|
bodyWriter, err := mail.CreateSingleInlineWriter(&buf, h)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("email build message: %w", err)
|
||||||
|
}
|
||||||
|
if _, err = bodyWriter.Write([]byte(msg.Content)); err != nil {
|
||||||
|
_ = bodyWriter.Close()
|
||||||
|
return fmt.Errorf("email write body: %w", err)
|
||||||
|
}
|
||||||
|
if err = bodyWriter.Close(); err != nil {
|
||||||
|
return fmt.Errorf("email close message: %w", err)
|
||||||
|
}
|
||||||
|
body := buf.Bytes()
|
||||||
|
|
||||||
port := c.config.SMTPPort
|
port := c.config.SMTPPort
|
||||||
if port <= 0 {
|
if port <= 0 {
|
||||||
|
|
@ -154,17 +186,17 @@ func (c *EmailChannel) Send(ctx context.Context, msg bus.OutboundMessage) error
|
||||||
if err = client.Auth(auth); err != nil {
|
if err = client.Auth(auth); err != nil {
|
||||||
return fmt.Errorf("smtp auth: %w", err)
|
return fmt.Errorf("smtp auth: %w", err)
|
||||||
}
|
}
|
||||||
if err = client.Mail(from); err != nil {
|
if err = client.Mail(fromRaw); err != nil {
|
||||||
return fmt.Errorf("smtp mail: %w", err)
|
return fmt.Errorf("smtp mail: %w", err)
|
||||||
}
|
}
|
||||||
if err = client.Rcpt(to); err != nil {
|
if err = client.Rcpt(toRaw); err != nil {
|
||||||
return fmt.Errorf("smtp rcpt: %w", err)
|
return fmt.Errorf("smtp rcpt: %w", err)
|
||||||
}
|
}
|
||||||
w, err := client.Data()
|
w, err := client.Data()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("smtp data: %w", err)
|
return fmt.Errorf("smtp data: %w", err)
|
||||||
}
|
}
|
||||||
if _, err = w.Write([]byte(body)); err != nil {
|
if _, err = w.Write(body); err != nil {
|
||||||
_ = w.Close()
|
_ = w.Close()
|
||||||
return fmt.Errorf("smtp write: %w", err)
|
return fmt.Errorf("smtp write: %w", err)
|
||||||
}
|
}
|
||||||
|
|
@ -187,23 +219,26 @@ func (c *EmailChannel) Send(ctx context.Context, msg bus.OutboundMessage) error
|
||||||
defer client.Close()
|
defer client.Close()
|
||||||
if err = client.StartTLS(&tls.Config{ServerName: host}); err != nil {
|
if err = client.StartTLS(&tls.Config{ServerName: host}); err != nil {
|
||||||
// Some servers on 587 do not require STARTTLS; continue anyway
|
// Some servers on 587 do not require STARTTLS; continue anyway
|
||||||
|
logger.WarnCF("email", "STARTTLS failed, connection may be unencrypted; credentials could be sent in plaintext", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
_ = err
|
_ = err
|
||||||
}
|
}
|
||||||
auth := smtp.PlainAuth("", c.config.Username, c.config.Password, host)
|
auth := smtp.PlainAuth("", c.config.Username, c.config.Password, host)
|
||||||
if err = client.Auth(auth); err != nil {
|
if err = client.Auth(auth); err != nil {
|
||||||
return fmt.Errorf("smtp auth: %w", err)
|
return fmt.Errorf("smtp auth: %w", err)
|
||||||
}
|
}
|
||||||
if err = client.Mail(from); err != nil {
|
if err = client.Mail(fromRaw); err != nil {
|
||||||
return fmt.Errorf("smtp mail: %w", err)
|
return fmt.Errorf("smtp mail: %w", err)
|
||||||
}
|
}
|
||||||
if err = client.Rcpt(to); err != nil {
|
if err = client.Rcpt(toRaw); err != nil {
|
||||||
return fmt.Errorf("smtp rcpt: %w", err)
|
return fmt.Errorf("smtp rcpt: %w", err)
|
||||||
}
|
}
|
||||||
w, err := client.Data()
|
w, err := client.Data()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("smtp data: %w", err)
|
return fmt.Errorf("smtp data: %w", err)
|
||||||
}
|
}
|
||||||
if _, err = w.Write([]byte(body)); err != nil {
|
if _, err = w.Write(body); err != nil {
|
||||||
_ = w.Close()
|
_ = w.Close()
|
||||||
return fmt.Errorf("smtp write: %w", err)
|
return fmt.Errorf("smtp write: %w", err)
|
||||||
}
|
}
|
||||||
|
|
@ -256,7 +291,10 @@ func (c *EmailChannel) connect() error {
|
||||||
// First connect: init lastUID from Select's UidNext (max current UID = UidNext-1) to avoid full UidSearch
|
// First connect: init lastUID from Select's UidNext (max current UID = UidNext-1) to avoid full UidSearch
|
||||||
if status != nil && status.UidNext > 0 {
|
if status != nil && status.UidNext > 0 {
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
|
// only init lastUID once
|
||||||
|
if c.lastUID == 0 {
|
||||||
c.lastUID = status.UidNext - 1
|
c.lastUID = status.UidNext - 1
|
||||||
|
}
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
} else {
|
} else {
|
||||||
// Fallback: some servers do not return UidNext, search all to get max UID
|
// Fallback: some servers do not return UidNext, search all to get max UID
|
||||||
|
|
@ -277,6 +315,13 @@ func (c *EmailChannel) connect() error {
|
||||||
|
|
||||||
// syncLastUID fetches the mailbox max UID and sets lastUID so only mail after connect is processed.
|
// syncLastUID fetches the mailbox max UID and sets lastUID so only mail after connect is processed.
|
||||||
func (c *EmailChannel) syncLastUID(cl *client.Client) error {
|
func (c *EmailChannel) syncLastUID(cl *client.Client) error {
|
||||||
|
c.mu.Lock()
|
||||||
|
// init lastUID once
|
||||||
|
if c.lastUID != 0 {
|
||||||
|
c.mu.Unlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
c.mu.Unlock()
|
||||||
criteria := imap.NewSearchCriteria()
|
criteria := imap.NewSearchCriteria()
|
||||||
uids, err := cl.UidSearch(criteria)
|
uids, err := cl.UidSearch(criteria)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -296,11 +341,74 @@ func (c *EmailChannel) syncLastUID(cl *client.Client) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
|
if c.lastUID == 0 {
|
||||||
c.lastUID = maxUID
|
c.lastUID = maxUID
|
||||||
|
}
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// closeIMAPClient logs out and clears the current IMAP client. Caller must not hold c.mu.
|
||||||
|
func (c *EmailChannel) closeIMAPClient() {
|
||||||
|
c.mu.Lock()
|
||||||
|
cl := c.imapClient
|
||||||
|
c.imapClient = nil
|
||||||
|
c.mu.Unlock()
|
||||||
|
if cl != nil {
|
||||||
|
_ = cl.Logout()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// reconnectWithBackoff closes the current IMAP client and reconnects with exponential backoff until success or ctx is done.
|
||||||
|
// when muti goroutine reconnect, only one goroutine can reconnect at a time, other goroutine will wait for the reconnect success.
|
||||||
|
func (c *EmailChannel) reconnectWithBackoff(ctx context.Context) error {
|
||||||
|
currentClientVersion := c.reconnectClientVersion
|
||||||
|
// singleflight reconnect, only one goroutine can reconnect at a time
|
||||||
|
c.reconnectMutex.Lock()
|
||||||
|
defer c.reconnectMutex.Unlock()
|
||||||
|
if currentClientVersion != c.reconnectClientVersion {
|
||||||
|
// other goroutine has already reconnect, check state is selected
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
isOk := c.imapClient != nil && c.imapClient.State() == imap.SelectedState
|
||||||
|
c.mu.Unlock()
|
||||||
|
if isOk {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
c.reconnectClientVersion++
|
||||||
|
|
||||||
|
c.closeIMAPClient()
|
||||||
|
backoff := reconnectBackoffInitial
|
||||||
|
for {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
err := c.connect()
|
||||||
|
if err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
logger.ErrorCF("email", "IMAP reconnect failed, retrying with backoff", map[string]interface{}{
|
||||||
|
"error": err.Error(), "backoff": backoff.String(),
|
||||||
|
})
|
||||||
|
timer := time.NewTimer(backoff)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
timer.Stop()
|
||||||
|
return ctx.Err()
|
||||||
|
case <-timer.C:
|
||||||
|
if backoff < reconnectBackoffMax {
|
||||||
|
backoff *= 2
|
||||||
|
if backoff > reconnectBackoffMax {
|
||||||
|
backoff = reconnectBackoffMax
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (c *EmailChannel) checkLoop(ctx context.Context) {
|
func (c *EmailChannel) checkLoop(ctx context.Context) {
|
||||||
interval := time.Duration(c.config.CheckInterval) * time.Second
|
interval := time.Duration(c.config.CheckInterval) * time.Second
|
||||||
if interval <= 0 {
|
if interval <= 0 {
|
||||||
|
|
@ -308,7 +416,7 @@ func (c *EmailChannel) checkLoop(ctx context.Context) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run one check immediately
|
// Run one check immediately
|
||||||
c.checkNewEmails()
|
c.checkNewEmails(ctx)
|
||||||
|
|
||||||
if !c.config.ForcedPolling {
|
if !c.config.ForcedPolling {
|
||||||
// support IDLE user idle loop, waiting for server push update
|
// support IDLE user idle loop, waiting for server push update
|
||||||
|
|
@ -328,7 +436,7 @@ func (c *EmailChannel) checkLoop(ctx context.Context) {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
c.checkNewEmails()
|
c.checkNewEmails(ctx)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -357,8 +465,8 @@ func (c *EmailChannel) runIdleLoop(ctx context.Context, pollInterval time.Durati
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if cl.State() != imap.SelectedState {
|
if cl.State() != imap.SelectedState {
|
||||||
if err := c.connect(); err != nil {
|
if err := c.reconnectWithBackoff(ctx); err != nil {
|
||||||
logger.ErrorCF("email", "Failed to reconnect in IDLE loop", map[string]interface{}{"error": err.Error()})
|
logger.ErrorCF("email", "Failed to reconnect after IDLE error", map[string]interface{}{"error": err.Error()})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
|
|
@ -369,11 +477,6 @@ func (c *EmailChannel) runIdleLoop(ctx context.Context, pollInterval time.Durati
|
||||||
go func() {
|
go func() {
|
||||||
idleDone <- cl.Idle(stop, opts)
|
idleDone <- cl.Idle(stop, opts)
|
||||||
}()
|
}()
|
||||||
go func() {
|
|
||||||
<-ctx.Done()
|
|
||||||
close(stop)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
close(stop)
|
close(stop)
|
||||||
|
|
@ -394,9 +497,13 @@ func (c *EmailChannel) runIdleLoop(ctx context.Context, pollInterval time.Durati
|
||||||
}
|
}
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
logger.ErrorCF("email", "IDLE ended with error after update", map[string]interface{}{"error": err.Error()})
|
logger.ErrorCF("email", "IDLE ended with error after update", map[string]interface{}{"error": err.Error()})
|
||||||
|
if err := c.reconnectWithBackoff(ctx); err != nil {
|
||||||
|
// reconnect failed, exit IDLE loop
|
||||||
|
logger.ErrorCF("email", "Failed to reconnect after IDLE error", map[string]interface{}{"error": err.Error()})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
c.checkNewEmails()
|
}
|
||||||
|
c.checkNewEmails(ctx)
|
||||||
case err := <-idleDone:
|
case err := <-idleDone:
|
||||||
// Idle returned (timeout restart or error)
|
// Idle returned (timeout restart or error)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -406,14 +513,22 @@ func (c *EmailChannel) runIdleLoop(ctx context.Context, pollInterval time.Durati
|
||||||
}
|
}
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
logger.ErrorCF("email", "IDLE ended with error", map[string]interface{}{"error": err.Error()})
|
logger.ErrorCF("email", "IDLE ended with error", map[string]interface{}{"error": err.Error()})
|
||||||
|
if err := c.reconnectWithBackoff(ctx); err != nil {
|
||||||
|
// reconnect failed , exit IDLE loop
|
||||||
|
logger.ErrorCF("email", "Failed to reconnect after IDLE error", map[string]interface{}{"error": err.Error()})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
c.checkNewEmails()
|
}
|
||||||
|
c.checkNewEmails(ctx)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *EmailChannel) checkNewEmails() {
|
func (c *EmailChannel) checkNewEmails(ctx context.Context) {
|
||||||
|
for {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
cl := c.imapClient
|
cl := c.imapClient
|
||||||
lastUID := c.lastUID
|
lastUID := c.lastUID
|
||||||
|
|
@ -423,31 +538,22 @@ func (c *EmailChannel) checkNewEmails() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check connection state
|
// Check connection state; reconnect with backoff if needed
|
||||||
if cl.State() != imap.SelectedState {
|
if cl.State() != imap.SelectedState {
|
||||||
// Reconnect
|
if err := c.reconnectWithBackoff(ctx); err != nil {
|
||||||
if err := c.connect(); err != nil {
|
|
||||||
logger.ErrorCF("email", "Failed to reconnect to IMAP server", map[string]interface{}{
|
|
||||||
"error": err.Error(),
|
|
||||||
})
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
c.mu.Lock()
|
continue
|
||||||
cl = c.imapClient
|
|
||||||
c.mu.Unlock()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only process mail after recorded lastUID (search by UID range, not by unread)
|
// Only process mail after recorded lastUID (search by UID range, not by unread)
|
||||||
criteria := imap.NewSearchCriteria()
|
criteria := imap.NewSearchCriteria()
|
||||||
|
criteria.WithoutFlags = []string{imap.SeenFlag}
|
||||||
if lastUID > 0 {
|
if lastUID > 0 {
|
||||||
// Build SeqSet for UID range (lastUID+1 to max)
|
// Build SeqSet for UID range (lastUID+1 to max)
|
||||||
seqset := new(imap.SeqSet)
|
seqset := new(imap.SeqSet)
|
||||||
seqset.AddRange(lastUID+1, 0)
|
seqset.AddRange(lastUID+1, 0)
|
||||||
criteria.Uid = seqset
|
criteria.Uid = seqset
|
||||||
criteria.WithoutFlags = []string{imap.SeenFlag}
|
|
||||||
} else {
|
|
||||||
// First run: fetch only unread
|
|
||||||
criteria.WithoutFlags = []string{imap.SeenFlag}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
uids, err := cl.UidSearch(criteria)
|
uids, err := cl.UidSearch(criteria)
|
||||||
|
|
@ -455,8 +561,13 @@ func (c *EmailChannel) checkNewEmails() {
|
||||||
logger.ErrorCF("email", "Failed to search emails", map[string]interface{}{
|
logger.ErrorCF("email", "Failed to search emails", map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
|
c.closeIMAPClient()
|
||||||
|
if err := c.reconnectWithBackoff(ctx); err != nil {
|
||||||
|
logger.ErrorCF("email", "Failed to reconnect after search emails error", map[string]interface{}{"error": err.Error()})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
if len(uids) == 0 {
|
if len(uids) == 0 {
|
||||||
return
|
return
|
||||||
|
|
@ -500,15 +611,24 @@ func (c *EmailChannel) checkNewEmails() {
|
||||||
logger.ErrorCF("email", "Failed to fetch emails", map[string]interface{}{
|
logger.ErrorCF("email", "Failed to fetch emails", map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
|
c.closeIMAPClient()
|
||||||
|
if err := c.reconnectWithBackoff(ctx); err != nil {
|
||||||
|
logger.ErrorCF("email", "Failed to reconnect after fetch emails error", map[string]interface{}{"error": err.Error()})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
// Update last processed UID
|
// Update last processed UID
|
||||||
if maxUID > 0 {
|
if maxUID > 0 {
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
|
if c.lastUID < maxUID {
|
||||||
c.lastUID = maxUID
|
c.lastUID = maxUID
|
||||||
|
}
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
}
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *EmailChannel) processEmail(msg *imap.Message) {
|
func (c *EmailChannel) processEmail(msg *imap.Message) {
|
||||||
|
|
@ -636,7 +756,7 @@ func (c *EmailChannel) extractEmailBodyAndAttachments(msg *imap.Message) (conten
|
||||||
mediaPaths = append(mediaPaths, localPath)
|
mediaPaths = append(mediaPaths, localPath)
|
||||||
attachmentRefs = append(attachmentRefs, fmt.Sprintf("[attachment: %s]", filepath.Base(localPath)))
|
attachmentRefs = append(attachmentRefs, fmt.Sprintf("[attachment: %s]", filepath.Base(localPath)))
|
||||||
} else {
|
} else {
|
||||||
attachmentRefs = append(attachmentRefs, fmt.Sprintf("[attachment: %s (save failed)]", filename))
|
attachmentRefs = append(attachmentRefs, fmt.Sprintf("[attachment: %s (save failed, you can check the attachment size limit in the config(attachment_max_bytes))]", filename))
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
attachmentRefs = append(attachmentRefs, fmt.Sprintf("[attachment: %s]", filename))
|
attachmentRefs = append(attachmentRefs, fmt.Sprintf("[attachment: %s]", filename))
|
||||||
|
|
@ -688,7 +808,7 @@ func (c *EmailChannel) extractEmailBodyAndAttachments(msg *imap.Message) (conten
|
||||||
return bodyContent, mediaPaths
|
return bodyContent, mediaPaths
|
||||||
}
|
}
|
||||||
|
|
||||||
// saveAttachmentToLocal writes the attachment stream to AttachmentDir; returns local path or empty on failure.
|
// saveAttachmentToLocal writes the attachment stream to AttachmentDir with size limit; returns local path or empty on failure or if over limit.
|
||||||
func (c *EmailChannel) saveAttachmentToLocal(uid uint32, index int, filename string, r io.Reader) string {
|
func (c *EmailChannel) saveAttachmentToLocal(uid uint32, index int, filename string, r io.Reader) string {
|
||||||
dir := strings.TrimSpace(c.config.AttachmentDir)
|
dir := strings.TrimSpace(c.config.AttachmentDir)
|
||||||
if dir == "" {
|
if dir == "" {
|
||||||
|
|
@ -698,14 +818,15 @@ func (c *EmailChannel) saveAttachmentToLocal(uid uint32, index int, filename str
|
||||||
logger.DebugCF("email", "Failed to create attachment dir", map[string]interface{}{"error": err.Error(), "dir": dir})
|
logger.DebugCF("email", "Failed to create attachment dir", map[string]interface{}{"error": err.Error(), "dir": dir})
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
limit := int64(c.config.AttachmentMaxBytes)
|
||||||
|
if limit <= 0 {
|
||||||
|
limit = defaultAttachmentMaxBytes
|
||||||
|
}
|
||||||
safeName := utils.SanitizeFilename(filename)
|
safeName := utils.SanitizeFilename(filename)
|
||||||
if safeName == "" {
|
if safeName == "" {
|
||||||
safeName = "attachment"
|
safeName = "attachment"
|
||||||
}
|
}
|
||||||
ext := filepath.Ext(safeName)
|
ext := filepath.Ext(safeName)
|
||||||
if ext == "" && filename != "" {
|
|
||||||
ext = filepath.Ext(filename)
|
|
||||||
}
|
|
||||||
localName := fmt.Sprintf("%d_%d_%s%s", uid, index, strings.TrimSuffix(safeName, ext), ext)
|
localName := fmt.Sprintf("%d_%d_%s%s", uid, index, strings.TrimSuffix(safeName, ext), ext)
|
||||||
localPath := filepath.Join(dir, localName)
|
localPath := filepath.Join(dir, localName)
|
||||||
f, err := os.Create(localPath)
|
f, err := os.Create(localPath)
|
||||||
|
|
@ -714,11 +835,19 @@ func (c *EmailChannel) saveAttachmentToLocal(uid uint32, index int, filename str
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
if _, err := io.Copy(f, r); err != nil {
|
// +1 to detect if the attachment exceeds the limit
|
||||||
|
limited := io.LimitReader(r, limit+1)
|
||||||
|
n, err := io.Copy(f, limited)
|
||||||
|
if err != nil {
|
||||||
_ = os.Remove(localPath)
|
_ = os.Remove(localPath)
|
||||||
logger.DebugCF("email", "Failed to write attachment", map[string]interface{}{"error": err.Error(), "path": localPath})
|
logger.DebugCF("email", "Failed to write attachment", map[string]interface{}{"error": err.Error(), "path": localPath})
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
if n > limit {
|
||||||
|
_ = os.Remove(localPath)
|
||||||
|
logger.DebugCF("email", "Attachment exceeds size limit, skipped", map[string]interface{}{"path": localPath, "limit": limit})
|
||||||
|
return ""
|
||||||
|
}
|
||||||
return localPath
|
return localPath
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -755,6 +884,7 @@ func parseFilenameFromDisposition(disp string) string {
|
||||||
if i < 0 {
|
if i < 0 {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
disp = disp[i+len(fn):]
|
disp = disp[i+len(fn):]
|
||||||
disp = strings.TrimLeft(disp, " \t")
|
disp = strings.TrimLeft(disp, " \t")
|
||||||
if len(disp) >= 2 && (disp[0] == '"' || disp[0] == '\'') {
|
if len(disp) >= 2 && (disp[0] == '"' || disp[0] == '\'') {
|
||||||
|
|
|
||||||
217
pkg/channels/email_test.go
Normal file
217
pkg/channels/email_test.go
Normal file
|
|
@ -0,0 +1,217 @@
|
||||||
|
package channels
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/emersion/go-imap"
|
||||||
|
"github.com/sipeed/picoclaw/pkg/config"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestEmailChannel_sanitizeHeaderValue(t *testing.T) {
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
s string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "empty", s: "", want: ""},
|
||||||
|
{name: "simple", s: "test", want: "test"},
|
||||||
|
{name: "crlf", s: "test\r\n", want: "test"},
|
||||||
|
{name: "lf", s: "test\n", want: "test"},
|
||||||
|
{name: "cr", s: "test\r", want: "test"},
|
||||||
|
{name: "crlf", s: "test\r\n", want: "test"},
|
||||||
|
{name: "lfcr", s: "test\n\r", want: "test"},
|
||||||
|
{name: "crlfcr", s: "test\r\n\r", want: "test"},
|
||||||
|
{name: "lfcrlf", s: "test\n\r\n", want: "test"},
|
||||||
|
{name: "crlfcrlf", s: "test\r\n\r\n", want: "test"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got := sanitizeHeaderValue(tt.s); got != tt.want {
|
||||||
|
t.Errorf("sanitizeHeaderValue(%q) = %q, want %q", tt.s, got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEmailChannel_parseFilenameFromDisposition(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
s string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "inline", s: "inline; filename=\"pico.png\"", want: "pico.png"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got := parseFilenameFromDisposition(tt.s); got != tt.want {
|
||||||
|
t.Errorf("parseFilenameFromDisposition(%q) = %q, want %q", tt.s, got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestEmailChannel_decodeRFC2047Filename(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
s string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "normal", s: "正常.png", want: "正常.png"},
|
||||||
|
{name: "GB2312-Quoted-Printable", s: "=?GB2312?Q?gb2312=B2=E2=CA=D4=B2=E2=CA=D4.png?=", want: "gb2312测试测试.png"},
|
||||||
|
{name: "GBK-Base64", s: "=?GBK?B?yfqzybLiytTNvMasLnBuZw==?=", want: "生成测试图片.png"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got := decodeRFC2047Filename(tt.s); got != tt.want {
|
||||||
|
t.Errorf("decodeRFC2047Filename(%q) = %q, want %q", tt.s, got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEmailChannel_extractEmailBodyAndAttachments(t *testing.T) {
|
||||||
|
c := &EmailChannel{
|
||||||
|
config: config.EmailConfig{
|
||||||
|
AttachmentDir: t.TempDir(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
t.Run("nil message", func(t *testing.T) {
|
||||||
|
content, paths := c.extractEmailBodyAndAttachments(nil)
|
||||||
|
assert.Empty(t, content)
|
||||||
|
assert.Nil(t, paths)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("plain text body", func(t *testing.T) {
|
||||||
|
mimeBytes := []byte("From: a@b.com\r\nTo: c@d.com\r\nSubject: Test\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nHello world")
|
||||||
|
section := &imap.BodySectionName{}
|
||||||
|
msg := &imap.Message{
|
||||||
|
Uid: 1,
|
||||||
|
Envelope: &imap.Envelope{Subject: "Test"},
|
||||||
|
Body: map[*imap.BodySectionName]imap.Literal{section: bytes.NewReader(mimeBytes)},
|
||||||
|
}
|
||||||
|
content, paths := c.extractEmailBodyAndAttachments(msg)
|
||||||
|
assert.Contains(t, content, "Subject: Test")
|
||||||
|
assert.Contains(t, content, "Hello world")
|
||||||
|
assert.Empty(t, paths)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("html body", func(t *testing.T) {
|
||||||
|
mimeBytes := []byte("From: a@b.com\r\nTo: c@d.com\r\nSubject: Test\r\nContent-Type: text/html; charset=utf-8\r\n\r\n<html><body><h1>Hello world</h1></body></html>")
|
||||||
|
section := &imap.BodySectionName{}
|
||||||
|
msg := &imap.Message{
|
||||||
|
Uid: 1,
|
||||||
|
Envelope: &imap.Envelope{Subject: "Test"},
|
||||||
|
Body: map[*imap.BodySectionName]imap.Literal{section: bytes.NewReader(mimeBytes)},
|
||||||
|
}
|
||||||
|
content, paths := c.extractEmailBodyAndAttachments(msg)
|
||||||
|
assert.Contains(t, content, "Subject: Test")
|
||||||
|
assert.Contains(t, content, "Hello world")
|
||||||
|
assert.Empty(t, paths)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("attachment and text body", func(t *testing.T) {
|
||||||
|
mimeBytes := []byte(
|
||||||
|
"From: a@b.com\r\n" +
|
||||||
|
"To: c@d.com\r\n" +
|
||||||
|
"Subject: test-file-and-body\r\n" +
|
||||||
|
"Content-Type: multipart/mixed; boundary=\"outer\"\r\n" +
|
||||||
|
"MIME-Version: 1.0\r\n" +
|
||||||
|
"\r\n" +
|
||||||
|
"--outer\r\n" +
|
||||||
|
"Content-Type: multipart/alternative; boundary=\"alt\"\r\n" +
|
||||||
|
"\r\n" +
|
||||||
|
"--alt\r\n" +
|
||||||
|
"Content-Type: text/plain; charset=GBK\r\n" +
|
||||||
|
"Content-Transfer-Encoding: 7bit\r\n" +
|
||||||
|
"\r\n" +
|
||||||
|
"this body\r\n" +
|
||||||
|
"--alt\r\n" +
|
||||||
|
"Content-Type: text/html; charset=GBK\r\n" +
|
||||||
|
"Content-Transfer-Encoding: 7bit\r\n" +
|
||||||
|
"\r\n" +
|
||||||
|
"<div>this body</div>\r\n" +
|
||||||
|
"--alt--\r\n" +
|
||||||
|
"\r\n" +
|
||||||
|
"--outer\r\n" +
|
||||||
|
"Content-Type: text/plain; name=test.txt\r\n" +
|
||||||
|
"Content-Transfer-Encoding: base64\r\n" +
|
||||||
|
"Content-Disposition: attachment; filename=\"test.txt\"\r\n" +
|
||||||
|
"\r\n" +
|
||||||
|
"VGVzdC0xMTEx\r\n" +
|
||||||
|
"--outer--\r\n")
|
||||||
|
section := &imap.BodySectionName{}
|
||||||
|
msg := &imap.Message{
|
||||||
|
Uid: 1,
|
||||||
|
Envelope: &imap.Envelope{Subject: "Test"},
|
||||||
|
Body: map[*imap.BodySectionName]imap.Literal{section: bytes.NewReader(mimeBytes)},
|
||||||
|
}
|
||||||
|
content, paths := c.extractEmailBodyAndAttachments(msg)
|
||||||
|
assert.Contains(t, content, "this body")
|
||||||
|
assert.NotEmpty(t, paths)
|
||||||
|
assert.Equal(t, 1, len(paths))
|
||||||
|
assert.Contains(t, paths[0], filepath.Base(paths[0]))
|
||||||
|
})
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEmailChannel_extractTextFromHTML(t *testing.T) {
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
s string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "with-script-and-style", s: `<!DOCTYPE html>
|
||||||
|
<html lang="zh-CN">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<title>Minimal</title>
|
||||||
|
<style>
|
||||||
|
body { margin: 0; font-family: sans-serif; }
|
||||||
|
.box { padding: 1rem; background: #eee; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="box">Hello</div>
|
||||||
|
<script>
|
||||||
|
document.querySelector('.box').onclick = function() {
|
||||||
|
this.textContent = 'Clicked';
|
||||||
|
};
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>`, want: "Minimal\n \n\n Hello"},
|
||||||
|
}
|
||||||
|
c := &EmailChannel{}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got := c.extractTextFromHTML(tt.s); got != tt.want {
|
||||||
|
t.Errorf("extractTextFromHTML(%q) = %q, want %q", tt.s, got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_saveAttachmentToLocal(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
c := &EmailChannel{
|
||||||
|
config: config.EmailConfig{
|
||||||
|
AttachmentDir: tmpDir,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
content := "Hello world"
|
||||||
|
body := bytes.NewReader([]byte(content))
|
||||||
|
// test attachment max bytes
|
||||||
|
c.config.AttachmentMaxBytes = len([]byte(content))
|
||||||
|
path := c.saveAttachmentToLocal(1, 1, "test.txt", body)
|
||||||
|
assert.NotEmpty(t, path)
|
||||||
|
assert.Equal(t, path, filepath.Join(tmpDir, "1_1_test.txt"))
|
||||||
|
|
||||||
|
// test greater than attachment max bytes
|
||||||
|
c.config.AttachmentMaxBytes = len([]byte(content)) - 1
|
||||||
|
body = bytes.NewReader([]byte(content))
|
||||||
|
path = c.saveAttachmentToLocal(1, 1, "test.txt", body)
|
||||||
|
assert.Empty(t, path)
|
||||||
|
}
|
||||||
|
|
@ -161,6 +161,8 @@ type EmailConfig struct {
|
||||||
ForcedPolling bool `json:"forced_polling" env:"PICOCLAW_CHANNELS_EMAIL_FORCED_POLLING"`
|
ForcedPolling bool `json:"forced_polling" env:"PICOCLAW_CHANNELS_EMAIL_FORCED_POLLING"`
|
||||||
AllowFrom FlexibleStringSlice `json:"allow_from" env:"PICOCLAW_CHANNELS_EMAIL_ALLOW_FROM"`
|
AllowFrom FlexibleStringSlice `json:"allow_from" env:"PICOCLAW_CHANNELS_EMAIL_ALLOW_FROM"`
|
||||||
AttachmentDir string `json:"attachment_dir" env:"PICOCLAW_CHANNELS_EMAIL_ATTACHMENT_DIR"`
|
AttachmentDir string `json:"attachment_dir" env:"PICOCLAW_CHANNELS_EMAIL_ATTACHMENT_DIR"`
|
||||||
|
// max size per attachment (default 25*1024*1024(25MB)), 0 = use default
|
||||||
|
AttachmentMaxBytes int `json:"attachment_max_bytes" env:"PICOCLAW_CHANNELS_EMAIL_ATTACHMENT_MAX_BYTES"` // max size per attachment (default 25MB), 0 = use default
|
||||||
// SMTP send (optional, if not configured, Send is not available)
|
// SMTP send (optional, if not configured, Send is not available)
|
||||||
SMTPServer string `json:"smtp_server" env:"PICOCLAW_CHANNELS_EMAIL_SMTP_SERVER"`
|
SMTPServer string `json:"smtp_server" env:"PICOCLAW_CHANNELS_EMAIL_SMTP_SERVER"`
|
||||||
SMTPPort int `json:"smtp_port" env:"PICOCLAW_CHANNELS_EMAIL_SMTP_PORT"` // 465 或 587
|
SMTPPort int `json:"smtp_port" env:"PICOCLAW_CHANNELS_EMAIL_SMTP_PORT"` // 465 或 587
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue