fix(channels): cap email body part size to avoid unbounded io.ReadAll
- Add defaultBodyPartMaxBytes (1MB) and config body_part_max_bytes; 0 = use default - Use io.LimitReader when reading text/plain and text/html parts; show placeholder when part exceeds limit - Add BodyPartMaxBytes to EmailConfig (config.go) - Add test "body part max bytes" for extractEmailBodyAndAttachments with limit
This commit is contained in:
parent
8417229012
commit
a27e191214
3 changed files with 65 additions and 30 deletions
|
|
@ -38,6 +38,8 @@ const (
|
||||||
reconnectBackoffMax = 10 * time.Minute
|
reconnectBackoffMax = 10 * time.Minute
|
||||||
// default attachment max bytes
|
// default attachment max bytes
|
||||||
defaultAttachmentMaxBytes = 25 * 1024 * 1024 // 25MB
|
defaultAttachmentMaxBytes = 25 * 1024 * 1024 // 25MB
|
||||||
|
// max bytes to read per body part (text/plain, text/html) to avoid unbounded io.ReadAll
|
||||||
|
defaultBodyPartMaxBytes = 1 * 1024 * 1024 // 1MB
|
||||||
)
|
)
|
||||||
|
|
||||||
type EmailChannel struct {
|
type EmailChannel struct {
|
||||||
|
|
@ -764,10 +766,19 @@ func (c *EmailChannel) extractEmailBodyAndAttachments(msg *imap.Message) (conten
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
body, err := io.ReadAll(p.Body)
|
limit := int64(c.config.BodyPartMaxBytes)
|
||||||
|
if limit <= 0 {
|
||||||
|
limit = int64(defaultBodyPartMaxBytes)
|
||||||
|
}
|
||||||
|
limitedBody := io.LimitReader(p.Body, limit+1)
|
||||||
|
body, err := io.ReadAll(limitedBody)
|
||||||
if err != nil || len(body) == 0 {
|
if err != nil || len(body) == 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if len(body) > int(limit) {
|
||||||
|
textParts = append(textParts, fmt.Sprintf("[body part exceeds size limit (max %d bytes), you can check body_part_max_bytes in config]", limit))
|
||||||
|
continue
|
||||||
|
}
|
||||||
bodyStr := strings.TrimSpace(string(body))
|
bodyStr := strings.TrimSpace(string(body))
|
||||||
if bodyStr == "" {
|
if bodyStr == "" {
|
||||||
continue
|
continue
|
||||||
|
|
|
||||||
|
|
@ -112,36 +112,37 @@ func TestEmailChannel_extractEmailBodyAndAttachments(t *testing.T) {
|
||||||
assert.Empty(t, paths)
|
assert.Empty(t, paths)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
attachmentwithText := []byte(
|
||||||
|
"From: a@b.com\r\n" +
|
||||||
|
"To: c@d.com\r\n" +
|
||||||
|
"Subject: test-file-and-body\r\n" +
|
||||||
|
"Content-Type: multipart/mixed; boundary=\"outer\"\r\n" +
|
||||||
|
"MIME-Version: 1.0\r\n" +
|
||||||
|
"\r\n" +
|
||||||
|
"--outer\r\n" +
|
||||||
|
"Content-Type: multipart/alternative; boundary=\"alt\"\r\n" +
|
||||||
|
"\r\n" +
|
||||||
|
"--alt\r\n" +
|
||||||
|
"Content-Type: text/plain; charset=GBK\r\n" +
|
||||||
|
"Content-Transfer-Encoding: 7bit\r\n" +
|
||||||
|
"\r\n" +
|
||||||
|
"this body\r\n" +
|
||||||
|
"--alt\r\n" +
|
||||||
|
"Content-Type: text/html; charset=GBK\r\n" +
|
||||||
|
"Content-Transfer-Encoding: 7bit\r\n" +
|
||||||
|
"\r\n" +
|
||||||
|
"<div>this body</div>\r\n" +
|
||||||
|
"--alt--\r\n" +
|
||||||
|
"\r\n" +
|
||||||
|
"--outer\r\n" +
|
||||||
|
"Content-Type: text/plain; name=test.txt\r\n" +
|
||||||
|
"Content-Transfer-Encoding: base64\r\n" +
|
||||||
|
"Content-Disposition: attachment; filename=\"test.txt\"\r\n" +
|
||||||
|
"\r\n" +
|
||||||
|
"VGVzdC0xMTEx\r\n" +
|
||||||
|
"--outer--\r\n")
|
||||||
t.Run("attachment and text body", func(t *testing.T) {
|
t.Run("attachment and text body", func(t *testing.T) {
|
||||||
mimeBytes := []byte(
|
mimeBytes := attachmentwithText
|
||||||
"From: a@b.com\r\n" +
|
|
||||||
"To: c@d.com\r\n" +
|
|
||||||
"Subject: test-file-and-body\r\n" +
|
|
||||||
"Content-Type: multipart/mixed; boundary=\"outer\"\r\n" +
|
|
||||||
"MIME-Version: 1.0\r\n" +
|
|
||||||
"\r\n" +
|
|
||||||
"--outer\r\n" +
|
|
||||||
"Content-Type: multipart/alternative; boundary=\"alt\"\r\n" +
|
|
||||||
"\r\n" +
|
|
||||||
"--alt\r\n" +
|
|
||||||
"Content-Type: text/plain; charset=GBK\r\n" +
|
|
||||||
"Content-Transfer-Encoding: 7bit\r\n" +
|
|
||||||
"\r\n" +
|
|
||||||
"this body\r\n" +
|
|
||||||
"--alt\r\n" +
|
|
||||||
"Content-Type: text/html; charset=GBK\r\n" +
|
|
||||||
"Content-Transfer-Encoding: 7bit\r\n" +
|
|
||||||
"\r\n" +
|
|
||||||
"<div>this body</div>\r\n" +
|
|
||||||
"--alt--\r\n" +
|
|
||||||
"\r\n" +
|
|
||||||
"--outer\r\n" +
|
|
||||||
"Content-Type: text/plain; name=test.txt\r\n" +
|
|
||||||
"Content-Transfer-Encoding: base64\r\n" +
|
|
||||||
"Content-Disposition: attachment; filename=\"test.txt\"\r\n" +
|
|
||||||
"\r\n" +
|
|
||||||
"VGVzdC0xMTEx\r\n" +
|
|
||||||
"--outer--\r\n")
|
|
||||||
section := &imap.BodySectionName{}
|
section := &imap.BodySectionName{}
|
||||||
msg := &imap.Message{
|
msg := &imap.Message{
|
||||||
Uid: 1,
|
Uid: 1,
|
||||||
|
|
@ -155,6 +156,28 @@ func TestEmailChannel_extractEmailBodyAndAttachments(t *testing.T) {
|
||||||
assert.Contains(t, paths[0], filepath.Base(paths[0]))
|
assert.Contains(t, paths[0], filepath.Base(paths[0]))
|
||||||
})
|
})
|
||||||
|
|
||||||
|
newLimitClient := &EmailChannel{
|
||||||
|
config: config.EmailConfig{
|
||||||
|
BodyPartMaxBytes: 1,
|
||||||
|
AttachmentDir: t.TempDir(),
|
||||||
|
AttachmentMaxBytes: 1024,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
t.Run("body part max bytes", func(t *testing.T) {
|
||||||
|
|
||||||
|
mimeBytes := attachmentwithText
|
||||||
|
section := &imap.BodySectionName{}
|
||||||
|
msg := &imap.Message{
|
||||||
|
Uid: 1,
|
||||||
|
Envelope: &imap.Envelope{Subject: "Test"},
|
||||||
|
Body: map[*imap.BodySectionName]imap.Literal{section: bytes.NewReader(mimeBytes)},
|
||||||
|
}
|
||||||
|
content, paths := newLimitClient.extractEmailBodyAndAttachments(msg)
|
||||||
|
assert.Contains(t, content, "[body part exceeds size limit (max 1 bytes), you can check body_part_max_bytes in config]")
|
||||||
|
assert.NotEmpty(t, paths)
|
||||||
|
assert.Equal(t, 1, len(paths))
|
||||||
|
assert.Contains(t, paths[0], filepath.Base(paths[0]))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEmailChannel_extractTextFromHTML(t *testing.T) {
|
func TestEmailChannel_extractTextFromHTML(t *testing.T) {
|
||||||
|
|
|
||||||
|
|
@ -163,6 +163,7 @@ type EmailConfig struct {
|
||||||
AttachmentDir string `json:"attachment_dir" env:"PICOCLAW_CHANNELS_EMAIL_ATTACHMENT_DIR"`
|
AttachmentDir string `json:"attachment_dir" env:"PICOCLAW_CHANNELS_EMAIL_ATTACHMENT_DIR"`
|
||||||
// max size per attachment (default 25*1024*1024(25MB)), 0 = use default
|
// max size per attachment (default 25*1024*1024(25MB)), 0 = use default
|
||||||
AttachmentMaxBytes int `json:"attachment_max_bytes" env:"PICOCLAW_CHANNELS_EMAIL_ATTACHMENT_MAX_BYTES"` // max size per attachment (default 25MB), 0 = use default
|
AttachmentMaxBytes int `json:"attachment_max_bytes" env:"PICOCLAW_CHANNELS_EMAIL_ATTACHMENT_MAX_BYTES"` // max size per attachment (default 25MB), 0 = use default
|
||||||
|
BodyPartMaxBytes int `json:"body_part_max_bytes" env:"PICOCLAW_CHANNELS_EMAIL_BODY_PART_MAX_BYTES"` // max size per body part (text/plain, text/html) to avoid unbounded io.ReadAll (default 1MB), 0 = use default
|
||||||
// SMTP send (optional, if not configured, Send is not available)
|
// SMTP send (optional, if not configured, Send is not available)
|
||||||
SMTPServer string `json:"smtp_server" env:"PICOCLAW_CHANNELS_EMAIL_SMTP_SERVER"`
|
SMTPServer string `json:"smtp_server" env:"PICOCLAW_CHANNELS_EMAIL_SMTP_SERVER"`
|
||||||
SMTPPort int `json:"smtp_port" env:"PICOCLAW_CHANNELS_EMAIL_SMTP_PORT"` // 465 或 587
|
SMTPPort int `json:"smtp_port" env:"PICOCLAW_CHANNELS_EMAIL_SMTP_PORT"` // 465 或 587
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue