From 707d3a3fa06dd15d603128d0dd635d5d4bd61899 Mon Sep 17 00:00:00 2001 From: Mechiel Lukkien Date: Mon, 13 Feb 2023 10:47:20 +0100 Subject: [PATCH] store rejects for 14 days, and don't keep them as neutral by default so they won't cause outright rejects for repeated delivery attempts of spam messages the previous default, marking the messages as junk had the interesting effect of training the junk filter. rejecting could have been the result of the sending IP being in the DNSBL. so the DNSBL helped to automatically train the junk filter. perhaps we can keep that in the future and just not take messages from the rejects mailbox into account when evaluating the reputation for incoming deliveries. --- config/config.go | 6 +++--- config/doc.go | 13 +++++++------ mox-/admin.go | 4 ++-- smtpserver/server.go | 4 +++- store/account.go | 2 +- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/config/config.go b/config/config.go index 7245767..4a014c1 100644 --- a/config/config.go +++ b/config/config.go @@ -200,9 +200,9 @@ type Account struct { } `sconf:"optional" sconf-doc:"If configured, messages classified as weakly spam are rejected with instructions to retry delivery, but this time with a signed token added to the subject. During the next delivery attempt, the signed token will bypass the spam filter. Messages with a clear spam signal, such as a known bad reputation, are rejected/delayed without a signed token."` RejectsMailbox string `sconf:"optional" sconf-doc:"Mail that looks like spam will be rejected, but a copy can be stored temporarily in a mailbox, e.g. Rejects. If mail isn't coming in when you expect, you can look there. The mail still isn't accepted, so the remote mail server may retry (hopefully, if legitimate), or give up (hopefully, if indeed a spammer). Messages are automatically removed from this mailbox, so do not set it to a mailbox that has messages you want to keep."` AutomaticJunkFlags struct { - Enabled bool `sconf-doc:"If enabled, flags will be set automatically if they match a regular expression below. When two lists are set, the empty list will match all remaining messages. Messages are matched in the order specified and the search stops on the first match. Mailboxes are lowercased before matching."` - JunkMailboxRegexp string `sconf:"optional" sconf-doc:"Example: ^(junk|spam|rejects)."` - NeutralMailboxRegexp string `sconf:"optional" sconf-doc:"Example: ^(inbox|neutral|postmaster|dmarc|tlsrpt), and you may wish to add trash depending on how you use it, or leave this empty."` + Enabled bool `sconf-doc:"If enabled, flags will be set automatically if they match a regular expression below. When two of the three mailbox regular expressions are set, the remaining one will match all unmatched messages. Messages are matched in the order specified and the search stops on the first match. Mailboxes are lowercased before matching."` + JunkMailboxRegexp string `sconf:"optional" sconf-doc:"Example: ^(junk|spam)."` + NeutralMailboxRegexp string `sconf:"optional" sconf-doc:"Example: ^(inbox|neutral|postmaster|dmarc|tlsrpt|rejects), and you may wish to add trash depending on how you use it, or leave this empty."` NotJunkMailboxRegexp string `sconf:"optional" sconf-doc:"Example: .* or an empty string."` } `sconf:"optional" sconf-doc:"Automatically set $Junk and $NotJunk flags based on mailbox messages are delivered/moved/copied to. Email clients typically have too limited functionality to conveniently set these flags, especially $NonJunk, but they can all move messages to a different mailbox, so this helps them."` JunkFilter *JunkFilter `sconf:"optional" sconf-doc:"Content-based filtering, using the junk-status of individual messages to rank words in such messages as spam or ham. It is recommended you always set the applicable (non)-junk status on messages, and that you do not empty your Trash because those messages contain valuable ham/spam training information."` // todo: sane defaults for junkfilter diff --git a/config/doc.go b/config/doc.go index fd3ad64..73d590e 100644 --- a/config/doc.go +++ b/config/doc.go @@ -435,16 +435,17 @@ describe-static" and "mox config describe-domains": AutomaticJunkFlags: # If enabled, flags will be set automatically if they match a regular expression - # below. When two lists are set, the empty list will match all remaining messages. - # Messages are matched in the order specified and the search stops on the first - # match. Mailboxes are lowercased before matching. + # below. When two of the three mailbox regular expressions are set, the remaining + # one will match all unmatched messages. Messages are matched in the order + # specified and the search stops on the first match. Mailboxes are lowercased + # before matching. Enabled: false - # Example: ^(junk|spam|rejects). (optional) + # Example: ^(junk|spam). (optional) JunkMailboxRegexp: - # Example: ^(inbox|neutral|postmaster|dmarc|tlsrpt), and you may wish to add trash - # depending on how you use it, or leave this empty. (optional) + # Example: ^(inbox|neutral|postmaster|dmarc|tlsrpt|rejects), and you may wish to + # add trash depending on how you use it, or leave this empty. (optional) NeutralMailboxRegexp: # Example: .* or an empty string. (optional) diff --git a/mox-/admin.go b/mox-/admin.go index 4babcdd..6ec1af4 100644 --- a/mox-/admin.go +++ b/mox-/admin.go @@ -130,8 +130,8 @@ func MakeAccountConfig(addr smtp.Address) config.Account { }, } account.AutomaticJunkFlags.Enabled = true - account.AutomaticJunkFlags.JunkMailboxRegexp = "^(junk|spam|rejects)" - account.AutomaticJunkFlags.NeutralMailboxRegexp = "^(inbox|neutral|postmaster|dmarc|tlsrpt)" + account.AutomaticJunkFlags.JunkMailboxRegexp = "^(junk|spam)" + account.AutomaticJunkFlags.NeutralMailboxRegexp = "^(inbox|neutral|postmaster|dmarc|tlsrpt|rejects)" account.SubjectPass.Period = 12 * time.Hour return account } diff --git a/smtpserver/server.go b/smtpserver/server.go index 3c78f13..5945d7d 100644 --- a/smtpserver/server.go +++ b/smtpserver/server.go @@ -2151,7 +2151,9 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW log.Errorx("checking whether reject is already present", err) } else if !present { m.Seen = true // We don't want to draw attention. - m.Junk = true // This is junk, also train as such. + // Regular automatic junk flags configuration applies to these messages. The + // default is to treat these are neutral, so they won't cause outright rejections + // due to reputation for later delivery attempts. m.MessageID = messageid m.MessageHash = messagehash acc.WithWLock(func() { diff --git a/store/account.go b/store/account.go index 6f27404..ba09fa0 100644 --- a/store/account.go +++ b/store/account.go @@ -1039,7 +1039,7 @@ func (a *Account) TidyRejectsMailbox(log *mlog.Log, rejectsMailbox string) (hasS } // Gather old messages to remove. - old := time.Now().Add(-24 * time.Hour) + old := time.Now().Add(-14 * 24 * time.Hour) qdel := bstore.QueryTx[Message](tx) qdel.FilterNonzero(Message{MailboxID: mb.ID}) qdel.FilterLess("Received", old)