infrastructure/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2025-07-10 07:14:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

improve training of junk filter

Browse Source 
before, we used heuristics to decide when to train/untrain a message as junk or
nonjunk: the message had to be seen, be in certain mailboxes. then if a message
was marked as junk, it was junk. and otherwise it was nonjunk. this wasn't good
enough: you may want to keep some messages around as neither junk or nonjunk.
and that wasn't possible.

ideally, we would just look at the imap $Junk and $NotJunk flags. the problem
is that mail clients don't set these flags, or don't make it easy. thunderbird
can set the flags based on its own bayesian filter. it has a shortcut for
marking Junk and moving it to the junk folder (good), but the counterpart of
notjunk only marks a message as notjunk without showing in the UI that it was
marked as notjunk. there is also no "move and mark as notjunk" mechanism. e.g.
"archive" does not mark a message as notjunk. ios mail and mutt don't appear to
have any way to see or change the $Junk and $NotJunk flags.

what email clients do have is the ability to move messages to other
mailboxes/folders. so mox now has a mechanism that allows you to configure
mailboxes that automatically set $Junk or $NotJunk (or clear both) when a
message is moved/copied/delivered to that folder. e.g. a mailbox called junk or
spam or rejects marks its messags as junk. inbox, postmaster, dmarc, tlsrpt,
neutral* mark their messages as neither junk or notjunk. other folders mark
their messages as notjunk. e.g. list/*, archive. this functionality is
optional, but enabled with the quickstart and for new accounts.

also, mox now keeps track of the previous training of a message and will only
untrain/train if needed. before, there probably have been duplicate or missing
(un)trainings.

this also includes a new subcommand "retrain" to recreate the junkfilter for an
account. you should run it after updating to this version. and you should
probably also modify your account config to include the AutomaticJunkFlags.
This commit is contained in:
Mechiel Lukkien
2023-02-11 23:00:12 +01:00
parent a4306ef783
commit bf04fb8a1a
26 changed files with 410 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
import.go
View File

@ -216,7 +216,10 @@ func importctl(ctl *ctl, mbox bool) {
// Messages don't always have a junk flag set. We'll assume anything in a mailbox
// starting with junk or spam is junk mail.
isjunk := strings.HasPrefix(strings.ToLower(mailbox), "junk") || strings.HasPrefix(strings.ToLower(mailbox), "spam")
var msgJunkFlags store.Message
conf, _ := a.Conf()
msgJunkFlags.JunkFlagsForMailbox(mailbox, conf)
// First check if we can access the mbox/maildir.
// Mox needs to be able to access those files, the user running the import command
@ -224,13 +227,13 @@ func importctl(ctl *ctl, mbox bool) {
if mbox {
mboxf, err = os.Open(src)
ctl.xcheck(err, "open mbox file")
msgreader = newMboxReader(isjunk, store.CreateMessageTemp, mboxf, ctl.log)
msgreader = newMboxReader(msgJunkFlags.Junk, msgJunkFlags.Notjunk, store.CreateMessageTemp, mboxf, ctl.log)
} else {
mdnewf, err = os.Open(filepath.Join(src, "new"))
ctl.xcheck(err, "open subdir new of maildir")
mdcurf, err = os.Open(filepath.Join(src, "cur"))
ctl.xcheck(err, "open subdir cur of maildir")
msgreader = newMaildirReader(isjunk, store.CreateMessageTemp, mdnewf, mdcurf, ctl.log)
msgreader = newMaildirReader(msgJunkFlags.Junk, msgJunkFlags.Notjunk, store.CreateMessageTemp, mdnewf, mdcurf, ctl.log)
}
tx, err := a.DB.Begin(true)
@ -276,8 +279,7 @@ func importctl(ctl *ctl, mbox bool) {
const consumeFile = true
isSent := mailbox == "Sent"
const sync = false
const train = false
a.DeliverX(ctl.log, tx, m, mf, consumeFile, isSent, sync, train)
a.DeliverX(ctl.log, tx, m, mf, consumeFile, isSent, sync)
deliveredIDs = append(deliveredIDs, m.ID)
ctl.log.Debug("delivered message", mlog.Field("id", m.ID))
changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, Flags: m.Flags})
@ -337,12 +339,13 @@ func importctl(ctl *ctl, mbox bool) {
}
}
if jf != nil && (m.Seen || m.Junk) {
if jf != nil && m.NeedsTraining() {
if words, err := jf.ParseMessage(p); err != nil {
ctl.log.Infox("parsing message for updating junk filter", err, mlog.Field("parse", ""), mlog.Field("path", origPath))
} else {
err = jf.Train(!m.Junk, words)
ctl.xcheck(err, "training junk filter")
m.TrainedJunk = &m.Junk
}
}
@ -402,10 +405,11 @@ type mboxReader struct {
log *mlog.Log
eof bool
junk bool
notjunk bool
}
func newMboxReader(isjunk bool, createTemp func(pattern string) (*os.File, error), f *os.File, log *mlog.Log) *mboxReader {
return &mboxReader{createTemp: createTemp, path: f.Name(), line: 1, r: bufio.NewReader(f), log: log, junk: isjunk}
func newMboxReader(isjunk, isnotjunk bool, createTemp func(pattern string) (*os.File, error), f *os.File, log *mlog.Log) *mboxReader {
return &mboxReader{createTemp: createTemp, path: f.Name(), line: 1, r: bufio.NewReader(f), log: log, junk: isjunk, notjunk: isnotjunk}
}
func (mr *mboxReader) position() string {
@ -486,7 +490,7 @@ func (mr *mboxReader) Next() (*store.Message, *os.File, string, error) {
// todo: look at Status or X-Status header in message?
// todo: take Received from the "From " line if present?
flags := store.Flags{Seen: true, Junk: mr.junk}
flags := store.Flags{Seen: true, Junk: mr.junk, Notjunk: mr.notjunk}
m := &store.Message{Flags: flags, Size: size}
// Prevent cleanup by defer.
@ -505,9 +509,10 @@ type maildirReader struct {
dovecotKeywords []string
log *mlog.Log
junk bool
notjunk bool
}
func newMaildirReader(isjunk bool, createTemp func(pattern string) (*os.File, error), newf, curf *os.File, log *mlog.Log) *maildirReader {
func newMaildirReader(isjunk, isnotjunk bool, createTemp func(pattern string) (*os.File, error), newf, curf *os.File, log *mlog.Log) *maildirReader {
mr := &maildirReader{createTemp: createTemp, newf: newf, curf: curf, f: newf, log: log, junk: isjunk}
// Best-effort parsing of dovecot keywords.
@ -642,6 +647,9 @@ func (mr *maildirReader) Next() (*store.Message, *os.File, string, error) {
if mr.junk {
flags.Junk = true
}
if mr.notjunk {
flags.Notjunk = true
}
m := &store.Message{Received: received, Flags: flags, Size: size}