mox!

store/account_test.go

@@ -0,0 +1,273 @@
+package store
+
+import (
+	"os"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mjl-/bstore"
+	"github.com/mjl-/sconf"
+
+	"github.com/mjl-/mox/config"
+	"github.com/mjl-/mox/message"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
+)
+
+func tcheck(t *testing.T, err error, msg string) {
+	t.Helper()
+	if err != nil {
+		t.Fatalf("%s: %s", msg, err)
+	}
+}
+
+func TestMailbox(t *testing.T) {
+	os.RemoveAll("../testdata/store/data")
+	mox.ConfigStaticPath = "../testdata/store/mox.conf"
+	mox.MustLoadConfig()
+	acc, err := OpenAccount("mjl")
+	tcheck(t, err, "open account")
+	defer acc.Close()
+	switchDone := Switchboard()
+	defer close(switchDone)
+
+	log := mlog.New("store")
+
+	msgFile, err := CreateMessageTemp("account-test")
+	if err != nil {
+		t.Fatalf("creating temp msg file: %s", err)
+	}
+	defer msgFile.Close()
+	msgWriter := &message.Writer{Writer: msgFile}
+	if _, err := msgWriter.Write([]byte(" message")); err != nil {
+		t.Fatalf("writing to temp message: %s", err)
+	}
+
+	msgPrefix := []byte("From: <mjl@mox.example\r\nTo: <mjl@mox.example>\r\nCc: <mjl@mox.example>Subject: test\r\nMessage-Id: <m01@mox.example>\r\n\r\n")
+	msgPrefixCatchall := []byte("Subject: catchall\r\n\r\n")
+	m := Message{
+		Received:  time.Now(),
+		Size:      int64(len(msgPrefix)) + msgWriter.Size,
+		MsgPrefix: msgPrefix,
+	}
+	msent := m
+	var mbsent Mailbox
+	mbrejects := Mailbox{Name: "Rejects", UIDValidity: 1, UIDNext: 1}
+	mreject := m
+	mconsumed := Message{
+		Received:  m.Received,
+		Size:      int64(len(msgPrefixCatchall)) + msgWriter.Size,
+		MsgPrefix: msgPrefixCatchall,
+	}
+	acc.WithWLock(func() {
+		conf, _ := acc.Conf()
+		err := acc.Deliver(xlog, conf.Destinations["mjl"], &m, msgFile, false)
+		tcheck(t, err, "deliver without consume")
+
+		err = acc.DB.Write(func(tx *bstore.Tx) error {
+			var err error
+			mbsent, err = bstore.QueryTx[Mailbox](tx).FilterNonzero(Mailbox{Name: "Sent"}).Get()
+			tcheck(t, err, "sent mailbox")
+			msent.MailboxID = mbsent.ID
+			msent.MailboxOrigID = mbsent.ID
+			acc.DeliverX(xlog, tx, &msent, msgFile, false, true, true, true)
+
+			err = tx.Insert(&mbrejects)
+			tcheck(t, err, "insert rejects mailbox")
+			mreject.MailboxID = mbrejects.ID
+			mreject.MailboxOrigID = mbrejects.ID
+			acc.DeliverX(xlog, tx, &mreject, msgFile, false, false, true, true)
+
+			return nil
+		})
+		tcheck(t, err, "deliver as sent and rejects")
+
+		err = acc.Deliver(xlog, conf.Destinations["mjl"], &mconsumed, msgFile, true)
+		tcheck(t, err, "deliver with consume")
+	})
+
+	m.Junk = true
+	err = acc.Train(log, []Message{m})
+	tcheck(t, err, "train as junk")
+
+	flags := m.Flags
+
+	m.Seen = true
+	m.Junk = false
+	jf, _, err := acc.OpenJunkFilter(log)
+	tcheck(t, err, "open junk filter")
+	err = acc.Retrain(log, jf, flags, m)
+	tcheck(t, err, "retrain as non-junk")
+	err = jf.Close()
+	tcheck(t, err, "close junk filter")
+
+	err = acc.Untrain(log, []Message{m})
+	tcheck(t, err, "untrain non-junk")
+
+	err = acc.SetPassword("testtest")
+	tcheck(t, err, "set password")
+
+	key0, err := acc.Subjectpass("test@localhost")
+	tcheck(t, err, "subjectpass")
+	key1, err := acc.Subjectpass("test@localhost")
+	tcheck(t, err, "subjectpass")
+	if key0 != key1 {
+		t.Fatalf("different keys for same address")
+	}
+	key2, err := acc.Subjectpass("test2@localhost")
+	tcheck(t, err, "subjectpass")
+	if key2 == key0 {
+		t.Fatalf("same key for different address")
+	}
+
+	acc.WithWLock(func() {
+		err := acc.DB.Write(func(tx *bstore.Tx) error {
+			acc.MailboxEnsureX(tx, "Testbox", true)
+			return nil
+		})
+		tcheck(t, err, "ensure mailbox exists")
+		err = acc.DB.Read(func(tx *bstore.Tx) error {
+			acc.MailboxEnsureX(tx, "Testbox", true)
+			return nil
+		})
+		tcheck(t, err, "ensure mailbox exists")
+
+		err = acc.DB.Write(func(tx *bstore.Tx) error {
+			acc.MailboxEnsureX(tx, "Testbox2", false)
+			tcheck(t, err, "create mailbox")
+
+			exists := acc.MailboxExistsX(tx, "Testbox2")
+			if !exists {
+				t.Fatalf("mailbox does not exist")
+			}
+
+			exists = acc.MailboxExistsX(tx, "Testbox3")
+			if exists {
+				t.Fatalf("mailbox does exist")
+			}
+
+			xmb := acc.MailboxFindX(tx, "Testbox3")
+			if xmb != nil {
+				t.Fatalf("did find Testbox3: %v", xmb)
+			}
+			xmb = acc.MailboxFindX(tx, "Testbox2")
+			if xmb == nil {
+				t.Fatalf("did not find Testbox2")
+			}
+
+			changes := acc.SubscriptionEnsureX(tx, "Testbox2")
+			if len(changes) == 0 {
+				t.Fatalf("new subscription did not result in changes")
+			}
+			changes = acc.SubscriptionEnsureX(tx, "Testbox2")
+			if len(changes) != 0 {
+				t.Fatalf("already present subscription resulted in changes")
+			}
+
+			return nil
+		})
+		tcheck(t, err, "write tx")
+
+		// todo: check that messages are removed and changes sent.
+		hasSpace, err := acc.TidyRejectsMailbox("Rejects")
+		tcheck(t, err, "tidy rejects mailbox")
+		if !hasSpace {
+			t.Fatalf("no space for more rejects")
+		}
+
+		acc.RejectsRemove(log, "Rejects", "m01@mox.example")
+	})
+
+	// Run the auth tests twice for possible cache effects.
+	for i := 0; i < 2; i++ {
+		_, err := OpenEmailAuth("mjl@mox.example", "bogus")
+		if err != ErrUnknownCredentials {
+			t.Fatalf("got %v, expected ErrUnknownCredentials", err)
+		}
+	}
+
+	for i := 0; i < 2; i++ {
+		acc2, err := OpenEmailAuth("mjl@mox.example", "testtest")
+		tcheck(t, err, "open for email with auth")
+		err = acc2.Close()
+		tcheck(t, err, "close account")
+	}
+
+	acc2, err := OpenEmailAuth("other@mox.example", "testtest")
+	tcheck(t, err, "open for email with auth")
+	err = acc2.Close()
+	tcheck(t, err, "close account")
+
+	_, err = OpenEmailAuth("bogus@mox.example", "testtest")
+	if err != ErrUnknownCredentials {
+		t.Fatalf("got %v, expected ErrUnknownCredentials", err)
+	}
+
+	_, err = OpenEmailAuth("mjl@test.example", "testtest")
+	if err != ErrUnknownCredentials {
+		t.Fatalf("got %v, expected ErrUnknownCredentials", err)
+	}
+}
+
+func TestWriteFile(t *testing.T) {
+	name := "../testdata/account.test"
+	os.Remove(name)
+	defer os.Remove(name)
+	err := writeFile(name, strings.NewReader("test"))
+	if err != nil {
+		t.Fatalf("writeFile, unexpected error %v", err)
+	}
+	buf, err := os.ReadFile(name)
+	if err != nil || string(buf) != "test" {
+		t.Fatalf("writeFile, read file, got err %v, data %q", err, buf)
+	}
+}
+
+func TestMessageRuleset(t *testing.T) {
+	f, err := os.Open("/dev/null")
+	tcheck(t, err, "open")
+	defer f.Close()
+	msgBuf := []byte(strings.ReplaceAll(`List-ID:  <test.mox.example>
+
+test
+`, "\n", "\r\n"))
+
+	const destConf = `
+Rulesets:
+	-
+		HeadersRegexp:
+			list-id: <test\.mox\.example>
+		Mailbox: test
+`
+	var dest config.Destination
+	err = sconf.Parse(strings.NewReader(destConf), &dest)
+	tcheck(t, err, "parse config")
+	// todo: should use regular config initialization functions for this.
+	var hdrs [][2]*regexp.Regexp
+	for k, v := range dest.Rulesets[0].HeadersRegexp {
+		rk, err := regexp.Compile(k)
+		tcheck(t, err, "compile key")
+		rv, err := regexp.Compile(v)
+		tcheck(t, err, "compile value")
+		hdrs = append(hdrs, [...]*regexp.Regexp{rk, rv})
+	}
+	dest.Rulesets[0].HeadersRegexpCompiled = hdrs
+
+	c := MessageRuleset(xlog, dest, &Message{}, msgBuf, f)
+	if c == nil {
+		t.Fatalf("expected ruleset match")
+	}
+
+	msg2Buf := []byte(strings.ReplaceAll(`From: <mjl@mox.example>
+
+test
+`, "\n", "\r\n"))
+	c = MessageRuleset(xlog, dest, &Message{}, msg2Buf, f)
+	if c != nil {
+		t.Fatalf("expected no ruleset match")
+	}
+
+	// todo: test the SMTPMailFrom and VerifiedDomains rule.
+}

store/msgreader.go

@@ -0,0 +1,135 @@
+package store
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+)
+
+// MsgReader provides access to a message. Reads return the "msg_prefix" in the
+// database (typically received headers), followed by the on-disk msg file
+// contents. MsgReader is an io.Reader, io.ReaderAt and io.Closer.
+type MsgReader struct {
+	prefix []byte   // First part of the message. Typically contains received headers.
+	path   string   // To on-disk message file.
+	size   int64    // Total size of message, including prefix and contents from path.
+	offset int64    // Current reading offset.
+	f      *os.File // Opened path, automatically opened after prefix has been read.
+	err    error    // If set, error to return for reads. Sets io.EOF for readers, but ReadAt ignores them.
+}
+
+var errMsgClosed = errors.New("msg is closed")
+
+// FileMsgReader makes a MsgReader for an open file.
+// If initialization fails, reads will return the error.
+// Only call close on the returned MsgReader if you want to close msgFile.
+func FileMsgReader(prefix []byte, msgFile *os.File) *MsgReader {
+	mr := &MsgReader{prefix: prefix, path: msgFile.Name(), f: msgFile}
+	fi, err := msgFile.Stat()
+	if err != nil {
+		mr.err = err
+		return mr
+	}
+	mr.size = int64(len(prefix)) + fi.Size()
+	return mr
+}
+
+// Read reads data from the msg, taking prefix and on-disk msg file into account.
+// The read offset is adjusted after the read.
+func (m *MsgReader) Read(buf []byte) (int, error) {
+	return m.read(buf, m.offset, false)
+}
+
+// ReadAt reads data from the msg, taking prefix and on-disk msg file into account.
+// The read offset is not affected by ReadAt.
+func (m *MsgReader) ReadAt(buf []byte, off int64) (n int, err error) {
+	return m.read(buf, off, true)
+}
+
+// read always fill buf as far as possible, for ReadAt semantics.
+func (m *MsgReader) read(buf []byte, off int64, pread bool) (int, error) {
+	// If a reader has consumed the file and reached EOF, further ReadAt must not return eof.
+	if m.err != nil && (!pread || m.err != io.EOF) {
+		return 0, m.err
+	}
+	var o int
+	for o < len(buf) {
+		// First attempt to read from m.prefix.
+		pn := int64(len(m.prefix)) - off
+		if pn > 0 {
+			n := len(buf)
+			if int64(n) > pn {
+				n = int(pn)
+			}
+			copy(buf[o:], m.prefix[int(off):int(off)+n])
+			o += n
+			off += int64(n)
+			if !pread {
+				m.offset += int64(n)
+			}
+			continue
+		}
+
+		// Now we need to read from file. Ensure it is open.
+		if m.f == nil {
+			f, err := os.Open(m.path)
+			if err != nil {
+				m.err = err
+				break
+			}
+			m.f = f
+		}
+		n, err := m.f.ReadAt(buf[o:], off-int64(len(m.prefix)))
+		if !pread && n > 0 {
+			m.offset += int64(n)
+		}
+		if !pread || err != io.EOF {
+			m.err = err
+		}
+		if n > 0 {
+			o += n
+			off += int64(n)
+		}
+		if err == io.EOF {
+			if off > m.size && (m.err == nil || m.err == io.EOF) {
+				err = fmt.Errorf("on-disk message larger than expected (off %d, size %d)", off, m.size)
+				m.err = err
+			}
+			return o, err
+		}
+		if n <= 0 {
+			break
+		}
+	}
+	if off > m.size && (m.err == nil || m.err == io.EOF) {
+		m.err = fmt.Errorf("on-disk message larger than expected (off %d, size %d, prefix %d)", off, m.size, len(m.prefix))
+	}
+	return o, m.err
+}
+
+// Close ensures the msg file is closed. Further reads will fail.
+func (m *MsgReader) Close() error {
+	if m.f != nil {
+		if err := m.f.Close(); err != nil {
+			return err
+		}
+		m.f = nil
+	}
+	if m.err == errMsgClosed {
+		return m.err
+	}
+	m.err = errMsgClosed
+	return nil
+}
+
+// Reset rewinds the offset and clears error conditions, making it usable as a fresh reader.
+func (m *MsgReader) Reset() {
+	m.offset = 0
+	m.err = nil
+}
+
+// Size returns the total size of the contents of the message.
+func (m *MsgReader) Size() int64 {
+	return m.size
+}

store/msgreader_test.go

@@ -0,0 +1,77 @@
+package store
+
+import (
+	"io"
+	"os"
+	"testing"
+)
+
+func TestMsgreader(t *testing.T) {
+	_, err := io.Copy(io.Discard, &MsgReader{prefix: []byte("hello"), path: "bogus.txt", size: int64(len("hello"))})
+	if err == nil || !os.IsNotExist(err) {
+		t.Fatalf("expected error for non-existing file, got %s", err)
+	}
+
+	if buf, err := io.ReadAll(&MsgReader{prefix: []byte("hello"), path: "/dev/null", size: int64(len("hello"))}); err != nil {
+		t.Fatalf("readall: %s", err)
+	} else if string(buf) != "hello" {
+		t.Fatalf("got %q, expected %q", buf, "hello")
+	}
+
+	if err := os.WriteFile("msgreader_test.txt", []byte(" world"), 0660); err != nil {
+		t.Fatalf("writing msgreader_test.txt: %s", err)
+	}
+	defer os.Remove("msgreader_test.txt")
+	mr := &MsgReader{prefix: []byte("hello"), path: "msgreader_test.txt", size: int64(len("hello world"))}
+	if buf, err := io.ReadAll(mr); err != nil {
+		t.Fatalf("readall: %s", err)
+	} else if string(buf) != "hello world" {
+		t.Fatalf("got %q, expected %q", buf, "hello world")
+	}
+
+	mr.Reset()
+	buf := make([]byte, 32)
+	if n, err := mr.ReadAt(buf, 1); err != nil && err != io.EOF {
+		t.Fatalf("readat: n %d, s %q, err %s", n, buf[:n], err)
+	} else if n != len("ello world") || string(buf[:n]) != "ello world" {
+		t.Fatalf("readat: got %d bytes (%q), expected %d (%q)", n, buf, int64(len("ello world")), "ello world")
+	}
+
+	// Read with 1 byte at a time to exercise the offset/buffer-length calculations.
+	buf = make([]byte, 1)
+	var result []byte
+	mr = &MsgReader{prefix: []byte("hello"), path: "msgreader_test.txt", size: int64(len("hello world"))}
+	for {
+		n, err := mr.Read(buf)
+		if n > 0 {
+			result = append(result, buf...)
+		}
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatalf("read: %s", err)
+		}
+	}
+	if string(result) != "hello world" {
+		t.Fatalf("got %q, want %q", result, "hello world")
+	}
+
+	if err := mr.Close(); err != nil {
+		t.Fatalf("close: %v", err)
+	}
+
+	f, err := os.Open("msgreader_test.txt")
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	mr = FileMsgReader([]byte("hello"), f)
+
+	if mr.Size() != int64(len("hello world")) {
+		t.Fatalf("size, got %d, expect %d", mr.Size(), len("hello world"))
+	}
+
+	if err := mr.Close(); err != nil {
+		t.Fatalf("close: %v", err)
+	}
+}

store/state.go

@@ -0,0 +1,157 @@
+package store
+
+import (
+	"sync/atomic"
+)
+
+var (
+	register   = make(chan *Comm)
+	unregister = make(chan *Comm)
+	broadcast  = make(chan changeReq)
+	get        = make(chan *Comm)
+)
+
+type changeReq struct {
+	comm    *Comm
+	changes []Change
+}
+
+type UID uint32 // IMAP UID.
+
+// Change to mailboxes/subscriptions/messages in an account. One of the Change*
+// types in this package.
+type Change any
+
+// ChangeAddUID is sent for a new message in a mailbox.
+type ChangeAddUID struct {
+	MailboxID int64
+	UID       UID
+	Flags     Flags
+}
+
+// ChangeRemoveUIDs is sent for removal of one or more messages from a mailbox.
+type ChangeRemoveUIDs struct {
+	MailboxID int64
+	UIDs      []UID
+}
+
+// ChangeFlags is sent for an update to flags for a message, e.g. "Seen".
+type ChangeFlags struct {
+	MailboxID int64
+	UID       UID
+	Mask      Flags // Which flags are actually modified.
+	Flags     Flags // New flag values. All are set, not just mask.
+}
+
+// ChangeRemoveMailbox is sent for a removed mailbox.
+type ChangeRemoveMailbox struct {
+	Name string
+}
+
+// ChangeAddMailbox is sent for a newly created mailbox.
+type ChangeAddMailbox struct {
+	Name  string
+	Flags []string
+}
+
+// ChangeRenameMailbox is sent for a rename mailbox.
+type ChangeRenameMailbox struct {
+	OldName string
+	NewName string
+	Flags   []string
+}
+
+// ChangeAddSubscription is sent for an added subscription to a mailbox.
+type ChangeAddSubscription struct {
+	Name string
+}
+
+var switchboardBusy atomic.Bool
+
+// Switchboard distributes changes to accounts to interested listeners. See Comm and Change.
+func Switchboard() chan struct{} {
+	regs := map[*Account]map[*Comm][]Change{}
+	done := make(chan struct{})
+
+	if !switchboardBusy.CompareAndSwap(false, true) {
+		panic("switchboard already busy")
+	}
+
+	go func() {
+		for {
+			select {
+			case c := <-register:
+				if _, ok := regs[c.acc]; !ok {
+					regs[c.acc] = map[*Comm][]Change{}
+				}
+				regs[c.acc][c] = nil
+			case c := <-unregister:
+				delete(regs[c.acc], c)
+				if len(regs[c.acc]) == 0 {
+					delete(regs, c.acc)
+				}
+			case chReq := <-broadcast:
+				acc := chReq.comm.acc
+				for c, changes := range regs[acc] {
+					// Do not send the broadcaster back their own changes.
+					if c == chReq.comm {
+						continue
+					}
+					regs[acc][c] = append(changes, chReq.changes...)
+					select {
+					case c.Changes <- regs[acc][c]:
+						regs[acc][c] = nil
+					default:
+					}
+				}
+				chReq.comm.r <- struct{}{}
+			case c := <-get:
+				c.Changes <- regs[c.acc][c]
+				regs[c.acc][c] = nil
+			case <-done:
+				if !switchboardBusy.CompareAndSwap(true, false) {
+					panic("switchboard already unregistered?")
+				}
+				return
+			}
+		}
+	}()
+	return done
+}
+
+// Comm handles communication with the goroutine that maintains the
+// account/mailbox/message state.
+type Comm struct {
+	Changes chan []Change // Receives block until changes come in, e.g. for IMAP IDLE.
+	acc     *Account
+	r       chan struct{}
+}
+
+// Register starts a Comm for the account. Unregister must be called.
+func RegisterComm(acc *Account) *Comm {
+	c := &Comm{make(chan []Change), acc, make(chan struct{})}
+	register <- c
+	return c
+}
+
+// Unregister stops this Comm.
+func (c *Comm) Unregister() {
+	unregister <- c
+}
+
+// Broadcast ensures changes are sent to other Comms.
+func (c *Comm) Broadcast(ch []Change) {
+	if len(ch) == 0 {
+		return
+	}
+	broadcast <- changeReq{c, ch}
+	<-c.r
+}
+
+// Get retrieves pending changes. If no changes are pending a nil or empty list
+// is returned.
+func (c *Comm) Get() []Change {
+	get <- c
+	changes := <-c.Changes
+	return changes
+}

store/tmp.go

@@ -0,0 +1,26 @@
+package store
+
+import (
+	"os"
+
+	"github.com/mjl-/mox/mox-"
+)
+
+// CreateMessageTemp creates a temporary file for a message to be delivered.
+// Caller is responsible for removing the temporary file on error, and for closing the file.
+// Caller should ensure the contents of the file are synced to disk before
+// attempting to deliver the message.
+func CreateMessageTemp(pattern string) (*os.File, error) {
+	dir := mox.DataDirPath("tmp")
+	os.MkdirAll(dir, 0770)
+	f, err := os.CreateTemp(dir, pattern)
+	if err != nil {
+		return nil, err
+	}
+	err = f.Chmod(0660)
+	if err != nil {
+		f.Close()
+		return nil, err
+	}
+	return f, err
+}

store/train.go

@@ -0,0 +1,136 @@
+package store
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+
+	"github.com/mjl-/mox/config"
+	"github.com/mjl-/mox/junk"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
+)
+
+// ErrNoJunkFilter indicates user did not configure/enable a junk filter.
+var ErrNoJunkFilter = errors.New("junkfilter: not configured")
+
+// OpenJunkFilter returns an opened junk filter for the account.
+// If the account does not have a junk filter enabled, ErrNotConfigured is returned.
+// Do not forget to save the filter after modifying, and to always close the filter when done.
+// An empty filter is initialized on first access of the filter.
+func (a *Account) OpenJunkFilter(log *mlog.Log) (*junk.Filter, *config.JunkFilter, error) {
+	conf, ok := mox.Conf.Account(a.Name)
+	if !ok {
+		return nil, nil, ErrAccountUnknown
+	}
+	jf := conf.JunkFilter
+	if jf == nil {
+		return nil, jf, ErrNoJunkFilter
+	}
+
+	basePath := mox.DataDirPath("accounts")
+	dbPath := filepath.Join(basePath, a.Name, "junkfilter.db")
+	bloomPath := filepath.Join(basePath, a.Name, "junkfilter.bloom")
+
+	if _, xerr := os.Stat(dbPath); xerr != nil && os.IsNotExist(xerr) {
+		f, err := junk.NewFilter(log, jf.Params, dbPath, bloomPath)
+		return f, jf, err
+	}
+	f, err := junk.OpenFilter(log, jf.Params, dbPath, bloomPath, false)
+	return f, jf, err
+}
+
+// Train new messages, if relevant given their flags.
+func (a *Account) Train(log *mlog.Log, msgs []Message) error {
+	return a.xtrain(log, msgs, false, true)
+}
+
+// Untrain removed messages, if relevant given their flags.
+func (a *Account) Untrain(log *mlog.Log, msgs []Message) error {
+	return a.xtrain(log, msgs, true, false)
+}
+
+// train or untrain messages, if relevant given their flags.
+func (a *Account) xtrain(log *mlog.Log, msgs []Message, untrain, train bool) (rerr error) {
+	if len(msgs) == 0 {
+		return nil
+	}
+
+	var jf *junk.Filter
+
+	for _, m := range msgs {
+		if !m.Seen && !m.Junk {
+			continue
+		}
+		// Lazy open the junk filter.
+		if jf == nil {
+			var err error
+			jf, _, err = a.OpenJunkFilter(log)
+			if err != nil && errors.Is(err, ErrNoJunkFilter) {
+				// No junk filter configured. Nothing more to do.
+				return nil
+			}
+			defer func() {
+				if jf != nil {
+					err := jf.Close()
+					if rerr == nil {
+						rerr = err
+					}
+				}
+			}()
+		}
+		ham := !m.Junk
+		err := xtrainMessage(log, a, jf, m, untrain, ham, train, ham)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Retrain message, if relevant given old flags and the new flags in m.
+func (a *Account) Retrain(log *mlog.Log, jf *junk.Filter, old Flags, m Message) error {
+	untrain := old.Seen || old.Junk
+	train := m.Seen || m.Junk
+	untrainHam := !old.Junk
+	trainHam := !m.Junk
+
+	if !untrain && !train || (untrain && train && trainHam == untrainHam) {
+		return nil
+	}
+
+	return xtrainMessage(log, a, jf, m, untrain, untrainHam, train, trainHam)
+}
+
+func xtrainMessage(log *mlog.Log, a *Account, jf *junk.Filter, m Message, untrain, untrainHam, train, trainHam bool) error {
+	log.Info("updating junk filter", mlog.Field("untrain", untrain), mlog.Field("untrainHam", untrainHam), mlog.Field("train", train), mlog.Field("trainHam", trainHam))
+
+	mr := a.MessageReader(m)
+	defer mr.Close()
+
+	p, err := m.LoadPart(mr)
+	if err != nil {
+		log.Errorx("loading part for message", err)
+		return nil
+	}
+
+	words, err := jf.ParseMessage(p)
+	if err != nil {
+		log.Errorx("parsing message for updating junk filter", err, mlog.Field("parse", ""))
+		return nil
+	}
+
+	if untrain {
+		err := jf.Untrain(untrainHam, words)
+		if err != nil {
+			return err
+		}
+	}
+	if train {
+		err := jf.Train(trainHam, words)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

store/transact.go

@@ -0,0 +1,24 @@
+package store
+
+import (
+	"github.com/mjl-/bstore"
+)
+
+// todo: get rid of this. it's a bad idea to indiscriminately turn all panics into an error.
+func extransact(db *bstore.DB, write bool, fn func(tx *bstore.Tx) error) (rerr error) {
+	defer func() {
+		x := recover()
+		if x == nil {
+			return
+		}
+		if err, ok := x.(error); ok {
+			rerr = err
+		} else {
+			panic(x)
+		}
+	}()
+	if write {
+		return db.Write(fn)
+	}
+	return db.Read(fn)
+}

store/validation.go

@@ -0,0 +1,24 @@
+package store
+
+import (
+	"github.com/mjl-/mox/spf"
+)
+
+var spfValidations = map[spf.Status]Validation{
+	spf.StatusNone:      ValidationNone,
+	spf.StatusNeutral:   ValidationNeutral,
+	spf.StatusPass:      ValidationPass,
+	spf.StatusFail:      ValidationFail,
+	spf.StatusSoftfail:  ValidationSoftfail,
+	spf.StatusTemperror: ValidationTemperror,
+	spf.StatusPermerror: ValidationPermerror,
+}
+
+// SPFValidation returns a Validation for an spf.Status.
+func SPFValidation(status spf.Status) Validation {
+	v, ok := spfValidations[status]
+	if !ok {
+		panic("missing spf status validation")
+	}
+	return v
+}