Improve expunged message/UID tracking in IMAP sessions, track synchronization history for mailboxes/annotations.

Keeping the message files around, and the message details in the database, is
useful for IMAP sessions that haven't seen/processed the removal of a message
yet and try to fetch it. Before, we would return errors. Similarly, a session
that has a mailbox selected that is removed can (at least in theory) still read
messages.

The mechanics to do this need keeping removed mailboxes around too. JMAP needs
that anyway, so we now keep modseq/createseq/expunged history for mailboxes
too. And while we're at it, for annotations as well.

For future JMAP support, we now also keep the mailbox parent id around for a
mailbox, with an upgrade step to set the field for existing mailboxes and
fixing up potential missing parents (which could possibly have happened in an
obscure corner case that I doubt anyone ran into).

smtpserver/fuzz_test.go

@@ -43,7 +43,7 @@ func FuzzServer(f *testing.F) {
 	}
 	defer func() {
 		acc.Close()
-		acc.CheckClosed()
+		acc.WaitClosed()
 	}()
 	err = acc.SetPassword(log, "testtest")
 	if err != nil {

smtpserver/rejects.go

@@ -46,14 +46,11 @@ func rejectPresent(log mlog.Log, acc *store.Account, rejectsMailbox string, m *s
 	var err error
 	acc.WithRLock(func() {
 		err = acc.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
-			mbq := bstore.QueryTx[store.Mailbox](tx)
-			mbq.FilterNonzero(store.Mailbox{Name: rejectsMailbox})
-			mb, err := mbq.Get()
-			if err == bstore.ErrAbsent {
-				return nil
-			}
+			mb, err := acc.MailboxFind(tx, rejectsMailbox)
 			if err != nil {
 				return fmt.Errorf("looking for rejects mailbox: %w", err)
+			} else if mb == nil {
+				return nil
 			}
 
 			q := bstore.QueryTx[store.Message](tx)

smtpserver/server.go

@@ -3272,6 +3272,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 					// See if we received a non-junk message from this organizational domain.
 					q := bstore.QueryTx[store.Message](tx)
 					q.FilterNonzero(store.Message{MsgFromOrgDomain: a0.d.m.MsgFromOrgDomain})
+					q.FilterEqual("Expunged", false)
 					q.FilterEqual("Notjunk", true)
 					q.FilterEqual("IsReject", false)
 					exists, err := q.Exists()
@@ -3406,25 +3407,37 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 				// due to reputation for later delivery attempts.
 				a.d.m.MessageHash = messagehash
 				a.d.acc.WithWLock(func() {
-					hasSpace := true
-					var err error
-					if !conf.KeepRejects {
-						hasSpace, err = a.d.acc.TidyRejectsMailbox(c.log, conf.RejectsMailbox)
-					}
-					if err != nil {
-						log.Errorx("tidying rejects mailbox", err)
-					} else if !hasSpace {
-						log.Info("not storing spammy mail to full rejects mailbox")
-						return
-					}
-
 					var changes []store.Change
 					var stored bool
-					err = a.d.acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
+
+					var newID int64
+					defer func() {
+						if newID != 0 {
+							p := a.d.acc.MessagePath(newID)
+							err := os.Remove(p)
+							c.log.Check(err, "remove message after error delivering to rejects", slog.String("path", p))
+						}
+					}()
+
+					err := a.d.acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
 						mbrej, err := a.d.acc.MailboxFind(tx, conf.RejectsMailbox)
 						if err != nil {
 							return fmt.Errorf("finding rejects mailbox: %v", err)
 						}
+
+						hasSpace := true
+						if !conf.KeepRejects && mbrej != nil {
+							var chl []store.Change
+							chl, hasSpace, err = a.d.acc.TidyRejectsMailbox(c.log, tx, mbrej)
+							if err != nil {
+								return fmt.Errorf("tidying rejects mailbox: %v", err)
+							}
+							if !hasSpace {
+								log.Info("not storing spammy mail to full rejects mailbox")
+								return nil
+							}
+							changes = append(changes, chl...)
+						}
 						if mbrej == nil {
 							nmb, chl, _, _, err := a.d.acc.MailboxCreate(tx, conf.RejectsMailbox, store.SpecialUse{})
 							if err != nil {
@@ -3438,6 +3451,8 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 						if err := a.d.acc.MessageAdd(log, tx, mbrej, a.d.m, dataFile, store.AddOpts{}); err != nil {
 							return fmt.Errorf("delivering spammy mail to rejects mailbox: %v", err)
 						}
+						newID = a.d.m.ID
+
 						if err := tx.Update(mbrej); err != nil {
 							return fmt.Errorf("updating rejects mailbox: %v", err)
 						}
@@ -3451,6 +3466,8 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 					} else if stored {
 						log.Info("stored spammy mail in rejects mailbox")
 					}
+					newID = 0
+
 					store.BroadcastChanges(a.d.acc, changes)
 				})
 			}

smtpserver/server_test.go

@@ -184,7 +184,7 @@ func (ts *testserver) close() {
 	ts.switchStop()
 	err = ts.acc.Close()
 	tcheck(ts.t, err, "closing account")
-	ts.acc.CheckClosed()
+	ts.acc.WaitClosed()
 	ts.acc = nil
 }
 
@@ -193,6 +193,7 @@ func (ts *testserver) checkCount(mailboxName string, expect int) {
 	t.Helper()
 	q := bstore.QueryDB[store.Mailbox](ctxbg, ts.acc.DB)
 	q.FilterNonzero(store.Mailbox{Name: mailboxName})
+	q.FilterEqual("Expunged", false)
 	mb, err := q.Get()
 	tcheck(t, err, "get mailbox")
 	qm := bstore.QueryDB[store.Message](ctxbg, ts.acc.DB)