Improve expunged message/UID tracking in IMAP sessions, track synchronization history for mailboxes/annotations.

Keeping the message files around, and the message details in the database, is
useful for IMAP sessions that haven't seen/processed the removal of a message
yet and try to fetch it. Before, we would return errors. Similarly, a session
that has a mailbox selected that is removed can (at least in theory) still read
messages.

The mechanics to do this need keeping removed mailboxes around too. JMAP needs
that anyway, so we now keep modseq/createseq/expunged history for mailboxes
too. And while we're at it, for annotations as well.

For future JMAP support, we now also keep the mailbox parent id around for a
mailbox, with an upgrade step to set the field for existing mailboxes and
fixing up potential missing parents (which could possibly have happened in an
obscure corner case that I doubt anyone ran into).

backup.go

@@ -532,8 +532,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
 			}
 		}()
 
-		// Link/copy known message files. If a message has been removed while we read the
+		// Link/copy known message files.
-		// database, our backup is not consistent and the backup will be marked failed.
 		tmMsgs := time.Now()
 		seen := map[string]struct{}{}
 		var maxID int64
@@ -565,6 +564,15 @@ func backupctl(ctx context.Context, ctl *ctl) {
 				slog.Duration("duration", time.Since(tmMsgs)))
 		}
 
+		eraseIDs := map[int64]struct{}{}
+		err = bstore.QueryDB[store.MessageErase](ctx, db).ForEach(func(me store.MessageErase) error {
+			eraseIDs[me.ID] = struct{}{}
+			return nil
+		})
+		if err != nil {
+			xerrx("listing erased messages", err)
+		}
+
 		// Read through all files in queue directory and warn about anything we haven't
 		// handled yet. Message files that are newer than we expect from our consistent
 		// database snapshot are ignored.
@@ -586,9 +594,13 @@ func backupctl(ctx context.Context, ctl *ctl) {
 					return nil
 				}
 
-				// Skip any messages that were added since we started on our consistent snapshot.
+				// Skip any messages that were added since we started on our consistent snapshot,
-				// We don't want to cause spurious backup warnings.
+				// or messages that will be erased. We don't want to cause spurious backup
-				if id, err := strconv.ParseInt(l[len(l)-1], 10, 64); err == nil && id > maxID && mp == store.MessagePath(id) {
+				// warnings.
+				id, err := strconv.ParseInt(l[len(l)-1], 10, 64)
+				if err == nil && id > maxID && mp == store.MessagePath(id) {
+					return nil
+				} else if _, ok := eraseIDs[id]; err == nil && ok {
 					return nil
 				}
 			}

config/config.go

@@ -65,7 +65,7 @@ type Static struct {
 
 		ParsedLocalpart smtp.Localpart `sconf:"-"`
 	} `sconf:"optional" sconf-doc:"Destination for per-host TLS reports (TLSRPT). TLS reports can be per recipient domain (for MTA-STS), or per MX host (for DANE). The per-domain TLS reporting configuration is in domains.conf. This is the TLS reporting configuration for this host. If absent, no host-based TLSRPT address is configured, and no host TLSRPT DNS record is suggested."`
-	InitialMailboxes InitialMailboxes     `sconf:"optional" sconf-doc:"Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be given a 'special-use' role, which are understood by most mail clients. If absent/empty, the following mailboxes are created: Sent, Archive, Trash, Drafts and Junk."`
+	InitialMailboxes InitialMailboxes     `sconf:"optional" sconf-doc:"Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be given a 'special-use' role, which are understood by most mail clients. If absent/empty, the following additional mailboxes are created: Sent, Archive, Trash, Drafts and Junk."`
 	DefaultMailboxes []string             `sconf:"optional" sconf-doc:"Deprecated in favor of InitialMailboxes. Mailboxes to create when adding an account. Inbox is always created. If no mailboxes are specified, the following are automatically created: Sent, Archive, Trash, Drafts and Junk."`
 	Transports       map[string]Transport `sconf:"optional" sconf-doc:"Transport are mechanisms for delivering messages. Transports can be referenced from Routes in accounts, domains and the global configuration. There is always an implicit/fallback delivery transport doing direct delivery with SMTP from the outgoing message queue. Transports are typically only configured when using smarthosts, i.e. when delivering through another SMTP server. Zero or one transport methods must be set in a transport, never multiple. When using an external party to send email for a domain, keep in mind you may have to add their IP address to your domain's SPF record, and possibly additional DKIM records."`
 	// Awkward naming of fields to get intended default behaviour for zero values.

config/doc.go

@@ -543,8 +543,8 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 
 	# Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be
 	# given a 'special-use' role, which are understood by most mail clients. If
-	# absent/empty, the following mailboxes are created: Sent, Archive, Trash, Drafts
+	# absent/empty, the following additional mailboxes are created: Sent, Archive,
-	# and Junk. (optional)
+	# Trash, Drafts and Junk. (optional)
 	InitialMailboxes:
 
 		# Special-use roles to mailbox to create. (optional)

ctl.go

@@ -1454,17 +1454,28 @@ func servectlcmd(ctx context.Context, ctl *ctl, cid int64, shutdown func()) {
 					log.Check(err, "closing junk filter during cleanup")
 				}()
 
-				// Read through messages with junk or nonjunk flag set, and train them.
+				// Read through messages with either junk or nonjunk flag set, and train them.
 				var total, trained int
-				q := bstore.QueryDB[store.Message](ctx, acc.DB)
+				err = acc.DB.Write(ctx, func(tx *bstore.Tx) error {
-				q.FilterEqual("Expunged", false)
+					q := bstore.QueryTx[store.Message](tx)
-				err = q.ForEach(func(m store.Message) error {
+					q.FilterEqual("Expunged", false)
-					total++
+					return q.ForEach(func(m store.Message) error {
-					ok, err := acc.TrainMessage(ctx, log, jf, m)
+						total++
-					if ok {
+						if m.Junk == m.Notjunk {
-						trained++
+							return nil
-					}
+						}
-					return err
+						ok, err := acc.TrainMessage(ctx, log, jf, m.Notjunk, m)
+						if ok {
+							trained++
+						}
+						if m.TrainedJunk == nil || *m.TrainedJunk != m.Junk {
+							m.TrainedJunk = &m.Junk
+							if err := tx.Update(&m); err != nil {
+								return fmt.Errorf("marking message as trained: %v", err)
+							}
+						}
+						return err
+					})
 				})
 				ctl.xcheck(err, "training messages")
 				log.Info("retrained messages", slog.Int("total", total), slog.Int("trained", trained))
@@ -1509,7 +1520,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, cid int64, shutdown func()) {
 			var changes []store.Change
 			err = acc.DB.Write(ctx, func(tx *bstore.Tx) error {
 				var totalSize int64
-				err := bstore.QueryTx[store.Mailbox](tx).ForEach(func(mb store.Mailbox) error {
+				err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb store.Mailbox) error {
 					mc, err := mb.CalculateCounts(tx)
 					if err != nil {
 						return fmt.Errorf("calculating counts for mailbox %q: %w", mb.Name, err)
@@ -1618,6 +1629,11 @@ func servectlcmd(ctx context.Context, ctl *ctl, cid int64, shutdown func()) {
 							if err := tx.Get(&mb); err != nil {
 								_, werr := fmt.Fprintf(w, "get mailbox id %d for message with file size mismatch: %v\n", mb.ID, err)
 								ctl.xcheck(werr, "write")
+								return nil
+							}
+							if mb.Expunged {
+								_, err := fmt.Fprintf(w, "message %d is in expunged mailbox %q (id %d) (continuing)\n", m.ID, mb.Name, mb.ID)
+								ctl.xcheck(err, "write")
 							}
 							_, err = fmt.Fprintf(w, "fixing message %d in mailbox %q (id %d) with incorrect size %d, should be %d (len msg prefix %d + on-disk file %s size %d)\n", m.ID, mb.Name, mb.ID, m.Size, correctSize, len(m.MsgPrefix), p, filesize)
 							ctl.xcheck(err, "write")
@@ -1650,7 +1666,6 @@ func servectlcmd(ctx context.Context, ctl *ctl, cid int64, shutdown func()) {
 							}
 							return nil
 						})
-
 					})
 					ctl.xcheck(err, "find and fix wrong message sizes")
 

ctl_test.go

@@ -436,7 +436,7 @@ func TestCtl(t *testing.T) {
 		tcheck(t, err, "open account")
 		defer func() {
 			acc.Close()
-			acc.CheckClosed()
+			acc.WaitClosed()
 		}()
 
 		content := []byte("Subject: hi\r\n\r\nbody\r\n")

imapserver/append_test.go

@@ -13,10 +13,10 @@ func TestAppend(t *testing.T) {
 	defer tc.close()
 
 	tc2 := startNoSwitchboard(t) // note: without switchboard because this connection will break during tests.
-	defer tc2.close()
+	defer tc2.closeNoWait()
 
 	tc3 := startNoSwitchboard(t)
-	defer tc3.close()
+	defer tc3.closeNoWait()
 
 	tc2.client.Login("mjl@mox.example", password0)
 	tc2.client.Select("inbox")
@@ -31,19 +31,22 @@ func TestAppend(t *testing.T) {
 	// Syntax error for line ending in literal causes connection abort.
 	tc2.transactf("bad", "append inbox (\\Badflag) {1+}\r\nx") // Unknown flag.
 	tc2 = startNoSwitchboard(t)
-	defer tc2.close()
+	defer tc2.closeNoWait()
 	tc2.client.Login("mjl@mox.example", password0)
 	tc2.client.Select("inbox")
 
 	tc2.transactf("bad", "append inbox () \"bad time\" {1+}\r\nx") // Bad time.
 	tc2 = startNoSwitchboard(t)
-	defer tc2.close()
+	defer tc2.closeNoWait()
 	tc2.client.Login("mjl@mox.example", password0)
 	tc2.client.Select("inbox")
 
 	tc2.transactf("no", "append nobox (\\Seen) \" 1-Jan-2022 10:10:00 +0100\" {1}")
 	tc2.xcode("TRYCREATE")
 
+	tc2.transactf("no", "append expungebox (\\Seen) {1}")
+	tc2.xcode("TRYCREATE")
+
 	tc2.transactf("ok", "append inbox (\\Seen Label1 $label2) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
 	tc2.xuntagged(imapclient.UntaggedExists(1))
 	tc2.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UIDs: xparseUIDRange("1")})

imapserver/condstore_test.go

@@ -85,6 +85,8 @@ func testCondstoreQresync(t *testing.T, qresync bool) {
 	// unmodified. Those messages have modseq 0 in the database. We use append for
 	// convenience, then adjust the records in the database.
 	// We have a workaround below to prevent triggering the consistency checker.
+	tc.account.SetSkipMessageModSeqZeroCheck(true)
+	defer tc.account.SetSkipMessageModSeqZeroCheck(false)
 	tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
 	tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
 	tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
@@ -100,13 +102,13 @@ func testCondstoreQresync(t *testing.T, qresync bool) {
 
 	// tc2 is a client without condstore, so no modseq responses.
 	tc2 := startNoSwitchboard(t)
-	defer tc2.close()
+	defer tc2.closeNoWait()
 	tc2.client.Login("mjl@mox.example", password0)
 	tc2.client.Select("inbox")
 
 	// tc3 is a client with condstore, so with modseq responses.
 	tc3 := startNoSwitchboard(t)
-	defer tc3.close()
+	defer tc3.closeNoWait()
 	tc3.client.Login("mjl@mox.example", password0)
 	tc3.client.Enable(capability)
 	tc3.client.Select("inbox")
@@ -124,7 +126,7 @@ func testCondstoreQresync(t *testing.T, qresync bool) {
 
 	tc.transactf("ok", "Append otherbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
 	tc.xuntagged()
-	tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 2, UIDs: xparseUIDRange("1")})
+	tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 3, UIDs: xparseUIDRange("1")})
 
 	tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
 	tc.xuntagged(imapclient.UntaggedExists(5))
@@ -353,11 +355,7 @@ func testCondstoreQresync(t *testing.T, qresync bool) {
 		xtc := startNoSwitchboard(t)
 		// We have modified modseq & createseq to 0 above for testing that case. Don't
 		// trigger the consistency checker.
-		store.CheckConsistencyOnClose = false
+		defer xtc.closeNoWait()
-		defer func() {
-			xtc.close()
-			store.CheckConsistencyOnClose = true
-		}()
 		xtc.client.Login("mjl@mox.example", password0)
 		fn(xtc)
 		tagcount++
@@ -444,13 +442,13 @@ func testCondstoreQresync(t *testing.T, qresync bool) {
 	// and untagged fetch with modseq in destination mailbox.
 	// tc2o is a client without condstore, so no modseq responses.
 	tc2o := startNoSwitchboard(t)
-	defer tc2o.close()
+	defer tc2o.closeNoWait()
 	tc2o.client.Login("mjl@mox.example", password0)
 	tc2o.client.Select("otherbox")
 
 	// tc3o is a client with condstore, so with modseq responses.
 	tc3o := startNoSwitchboard(t)
-	defer tc3o.close()
+	defer tc3o.closeNoWait()
 	tc3o.client.Login("mjl@mox.example", password0)
 	tc3o.client.Enable(capability)
 	tc3o.client.Select("otherbox")
@@ -484,14 +482,9 @@ func testCondstoreQresync(t *testing.T, qresync bool) {
 		imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), noflags, imapclient.FetchModSeq(clientModseq)}},
 	)
 
-	// Restore valid modseq/createseq for the consistency checker.
+	tc2o.closeNoWait()
-	_, err = bstore.QueryDB[store.Message](ctxbg, tc.account.DB).FilterEqual("CreateSeq", int64(0)).UpdateNonzero(store.Message{CreateSeq: 2})
-	tcheck(t, err, "updating modseq/createseq to valid values")
-	_, err = bstore.QueryDB[store.Message](ctxbg, tc.account.DB).FilterEqual("ModSeq", int64(0)).UpdateNonzero(store.Message{ModSeq: 2})
-	tcheck(t, err, "updating modseq/createseq to valid values")
-	tc2o.close()
 	tc2o = nil
-	tc3o.close()
+	tc3o.closeNoWait()
 	tc3o = nil
 
 	// Then we rename inbox, which is special because it moves messages away instead of
@@ -533,10 +526,7 @@ func testQresync(t *testing.T, tc *testconn, clientModseq int64) {
 	xtc.client.Login("mjl@mox.example", password0)
 	xtc.transactf("ok", "Select inbox (Condstore)")
 	xtc.transactf("bad", "Uid Fetch 1:* (Flags) (Changedsince 1 Vanished)")
-	// Prevent triggering the consistency checker, we still have modseq/createseq at 0.
+	xtc.closeNoWait()
-	store.CheckConsistencyOnClose = false
-	xtc.close()
-	store.CheckConsistencyOnClose = true
 	xtc = nil
 
 	// Check that we get proper vanished responses.
@@ -557,9 +547,7 @@ func testQresync(t *testing.T, tc *testconn, clientModseq int64) {
 	xtc.client.Login("mjl@mox.example", password0)
 	xtc.transactf("bad", "Select inbox (Qresync 1 0)")
 	// Prevent triggering the consistency checker, we still have modseq/createseq at 0.
-	store.CheckConsistencyOnClose = false
+	xtc.closeNoWait()
-	xtc.close()
-	store.CheckConsistencyOnClose = true
 	xtc = nil
 
 	tc.transactf("bad", "Select inbox (Qresync (0 1))")               // Both args must be > 0.

imapserver/copy_test.go

@@ -12,7 +12,7 @@ func TestCopy(t *testing.T) {
 	defer tc.close()
 
 	tc2 := startNoSwitchboard(t)
-	defer tc2.close()
+	defer tc2.closeNoWait()
 
 	tc.client.Login("mjl@mox.example", password0)
 	tc.client.Select("inbox")
@@ -34,6 +34,8 @@ func TestCopy(t *testing.T) {
 
 	tc.transactf("no", "copy 1 nonexistent")
 	tc.xcode("TRYCREATE")
+	tc.transactf("no", "copy 1 expungebox")
+	tc.xcode("TRYCREATE")
 
 	tc.transactf("no", "copy 1 inbox") // Cannot copy to same mailbox.
 

imapserver/create_test.go

@@ -11,7 +11,7 @@ func TestCreate(t *testing.T) {
 	defer tc.close()
 
 	tc2 := startNoSwitchboard(t)
-	defer tc2.close()
+	defer tc2.closeNoWait()
 
 	tc.client.Login("mjl@mox.example", password0)
 	tc2.client.Login("mjl@mox.example", password0)
@@ -84,4 +84,7 @@ func TestCreate(t *testing.T) {
 	tc.transactf("ok", `create "&"`)      // Interpreted as UTF-8, no UTF-7.
 	tc2.transactf("bad", `create "&"`)    // Bad UTF-7.
 	tc2.transactf("ok", `create "&Jjo-"`) // ☺, valid UTF-7.
+
+	tc.transactf("ok", "create expungebox") // Existed in past.
+	tc.transactf("ok", "delete expungebox") // Gone again.
 }

imapserver/delete_test.go

@@ -11,10 +11,10 @@ func TestDelete(t *testing.T) {
 	defer tc.close()
 
 	tc2 := startNoSwitchboard(t)
-	defer tc2.close()
+	defer tc2.closeNoWait()
 
 	tc3 := startNoSwitchboard(t)
-	defer tc3.close()
+	defer tc3.closeNoWait()
 
 	tc.client.Login("mjl@mox.example", password0)
 	tc2.client.Login("mjl@mox.example", password0)
@@ -24,6 +24,7 @@ func TestDelete(t *testing.T) {
 	tc.transactf("no", "delete inbox")         // Cannot delete inbox.
 	tc.transactf("no", "delete nonexistent")   // Cannot delete mailbox that does not exist.
 	tc.transactf("no", `delete "nonexistent"`) // Again, with quoted string syntax.
+	tc.transactf("no", `delete "expungebox"`)  // Already removed.
 
 	tc.client.Subscribe("x")
 	tc.transactf("no", "delete x") // Subscription does not mean there is a mailbox that can be deleted.

imapserver/expunge_test.go

@@ -12,7 +12,7 @@ func TestExpunge(t *testing.T) {
 	defer tc.close()
 
 	tc2 := startNoSwitchboard(t)
-	defer tc2.close()
+	defer tc2.closeNoWait()
 
 	tc.client.Login("mjl@mox.example", password0)
 	tc.client.Select("inbox")

imapserver/fetch.go

@@ -5,7 +5,6 @@ package imapserver
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"log/slog"
@@ -26,11 +25,11 @@ import (
 // functions to handle fetch attribute requests are defined on fetchCmd.
 type fetchCmd struct {
 	conn            *conn
-	isUID           bool       // If this is a UID FETCH command.
+	isUID           bool        // If this is a UID FETCH command.
-	rtx             *bstore.Tx // Read-only transaction, kept open while processing all messages.
+	rtx             *bstore.Tx  // Read-only transaction, kept open while processing all messages.
-	updateSeen      []int64    // IDs of messages to mark as seen, after processing all messages.
+	updateSeen      []store.UID // To mark as seen after processing all messages. UID instead of message ID since moved messages keep their ID and insert a new ID in the original mailbox.
-	hasChangedSince bool       // Whether CHANGEDSINCE was set. Enables MODSEQ in response.
+	hasChangedSince bool        // Whether CHANGEDSINCE was set. Enables MODSEQ in response.
-	expungeIssued   bool       // Set if any message cannot be read. Can happen for expunged messages.
+	expungeIssued   bool        // Set if any message has been expunged. Can happen for expunged messages.
 
 	uid        store.UID // UID currently processing.
 	markSeen   bool
@@ -271,15 +270,16 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
 			changes := make([]store.Change, 0, len(cmd.updateSeen)+1)
 
 			c.xdbwrite(func(wtx *bstore.Tx) {
-				mb := store.Mailbox{ID: c.mailboxID}
+				mb, err := store.MailboxID(wtx, c.mailboxID)
-				err = wtx.Get(&mb)
+				if err == store.ErrMailboxExpunged {
+					xusercodeErrorf("NONEXISTENT", "mailbox has been expunged")
+				}
 				xcheckf(err, "get mailbox for updating counts after marking as seen")
 
 				var modseq store.ModSeq
 
-				for _, id := range cmd.updateSeen {
+				for _, uid := range cmd.updateSeen {
-					m := store.Message{ID: id}
+					m, err := bstore.QueryTx[store.Message](wtx).FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: uid}).Get()
-					err := wtx.Get(&m)
 					xcheckf(err, "get message")
 					if m.Expunged {
 						// Message has been deleted in the mean time.
@@ -322,7 +322,8 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
 
 	if cmd.expungeIssued {
 		// ../rfc/2180:343
-		c.writeresultf("%s NO [EXPUNGEISSUED] at least one message was expunged", tag)
+		// ../rfc/9051:5102
+		c.writeresultf("%s OK [EXPUNGEISSUED] at least one message was expunged", tag)
 	} else {
 		c.ok(tag, cmdstr)
 	}
@@ -333,12 +334,16 @@ func (cmd *fetchCmd) xensureMessage() *store.Message {
 		return cmd.m
 	}
 
+	// We do not filter by Expunged, the message may have been deleted in other
+	// sessions, but not in ours.
 	q := bstore.QueryTx[store.Message](cmd.rtx)
 	q.FilterNonzero(store.Message{MailboxID: cmd.conn.mailboxID, UID: cmd.uid})
-	q.FilterEqual("Expunged", false)
 	m, err := q.Get()
 	cmd.xcheckf(err, "get message for uid %d", cmd.uid)
 	cmd.m = &m
+	if m.Expunged {
+		cmd.expungeIssued = true
+	}
 	return cmd.m
 }
 
@@ -382,10 +387,6 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
 		if !ok {
 			panic(x)
 		}
-		if errors.Is(err, bstore.ErrAbsent) {
-			cmd.expungeIssued = true
-			return
-		}
 		cmd.conn.log.Infox("processing fetch attribute", err, slog.Any("uid", cmd.uid))
 		xuserErrorf("processing fetch attribute: %v", err)
 	}()
@@ -401,8 +402,7 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
 	}
 
 	if cmd.markSeen {
-		m := cmd.xensureMessage()
+		cmd.updateSeen = append(cmd.updateSeen, cmd.uid)
-		cmd.updateSeen = append(cmd.updateSeen, m.ID)
 	}
 
 	if cmd.needFlags {

imapserver/fetch_test.go

@@ -444,6 +444,16 @@ Content-Transfer-Encoding: Quoted-printable
 	tc.client.Unselect()
 	tc.client.Examine("inbox")
 
+	// Start a second session. Use it to remove the message. First session should still
+	// be able to access the messages.
+	tc2 := startNoSwitchboard(t)
+	defer tc2.closeNoWait()
+	tc2.client.Login("mjl@mox.example", password0)
+	tc2.client.Select("inbox")
+	tc2.client.StoreFlagsSet("1", true, `\Deleted`)
+	tc2.client.Expunge()
+	tc2.client.Logout()
+
 	tc.transactf("ok", "fetch 1 binary[]")
 	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binary1}})
 

imapserver/fuzz_test.go

@@ -70,7 +70,7 @@ func FuzzServer(f *testing.F) {
 	}
 	defer func() {
 		acc.Close()
-		acc.CheckClosed()
+		acc.WaitClosed()
 	}()
 	err = acc.SetPassword(log, password0)
 	if err != nil {

imapserver/idle_test.go

@@ -13,7 +13,7 @@ func TestIdle(t *testing.T) {
 	defer tc1.close()
 
 	tc2 := startNoSwitchboard(t)
-	defer tc2.close()
+	defer tc2.closeNoWait()
 
 	tc1.client.Login("mjl@mox.example", password0)
 	tc2.client.Login("mjl@mox.example", password0)

imapserver/list.go

@@ -3,11 +3,12 @@ package imapserver
 import (
 	"bytes"
 	"fmt"
-	"path"
 	"sort"
 	"strings"
 
 	"github.com/mjl-/bstore"
+
+	"github.com/mjl-/mox/mox-"
 	"github.com/mjl-/mox/store"
 )
 
@@ -145,10 +146,11 @@ func (c *conn) cmdList(tag, cmd string, p *parser) {
 			var nameList []string
 
 			q := bstore.QueryTx[store.Mailbox](tx)
+			q.FilterEqual("Expunged", false)
 			err := q.ForEach(func(mb store.Mailbox) error {
 				names[mb.Name] = info{mailbox: &mb}
 				nameList = append(nameList, mb.Name)
-				for p := path.Dir(mb.Name); p != "."; p = path.Dir(p) {
+				for p := mox.ParentMailboxName(mb.Name); p != ""; p = mox.ParentMailboxName(p) {
 					hasChild[p] = true
 				}
 				return nil
@@ -163,7 +165,7 @@ func (c *conn) cmdList(tag, cmd string, p *parser) {
 				if !ok {
 					nameList = append(nameList, sub.Name)
 				}
-				for p := path.Dir(sub.Name); p != "."; p = path.Dir(p) {
+				for p := mox.ParentMailboxName(sub.Name); p != ""; p = mox.ParentMailboxName(p) {
 					hasSubscribedChild[p] = true
 				}
 				return nil
@@ -234,7 +236,10 @@ func (c *conn) cmdList(tag, cmd string, p *parser) {
 				if info.mailbox != nil && len(retMetadata) > 0 {
 					var meta listspace
 					for _, k := range retMetadata {
-						a, err := bstore.QueryTx[store.Annotation](tx).FilterNonzero(store.Annotation{MailboxID: info.mailbox.ID, Key: k}).Get()
+						q := bstore.QueryTx[store.Annotation](tx)
+						q.FilterNonzero(store.Annotation{MailboxID: info.mailbox.ID, Key: k})
+						q.FilterEqual("Expunged", false)
+						a, err := q.Get()
 						var v token
 						if err == bstore.ErrAbsent {
 							v = nilt

imapserver/list_test.go

@@ -26,6 +26,9 @@ func TestListBasic(t *testing.T) {
 	tc.last(tc.client.List("Inbox"))
 	tc.xuntagged(ulist("Inbox"))
 
+	tc.last(tc.client.List("expungebox"))
+	tc.xuntagged()
+
 	tc.last(tc.client.List("%"))
 	tc.xuntagged(ulist("Archive", `\Archive`), ulist("Drafts", `\Drafts`), ulist("Inbox"), ulist("Junk", `\Junk`), ulist("Sent", `\Sent`), ulist("Trash", `\Trash`))
 
@@ -78,7 +81,7 @@ func TestListExtended(t *testing.T) {
 	for _, name := range store.DefaultInitialMailboxes.Regular {
 		uidvals[name] = 1
 	}
-	var uidvalnext uint32 = 2
+	var uidvalnext uint32 = 3
 	uidval := func(name string) uint32 {
 		v, ok := uidvals[name]
 		if !ok {

imapserver/lsub_test.go

@@ -19,6 +19,9 @@ func TestLsub(t *testing.T) {
 	tc.transactf("ok", `lsub "" x*`)
 	tc.xuntagged()
 
+	tc.transactf("ok", `lsub "" expungebox`)
+	tc.xuntagged(imapclient.UntaggedLsub{Separator: '/', Mailbox: "expungebox"})
+
 	tc.transactf("ok", "create a/b/c")
 	tc.transactf("ok", `lsub "" a/*`)
 	tc.xuntagged(imapclient.UntaggedLsub{Separator: '/', Mailbox: "a/b"}, imapclient.UntaggedLsub{Separator: '/', Mailbox: "a/b/c"})

imapserver/metadata.go

@@ -100,7 +100,7 @@ func (c *conn) cmdGetmetadata(tag, cmd string, p *parser) {
 				mb := c.xmailbox(tx, mailboxName, "TRYCREATE")
 				q.FilterNonzero(store.Annotation{MailboxID: mb.ID})
 			}
-
+			q.FilterEqual("Expunged", false)
 			q.SortAsc("MailboxID", "Key") // For tests.
 			err := q.ForEach(func(a store.Annotation) error {
 				// ../rfc/5464:516
@@ -246,41 +246,35 @@ func (c *conn) cmdSetmetadata(tag, cmd string, p *parser) {
 				q := bstore.QueryTx[store.Annotation](tx)
 				q.FilterNonzero(store.Annotation{Key: a.Key})
 				q.FilterEqual("MailboxID", mb.ID) // Can be zero.
-
+				q.FilterEqual("Expunged", false)
+				oa, err := q.Get()
 				// Nil means remove. ../rfc/5464:579
-				if a.Value == nil {
+				if err == bstore.ErrAbsent && a.Value == nil {
-					var deleted []store.Annotation
-					q.Gather(&deleted)
-					_, err := q.Delete()
-					xcheckf(err, "deleting annotation")
-					for _, oa := range deleted {
-						changes = append(changes, oa.Change(mailboxName))
-					}
 					continue
 				}
-
 				if modseq == 0 {
 					var err error
 					modseq, err = c.account.NextModSeq(tx)
 					xcheckf(err, "get next modseq")
 				}
-
-				a.MailboxID = mb.ID
-				a.ModSeq = modseq
-				a.CreateSeq = modseq
-
-				oa, err := q.Get()
 				if err == bstore.ErrAbsent {
+					a.MailboxID = mb.ID
+					a.CreateSeq = modseq
+					a.ModSeq = modseq
 					err = tx.Insert(&a)
 					xcheckf(err, "inserting annotation")
-					changes = append(changes, a.Change(mailboxName))
+				} else {
-					continue
+					xcheckf(err, "get metadata")
+					oa.ModSeq = modseq
+					if a.Value == nil {
+						oa.Expunged = true
+					}
+					oa.IsString = a.IsString
+					oa.Value = a.Value
+					err = tx.Update(&oa)
+					xcheckf(err, "updating metdata")
 				}
-				xcheckf(err, "looking up existing annotation for entry name")
 				changes = append(changes, a.Change(mailboxName))
-				oa.Value = a.Value
-				err = tx.Update(&oa)
-				xcheckf(err, "updating metadata annotation")
 			}
 
 			c.xcheckMetadataSize(tx)
@@ -304,7 +298,7 @@ func (c *conn) xcheckMetadataSize(tx *bstore.Tx) {
 	// ../rfc/5464:383
 	var n int
 	var size int
-	err := bstore.QueryTx[store.Annotation](tx).ForEach(func(a store.Annotation) error {
+	err := bstore.QueryTx[store.Annotation](tx).FilterEqual("Expunged", false).ForEach(func(a store.Annotation) error {
 		n++
 		if n > metadataMaxKeys {
 			// ../rfc/5464:590

imapserver/metadata_test.go

@@ -23,6 +23,15 @@ func TestMetadata(t *testing.T) {
 	tc.transactf("ok", `setmetadata "" (/PRIVATE/COMMENT "global value")`)
 	tc.transactf("ok", `setmetadata inbox (/private/comment "mailbox value")`)
 
+	tc.transactf("ok", `create metabox`)
+	tc.transactf("ok", `setmetadata metabox (/private/comment "mailbox value")`)
+	tc.transactf("ok", `setmetadata metabox (/shared/comment "mailbox value")`)
+	tc.transactf("ok", `setmetadata metabox (/shared/comment nil)`) // Remove.
+	tc.transactf("ok", `delete metabox`)                            // Delete mailbox with live and expunged metadata.
+
+	tc.transactf("no", `setmetadata expungebox (/private/comment "mailbox value")`)
+	tc.xcode("TRYCREATE")
+
 	tc.transactf("ok", `getmetadata "" ("/private/comment")`)
 	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
 		Mailbox: "",
@@ -176,7 +185,7 @@ func TestMetadata(t *testing.T) {
 
 	// Broadcast should not happen when metadata capability is not enabled.
 	tc2 := startNoSwitchboard(t)
-	defer tc2.close()
+	defer tc2.closeNoWait()
 	tc2.client.Login("mjl@mox.example", password0)
 	tc2.client.Select("inbox")
 

imapserver/move_test.go

@@ -12,10 +12,10 @@ func TestMove(t *testing.T) {
 	defer tc.close()
 
 	tc2 := startNoSwitchboard(t)
-	defer tc2.close()
+	defer tc2.closeNoWait()
 
 	tc3 := startNoSwitchboard(t)
-	defer tc3.close()
+	defer tc3.closeNoWait()
 
 	tc.client.Login("mjl@mox.example", password0)
 	tc.client.Select("inbox")
@@ -47,6 +47,9 @@ func TestMove(t *testing.T) {
 	tc.transactf("no", "move 1 nonexistent")
 	tc.xcode("TRYCREATE")
 
+	tc.transactf("no", "move 1 expungebox")
+	tc.xcode("TRYCREATE")
+
 	tc.transactf("no", "move 1 inbox") // Cannot move to same mailbox.
 
 	tc2.transactf("ok", "noop") // Drain.

imapserver/rename_test.go

@@ -12,7 +12,7 @@ func TestRename(t *testing.T) {
 	defer tc.close()
 
 	tc2 := startNoSwitchboard(t)
-	defer tc2.close()
+	defer tc2.closeNoWait()
 
 	tc.client.Login("mjl@mox.example", password0)
 	tc2.client.Login("mjl@mox.example", password0)
@@ -23,7 +23,9 @@ func TestRename(t *testing.T) {
 
 	tc.transactf("no", "rename doesnotexist newbox") // Does not exist.
 	tc.xcode("NONEXISTENT")                          // ../rfc/9051:5140
-	tc.transactf("no", `rename "Sent" "Trash"`)      // Already exists.
+	tc.transactf("no", "rename expungebox newbox")   // No longer exists.
+	tc.xcode("NONEXISTENT")
+	tc.transactf("no", `rename "Sent" "Trash"`) // Already exists.
 	tc.xcode("ALREADYEXISTS")
 
 	tc.client.Create("x", nil)
@@ -70,28 +72,51 @@ func TestRename(t *testing.T) {
 	tc.client.Unsubscribe("k")
 	tc.transactf("ok", "rename k/l k/l/m") // With "l" renamed, a new "k" will be created.
 	tc.transactf("ok", `list "" "k*" return (subscribed)`)
-	tc.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "k"}, imapclient.UntaggedList{Flags: []string{"\\Subscribed"}, Separator: '/', Mailbox: "k/l"}, imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/m"})
+	tc.xuntagged(
+		imapclient.UntaggedList{Separator: '/', Mailbox: "k"},
+		imapclient.UntaggedList{Flags: []string{"\\Subscribed"}, Separator: '/', Mailbox: "k/l"},
+		imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/m"},
+	)
+
+	tc.transactf("ok", "rename k/l/m k/l/x/y/m") // k/l/x and k/l/x/y will be created.
+	tc.transactf("ok", `list "" "k/l/x*" return (subscribed)`)
+	tc.xuntagged(
+		imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/x"},
+		imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/x/y"},
+		imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/x/y/m"},
+	)
 
 	// Renaming inbox keeps inbox in existence, moves messages, and does not rename children.
 	tc.transactf("ok", "create inbox/a")
 	// To check if UIDs are renumbered properly, we add UIDs 1 and 2. Expunge 1,
 	// keeping only 2. Then rename the inbox, which should renumber UID 2 in the old
 	// inbox to UID 1 in the newly created mailbox.
-	tc.transactf("ok", "append inbox (\\deleted) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
+	tc.transactf("ok", "append inbox (\\deleted) {1+}\r\nx")
-	tc.transactf("ok", "append inbox (label1) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
+	tc.transactf("ok", "append inbox (label1) {1+}\r\nx")
 	tc.transactf("ok", `select inbox`)
 	tc.transactf("ok", "expunge")
-	tc.transactf("ok", "rename inbox minbox")
+	tc.transactf("ok", "rename inbox x/minbox")
-	tc.transactf("ok", `list "" (inbox inbox/a minbox)`)
+	tc.transactf("ok", `list "" (inbox inbox/a x/minbox)`)
-	tc.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"}, imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox/a"}, imapclient.UntaggedList{Separator: '/', Mailbox: "minbox"})
+	tc.xuntagged(
-	tc.transactf("ok", `select minbox`)
+		imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"},
+		imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox/a"},
+		imapclient.UntaggedList{Separator: '/', Mailbox: "x/minbox"},
+	)
+	tc.transactf("ok", `select x/minbox`)
 	tc.transactf("ok", `uid fetch 1:* flags`)
 	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}}})
 
 	// Renaming to new hiearchy that does not have any subscribes.
-	tc.transactf("ok", "rename minbox w/w")
+	tc.transactf("ok", "rename x/minbox w/w")
 	tc.transactf("ok", `list "" "w*"`)
 	tc.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "w"}, imapclient.UntaggedList{Separator: '/', Mailbox: "w/w"})
 
+	tc.transactf("ok", "rename inbox misc/old/inbox")
+	tc.transactf("ok", `list "" (misc misc/old/inbox)`)
+	tc.xuntagged(
+		imapclient.UntaggedList{Separator: '/', Mailbox: "misc"},
+		imapclient.UntaggedList{Separator: '/', Mailbox: "misc/old/inbox"},
+	)
+
 	// todo: test create+delete+rename of/to a name results in a higher uidvalidity.
 }

imapserver/replace.go

@@ -1,7 +1,6 @@
 package imapserver
 
 import (
-	"context"
 	"errors"
 	"io"
 	"os"
@@ -116,7 +115,7 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) {
 		q := bstore.QueryTx[store.Message](tx)
 		q.FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: uidOld})
 		q.FilterEqual("Expunged", false)
-		om, err := q.Get()
+		_, err = q.Get()
 		if err == bstore.ErrAbsent {
 			return nil
 		}
@@ -124,8 +123,8 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) {
 			return func() { xserverErrorf("get message to replace: %v", err) }
 		}
 
-		delta := size - om.Size
+		// Check if we can add size bytes. We can't necessarily remove the current message yet.
-		ok, maxSize, err := c.account.CanAddMessageSize(tx, delta)
+		ok, maxSize, err := c.account.CanAddMessageSize(tx, size)
 		if err != nil {
 			return func() { xserverErrorf("check quota: %v", err) }
 		}
@@ -169,9 +168,7 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) {
 	var file *os.File
 	var newMsgPath string
 	var f io.Writer
-	var committed bool
+	var commit bool
-
-	var oldMsgPath string // To remove on success.
 
 	if errfn != nil {
 		// We got a non-sync literal, we will consume some data, but abort if there's too
@@ -197,14 +194,10 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) {
 				err := file.Close()
 				c.xsanity(err, "close temporary file for replace")
 			}
-			if newMsgPath != "" && !committed {
+			if newMsgPath != "" && !commit {
 				err := os.Remove(newMsgPath)
 				c.xsanity(err, "remove temporary file for replace")
 			}
-			if committed {
-				err := os.Remove(oldMsgPath)
-				c.xsanity(err, "remove old message")
-			}
 		}()
 	}
 
@@ -258,8 +251,8 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) {
 				return
 			}
 
-			// Check quota. Even if the delta is negative, the quota may have changed.
+			// Check quota for addition of new message. We can't necessarily yet remove the old message.
-			ok, maxSize, err := c.account.CanAddMessageSize(tx, mw.Size-om.Size)
+			ok, maxSize, err := c.account.CanAddMessageSize(tx, mw.Size)
 			xcheckf(err, "checking quota")
 			if !ok {
 				// ../rfc/9208:472
@@ -269,31 +262,11 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) {
 			modseq, err := c.account.NextModSeq(tx)
 			xcheckf(err, "get next mod seq")
 
-			// Subtract counts for message from source mailbox.
+			chremuids, _, err := c.account.MessageRemove(c.log, tx, modseq, &mbSrc, store.RemoveOpts{}, om)
-			mbSrc.Sub(om.MailboxCounts())
+			xcheckf(err, "expunge old message")
+			changes = append(changes, chremuids)
+			// Note: we only add a mbSrc counts change later on, if it is not equal to mbDst.
 
-			// Remove message recipients for old message.
-			_, err = bstore.QueryTx[store.Recipient](tx).FilterNonzero(store.Recipient{MessageID: om.ID}).Delete()
-			xcheckf(err, "removing message recipients")
-
-			// Subtract size of old message from account.
-			err = c.account.AddMessageSize(c.log, tx, -om.Size)
-			xcheckf(err, "updating disk usage")
-
-			// Undo any junk filter training for the old message.
-			om.Junk = false
-			om.Notjunk = false
-			err = c.account.RetrainMessages(context.TODO(), c.log, tx, []store.Message{om})
-			xcheckf(err, "untraining expunged messages")
-
-			// Mark old message expunged.
-			om.ModSeq = modseq
-			om.PrepareExpunge()
-			err = tx.Update(&om)
-			xcheckf(err, "mark old message as expunged")
-
-			// Update source mailbox.
-			mbSrc.ModSeq = modseq
 			err = tx.Update(&mbSrc)
 			xcheckf(err, "updating source mailbox counts")
 
@@ -316,22 +289,16 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) {
 
 			err = c.account.MessageAdd(c.log, tx, &mbDst, &nm, file, store.AddOpts{})
 			xcheckf(err, "delivering message")
+			// Update path to what is stored in the account. We may still have to clean it up on errors.
+			newMsgPath = c.account.MessagePath(nm.ID)
 
-			changes = append(changes,
+			changes = append(changes, nm.ChangeAddUID(), mbDst.ChangeCounts())
-				store.ChangeRemoveUIDs{MailboxID: om.MailboxID, UIDs: []store.UID{om.UID}, ModSeq: om.ModSeq},
-				nm.ChangeAddUID(),
-				mbDst.ChangeCounts(),
-			)
 			if nkeywords != len(mbDst.Keywords) {
 				changes = append(changes, mbDst.ChangeKeywords())
 			}
 
 			err = tx.Update(&mbDst)
 			xcheckf(err, "updating destination mailbox")
-
-			// Update path to what is stored in the account. We may still have to clean it up on errors.
-			newMsgPath = c.account.MessagePath(nm.ID)
-			oldMsgPath = c.account.MessagePath(om.ID)
 		})
 
 		// Fetch pending changes, possibly with new UIDs, so we can apply them before adding our own new UID.
@@ -342,7 +309,7 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) {
 		}
 
 		// Success, make sure messages aren't cleaned up anymore.
-		committed = true
+		commit = true
 
 		// Broadcast the change to other connections.
 		if mbSrc.ID != mbDst.ID {

imapserver/replace_test.go

@@ -13,7 +13,7 @@ func TestReplace(t *testing.T) {
 	defer tc.close()
 
 	tc2 := startNoSwitchboard(t)
-	defer tc2.close()
+	defer tc2.closeNoWait()
 
 	tc.client.Login("mjl@mox.example", password0)
 	tc.client.Select("inbox")
@@ -23,6 +23,9 @@ func TestReplace(t *testing.T) {
 	tc.client.StoreFlagsSet("1", true, `\deleted`)
 	tc.client.Expunge()
 
+	tc.transactf("no", "replace 2 expungebox {1}") // Mailbox no longer exists.
+	tc.xcode("TRYCREATE")
+
 	tc2.client.Login("mjl@mox.example", password0)
 	tc2.client.Select("inbox")
 
@@ -34,7 +37,7 @@ func TestReplace(t *testing.T) {
 		imapclient.UntaggedExists(3),
 		imapclient.UntaggedExpunge(2),
 	)
-	tc.xcodeArg(imapclient.CodeHighestModSeq(6))
+	tc.xcodeArg(imapclient.CodeHighestModSeq(8))
 
 	// Check that other client sees Exists and Expunge.
 	tc2.transactf("ok", "noop")
@@ -53,7 +56,7 @@ func TestReplace(t *testing.T) {
 		imapclient.UntaggedExists(3),
 		imapclient.UntaggedVanished{UIDs: xparseNumSet("2")},
 	)
-	tc.xcodeArg(imapclient.CodeHighestModSeq(7))
+	tc.xcodeArg(imapclient.CodeHighestModSeq(9))
 
 	// Leftover data.
 	tc.transactf("bad", "replace 1 inbox () {6+}\r\ntest\r\n ")
@@ -125,7 +128,7 @@ func TestReplaceExpunged(t *testing.T) {
 
 	// Get in with second client and remove the message we are replacing.
 	tc2 := startNoSwitchboard(t)
-	defer tc2.close()
+	defer tc2.closeNoWait()
 	tc2.client.Login("mjl@mox.example", password0)
 	tc2.client.Select("inbox")
 	tc2.client.StoreFlagsSet("1", true, `\Deleted`)

imapserver/search_test.go

@@ -406,7 +406,6 @@ func TestSearch(t *testing.T) {
 		writeTextLit(1, true)
 	}
 	writeTextLit(1, false)
-
 }
 
 // esearchall makes an UntaggedEsearch response with All set, for comparisons.

imapserver/selectexamine_test.go

@@ -49,6 +49,7 @@ func testSelectExamine(t *testing.T, examine bool) {
 
 	// Mailbox does not exist.
 	tc.transactf("no", "%s bogus", cmd)
+	tc.transactf("no", "%s expungebox", cmd)
 
 	tc.transactf("ok", "%s inbox", cmd)
 	tc.xuntagged(uflags, upermflags, urecent, uexists0, uuidval1, uuidnext1, ulist)

imapserver/server.go

@@ -26,7 +26,6 @@ non-ASCII UTF-8. Until that's enabled, we do use UTF-7 for mailbox names. See
 
 /*
 - todo: do not return binary data for a fetch body. at least not for imap4rev1. we should be encoding it as base64?
-- todo: on expunge we currently remove the message even if other sessions still have a reference to the uid. if they try to query the uid, they'll get an error. we could be nicer and only actually remove the message when the last reference has gone. we could add a new flag to store.Message marking the message as expunged, not give new session access to such messages, and make store remove them at startup, and clean them when the last session referencing the session goes. however, it will get much more complicated. renaming messages would need special handling. and should we do the same for removed mailboxes?
 - todo: try to recover from syntax errors when the last command line ends with a }, i.e. a literal. we currently abort the entire connection. we may want to read some amount of literal data and continue with a next command.
 */
 
@@ -69,6 +68,7 @@ import (
 	"github.com/mjl-/flate"
 
 	"github.com/mjl-/mox/config"
+	"github.com/mjl-/mox/junk"
 	"github.com/mjl-/mox/message"
 	"github.com/mjl-/mox/metrics"
 	"github.com/mjl-/mox/mlog"
@@ -1557,10 +1557,12 @@ func (c *conn) xmailbox(tx *bstore.Tx, name string, missingErrCode string) store
 // If the mailbox does not exist, panic is called with a user error.
 // Must be called with account rlock held.
 func (c *conn) xmailboxID(tx *bstore.Tx, id int64) store.Mailbox {
-	mb := store.Mailbox{ID: id}
+	mb, err := store.MailboxID(tx, id)
-	err := tx.Get(&mb)
 	if err == bstore.ErrAbsent {
 		xuserErrorf("%w", store.ErrUnknownMailbox)
+	} else if err == store.ErrMailboxExpunged {
+		// ../rfc/9051:5140
+		xusercodeErrorf("NONEXISTENT", "mailbox has been deleted")
 	}
 	return mb
 }
@@ -1589,6 +1591,7 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) {
 			mbID = ch.MailboxID
 		case store.ChangeRemoveUIDs:
 			mbID = ch.MailboxID
+			c.comm.RemovalSeen(ch)
 		case store.ChangeFlags:
 			mbID = ch.MailboxID
 		case store.ChangeRemoveMailbox, store.ChangeAddMailbox, store.ChangeRenameMailbox, store.ChangeAddSubscription:
@@ -2881,9 +2884,6 @@ func (c *conn) cmdDelete(tag, cmd string, p *parser) {
 
 	name = xcheckmailboxname(name, false)
 
-	// Messages to remove after having broadcasted the removal of messages.
-	var removeMessageIDs []int64
-
 	c.account.WithWLock(func() {
 		var mb store.Mailbox
 		var changes []store.Change
@@ -2893,7 +2893,7 @@ func (c *conn) cmdDelete(tag, cmd string, p *parser) {
 
 			var hasChildren bool
 			var err error
-			changes, removeMessageIDs, hasChildren, err = c.account.MailboxDelete(context.TODO(), c.log, tx, mb)
+			changes, hasChildren, err = c.account.MailboxDelete(context.TODO(), c.log, tx, &mb)
 			if hasChildren {
 				xusercodeErrorf("HASCHILDREN", "mailbox has a child, only leaf mailboxes can be deleted")
 			}
@@ -2903,12 +2903,6 @@ func (c *conn) cmdDelete(tag, cmd string, p *parser) {
 		c.broadcast(changes)
 	})
 
-	for _, mID := range removeMessageIDs {
-		p := c.account.MessagePath(mID)
-		err := os.Remove(p)
-		c.xsanity(err, "removing message file %q for mailbox delete", p)
-	}
-
 	c.ok(tag, cmd)
 }
 
@@ -2934,14 +2928,21 @@ func (c *conn) cmdRename(tag, cmd string, p *parser) {
 	src = xcheckmailboxname(src, true)
 	dst = xcheckmailboxname(dst, false)
 
+	var cleanupIDs []int64
+	defer func() {
+		for _, id := range cleanupIDs {
+			p := c.account.MessagePath(id)
+			err := os.Remove(p)
+			c.xsanity(err, "cleaning up message")
+		}
+	}()
+
 	c.account.WithWLock(func() {
 		var changes []store.Change
 
 		c.xdbwrite(func(tx *bstore.Tx) {
 			srcMB := c.xmailbox(tx, src, "NONEXISTENT")
 
-			var modseq store.ModSeq
-
 			// Inbox is very special. Unlike other mailboxes, its children are not moved. And
 			// unlike a regular move, its messages are moved to a newly created mailbox. We do
 			// indeed create a new destination mailbox and actually move the messages.
@@ -2956,111 +2957,53 @@ func (c *conn) cmdRename(tag, cmd string, p *parser) {
 					xuserErrorf("cannot move inbox to itself")
 				}
 
-				uidval, err := c.account.NextUIDValidity(tx)
+				var modseq store.ModSeq
-				xcheckf(err, "next uid validity")
+				dstMB, chl, err := c.account.MailboxEnsure(tx, dst, false, store.SpecialUse{}, &modseq)
+				xcheckf(err, "creating destination mailbox")
+				changes = chl
 
-				modseq, err = c.account.NextModSeq(tx)
+				// Copy mailbox annotations. ../rfc/5464:368
-				xcheckf(err, "assigning next modseq")
+				qa := bstore.QueryTx[store.Annotation](tx)
-
+				qa.FilterNonzero(store.Annotation{MailboxID: srcMB.ID})
-				dstMB := store.Mailbox{
+				qa.FilterEqual("Expunged", false)
-					Name:        dst,
+				annotations, err := qa.List()
-					UIDValidity: uidval,
+				xcheckf(err, "get annotations to copy for inbox")
-					UIDNext:     1,
+				for _, a := range annotations {
-					Keywords:    srcMB.Keywords,
+					a.ID = 0
-					ModSeq:      modseq,
+					a.MailboxID = dstMB.ID
-					CreateSeq:   modseq,
+					a.ModSeq = modseq
-					HaveCounts:  true,
+					a.CreateSeq = modseq
+					err := tx.Insert(&a)
+					xcheckf(err, "copy annotation to destination mailbox")
+					changes = append(changes, a.Change(dstMB.Name))
 				}
-				err = tx.Insert(&dstMB)
+				c.xcheckMetadataSize(tx)
-				xcheckf(err, "create new destination mailbox")
 
-				changes = make([]store.Change, 2) // Placeholders filled in below.
+				// Build query that selects messages to move.
-
-				// Move existing messages, with their ID's and on-disk files intact, to the new
-				// mailbox. We keep the expunged messages, the destination mailbox doesn't care
-				// about them.
-				var oldUIDs []store.UID
 				q := bstore.QueryTx[store.Message](tx)
 				q.FilterNonzero(store.Message{MailboxID: srcMB.ID})
 				q.FilterEqual("Expunged", false)
 				q.SortAsc("UID")
-				err = q.ForEach(func(m store.Message) error {
-					om := m
-					om.ID = 0
-					om.ModSeq = modseq
-					om.PrepareExpunge()
-					oldUIDs = append(oldUIDs, om.UID)
 
-					mc := m.MailboxCounts()
+				newIDs, chl := c.xmoveMessages(tx, q, 0, modseq, &srcMB, &dstMB)
-					srcMB.Sub(mc)
+				changes = append(changes, chl...)
-					dstMB.Add(mc)
+				cleanupIDs = newIDs
-
-					m.MailboxID = dstMB.ID
-					m.UID = dstMB.UIDNext
-					dstMB.UIDNext++
-					m.CreateSeq = modseq
-					m.ModSeq = modseq
-					if err := tx.Update(&m); err != nil {
-						return fmt.Errorf("updating message to move to new mailbox: %w", err)
-					}
-
-					changes = append(changes, m.ChangeAddUID())
-
-					if err := tx.Insert(&om); err != nil {
-						return fmt.Errorf("adding empty expunge message record to inbox: %w", err)
-					}
-					return nil
-				})
-				xcheckf(err, "moving messages from inbox to destination mailbox")
-
-				err = tx.Update(&dstMB)
-				xcheckf(err, "updating uidnext and counts in destination mailbox")
-
-				srcMB.ModSeq = modseq
-				err = tx.Update(&srcMB)
-				xcheckf(err, "updating counts for inbox")
-
-				var dstFlags []string
-				if tx.Get(&store.Subscription{Name: dstMB.Name}) == nil {
-					dstFlags = []string{`\Subscribed`}
-				}
-
-				// Copy any annotations. ../rfc/5464:368
-				annotations, err := bstore.QueryTx[store.Annotation](tx).FilterNonzero(store.Annotation{MailboxID: srcMB.ID}).List()
-				xcheckf(err, "get annotations to copy for inbox")
-				for i := range annotations {
-					annotations[i].ID = 0
-					annotations[i].MailboxID = dstMB.ID
-					annotations[i].ModSeq = modseq
-					annotations[i].CreateSeq = modseq
-					err := tx.Insert(&annotations[i])
-					xcheckf(err, "copy annotation to destination mailbox")
-				}
-
-				c.xcheckMetadataSize(tx)
-
-				changes[0] = store.ChangeRemoveUIDs{MailboxID: srcMB.ID, UIDs: oldUIDs, ModSeq: modseq}
-				changes[1] = store.ChangeAddMailbox{Mailbox: dstMB, Flags: dstFlags, ModSeq: modseq}
-				// changes[2:...] are ChangeAddUIDs
-				changes = append(changes, srcMB.ChangeCounts(), dstMB.ChangeCounts())
-				for _, a := range annotations {
-					changes = append(changes, a.Change(dstMB.Name))
-				}
 
 				return
 			}
 
-			var notExists, alreadyExists bool
+			var modseq store.ModSeq
+			var alreadyExists bool
 			var err error
-			changes, _, notExists, alreadyExists, err = c.account.MailboxRename(tx, srcMB, dst, &modseq)
+			changes, _, alreadyExists, err = c.account.MailboxRename(tx, &srcMB, dst, &modseq)
-			if notExists {
+			if alreadyExists {
-				// ../rfc/9051:5140
-				xusercodeErrorf("NONEXISTENT", "%s", err)
-			} else if alreadyExists {
 				xusercodeErrorf("ALREADYEXISTS", "%s", err)
 			}
 			xcheckf(err, "renaming mailbox")
 		})
+
+		cleanupIDs = nil
+
 		c.broadcast(changes)
 	})
 
@@ -3163,7 +3106,7 @@ func (c *conn) cmdLsub(tag, cmd string, p *parser) {
 		for _, sub := range subscriptions {
 			name := sub.Name
 			if ispercent {
-				for p := path.Dir(name); p != "."; p = path.Dir(p) {
+				for p := mox.ParentMailboxName(name); p != ""; p = mox.ParentMailboxName(p) {
 					subscribedKids[p] = true
 				}
 			}
@@ -3181,6 +3124,7 @@ func (c *conn) cmdLsub(tag, cmd string, p *parser) {
 			return
 		}
 		qmb := bstore.QueryTx[store.Mailbox](tx)
+		qmb.FilterEqual("Expunged", false)
 		qmb.SortAsc("Name")
 		err = qmb.ForEach(func(mb store.Mailbox) error {
 			if have[mb.Name] || !subscribedKids[mb.Name] || !re.MatchString(mb.Name) {
@@ -3544,12 +3488,11 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) {
 				// todo: do a single junk training
 				err = c.account.MessageAdd(c.log, tx, &mb, &a.m, a.file, store.AddOpts{SkipDirSync: true})
 				xcheckf(err, "delivering message")
-
-				changes = append(changes, a.m.ChangeAddUID())
-
 				// Update path to what is stored in the account. We may still have to clean it up on errors.
 				a.path = c.account.MessagePath(a.m.ID)
 
+				changes = append(changes, a.m.ChangeAddUID())
+
 				msgDirs[filepath.Dir(a.path)] = struct{}{}
 			}
 
@@ -3759,29 +3702,29 @@ func (c *conn) cmdClose(tag, cmd string, p *parser) {
 }
 
 // expunge messages marked for deletion in currently selected/active mailbox.
-// if uidSet is not nil, only messages matching the set are deleted.
+// if uidSet is not nil, only messages matching the set are expunged.
 //
-// messages that have been marked expunged from the database are returned and
+// Messages that have been marked expunged from the database are returned. While
-// have already been removed.
+// other sessions still reference the message, it is not cleared from the database
+// yet, and the message file is not yet removed.
 //
-// the highest modseq in the mailbox is returned, typically associated with the
+// The highest modseq in the mailbox is returned, typically associated with the
 // removal of the messages, but if no messages were expunged the current latest max
 // modseq for the mailbox is returned.
-func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) (removed []store.Message, highestModSeq store.ModSeq) {
+func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) (expunged []store.Message, highestModSeq store.ModSeq) {
-	var modseq store.ModSeq
-
 	c.account.WithWLock(func() {
-		var mb store.Mailbox
+		var changes []store.Change
 
 		c.xdbwrite(func(tx *bstore.Tx) {
-			mb = store.Mailbox{ID: c.mailboxID}
+			mb, err := store.MailboxID(tx, c.mailboxID)
-			err := tx.Get(&mb)
+			if err == bstore.ErrAbsent || err == store.ErrMailboxExpunged {
-			if err == bstore.ErrAbsent {
 				if missingMailboxOK {
 					return
 				}
-				xuserErrorf("%w", store.ErrUnknownMailbox)
+				// ../rfc/9051:5140
+				xusercodeErrorf("NONEXISTENT", "%w", store.ErrUnknownMailbox)
 			}
+			xcheckf(err, "get mailbox")
 
 			qm := bstore.QueryTx[store.Message](tx)
 			qm.FilterNonzero(store.Message{MailboxID: c.mailboxID})
@@ -3792,82 +3735,32 @@ func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) (removed []store.
 				return uidSearch(c.uids, m.UID) > 0 && (uidSet == nil || uidSet.containsUID(m.UID, c.uids, c.searchResult))
 			})
 			qm.SortAsc("UID")
-			removed, err = qm.List()
+			expunged, err = qm.List()
-			xcheckf(err, "listing messages to delete")
+			xcheckf(err, "listing messages to expunge")
 
-			if len(removed) == 0 {
+			if len(expunged) == 0 {
 				highestModSeq = mb.ModSeq
 				return
 			}
 
 			// Assign new modseq.
-			modseq, err = c.account.NextModSeq(tx)
+			modseq, err := c.account.NextModSeq(tx)
 			xcheckf(err, "assigning next modseq")
 			highestModSeq = modseq
 			mb.ModSeq = modseq
 
-			removeIDs := make([]int64, len(removed))
+			chremuids, chmbcounts, err := c.account.MessageRemove(c.log, tx, modseq, &mb, store.RemoveOpts{}, expunged...)
-			anyIDs := make([]any, len(removed))
+			xcheckf(err, "expunging messages")
-			var totalSize int64
+			changes = append(changes, chremuids, chmbcounts)
-			for i, m := range removed {
-				removeIDs[i] = m.ID
-				anyIDs[i] = m.ID
-				mb.Sub(m.MailboxCounts())
-				totalSize += m.Size
-				// Update "remove", because RetrainMessage below will save the message.
-				removed[i].Expunged = true
-				removed[i].ModSeq = modseq
-			}
-			qmr := bstore.QueryTx[store.Recipient](tx)
-			qmr.FilterEqual("MessageID", anyIDs...)
-			_, err = qmr.Delete()
-			xcheckf(err, "removing message recipients")
-
-			qm = bstore.QueryTx[store.Message](tx)
-			qm.FilterIDs(removeIDs)
-			n, err := qm.UpdateNonzero(store.Message{Expunged: true, ModSeq: modseq})
-			if err == nil && n != len(removeIDs) {
-				err = fmt.Errorf("only %d messages set to expunged, expected %d", n, len(removeIDs))
-			}
-			xcheckf(err, "marking messages marked for deleted as expunged")
 
 			err = tx.Update(&mb)
-			xcheckf(err, "updating mailbox counts")
+			xcheckf(err, "update mailbox")
-
-			err = c.account.AddMessageSize(c.log, tx, -totalSize)
-			xcheckf(err, "updating disk usage")
-
-			// Mark expunged messages as not needing training, then retrain them, so if they
-			// were trained, they get untrained.
-			for i := range removed {
-				removed[i].Junk = false
-				removed[i].Notjunk = false
-			}
-			err = c.account.RetrainMessages(context.TODO(), c.log, tx, removed)
-			xcheckf(err, "untraining expunged messages")
 		})
 
-		// Broadcast changes to other connections. We may not have actually removed any
+		c.broadcast(changes)
-		// messages, so take care not to send an empty update.
-		if len(removed) > 0 {
-			ouids := make([]store.UID, len(removed))
-			for i, m := range removed {
-				ouids[i] = m.UID
-			}
-			changes := []store.Change{
-				store.ChangeRemoveUIDs{MailboxID: c.mailboxID, UIDs: ouids, ModSeq: modseq},
-				mb.ChangeCounts(),
-			}
-			c.broadcast(changes)
-		}
-
-		for _, m := range removed {
-			p := c.account.MessagePath(m.ID)
-			err := os.Remove(p)
-			c.xsanity(err, "removing message file for expunge")
-		}
 	})
-	return removed, highestModSeq
+
+	return expunged, highestModSeq
 }
 
 // Unselect is similar to close in that it closes the currently active mailbox, but
@@ -4052,9 +3945,9 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) {
 	uids, uidargs := c.gatherCopyMoveUIDs(isUID, nums)
 
 	// Files that were created during the copy. Remove them if the operation fails.
-	var createdIDs []int64
+	var newIDs []int64
 	defer func() {
-		for _, id := range createdIDs {
+		for _, id := range newIDs {
 			p := c.account.MessagePath(id)
 			err := os.Remove(p)
 			c.xsanity(err, "cleaning up created file")
@@ -4200,7 +4093,7 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) {
 				}
 				err := moxio.LinkOrCopy(c.log, dst, src, nil, true)
 				xcheckf(err, "link or copy file %q to %q", src, dst)
-				createdIDs = append(createdIDs, newMsgIDs[i])
+				newIDs = append(newIDs, newMsgIDs[i])
 			}
 
 			for dir := range syncDirs {
@@ -4212,7 +4105,7 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) {
 			xcheckf(err, "train copied messages")
 		})
 
-		createdIDs = nil
+		newIDs = nil
 
 		// Broadcast changes to other connections.
 		if len(newUIDs) > 0 {
@@ -4253,127 +4146,61 @@ func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) {
 
 	uids, uidargs := c.gatherCopyMoveUIDs(isUID, nums)
 
-	var mbSrc, mbDst store.Mailbox
+	var mbDst store.Mailbox
-	var changes []store.Change
+	var uidFirst store.UID
-	var newUIDs []store.UID
 	var modseq store.ModSeq
 
+	var cleanupIDs []int64
+	defer func() {
+		for _, id := range cleanupIDs {
+			p := c.account.MessagePath(id)
+			err := os.Remove(p)
+			c.xsanity(err, "removing destination message file %v", p)
+		}
+	}()
+
 	c.account.WithWLock(func() {
+		var changes []store.Change
+
 		c.xdbwrite(func(tx *bstore.Tx) {
-			mbSrc = c.xmailboxID(tx, c.mailboxID) // Validate.
+			mbSrc := c.xmailboxID(tx, c.mailboxID) // Validate.
 			mbDst = c.xmailbox(tx, name, "TRYCREATE")
 			if mbDst.ID == c.mailboxID {
 				xuserErrorf("cannot move to currently selected mailbox")
 			}
 
-			if len(uidargs) == 0 {
+			if len(uids) == 0 {
 				xuserErrorf("no matching messages to move")
 			}
 
-			// Reserve the uids in the destination mailbox.
+			uidFirst = mbDst.UIDNext
-			uidFirst := mbDst.UIDNext
-			uidnext := uidFirst
-			mbDst.UIDNext += store.UID(len(uids))
 
 			// Assign a new modseq, for the new records and for the expunged records.
 			var err error
 			modseq, err = c.account.NextModSeq(tx)
 			xcheckf(err, "assigning next modseq")
-			mbSrc.ModSeq = modseq
-			mbDst.ModSeq = modseq
 
-			// Update existing record with new UID and MailboxID in database for messages. We
+			// Make query selecting messages to move.
-			// add a new but expunged record again in the original/source mailbox, for qresync.
-			// Keeping the original ID for the live message means we don't have to move the
-			// on-disk message contents file.
 			q := bstore.QueryTx[store.Message](tx)
-			q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
+			q.FilterNonzero(store.Message{MailboxID: mbSrc.ID})
 			q.FilterEqual("UID", uidargs...)
 			q.FilterEqual("Expunged", false)
 			q.SortAsc("UID")
-			msgs, err := q.List()
-			xcheckf(err, "listing messages to move")
 
-			if len(msgs) != len(uidargs) {
+			newIDs, chl := c.xmoveMessages(tx, q, len(uidargs), modseq, &mbSrc, &mbDst)
-				xserverErrorf("uid and message mismatch")
+			changes = append(changes, chl...)
-			}
+			cleanupIDs = newIDs
-
-			keywords := map[string]struct{}{}
-			now := time.Now()
-
-			conf, _ := c.account.Conf()
-			for i := range msgs {
-				m := &msgs[i]
-				if m.UID != uids[i] {
-					xserverErrorf("internal error: got uid %d, expected %d, for index %d", m.UID, uids[i], i)
-				}
-
-				mbSrc.Sub(m.MailboxCounts())
-
-				// Copy of message record that we'll insert when UID is freed up.
-				om := *m
-				om.PrepareExpunge()
-				om.ID = 0 // Assign new ID.
-				om.ModSeq = modseq
-
-				m.MailboxID = mbDst.ID
-				if m.IsReject && m.MailboxDestinedID != 0 {
-					// Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message
-					// is used for reputation calculation during future deliveries.
-					m.MailboxOrigID = m.MailboxDestinedID
-					m.IsReject = false
-					m.Seen = false
-				}
-				mbDst.Add(m.MailboxCounts())
-				m.UID = uidnext
-				m.ModSeq = modseq
-				m.JunkFlagsForMailbox(mbDst, conf)
-				m.SaveDate = &now
-				uidnext++
-				err := tx.Update(m)
-				xcheckf(err, "updating moved message in database")
-
-				// Now that UID is unused, we can insert the old record again.
-				err = tx.Insert(&om)
-				xcheckf(err, "inserting record for expunge after moving message")
-
-				for _, kw := range m.Keywords {
-					keywords[kw] = struct{}{}
-				}
-			}
-
-			// Ensure destination mailbox has keywords of the moved messages.
-			var mbKwChanged bool
-			mbDst.Keywords, mbKwChanged = store.MergeKeywords(mbDst.Keywords, maps.Keys(keywords))
-			if mbKwChanged {
-				changes = append(changes, mbDst.ChangeKeywords())
-			}
-
-			err = tx.Update(&mbSrc)
-			xcheckf(err, "updating source mailbox counts and modseq")
-
-			err = tx.Update(&mbDst)
-			xcheckf(err, "updating destination mailbox for uids, keywords and counts")
-
-			err = c.account.RetrainMessages(context.TODO(), c.log, tx, msgs)
-			xcheckf(err, "retraining messages after move")
-
-			// Prepare broadcast changes to other connections.
-			changes = make([]store.Change, 0, 1+len(msgs)+2)
-			changes = append(changes, store.ChangeRemoveUIDs{MailboxID: c.mailboxID, UIDs: uids, ModSeq: modseq})
-			for _, m := range msgs {
-				newUIDs = append(newUIDs, m.UID)
-				changes = append(changes, m.ChangeAddUID())
-			}
-			changes = append(changes, mbSrc.ChangeCounts(), mbDst.ChangeCounts())
 		})
 
+		cleanupIDs = nil
+
 		c.broadcast(changes)
 	})
 
 	// ../rfc/9051:4708 ../rfc/6851:254
 	// ../rfc/9051:4713
-	c.bwritelinef("* OK [COPYUID %d %s %s] moved", mbDst.UIDValidity, compactUIDSet(uids).String(), compactUIDSet(newUIDs).String())
+	newUIDs := numSet{ranges: []numRange{{setNumber{number: uint32(uidFirst)}, &setNumber{number: uint32(mbDst.UIDNext - 1)}}}}
+	c.bwritelinef("* OK [COPYUID %d %s %s] moved", mbDst.UIDValidity, compactUIDSet(uids).String(), newUIDs.String())
 	qresync := c.enabled[capQresync]
 	var vanishedUIDs numSet
 	for i := 0; i < len(uids); i++ {
@@ -4400,6 +4227,132 @@ func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) {
 	}
 }
 
+// q must yield messages from a single mailbox.
+func (c *conn) xmoveMessages(tx *bstore.Tx, q *bstore.Query[store.Message], expectCount int, modseq store.ModSeq, mbSrc, mbDst *store.Mailbox) (newIDs []int64, changes []store.Change) {
+	newIDs = make([]int64, 0, expectCount)
+	var commit bool
+	defer func() {
+		if commit {
+			return
+		}
+		for _, id := range newIDs {
+			p := c.account.MessagePath(id)
+			err := os.Remove(p)
+			c.xsanity(err, "removing added message file %v", p)
+		}
+		newIDs = nil
+	}()
+
+	mbSrc.ModSeq = modseq
+	mbDst.ModSeq = modseq
+
+	var jf *junk.Filter
+	defer func() {
+		if jf != nil {
+			err := jf.CloseDiscard()
+			c.log.Check(err, "closing junk filter after error")
+		}
+	}()
+
+	accConf, _ := c.account.Conf()
+
+	changeRemoveUIDs := store.ChangeRemoveUIDs{
+		MailboxID: mbSrc.ID,
+		ModSeq:    modseq,
+	}
+	changes = make([]store.Change, 0, expectCount+4) // mbsrc removeuids, mbsrc counts, mbdst counts, mbdst keywords
+
+	nkeywords := len(mbDst.Keywords)
+	now := time.Now()
+
+	l, err := q.List()
+	xcheckf(err, "listing messages to move")
+
+	if expectCount > 0 && len(l) != expectCount {
+		xcheckf(fmt.Errorf("moved %d messages, expected %d", len(l), expectCount), "move messages")
+	}
+
+	for _, om := range l {
+		nm := om
+		nm.MailboxID = mbDst.ID
+		nm.UID = mbDst.UIDNext
+		mbDst.UIDNext++
+		nm.ModSeq = modseq
+		nm.CreateSeq = modseq
+		nm.SaveDate = &now
+		if nm.IsReject && nm.MailboxDestinedID != 0 {
+			// Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message
+			// is used for reputation calculation during future deliveries.
+			nm.MailboxOrigID = nm.MailboxDestinedID
+			nm.IsReject = false
+			nm.Seen = false
+		}
+
+		nm.JunkFlagsForMailbox(*mbDst, accConf)
+
+		err := tx.Update(&nm)
+		xcheckf(err, "updating message with new mailbox")
+
+		mbDst.Add(nm.MailboxCounts())
+
+		mbSrc.Sub(om.MailboxCounts())
+		om.ID = 0
+		om.Expunged = true
+		om.ModSeq = modseq
+		om.TrainedJunk = nil
+		err = tx.Insert(&om)
+		xcheckf(err, "inserting expunged message in old mailbox")
+
+		err = moxio.LinkOrCopy(c.log, c.account.MessagePath(om.ID), c.account.MessagePath(nm.ID), nil, false)
+		xcheckf(err, "duplicating message in old mailbox for current sessions")
+		newIDs = append(newIDs, nm.ID)
+		// We don't sync the directory. In case of a crash and files disappearing, the
+		// eraser will simply not find the file at next startup.
+
+		err = tx.Insert(&store.MessageErase{ID: om.ID, SkipUpdateDiskUsage: true})
+		xcheckf(err, "insert message erase")
+
+		mbDst.Keywords, _ = store.MergeKeywords(mbDst.Keywords, nm.Keywords)
+
+		if accConf.JunkFilter != nil && nm.NeedsTraining() {
+			// Lazily open junk filter.
+			if jf == nil {
+				jf, _, err = c.account.OpenJunkFilter(context.TODO(), c.log)
+				xcheckf(err, "open junk filter")
+			}
+			err := c.account.RetrainMessage(context.TODO(), c.log, tx, jf, &nm)
+			xcheckf(err, "retrain message after moving")
+		}
+
+		changeRemoveUIDs.UIDs = append(changeRemoveUIDs.UIDs, om.UID)
+		changeRemoveUIDs.MsgIDs = append(changeRemoveUIDs.MsgIDs, om.ID)
+		changes = append(changes, nm.ChangeAddUID())
+	}
+	xcheckf(err, "move messages")
+
+	changes = append(changes, changeRemoveUIDs, mbSrc.ChangeCounts())
+
+	err = tx.Update(mbSrc)
+	xcheckf(err, "updating counts for inbox")
+
+	changes = append(changes, mbDst.ChangeCounts())
+	if len(mbDst.Keywords) > nkeywords {
+		changes = append(changes, mbDst.ChangeKeywords())
+	}
+
+	err = tx.Update(mbDst)
+	xcheckf(err, "updating uidnext and counts in destination mailbox")
+
+	if jf != nil {
+		err := jf.Close()
+		jf = nil
+		xcheckf(err, "saving junk filter")
+	}
+
+	commit = true
+	return
+}
+
 // Store sets a full set of flags, or adds/removes specific flags.
 //
 // State: Selected

imapserver/server_test.go

@@ -18,6 +18,8 @@ import (
 
 	"golang.org/x/sys/unix"
 
+	"github.com/mjl-/bstore"
+
 	"github.com/mjl-/mox/imapclient"
 	"github.com/mjl-/mox/mlog"
 	"github.com/mjl-/mox/mox-"
@@ -309,6 +311,14 @@ func (tc *testconn) waitDone() {
 }
 
 func (tc *testconn) close() {
+	tc.close0(true)
+}
+
+func (tc *testconn) closeNoWait() {
+	tc.close0(false)
+}
+
+func (tc *testconn) close0(waitclose bool) {
 	defer func() {
 		if unhandledPanics.Swap(0) > 0 {
 			tc.t.Fatalf("handled panic in server")
@@ -319,13 +329,16 @@ func (tc *testconn) close() {
 		// Already closed, we are not strict about closing multiple times.
 		return
 	}
-	err := tc.account.Close()
-	tc.check(err, "close account")
-	// no account.CheckClosed(), the tests open accounts multiple times.
-	tc.account = nil
 	if tc.client != nil {
 		tc.client.Close()
 	}
+	err := tc.account.Close()
+	tc.check(err, "close account")
+	if waitclose {
+		tc.account.WaitClosed()
+	}
+	// no account.CheckClosed(), the tests open accounts multiple times.
+	tc.account = nil
 	tc.serverConn.Close()
 	tc.waitDone()
 	if tc.switchStop != nil {
@@ -406,6 +419,22 @@ func startArgsMore(t *testing.T, first, immediateTLS bool, serverConfig, clientC
 	switchStop := func() {}
 	if first {
 		switchStop = store.Switchboard()
+
+		// Add a deleted mailbox, may excercise some code paths.
+		err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
+			// todo: add a message to inbox and remove it again. need to change all uids in the tests.
+			// todo: add tests for operating on an expunged mailbox. it should say it doesn't exist.
+
+			mb, _, _, _, err := acc.MailboxCreate(tx, "expungebox", store.SpecialUse{})
+			if err != nil {
+				return fmt.Errorf("create mailbox: %v", err)
+			}
+			if _, _, err := acc.MailboxDelete(ctxbg, pkglog, tx, &mb); err != nil {
+				return fmt.Errorf("delete mailbox: %v", err)
+			}
+			return nil
+		})
+		tcheck(t, err, "add expunged mailbox")
 	}
 
 	if afterInit != nil {
@@ -698,7 +727,7 @@ func TestMailboxDeleted(t *testing.T) {
 	defer tc.close()
 
 	tc2 := startNoSwitchboard(t)
-	defer tc2.close()
+	defer tc2.closeNoWait()
 
 	tc.client.Login("mjl@mox.example", password0)
 	tc2.client.Login("mjl@mox.example", password0)

imapserver/status_test.go

@@ -19,6 +19,8 @@ func TestStatus(t *testing.T) {
 	tc.transactf("bad", "status inbox (uidvalidity) ") // Leftover data.
 	tc.transactf("bad", "status inbox (unknown)")      // Unknown attribute.
 
+	tc.transactf("no", "status expungebox (messages)") // No longer exists.
+
 	tc.transactf("ok", "status inbox (messages uidnext uidvalidity unseen deleted size recent appendlimit)")
 	tc.xuntagged(imapclient.UntaggedStatus{
 		Mailbox: "Inbox",

imapserver/subscribe_test.go

@@ -11,7 +11,7 @@ func TestSubscribe(t *testing.T) {
 	defer tc.close()
 
 	tc2 := startNoSwitchboard(t)
-	defer tc2.close()
+	defer tc2.closeNoWait()
 
 	tc.client.Login("mjl@mox.example", password0)
 	tc2.client.Login("mjl@mox.example", password0)

imapserver/unsubscribe_test.go

@@ -14,10 +14,12 @@ func TestUnsubscribe(t *testing.T) {
 	tc.transactf("bad", "unsubscribe ")      // Missing param.
 	tc.transactf("bad", "unsubscribe fine ") // Leftover data.
 
-	tc.transactf("no", "unsubscribe a/b") // Does not exist and is not subscribed.
+	tc.transactf("no", "unsubscribe a/b")        // Does not exist and is not subscribed.
+	tc.transactf("ok", "unsubscribe expungebox") // Does not exist anymore but is still subscribed.
+	tc.transactf("no", "unsubscribe expungebox") // Not subscribed.
 	tc.transactf("ok", "create a/b")
 	tc.transactf("ok", "unsubscribe a/b")
-	tc.transactf("ok", "unsubscribe a/b") // Can unsubscribe even if it does not exist.
+	tc.transactf("ok", "unsubscribe a/b") // Can unsubscribe even if there is no subscription.
 	tc.transactf("ok", "subscribe a/b")
 	tc.transactf("ok", "unsubscribe a/b")
 }

import.go

@@ -254,7 +254,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
 		ctl.xwriteok()
 
 		// We will be delivering messages. If we fail halfway, we need to remove the created msg files.
-		var deliveredIDs []int64
+		var newIDs []int64
 		defer func() {
 			x := recover()
 			if x == nil {
@@ -269,12 +269,12 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
 				ctl.log.Error("import error")
 			}
 
-			for _, id := range deliveredIDs {
+			for _, id := range newIDs {
 				p := a.MessagePath(id)
 				err := os.Remove(p)
 				ctl.log.Check(err, "closing message file after import error", slog.String("path", p))
 			}
-			deliveredIDs = nil
+			newIDs = nil
 
 			ctl.xerror(fmt.Sprintf("import error: %v", x))
 		}()
@@ -371,7 +371,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
 			}
 			err = a.MessageAdd(ctl.log, tx, &mb, m, msgf, opts)
 			ctl.xcheck(err, "delivering message")
-			deliveredIDs = append(deliveredIDs, m.ID)
+			newIDs = append(newIDs, m.ID)
 			changes = append(changes, m.ChangeAddUID())
 
 			msgDirs[filepath.Dir(a.MessagePath(m.ID))] = struct{}{}
@@ -393,8 +393,8 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
 		}
 
 		// Match threads.
-		if len(deliveredIDs) > 0 {
+		if len(newIDs) > 0 {
-			err = a.AssignThreads(ctx, ctl.log, tx, deliveredIDs[0], 0, io.Discard)
+			err = a.AssignThreads(ctx, ctl.log, tx, newIDs[0], 0, io.Discard)
 			ctl.xcheck(err, "assigning messages to threads")
 		}
 
@@ -423,8 +423,8 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
 		err = tx.Commit()
 		ctl.xcheck(err, "commit")
 		tx = nil
-		ctl.log.Info("delivered messages through import", slog.Int("count", len(deliveredIDs)))
+		ctl.log.Info("delivered messages through import", slog.Int("count", len(newIDs)))
-		deliveredIDs = nil
+		newIDs = nil
 
 		store.BroadcastChanges(a, changes)
 	})

main.go

@@ -3340,6 +3340,7 @@ open, or is not running.
 		}
 
 		q := bstore.QueryTx[store.Mailbox](tx)
+		q.FilterEqual("Expunged", false)
 		if len(args) == 2 {
 			q.FilterEqual("Name", args[1])
 		}
@@ -3398,7 +3399,8 @@ open, or is not running.
 		// Reassign UIDs, going per mailbox. We assign starting at 1, only changing the
 		// message if it isn't already at the intended UID. Doing it in this order ensures
 		// we don't get into trouble with duplicate UIDs for a mailbox. We assign a new
-		// modseq. Not strictly needed, but doesn't hurt.
+		// modseq. Not strictly needed, but doesn't hurt. It's also why we assign a UID to
+		// expunged messages.
 		modseq, err := a.NextModSeq(tx)
 		xcheckf(err, "assigning next modseq")
 
@@ -3424,7 +3426,7 @@ open, or is not running.
 		}
 
 		// Now update the uidnext, uidvalidity and modseq for each mailbox.
-		err = bstore.QueryTx[store.Mailbox](tx).ForEach(func(mb store.Mailbox) error {
+		err = bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb store.Mailbox) error {
 			// Assign each mailbox a completely new uidvalidity.
 			uidvalidity, err := a.NextUIDValidity(tx)
 			if err != nil {
@@ -3491,7 +3493,7 @@ open, or is not running.
 	err = a.DB.Write(context.Background(), func(tx *bstore.Tx) error {
 		// We look at each mailbox, retrieve its max UID and compare against the mailbox
 		// UIDNEXT.
-		err := bstore.QueryTx[store.Mailbox](tx).ForEach(func(mb store.Mailbox) error {
+		err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb store.Mailbox) error {
 			if mb.UIDValidity > maxUIDValidity {
 				maxUIDValidity = mb.UIDValidity
 			}

mox-/config.go

@@ -933,6 +933,10 @@ func PrepareStaticConfig(ctx context.Context, log mlog.Log, configFile string, c
 	// DefaultMailboxes is deprecated.
 	for _, mb := range c.DefaultMailboxes {
 		checkMailboxNormf(mb, "default mailbox")
+		// We don't create parent mailboxes for default mailboxes.
+		if ParentMailboxName(mb) != "" {
+			addErrorf("default mailbox cannot be a child mailbox")
+		}
 	}
 	checkSpecialUseMailbox := func(nameOpt string) {
 		if nameOpt != "" {
@@ -940,6 +944,10 @@ func PrepareStaticConfig(ctx context.Context, log mlog.Log, configFile string, c
 			if strings.EqualFold(nameOpt, "inbox") {
 				addErrorf("initial mailbox cannot be set to Inbox (Inbox is always created)")
 			}
+			// We don't currently create parent mailboxes for initial mailboxes.
+			if ParentMailboxName(nameOpt) != "" {
+				addErrorf("initial mailboxes cannot be child mailboxes")
+			}
 		}
 	}
 	checkSpecialUseMailbox(c.InitialMailboxes.SpecialUse.Archive)
@@ -952,6 +960,9 @@ func PrepareStaticConfig(ctx context.Context, log mlog.Log, configFile string, c
 		if strings.EqualFold(name, "inbox") {
 			addErrorf("initial regular mailbox cannot be set to Inbox (Inbox is always created)")
 		}
+		if ParentMailboxName(name) != "" {
+			addErrorf("initial mailboxes cannot be child mailboxes")
+		}
 	}
 
 	checkTransportSMTP := func(name string, isTLS bool, t *config.TransportSMTP) {

mox-/parentname.go

@@ -0,0 +1,15 @@
+package mox
+
+import (
+	"strings"
+)
+
+// ParentMailboxName returns the name of the parent mailbox, returning empty if
+// there is no parent.
+func ParentMailboxName(name string) string {
+	i := strings.LastIndex(name, "/")
+	if i < 0 {
+		return ""
+	}
+	return name[:i]
+}

queue/hook_test.go

@@ -29,7 +29,7 @@ func TestHookIncoming(t *testing.T) {
 	tcheck(t, err, "open account for retired")
 	defer func() {
 		accret.Close()
-		accret.CheckClosed()
+		accret.WaitClosed()
 	}()
 
 	testIncoming := func(a *store.Account, expIn bool) {
@@ -125,7 +125,7 @@ func TestFromIDIncomingDelivery(t *testing.T) {
 	tcheck(t, err, "open account for retired")
 	defer func() {
 		accret.Close()
-		accret.CheckClosed()
+		accret.WaitClosed()
 	}()
 
 	// Account that only gets webhook calls, but no retired webhooks.
@@ -133,7 +133,7 @@ func TestFromIDIncomingDelivery(t *testing.T) {
 	tcheck(t, err, "open account for hook")
 	defer func() {
 		acchook.Close()
-		acchook.CheckClosed()
+		acchook.WaitClosed()
 	}()
 
 	addr, err := smtp.ParseAddress("mjl@mox.example")

queue/queue_test.go

@@ -75,7 +75,7 @@ func setup(t *testing.T) (*store.Account, func()) {
 	mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg)
 	return acc, func() {
 		acc.Close()
-		acc.CheckClosed()
+		acc.WaitClosed()
 		mox.ShutdownCancel()
 		mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg)
 		Shutdown()

smtpserver/fuzz_test.go

@@ -43,7 +43,7 @@ func FuzzServer(f *testing.F) {
 	}
 	defer func() {
 		acc.Close()
-		acc.CheckClosed()
+		acc.WaitClosed()
 	}()
 	err = acc.SetPassword(log, "testtest")
 	if err != nil {

smtpserver/rejects.go

@@ -46,14 +46,11 @@ func rejectPresent(log mlog.Log, acc *store.Account, rejectsMailbox string, m *s
 	var err error
 	acc.WithRLock(func() {
 		err = acc.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
-			mbq := bstore.QueryTx[store.Mailbox](tx)
+			mb, err := acc.MailboxFind(tx, rejectsMailbox)
-			mbq.FilterNonzero(store.Mailbox{Name: rejectsMailbox})
-			mb, err := mbq.Get()
-			if err == bstore.ErrAbsent {
-				return nil
-			}
 			if err != nil {
 				return fmt.Errorf("looking for rejects mailbox: %w", err)
+			} else if mb == nil {
+				return nil
 			}
 
 			q := bstore.QueryTx[store.Message](tx)

smtpserver/server.go

@@ -3272,6 +3272,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 					// See if we received a non-junk message from this organizational domain.
 					q := bstore.QueryTx[store.Message](tx)
 					q.FilterNonzero(store.Message{MsgFromOrgDomain: a0.d.m.MsgFromOrgDomain})
+					q.FilterEqual("Expunged", false)
 					q.FilterEqual("Notjunk", true)
 					q.FilterEqual("IsReject", false)
 					exists, err := q.Exists()
@@ -3406,25 +3407,37 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 				// due to reputation for later delivery attempts.
 				a.d.m.MessageHash = messagehash
 				a.d.acc.WithWLock(func() {
-					hasSpace := true
-					var err error
-					if !conf.KeepRejects {
-						hasSpace, err = a.d.acc.TidyRejectsMailbox(c.log, conf.RejectsMailbox)
-					}
-					if err != nil {
-						log.Errorx("tidying rejects mailbox", err)
-					} else if !hasSpace {
-						log.Info("not storing spammy mail to full rejects mailbox")
-						return
-					}
-
 					var changes []store.Change
 					var stored bool
-					err = a.d.acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
+
+					var newID int64
+					defer func() {
+						if newID != 0 {
+							p := a.d.acc.MessagePath(newID)
+							err := os.Remove(p)
+							c.log.Check(err, "remove message after error delivering to rejects", slog.String("path", p))
+						}
+					}()
+
+					err := a.d.acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
 						mbrej, err := a.d.acc.MailboxFind(tx, conf.RejectsMailbox)
 						if err != nil {
 							return fmt.Errorf("finding rejects mailbox: %v", err)
 						}
+
+						hasSpace := true
+						if !conf.KeepRejects && mbrej != nil {
+							var chl []store.Change
+							chl, hasSpace, err = a.d.acc.TidyRejectsMailbox(c.log, tx, mbrej)
+							if err != nil {
+								return fmt.Errorf("tidying rejects mailbox: %v", err)
+							}
+							if !hasSpace {
+								log.Info("not storing spammy mail to full rejects mailbox")
+								return nil
+							}
+							changes = append(changes, chl...)
+						}
 						if mbrej == nil {
 							nmb, chl, _, _, err := a.d.acc.MailboxCreate(tx, conf.RejectsMailbox, store.SpecialUse{})
 							if err != nil {
@@ -3438,6 +3451,8 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 						if err := a.d.acc.MessageAdd(log, tx, mbrej, a.d.m, dataFile, store.AddOpts{}); err != nil {
 							return fmt.Errorf("delivering spammy mail to rejects mailbox: %v", err)
 						}
+						newID = a.d.m.ID
+
 						if err := tx.Update(mbrej); err != nil {
 							return fmt.Errorf("updating rejects mailbox: %v", err)
 						}
@@ -3451,6 +3466,8 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 					} else if stored {
 						log.Info("stored spammy mail in rejects mailbox")
 					}
+					newID = 0
+
 					store.BroadcastChanges(a.d.acc, changes)
 				})
 			}

smtpserver/server_test.go

@@ -184,7 +184,7 @@ func (ts *testserver) close() {
 	ts.switchStop()
 	err = ts.acc.Close()
 	tcheck(ts.t, err, "closing account")
-	ts.acc.CheckClosed()
+	ts.acc.WaitClosed()
 	ts.acc = nil
 }
 
@@ -193,6 +193,7 @@ func (ts *testserver) checkCount(mailboxName string, expect int) {
 	t.Helper()
 	q := bstore.QueryDB[store.Mailbox](ctxbg, ts.acc.DB)
 	q.FilterNonzero(store.Mailbox{Name: mailboxName})
+	q.FilterEqual("Expunged", false)
 	mb, err := q.Get()
 	tcheck(t, err, "get mailbox")
 	qm := bstore.QueryDB[store.Message](ctxbg, ts.acc.DB)

store/account_test.go

@@ -46,7 +46,7 @@ func TestMailbox(t *testing.T) {
 	defer func() {
 		err = acc.Close()
 		tcheck(t, err, "closing account")
-		acc.CheckClosed()
+		acc.WaitClosed()
 	}()
 	defer Switchboard()()
 
@@ -162,6 +162,8 @@ func TestMailbox(t *testing.T) {
 
 	var modseq ModSeq
 	acc.WithWLock(func() {
+		var changes []Change
+
 		err := acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
 			_, _, err := acc.MailboxEnsure(tx, "Testbox", true, SpecialUse{}, &modseq)
 			return err
@@ -200,27 +202,33 @@ func TestMailbox(t *testing.T) {
 				t.Fatalf("did not find Testbox2")
 			}
 
-			changes, err := acc.SubscriptionEnsure(tx, "Testbox2")
+			nchanges, err := acc.SubscriptionEnsure(tx, "Testbox2")
 			tcheck(t, err, "ensuring new subscription")
-			if len(changes) == 0 {
+			if len(nchanges) == 0 {
 				t.Fatalf("new subscription did not result in changes")
 			}
-			changes, err = acc.SubscriptionEnsure(tx, "Testbox2")
+			changes = append(changes, nchanges...)
+			nchanges, err = acc.SubscriptionEnsure(tx, "Testbox2")
 			tcheck(t, err, "ensuring already present subscription")
-			if len(changes) != 0 {
+			if len(nchanges) != 0 {
 				t.Fatalf("already present subscription resulted in changes")
 			}
 
+			// todo: check that messages are removed.
+			mbRej, err := bstore.QueryTx[Mailbox](tx).FilterNonzero(Mailbox{Name: "Rejects"}).Get()
+			tcheck(t, err, "get rejects mailbox")
+			nchanges, hasSpace, err := acc.TidyRejectsMailbox(log, tx, &mbRej)
+			tcheck(t, err, "tidy rejects mailbox")
+			changes = append(changes, nchanges...)
+			if !hasSpace {
+				t.Fatalf("no space for more rejects")
+			}
+
 			return nil
 		})
 		tcheck(t, err, "write tx")
 
-		// todo: check that messages are removed.
+		BroadcastChanges(acc, changes)
-		hasSpace, err := acc.TidyRejectsMailbox(log, "Rejects")
-		tcheck(t, err, "tidy rejects mailbox")
-		if !hasSpace {
-			t.Fatalf("no space for more rejects")
-		}
 
 		acc.RejectsRemove(log, "Rejects", "m01@mox.example")
 	})

store/export.go

@@ -10,7 +10,6 @@ import (
 	"io"
 	"log/slog"
 	"os"
-	"path"
 	"path/filepath"
 	"strings"
 	"time"
@@ -18,6 +17,7 @@ import (
 	"github.com/mjl-/bstore"
 
 	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
 )
 
 // Archiver can archive multiple mailboxes and their messages.
@@ -158,9 +158,10 @@ func ExportMessages(ctx context.Context, log mlog.Log, db *bstore.DB, accountDir
 	var trimPrefix string
 	if mailboxOpt != "" {
 		// If exporting a specific mailbox, trim its parent path from stored file names.
-		trimPrefix = path.Dir(mailboxOpt) + "/"
+		trimPrefix = mox.ParentMailboxName(mailboxOpt) + "/"
 	}
 	q := bstore.QueryTx[Mailbox](tx)
+	q.FilterEqual("Expunged", false)
 	q.FilterFn(func(mb Mailbox) bool {
 		return mailboxOpt == "" || mb.Name == mailboxOpt || recursive && strings.HasPrefix(mb.Name, prefix)
 	})

store/state.go

@@ -1,14 +1,24 @@
 package store
 
 import (
+	"fmt"
+	"log/slog"
+	"os"
 	"sync"
 	"sync/atomic"
+
+	"github.com/mjl-/bstore"
+
+	"github.com/mjl-/mox/metrics"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
 )
 
 var (
 	register   = make(chan *Comm)
 	unregister = make(chan *Comm)
 	broadcast  = make(chan changeReq)
+	applied    = make(chan removalApplied)
 )
 
 type changeReq struct {
@@ -18,6 +28,11 @@ type changeReq struct {
 	done    chan struct{}
 }
 
+type removalApplied struct {
+	Account *Account
+	MsgIDs  []int64
+}
+
 type UID uint32 // IMAP UID.
 
 // Change to mailboxes/subscriptions/messages in an account. One of the Change*
@@ -38,6 +53,7 @@ type ChangeRemoveUIDs struct {
 	MailboxID int64
 	UIDs      []UID // Must be in increasing UID order, for IMAP.
 	ModSeq    ModSeq
+	MsgIDs    []int64 // Message.ID, for erasing, order does not necessarily correspond with UIDs!
 }
 
 // ChangeFlags is sent for an update to flags for a message, e.g. "Seen".
@@ -118,61 +134,295 @@ type ChangeAnnotation struct {
 	ModSeq      ModSeq
 }
 
+func messageEraser(donec chan struct{}, cleanc chan map[*Account][]int64) {
+	log := mlog.New("store", nil)
+
+	for {
+		clean, ok := <-cleanc
+		if !ok {
+			donec <- struct{}{}
+			return
+		}
+
+		for acc, ids := range clean {
+			eraseMessages(log, acc, ids)
+		}
+	}
+}
+
+func eraseMessages(log mlog.Log, acc *Account, ids []int64) {
+	// We are responsible for closing the accounts.
+	defer func() {
+		err := acc.Close()
+		log.Check(err, "close account after erasing expunged messages", slog.String("account", acc.Name))
+	}()
+
+	acc.Lock()
+	defer acc.Unlock()
+	err := acc.DB.Write(mox.Context, func(tx *bstore.Tx) error {
+		du := DiskUsage{ID: 1}
+		if err := tx.Get(&du); err != nil {
+			return fmt.Errorf("get disk usage: %v", err)
+		}
+		var duchanged bool
+
+		for _, id := range ids {
+			me := MessageErase{ID: id}
+			if err := tx.Get(&me); err != nil {
+				return fmt.Errorf("delete message erase record %d: %v", id, err)
+			}
+
+			m := Message{ID: id}
+			if err := tx.Get(&m); err != nil {
+				return fmt.Errorf("get message %d to erase: %v", id, err)
+			} else if !m.Expunged {
+				return fmt.Errorf("message %d to erase is not marked expunged", id)
+			}
+			if !me.SkipUpdateDiskUsage {
+				du.MessageSize -= m.Size
+				duchanged = true
+			}
+			m.erase()
+			if err := tx.Update(&m); err != nil {
+				return fmt.Errorf("mark message %d erase in database: %v", id, err)
+			}
+
+			if err := tx.Delete(&me); err != nil {
+				return fmt.Errorf("deleting message erase record %d: %v", id, err)
+			}
+		}
+
+		if duchanged {
+			if err := tx.Update(&du); err != nil {
+				return fmt.Errorf("update disk usage after erasing: %v", err)
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		log.Errorx("erasing expunged messages", err,
+			slog.String("account", acc.Name),
+			slog.Any("ids", ids),
+		)
+		return
+	}
+
+	// We remove the files after the database commit. It's better to have the files
+	// still around without being referenced from the database than references in the
+	// database to non-existent files.
+	for _, id := range ids {
+		p := acc.MessagePath(id)
+		err := os.Remove(p)
+		log.Check(err, "removing expunged message file from disk", slog.String("path", p))
+	}
+}
+
+func switchboard(stopc, donec chan struct{}, cleanc chan map[*Account][]int64) {
+	regs := map[*Account]map[*Comm]struct{}{}
+
+	// We don't remove message files or clear fields in the Message stored in the
+	// database until all references, from all sessions have gone away. When we see
+	// an expunge of a message, we count how many comms are active (i.e. how many
+	// sessions reference the message). We require each of them to tell us they are no
+	// longer referencing that message. Once we've seen that from all Comms, we remove
+	// the on-disk file and the fields from the database.
+	//
+	// During the initial account open (when there are no active sessions/Comms yet,
+	// and we open the message database file), the message erases will also be applied.
+	//
+	// When we add an account to eraseRefs, we increase the refcount, and we decrease
+	// it again when removing the account.
+	eraseRefs := map[*Account]map[int64]int{}
+
+	// We collect which messages can be erased per account, for sending them off to the
+	// eraser goroutine. When an account is added to this map, its refcount is
+	// increased. It is decreased again by the eraser goroutine.
+	eraseIDs := map[*Account][]int64{}
+
+	addEraseIDs := func(acc *Account, ids ...int64) {
+		if _, ok := eraseIDs[acc]; !ok {
+			openAccounts.Lock()
+			acc.nused++
+			openAccounts.Unlock()
+		}
+		eraseIDs[acc] = append(eraseIDs[acc], ids...)
+	}
+
+	decreaseEraseRefs := func(acc *Account, ids ...int64) {
+		for _, id := range ids {
+			v := eraseRefs[acc][id] - 1
+			if v < 0 {
+				metrics.PanicInc(metrics.Store) // For tests.
+				panic(fmt.Sprintf("negative expunged message references for account %q, message id %d", acc.Name, id))
+			}
+			if v > 0 {
+				eraseRefs[acc][id] = v
+				continue
+			}
+
+			addEraseIDs(acc, id)
+			delete(eraseRefs[acc], id)
+			if len(eraseRefs[acc]) > 0 {
+				continue
+			}
+			delete(eraseRefs, acc)
+			// Note: cannot use acc.Close, it tries to lock acc, but someone broadcasting to
+			// this goroutine will likely have the lock.
+			openAccounts.Lock()
+			acc.nused--
+			n := acc.nused
+			openAccounts.Unlock()
+			if n < 0 {
+				metrics.PanicInc(metrics.Store) // For tests.
+				panic(fmt.Sprintf("negative reference count for account %q, after removing message id %d", acc.Name, id))
+			}
+		}
+	}
+
+	for {
+		// If we have messages to clean, try sending to the eraser.
+		cc := cleanc
+		if len(eraseIDs) == 0 {
+			cc = nil
+		}
+
+		select {
+		case cc <- eraseIDs:
+			eraseIDs = map[*Account][]int64{}
+
+		case c := <-register:
+			if _, ok := regs[c.acc]; !ok {
+				regs[c.acc] = map[*Comm]struct{}{}
+			}
+			regs[c.acc][c] = struct{}{}
+
+		case c := <-unregister:
+			// Drain any ChangeRemoveUIDs references from the comm, to update our eraseRefs and
+			// possibly queue messages for cleaning. No need to take a lock, the caller does
+			// not use the comm anymore.
+			for _, ch := range c.changes {
+				rem, ok := ch.(ChangeRemoveUIDs)
+				if !ok {
+					continue
+				}
+				decreaseEraseRefs(c.acc, rem.MsgIDs...)
+			}
+
+			delete(regs[c.acc], c)
+			if len(regs[c.acc]) == 0 {
+				delete(regs, c.acc)
+			}
+
+		case chReq := <-broadcast:
+			acc := chReq.acc
+
+			// Track references to removed messages in sessions (mostly IMAP) so we can pass
+			// them to the eraser.
+			for _, ch := range chReq.changes {
+				rem, ok := ch.(ChangeRemoveUIDs)
+				if !ok {
+					continue
+				}
+
+				refs := len(regs[acc])
+				if chReq.comm != nil {
+					// The sender does not get this change and doesn't have to notify us of having
+					// processed the removal.
+					refs--
+				}
+				if refs <= 0 {
+					addEraseIDs(acc, rem.MsgIDs...)
+					continue
+				}
+
+				// Comms/sessions still reference these messages, track how many.
+				for _, id := range rem.MsgIDs {
+					if _, ok := eraseRefs[acc]; !ok {
+						openAccounts.Lock()
+						acc.nused++
+						openAccounts.Unlock()
+
+						eraseRefs[acc] = map[int64]int{}
+					}
+					if _, ok := eraseRefs[acc][id]; ok {
+						metrics.PanicInc(metrics.Store) // For tests.
+						panic(fmt.Sprintf("already have eraseRef for message id %d, account %q", id, acc.Name))
+					}
+					eraseRefs[acc][id] = refs
+				}
+			}
+
+			for c := range regs[acc] {
+				// Do not send the broadcaster back their own changes. chReq.comm is nil if not
+				// originating from a comm, so won't match in that case.
+				if c == chReq.comm {
+					continue
+				}
+
+				c.Lock()
+				c.changes = append(c.changes, chReq.changes...)
+				c.Unlock()
+
+				select {
+				case c.Pending <- struct{}{}:
+				default:
+				}
+			}
+			chReq.done <- struct{}{}
+
+		case removal := <-applied:
+			acc := removal.Account
+
+			// Decrease references of messages, queueing for erasure when the last reference
+			// goes away.
+			decreaseEraseRefs(acc, removal.MsgIDs...)
+
+		case <-stopc:
+			// We may still have eraseRefs, messages currently referenced in a session. Those
+			// messages will be erased when the database file is opened again in the future. If
+			// we have messages ready to erase now, we'll do that first.
+
+			if len(eraseIDs) > 0 {
+				cleanc <- eraseIDs
+				eraseIDs = nil
+			}
+
+			for acc := range eraseRefs {
+				err := acc.Close()
+				log := mlog.New("store", nil)
+				log.Check(err, "closing account")
+			}
+
+			close(cleanc)       // Tell eraser to stop.
+			donec <- struct{}{} // Say we are now done.
+			return
+		}
+	}
+}
+
 var switchboardBusy atomic.Bool
 
 // Switchboard distributes changes to accounts to interested listeners. See Comm and Change.
 func Switchboard() (stop func()) {
-	regs := map[*Account]map[*Comm]struct{}{}
-	done := make(chan struct{})
-
 	if !switchboardBusy.CompareAndSwap(false, true) {
 		panic("switchboard already busy")
 	}
 
-	go func() {
+	stopc := make(chan struct{})
-		for {
+	donec := make(chan struct{})
-			select {
+	cleanc := make(chan map[*Account][]int64)
-			case c := <-register:
-				if _, ok := regs[c.acc]; !ok {
-					regs[c.acc] = map[*Comm]struct{}{}
-				}
-				regs[c.acc][c] = struct{}{}
 
-			case c := <-unregister:
+	go messageEraser(donec, cleanc)
-				delete(regs[c.acc], c)
+	go switchboard(stopc, donec, cleanc)
-				if len(regs[c.acc]) == 0 {
-					delete(regs, c.acc)
-				}
 
-			case chReq := <-broadcast:
-				acc := chReq.acc
-				for c := range regs[acc] {
-					// Do not send the broadcaster back their own changes. chReq.comm is nil if not
-					// originating from a comm, so won't match in that case.
-					if c == chReq.comm {
-						continue
-					}
-
-					c.Lock()
-					c.changes = append(c.changes, chReq.changes...)
-					c.Unlock()
-
-					select {
-					case c.Pending <- struct{}{}:
-					default:
-					}
-				}
-				chReq.done <- struct{}{}
-
-			case <-done:
-				done <- struct{}{}
-				return
-			}
-		}
-	}()
 	return func() {
-		done <- struct{}{}
+		stopc <- struct{}{}
-		<-done
+
+		// Wait for switchboard and eraser goroutines to be ready.
+		<-donec
+		<-donec
+
 		if !switchboardBusy.CompareAndSwap(true, false) {
 			panic("switchboard already unregistered?")
 		}
@@ -225,6 +475,13 @@ func (c *Comm) Get() []Change {
 	return l
 }
 
+// RemovalSeen must be called by consumers when they have applied the removal to
+// their session. The switchboard tracks references of expunged messages, and
+// removes/cleans the message up when the last reference is gone.
+func (c *Comm) RemovalSeen(ch ChangeRemoveUIDs) {
+	applied <- removalApplied{c.acc, ch.MsgIDs}
+}
+
 // BroadcastChanges ensures changes are sent to all listeners on the accoount.
 func BroadcastChanges(acc *Account, ch []Change) {
 	if len(ch) == 0 {

store/threads.go

@@ -357,6 +357,8 @@ func (a *Account) AssignThreads(ctx context.Context, log mlog.Log, txOpt *bstore
 			m := Message{ID: mi.ID}
 			if err := tx.Get(&m); err != nil {
 				return fmt.Errorf("get message %d for resolving pending thread for message-id %s, %d: %w", mi.ID, tm.MessageID, tm.ID, err)
+			} else if m.Expunged {
+				return fmt.Errorf("message %d marked as expunged", mi.ID)
 			}
 			if m.ThreadID != 0 {
 				// ThreadID already set because this is a cyclic message. If we would assign a

store/train.go

@@ -155,11 +155,7 @@ func (a *Account) RetrainMessage(ctx context.Context, log mlog.Log, tx *bstore.T
 
 // TrainMessage trains the junk filter based on the current m.Junk/m.Notjunk flags,
 // disregarding m.TrainedJunk and not updating that field.
-func (a *Account) TrainMessage(ctx context.Context, log mlog.Log, jf *junk.Filter, m Message) (bool, error) {
+func (a *Account) TrainMessage(ctx context.Context, log mlog.Log, jf *junk.Filter, ham bool, m Message) (bool, error) {
-	if m.Junk == m.Notjunk {
-		return false, nil
-	}
-
 	mr := a.MessageReader(m)
 	defer func() {
 		err := mr.Close()
@@ -178,5 +174,5 @@ func (a *Account) TrainMessage(ctx context.Context, log mlog.Log, jf *junk.Filte
 		return false, nil
 	}
 
-	return true, jf.Train(ctx, m.Notjunk, words)
+	return true, jf.Train(ctx, ham, words)
 }

verifydata.go

@@ -285,6 +285,9 @@ possibly making them potentially no longer readable by the previous version.
 				if m.Expunged {
 					return nil
 				}
+				if mb.Expunged {
+					checkf(errors.New("mailbox is expunged but message is not"), dbpath, "message id %d is in expunged mailbox %q (id %d)", m.ID, mb.Name, mb.ID)
+				}
 				totalSize += m.Size
 
 				mp := store.MessagePath(m.ID)

webaccount/account_test.go

@@ -111,7 +111,7 @@ func TestAccount(t *testing.T) {
 	defer func() {
 		err = acc.Close()
 		tcheck(t, err, "closing account")
-		acc.CheckClosed()
+		acc.WaitClosed()
 	}()
 	defer store.Switchboard()()
 

webaccount/import.go

@@ -291,7 +291,7 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store.
 	var changes []store.Change
 
 	// ID's of delivered messages. If we have to rollback, we have to remove this files.
-	var deliveredIDs []int64
+	var newIDs []int64
 
 	sendEvent := func(kind string, v any) {
 		buf, err := json.Marshal(v)
@@ -321,7 +321,7 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store.
 	defer func() {
 		store.CloseRemoveTempFile(log, f, "uploaded messages")
 
-		for _, id := range deliveredIDs {
+		for _, id := range newIDs {
 			p := acc.MessagePath(id)
 			err := os.Remove(p)
 			log.Check(err, "closing message file after import error", slog.String("path", p))
@@ -444,6 +444,7 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store.
 
 		var p string
 		var mb *store.Mailbox
+		var parent store.Mailbox
 		for i, e := range strings.Split(name, "/") {
 			if i == 0 {
 				p = e
@@ -454,10 +455,9 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store.
 				continue
 			}
 
-			q := bstore.QueryTx[store.Mailbox](tx)
+			mb, err = acc.MailboxFind(tx, p)
-			q.FilterNonzero(store.Mailbox{Name: p})
+			ximportcheckf(err, "looking up mailbox %s to import to (aborting)", p)
-			xmb, err := q.Get()
+			if mb == nil {
-			if err == bstore.ErrAbsent {
 				uidvalidity, err := acc.NextUIDValidity(tx)
 				ximportcheckf(err, "finding next uid validity")
 
@@ -468,26 +468,24 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store.
 				}
 
 				mb = &store.Mailbox{
+					CreateSeq:   modseq,
+					ModSeq:      modseq,
+					ParentID:    parent.ID,
 					Name:        p,
 					UIDValidity: uidvalidity,
 					UIDNext:     1,
-					ModSeq:      modseq,
-					CreateSeq:   modseq,
 					HaveCounts:  true,
 					// Do not assign special-use flags. This existing account probably already has such mailboxes.
 				}
 				err = tx.Insert(mb)
 				ximportcheckf(err, "inserting mailbox in database")
+				parent = *mb
 
 				if tx.Get(&store.Subscription{Name: p}) != nil {
 					err := tx.Insert(&store.Subscription{Name: p})
 					ximportcheckf(err, "subscribing to imported mailbox")
 				}
 				changes = append(changes, store.ChangeAddMailbox{Mailbox: *mb, Flags: []string{`\Subscribed`}, ModSeq: modseq})
-			} else if err != nil {
-				ximportcheckf(err, "creating mailbox %s (aborting)", p)
-			} else {
-				mb = &xmb
 			}
 			if prevMailbox != "" && mb.Name != prevMailbox {
 				sendEvent("count", importCount{prevMailbox, messages[prevMailbox]})
@@ -559,7 +557,7 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store.
 			problemf("delivering message %s: %s (continuing)", pos, err)
 			return
 		}
-		deliveredIDs = append(deliveredIDs, m.ID)
+		newIDs = append(newIDs, m.ID)
 		changes = append(changes, m.ChangeAddUID())
 		messages[mb.Name]++
 		if messages[mb.Name]%100 == 0 || prevMailbox != mb.Name {
@@ -814,9 +812,9 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store.
 	}
 
 	// Match threads.
-	if len(deliveredIDs) > 0 {
+	if len(newIDs) > 0 {
 		sendEvent("step", importStep{"matching messages with threads"})
-		err = acc.AssignThreads(ctx, log, tx, deliveredIDs[0], 0, io.Discard)
+		err = acc.AssignThreads(ctx, log, tx, newIDs[0], 0, io.Discard)
 		ximportcheckf(err, "assigning messages to threads")
 	}
 
@@ -837,7 +835,7 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store.
 	err = tx.Commit()
 	tx = nil
 	ximportcheckf(err, "commit")
-	deliveredIDs = nil
+	newIDs = nil
 
 	if jf != nil {
 		if err := jf.Close(); err != nil {

webapisrv/server.go

@@ -21,6 +21,7 @@ import (
 	"net"
 	"net/http"
 	"net/textproto"
+	"os"
 	"reflect"
 	"runtime/debug"
 	"slices"
@@ -1058,8 +1059,15 @@ func (s server) Send(ctx context.Context, req webapi.SendRequest) (resp webapi.S
 		acc.WithRLock(func() {
 			var changes []store.Change
 
+			var sentID int64
 			metricked := false
 			defer func() {
+				if sentID != 0 {
+					p := acc.MessagePath(sentID)
+					err := os.Remove(p)
+					log.Check(err, "removing sent message file after error", slog.String("path", p))
+				}
+
 				if x := recover(); x != nil {
 					if !metricked {
 						metricServerErrors.WithLabelValues("submit").Inc()
@@ -1068,7 +1076,7 @@ func (s server) Send(ctx context.Context, req webapi.SendRequest) (resp webapi.S
 				}
 			}()
 			xdbwrite(ctx, reqInfo.Account, func(tx *bstore.Tx) {
-				sentmb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Sent", true).Get()
+				sentmb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).FilterEqual("Sent", true).Get()
 				if err == bstore.ErrAbsent {
 					// There is no mailbox designated as Sent mailbox, so we're done.
 					return
@@ -1107,12 +1115,14 @@ func (s server) Send(ctx context.Context, req webapi.SendRequest) (resp webapi.S
 					metricked = true
 				}
 				xcheckf(err, "message submitted to queue, appending message to Sent mailbox")
+				sentID = sentm.ID
 
 				err = tx.Update(&sentmb)
 				xcheckf(err, "updating mailbox")
 
 				changes = append(changes, sentm.ChangeAddUID(), sentmb.ChangeCounts())
 			})
+			sentID = 0 // Commit.
 
 			store.BroadcastChanges(acc, changes)
 		})
@@ -1190,8 +1200,12 @@ func xmessageGet(ctx context.Context, acc *store.Account, msgID int64) (store.Me
 		if err := tx.Get(&m); err == bstore.ErrAbsent || err == nil && m.Expunged {
 			panic(webapi.Error{Code: "messageNotFound", Message: "message not found"})
 		}
-		mb = store.Mailbox{ID: m.MailboxID}
+		var err error
-		return tx.Get(&mb)
+		mb, err = store.MailboxID(tx, m.MailboxID)
+		if err != nil {
+			return fmt.Errorf("get mailbox: %v", err)
+		}
+		return nil
 	})
 	xcheckf(err, "get message")
 	return m, mb

webapisrv/server_test.go

@@ -82,7 +82,7 @@ func TestServer(t *testing.T) {
 	defer func() {
 		err := acc.Close()
 		log.Check(err, "closing account")
-		acc.CheckClosed()
+		acc.WaitClosed()
 	}()
 
 	s := NewServer(100*1024, "/webapi/", false).(server)

webmail/api.go

@@ -245,7 +245,10 @@ func (Webmail) MessageFindMessageID(ctx context.Context, messageID string) (id i
 	}
 
 	xdbread(ctx, acc, func(tx *bstore.Tx) {
-		m, err := bstore.QueryTx[store.Message](tx).FilterNonzero(store.Message{MessageID: messageID}).Get()
+		q := bstore.QueryTx[store.Message](tx)
+		q.FilterEqual("Expunged", false)
+		q.FilterNonzero(store.Message{MessageID: messageID})
+		m, err := q.Get()
 		if err == bstore.ErrAbsent {
 			return
 		}
@@ -417,24 +420,27 @@ func (w Webmail) MessageCompose(ctx context.Context, m ComposeMessage, mailboxID
 	var nm store.Message
 
 	// Remove previous draft message, append message to destination mailbox.
-	acc.WithRLock(func() {
+	acc.WithWLock(func() {
 		var changes []store.Change
 
+		var newIDs []int64
+		defer func() {
+			for _, id := range newIDs {
+				p := acc.MessagePath(id)
+				err := os.Remove(p)
+				log.Check(err, "removing added message aftr error", slog.String("path", p))
+			}
+		}()
+
 		xdbwrite(ctx, acc, func(tx *bstore.Tx) {
 			var modseq store.ModSeq // Only set if needed.
 
 			if m.DraftMessageID > 0 {
 				nchanges := xops.MessageDeleteTx(ctx, log, tx, acc, []int64{m.DraftMessageID}, &modseq)
 				changes = append(changes, nchanges...)
-				// On-disk file is removed after lock.
 			}
 
-			// Find mailbox to write to.
+			mb, err := store.MailboxID(tx, mailboxID)
-			mb := store.Mailbox{ID: mailboxID}
-			err := tx.Get(&mb)
-			if err == bstore.ErrAbsent {
-				xcheckuserf(ctx, err, "looking up mailbox")
-			}
 			xcheckf(ctx, err, "looking up mailbox")
 
 			if modseq == 0 {
@@ -456,23 +462,18 @@ func (w Webmail) MessageCompose(ctx context.Context, m ComposeMessage, mailboxID
 				xcheckuserf(ctx, err, "checking quota")
 			}
 			xcheckf(ctx, err, "storing message in mailbox")
+			newIDs = append(newIDs, nm.ID)
 
 			err = tx.Update(&mb)
 			xcheckf(ctx, err, "updating sent mailbox for counts")
 
 			changes = append(changes, nm.ChangeAddUID(), mb.ChangeCounts())
 		})
+		newIDs = nil
 
 		store.BroadcastChanges(acc, changes)
 	})
 
-	// Remove on-disk file for removed draft message.
-	if m.DraftMessageID > 0 {
-		p := acc.MessagePath(m.DraftMessageID)
-		err := os.Remove(p)
-		log.Check(err, "removing draft message file")
-	}
-
 	return nm.ID
 }
 
@@ -545,9 +546,8 @@ func xmailboxID(ctx context.Context, tx *bstore.Tx, mailboxID int64) store.Mailb
 	if mailboxID == 0 {
 		xcheckuserf(ctx, errors.New("invalid zero mailbox ID"), "getting mailbox")
 	}
-	mb := store.Mailbox{ID: mailboxID}
+	mb, err := store.MailboxID(tx, mailboxID)
-	err := tx.Get(&mb)
+	if err == bstore.ErrAbsent || err == store.ErrMailboxExpunged {
-	if err == bstore.ErrAbsent {
 		xcheckuserf(ctx, err, "getting mailbox")
 	}
 	xcheckf(ctx, err, "getting mailbox")
@@ -1010,7 +1010,7 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
 
 	// Append message to Sent mailbox, mark original messages as answered/forwarded,
 	// remove any draft message.
-	acc.WithRLock(func() {
+	acc.WithWLock(func() {
 		var changes []store.Change
 
 		metricked := false
@@ -1023,9 +1023,9 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
 			}
 		}()
 
-		var deliveredIDs []int64
+		var newIDs []int64
 		defer func() {
-			for _, id := range deliveredIDs {
+			for _, id := range newIDs {
 				p := acc.MessagePath(id)
 				err := os.Remove(p)
 				log.Check(err, "removing delivered message on error", slog.String("path", p))
@@ -1036,7 +1036,6 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
 			if m.DraftMessageID > 0 {
 				nchanges := xops.MessageDeleteTx(ctx, log, tx, acc, []int64{m.DraftMessageID}, &modseq)
 				changes = append(changes, nchanges...)
-				// On-disk file is removed after lock.
 			}
 
 			if m.ResponseMessageID > 0 {
@@ -1061,12 +1060,11 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
 					changes = append(changes, rm.ChangeFlags(oflags))
 
 					// Update modseq of mailbox of replied/forwarded message.
-					rmb := store.Mailbox{ID: rm.MailboxID}
+					rmb, err := store.MailboxID(tx, rm.MailboxID)
-					err = tx.Get(&rmb)
 					xcheckf(ctx, err, "get mailbox of replied/forwarded message for modseq update")
 					rmb.ModSeq = modseq
 					err = tx.Update(&rmb)
-					xcheckf(ctx, err, "update modseqo of mailbox of replied/forwarded message")
+					xcheckf(ctx, err, "update modseq of mailbox of replied/forwarded message")
 
 					err = acc.RetrainMessages(ctx, log, tx, []store.Message{rm})
 					xcheckf(ctx, err, "retraining messages after reply/forward")
@@ -1075,8 +1073,8 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
 				// Move messages from this thread still in this mailbox to the designated Archive
 				// mailbox.
 				if m.ArchiveThread {
-					mbArchive, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Archive", true).Get()
+					mbArchive, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).FilterEqual("Archive", true).Get()
-					if err == bstore.ErrAbsent {
+					if err == bstore.ErrAbsent || err == store.ErrMailboxExpunged {
 						xcheckuserf(ctx, errors.New("not configured"), "looking up designated archive mailbox")
 					}
 					xcheckf(ctx, err, "looking up designated archive mailbox")
@@ -1088,14 +1086,15 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
 					err = q.IDs(&msgIDs)
 					xcheckf(ctx, err, "listing messages in thread to archive")
 					if len(msgIDs) > 0 {
-						nchanges := xops.MessageMoveTx(ctx, log, acc, tx, msgIDs, mbArchive, &modseq)
+						ids, nchanges := xops.MessageMoveTx(ctx, log, acc, tx, msgIDs, mbArchive, &modseq)
+						newIDs = append(newIDs, ids...)
 						changes = append(changes, nchanges...)
 					}
 				}
 			}
 
-			sentmb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Sent", true).Get()
+			sentmb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).FilterEqual("Sent", true).Get()
-			if err == bstore.ErrAbsent {
+			if err == bstore.ErrAbsent || err == store.ErrMailboxExpunged {
 				// There is no mailbox designated as Sent mailbox, so we're done.
 				return
 			}
@@ -1135,24 +1134,17 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
 				metricked = true
 			}
 			xcheckf(ctx, err, "message submitted to queue, appending message to Sent mailbox")
-			deliveredIDs = append(deliveredIDs, sentm.ID)
+			newIDs = append(newIDs, sentm.ID)
 
 			err = tx.Update(&sentmb)
 			xcheckf(ctx, err, "updating sent mailbox for counts")
 
 			changes = append(changes, sentm.ChangeAddUID(), sentmb.ChangeCounts())
 		})
-		deliveredIDs = nil
+		newIDs = nil
 
 		store.BroadcastChanges(acc, changes)
 	})
-
-	// Remove on-disk file for removed draft message.
-	if m.DraftMessageID > 0 {
-		p := acc.MessagePath(m.DraftMessageID)
-		err := os.Remove(p)
-		log.Check(err, "removing draft message file")
-	}
 }
 
 // MessageMove moves messages to another mailbox. If the message is already in
@@ -1244,9 +1236,6 @@ func (Webmail) MailboxDelete(ctx context.Context, mailboxID int64) {
 	acc := reqInfo.Account
 	log := reqInfo.Log
 
-	// Messages to remove after having broadcasted the removal of messages.
-	var removeMessageIDs []int64
-
 	acc.WithWLock(func() {
 		var changes []store.Change
 
@@ -1259,7 +1248,7 @@ func (Webmail) MailboxDelete(ctx context.Context, mailboxID int64) {
 
 			var hasChildren bool
 			var err error
-			changes, removeMessageIDs, hasChildren, err = acc.MailboxDelete(ctx, log, tx, mb)
+			changes, hasChildren, err = acc.MailboxDelete(ctx, log, tx, &mb)
 			if hasChildren {
 				xcheckuserf(ctx, errors.New("mailbox has children"), "deleting mailbox")
 			}
@@ -1268,12 +1257,6 @@ func (Webmail) MailboxDelete(ctx context.Context, mailboxID int64) {
 
 		store.BroadcastChanges(acc, changes)
 	})
-
-	for _, mID := range removeMessageIDs {
-		p := acc.MessagePath(mID)
-		err := os.Remove(p)
-		log.Check(err, "removing message file for mailbox delete", slog.String("path", p))
-	}
 }
 
 // MailboxEmpty empties a mailbox, removing all messages from the mailbox, but not
@@ -1283,76 +1266,36 @@ func (Webmail) MailboxEmpty(ctx context.Context, mailboxID int64) {
 	acc := reqInfo.Account
 	log := reqInfo.Log
 
-	var expunged []store.Message
-
 	acc.WithWLock(func() {
 		var changes []store.Change
 
 		xdbwrite(ctx, acc, func(tx *bstore.Tx) {
 			mb := xmailboxID(ctx, tx, mailboxID)
 
-			modseq, err := acc.NextModSeq(tx)
-			xcheckf(ctx, err, "next modseq")
-
-			// Mark messages as expunged.
 			qm := bstore.QueryTx[store.Message](tx)
 			qm.FilterNonzero(store.Message{MailboxID: mb.ID})
 			qm.FilterEqual("Expunged", false)
 			qm.SortAsc("UID")
-			qm.Gather(&expunged)
+			l, err := qm.List()
-			n, err := qm.UpdateNonzero(store.Message{ModSeq: modseq, Expunged: true})
+			xcheckf(ctx, err, "listing messages to remove")
-			xcheckf(ctx, err, "deleting messages")
 
-			if n == 0 {
+			if len(l) == 0 {
 				xcheckuserf(ctx, errors.New("no messages in mailbox"), "emptying mailbox")
 			}
 
-			mb.ModSeq = modseq
+			modseq, err := acc.NextModSeq(tx)
+			xcheckf(ctx, err, "next modseq")
 
-			// Remove Recipients.
+			chrem, chmbcounts, err := acc.MessageRemove(log, tx, modseq, &mb, store.RemoveOpts{}, l...)
-			anyIDs := make([]any, len(expunged))
+			xcheckf(ctx, err, "expunge messages")
-			for i, m := range expunged {
+			changes = append(changes, chrem, chmbcounts)
-				anyIDs[i] = m.ID
-			}
-			qmr := bstore.QueryTx[store.Recipient](tx)
-			qmr.FilterEqual("MessageID", anyIDs...)
-			_, err = qmr.Delete()
-			xcheckf(ctx, err, "removing message recipients")
-
-			// Adjust mailbox counts, gather UIDs for broadcasted change, prepare for untraining.
-			var totalSize int64
-			uids := make([]store.UID, len(expunged))
-			for i, m := range expunged {
-				m.Expunged = false // Gather returns updated values.
-				mb.Sub(m.MailboxCounts())
-				totalSize += m.Size
-				uids[i] = m.UID
-
-				expunged[i].Junk = false
-				expunged[i].Notjunk = false
-			}
 
 			err = tx.Update(&mb)
 			xcheckf(ctx, err, "updating mailbox for counts")
-
-			err = acc.AddMessageSize(log, tx, -totalSize)
-			xcheckf(ctx, err, "updating disk usage")
-
-			err = acc.RetrainMessages(ctx, log, tx, expunged)
-			xcheckf(ctx, err, "retraining expunged messages")
-
-			chremove := store.ChangeRemoveUIDs{MailboxID: mb.ID, UIDs: uids, ModSeq: modseq}
-			changes = []store.Change{chremove, mb.ChangeCounts()}
 		})
 
 		store.BroadcastChanges(acc, changes)
 	})
-
-	for _, m := range expunged {
-		p := acc.MessagePath(m.ID)
-		err := os.Remove(p)
-		log.Check(err, "removing message file after emptying mailbox", slog.String("path", p))
-	}
 }
 
 // MailboxRename renames a mailbox, possibly moving it to a new parent. The mailbox
@@ -1373,10 +1316,10 @@ func (Webmail) MailboxRename(ctx context.Context, mailboxID int64, newName strin
 		xdbwrite(ctx, acc, func(tx *bstore.Tx) {
 			mbsrc := xmailboxID(ctx, tx, mailboxID)
 			var err error
-			var isInbox, notExists, alreadyExists bool
+			var isInbox, alreadyExists bool
 			var modseq store.ModSeq
-			changes, isInbox, notExists, alreadyExists, err = acc.MailboxRename(tx, mbsrc, newName, &modseq)
+			changes, isInbox, alreadyExists, err = acc.MailboxRename(tx, &mbsrc, newName, &modseq)
-			if isInbox || notExists || alreadyExists {
+			if isInbox || alreadyExists {
 				xcheckuserf(ctx, err, "renaming mailbox")
 			}
 			xcheckf(ctx, err, "renaming mailbox")
@@ -1555,8 +1498,8 @@ func (Webmail) ThreadCollapse(ctx context.Context, messageIDs []int64, collapse
 			for _, id := range messageIDs {
 				m := store.Message{ID: id}
 				err := tx.Get(&m)
-				if err == bstore.ErrAbsent {
+				if err == bstore.ErrAbsent || err == nil && m.Expunged {
-					xcheckuserf(ctx, err, "get message")
+					xcheckuserf(ctx, bstore.ErrAbsent, "get message")
 				}
 				xcheckf(ctx, err, "get message")
 				threadIDs[m.ThreadID] = struct{}{}
@@ -1565,6 +1508,7 @@ func (Webmail) ThreadCollapse(ctx context.Context, messageIDs []int64, collapse
 
 			var updated []store.Message
 			q := bstore.QueryTx[store.Message](tx)
+			q.FilterEqual("Expunged", false)
 			q.FilterEqual("ThreadID", slicesAny(maps.Keys(threadIDs))...)
 			q.FilterNotEqual("ThreadCollapsed", collapse)
 			q.FilterFn(func(tm store.Message) bool {
@@ -1607,8 +1551,8 @@ func (Webmail) ThreadMute(ctx context.Context, messageIDs []int64, mute bool) {
 			for _, id := range messageIDs {
 				m := store.Message{ID: id}
 				err := tx.Get(&m)
-				if err == bstore.ErrAbsent {
+				if err == bstore.ErrAbsent || err == nil && m.Expunged {
-					xcheckuserf(ctx, err, "get message")
+					xcheckuserf(ctx, bstore.ErrAbsent, "get message")
 				}
 				xcheckf(ctx, err, "get message")
 				threadIDs[m.ThreadID] = struct{}{}
@@ -1618,6 +1562,7 @@ func (Webmail) ThreadMute(ctx context.Context, messageIDs []int64, mute bool) {
 			var updated []store.Message
 
 			q := bstore.QueryTx[store.Message](tx)
+			q.FilterEqual("Expunged", false)
 			q.FilterEqual("ThreadID", slicesAny(maps.Keys(threadIDs))...)
 			q.FilterFn(func(tm store.Message) bool {
 				if tm.ThreadMuted == mute && (!mute || tm.ThreadCollapsed) {

webmail/api.json

@@ -1601,9 +1601,37 @@
 						"int64"
 					]
 				},
+				{
+					"Name": "CreateSeq",
+					"Docs": "",
+					"Typewords": [
+						"ModSeq"
+					]
+				},
+				{
+					"Name": "ModSeq",
+					"Docs": "Of last change, or when deleted.",
+					"Typewords": [
+						"ModSeq"
+					]
+				},
+				{
+					"Name": "Expunged",
+					"Docs": "",
+					"Typewords": [
+						"bool"
+					]
+				},
+				{
+					"Name": "ParentID",
+					"Docs": "Zero for top-level mailbox.",
+					"Typewords": [
+						"int64"
+					]
+				},
 				{
 					"Name": "Name",
-					"Docs": "\"Inbox\" is the name for the special IMAP \"INBOX\". Slash separated for hierarchy.",
+					"Docs": "\"Inbox\" is the name for the special IMAP \"INBOX\". Slash separated for hierarchy. Names must be unique for mailboxes that are not expunged.",
 					"Typewords": [
 						"string"
 					]
@@ -1665,20 +1693,6 @@
 						"string"
 					]
 				},
-				{
-					"Name": "ModSeq",
-					"Docs": "ModSeq matches that of last message (including deleted), or changes to mailbox such as after metadata changes.",
-					"Typewords": [
-						"ModSeq"
-					]
-				},
-				{
-					"Name": "CreateSeq",
-					"Docs": "",
-					"Typewords": [
-						"ModSeq"
-					]
-				},
 				{
 					"Name": "HaveCounts",
 					"Docs": "Whether MailboxCounts have been initialized.",
@@ -2165,14 +2179,14 @@
 			"Fields": [
 				{
 					"Name": "ID",
-					"Docs": "ID, unchanged over lifetime, determines path to on-disk msg file. Set during deliver.",
+					"Docs": "ID of the message, determines path to on-disk message file. Set when adding to a mailbox. When a message is moved to another mailbox, the mailbox ID is changed, but for synchronization purposes, a new Message record is inserted (which gets a new ID) with the Expunged field set and the MailboxID and UID copied.",
 					"Typewords": [
 						"int64"
 					]
 				},
 				{
 					"Name": "UID",
-					"Docs": "UID, for IMAP. Set during deliver.",
+					"Docs": "UID, for IMAP. Set when adding to mailbox. Strictly increasing values, per mailbox. The UID of a message can never change (though messages can be copied), and the contents of a message/UID also never changes.",
 					"Typewords": [
 						"UID"
 					]
@@ -2435,7 +2449,7 @@
 				},
 				{
 					"Name": "ThreadParentIDs",
-					"Docs": "IDs of parent messages, from closest parent to the root message. Parent messages may be in a different mailbox, or may no longer exist. ThreadParentIDs must never contain the message id itself (a cycle), and parent messages must reference the same ancestors.",
+					"Docs": "IDs of parent messages, from closest parent to the root message. Parent messages may be in a different mailbox, or may no longer exist. ThreadParentIDs must never contain the message id itself (a cycle), and parent messages must reference the same ancestors. Moving a message to another mailbox keeps the message ID and changes the MailboxID (and UID) of the message, leaving threading parent ids intact.",
 					"Typewords": [
 						"[]",
 						"int64"
@@ -2891,6 +2905,14 @@
 					"Typewords": [
 						"ModSeq"
 					]
+				},
+				{
+					"Name": "MsgIDs",
+					"Docs": "Message.ID, for erasing, order does not necessarily correspond with UIDs!",
+					"Typewords": [
+						"[]",
+						"int64"
+					]
 				}
 			]
 		},
@@ -3214,13 +3236,13 @@
 	],
 	"Ints": [
 		{
-			"Name": "UID",
+			"Name": "ModSeq",
-			"Docs": "IMAP UID.",
+			"Docs": "ModSeq represents a modseq as stored in the database. ModSeq 0 in the\ndatabase is sent to the client as 1, because modseq 0 is special in IMAP.\nModSeq coming from the client are of type int64.",
 			"Values": null
 		},
 		{
-			"Name": "ModSeq",
+			"Name": "UID",
-			"Docs": "ModSeq represents a modseq as stored in the database. ModSeq 0 in the\ndatabase is sent to the client as 1, because modseq 0 is special in IMAP.\nModSeq coming from the client are of type int64.",
+			"Docs": "IMAP UID.",
 			"Values": null
 		},
 		{

webmail/api.ts

@@ -188,7 +188,11 @@ export interface ForwardAttachments {
 // Mailbox is collection of messages, e.g. Inbox or Sent.
 export interface Mailbox {
 	ID: number
-	Name: string  // "Inbox" is the name for the special IMAP "INBOX". Slash separated for hierarchy.
+	CreateSeq: ModSeq
+	ModSeq: ModSeq  // Of last change, or when deleted.
+	Expunged: boolean
+	ParentID: number  // Zero for top-level mailbox.
+	Name: string  // "Inbox" is the name for the special IMAP "INBOX". Slash separated for hierarchy. Names must be unique for mailboxes that are not expunged.
 	UIDValidity: number  // If UIDs are invalidated, e.g. when renaming a mailbox to a previously existing name, UIDValidity must be changed. Used by IMAP for synchronization.
 	UIDNext: UID  // UID likely to be assigned to next message. Used by IMAP to detect messages delivered to a mailbox.
 	Archive: boolean
@@ -197,8 +201,6 @@ export interface Mailbox {
 	Sent: boolean
 	Trash: boolean
 	Keywords?: string[] | null  // Keywords as used in messages. Storing a non-system keyword for a message automatically adds it to this list. Used in the IMAP FLAGS response. Only "atoms" are allowed (IMAP syntax), keywords are case-insensitive, only stored in lower case (for JMAP), sorted.
-	ModSeq: ModSeq  // ModSeq matches that of last message (including deleted), or changes to mailbox such as after metadata changes.
-	CreateSeq: ModSeq
 	HaveCounts: boolean  // Whether MailboxCounts have been initialized.
 	Total: number  // Total number of messages, excluding \Deleted. For JMAP.
 	Deleted: number  // Number of messages with \Deleted flag. Used for IMAP message count that includes messages with \Deleted.
@@ -315,8 +317,8 @@ export interface MessageItem {
 // Messages always have a header section, even if empty. Incoming messages without
 // header section must get an empty header section added before inserting.
 export interface Message {
-	ID: number  // ID, unchanged over lifetime, determines path to on-disk msg file. Set during deliver.
+	ID: number  // ID of the message, determines path to on-disk message file. Set when adding to a mailbox. When a message is moved to another mailbox, the mailbox ID is changed, but for synchronization purposes, a new Message record is inserted (which gets a new ID) with the Expunged field set and the MailboxID and UID copied.
-	UID: UID  // UID, for IMAP. Set during deliver.
+	UID: UID  // UID, for IMAP. Set when adding to mailbox. Strictly increasing values, per mailbox. The UID of a message can never change (though messages can be copied), and the contents of a message/UID also never changes.
 	MailboxID: number
 	ModSeq: ModSeq  // Modification sequence, for faster syncing with IMAP QRESYNC and JMAP. ModSeq is the last modification. CreateSeq is the Seq the message was inserted, always <= ModSeq. If Expunged is set, the message has been removed and should not be returned to the user. In this case, ModSeq is the Seq where the message is removed, and will never be changed again. We have an index on both ModSeq (for JMAP that synchronizes per account) and MailboxID+ModSeq (for IMAP that synchronizes per mailbox). The index on CreateSeq helps efficiently finding created messages for JMAP. The value of ModSeq is special for IMAP. Messages that existed before ModSeq was added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If we get modseq 1 from a client, the IMAP server will translate it to 0. When we return modseq to clients, we turn 0 into 1.
 	CreateSeq: ModSeq
@@ -353,7 +355,7 @@ export interface Message {
 	SubjectBase: string  // For matching threads in case there is no References/In-Reply-To header. It is lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.
 	MessageHash?: string | null  // Hash of message. For rejects delivery in case there is no Message-ID, only set when delivered as reject.
 	ThreadID: number  // ID of message starting this thread.
-	ThreadParentIDs?: number[] | null  // IDs of parent messages, from closest parent to the root message. Parent messages may be in a different mailbox, or may no longer exist. ThreadParentIDs must never contain the message id itself (a cycle), and parent messages must reference the same ancestors.
+	ThreadParentIDs?: number[] | null  // IDs of parent messages, from closest parent to the root message. Parent messages may be in a different mailbox, or may no longer exist. ThreadParentIDs must never contain the message id itself (a cycle), and parent messages must reference the same ancestors. Moving a message to another mailbox keeps the message ID and changes the MailboxID (and UID) of the message, leaving threading parent ids intact.
 	ThreadMissingLink: boolean  // ThreadMissingLink is true if there is no match with a direct parent. E.g. first ID in ThreadParentIDs is not the direct ancestor (an intermediate message may have been deleted), or subject-based matching was done.
 	ThreadMuted: boolean  // If set, newly delivered child messages are automatically marked as read. This field is copied to new child messages. Changes are propagated to the webmail client.
 	ThreadCollapsed: boolean  // If set, this (sub)thread is collapsed in the webmail client, for threading mode "on" (mode "unread" ignores it). This field is copied to new child message. Changes are propagated to the webmail client.
@@ -439,6 +441,7 @@ export interface ChangeMsgRemove {
 	MailboxID: number
 	UIDs?: UID[] | null  // Must be in increasing UID order, for IMAP.
 	ModSeq: ModSeq
+	MsgIDs?: number[] | null  // Message.ID, for erasing, order does not necessarily correspond with UIDs!
 }
 
 // ChangeMsgFlags updates flags for one message.
@@ -517,14 +520,14 @@ export interface ChangeMailboxKeywords {
 	Keywords?: string[] | null
 }
 
-// IMAP UID.
-export type UID = number
-
 // ModSeq represents a modseq as stored in the database. ModSeq 0 in the
 // database is sent to the client as 1, because modseq 0 is special in IMAP.
 // ModSeq coming from the client are of type int64.
 export type ModSeq = number
 
+// IMAP UID.
+export type UID = number
+
 // Validation of "message From" domain.
 export enum Validation {
 	ValidationUnknown = 0,
@@ -613,7 +616,7 @@ export const types: TypenameMap = {
 	"SubmitMessage": {"Name":"SubmitMessage","Docs":"","Fields":[{"Name":"From","Docs":"","Typewords":["string"]},{"Name":"To","Docs":"","Typewords":["[]","string"]},{"Name":"Cc","Docs":"","Typewords":["[]","string"]},{"Name":"Bcc","Docs":"","Typewords":["[]","string"]},{"Name":"ReplyTo","Docs":"","Typewords":["string"]},{"Name":"Subject","Docs":"","Typewords":["string"]},{"Name":"TextBody","Docs":"","Typewords":["string"]},{"Name":"Attachments","Docs":"","Typewords":["[]","File"]},{"Name":"ForwardAttachments","Docs":"","Typewords":["ForwardAttachments"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"ResponseMessageID","Docs":"","Typewords":["int64"]},{"Name":"UserAgent","Docs":"","Typewords":["string"]},{"Name":"RequireTLS","Docs":"","Typewords":["nullable","bool"]},{"Name":"FutureRelease","Docs":"","Typewords":["nullable","timestamp"]},{"Name":"ArchiveThread","Docs":"","Typewords":["bool"]},{"Name":"ArchiveReferenceMailboxID","Docs":"","Typewords":["int64"]},{"Name":"DraftMessageID","Docs":"","Typewords":["int64"]}]},
 	"File": {"Name":"File","Docs":"","Fields":[{"Name":"Filename","Docs":"","Typewords":["string"]},{"Name":"DataURI","Docs":"","Typewords":["string"]}]},
 	"ForwardAttachments": {"Name":"ForwardAttachments","Docs":"","Fields":[{"Name":"MessageID","Docs":"","Typewords":["int64"]},{"Name":"Paths","Docs":"","Typewords":["[]","[]","int32"]}]},
-	"Mailbox": {"Name":"Mailbox","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"Name","Docs":"","Typewords":["string"]},{"Name":"UIDValidity","Docs":"","Typewords":["uint32"]},{"Name":"UIDNext","Docs":"","Typewords":["UID"]},{"Name":"Archive","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Sent","Docs":"","Typewords":["bool"]},{"Name":"Trash","Docs":"","Typewords":["bool"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"CreateSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"HaveCounts","Docs":"","Typewords":["bool"]},{"Name":"Total","Docs":"","Typewords":["int64"]},{"Name":"Deleted","Docs":"","Typewords":["int64"]},{"Name":"Unread","Docs":"","Typewords":["int64"]},{"Name":"Unseen","Docs":"","Typewords":["int64"]},{"Name":"Size","Docs":"","Typewords":["int64"]}]},
+	"Mailbox": {"Name":"Mailbox","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"CreateSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Expunged","Docs":"","Typewords":["bool"]},{"Name":"ParentID","Docs":"","Typewords":["int64"]},{"Name":"Name","Docs":"","Typewords":["string"]},{"Name":"UIDValidity","Docs":"","Typewords":["uint32"]},{"Name":"UIDNext","Docs":"","Typewords":["UID"]},{"Name":"Archive","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Sent","Docs":"","Typewords":["bool"]},{"Name":"Trash","Docs":"","Typewords":["bool"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"HaveCounts","Docs":"","Typewords":["bool"]},{"Name":"Total","Docs":"","Typewords":["int64"]},{"Name":"Deleted","Docs":"","Typewords":["int64"]},{"Name":"Unread","Docs":"","Typewords":["int64"]},{"Name":"Unseen","Docs":"","Typewords":["int64"]},{"Name":"Size","Docs":"","Typewords":["int64"]}]},
 	"RecipientSecurity": {"Name":"RecipientSecurity","Docs":"","Fields":[{"Name":"STARTTLS","Docs":"","Typewords":["SecurityResult"]},{"Name":"MTASTS","Docs":"","Typewords":["SecurityResult"]},{"Name":"DNSSEC","Docs":"","Typewords":["SecurityResult"]},{"Name":"DANE","Docs":"","Typewords":["SecurityResult"]},{"Name":"RequireTLS","Docs":"","Typewords":["SecurityResult"]}]},
 	"Settings": {"Name":"Settings","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["uint8"]},{"Name":"Signature","Docs":"","Typewords":["string"]},{"Name":"Quoting","Docs":"","Typewords":["Quoting"]},{"Name":"ShowAddressSecurity","Docs":"","Typewords":["bool"]},{"Name":"ShowHTML","Docs":"","Typewords":["bool"]},{"Name":"NoShowShortcuts","Docs":"","Typewords":["bool"]},{"Name":"ShowHeaders","Docs":"","Typewords":["[]","string"]}]},
 	"Ruleset": {"Name":"Ruleset","Docs":"","Fields":[{"Name":"SMTPMailFromRegexp","Docs":"","Typewords":["string"]},{"Name":"MsgFromRegexp","Docs":"","Typewords":["string"]},{"Name":"VerifiedDomain","Docs":"","Typewords":["string"]},{"Name":"HeadersRegexp","Docs":"","Typewords":["{}","string"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"ListAllowDomain","Docs":"","Typewords":["string"]},{"Name":"AcceptRejectsToMailbox","Docs":"","Typewords":["string"]},{"Name":"Mailbox","Docs":"","Typewords":["string"]},{"Name":"Comment","Docs":"","Typewords":["string"]},{"Name":"VerifiedDNSDomain","Docs":"","Typewords":["Domain"]},{"Name":"ListAllowDNSDomain","Docs":"","Typewords":["Domain"]}]},
@@ -629,7 +632,7 @@ export const types: TypenameMap = {
 	"EventViewChanges": {"Name":"EventViewChanges","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"Changes","Docs":"","Typewords":["[]","[]","any"]}]},
 	"ChangeMsgAdd": {"Name":"ChangeMsgAdd","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Flags","Docs":"","Typewords":["Flags"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"MessageItems","Docs":"","Typewords":["[]","MessageItem"]}]},
 	"Flags": {"Name":"Flags","Docs":"","Fields":[{"Name":"Seen","Docs":"","Typewords":["bool"]},{"Name":"Answered","Docs":"","Typewords":["bool"]},{"Name":"Flagged","Docs":"","Typewords":["bool"]},{"Name":"Forwarded","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Notjunk","Docs":"","Typewords":["bool"]},{"Name":"Deleted","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Phishing","Docs":"","Typewords":["bool"]},{"Name":"MDNSent","Docs":"","Typewords":["bool"]}]},
-	"ChangeMsgRemove": {"Name":"ChangeMsgRemove","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UIDs","Docs":"","Typewords":["[]","UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]}]},
+	"ChangeMsgRemove": {"Name":"ChangeMsgRemove","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UIDs","Docs":"","Typewords":["[]","UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"MsgIDs","Docs":"","Typewords":["[]","int64"]}]},
 	"ChangeMsgFlags": {"Name":"ChangeMsgFlags","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Mask","Docs":"","Typewords":["Flags"]},{"Name":"Flags","Docs":"","Typewords":["Flags"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]}]},
 	"ChangeMsgThread": {"Name":"ChangeMsgThread","Docs":"","Fields":[{"Name":"MessageIDs","Docs":"","Typewords":["[]","int64"]},{"Name":"Muted","Docs":"","Typewords":["bool"]},{"Name":"Collapsed","Docs":"","Typewords":["bool"]}]},
 	"ChangeMailboxRemove": {"Name":"ChangeMailboxRemove","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"Name","Docs":"","Typewords":["string"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]}]},
@@ -639,8 +642,8 @@ export const types: TypenameMap = {
 	"ChangeMailboxSpecialUse": {"Name":"ChangeMailboxSpecialUse","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"MailboxName","Docs":"","Typewords":["string"]},{"Name":"SpecialUse","Docs":"","Typewords":["SpecialUse"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]}]},
 	"SpecialUse": {"Name":"SpecialUse","Docs":"","Fields":[{"Name":"Archive","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Sent","Docs":"","Typewords":["bool"]},{"Name":"Trash","Docs":"","Typewords":["bool"]}]},
 	"ChangeMailboxKeywords": {"Name":"ChangeMailboxKeywords","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"MailboxName","Docs":"","Typewords":["string"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]}]},
-	"UID": {"Name":"UID","Docs":"","Values":null},
 	"ModSeq": {"Name":"ModSeq","Docs":"","Values":null},
+	"UID": {"Name":"UID","Docs":"","Values":null},
 	"Validation": {"Name":"Validation","Docs":"","Values":[{"Name":"ValidationUnknown","Value":0,"Docs":""},{"Name":"ValidationStrict","Value":1,"Docs":""},{"Name":"ValidationDMARC","Value":2,"Docs":""},{"Name":"ValidationRelaxed","Value":3,"Docs":""},{"Name":"ValidationPass","Value":4,"Docs":""},{"Name":"ValidationNeutral","Value":5,"Docs":""},{"Name":"ValidationTemperror","Value":6,"Docs":""},{"Name":"ValidationPermerror","Value":7,"Docs":""},{"Name":"ValidationFail","Value":8,"Docs":""},{"Name":"ValidationSoftfail","Value":9,"Docs":""},{"Name":"ValidationNone","Value":10,"Docs":""}]},
 	"CSRFToken": {"Name":"CSRFToken","Docs":"","Values":null},
 	"ThreadMode": {"Name":"ThreadMode","Docs":"","Values":[{"Name":"ThreadOff","Value":"off","Docs":""},{"Name":"ThreadOn","Value":"on","Docs":""},{"Name":"ThreadUnread","Value":"unread","Docs":""}]},
@@ -694,8 +697,8 @@ export const parser = {
 	ChangeMailboxSpecialUse: (v: any) => parse("ChangeMailboxSpecialUse", v) as ChangeMailboxSpecialUse,
 	SpecialUse: (v: any) => parse("SpecialUse", v) as SpecialUse,
 	ChangeMailboxKeywords: (v: any) => parse("ChangeMailboxKeywords", v) as ChangeMailboxKeywords,
-	UID: (v: any) => parse("UID", v) as UID,
 	ModSeq: (v: any) => parse("ModSeq", v) as ModSeq,
+	UID: (v: any) => parse("UID", v) as UID,
 	Validation: (v: any) => parse("Validation", v) as Validation,
 	CSRFToken: (v: any) => parse("CSRFToken", v) as CSRFToken,
 	ThreadMode: (v: any) => parse("ThreadMode", v) as ThreadMode,

webmail/api_test.go

@@ -78,7 +78,7 @@ func TestAPI(t *testing.T) {
 		tcheck(t, err, "mtastsdb close")
 		err = acc.Close()
 		pkglog.Check(err, "closing account")
-		acc.CheckClosed()
+		acc.WaitClosed()
 	}()
 
 	var zerom store.Message
@@ -206,7 +206,7 @@ func TestAPI(t *testing.T) {
 	var inbox, archive, sent, drafts, testbox1 store.Mailbox
 	err = acc.DB.Read(ctx, func(tx *bstore.Tx) error {
 		get := func(k string, v any) store.Mailbox {
-			mb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual(k, v).Get()
+			mb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).FilterEqual(k, v).Get()
 			tcheck(t, err, "get special-use mailbox")
 			return mb
 		}
@@ -276,7 +276,7 @@ func TestAPI(t *testing.T) {
 	tneedError(t, func() { api.ParsedMessage(ctx, testbox1Alt.ID) }) // Message was removed and no longer exists.
 
 	api.MailboxCreate(ctx, "Testbox1")
-	testbox1, err = bstore.QueryDB[store.Mailbox](ctx, acc.DB).FilterEqual("Name", "Testbox1").Get()
+	testbox1, err = bstore.QueryDB[store.Mailbox](ctx, acc.DB).FilterEqual("Expunged", false).FilterEqual("Name", "Testbox1").Get()
 	tcheck(t, err, "get testbox1")
 	tdeliver(t, acc, testbox1Alt)
 

webmail/msg.js

@@ -309,7 +309,7 @@ var api;
 		"SubmitMessage": { "Name": "SubmitMessage", "Docs": "", "Fields": [{ "Name": "From", "Docs": "", "Typewords": ["string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Cc", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Bcc", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "TextBody", "Docs": "", "Typewords": ["string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "File"] }, { "Name": "ForwardAttachments", "Docs": "", "Typewords": ["ForwardAttachments"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ResponseMessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UserAgent", "Docs": "", "Typewords": ["string"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "FutureRelease", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "ArchiveThread", "Docs": "", "Typewords": ["bool"] }, { "Name": "ArchiveReferenceMailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "DraftMessageID", "Docs": "", "Typewords": ["int64"] }] },
 		"File": { "Name": "File", "Docs": "", "Fields": [{ "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "DataURI", "Docs": "", "Typewords": ["string"] }] },
 		"ForwardAttachments": { "Name": "ForwardAttachments", "Docs": "", "Fields": [{ "Name": "MessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Paths", "Docs": "", "Typewords": ["[]", "[]", "int32"] }] },
-		"Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] },
+		"Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "ParentID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] },
 		"RecipientSecurity": { "Name": "RecipientSecurity", "Docs": "", "Fields": [{ "Name": "STARTTLS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "MTASTS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DNSSEC", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DANE", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["SecurityResult"] }] },
 		"Settings": { "Name": "Settings", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["uint8"] }, { "Name": "Signature", "Docs": "", "Typewords": ["string"] }, { "Name": "Quoting", "Docs": "", "Typewords": ["Quoting"] }, { "Name": "ShowAddressSecurity", "Docs": "", "Typewords": ["bool"] }, { "Name": "ShowHTML", "Docs": "", "Typewords": ["bool"] }, { "Name": "NoShowShortcuts", "Docs": "", "Typewords": ["bool"] }, { "Name": "ShowHeaders", "Docs": "", "Typewords": ["[]", "string"] }] },
 		"Ruleset": { "Name": "Ruleset", "Docs": "", "Fields": [{ "Name": "SMTPMailFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "HeadersRegexp", "Docs": "", "Typewords": ["{}", "string"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ListAllowDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "AcceptRejectsToMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Comment", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDNSDomain", "Docs": "", "Typewords": ["Domain"] }, { "Name": "ListAllowDNSDomain", "Docs": "", "Typewords": ["Domain"] }] },
@@ -325,7 +325,7 @@ var api;
 		"EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] },
 		"ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] },
 		"Flags": { "Name": "Flags", "Docs": "", "Fields": [{ "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }] },
-		"ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
+		"ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "MsgIDs", "Docs": "", "Typewords": ["[]", "int64"] }] },
 		"ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] },
 		"ChangeMsgThread": { "Name": "ChangeMsgThread", "Docs": "", "Fields": [{ "Name": "MessageIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "Muted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Collapsed", "Docs": "", "Typewords": ["bool"] }] },
 		"ChangeMailboxRemove": { "Name": "ChangeMailboxRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
@@ -335,8 +335,8 @@ var api;
 		"ChangeMailboxSpecialUse": { "Name": "ChangeMailboxSpecialUse", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "SpecialUse", "Docs": "", "Typewords": ["SpecialUse"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
 		"SpecialUse": { "Name": "SpecialUse", "Docs": "", "Fields": [{ "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }] },
 		"ChangeMailboxKeywords": { "Name": "ChangeMailboxKeywords", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] },
-		"UID": { "Name": "UID", "Docs": "", "Values": null },
 		"ModSeq": { "Name": "ModSeq", "Docs": "", "Values": null },
+		"UID": { "Name": "UID", "Docs": "", "Values": null },
 		"Validation": { "Name": "Validation", "Docs": "", "Values": [{ "Name": "ValidationUnknown", "Value": 0, "Docs": "" }, { "Name": "ValidationStrict", "Value": 1, "Docs": "" }, { "Name": "ValidationDMARC", "Value": 2, "Docs": "" }, { "Name": "ValidationRelaxed", "Value": 3, "Docs": "" }, { "Name": "ValidationPass", "Value": 4, "Docs": "" }, { "Name": "ValidationNeutral", "Value": 5, "Docs": "" }, { "Name": "ValidationTemperror", "Value": 6, "Docs": "" }, { "Name": "ValidationPermerror", "Value": 7, "Docs": "" }, { "Name": "ValidationFail", "Value": 8, "Docs": "" }, { "Name": "ValidationSoftfail", "Value": 9, "Docs": "" }, { "Name": "ValidationNone", "Value": 10, "Docs": "" }] },
 		"CSRFToken": { "Name": "CSRFToken", "Docs": "", "Values": null },
 		"ThreadMode": { "Name": "ThreadMode", "Docs": "", "Values": [{ "Name": "ThreadOff", "Value": "off", "Docs": "" }, { "Name": "ThreadOn", "Value": "on", "Docs": "" }, { "Name": "ThreadUnread", "Value": "unread", "Docs": "" }] },
@@ -389,8 +389,8 @@ var api;
 		ChangeMailboxSpecialUse: (v) => api.parse("ChangeMailboxSpecialUse", v),
 		SpecialUse: (v) => api.parse("SpecialUse", v),
 		ChangeMailboxKeywords: (v) => api.parse("ChangeMailboxKeywords", v),
-		UID: (v) => api.parse("UID", v),
 		ModSeq: (v) => api.parse("ModSeq", v),
+		UID: (v) => api.parse("UID", v),
 		Validation: (v) => api.parse("Validation", v),
 		CSRFToken: (v) => api.parse("CSRFToken", v),
 		ThreadMode: (v) => api.parse("ThreadMode", v),

webmail/text.js

@@ -309,7 +309,7 @@ var api;
 		"SubmitMessage": { "Name": "SubmitMessage", "Docs": "", "Fields": [{ "Name": "From", "Docs": "", "Typewords": ["string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Cc", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Bcc", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "TextBody", "Docs": "", "Typewords": ["string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "File"] }, { "Name": "ForwardAttachments", "Docs": "", "Typewords": ["ForwardAttachments"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ResponseMessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UserAgent", "Docs": "", "Typewords": ["string"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "FutureRelease", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "ArchiveThread", "Docs": "", "Typewords": ["bool"] }, { "Name": "ArchiveReferenceMailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "DraftMessageID", "Docs": "", "Typewords": ["int64"] }] },
 		"File": { "Name": "File", "Docs": "", "Fields": [{ "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "DataURI", "Docs": "", "Typewords": ["string"] }] },
 		"ForwardAttachments": { "Name": "ForwardAttachments", "Docs": "", "Fields": [{ "Name": "MessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Paths", "Docs": "", "Typewords": ["[]", "[]", "int32"] }] },
-		"Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] },
+		"Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "ParentID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] },
 		"RecipientSecurity": { "Name": "RecipientSecurity", "Docs": "", "Fields": [{ "Name": "STARTTLS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "MTASTS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DNSSEC", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DANE", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["SecurityResult"] }] },
 		"Settings": { "Name": "Settings", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["uint8"] }, { "Name": "Signature", "Docs": "", "Typewords": ["string"] }, { "Name": "Quoting", "Docs": "", "Typewords": ["Quoting"] }, { "Name": "ShowAddressSecurity", "Docs": "", "Typewords": ["bool"] }, { "Name": "ShowHTML", "Docs": "", "Typewords": ["bool"] }, { "Name": "NoShowShortcuts", "Docs": "", "Typewords": ["bool"] }, { "Name": "ShowHeaders", "Docs": "", "Typewords": ["[]", "string"] }] },
 		"Ruleset": { "Name": "Ruleset", "Docs": "", "Fields": [{ "Name": "SMTPMailFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "HeadersRegexp", "Docs": "", "Typewords": ["{}", "string"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ListAllowDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "AcceptRejectsToMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Comment", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDNSDomain", "Docs": "", "Typewords": ["Domain"] }, { "Name": "ListAllowDNSDomain", "Docs": "", "Typewords": ["Domain"] }] },
@@ -325,7 +325,7 @@ var api;
 		"EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] },
 		"ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] },
 		"Flags": { "Name": "Flags", "Docs": "", "Fields": [{ "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }] },
-		"ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
+		"ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "MsgIDs", "Docs": "", "Typewords": ["[]", "int64"] }] },
 		"ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] },
 		"ChangeMsgThread": { "Name": "ChangeMsgThread", "Docs": "", "Fields": [{ "Name": "MessageIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "Muted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Collapsed", "Docs": "", "Typewords": ["bool"] }] },
 		"ChangeMailboxRemove": { "Name": "ChangeMailboxRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
@@ -335,8 +335,8 @@ var api;
 		"ChangeMailboxSpecialUse": { "Name": "ChangeMailboxSpecialUse", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "SpecialUse", "Docs": "", "Typewords": ["SpecialUse"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
 		"SpecialUse": { "Name": "SpecialUse", "Docs": "", "Fields": [{ "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }] },
 		"ChangeMailboxKeywords": { "Name": "ChangeMailboxKeywords", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] },
-		"UID": { "Name": "UID", "Docs": "", "Values": null },
 		"ModSeq": { "Name": "ModSeq", "Docs": "", "Values": null },
+		"UID": { "Name": "UID", "Docs": "", "Values": null },
 		"Validation": { "Name": "Validation", "Docs": "", "Values": [{ "Name": "ValidationUnknown", "Value": 0, "Docs": "" }, { "Name": "ValidationStrict", "Value": 1, "Docs": "" }, { "Name": "ValidationDMARC", "Value": 2, "Docs": "" }, { "Name": "ValidationRelaxed", "Value": 3, "Docs": "" }, { "Name": "ValidationPass", "Value": 4, "Docs": "" }, { "Name": "ValidationNeutral", "Value": 5, "Docs": "" }, { "Name": "ValidationTemperror", "Value": 6, "Docs": "" }, { "Name": "ValidationPermerror", "Value": 7, "Docs": "" }, { "Name": "ValidationFail", "Value": 8, "Docs": "" }, { "Name": "ValidationSoftfail", "Value": 9, "Docs": "" }, { "Name": "ValidationNone", "Value": 10, "Docs": "" }] },
 		"CSRFToken": { "Name": "CSRFToken", "Docs": "", "Values": null },
 		"ThreadMode": { "Name": "ThreadMode", "Docs": "", "Values": [{ "Name": "ThreadOff", "Value": "off", "Docs": "" }, { "Name": "ThreadOn", "Value": "on", "Docs": "" }, { "Name": "ThreadUnread", "Value": "unread", "Docs": "" }] },
@@ -389,8 +389,8 @@ var api;
 		ChangeMailboxSpecialUse: (v) => api.parse("ChangeMailboxSpecialUse", v),
 		SpecialUse: (v) => api.parse("SpecialUse", v),
 		ChangeMailboxKeywords: (v) => api.parse("ChangeMailboxKeywords", v),
-		UID: (v) => api.parse("UID", v),
 		ModSeq: (v) => api.parse("ModSeq", v),
+		UID: (v) => api.parse("UID", v),
 		Validation: (v) => api.parse("Validation", v),
 		CSRFToken: (v) => api.parse("CSRFToken", v),
 		ThreadMode: (v) => api.parse("ThreadMode", v),

webmail/view.go

@@ -708,7 +708,7 @@ func serveEvents(ctx context.Context, log mlog.Log, accountPath string, w http.R
 		qtx, err = acc.DB.Begin(reqctx, false)
 		xcheckf(ctx, err, "begin transaction")
 
-		mbl, err = bstore.QueryTx[store.Mailbox](qtx).List()
+		mbl, err = bstore.QueryTx[store.Mailbox](qtx).FilterEqual("Expunged", false).List()
 		xcheckf(ctx, err, "list mailboxes")
 
 		err = qtx.Get(&settings)
@@ -916,6 +916,8 @@ func serveEvents(ctx context.Context, log mlog.Log, accountPath string, w http.R
 				}
 
 			case store.ChangeRemoveUIDs:
+				comm.RemovalSeen(c)
+
 				// We may send changes for uids the client doesn't know, that's fine.
 				changedUIDs := xchangedUIDs(c.MailboxID, c.UIDs, true)
 				if len(changedUIDs) == 0 {
@@ -1126,7 +1128,7 @@ func xprepareMailboxIDs(ctx context.Context, tx *bstore.Tx, f Filter, rejectsMai
 	if f.MailboxID == -1 {
 		matchMailboxes = false
 		// Add the trash, junk and account rejects mailbox.
-		err := bstore.QueryTx[store.Mailbox](tx).ForEach(func(mb store.Mailbox) error {
+		err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb store.Mailbox) error {
 			if mb.Trash || mb.Junk || mb.Name == rejectsMailbox {
 				mailboxPrefixes = append(mailboxPrefixes, mb.Name+"/")
 				mailboxIDs[mb.ID] = true
@@ -1135,8 +1137,7 @@ func xprepareMailboxIDs(ctx context.Context, tx *bstore.Tx, f Filter, rejectsMai
 		})
 		xcheckf(ctx, err, "finding trash/junk/rejects mailbox")
 	} else if f.MailboxID > 0 {
-		mb := store.Mailbox{ID: f.MailboxID}
+		mb, err := store.MailboxID(tx, f.MailboxID)
-		err := tx.Get(&mb)
 		xcheckf(ctx, err, "get mailbox")
 		mailboxIDs[f.MailboxID] = true
 		mailboxPrefixes = []string{mb.Name + "/"}
@@ -1151,7 +1152,7 @@ func xgatherMailboxIDs(ctx context.Context, tx *bstore.Tx, mailboxIDs map[int64]
 	if len(mailboxPrefixes) == 0 {
 		return
 	}
-	err := bstore.QueryTx[store.Mailbox](tx).ForEach(func(mb store.Mailbox) error {
+	err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb store.Mailbox) error {
 		for _, p := range mailboxPrefixes {
 			if strings.HasPrefix(mb.Name, p) {
 				mailboxIDs[mb.ID] = true

webmail/view_test.go

@@ -45,7 +45,7 @@ func TestView(t *testing.T) {
 	defer func() {
 		err := acc.Close()
 		pkglog.Check(err, "closing account")
-		acc.CheckClosed()
+		acc.WaitClosed()
 	}()
 
 	api := Webmail{maxMessageSize: 1024 * 1024, cookiePath: "/"}

webmail/webmail.go

@@ -407,6 +407,8 @@ func handle(apiHandler http.Handler, isForwarded bool, accountPath string, w htt
 		err = acc.DB.Read(ctx, func(tx *bstore.Tx) error {
 			if err := tx.Get(&m); err != nil {
 				return err
+			} else if m.Expunged {
+				return fmt.Errorf("message was removed")
 			}
 			s := store.Settings{ID: 1}
 			if err := tx.Get(&s); err != nil {

webmail/webmail.js

@@ -309,7 +309,7 @@ var api;
 		"SubmitMessage": { "Name": "SubmitMessage", "Docs": "", "Fields": [{ "Name": "From", "Docs": "", "Typewords": ["string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Cc", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Bcc", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "TextBody", "Docs": "", "Typewords": ["string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "File"] }, { "Name": "ForwardAttachments", "Docs": "", "Typewords": ["ForwardAttachments"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ResponseMessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UserAgent", "Docs": "", "Typewords": ["string"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "FutureRelease", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "ArchiveThread", "Docs": "", "Typewords": ["bool"] }, { "Name": "ArchiveReferenceMailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "DraftMessageID", "Docs": "", "Typewords": ["int64"] }] },
 		"File": { "Name": "File", "Docs": "", "Fields": [{ "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "DataURI", "Docs": "", "Typewords": ["string"] }] },
 		"ForwardAttachments": { "Name": "ForwardAttachments", "Docs": "", "Fields": [{ "Name": "MessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Paths", "Docs": "", "Typewords": ["[]", "[]", "int32"] }] },
-		"Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] },
+		"Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "ParentID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] },
 		"RecipientSecurity": { "Name": "RecipientSecurity", "Docs": "", "Fields": [{ "Name": "STARTTLS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "MTASTS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DNSSEC", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DANE", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["SecurityResult"] }] },
 		"Settings": { "Name": "Settings", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["uint8"] }, { "Name": "Signature", "Docs": "", "Typewords": ["string"] }, { "Name": "Quoting", "Docs": "", "Typewords": ["Quoting"] }, { "Name": "ShowAddressSecurity", "Docs": "", "Typewords": ["bool"] }, { "Name": "ShowHTML", "Docs": "", "Typewords": ["bool"] }, { "Name": "NoShowShortcuts", "Docs": "", "Typewords": ["bool"] }, { "Name": "ShowHeaders", "Docs": "", "Typewords": ["[]", "string"] }] },
 		"Ruleset": { "Name": "Ruleset", "Docs": "", "Fields": [{ "Name": "SMTPMailFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "HeadersRegexp", "Docs": "", "Typewords": ["{}", "string"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ListAllowDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "AcceptRejectsToMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Comment", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDNSDomain", "Docs": "", "Typewords": ["Domain"] }, { "Name": "ListAllowDNSDomain", "Docs": "", "Typewords": ["Domain"] }] },
@@ -325,7 +325,7 @@ var api;
 		"EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] },
 		"ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] },
 		"Flags": { "Name": "Flags", "Docs": "", "Fields": [{ "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }] },
-		"ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
+		"ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "MsgIDs", "Docs": "", "Typewords": ["[]", "int64"] }] },
 		"ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] },
 		"ChangeMsgThread": { "Name": "ChangeMsgThread", "Docs": "", "Fields": [{ "Name": "MessageIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "Muted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Collapsed", "Docs": "", "Typewords": ["bool"] }] },
 		"ChangeMailboxRemove": { "Name": "ChangeMailboxRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
@@ -335,8 +335,8 @@ var api;
 		"ChangeMailboxSpecialUse": { "Name": "ChangeMailboxSpecialUse", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "SpecialUse", "Docs": "", "Typewords": ["SpecialUse"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
 		"SpecialUse": { "Name": "SpecialUse", "Docs": "", "Fields": [{ "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }] },
 		"ChangeMailboxKeywords": { "Name": "ChangeMailboxKeywords", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] },
-		"UID": { "Name": "UID", "Docs": "", "Values": null },
 		"ModSeq": { "Name": "ModSeq", "Docs": "", "Values": null },
+		"UID": { "Name": "UID", "Docs": "", "Values": null },
 		"Validation": { "Name": "Validation", "Docs": "", "Values": [{ "Name": "ValidationUnknown", "Value": 0, "Docs": "" }, { "Name": "ValidationStrict", "Value": 1, "Docs": "" }, { "Name": "ValidationDMARC", "Value": 2, "Docs": "" }, { "Name": "ValidationRelaxed", "Value": 3, "Docs": "" }, { "Name": "ValidationPass", "Value": 4, "Docs": "" }, { "Name": "ValidationNeutral", "Value": 5, "Docs": "" }, { "Name": "ValidationTemperror", "Value": 6, "Docs": "" }, { "Name": "ValidationPermerror", "Value": 7, "Docs": "" }, { "Name": "ValidationFail", "Value": 8, "Docs": "" }, { "Name": "ValidationSoftfail", "Value": 9, "Docs": "" }, { "Name": "ValidationNone", "Value": 10, "Docs": "" }] },
 		"CSRFToken": { "Name": "CSRFToken", "Docs": "", "Values": null },
 		"ThreadMode": { "Name": "ThreadMode", "Docs": "", "Values": [{ "Name": "ThreadOff", "Value": "off", "Docs": "" }, { "Name": "ThreadOn", "Value": "on", "Docs": "" }, { "Name": "ThreadUnread", "Value": "unread", "Docs": "" }] },
@@ -389,8 +389,8 @@ var api;
 		ChangeMailboxSpecialUse: (v) => api.parse("ChangeMailboxSpecialUse", v),
 		SpecialUse: (v) => api.parse("SpecialUse", v),
 		ChangeMailboxKeywords: (v) => api.parse("ChangeMailboxKeywords", v),
-		UID: (v) => api.parse("UID", v),
 		ModSeq: (v) => api.parse("ModSeq", v),
+		UID: (v) => api.parse("UID", v),
 		Validation: (v) => api.parse("Validation", v),
 		CSRFToken: (v) => api.parse("CSRFToken", v),
 		ThreadMode: (v) => api.parse("ThreadMode", v),

webops/xops.go

@@ -6,16 +6,18 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log/slog"
 	"os"
+	"slices"
 	"sort"
 	"time"
 
-	"golang.org/x/exp/maps"
-
 	"github.com/mjl-/bstore"
 
+	"github.com/mjl-/mox/junk"
 	"github.com/mjl-/mox/message"
 	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/moxio"
 	"github.com/mjl-/mox/store"
 )
 
@@ -31,9 +33,8 @@ func (x XOps) mailboxID(ctx context.Context, tx *bstore.Tx, mailboxID int64) sto
 	if mailboxID == 0 {
 		x.Checkuserf(ctx, errors.New("invalid zero mailbox ID"), "getting mailbox")
 	}
-	mb := store.Mailbox{ID: mailboxID}
+	mb, err := store.MailboxID(tx, mailboxID)
-	err := tx.Get(&mb)
+	if err == bstore.ErrAbsent || err == store.ErrMailboxExpunged {
-	if err == bstore.ErrAbsent {
 		x.Checkuserf(ctx, err, "getting mailbox")
 	}
 	x.Checkf(ctx, err, "getting mailbox")
@@ -67,25 +68,32 @@ func (x XOps) MessageDelete(ctx context.Context, log mlog.Log, acc *store.Accoun
 
 		store.BroadcastChanges(acc, changes)
 	})
-
-	for _, mID := range messageIDs {
-		p := acc.MessagePath(mID)
-		err := os.Remove(p)
-		log.Check(err, "removing message file for expunge")
-	}
 }
 
 func (x XOps) MessageDeleteTx(ctx context.Context, log mlog.Log, tx *bstore.Tx, acc *store.Account, messageIDs []int64, modseq *store.ModSeq) []store.Change {
-	removeChanges := map[int64]store.ChangeRemoveUIDs{}
+	changes := make([]store.Change, 0, 1+1) // 1 remove, 1 mailbox counts, optimistic that all messages are in 1 mailbox.
-	changes := make([]store.Change, 0, len(messageIDs)+1) // n remove, 1 mailbox counts
+
+	var jf *junk.Filter
+	defer func() {
+		if jf != nil {
+			err := jf.CloseDiscard()
+			log.Check(err, "close junk filter")
+		}
+	}()
+
+	conf, _ := acc.Conf()
 
 	var mb store.Mailbox
-	remove := make([]store.Message, 0, len(messageIDs))
+	var changeRemoveUIDs store.ChangeRemoveUIDs
+	xflushMailbox := func() {
+		err := tx.Update(&mb)
+		x.Checkf(ctx, err, "updating mailbox counts")
+		slices.Sort(changeRemoveUIDs.UIDs)
+		changes = append(changes, mb.ChangeCounts(), changeRemoveUIDs)
+	}
 
-	var totalSize int64
+	for _, id := range messageIDs {
-	for _, mid := range messageIDs {
+		m := x.messageID(ctx, tx, id)
-		m := x.messageID(ctx, tx, mid)
-		totalSize += m.Size
 
 		if *modseq == 0 {
 			var err error
@@ -95,58 +103,33 @@ func (x XOps) MessageDeleteTx(ctx context.Context, log mlog.Log, tx *bstore.Tx,
 
 		if m.MailboxID != mb.ID {
 			if mb.ID != 0 {
-				mb.ModSeq = *modseq
+				xflushMailbox()
-				err := tx.Update(&mb)
-				x.Checkf(ctx, err, "updating mailbox counts")
-				changes = append(changes, mb.ChangeCounts())
 			}
 			mb = x.mailboxID(ctx, tx, m.MailboxID)
+			mb.ModSeq = *modseq
+			changeRemoveUIDs = store.ChangeRemoveUIDs{MailboxID: mb.ID, ModSeq: *modseq}
 		}
 
-		qmr := bstore.QueryTx[store.Recipient](tx)
+		if m.Junk != m.Notjunk && jf == nil && conf.JunkFilter != nil {
-		qmr.FilterEqual("MessageID", m.ID)
+			var err error
-		_, err := qmr.Delete()
+			jf, _, err = acc.OpenJunkFilter(ctx, log)
-		x.Checkf(ctx, err, "removing message recipients")
+			x.Checkf(ctx, err, "open junk filter")
+		}
 
-		mb.Sub(m.MailboxCounts())
+		opts := store.RemoveOpts{JunkFilter: jf}
+		_, _, err := acc.MessageRemove(log, tx, *modseq, &mb, opts, m)
+		x.Checkf(ctx, err, "expunge message")
 
-		m.Expunged = true
+		changeRemoveUIDs.UIDs = append(changeRemoveUIDs.UIDs, m.UID)
-		m.ModSeq = *modseq
+		changeRemoveUIDs.MsgIDs = append(changeRemoveUIDs.MsgIDs, m.ID)
-		err = tx.Update(&m)
-		x.Checkf(ctx, err, "marking message as expunged")
-
-		ch := removeChanges[m.MailboxID]
-		ch.UIDs = append(ch.UIDs, m.UID)
-		ch.MailboxID = m.MailboxID
-		ch.ModSeq = *modseq
-		removeChanges[m.MailboxID] = ch
-		remove = append(remove, m)
 	}
 
-	if mb.ID != 0 {
+	xflushMailbox()
-		mb.ModSeq = *modseq
-		err := tx.Update(&mb)
-		x.Checkf(ctx, err, "updating count in mailbox")
-		changes = append(changes, mb.ChangeCounts())
-	}
 
-	err := acc.AddMessageSize(log, tx, -totalSize)
+	if jf != nil {
-	x.Checkf(ctx, err, "updating disk usage")
+		err := jf.Close()
-
+		jf = nil
-	// Mark removed messages as not needing training, then retrain them, so if they
+		x.Checkf(ctx, err, "close junk filter")
-	// were trained, they get untrained.
-	for i := range remove {
-		remove[i].Junk = false
-		remove[i].Notjunk = false
-	}
-	err = acc.RetrainMessages(ctx, log, tx, remove)
-	x.Checkf(ctx, err, "untraining deleted messages")
-
-	for _, ch := range removeChanges {
-		sort.Slice(ch.UIDs, func(i, j int) bool {
-			return ch.UIDs[i] < ch.UIDs[j]
-		})
-		changes = append(changes, ch)
 	}
 
 	return changes
@@ -361,6 +344,15 @@ func (x XOps) MessageMove(ctx context.Context, log mlog.Log, acc *store.Account,
 	acc.WithWLock(func() {
 		var changes []store.Change
 
+		var newIDs []int64
+		defer func() {
+			for _, id := range newIDs {
+				p := acc.MessagePath(id)
+				err := os.Remove(p)
+				log.Check(err, "removing delivered message after failure", slog.String("path", p))
+			}
+		}()
+
 		x.DBWrite(ctx, acc, func(tx *bstore.Tx) {
 			if mailboxName != "" {
 				mb, err := acc.MailboxFind(tx, mailboxName)
@@ -379,23 +371,37 @@ func (x XOps) MessageMove(ctx context.Context, log mlog.Log, acc *store.Account,
 			}
 
 			var modseq store.ModSeq
-			changes = x.MessageMoveTx(ctx, log, acc, tx, messageIDs, mbDst, &modseq)
+			newIDs, changes = x.MessageMoveTx(ctx, log, acc, tx, messageIDs, mbDst, &modseq)
 		})
+		newIDs = nil
 
 		store.BroadcastChanges(acc, changes)
 	})
 }
 
-func (x XOps) MessageMoveTx(ctx context.Context, log mlog.Log, acc *store.Account, tx *bstore.Tx, messageIDs []int64, mbDst store.Mailbox, modseq *store.ModSeq) []store.Change {
+// MessageMoveTx moves message to a new mailbox, which must be different than their
-	retrain := make([]store.Message, 0, len(messageIDs))
+// current mailbox. Moving a message is done by changing the MailboxID and
-	removeChanges := map[int64]store.ChangeRemoveUIDs{}
+// assigning an appriorate new UID, and then inserting a replacement Message record
-	// n adds, 1 remove, 2 mailboxcounts, optimistic and at least for a single message.
+// with new ID that is marked expunged in the original mailbox, along with a
-	changes := make([]store.Change, 0, len(messageIDs)+3)
+// MessageErase record so the message gets erased when all sessions stopped
+// referencing the message.
+func (x XOps) MessageMoveTx(ctx context.Context, log mlog.Log, acc *store.Account, tx *bstore.Tx, messageIDs []int64, mbDst store.Mailbox, modseq *store.ModSeq) ([]int64, []store.Change) {
+	var newIDs []int64
+	var commit bool
+	defer func() {
+		if commit {
+			return
+		}
+		for _, id := range newIDs {
+			p := acc.MessagePath(id)
+			err := os.Remove(p)
+			log.Check(err, "removing delivered message after failure", slog.String("path", p))
+		}
+		newIDs = nil
+	}()
 
-	var mbSrc store.Mailbox
+	// n adds, 1 remove, 2 mailboxcounts, 1 mailboxkeywords, optimistic that messages are in a single source mailbox.
-
+	changes := make([]store.Change, 0, len(messageIDs)+4)
-	keywords := map[string]struct{}{}
-	now := time.Now()
 
 	var err error
 	if *modseq == 0 {
@@ -403,104 +409,135 @@ func (x XOps) MessageMoveTx(ctx context.Context, log mlog.Log, acc *store.Accoun
 		x.Checkf(ctx, err, "assigning next modseq")
 	}
 
-	for _, mid := range messageIDs {
+	mbDst.ModSeq = *modseq
-		m := x.messageID(ctx, tx, mid)
 
-		// We may have loaded this mailbox in the previous iteration of this loop.
+	// Get messages. group them by mailbox.
-		if m.MailboxID != mbSrc.ID {
+	l := make([]store.Message, len(messageIDs))
-			if mbSrc.ID != 0 {
+	for i, id := range messageIDs {
-				mbSrc.ModSeq = *modseq
+		l[i] = x.messageID(ctx, tx, id)
-				err := tx.Update(&mbSrc)
+		if l[i].MailboxID == mbDst.ID {
-				x.Checkf(ctx, err, "updating source mailbox counts")
-				changes = append(changes, mbSrc.ChangeCounts())
-			}
-			mbSrc = x.mailboxID(ctx, tx, m.MailboxID)
-		}
-
-		if mbSrc.ID == mbDst.ID {
 			// Client should filter out messages that are already in mailbox.
-			x.Checkuserf(ctx, errors.New("already in destination mailbox"), "moving message")
+			x.Checkuserf(ctx, fmt.Errorf("message %d already in destination mailbox", l[i].ID), "moving message")
-		}
-
-		ch := removeChanges[m.MailboxID]
-		ch.UIDs = append(ch.UIDs, m.UID)
-		ch.ModSeq = *modseq
-		ch.MailboxID = m.MailboxID
-		removeChanges[m.MailboxID] = ch
-
-		// Copy of message record that we'll insert when UID is freed up.
-		om := m
-		om.PrepareExpunge()
-		om.ID = 0 // Assign new ID.
-		om.ModSeq = *modseq
-
-		mbSrc.Sub(m.MailboxCounts())
-
-		if mbDst.Trash {
-			m.Seen = true
-		}
-		conf, _ := acc.Conf()
-		m.MailboxID = mbDst.ID
-		if m.IsReject && m.MailboxDestinedID != 0 {
-			// Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message
-			// is used for reputation calculation during future deliveries.
-			m.MailboxOrigID = m.MailboxDestinedID
-			m.IsReject = false
-			m.Seen = false
-		}
-		m.UID = mbDst.UIDNext
-		m.ModSeq = *modseq
-		mbDst.UIDNext++
-		m.JunkFlagsForMailbox(mbDst, conf)
-		m.SaveDate = &now
-		err = tx.Update(&m)
-		x.Checkf(ctx, err, "updating moved message in database")
-
-		// Now that UID is unused, we can insert the old record again.
-		err = tx.Insert(&om)
-		x.Checkf(ctx, err, "inserting record for expunge after moving message")
-
-		mbDst.Add(m.MailboxCounts())
-
-		changes = append(changes, m.ChangeAddUID())
-		retrain = append(retrain, m)
-
-		for _, kw := range m.Keywords {
-			keywords[kw] = struct{}{}
 		}
 	}
 
-	mbSrc.ModSeq = *modseq
+	// Sort (group) by mailbox, sort by UID.
-	err = tx.Update(&mbSrc)
+	sort.Slice(l, func(i, j int) bool {
-	x.Checkf(ctx, err, "updating source mailbox counts and modseq")
+		if l[i].MailboxID != l[j].MailboxID {
+			return l[i].MailboxID < l[j].MailboxID
+		}
+		return l[i].UID < l[j].UID
+	})
 
-	changes = append(changes, mbSrc.ChangeCounts(), mbDst.ChangeCounts())
+	var jf *junk.Filter
+	defer func() {
+		if jf != nil {
+			err := jf.CloseDiscard()
+			log.Check(err, "close junk filter")
+		}
+	}()
 
-	// Ensure destination mailbox has keywords of the moved messages.
+	accConf, _ := acc.Conf()
-	var mbKwChanged bool
+
-	mbDst.Keywords, mbKwChanged = store.MergeKeywords(mbDst.Keywords, maps.Keys(keywords))
+	var mbSrc store.Mailbox
-	if mbKwChanged {
+	var changeRemoveUIDs store.ChangeRemoveUIDs
+	xflushMailbox := func() {
+		changes = append(changes, changeRemoveUIDs, mbSrc.ChangeCounts())
+
+		err = tx.Update(&mbSrc)
+		x.Checkf(ctx, err, "updating source mailbox counts")
+	}
+
+	nkeywords := len(mbDst.Keywords)
+	now := time.Now()
+
+	for _, om := range l {
+		if om.MailboxID != mbSrc.ID {
+			if mbSrc.ID != 0 {
+				xflushMailbox()
+			}
+			mbSrc = x.mailboxID(ctx, tx, om.MailboxID)
+			mbSrc.ModSeq = *modseq
+			changeRemoveUIDs = store.ChangeRemoveUIDs{MailboxID: mbSrc.ID, ModSeq: *modseq}
+		}
+
+		nm := om
+		nm.MailboxID = mbDst.ID
+		nm.UID = mbDst.UIDNext
+		mbDst.UIDNext++
+		nm.ModSeq = *modseq
+		nm.CreateSeq = *modseq
+		nm.SaveDate = &now
+		if nm.IsReject && nm.MailboxDestinedID != 0 {
+			// Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message
+			// is used for reputation calculation during future deliveries.
+			nm.MailboxOrigID = nm.MailboxDestinedID
+			nm.IsReject = false
+			nm.Seen = false
+		}
+		if mbDst.Trash {
+			nm.Seen = true
+		}
+
+		nm.JunkFlagsForMailbox(mbDst, accConf)
+
+		err := tx.Update(&nm)
+		x.Checkf(ctx, err, "updating message with new mailbox")
+
+		mbDst.Add(nm.MailboxCounts())
+
+		mbSrc.Sub(om.MailboxCounts())
+		om.ID = 0
+		om.Expunged = true
+		om.ModSeq = *modseq
+		om.TrainedJunk = nil
+		err = tx.Insert(&om)
+		x.Checkf(ctx, err, "inserting expunged message in old mailbox")
+
+		err = moxio.LinkOrCopy(log, acc.MessagePath(om.ID), acc.MessagePath(nm.ID), nil, false)
+		x.Checkf(ctx, err, "duplicating message in old mailbox for current sessions")
+		newIDs = append(newIDs, nm.ID)
+		// We don't sync the directory. In case of a crash and files disappearing, the
+		// eraser will simply not find the file at next startup.
+
+		err = tx.Insert(&store.MessageErase{ID: om.ID, SkipUpdateDiskUsage: true})
+		x.Checkf(ctx, err, "insert message erase")
+
+		mbDst.Keywords, _ = store.MergeKeywords(mbDst.Keywords, nm.Keywords)
+
+		if accConf.JunkFilter != nil && nm.NeedsTraining() {
+			// Lazily open junk filter.
+			if jf == nil {
+				jf, _, err = acc.OpenJunkFilter(ctx, log)
+				x.Checkf(ctx, err, "open junk filter")
+			}
+			err := acc.RetrainMessage(ctx, log, tx, jf, &nm)
+			x.Checkf(ctx, err, "retrain message after moving")
+		}
+
+		changeRemoveUIDs.UIDs = append(changeRemoveUIDs.UIDs, om.UID)
+		changeRemoveUIDs.MsgIDs = append(changeRemoveUIDs.MsgIDs, om.ID)
+		changes = append(changes, nm.ChangeAddUID())
+	}
+
+	xflushMailbox()
+
+	changes = append(changes, mbDst.ChangeCounts())
+	if nkeywords > len(mbDst.Keywords) {
 		changes = append(changes, mbDst.ChangeKeywords())
 	}
 
-	mbDst.ModSeq = *modseq
 	err = tx.Update(&mbDst)
 	x.Checkf(ctx, err, "updating destination mailbox with uidnext and modseq")
 
-	err = acc.RetrainMessages(ctx, log, tx, retrain)
+	if jf != nil {
-	x.Checkf(ctx, err, "retraining messages after move")
+		err := jf.Close()
-
+		x.Checkf(ctx, err, "saving junk filter")
-	// Ensure UIDs of the removed message are in increasing order. It is quite common
+		jf = nil
-	// for all messages to be from a single source mailbox, meaning this is just one
-	// change, for which we preallocated space.
-	for _, ch := range removeChanges {
-		sort.Slice(ch.UIDs, func(i, j int) bool {
-			return ch.UIDs[i] < ch.UIDs[j]
-		})
-		changes = append(changes, ch)
 	}
 
-	return changes
+	commit = true
+	return newIDs, changes
 }
 
 func isText(p message.Part) bool {