diff --git a/backup.go b/backup.go index 8baa1ac..29a7c2c 100644 --- a/backup.go +++ b/backup.go @@ -532,8 +532,7 @@ func backupctl(ctx context.Context, ctl *ctl) { } }() - // Link/copy known message files. If a message has been removed while we read the - // database, our backup is not consistent and the backup will be marked failed. + // Link/copy known message files. tmMsgs := time.Now() seen := map[string]struct{}{} var maxID int64 @@ -565,6 +564,15 @@ func backupctl(ctx context.Context, ctl *ctl) { slog.Duration("duration", time.Since(tmMsgs))) } + eraseIDs := map[int64]struct{}{} + err = bstore.QueryDB[store.MessageErase](ctx, db).ForEach(func(me store.MessageErase) error { + eraseIDs[me.ID] = struct{}{} + return nil + }) + if err != nil { + xerrx("listing erased messages", err) + } + // Read through all files in queue directory and warn about anything we haven't // handled yet. Message files that are newer than we expect from our consistent // database snapshot are ignored. @@ -586,9 +594,13 @@ func backupctl(ctx context.Context, ctl *ctl) { return nil } - // Skip any messages that were added since we started on our consistent snapshot. - // We don't want to cause spurious backup warnings. - if id, err := strconv.ParseInt(l[len(l)-1], 10, 64); err == nil && id > maxID && mp == store.MessagePath(id) { + // Skip any messages that were added since we started on our consistent snapshot, + // or messages that will be erased. We don't want to cause spurious backup + // warnings. + id, err := strconv.ParseInt(l[len(l)-1], 10, 64) + if err == nil && id > maxID && mp == store.MessagePath(id) { + return nil + } else if _, ok := eraseIDs[id]; err == nil && ok { return nil } } diff --git a/config/config.go b/config/config.go index bb15e94..189bd96 100644 --- a/config/config.go +++ b/config/config.go @@ -65,7 +65,7 @@ type Static struct { ParsedLocalpart smtp.Localpart `sconf:"-"` } `sconf:"optional" sconf-doc:"Destination for per-host TLS reports (TLSRPT). TLS reports can be per recipient domain (for MTA-STS), or per MX host (for DANE). The per-domain TLS reporting configuration is in domains.conf. This is the TLS reporting configuration for this host. If absent, no host-based TLSRPT address is configured, and no host TLSRPT DNS record is suggested."` - InitialMailboxes InitialMailboxes `sconf:"optional" sconf-doc:"Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be given a 'special-use' role, which are understood by most mail clients. If absent/empty, the following mailboxes are created: Sent, Archive, Trash, Drafts and Junk."` + InitialMailboxes InitialMailboxes `sconf:"optional" sconf-doc:"Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be given a 'special-use' role, which are understood by most mail clients. If absent/empty, the following additional mailboxes are created: Sent, Archive, Trash, Drafts and Junk."` DefaultMailboxes []string `sconf:"optional" sconf-doc:"Deprecated in favor of InitialMailboxes. Mailboxes to create when adding an account. Inbox is always created. If no mailboxes are specified, the following are automatically created: Sent, Archive, Trash, Drafts and Junk."` Transports map[string]Transport `sconf:"optional" sconf-doc:"Transport are mechanisms for delivering messages. Transports can be referenced from Routes in accounts, domains and the global configuration. There is always an implicit/fallback delivery transport doing direct delivery with SMTP from the outgoing message queue. Transports are typically only configured when using smarthosts, i.e. when delivering through another SMTP server. Zero or one transport methods must be set in a transport, never multiple. When using an external party to send email for a domain, keep in mind you may have to add their IP address to your domain's SPF record, and possibly additional DKIM records."` // Awkward naming of fields to get intended default behaviour for zero values. diff --git a/config/doc.go b/config/doc.go index 7fb271e..0df0c65 100644 --- a/config/doc.go +++ b/config/doc.go @@ -543,8 +543,8 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details. # Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be # given a 'special-use' role, which are understood by most mail clients. If - # absent/empty, the following mailboxes are created: Sent, Archive, Trash, Drafts - # and Junk. (optional) + # absent/empty, the following additional mailboxes are created: Sent, Archive, + # Trash, Drafts and Junk. (optional) InitialMailboxes: # Special-use roles to mailbox to create. (optional) diff --git a/ctl.go b/ctl.go index 1b43f71..b16cb72 100644 --- a/ctl.go +++ b/ctl.go @@ -1454,17 +1454,28 @@ func servectlcmd(ctx context.Context, ctl *ctl, cid int64, shutdown func()) { log.Check(err, "closing junk filter during cleanup") }() - // Read through messages with junk or nonjunk flag set, and train them. + // Read through messages with either junk or nonjunk flag set, and train them. var total, trained int - q := bstore.QueryDB[store.Message](ctx, acc.DB) - q.FilterEqual("Expunged", false) - err = q.ForEach(func(m store.Message) error { - total++ - ok, err := acc.TrainMessage(ctx, log, jf, m) - if ok { - trained++ - } - return err + err = acc.DB.Write(ctx, func(tx *bstore.Tx) error { + q := bstore.QueryTx[store.Message](tx) + q.FilterEqual("Expunged", false) + return q.ForEach(func(m store.Message) error { + total++ + if m.Junk == m.Notjunk { + return nil + } + ok, err := acc.TrainMessage(ctx, log, jf, m.Notjunk, m) + if ok { + trained++ + } + if m.TrainedJunk == nil || *m.TrainedJunk != m.Junk { + m.TrainedJunk = &m.Junk + if err := tx.Update(&m); err != nil { + return fmt.Errorf("marking message as trained: %v", err) + } + } + return err + }) }) ctl.xcheck(err, "training messages") log.Info("retrained messages", slog.Int("total", total), slog.Int("trained", trained)) @@ -1509,7 +1520,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, cid int64, shutdown func()) { var changes []store.Change err = acc.DB.Write(ctx, func(tx *bstore.Tx) error { var totalSize int64 - err := bstore.QueryTx[store.Mailbox](tx).ForEach(func(mb store.Mailbox) error { + err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb store.Mailbox) error { mc, err := mb.CalculateCounts(tx) if err != nil { return fmt.Errorf("calculating counts for mailbox %q: %w", mb.Name, err) @@ -1618,6 +1629,11 @@ func servectlcmd(ctx context.Context, ctl *ctl, cid int64, shutdown func()) { if err := tx.Get(&mb); err != nil { _, werr := fmt.Fprintf(w, "get mailbox id %d for message with file size mismatch: %v\n", mb.ID, err) ctl.xcheck(werr, "write") + return nil + } + if mb.Expunged { + _, err := fmt.Fprintf(w, "message %d is in expunged mailbox %q (id %d) (continuing)\n", m.ID, mb.Name, mb.ID) + ctl.xcheck(err, "write") } _, err = fmt.Fprintf(w, "fixing message %d in mailbox %q (id %d) with incorrect size %d, should be %d (len msg prefix %d + on-disk file %s size %d)\n", m.ID, mb.Name, mb.ID, m.Size, correctSize, len(m.MsgPrefix), p, filesize) ctl.xcheck(err, "write") @@ -1650,7 +1666,6 @@ func servectlcmd(ctx context.Context, ctl *ctl, cid int64, shutdown func()) { } return nil }) - }) ctl.xcheck(err, "find and fix wrong message sizes") diff --git a/ctl_test.go b/ctl_test.go index 9b4d1eb..48be61d 100644 --- a/ctl_test.go +++ b/ctl_test.go @@ -436,7 +436,7 @@ func TestCtl(t *testing.T) { tcheck(t, err, "open account") defer func() { acc.Close() - acc.CheckClosed() + acc.WaitClosed() }() content := []byte("Subject: hi\r\n\r\nbody\r\n") diff --git a/imapserver/append_test.go b/imapserver/append_test.go index 45a883f..14466d9 100644 --- a/imapserver/append_test.go +++ b/imapserver/append_test.go @@ -13,10 +13,10 @@ func TestAppend(t *testing.T) { defer tc.close() tc2 := startNoSwitchboard(t) // note: without switchboard because this connection will break during tests. - defer tc2.close() + defer tc2.closeNoWait() tc3 := startNoSwitchboard(t) - defer tc3.close() + defer tc3.closeNoWait() tc2.client.Login("mjl@mox.example", password0) tc2.client.Select("inbox") @@ -31,19 +31,22 @@ func TestAppend(t *testing.T) { // Syntax error for line ending in literal causes connection abort. tc2.transactf("bad", "append inbox (\\Badflag) {1+}\r\nx") // Unknown flag. tc2 = startNoSwitchboard(t) - defer tc2.close() + defer tc2.closeNoWait() tc2.client.Login("mjl@mox.example", password0) tc2.client.Select("inbox") tc2.transactf("bad", "append inbox () \"bad time\" {1+}\r\nx") // Bad time. tc2 = startNoSwitchboard(t) - defer tc2.close() + defer tc2.closeNoWait() tc2.client.Login("mjl@mox.example", password0) tc2.client.Select("inbox") tc2.transactf("no", "append nobox (\\Seen) \" 1-Jan-2022 10:10:00 +0100\" {1}") tc2.xcode("TRYCREATE") + tc2.transactf("no", "append expungebox (\\Seen) {1}") + tc2.xcode("TRYCREATE") + tc2.transactf("ok", "append inbox (\\Seen Label1 $label2) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx") tc2.xuntagged(imapclient.UntaggedExists(1)) tc2.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UIDs: xparseUIDRange("1")}) diff --git a/imapserver/condstore_test.go b/imapserver/condstore_test.go index aa6ce18..7195fa5 100644 --- a/imapserver/condstore_test.go +++ b/imapserver/condstore_test.go @@ -85,6 +85,8 @@ func testCondstoreQresync(t *testing.T, qresync bool) { // unmodified. Those messages have modseq 0 in the database. We use append for // convenience, then adjust the records in the database. // We have a workaround below to prevent triggering the consistency checker. + tc.account.SetSkipMessageModSeqZeroCheck(true) + defer tc.account.SetSkipMessageModSeqZeroCheck(false) tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx") tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx") tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx") @@ -100,13 +102,13 @@ func testCondstoreQresync(t *testing.T, qresync bool) { // tc2 is a client without condstore, so no modseq responses. tc2 := startNoSwitchboard(t) - defer tc2.close() + defer tc2.closeNoWait() tc2.client.Login("mjl@mox.example", password0) tc2.client.Select("inbox") // tc3 is a client with condstore, so with modseq responses. tc3 := startNoSwitchboard(t) - defer tc3.close() + defer tc3.closeNoWait() tc3.client.Login("mjl@mox.example", password0) tc3.client.Enable(capability) tc3.client.Select("inbox") @@ -124,7 +126,7 @@ func testCondstoreQresync(t *testing.T, qresync bool) { tc.transactf("ok", "Append otherbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx") tc.xuntagged() - tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 2, UIDs: xparseUIDRange("1")}) + tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 3, UIDs: xparseUIDRange("1")}) tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx") tc.xuntagged(imapclient.UntaggedExists(5)) @@ -353,11 +355,7 @@ func testCondstoreQresync(t *testing.T, qresync bool) { xtc := startNoSwitchboard(t) // We have modified modseq & createseq to 0 above for testing that case. Don't // trigger the consistency checker. - store.CheckConsistencyOnClose = false - defer func() { - xtc.close() - store.CheckConsistencyOnClose = true - }() + defer xtc.closeNoWait() xtc.client.Login("mjl@mox.example", password0) fn(xtc) tagcount++ @@ -444,13 +442,13 @@ func testCondstoreQresync(t *testing.T, qresync bool) { // and untagged fetch with modseq in destination mailbox. // tc2o is a client without condstore, so no modseq responses. tc2o := startNoSwitchboard(t) - defer tc2o.close() + defer tc2o.closeNoWait() tc2o.client.Login("mjl@mox.example", password0) tc2o.client.Select("otherbox") // tc3o is a client with condstore, so with modseq responses. tc3o := startNoSwitchboard(t) - defer tc3o.close() + defer tc3o.closeNoWait() tc3o.client.Login("mjl@mox.example", password0) tc3o.client.Enable(capability) tc3o.client.Select("otherbox") @@ -484,14 +482,9 @@ func testCondstoreQresync(t *testing.T, qresync bool) { imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), noflags, imapclient.FetchModSeq(clientModseq)}}, ) - // Restore valid modseq/createseq for the consistency checker. - _, err = bstore.QueryDB[store.Message](ctxbg, tc.account.DB).FilterEqual("CreateSeq", int64(0)).UpdateNonzero(store.Message{CreateSeq: 2}) - tcheck(t, err, "updating modseq/createseq to valid values") - _, err = bstore.QueryDB[store.Message](ctxbg, tc.account.DB).FilterEqual("ModSeq", int64(0)).UpdateNonzero(store.Message{ModSeq: 2}) - tcheck(t, err, "updating modseq/createseq to valid values") - tc2o.close() + tc2o.closeNoWait() tc2o = nil - tc3o.close() + tc3o.closeNoWait() tc3o = nil // Then we rename inbox, which is special because it moves messages away instead of @@ -533,10 +526,7 @@ func testQresync(t *testing.T, tc *testconn, clientModseq int64) { xtc.client.Login("mjl@mox.example", password0) xtc.transactf("ok", "Select inbox (Condstore)") xtc.transactf("bad", "Uid Fetch 1:* (Flags) (Changedsince 1 Vanished)") - // Prevent triggering the consistency checker, we still have modseq/createseq at 0. - store.CheckConsistencyOnClose = false - xtc.close() - store.CheckConsistencyOnClose = true + xtc.closeNoWait() xtc = nil // Check that we get proper vanished responses. @@ -557,9 +547,7 @@ func testQresync(t *testing.T, tc *testconn, clientModseq int64) { xtc.client.Login("mjl@mox.example", password0) xtc.transactf("bad", "Select inbox (Qresync 1 0)") // Prevent triggering the consistency checker, we still have modseq/createseq at 0. - store.CheckConsistencyOnClose = false - xtc.close() - store.CheckConsistencyOnClose = true + xtc.closeNoWait() xtc = nil tc.transactf("bad", "Select inbox (Qresync (0 1))") // Both args must be > 0. diff --git a/imapserver/copy_test.go b/imapserver/copy_test.go index f1d63c4..e93c999 100644 --- a/imapserver/copy_test.go +++ b/imapserver/copy_test.go @@ -12,7 +12,7 @@ func TestCopy(t *testing.T) { defer tc.close() tc2 := startNoSwitchboard(t) - defer tc2.close() + defer tc2.closeNoWait() tc.client.Login("mjl@mox.example", password0) tc.client.Select("inbox") @@ -34,6 +34,8 @@ func TestCopy(t *testing.T) { tc.transactf("no", "copy 1 nonexistent") tc.xcode("TRYCREATE") + tc.transactf("no", "copy 1 expungebox") + tc.xcode("TRYCREATE") tc.transactf("no", "copy 1 inbox") // Cannot copy to same mailbox. diff --git a/imapserver/create_test.go b/imapserver/create_test.go index 709ee4a..eab1ef7 100644 --- a/imapserver/create_test.go +++ b/imapserver/create_test.go @@ -11,7 +11,7 @@ func TestCreate(t *testing.T) { defer tc.close() tc2 := startNoSwitchboard(t) - defer tc2.close() + defer tc2.closeNoWait() tc.client.Login("mjl@mox.example", password0) tc2.client.Login("mjl@mox.example", password0) @@ -84,4 +84,7 @@ func TestCreate(t *testing.T) { tc.transactf("ok", `create "&"`) // Interpreted as UTF-8, no UTF-7. tc2.transactf("bad", `create "&"`) // Bad UTF-7. tc2.transactf("ok", `create "&Jjo-"`) // ☺, valid UTF-7. + + tc.transactf("ok", "create expungebox") // Existed in past. + tc.transactf("ok", "delete expungebox") // Gone again. } diff --git a/imapserver/delete_test.go b/imapserver/delete_test.go index c5924ac..cdd910c 100644 --- a/imapserver/delete_test.go +++ b/imapserver/delete_test.go @@ -11,10 +11,10 @@ func TestDelete(t *testing.T) { defer tc.close() tc2 := startNoSwitchboard(t) - defer tc2.close() + defer tc2.closeNoWait() tc3 := startNoSwitchboard(t) - defer tc3.close() + defer tc3.closeNoWait() tc.client.Login("mjl@mox.example", password0) tc2.client.Login("mjl@mox.example", password0) @@ -24,6 +24,7 @@ func TestDelete(t *testing.T) { tc.transactf("no", "delete inbox") // Cannot delete inbox. tc.transactf("no", "delete nonexistent") // Cannot delete mailbox that does not exist. tc.transactf("no", `delete "nonexistent"`) // Again, with quoted string syntax. + tc.transactf("no", `delete "expungebox"`) // Already removed. tc.client.Subscribe("x") tc.transactf("no", "delete x") // Subscription does not mean there is a mailbox that can be deleted. diff --git a/imapserver/expunge_test.go b/imapserver/expunge_test.go index 0ecb232..cae39b6 100644 --- a/imapserver/expunge_test.go +++ b/imapserver/expunge_test.go @@ -12,7 +12,7 @@ func TestExpunge(t *testing.T) { defer tc.close() tc2 := startNoSwitchboard(t) - defer tc2.close() + defer tc2.closeNoWait() tc.client.Login("mjl@mox.example", password0) tc.client.Select("inbox") diff --git a/imapserver/fetch.go b/imapserver/fetch.go index 8ddbbde..b83d554 100644 --- a/imapserver/fetch.go +++ b/imapserver/fetch.go @@ -5,7 +5,6 @@ package imapserver import ( "bytes" "context" - "errors" "fmt" "io" "log/slog" @@ -26,11 +25,11 @@ import ( // functions to handle fetch attribute requests are defined on fetchCmd. type fetchCmd struct { conn *conn - isUID bool // If this is a UID FETCH command. - rtx *bstore.Tx // Read-only transaction, kept open while processing all messages. - updateSeen []int64 // IDs of messages to mark as seen, after processing all messages. - hasChangedSince bool // Whether CHANGEDSINCE was set. Enables MODSEQ in response. - expungeIssued bool // Set if any message cannot be read. Can happen for expunged messages. + isUID bool // If this is a UID FETCH command. + rtx *bstore.Tx // Read-only transaction, kept open while processing all messages. + updateSeen []store.UID // To mark as seen after processing all messages. UID instead of message ID since moved messages keep their ID and insert a new ID in the original mailbox. + hasChangedSince bool // Whether CHANGEDSINCE was set. Enables MODSEQ in response. + expungeIssued bool // Set if any message has been expunged. Can happen for expunged messages. uid store.UID // UID currently processing. markSeen bool @@ -271,15 +270,16 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) { changes := make([]store.Change, 0, len(cmd.updateSeen)+1) c.xdbwrite(func(wtx *bstore.Tx) { - mb := store.Mailbox{ID: c.mailboxID} - err = wtx.Get(&mb) + mb, err := store.MailboxID(wtx, c.mailboxID) + if err == store.ErrMailboxExpunged { + xusercodeErrorf("NONEXISTENT", "mailbox has been expunged") + } xcheckf(err, "get mailbox for updating counts after marking as seen") var modseq store.ModSeq - for _, id := range cmd.updateSeen { - m := store.Message{ID: id} - err := wtx.Get(&m) + for _, uid := range cmd.updateSeen { + m, err := bstore.QueryTx[store.Message](wtx).FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: uid}).Get() xcheckf(err, "get message") if m.Expunged { // Message has been deleted in the mean time. @@ -322,7 +322,8 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) { if cmd.expungeIssued { // ../rfc/2180:343 - c.writeresultf("%s NO [EXPUNGEISSUED] at least one message was expunged", tag) + // ../rfc/9051:5102 + c.writeresultf("%s OK [EXPUNGEISSUED] at least one message was expunged", tag) } else { c.ok(tag, cmdstr) } @@ -333,12 +334,16 @@ func (cmd *fetchCmd) xensureMessage() *store.Message { return cmd.m } + // We do not filter by Expunged, the message may have been deleted in other + // sessions, but not in ours. q := bstore.QueryTx[store.Message](cmd.rtx) q.FilterNonzero(store.Message{MailboxID: cmd.conn.mailboxID, UID: cmd.uid}) - q.FilterEqual("Expunged", false) m, err := q.Get() cmd.xcheckf(err, "get message for uid %d", cmd.uid) cmd.m = &m + if m.Expunged { + cmd.expungeIssued = true + } return cmd.m } @@ -382,10 +387,6 @@ func (cmd *fetchCmd) process(atts []fetchAtt) { if !ok { panic(x) } - if errors.Is(err, bstore.ErrAbsent) { - cmd.expungeIssued = true - return - } cmd.conn.log.Infox("processing fetch attribute", err, slog.Any("uid", cmd.uid)) xuserErrorf("processing fetch attribute: %v", err) }() @@ -401,8 +402,7 @@ func (cmd *fetchCmd) process(atts []fetchAtt) { } if cmd.markSeen { - m := cmd.xensureMessage() - cmd.updateSeen = append(cmd.updateSeen, m.ID) + cmd.updateSeen = append(cmd.updateSeen, cmd.uid) } if cmd.needFlags { diff --git a/imapserver/fetch_test.go b/imapserver/fetch_test.go index 4d091f5..286d8c8 100644 --- a/imapserver/fetch_test.go +++ b/imapserver/fetch_test.go @@ -444,6 +444,16 @@ Content-Transfer-Encoding: Quoted-printable tc.client.Unselect() tc.client.Examine("inbox") + // Start a second session. Use it to remove the message. First session should still + // be able to access the messages. + tc2 := startNoSwitchboard(t) + defer tc2.closeNoWait() + tc2.client.Login("mjl@mox.example", password0) + tc2.client.Select("inbox") + tc2.client.StoreFlagsSet("1", true, `\Deleted`) + tc2.client.Expunge() + tc2.client.Logout() + tc.transactf("ok", "fetch 1 binary[]") tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binary1}}) diff --git a/imapserver/fuzz_test.go b/imapserver/fuzz_test.go index ef647d9..dad691a 100644 --- a/imapserver/fuzz_test.go +++ b/imapserver/fuzz_test.go @@ -70,7 +70,7 @@ func FuzzServer(f *testing.F) { } defer func() { acc.Close() - acc.CheckClosed() + acc.WaitClosed() }() err = acc.SetPassword(log, password0) if err != nil { diff --git a/imapserver/idle_test.go b/imapserver/idle_test.go index b733bd1..68fd135 100644 --- a/imapserver/idle_test.go +++ b/imapserver/idle_test.go @@ -13,7 +13,7 @@ func TestIdle(t *testing.T) { defer tc1.close() tc2 := startNoSwitchboard(t) - defer tc2.close() + defer tc2.closeNoWait() tc1.client.Login("mjl@mox.example", password0) tc2.client.Login("mjl@mox.example", password0) diff --git a/imapserver/list.go b/imapserver/list.go index d65cb72..2367beb 100644 --- a/imapserver/list.go +++ b/imapserver/list.go @@ -3,11 +3,12 @@ package imapserver import ( "bytes" "fmt" - "path" "sort" "strings" "github.com/mjl-/bstore" + + "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/store" ) @@ -145,10 +146,11 @@ func (c *conn) cmdList(tag, cmd string, p *parser) { var nameList []string q := bstore.QueryTx[store.Mailbox](tx) + q.FilterEqual("Expunged", false) err := q.ForEach(func(mb store.Mailbox) error { names[mb.Name] = info{mailbox: &mb} nameList = append(nameList, mb.Name) - for p := path.Dir(mb.Name); p != "."; p = path.Dir(p) { + for p := mox.ParentMailboxName(mb.Name); p != ""; p = mox.ParentMailboxName(p) { hasChild[p] = true } return nil @@ -163,7 +165,7 @@ func (c *conn) cmdList(tag, cmd string, p *parser) { if !ok { nameList = append(nameList, sub.Name) } - for p := path.Dir(sub.Name); p != "."; p = path.Dir(p) { + for p := mox.ParentMailboxName(sub.Name); p != ""; p = mox.ParentMailboxName(p) { hasSubscribedChild[p] = true } return nil @@ -234,7 +236,10 @@ func (c *conn) cmdList(tag, cmd string, p *parser) { if info.mailbox != nil && len(retMetadata) > 0 { var meta listspace for _, k := range retMetadata { - a, err := bstore.QueryTx[store.Annotation](tx).FilterNonzero(store.Annotation{MailboxID: info.mailbox.ID, Key: k}).Get() + q := bstore.QueryTx[store.Annotation](tx) + q.FilterNonzero(store.Annotation{MailboxID: info.mailbox.ID, Key: k}) + q.FilterEqual("Expunged", false) + a, err := q.Get() var v token if err == bstore.ErrAbsent { v = nilt diff --git a/imapserver/list_test.go b/imapserver/list_test.go index d7aa29f..f9dc90b 100644 --- a/imapserver/list_test.go +++ b/imapserver/list_test.go @@ -26,6 +26,9 @@ func TestListBasic(t *testing.T) { tc.last(tc.client.List("Inbox")) tc.xuntagged(ulist("Inbox")) + tc.last(tc.client.List("expungebox")) + tc.xuntagged() + tc.last(tc.client.List("%")) tc.xuntagged(ulist("Archive", `\Archive`), ulist("Drafts", `\Drafts`), ulist("Inbox"), ulist("Junk", `\Junk`), ulist("Sent", `\Sent`), ulist("Trash", `\Trash`)) @@ -78,7 +81,7 @@ func TestListExtended(t *testing.T) { for _, name := range store.DefaultInitialMailboxes.Regular { uidvals[name] = 1 } - var uidvalnext uint32 = 2 + var uidvalnext uint32 = 3 uidval := func(name string) uint32 { v, ok := uidvals[name] if !ok { diff --git a/imapserver/lsub_test.go b/imapserver/lsub_test.go index 4855142..bb61455 100644 --- a/imapserver/lsub_test.go +++ b/imapserver/lsub_test.go @@ -19,6 +19,9 @@ func TestLsub(t *testing.T) { tc.transactf("ok", `lsub "" x*`) tc.xuntagged() + tc.transactf("ok", `lsub "" expungebox`) + tc.xuntagged(imapclient.UntaggedLsub{Separator: '/', Mailbox: "expungebox"}) + tc.transactf("ok", "create a/b/c") tc.transactf("ok", `lsub "" a/*`) tc.xuntagged(imapclient.UntaggedLsub{Separator: '/', Mailbox: "a/b"}, imapclient.UntaggedLsub{Separator: '/', Mailbox: "a/b/c"}) diff --git a/imapserver/metadata.go b/imapserver/metadata.go index 8806542..8f7ffa7 100644 --- a/imapserver/metadata.go +++ b/imapserver/metadata.go @@ -100,7 +100,7 @@ func (c *conn) cmdGetmetadata(tag, cmd string, p *parser) { mb := c.xmailbox(tx, mailboxName, "TRYCREATE") q.FilterNonzero(store.Annotation{MailboxID: mb.ID}) } - + q.FilterEqual("Expunged", false) q.SortAsc("MailboxID", "Key") // For tests. err := q.ForEach(func(a store.Annotation) error { // ../rfc/5464:516 @@ -246,41 +246,35 @@ func (c *conn) cmdSetmetadata(tag, cmd string, p *parser) { q := bstore.QueryTx[store.Annotation](tx) q.FilterNonzero(store.Annotation{Key: a.Key}) q.FilterEqual("MailboxID", mb.ID) // Can be zero. - + q.FilterEqual("Expunged", false) + oa, err := q.Get() // Nil means remove. ../rfc/5464:579 - if a.Value == nil { - var deleted []store.Annotation - q.Gather(&deleted) - _, err := q.Delete() - xcheckf(err, "deleting annotation") - for _, oa := range deleted { - changes = append(changes, oa.Change(mailboxName)) - } + if err == bstore.ErrAbsent && a.Value == nil { continue } - if modseq == 0 { var err error modseq, err = c.account.NextModSeq(tx) xcheckf(err, "get next modseq") } - - a.MailboxID = mb.ID - a.ModSeq = modseq - a.CreateSeq = modseq - - oa, err := q.Get() if err == bstore.ErrAbsent { + a.MailboxID = mb.ID + a.CreateSeq = modseq + a.ModSeq = modseq err = tx.Insert(&a) xcheckf(err, "inserting annotation") - changes = append(changes, a.Change(mailboxName)) - continue + } else { + xcheckf(err, "get metadata") + oa.ModSeq = modseq + if a.Value == nil { + oa.Expunged = true + } + oa.IsString = a.IsString + oa.Value = a.Value + err = tx.Update(&oa) + xcheckf(err, "updating metdata") } - xcheckf(err, "looking up existing annotation for entry name") changes = append(changes, a.Change(mailboxName)) - oa.Value = a.Value - err = tx.Update(&oa) - xcheckf(err, "updating metadata annotation") } c.xcheckMetadataSize(tx) @@ -304,7 +298,7 @@ func (c *conn) xcheckMetadataSize(tx *bstore.Tx) { // ../rfc/5464:383 var n int var size int - err := bstore.QueryTx[store.Annotation](tx).ForEach(func(a store.Annotation) error { + err := bstore.QueryTx[store.Annotation](tx).FilterEqual("Expunged", false).ForEach(func(a store.Annotation) error { n++ if n > metadataMaxKeys { // ../rfc/5464:590 diff --git a/imapserver/metadata_test.go b/imapserver/metadata_test.go index 1144b7b..50090d2 100644 --- a/imapserver/metadata_test.go +++ b/imapserver/metadata_test.go @@ -23,6 +23,15 @@ func TestMetadata(t *testing.T) { tc.transactf("ok", `setmetadata "" (/PRIVATE/COMMENT "global value")`) tc.transactf("ok", `setmetadata inbox (/private/comment "mailbox value")`) + tc.transactf("ok", `create metabox`) + tc.transactf("ok", `setmetadata metabox (/private/comment "mailbox value")`) + tc.transactf("ok", `setmetadata metabox (/shared/comment "mailbox value")`) + tc.transactf("ok", `setmetadata metabox (/shared/comment nil)`) // Remove. + tc.transactf("ok", `delete metabox`) // Delete mailbox with live and expunged metadata. + + tc.transactf("no", `setmetadata expungebox (/private/comment "mailbox value")`) + tc.xcode("TRYCREATE") + tc.transactf("ok", `getmetadata "" ("/private/comment")`) tc.xuntagged(imapclient.UntaggedMetadataAnnotations{ Mailbox: "", @@ -176,7 +185,7 @@ func TestMetadata(t *testing.T) { // Broadcast should not happen when metadata capability is not enabled. tc2 := startNoSwitchboard(t) - defer tc2.close() + defer tc2.closeNoWait() tc2.client.Login("mjl@mox.example", password0) tc2.client.Select("inbox") diff --git a/imapserver/move_test.go b/imapserver/move_test.go index f49fcfd..ef135a2 100644 --- a/imapserver/move_test.go +++ b/imapserver/move_test.go @@ -12,10 +12,10 @@ func TestMove(t *testing.T) { defer tc.close() tc2 := startNoSwitchboard(t) - defer tc2.close() + defer tc2.closeNoWait() tc3 := startNoSwitchboard(t) - defer tc3.close() + defer tc3.closeNoWait() tc.client.Login("mjl@mox.example", password0) tc.client.Select("inbox") @@ -47,6 +47,9 @@ func TestMove(t *testing.T) { tc.transactf("no", "move 1 nonexistent") tc.xcode("TRYCREATE") + tc.transactf("no", "move 1 expungebox") + tc.xcode("TRYCREATE") + tc.transactf("no", "move 1 inbox") // Cannot move to same mailbox. tc2.transactf("ok", "noop") // Drain. diff --git a/imapserver/rename_test.go b/imapserver/rename_test.go index b74be26..47d09be 100644 --- a/imapserver/rename_test.go +++ b/imapserver/rename_test.go @@ -12,7 +12,7 @@ func TestRename(t *testing.T) { defer tc.close() tc2 := startNoSwitchboard(t) - defer tc2.close() + defer tc2.closeNoWait() tc.client.Login("mjl@mox.example", password0) tc2.client.Login("mjl@mox.example", password0) @@ -23,7 +23,9 @@ func TestRename(t *testing.T) { tc.transactf("no", "rename doesnotexist newbox") // Does not exist. tc.xcode("NONEXISTENT") // ../rfc/9051:5140 - tc.transactf("no", `rename "Sent" "Trash"`) // Already exists. + tc.transactf("no", "rename expungebox newbox") // No longer exists. + tc.xcode("NONEXISTENT") + tc.transactf("no", `rename "Sent" "Trash"`) // Already exists. tc.xcode("ALREADYEXISTS") tc.client.Create("x", nil) @@ -70,28 +72,51 @@ func TestRename(t *testing.T) { tc.client.Unsubscribe("k") tc.transactf("ok", "rename k/l k/l/m") // With "l" renamed, a new "k" will be created. tc.transactf("ok", `list "" "k*" return (subscribed)`) - tc.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "k"}, imapclient.UntaggedList{Flags: []string{"\\Subscribed"}, Separator: '/', Mailbox: "k/l"}, imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/m"}) + tc.xuntagged( + imapclient.UntaggedList{Separator: '/', Mailbox: "k"}, + imapclient.UntaggedList{Flags: []string{"\\Subscribed"}, Separator: '/', Mailbox: "k/l"}, + imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/m"}, + ) + + tc.transactf("ok", "rename k/l/m k/l/x/y/m") // k/l/x and k/l/x/y will be created. + tc.transactf("ok", `list "" "k/l/x*" return (subscribed)`) + tc.xuntagged( + imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/x"}, + imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/x/y"}, + imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/x/y/m"}, + ) // Renaming inbox keeps inbox in existence, moves messages, and does not rename children. tc.transactf("ok", "create inbox/a") // To check if UIDs are renumbered properly, we add UIDs 1 and 2. Expunge 1, // keeping only 2. Then rename the inbox, which should renumber UID 2 in the old // inbox to UID 1 in the newly created mailbox. - tc.transactf("ok", "append inbox (\\deleted) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx") - tc.transactf("ok", "append inbox (label1) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx") + tc.transactf("ok", "append inbox (\\deleted) {1+}\r\nx") + tc.transactf("ok", "append inbox (label1) {1+}\r\nx") tc.transactf("ok", `select inbox`) tc.transactf("ok", "expunge") - tc.transactf("ok", "rename inbox minbox") - tc.transactf("ok", `list "" (inbox inbox/a minbox)`) - tc.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"}, imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox/a"}, imapclient.UntaggedList{Separator: '/', Mailbox: "minbox"}) - tc.transactf("ok", `select minbox`) + tc.transactf("ok", "rename inbox x/minbox") + tc.transactf("ok", `list "" (inbox inbox/a x/minbox)`) + tc.xuntagged( + imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"}, + imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox/a"}, + imapclient.UntaggedList{Separator: '/', Mailbox: "x/minbox"}, + ) + tc.transactf("ok", `select x/minbox`) tc.transactf("ok", `uid fetch 1:* flags`) tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}}}) // Renaming to new hiearchy that does not have any subscribes. - tc.transactf("ok", "rename minbox w/w") + tc.transactf("ok", "rename x/minbox w/w") tc.transactf("ok", `list "" "w*"`) tc.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "w"}, imapclient.UntaggedList{Separator: '/', Mailbox: "w/w"}) + tc.transactf("ok", "rename inbox misc/old/inbox") + tc.transactf("ok", `list "" (misc misc/old/inbox)`) + tc.xuntagged( + imapclient.UntaggedList{Separator: '/', Mailbox: "misc"}, + imapclient.UntaggedList{Separator: '/', Mailbox: "misc/old/inbox"}, + ) + // todo: test create+delete+rename of/to a name results in a higher uidvalidity. } diff --git a/imapserver/replace.go b/imapserver/replace.go index a64e10c..26a6d31 100644 --- a/imapserver/replace.go +++ b/imapserver/replace.go @@ -1,7 +1,6 @@ package imapserver import ( - "context" "errors" "io" "os" @@ -116,7 +115,7 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) { q := bstore.QueryTx[store.Message](tx) q.FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: uidOld}) q.FilterEqual("Expunged", false) - om, err := q.Get() + _, err = q.Get() if err == bstore.ErrAbsent { return nil } @@ -124,8 +123,8 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) { return func() { xserverErrorf("get message to replace: %v", err) } } - delta := size - om.Size - ok, maxSize, err := c.account.CanAddMessageSize(tx, delta) + // Check if we can add size bytes. We can't necessarily remove the current message yet. + ok, maxSize, err := c.account.CanAddMessageSize(tx, size) if err != nil { return func() { xserverErrorf("check quota: %v", err) } } @@ -169,9 +168,7 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) { var file *os.File var newMsgPath string var f io.Writer - var committed bool - - var oldMsgPath string // To remove on success. + var commit bool if errfn != nil { // We got a non-sync literal, we will consume some data, but abort if there's too @@ -197,14 +194,10 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) { err := file.Close() c.xsanity(err, "close temporary file for replace") } - if newMsgPath != "" && !committed { + if newMsgPath != "" && !commit { err := os.Remove(newMsgPath) c.xsanity(err, "remove temporary file for replace") } - if committed { - err := os.Remove(oldMsgPath) - c.xsanity(err, "remove old message") - } }() } @@ -258,8 +251,8 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) { return } - // Check quota. Even if the delta is negative, the quota may have changed. - ok, maxSize, err := c.account.CanAddMessageSize(tx, mw.Size-om.Size) + // Check quota for addition of new message. We can't necessarily yet remove the old message. + ok, maxSize, err := c.account.CanAddMessageSize(tx, mw.Size) xcheckf(err, "checking quota") if !ok { // ../rfc/9208:472 @@ -269,31 +262,11 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) { modseq, err := c.account.NextModSeq(tx) xcheckf(err, "get next mod seq") - // Subtract counts for message from source mailbox. - mbSrc.Sub(om.MailboxCounts()) + chremuids, _, err := c.account.MessageRemove(c.log, tx, modseq, &mbSrc, store.RemoveOpts{}, om) + xcheckf(err, "expunge old message") + changes = append(changes, chremuids) + // Note: we only add a mbSrc counts change later on, if it is not equal to mbDst. - // Remove message recipients for old message. - _, err = bstore.QueryTx[store.Recipient](tx).FilterNonzero(store.Recipient{MessageID: om.ID}).Delete() - xcheckf(err, "removing message recipients") - - // Subtract size of old message from account. - err = c.account.AddMessageSize(c.log, tx, -om.Size) - xcheckf(err, "updating disk usage") - - // Undo any junk filter training for the old message. - om.Junk = false - om.Notjunk = false - err = c.account.RetrainMessages(context.TODO(), c.log, tx, []store.Message{om}) - xcheckf(err, "untraining expunged messages") - - // Mark old message expunged. - om.ModSeq = modseq - om.PrepareExpunge() - err = tx.Update(&om) - xcheckf(err, "mark old message as expunged") - - // Update source mailbox. - mbSrc.ModSeq = modseq err = tx.Update(&mbSrc) xcheckf(err, "updating source mailbox counts") @@ -316,22 +289,16 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) { err = c.account.MessageAdd(c.log, tx, &mbDst, &nm, file, store.AddOpts{}) xcheckf(err, "delivering message") + // Update path to what is stored in the account. We may still have to clean it up on errors. + newMsgPath = c.account.MessagePath(nm.ID) - changes = append(changes, - store.ChangeRemoveUIDs{MailboxID: om.MailboxID, UIDs: []store.UID{om.UID}, ModSeq: om.ModSeq}, - nm.ChangeAddUID(), - mbDst.ChangeCounts(), - ) + changes = append(changes, nm.ChangeAddUID(), mbDst.ChangeCounts()) if nkeywords != len(mbDst.Keywords) { changes = append(changes, mbDst.ChangeKeywords()) } err = tx.Update(&mbDst) xcheckf(err, "updating destination mailbox") - - // Update path to what is stored in the account. We may still have to clean it up on errors. - newMsgPath = c.account.MessagePath(nm.ID) - oldMsgPath = c.account.MessagePath(om.ID) }) // Fetch pending changes, possibly with new UIDs, so we can apply them before adding our own new UID. @@ -342,7 +309,7 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) { } // Success, make sure messages aren't cleaned up anymore. - committed = true + commit = true // Broadcast the change to other connections. if mbSrc.ID != mbDst.ID { diff --git a/imapserver/replace_test.go b/imapserver/replace_test.go index 5a0a691..2b6a31d 100644 --- a/imapserver/replace_test.go +++ b/imapserver/replace_test.go @@ -13,7 +13,7 @@ func TestReplace(t *testing.T) { defer tc.close() tc2 := startNoSwitchboard(t) - defer tc2.close() + defer tc2.closeNoWait() tc.client.Login("mjl@mox.example", password0) tc.client.Select("inbox") @@ -23,6 +23,9 @@ func TestReplace(t *testing.T) { tc.client.StoreFlagsSet("1", true, `\deleted`) tc.client.Expunge() + tc.transactf("no", "replace 2 expungebox {1}") // Mailbox no longer exists. + tc.xcode("TRYCREATE") + tc2.client.Login("mjl@mox.example", password0) tc2.client.Select("inbox") @@ -34,7 +37,7 @@ func TestReplace(t *testing.T) { imapclient.UntaggedExists(3), imapclient.UntaggedExpunge(2), ) - tc.xcodeArg(imapclient.CodeHighestModSeq(6)) + tc.xcodeArg(imapclient.CodeHighestModSeq(8)) // Check that other client sees Exists and Expunge. tc2.transactf("ok", "noop") @@ -53,7 +56,7 @@ func TestReplace(t *testing.T) { imapclient.UntaggedExists(3), imapclient.UntaggedVanished{UIDs: xparseNumSet("2")}, ) - tc.xcodeArg(imapclient.CodeHighestModSeq(7)) + tc.xcodeArg(imapclient.CodeHighestModSeq(9)) // Leftover data. tc.transactf("bad", "replace 1 inbox () {6+}\r\ntest\r\n ") @@ -125,7 +128,7 @@ func TestReplaceExpunged(t *testing.T) { // Get in with second client and remove the message we are replacing. tc2 := startNoSwitchboard(t) - defer tc2.close() + defer tc2.closeNoWait() tc2.client.Login("mjl@mox.example", password0) tc2.client.Select("inbox") tc2.client.StoreFlagsSet("1", true, `\Deleted`) diff --git a/imapserver/search_test.go b/imapserver/search_test.go index b0dae76..4af4203 100644 --- a/imapserver/search_test.go +++ b/imapserver/search_test.go @@ -406,7 +406,6 @@ func TestSearch(t *testing.T) { writeTextLit(1, true) } writeTextLit(1, false) - } // esearchall makes an UntaggedEsearch response with All set, for comparisons. diff --git a/imapserver/selectexamine_test.go b/imapserver/selectexamine_test.go index 3189940..40ca952 100644 --- a/imapserver/selectexamine_test.go +++ b/imapserver/selectexamine_test.go @@ -49,6 +49,7 @@ func testSelectExamine(t *testing.T, examine bool) { // Mailbox does not exist. tc.transactf("no", "%s bogus", cmd) + tc.transactf("no", "%s expungebox", cmd) tc.transactf("ok", "%s inbox", cmd) tc.xuntagged(uflags, upermflags, urecent, uexists0, uuidval1, uuidnext1, ulist) diff --git a/imapserver/server.go b/imapserver/server.go index eb702e6..6647279 100644 --- a/imapserver/server.go +++ b/imapserver/server.go @@ -26,7 +26,6 @@ non-ASCII UTF-8. Until that's enabled, we do use UTF-7 for mailbox names. See /* - todo: do not return binary data for a fetch body. at least not for imap4rev1. we should be encoding it as base64? -- todo: on expunge we currently remove the message even if other sessions still have a reference to the uid. if they try to query the uid, they'll get an error. we could be nicer and only actually remove the message when the last reference has gone. we could add a new flag to store.Message marking the message as expunged, not give new session access to such messages, and make store remove them at startup, and clean them when the last session referencing the session goes. however, it will get much more complicated. renaming messages would need special handling. and should we do the same for removed mailboxes? - todo: try to recover from syntax errors when the last command line ends with a }, i.e. a literal. we currently abort the entire connection. we may want to read some amount of literal data and continue with a next command. */ @@ -69,6 +68,7 @@ import ( "github.com/mjl-/flate" "github.com/mjl-/mox/config" + "github.com/mjl-/mox/junk" "github.com/mjl-/mox/message" "github.com/mjl-/mox/metrics" "github.com/mjl-/mox/mlog" @@ -1557,10 +1557,12 @@ func (c *conn) xmailbox(tx *bstore.Tx, name string, missingErrCode string) store // If the mailbox does not exist, panic is called with a user error. // Must be called with account rlock held. func (c *conn) xmailboxID(tx *bstore.Tx, id int64) store.Mailbox { - mb := store.Mailbox{ID: id} - err := tx.Get(&mb) + mb, err := store.MailboxID(tx, id) if err == bstore.ErrAbsent { xuserErrorf("%w", store.ErrUnknownMailbox) + } else if err == store.ErrMailboxExpunged { + // ../rfc/9051:5140 + xusercodeErrorf("NONEXISTENT", "mailbox has been deleted") } return mb } @@ -1589,6 +1591,7 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { mbID = ch.MailboxID case store.ChangeRemoveUIDs: mbID = ch.MailboxID + c.comm.RemovalSeen(ch) case store.ChangeFlags: mbID = ch.MailboxID case store.ChangeRemoveMailbox, store.ChangeAddMailbox, store.ChangeRenameMailbox, store.ChangeAddSubscription: @@ -2881,9 +2884,6 @@ func (c *conn) cmdDelete(tag, cmd string, p *parser) { name = xcheckmailboxname(name, false) - // Messages to remove after having broadcasted the removal of messages. - var removeMessageIDs []int64 - c.account.WithWLock(func() { var mb store.Mailbox var changes []store.Change @@ -2893,7 +2893,7 @@ func (c *conn) cmdDelete(tag, cmd string, p *parser) { var hasChildren bool var err error - changes, removeMessageIDs, hasChildren, err = c.account.MailboxDelete(context.TODO(), c.log, tx, mb) + changes, hasChildren, err = c.account.MailboxDelete(context.TODO(), c.log, tx, &mb) if hasChildren { xusercodeErrorf("HASCHILDREN", "mailbox has a child, only leaf mailboxes can be deleted") } @@ -2903,12 +2903,6 @@ func (c *conn) cmdDelete(tag, cmd string, p *parser) { c.broadcast(changes) }) - for _, mID := range removeMessageIDs { - p := c.account.MessagePath(mID) - err := os.Remove(p) - c.xsanity(err, "removing message file %q for mailbox delete", p) - } - c.ok(tag, cmd) } @@ -2934,14 +2928,21 @@ func (c *conn) cmdRename(tag, cmd string, p *parser) { src = xcheckmailboxname(src, true) dst = xcheckmailboxname(dst, false) + var cleanupIDs []int64 + defer func() { + for _, id := range cleanupIDs { + p := c.account.MessagePath(id) + err := os.Remove(p) + c.xsanity(err, "cleaning up message") + } + }() + c.account.WithWLock(func() { var changes []store.Change c.xdbwrite(func(tx *bstore.Tx) { srcMB := c.xmailbox(tx, src, "NONEXISTENT") - var modseq store.ModSeq - // Inbox is very special. Unlike other mailboxes, its children are not moved. And // unlike a regular move, its messages are moved to a newly created mailbox. We do // indeed create a new destination mailbox and actually move the messages. @@ -2956,111 +2957,53 @@ func (c *conn) cmdRename(tag, cmd string, p *parser) { xuserErrorf("cannot move inbox to itself") } - uidval, err := c.account.NextUIDValidity(tx) - xcheckf(err, "next uid validity") + var modseq store.ModSeq + dstMB, chl, err := c.account.MailboxEnsure(tx, dst, false, store.SpecialUse{}, &modseq) + xcheckf(err, "creating destination mailbox") + changes = chl - modseq, err = c.account.NextModSeq(tx) - xcheckf(err, "assigning next modseq") - - dstMB := store.Mailbox{ - Name: dst, - UIDValidity: uidval, - UIDNext: 1, - Keywords: srcMB.Keywords, - ModSeq: modseq, - CreateSeq: modseq, - HaveCounts: true, + // Copy mailbox annotations. ../rfc/5464:368 + qa := bstore.QueryTx[store.Annotation](tx) + qa.FilterNonzero(store.Annotation{MailboxID: srcMB.ID}) + qa.FilterEqual("Expunged", false) + annotations, err := qa.List() + xcheckf(err, "get annotations to copy for inbox") + for _, a := range annotations { + a.ID = 0 + a.MailboxID = dstMB.ID + a.ModSeq = modseq + a.CreateSeq = modseq + err := tx.Insert(&a) + xcheckf(err, "copy annotation to destination mailbox") + changes = append(changes, a.Change(dstMB.Name)) } - err = tx.Insert(&dstMB) - xcheckf(err, "create new destination mailbox") + c.xcheckMetadataSize(tx) - changes = make([]store.Change, 2) // Placeholders filled in below. - - // Move existing messages, with their ID's and on-disk files intact, to the new - // mailbox. We keep the expunged messages, the destination mailbox doesn't care - // about them. - var oldUIDs []store.UID + // Build query that selects messages to move. q := bstore.QueryTx[store.Message](tx) q.FilterNonzero(store.Message{MailboxID: srcMB.ID}) q.FilterEqual("Expunged", false) q.SortAsc("UID") - err = q.ForEach(func(m store.Message) error { - om := m - om.ID = 0 - om.ModSeq = modseq - om.PrepareExpunge() - oldUIDs = append(oldUIDs, om.UID) - mc := m.MailboxCounts() - srcMB.Sub(mc) - dstMB.Add(mc) - - m.MailboxID = dstMB.ID - m.UID = dstMB.UIDNext - dstMB.UIDNext++ - m.CreateSeq = modseq - m.ModSeq = modseq - if err := tx.Update(&m); err != nil { - return fmt.Errorf("updating message to move to new mailbox: %w", err) - } - - changes = append(changes, m.ChangeAddUID()) - - if err := tx.Insert(&om); err != nil { - return fmt.Errorf("adding empty expunge message record to inbox: %w", err) - } - return nil - }) - xcheckf(err, "moving messages from inbox to destination mailbox") - - err = tx.Update(&dstMB) - xcheckf(err, "updating uidnext and counts in destination mailbox") - - srcMB.ModSeq = modseq - err = tx.Update(&srcMB) - xcheckf(err, "updating counts for inbox") - - var dstFlags []string - if tx.Get(&store.Subscription{Name: dstMB.Name}) == nil { - dstFlags = []string{`\Subscribed`} - } - - // Copy any annotations. ../rfc/5464:368 - annotations, err := bstore.QueryTx[store.Annotation](tx).FilterNonzero(store.Annotation{MailboxID: srcMB.ID}).List() - xcheckf(err, "get annotations to copy for inbox") - for i := range annotations { - annotations[i].ID = 0 - annotations[i].MailboxID = dstMB.ID - annotations[i].ModSeq = modseq - annotations[i].CreateSeq = modseq - err := tx.Insert(&annotations[i]) - xcheckf(err, "copy annotation to destination mailbox") - } - - c.xcheckMetadataSize(tx) - - changes[0] = store.ChangeRemoveUIDs{MailboxID: srcMB.ID, UIDs: oldUIDs, ModSeq: modseq} - changes[1] = store.ChangeAddMailbox{Mailbox: dstMB, Flags: dstFlags, ModSeq: modseq} - // changes[2:...] are ChangeAddUIDs - changes = append(changes, srcMB.ChangeCounts(), dstMB.ChangeCounts()) - for _, a := range annotations { - changes = append(changes, a.Change(dstMB.Name)) - } + newIDs, chl := c.xmoveMessages(tx, q, 0, modseq, &srcMB, &dstMB) + changes = append(changes, chl...) + cleanupIDs = newIDs return } - var notExists, alreadyExists bool + var modseq store.ModSeq + var alreadyExists bool var err error - changes, _, notExists, alreadyExists, err = c.account.MailboxRename(tx, srcMB, dst, &modseq) - if notExists { - // ../rfc/9051:5140 - xusercodeErrorf("NONEXISTENT", "%s", err) - } else if alreadyExists { + changes, _, alreadyExists, err = c.account.MailboxRename(tx, &srcMB, dst, &modseq) + if alreadyExists { xusercodeErrorf("ALREADYEXISTS", "%s", err) } xcheckf(err, "renaming mailbox") }) + + cleanupIDs = nil + c.broadcast(changes) }) @@ -3163,7 +3106,7 @@ func (c *conn) cmdLsub(tag, cmd string, p *parser) { for _, sub := range subscriptions { name := sub.Name if ispercent { - for p := path.Dir(name); p != "."; p = path.Dir(p) { + for p := mox.ParentMailboxName(name); p != ""; p = mox.ParentMailboxName(p) { subscribedKids[p] = true } } @@ -3181,6 +3124,7 @@ func (c *conn) cmdLsub(tag, cmd string, p *parser) { return } qmb := bstore.QueryTx[store.Mailbox](tx) + qmb.FilterEqual("Expunged", false) qmb.SortAsc("Name") err = qmb.ForEach(func(mb store.Mailbox) error { if have[mb.Name] || !subscribedKids[mb.Name] || !re.MatchString(mb.Name) { @@ -3544,12 +3488,11 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) { // todo: do a single junk training err = c.account.MessageAdd(c.log, tx, &mb, &a.m, a.file, store.AddOpts{SkipDirSync: true}) xcheckf(err, "delivering message") - - changes = append(changes, a.m.ChangeAddUID()) - // Update path to what is stored in the account. We may still have to clean it up on errors. a.path = c.account.MessagePath(a.m.ID) + changes = append(changes, a.m.ChangeAddUID()) + msgDirs[filepath.Dir(a.path)] = struct{}{} } @@ -3759,29 +3702,29 @@ func (c *conn) cmdClose(tag, cmd string, p *parser) { } // expunge messages marked for deletion in currently selected/active mailbox. -// if uidSet is not nil, only messages matching the set are deleted. +// if uidSet is not nil, only messages matching the set are expunged. // -// messages that have been marked expunged from the database are returned and -// have already been removed. +// Messages that have been marked expunged from the database are returned. While +// other sessions still reference the message, it is not cleared from the database +// yet, and the message file is not yet removed. // -// the highest modseq in the mailbox is returned, typically associated with the +// The highest modseq in the mailbox is returned, typically associated with the // removal of the messages, but if no messages were expunged the current latest max // modseq for the mailbox is returned. -func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) (removed []store.Message, highestModSeq store.ModSeq) { - var modseq store.ModSeq - +func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) (expunged []store.Message, highestModSeq store.ModSeq) { c.account.WithWLock(func() { - var mb store.Mailbox + var changes []store.Change c.xdbwrite(func(tx *bstore.Tx) { - mb = store.Mailbox{ID: c.mailboxID} - err := tx.Get(&mb) - if err == bstore.ErrAbsent { + mb, err := store.MailboxID(tx, c.mailboxID) + if err == bstore.ErrAbsent || err == store.ErrMailboxExpunged { if missingMailboxOK { return } - xuserErrorf("%w", store.ErrUnknownMailbox) + // ../rfc/9051:5140 + xusercodeErrorf("NONEXISTENT", "%w", store.ErrUnknownMailbox) } + xcheckf(err, "get mailbox") qm := bstore.QueryTx[store.Message](tx) qm.FilterNonzero(store.Message{MailboxID: c.mailboxID}) @@ -3792,82 +3735,32 @@ func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) (removed []store. return uidSearch(c.uids, m.UID) > 0 && (uidSet == nil || uidSet.containsUID(m.UID, c.uids, c.searchResult)) }) qm.SortAsc("UID") - removed, err = qm.List() - xcheckf(err, "listing messages to delete") + expunged, err = qm.List() + xcheckf(err, "listing messages to expunge") - if len(removed) == 0 { + if len(expunged) == 0 { highestModSeq = mb.ModSeq return } // Assign new modseq. - modseq, err = c.account.NextModSeq(tx) + modseq, err := c.account.NextModSeq(tx) xcheckf(err, "assigning next modseq") highestModSeq = modseq mb.ModSeq = modseq - removeIDs := make([]int64, len(removed)) - anyIDs := make([]any, len(removed)) - var totalSize int64 - for i, m := range removed { - removeIDs[i] = m.ID - anyIDs[i] = m.ID - mb.Sub(m.MailboxCounts()) - totalSize += m.Size - // Update "remove", because RetrainMessage below will save the message. - removed[i].Expunged = true - removed[i].ModSeq = modseq - } - qmr := bstore.QueryTx[store.Recipient](tx) - qmr.FilterEqual("MessageID", anyIDs...) - _, err = qmr.Delete() - xcheckf(err, "removing message recipients") - - qm = bstore.QueryTx[store.Message](tx) - qm.FilterIDs(removeIDs) - n, err := qm.UpdateNonzero(store.Message{Expunged: true, ModSeq: modseq}) - if err == nil && n != len(removeIDs) { - err = fmt.Errorf("only %d messages set to expunged, expected %d", n, len(removeIDs)) - } - xcheckf(err, "marking messages marked for deleted as expunged") + chremuids, chmbcounts, err := c.account.MessageRemove(c.log, tx, modseq, &mb, store.RemoveOpts{}, expunged...) + xcheckf(err, "expunging messages") + changes = append(changes, chremuids, chmbcounts) err = tx.Update(&mb) - xcheckf(err, "updating mailbox counts") - - err = c.account.AddMessageSize(c.log, tx, -totalSize) - xcheckf(err, "updating disk usage") - - // Mark expunged messages as not needing training, then retrain them, so if they - // were trained, they get untrained. - for i := range removed { - removed[i].Junk = false - removed[i].Notjunk = false - } - err = c.account.RetrainMessages(context.TODO(), c.log, tx, removed) - xcheckf(err, "untraining expunged messages") + xcheckf(err, "update mailbox") }) - // Broadcast changes to other connections. We may not have actually removed any - // messages, so take care not to send an empty update. - if len(removed) > 0 { - ouids := make([]store.UID, len(removed)) - for i, m := range removed { - ouids[i] = m.UID - } - changes := []store.Change{ - store.ChangeRemoveUIDs{MailboxID: c.mailboxID, UIDs: ouids, ModSeq: modseq}, - mb.ChangeCounts(), - } - c.broadcast(changes) - } - - for _, m := range removed { - p := c.account.MessagePath(m.ID) - err := os.Remove(p) - c.xsanity(err, "removing message file for expunge") - } + c.broadcast(changes) }) - return removed, highestModSeq + + return expunged, highestModSeq } // Unselect is similar to close in that it closes the currently active mailbox, but @@ -4052,9 +3945,9 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) { uids, uidargs := c.gatherCopyMoveUIDs(isUID, nums) // Files that were created during the copy. Remove them if the operation fails. - var createdIDs []int64 + var newIDs []int64 defer func() { - for _, id := range createdIDs { + for _, id := range newIDs { p := c.account.MessagePath(id) err := os.Remove(p) c.xsanity(err, "cleaning up created file") @@ -4200,7 +4093,7 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) { } err := moxio.LinkOrCopy(c.log, dst, src, nil, true) xcheckf(err, "link or copy file %q to %q", src, dst) - createdIDs = append(createdIDs, newMsgIDs[i]) + newIDs = append(newIDs, newMsgIDs[i]) } for dir := range syncDirs { @@ -4212,7 +4105,7 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) { xcheckf(err, "train copied messages") }) - createdIDs = nil + newIDs = nil // Broadcast changes to other connections. if len(newUIDs) > 0 { @@ -4253,127 +4146,61 @@ func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) { uids, uidargs := c.gatherCopyMoveUIDs(isUID, nums) - var mbSrc, mbDst store.Mailbox - var changes []store.Change - var newUIDs []store.UID + var mbDst store.Mailbox + var uidFirst store.UID var modseq store.ModSeq + var cleanupIDs []int64 + defer func() { + for _, id := range cleanupIDs { + p := c.account.MessagePath(id) + err := os.Remove(p) + c.xsanity(err, "removing destination message file %v", p) + } + }() + c.account.WithWLock(func() { + var changes []store.Change + c.xdbwrite(func(tx *bstore.Tx) { - mbSrc = c.xmailboxID(tx, c.mailboxID) // Validate. + mbSrc := c.xmailboxID(tx, c.mailboxID) // Validate. mbDst = c.xmailbox(tx, name, "TRYCREATE") if mbDst.ID == c.mailboxID { xuserErrorf("cannot move to currently selected mailbox") } - if len(uidargs) == 0 { + if len(uids) == 0 { xuserErrorf("no matching messages to move") } - // Reserve the uids in the destination mailbox. - uidFirst := mbDst.UIDNext - uidnext := uidFirst - mbDst.UIDNext += store.UID(len(uids)) + uidFirst = mbDst.UIDNext // Assign a new modseq, for the new records and for the expunged records. var err error modseq, err = c.account.NextModSeq(tx) xcheckf(err, "assigning next modseq") - mbSrc.ModSeq = modseq - mbDst.ModSeq = modseq - // Update existing record with new UID and MailboxID in database for messages. We - // add a new but expunged record again in the original/source mailbox, for qresync. - // Keeping the original ID for the live message means we don't have to move the - // on-disk message contents file. + // Make query selecting messages to move. q := bstore.QueryTx[store.Message](tx) - q.FilterNonzero(store.Message{MailboxID: c.mailboxID}) + q.FilterNonzero(store.Message{MailboxID: mbSrc.ID}) q.FilterEqual("UID", uidargs...) q.FilterEqual("Expunged", false) q.SortAsc("UID") - msgs, err := q.List() - xcheckf(err, "listing messages to move") - if len(msgs) != len(uidargs) { - xserverErrorf("uid and message mismatch") - } - - keywords := map[string]struct{}{} - now := time.Now() - - conf, _ := c.account.Conf() - for i := range msgs { - m := &msgs[i] - if m.UID != uids[i] { - xserverErrorf("internal error: got uid %d, expected %d, for index %d", m.UID, uids[i], i) - } - - mbSrc.Sub(m.MailboxCounts()) - - // Copy of message record that we'll insert when UID is freed up. - om := *m - om.PrepareExpunge() - om.ID = 0 // Assign new ID. - om.ModSeq = modseq - - m.MailboxID = mbDst.ID - if m.IsReject && m.MailboxDestinedID != 0 { - // Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message - // is used for reputation calculation during future deliveries. - m.MailboxOrigID = m.MailboxDestinedID - m.IsReject = false - m.Seen = false - } - mbDst.Add(m.MailboxCounts()) - m.UID = uidnext - m.ModSeq = modseq - m.JunkFlagsForMailbox(mbDst, conf) - m.SaveDate = &now - uidnext++ - err := tx.Update(m) - xcheckf(err, "updating moved message in database") - - // Now that UID is unused, we can insert the old record again. - err = tx.Insert(&om) - xcheckf(err, "inserting record for expunge after moving message") - - for _, kw := range m.Keywords { - keywords[kw] = struct{}{} - } - } - - // Ensure destination mailbox has keywords of the moved messages. - var mbKwChanged bool - mbDst.Keywords, mbKwChanged = store.MergeKeywords(mbDst.Keywords, maps.Keys(keywords)) - if mbKwChanged { - changes = append(changes, mbDst.ChangeKeywords()) - } - - err = tx.Update(&mbSrc) - xcheckf(err, "updating source mailbox counts and modseq") - - err = tx.Update(&mbDst) - xcheckf(err, "updating destination mailbox for uids, keywords and counts") - - err = c.account.RetrainMessages(context.TODO(), c.log, tx, msgs) - xcheckf(err, "retraining messages after move") - - // Prepare broadcast changes to other connections. - changes = make([]store.Change, 0, 1+len(msgs)+2) - changes = append(changes, store.ChangeRemoveUIDs{MailboxID: c.mailboxID, UIDs: uids, ModSeq: modseq}) - for _, m := range msgs { - newUIDs = append(newUIDs, m.UID) - changes = append(changes, m.ChangeAddUID()) - } - changes = append(changes, mbSrc.ChangeCounts(), mbDst.ChangeCounts()) + newIDs, chl := c.xmoveMessages(tx, q, len(uidargs), modseq, &mbSrc, &mbDst) + changes = append(changes, chl...) + cleanupIDs = newIDs }) + cleanupIDs = nil + c.broadcast(changes) }) // ../rfc/9051:4708 ../rfc/6851:254 // ../rfc/9051:4713 - c.bwritelinef("* OK [COPYUID %d %s %s] moved", mbDst.UIDValidity, compactUIDSet(uids).String(), compactUIDSet(newUIDs).String()) + newUIDs := numSet{ranges: []numRange{{setNumber{number: uint32(uidFirst)}, &setNumber{number: uint32(mbDst.UIDNext - 1)}}}} + c.bwritelinef("* OK [COPYUID %d %s %s] moved", mbDst.UIDValidity, compactUIDSet(uids).String(), newUIDs.String()) qresync := c.enabled[capQresync] var vanishedUIDs numSet for i := 0; i < len(uids); i++ { @@ -4400,6 +4227,132 @@ func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) { } } +// q must yield messages from a single mailbox. +func (c *conn) xmoveMessages(tx *bstore.Tx, q *bstore.Query[store.Message], expectCount int, modseq store.ModSeq, mbSrc, mbDst *store.Mailbox) (newIDs []int64, changes []store.Change) { + newIDs = make([]int64, 0, expectCount) + var commit bool + defer func() { + if commit { + return + } + for _, id := range newIDs { + p := c.account.MessagePath(id) + err := os.Remove(p) + c.xsanity(err, "removing added message file %v", p) + } + newIDs = nil + }() + + mbSrc.ModSeq = modseq + mbDst.ModSeq = modseq + + var jf *junk.Filter + defer func() { + if jf != nil { + err := jf.CloseDiscard() + c.log.Check(err, "closing junk filter after error") + } + }() + + accConf, _ := c.account.Conf() + + changeRemoveUIDs := store.ChangeRemoveUIDs{ + MailboxID: mbSrc.ID, + ModSeq: modseq, + } + changes = make([]store.Change, 0, expectCount+4) // mbsrc removeuids, mbsrc counts, mbdst counts, mbdst keywords + + nkeywords := len(mbDst.Keywords) + now := time.Now() + + l, err := q.List() + xcheckf(err, "listing messages to move") + + if expectCount > 0 && len(l) != expectCount { + xcheckf(fmt.Errorf("moved %d messages, expected %d", len(l), expectCount), "move messages") + } + + for _, om := range l { + nm := om + nm.MailboxID = mbDst.ID + nm.UID = mbDst.UIDNext + mbDst.UIDNext++ + nm.ModSeq = modseq + nm.CreateSeq = modseq + nm.SaveDate = &now + if nm.IsReject && nm.MailboxDestinedID != 0 { + // Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message + // is used for reputation calculation during future deliveries. + nm.MailboxOrigID = nm.MailboxDestinedID + nm.IsReject = false + nm.Seen = false + } + + nm.JunkFlagsForMailbox(*mbDst, accConf) + + err := tx.Update(&nm) + xcheckf(err, "updating message with new mailbox") + + mbDst.Add(nm.MailboxCounts()) + + mbSrc.Sub(om.MailboxCounts()) + om.ID = 0 + om.Expunged = true + om.ModSeq = modseq + om.TrainedJunk = nil + err = tx.Insert(&om) + xcheckf(err, "inserting expunged message in old mailbox") + + err = moxio.LinkOrCopy(c.log, c.account.MessagePath(om.ID), c.account.MessagePath(nm.ID), nil, false) + xcheckf(err, "duplicating message in old mailbox for current sessions") + newIDs = append(newIDs, nm.ID) + // We don't sync the directory. In case of a crash and files disappearing, the + // eraser will simply not find the file at next startup. + + err = tx.Insert(&store.MessageErase{ID: om.ID, SkipUpdateDiskUsage: true}) + xcheckf(err, "insert message erase") + + mbDst.Keywords, _ = store.MergeKeywords(mbDst.Keywords, nm.Keywords) + + if accConf.JunkFilter != nil && nm.NeedsTraining() { + // Lazily open junk filter. + if jf == nil { + jf, _, err = c.account.OpenJunkFilter(context.TODO(), c.log) + xcheckf(err, "open junk filter") + } + err := c.account.RetrainMessage(context.TODO(), c.log, tx, jf, &nm) + xcheckf(err, "retrain message after moving") + } + + changeRemoveUIDs.UIDs = append(changeRemoveUIDs.UIDs, om.UID) + changeRemoveUIDs.MsgIDs = append(changeRemoveUIDs.MsgIDs, om.ID) + changes = append(changes, nm.ChangeAddUID()) + } + xcheckf(err, "move messages") + + changes = append(changes, changeRemoveUIDs, mbSrc.ChangeCounts()) + + err = tx.Update(mbSrc) + xcheckf(err, "updating counts for inbox") + + changes = append(changes, mbDst.ChangeCounts()) + if len(mbDst.Keywords) > nkeywords { + changes = append(changes, mbDst.ChangeKeywords()) + } + + err = tx.Update(mbDst) + xcheckf(err, "updating uidnext and counts in destination mailbox") + + if jf != nil { + err := jf.Close() + jf = nil + xcheckf(err, "saving junk filter") + } + + commit = true + return +} + // Store sets a full set of flags, or adds/removes specific flags. // // State: Selected diff --git a/imapserver/server_test.go b/imapserver/server_test.go index 7f81e01..88488d8 100644 --- a/imapserver/server_test.go +++ b/imapserver/server_test.go @@ -18,6 +18,8 @@ import ( "golang.org/x/sys/unix" + "github.com/mjl-/bstore" + "github.com/mjl-/mox/imapclient" "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" @@ -309,6 +311,14 @@ func (tc *testconn) waitDone() { } func (tc *testconn) close() { + tc.close0(true) +} + +func (tc *testconn) closeNoWait() { + tc.close0(false) +} + +func (tc *testconn) close0(waitclose bool) { defer func() { if unhandledPanics.Swap(0) > 0 { tc.t.Fatalf("handled panic in server") @@ -319,13 +329,16 @@ func (tc *testconn) close() { // Already closed, we are not strict about closing multiple times. return } - err := tc.account.Close() - tc.check(err, "close account") - // no account.CheckClosed(), the tests open accounts multiple times. - tc.account = nil if tc.client != nil { tc.client.Close() } + err := tc.account.Close() + tc.check(err, "close account") + if waitclose { + tc.account.WaitClosed() + } + // no account.CheckClosed(), the tests open accounts multiple times. + tc.account = nil tc.serverConn.Close() tc.waitDone() if tc.switchStop != nil { @@ -406,6 +419,22 @@ func startArgsMore(t *testing.T, first, immediateTLS bool, serverConfig, clientC switchStop := func() {} if first { switchStop = store.Switchboard() + + // Add a deleted mailbox, may excercise some code paths. + err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error { + // todo: add a message to inbox and remove it again. need to change all uids in the tests. + // todo: add tests for operating on an expunged mailbox. it should say it doesn't exist. + + mb, _, _, _, err := acc.MailboxCreate(tx, "expungebox", store.SpecialUse{}) + if err != nil { + return fmt.Errorf("create mailbox: %v", err) + } + if _, _, err := acc.MailboxDelete(ctxbg, pkglog, tx, &mb); err != nil { + return fmt.Errorf("delete mailbox: %v", err) + } + return nil + }) + tcheck(t, err, "add expunged mailbox") } if afterInit != nil { @@ -698,7 +727,7 @@ func TestMailboxDeleted(t *testing.T) { defer tc.close() tc2 := startNoSwitchboard(t) - defer tc2.close() + defer tc2.closeNoWait() tc.client.Login("mjl@mox.example", password0) tc2.client.Login("mjl@mox.example", password0) diff --git a/imapserver/status_test.go b/imapserver/status_test.go index 350bd76..e8f041c 100644 --- a/imapserver/status_test.go +++ b/imapserver/status_test.go @@ -19,6 +19,8 @@ func TestStatus(t *testing.T) { tc.transactf("bad", "status inbox (uidvalidity) ") // Leftover data. tc.transactf("bad", "status inbox (unknown)") // Unknown attribute. + tc.transactf("no", "status expungebox (messages)") // No longer exists. + tc.transactf("ok", "status inbox (messages uidnext uidvalidity unseen deleted size recent appendlimit)") tc.xuntagged(imapclient.UntaggedStatus{ Mailbox: "Inbox", diff --git a/imapserver/subscribe_test.go b/imapserver/subscribe_test.go index 8b7ec5d..ede490b 100644 --- a/imapserver/subscribe_test.go +++ b/imapserver/subscribe_test.go @@ -11,7 +11,7 @@ func TestSubscribe(t *testing.T) { defer tc.close() tc2 := startNoSwitchboard(t) - defer tc2.close() + defer tc2.closeNoWait() tc.client.Login("mjl@mox.example", password0) tc2.client.Login("mjl@mox.example", password0) diff --git a/imapserver/unsubscribe_test.go b/imapserver/unsubscribe_test.go index be09c7f..05e44f4 100644 --- a/imapserver/unsubscribe_test.go +++ b/imapserver/unsubscribe_test.go @@ -14,10 +14,12 @@ func TestUnsubscribe(t *testing.T) { tc.transactf("bad", "unsubscribe ") // Missing param. tc.transactf("bad", "unsubscribe fine ") // Leftover data. - tc.transactf("no", "unsubscribe a/b") // Does not exist and is not subscribed. + tc.transactf("no", "unsubscribe a/b") // Does not exist and is not subscribed. + tc.transactf("ok", "unsubscribe expungebox") // Does not exist anymore but is still subscribed. + tc.transactf("no", "unsubscribe expungebox") // Not subscribed. tc.transactf("ok", "create a/b") tc.transactf("ok", "unsubscribe a/b") - tc.transactf("ok", "unsubscribe a/b") // Can unsubscribe even if it does not exist. + tc.transactf("ok", "unsubscribe a/b") // Can unsubscribe even if there is no subscription. tc.transactf("ok", "subscribe a/b") tc.transactf("ok", "unsubscribe a/b") } diff --git a/import.go b/import.go index b767bcf..1b650a4 100644 --- a/import.go +++ b/import.go @@ -254,7 +254,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { ctl.xwriteok() // We will be delivering messages. If we fail halfway, we need to remove the created msg files. - var deliveredIDs []int64 + var newIDs []int64 defer func() { x := recover() if x == nil { @@ -269,12 +269,12 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { ctl.log.Error("import error") } - for _, id := range deliveredIDs { + for _, id := range newIDs { p := a.MessagePath(id) err := os.Remove(p) ctl.log.Check(err, "closing message file after import error", slog.String("path", p)) } - deliveredIDs = nil + newIDs = nil ctl.xerror(fmt.Sprintf("import error: %v", x)) }() @@ -371,7 +371,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { } err = a.MessageAdd(ctl.log, tx, &mb, m, msgf, opts) ctl.xcheck(err, "delivering message") - deliveredIDs = append(deliveredIDs, m.ID) + newIDs = append(newIDs, m.ID) changes = append(changes, m.ChangeAddUID()) msgDirs[filepath.Dir(a.MessagePath(m.ID))] = struct{}{} @@ -393,8 +393,8 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { } // Match threads. - if len(deliveredIDs) > 0 { - err = a.AssignThreads(ctx, ctl.log, tx, deliveredIDs[0], 0, io.Discard) + if len(newIDs) > 0 { + err = a.AssignThreads(ctx, ctl.log, tx, newIDs[0], 0, io.Discard) ctl.xcheck(err, "assigning messages to threads") } @@ -423,8 +423,8 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { err = tx.Commit() ctl.xcheck(err, "commit") tx = nil - ctl.log.Info("delivered messages through import", slog.Int("count", len(deliveredIDs))) - deliveredIDs = nil + ctl.log.Info("delivered messages through import", slog.Int("count", len(newIDs))) + newIDs = nil store.BroadcastChanges(a, changes) }) diff --git a/main.go b/main.go index 64205b8..84524da 100644 --- a/main.go +++ b/main.go @@ -3340,6 +3340,7 @@ open, or is not running. } q := bstore.QueryTx[store.Mailbox](tx) + q.FilterEqual("Expunged", false) if len(args) == 2 { q.FilterEqual("Name", args[1]) } @@ -3398,7 +3399,8 @@ open, or is not running. // Reassign UIDs, going per mailbox. We assign starting at 1, only changing the // message if it isn't already at the intended UID. Doing it in this order ensures // we don't get into trouble with duplicate UIDs for a mailbox. We assign a new - // modseq. Not strictly needed, but doesn't hurt. + // modseq. Not strictly needed, but doesn't hurt. It's also why we assign a UID to + // expunged messages. modseq, err := a.NextModSeq(tx) xcheckf(err, "assigning next modseq") @@ -3424,7 +3426,7 @@ open, or is not running. } // Now update the uidnext, uidvalidity and modseq for each mailbox. - err = bstore.QueryTx[store.Mailbox](tx).ForEach(func(mb store.Mailbox) error { + err = bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb store.Mailbox) error { // Assign each mailbox a completely new uidvalidity. uidvalidity, err := a.NextUIDValidity(tx) if err != nil { @@ -3491,7 +3493,7 @@ open, or is not running. err = a.DB.Write(context.Background(), func(tx *bstore.Tx) error { // We look at each mailbox, retrieve its max UID and compare against the mailbox // UIDNEXT. - err := bstore.QueryTx[store.Mailbox](tx).ForEach(func(mb store.Mailbox) error { + err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb store.Mailbox) error { if mb.UIDValidity > maxUIDValidity { maxUIDValidity = mb.UIDValidity } diff --git a/mox-/config.go b/mox-/config.go index 359ca72..e371c3e 100644 --- a/mox-/config.go +++ b/mox-/config.go @@ -933,6 +933,10 @@ func PrepareStaticConfig(ctx context.Context, log mlog.Log, configFile string, c // DefaultMailboxes is deprecated. for _, mb := range c.DefaultMailboxes { checkMailboxNormf(mb, "default mailbox") + // We don't create parent mailboxes for default mailboxes. + if ParentMailboxName(mb) != "" { + addErrorf("default mailbox cannot be a child mailbox") + } } checkSpecialUseMailbox := func(nameOpt string) { if nameOpt != "" { @@ -940,6 +944,10 @@ func PrepareStaticConfig(ctx context.Context, log mlog.Log, configFile string, c if strings.EqualFold(nameOpt, "inbox") { addErrorf("initial mailbox cannot be set to Inbox (Inbox is always created)") } + // We don't currently create parent mailboxes for initial mailboxes. + if ParentMailboxName(nameOpt) != "" { + addErrorf("initial mailboxes cannot be child mailboxes") + } } } checkSpecialUseMailbox(c.InitialMailboxes.SpecialUse.Archive) @@ -952,6 +960,9 @@ func PrepareStaticConfig(ctx context.Context, log mlog.Log, configFile string, c if strings.EqualFold(name, "inbox") { addErrorf("initial regular mailbox cannot be set to Inbox (Inbox is always created)") } + if ParentMailboxName(name) != "" { + addErrorf("initial mailboxes cannot be child mailboxes") + } } checkTransportSMTP := func(name string, isTLS bool, t *config.TransportSMTP) { diff --git a/mox-/parentname.go b/mox-/parentname.go new file mode 100644 index 0000000..6430599 --- /dev/null +++ b/mox-/parentname.go @@ -0,0 +1,15 @@ +package mox + +import ( + "strings" +) + +// ParentMailboxName returns the name of the parent mailbox, returning empty if +// there is no parent. +func ParentMailboxName(name string) string { + i := strings.LastIndex(name, "/") + if i < 0 { + return "" + } + return name[:i] +} diff --git a/queue/hook_test.go b/queue/hook_test.go index 6cd7ac7..58484ca 100644 --- a/queue/hook_test.go +++ b/queue/hook_test.go @@ -29,7 +29,7 @@ func TestHookIncoming(t *testing.T) { tcheck(t, err, "open account for retired") defer func() { accret.Close() - accret.CheckClosed() + accret.WaitClosed() }() testIncoming := func(a *store.Account, expIn bool) { @@ -125,7 +125,7 @@ func TestFromIDIncomingDelivery(t *testing.T) { tcheck(t, err, "open account for retired") defer func() { accret.Close() - accret.CheckClosed() + accret.WaitClosed() }() // Account that only gets webhook calls, but no retired webhooks. @@ -133,7 +133,7 @@ func TestFromIDIncomingDelivery(t *testing.T) { tcheck(t, err, "open account for hook") defer func() { acchook.Close() - acchook.CheckClosed() + acchook.WaitClosed() }() addr, err := smtp.ParseAddress("mjl@mox.example") diff --git a/queue/queue_test.go b/queue/queue_test.go index 20baf65..e791d00 100644 --- a/queue/queue_test.go +++ b/queue/queue_test.go @@ -75,7 +75,7 @@ func setup(t *testing.T) (*store.Account, func()) { mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg) return acc, func() { acc.Close() - acc.CheckClosed() + acc.WaitClosed() mox.ShutdownCancel() mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg) Shutdown() diff --git a/smtpserver/fuzz_test.go b/smtpserver/fuzz_test.go index c08e8b0..1a075a5 100644 --- a/smtpserver/fuzz_test.go +++ b/smtpserver/fuzz_test.go @@ -43,7 +43,7 @@ func FuzzServer(f *testing.F) { } defer func() { acc.Close() - acc.CheckClosed() + acc.WaitClosed() }() err = acc.SetPassword(log, "testtest") if err != nil { diff --git a/smtpserver/rejects.go b/smtpserver/rejects.go index 71013c2..e187b9b 100644 --- a/smtpserver/rejects.go +++ b/smtpserver/rejects.go @@ -46,14 +46,11 @@ func rejectPresent(log mlog.Log, acc *store.Account, rejectsMailbox string, m *s var err error acc.WithRLock(func() { err = acc.DB.Read(context.TODO(), func(tx *bstore.Tx) error { - mbq := bstore.QueryTx[store.Mailbox](tx) - mbq.FilterNonzero(store.Mailbox{Name: rejectsMailbox}) - mb, err := mbq.Get() - if err == bstore.ErrAbsent { - return nil - } + mb, err := acc.MailboxFind(tx, rejectsMailbox) if err != nil { return fmt.Errorf("looking for rejects mailbox: %w", err) + } else if mb == nil { + return nil } q := bstore.QueryTx[store.Message](tx) diff --git a/smtpserver/server.go b/smtpserver/server.go index faf3a01..8d2ec96 100644 --- a/smtpserver/server.go +++ b/smtpserver/server.go @@ -3272,6 +3272,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW // See if we received a non-junk message from this organizational domain. q := bstore.QueryTx[store.Message](tx) q.FilterNonzero(store.Message{MsgFromOrgDomain: a0.d.m.MsgFromOrgDomain}) + q.FilterEqual("Expunged", false) q.FilterEqual("Notjunk", true) q.FilterEqual("IsReject", false) exists, err := q.Exists() @@ -3406,25 +3407,37 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW // due to reputation for later delivery attempts. a.d.m.MessageHash = messagehash a.d.acc.WithWLock(func() { - hasSpace := true - var err error - if !conf.KeepRejects { - hasSpace, err = a.d.acc.TidyRejectsMailbox(c.log, conf.RejectsMailbox) - } - if err != nil { - log.Errorx("tidying rejects mailbox", err) - } else if !hasSpace { - log.Info("not storing spammy mail to full rejects mailbox") - return - } - var changes []store.Change var stored bool - err = a.d.acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error { + + var newID int64 + defer func() { + if newID != 0 { + p := a.d.acc.MessagePath(newID) + err := os.Remove(p) + c.log.Check(err, "remove message after error delivering to rejects", slog.String("path", p)) + } + }() + + err := a.d.acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error { mbrej, err := a.d.acc.MailboxFind(tx, conf.RejectsMailbox) if err != nil { return fmt.Errorf("finding rejects mailbox: %v", err) } + + hasSpace := true + if !conf.KeepRejects && mbrej != nil { + var chl []store.Change + chl, hasSpace, err = a.d.acc.TidyRejectsMailbox(c.log, tx, mbrej) + if err != nil { + return fmt.Errorf("tidying rejects mailbox: %v", err) + } + if !hasSpace { + log.Info("not storing spammy mail to full rejects mailbox") + return nil + } + changes = append(changes, chl...) + } if mbrej == nil { nmb, chl, _, _, err := a.d.acc.MailboxCreate(tx, conf.RejectsMailbox, store.SpecialUse{}) if err != nil { @@ -3438,6 +3451,8 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW if err := a.d.acc.MessageAdd(log, tx, mbrej, a.d.m, dataFile, store.AddOpts{}); err != nil { return fmt.Errorf("delivering spammy mail to rejects mailbox: %v", err) } + newID = a.d.m.ID + if err := tx.Update(mbrej); err != nil { return fmt.Errorf("updating rejects mailbox: %v", err) } @@ -3451,6 +3466,8 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW } else if stored { log.Info("stored spammy mail in rejects mailbox") } + newID = 0 + store.BroadcastChanges(a.d.acc, changes) }) } diff --git a/smtpserver/server_test.go b/smtpserver/server_test.go index f125b16..a2f58d0 100644 --- a/smtpserver/server_test.go +++ b/smtpserver/server_test.go @@ -184,7 +184,7 @@ func (ts *testserver) close() { ts.switchStop() err = ts.acc.Close() tcheck(ts.t, err, "closing account") - ts.acc.CheckClosed() + ts.acc.WaitClosed() ts.acc = nil } @@ -193,6 +193,7 @@ func (ts *testserver) checkCount(mailboxName string, expect int) { t.Helper() q := bstore.QueryDB[store.Mailbox](ctxbg, ts.acc.DB) q.FilterNonzero(store.Mailbox{Name: mailboxName}) + q.FilterEqual("Expunged", false) mb, err := q.Get() tcheck(t, err, "get mailbox") qm := bstore.QueryDB[store.Message](ctxbg, ts.acc.DB) diff --git a/store/account.go b/store/account.go index 3f79059..7b4f3a7 100644 --- a/store/account.go +++ b/store/account.go @@ -8,7 +8,7 @@ Layout of storage for accounts: /accounts//index.db /accounts//msg/[a-zA-Z0-9_-]+/ -Index.db holds tables for user information, mailboxes, and messages. Messages +Index.db holds tables for user information, mailboxes, and messages. Message contents are stored in the msg/ subdirectory, each in their own file. The on-disk message does not contain headers generated during an incoming SMTP transaction, such as Received and Authentication-Results headers. Those are in the database to @@ -195,9 +195,15 @@ type SyncState struct { type Mailbox struct { ID int64 - // "Inbox" is the name for the special IMAP "INBOX". Slash separated - // for hierarchy. - Name string `bstore:"nonzero,unique"` + CreateSeq ModSeq + ModSeq ModSeq `bstore:"index"` // Of last change, or when deleted. + Expunged bool + + ParentID int64 `bstore:"ref Mailbox"` // Zero for top-level mailbox. + + // "Inbox" is the name for the special IMAP "INBOX". Slash separated for hierarchy. + // Names must be unique for mailboxes that are not expunged. + Name string `bstore:"nonzero"` // If UIDs are invalidated, e.g. when renaming a mailbox to a previously existing // name, UIDValidity must be changed. Used by IMAP for synchronization. @@ -215,11 +221,6 @@ type Mailbox struct { // lower case (for JMAP), sorted. Keywords []string - // ModSeq matches that of last message (including deleted), or changes - // to mailbox such as after metadata changes. - ModSeq ModSeq - CreateSeq ModSeq - HaveCounts bool // Whether MailboxCounts have been initialized. MailboxCounts // Statistics about messages, kept up to date whenever a change happens. } @@ -229,8 +230,12 @@ type Mailbox struct { type Annotation struct { ID int64 + CreateSeq ModSeq + ModSeq ModSeq `bstore:"index"` + Expunged bool + // Can be zero, indicates global (per-account) annotation. - MailboxID int64 `bstore:"ref Mailbox,unique MailboxID+Key"` + MailboxID int64 `bstore:"ref Mailbox,index MailboxID+Key"` // "Entry name", always starts with "/private/" or "/shared/". Stored lower-case, // comparisons must be done case-insensitively. @@ -238,9 +243,6 @@ type Annotation struct { IsString bool // If true, the value is a string instead of bytes. Value []byte - - ModSeq ModSeq - CreateSeq ModSeq } // Change returns a broadcastable change for the annotation. @@ -314,6 +316,14 @@ func (mb Mailbox) ChangeKeywords() ChangeMailboxKeywords { return ChangeMailboxKeywords{mb.ID, mb.Name, mb.Keywords} } +func (mb Mailbox) ChangeAddMailbox(flags []string) ChangeAddMailbox { + return ChangeAddMailbox{Mailbox: mb, Flags: flags, ModSeq: mb.ModSeq} +} + +func (mb Mailbox) ChangeRemoveMailbox() ChangeRemoveMailbox { + return ChangeRemoveMailbox{mb.ID, mb.Name, mb.ModSeq} +} + // KeywordsChanged returns whether the keywords in a mailbox have changed. func (mb Mailbox) KeywordsChanged(origmb Mailbox) bool { if len(mb.Keywords) != len(origmb.Keywords) { @@ -380,11 +390,17 @@ const ( // Messages always have a header section, even if empty. Incoming messages without // header section must get an empty header section added before inserting. type Message struct { - // ID, unchanged over lifetime, determines path to on-disk msg file. - // Set during deliver. + // ID of the message, determines path to on-disk message file. Set when adding to a + // mailbox. When a message is moved to another mailbox, the mailbox ID is changed, + // but for synchronization purposes, a new Message record is inserted (which gets a + // new ID) with the Expunged field set and the MailboxID and UID copied. ID int64 - UID UID `bstore:"nonzero"` // UID, for IMAP. Set during deliver. + // UID, for IMAP. Set when adding to mailbox. Strictly increasing values, per + // mailbox. The UID of a message can never change (though messages can be copied), + // and the contents of a message/UID also never changes. + UID UID `bstore:"nonzero"` + MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,index MailboxID+ModSeq,ref Mailbox"` // Modification sequence, for faster syncing with IMAP QRESYNC and JMAP. @@ -506,7 +522,9 @@ type Message struct { // IDs of parent messages, from closest parent to the root message. Parent messages // may be in a different mailbox, or may no longer exist. ThreadParentIDs must // never contain the message id itself (a cycle), and parent messages must - // reference the same ancestors. + // reference the same ancestors. Moving a message to another mailbox keeps the + // message ID and changes the MailboxID (and UID) of the message, leaving threading + // parent ids intact. ThreadParentIDs []int64 // ThreadMissingLink is true if there is no match with a direct parent. E.g. first // ID in ThreadParentIDs is not the direct ancestor (an intermediate message may @@ -605,9 +623,13 @@ func ModSeqFromClient(modseq int64) ModSeq { return ModSeq(modseq) } -// PrepareExpunge clears fields that are no longer needed after an expunge, so -// almost all fields. Does not change ModSeq, but does set Expunged. -func (m *Message) PrepareExpunge() { +// Erase clears fields from a Message that are no longer needed after actually +// removing the message file from the file system, after all references to the +// message have gone away. Only the fields necessary for synchronisation are kept. +func (m *Message) erase() { + if !m.Expunged { + panic("erase called on non-expunged message") + } *m = Message{ ID: m.ID, UID: m.UID, @@ -842,6 +864,20 @@ type RulesetNoMailbox struct { ToMailbox bool // Whether MailboxID is the destination of the move (instead of source). } +// MessageErase represents the need to remove a message file from disk, and clear +// message fields from the database, but only when the last reference to the +// message is gone (all IMAP sessions need to have applied the changes indicating +// message removal). +type MessageErase struct { + ID int64 // Same ID as Message.ID. + + // Whether to subtract the size from the total disk usage. Useful for moving + // messages, which involves duplicating the message temporarily, while there are + // still references in the old mailbox, but which isn't counted as using twice the + // disk space.. + SkipUpdateDiskUsage bool +} + // Types stored in DB. var DBTypes = []any{ NextUIDValidity{}, @@ -863,6 +899,7 @@ var DBTypes = []any{ RulesetNoMsgFrom{}, RulesetNoMailbox{}, Annotation{}, + MessageErase{}, } // Account holds the information about a user, includings mailboxes, messages, imap subscriptions. @@ -883,6 +920,9 @@ type Account struct { // directory when delivering. lastMsgDir string + // If set, consistency checks won't fail on message ModSeq/CreateSeq being zero. + skipMessageZeroSeqCheck bool + // Write lock must be held when modifying account/mailbox/message/flags/annotations // if the change needs to be synchronized with client connections by broadcasting // the changes. Changes that are not protocol-visible do not require a lock, the @@ -897,20 +937,25 @@ type Account struct { // releasing the lock to ensure proper UID ordering. sync.RWMutex - nused int // Reference count, while >0, this account is alive and shared. + // Reference count, while >0, this account is alive and shared. Protected by + // openAccounts, not by account wlock. + nused int + closed chan struct{} // Closed when last reference is gone. } type Upgrade struct { - ID byte - Threads byte // 0: None, 1: Adding MessageID's completed, 2: Adding ThreadID's completed. - MailboxModSeq bool // Whether mailboxes have been assigned modseqs. + ID byte + Threads byte // 0: None, 1: Adding MessageID's completed, 2: Adding ThreadID's completed. + MailboxModSeq bool // Whether mailboxes have been assigned modseqs. + MailboxParentID bool // Setting ParentID on mailboxes. } // upgradeInit is the value to for new account database, that don't need any upgrading. var upgradeInit = Upgrade{ - ID: 1, // Singleton. - Threads: 2, - MailboxModSeq: true, + ID: 1, // Singleton. + Threads: 2, + MailboxModSeq: true, + MailboxParentID: true, } // InitialUIDValidity returns a UIDValidity used for initializing an account. @@ -928,15 +973,33 @@ var openAccounts = struct { func closeAccount(acc *Account) (rerr error) { openAccounts.Lock() - acc.nused-- defer openAccounts.Unlock() - if acc.nused == 0 { - // threadsCompleted must be closed now because it increased nused. - rerr = acc.DB.Close() + acc.nused-- + if acc.nused > 0 { + return + } + + // threadsCompleted must be closed now because it increased nused. + + defer func() { + err := acc.DB.Close() acc.DB = nil delete(openAccounts.names, acc.Name) + close(acc.closed) + + if rerr == nil { + rerr = err + } + }() + + // Verify there are no more pending MessageErase records. + l, err := bstore.QueryDB[MessageErase](context.TODO(), acc.DB).List() + if err != nil { + return fmt.Errorf("listing messageerase records: %v", err) + } else if len(l) > 0 { + return fmt.Errorf("messageerase records still present after last account reference is gone: %v", l) } - return + return nil } // OpenAccount opens an account by name. @@ -1005,6 +1068,7 @@ func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, re DBPath: dbpath, DB: db, nused: 1, + closed: make(chan struct{}), threadsCompleted: make(chan struct{}), } @@ -1017,6 +1081,7 @@ func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, re } // Ensure singletons are present. Mailbox counts and total message size, Settings. + // Process pending MessageErase records. var mentioned bool err = db.Write(context.TODO(), func(tx *bstore.Tx) error { if tx.Get(&Settings{ID: 1}) == bstore.ErrAbsent { @@ -1044,21 +1109,66 @@ func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, re du := DiskUsage{ID: 1} err = tx.Get(&du) - if err == nil || !errors.Is(err, bstore.ErrAbsent) { + if err == bstore.ErrAbsent { + // No DiskUsage record yet, calculate total size and insert. + err := bstore.QueryTx[Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb Mailbox) error { + du.MessageSize += mb.Size + return nil + }) + if err != nil { + return err + } + if err := tx.Insert(&du); err != nil { + return err + } + } else if err != nil { return err } - // No DiskUsage record yet, calculate total size and insert. - err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error { - du.MessageSize += mb.Size - return nil - }) - if err != nil { - return err + + var erase []MessageErase + if _, err := bstore.QueryTx[MessageErase](tx).Gather(&erase).Delete(); err != nil { + return fmt.Errorf("fetching messages to erase: %w", err) } - return tx.Insert(&du) + if len(erase) > 0 { + log.Debug("deleting message files from message erase records", slog.Int("count", len(erase))) + } + var duChanged bool + for _, me := range erase { + // Clear the fields from the message not needed for synchronization. + m := Message{ID: me.ID} + if err := tx.Get(&m); err != nil { + return fmt.Errorf("get message %d to expunge: %w", me.ID, err) + } else if !m.Expunged { + return fmt.Errorf("message %d to erase is not expunged", m.ID) + } + + // We remove before we update/commit the database, so we are sure we don't leave + // files behind in case of an error/crash. + p := acc.MessagePath(me.ID) + err := os.Remove(p) + log.Check(err, "removing message file for expunged message", slog.String("path", p)) + + if !me.SkipUpdateDiskUsage { + du.MessageSize -= m.Size + duChanged = true + } + + m.erase() + if err := tx.Update(&m); err != nil { + return fmt.Errorf("save erase of message %d in database: %w", m.ID, err) + } + } + + if duChanged { + if err := tx.Update(&du); err != nil { + return fmt.Errorf("saving disk usage after erasing messages: %w", err) + } + } + + return nil }) if err != nil { - return nil, fmt.Errorf("calculating counts for mailbox or inserting settings: %v", err) + return nil, fmt.Errorf("calculating counts for mailbox, inserting settings, expunging messages: %v", err) } // Start adding threading if needed. @@ -1078,13 +1188,13 @@ func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, re } // Ensure all mailboxes have a modseq based on highest modseq message in each - // mailbox, and a creatseq. + // mailbox, and a createseq. if !up.MailboxModSeq { log.Debug("upgrade: adding modseq to each mailbox") err := acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error { var modseq ModSeq - mbl, err := bstore.QueryTx[Mailbox](tx).List() + mbl, err := bstore.QueryTx[Mailbox](tx).FilterEqual("Expunged", false).List() if err != nil { return fmt.Errorf("listing mailboxes: %v", err) } @@ -1126,6 +1236,91 @@ func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, re } } + // Add ParentID to mailboxes. + if !up.MailboxParentID { + log.Debug("upgrade: setting parentid on each mailbox") + + err := acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error { + mbl, err := bstore.QueryTx[Mailbox](tx).FilterEqual("Expunged", false).SortAsc("Name").List() + if err != nil { + return fmt.Errorf("listing mailboxes: %w", err) + } + + names := map[string]Mailbox{} + for _, mb := range mbl { + names[mb.Name] = mb + } + + var modseq ModSeq + + // Ensure a parent mailbox for name exists, creating it if needed, including any + // grandparents, up to the top. + var ensureParentMailboxID func(name string) (int64, error) + ensureParentMailboxID = func(name string) (int64, error) { + parentName := mox.ParentMailboxName(name) + if parentName == "" { + return 0, nil + } + parent := names[parentName] + if parent.ID != 0 { + return parent.ID, nil + } + + parentParentID, err := ensureParentMailboxID(parentName) + if err != nil { + return 0, fmt.Errorf("creating parent mailbox %q: %w", parentName, err) + } + + if modseq == 0 { + modseq, err = a.NextModSeq(tx) + if err != nil { + return 0, fmt.Errorf("get next modseq: %w", err) + } + } + + uidvalidity, err := a.NextUIDValidity(tx) + if err != nil { + return 0, fmt.Errorf("next uid validity: %w", err) + } + + parent = Mailbox{ + CreateSeq: modseq, + ModSeq: modseq, + ParentID: parentParentID, + Name: parentName, + UIDValidity: uidvalidity, + UIDNext: 1, + SpecialUse: SpecialUse{}, + HaveCounts: true, + } + if err := tx.Insert(&parent); err != nil { + return 0, fmt.Errorf("creating parent mailbox: %w", err) + } + return parent.ID, nil + } + + for _, mb := range mbl { + parentID, err := ensureParentMailboxID(mb.Name) + if err != nil { + return fmt.Errorf("creating missing parent mailbox for mailbox %q: %w", mb.Name, err) + } + mb.ParentID = parentID + if err := tx.Update(&mb); err != nil { + return fmt.Errorf("update mailbox with parentid: %w", err) + } + } + + up.MailboxParentID = true + if err := tx.Update(&up); err != nil { + return fmt.Errorf("marking upgrade done: %w", err) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("upgrade: setting parentid on each mailbox: %w", err) + } + } + if up.Threads == 2 { close(acc.threadsCompleted) return acc, nil @@ -1216,7 +1411,15 @@ func initAccount(db *bstore.DB) error { mailboxes = append(mailboxes, name) } for _, name := range mailboxes { - mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, ModSeq: modseq, CreateSeq: modseq, HaveCounts: true} + mb := Mailbox{ + CreateSeq: modseq, + ModSeq: modseq, + ParentID: 0, + Name: name, + UIDValidity: uidvalidity, + UIDNext: 1, + HaveCounts: true, + } if strings.HasPrefix(name, "Archive") { mb.Archive = true } else if strings.HasPrefix(name, "Drafts") { @@ -1243,7 +1446,16 @@ func initAccount(db *bstore.DB) error { } add := func(name string, use SpecialUse) error { - mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, SpecialUse: use, ModSeq: modseq, CreateSeq: modseq, HaveCounts: true} + mb := Mailbox{ + CreateSeq: modseq, + ModSeq: modseq, + ParentID: 0, + Name: name, + UIDValidity: uidvalidity, + UIDNext: 1, + SpecialUse: use, + HaveCounts: true, + } if err := tx.Insert(&mb); err != nil { return fmt.Errorf("creating mailbox: %w", err) } @@ -1289,6 +1501,13 @@ func initAccount(db *bstore.DB) error { }) } +// WaitClosed waits until the last reference to this account is gone and the +// account is closed. Used during tests, to ensure the consistency checks run after +// expunged messages have been erased. +func (a *Account) WaitClosed() { + <-a.closed +} + // CheckClosed asserts that the account has a zero reference count. For use in tests. func (a *Account) CheckClosed() { openAccounts.Lock() @@ -1312,6 +1531,14 @@ func (a *Account) Close() error { return closeAccount(a) } +// SetSkipMessageModSeqZeroCheck skips consistency checks for Message.ModSeq and +// Message.CreateSeq being zero. +func (a *Account) SetSkipMessageModSeqZeroCheck(skip bool) { + a.Lock() + defer a.Unlock() + a.skipMessageZeroSeqCheck = true +} + // CheckConsistency checks the consistency of the database and returns a non-nil // error for these cases: // @@ -1323,11 +1550,15 @@ func (a *Account) Close() error { // - Message with UID >= mailbox uid next. // - Mailbox uidvalidity >= account uid validity. // - Mailbox ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq, and Modseq >= highest message ModSeq. +// - Mailbox must have a live parent ID if they are live themselves, live names must be unique. // - Message ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq. // - All messages have a nonzero ThreadID, and no cycles in ThreadParentID, and parent messages the same ThreadParentIDs tail. -// - Annotations must have ModSeq > 0, CreateSeq > 0, ModSeq >= CreateSeq. +// - Annotations must have ModSeq > 0, CreateSeq > 0, ModSeq >= CreateSeq and live keys must be unique per mailbox. // - Recalculate junk filter (words and counts) and check they are the same. func (a *Account) CheckConsistency() error { + a.Lock() + defer a.Unlock() + var uidErrors []string // With a limit, could be many. var modseqErrors []string // With limit. var fileErrors []string // With limit. @@ -1339,8 +1570,6 @@ func (a *Account) CheckConsistency() error { ctx := context.Background() log := mlog.New("store", nil) - a.Lock() - defer a.Unlock() err := a.DB.Read(ctx, func(tx *bstore.Tx) error { nuv := NextUIDValidity{ID: 1} err := tx.Get(&nuv) @@ -1348,9 +1577,17 @@ func (a *Account) CheckConsistency() error { return fmt.Errorf("fetching next uid validity: %v", err) } - mailboxes := map[int64]Mailbox{} + mailboxes := map[int64]Mailbox{} // Also expunged mailboxes. + mailboxNames := map[string]Mailbox{} // Only live names. err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error { mailboxes[mb.ID] = mb + if !mb.Expunged { + if xmb, ok := mailboxNames[mb.Name]; ok { + errmsg := fmt.Sprintf("mailbox %q exists as id %d and id %d", mb.Name, mb.ID, xmb.ID) + errmsgs = append(errmsgs, errmsg) + } + mailboxNames[mb.Name] = mb + } if mb.UIDValidity >= nuv.Next { errmsg := fmt.Sprintf("mailbox %q (id %d) has uidvalidity %d >= account next uidvalidity %d", mb.Name, mb.ID, mb.UIDValidity, nuv.Next) @@ -1377,7 +1614,37 @@ func (a *Account) CheckConsistency() error { return fmt.Errorf("checking mailboxes: %v", err) } + // Check ParentID and name of parent. + for _, mb := range mailboxNames { + if mox.ParentMailboxName(mb.Name) == "" { + if mb.ParentID == 0 { + continue + } + errmsg := fmt.Sprintf("mailbox %q (id %d) is a root mailbox but has parentid %d", mb.Name, mb.ID, mb.ParentID) + errmsgs = append(errmsgs, errmsg) + } else if mb.ParentID == 0 { + errmsg := fmt.Sprintf("mailbox %q (id %d) is not a root mailbox but has a zero parentid", mb.Name, mb.ID) + errmsgs = append(errmsgs, errmsg) + } else if mox.ParentMailboxName(mb.Name) != mailboxes[mb.ParentID].Name { + errmsg := fmt.Sprintf("mailbox %q (id %d) has parent mailbox id %d with name %q, but parent name should be %q", mb.Name, mb.ID, mb.ParentID, mailboxes[mb.ParentID].Name, mox.ParentMailboxName(mb.Name)) + errmsgs = append(errmsgs, errmsg) + } + } + + type annotation struct { + mailboxID int64 // Can be 0. + key string + } + annotations := map[annotation]struct{}{} err = bstore.QueryTx[Annotation](tx).ForEach(func(a Annotation) error { + if !a.Expunged { + k := annotation{a.MailboxID, a.Key} + if _, ok := annotations[k]; ok { + errmsg := fmt.Sprintf("duplicate live annotation key %q for mailbox id %d", a.Key, a.MailboxID) + errmsgs = append(errmsgs, errmsg) + } + annotations[k] = struct{}{} + } if a.ModSeq == 0 || a.CreateSeq == 0 || a.CreateSeq > a.ModSeq { errmsg := fmt.Sprintf("annotation %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and modseq >= createseq", a.ID, mailboxes[a.MailboxID].Name, a.MailboxID, a.ModSeq, a.CreateSeq) errmsgs = append(errmsgs, errmsg) @@ -1393,6 +1660,7 @@ func (a *Account) CheckConsistency() error { // All message id's from database. For checking for unexpected files afterwards. messageIDs := map[int64]struct{}{} + eraseMessageIDs := map[int64]bool{} // Value indicates whether to skip updating disk usage. // If configured, we'll be building up the junk filter for the messages, to compare // against the on-disk junk filter. @@ -1414,7 +1682,17 @@ func (a *Account) CheckConsistency() error { } var ntrained int + // Get IDs of erase messages not yet removed, they'll have a message file. + err = bstore.QueryTx[MessageErase](tx).ForEach(func(me MessageErase) error { + eraseMessageIDs[me.ID] = me.SkipUpdateDiskUsage + return nil + }) + if err != nil { + return fmt.Errorf("listing message erase records") + } + counts := map[int64]MailboxCounts{} + var totalExpungedSize int64 err = bstore.QueryTx[Message](tx).ForEach(func(m Message) error { mc := counts[m.MailboxID] mc.Add(m.MailboxCounts()) @@ -1422,7 +1700,7 @@ func (a *Account) CheckConsistency() error { mb := mailboxes[m.MailboxID] - if (m.ModSeq == 0 || m.CreateSeq == 0 || m.CreateSeq > m.ModSeq) && len(modseqErrors) < 20 { + if (!a.skipMessageZeroSeqCheck && (m.ModSeq == 0 || m.CreateSeq == 0) || m.CreateSeq > m.ModSeq) && len(modseqErrors) < 20 { modseqerr := fmt.Sprintf("message %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq) modseqErrors = append(modseqErrors, modseqerr) } @@ -1431,6 +1709,9 @@ func (a *Account) CheckConsistency() error { uidErrors = append(uidErrors, uiderr) } if m.Expunged { + if skip := eraseMessageIDs[m.ID]; !skip { + totalExpungedSize += m.Size + } return nil } @@ -1455,7 +1736,7 @@ func (a *Account) CheckConsistency() error { } for i, pid := range m.ThreadParentIDs { am := Message{ID: pid} - if err := tx.Get(&am); err == bstore.ErrAbsent { + if err := tx.Get(&am); err == bstore.ErrAbsent || err == nil && am.Expunged { continue } else if err != nil { return fmt.Errorf("get ancestor message: %v", err) @@ -1470,9 +1751,10 @@ func (a *Account) CheckConsistency() error { if jf != nil { if m.Junk != m.Notjunk { ntrained++ - if _, err := a.TrainMessage(ctx, log, jf, m); err != nil { + if _, err := a.TrainMessage(ctx, log, jf, m.Notjunk, m); err != nil { return fmt.Errorf("train message: %v", err) } + // We are not setting m.TrainedJunk, we were only recalculating the words. } } @@ -1483,21 +1765,23 @@ func (a *Account) CheckConsistency() error { } msgdir := filepath.Join(a.Dir, "msg") - err = filepath.Walk(msgdir, func(path string, info fs.FileInfo, err error) error { + err = filepath.WalkDir(msgdir, func(path string, entry fs.DirEntry, err error) error { if err != nil { if path == msgdir && errors.Is(err, fs.ErrNotExist) { return nil } return err } - if info.IsDir() { + if entry.IsDir() { return nil } id, err := strconv.ParseInt(filepath.Base(path), 10, 64) if err != nil { return fmt.Errorf("parsing message id from path %q: %v", path, err) } - if _, ok := messageIDs[id]; !ok { + _, mok := messageIDs[id] + _, meok := eraseMessageIDs[id] + if !mok && !meok { return fmt.Errorf("unexpected message file %q", path) } return nil @@ -1506,9 +1790,9 @@ func (a *Account) CheckConsistency() error { return fmt.Errorf("walking message dir: %v", err) } - var totalSize int64 - for _, mb := range mailboxes { - totalSize += mb.Size + var totalMailboxSize int64 + for _, mb := range mailboxNames { + totalMailboxSize += mb.Size if !mb.HaveCounts { errmsg := fmt.Sprintf("mailbox %q (id %d) does not have counts, should be %#v", mb.Name, mb.ID, counts[mb.ID]) errmsgs = append(errmsgs, errmsg) @@ -1522,8 +1806,8 @@ func (a *Account) CheckConsistency() error { if err := tx.Get(&du); err != nil { return fmt.Errorf("get diskusage") } - if du.MessageSize != totalSize { - errmsg := fmt.Sprintf("total message size in database is %d, sum of mailbox message sizes is %d", du.MessageSize, totalSize) + if du.MessageSize != totalMailboxSize+totalExpungedSize { + errmsg := fmt.Sprintf("total disk usage message size in database is %d != sum of mailbox message sizes %d + sum unerased expunged message sizes %d", du.MessageSize, totalMailboxSize, totalExpungedSize) errmsgs = append(errmsgs, errmsg) } @@ -1681,8 +1965,14 @@ type AddOpts struct { SkipTraining bool } +// todo optimization: when moving files, we open the original, call MessageAdd() which hardlinks it and close the file gain. when passing the filename, we could just use os.Link, saves 2 syscalls. + // MessageAdd delivers a mail message to the account. // +// If the message does not fit in the quota, an error with ErrOverQuota is returned +// and the mailbox and message are unchanged and the transaction can continue. For +// other errors, the caller must abort the transaction. +// // The message, with msg.MsgPrefix and msgFile combined, must have a header // section. The caller is responsible for adding a header separator to // msg.MsgPrefix if missing from an incoming message. @@ -1753,6 +2043,14 @@ func (a *Account) MessageAdd(log mlog.Log, tx *bstore.Tx, mb *Mailbox, m *Messag } mb.ModSeq = m.ModSeq + if m.SaveDate == nil { + now := time.Now() + m.SaveDate = &now + } + if m.Received.IsZero() { + m.Received = time.Now() + } + if len(m.Keywords) > 0 { mb.Keywords, _ = MergeKeywords(mb.Keywords, m.Keywords) } @@ -1880,7 +2178,7 @@ func (a *Account) MessageAdd(log mlog.Log, tx *bstore.Tx, mb *Mailbox, m *Messag if a.lastMsgDir != msgDir { os.MkdirAll(msgDir, 0770) if err := moxio.SyncDir(log, msgDir); err != nil { - return fmt.Errorf("sync message dir: %v", err) + return fmt.Errorf("sync message dir: %w", err) } a.lastMsgDir = msgDir } @@ -1912,7 +2210,7 @@ func (a *Account) MessageAdd(log mlog.Log, tx *bstore.Tx, mb *Mailbox, m *Messag if !opts.SkipTraining && m.NeedsTraining() && a.HasJunkFilter() { jf, opened, err := a.ensureJunkFilter(context.TODO(), log, opts.JunkFilter) if err != nil { - return fmt.Errorf("open junk filter: %v", err) + return fmt.Errorf("open junk filter: %w", err) } defer func() { if jf != nil && opened { @@ -2069,10 +2367,13 @@ func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool, spec return Mailbox{}, nil, fmt.Errorf("bad casing for inbox") } + // Get mailboxes with same name or prefix (parents). elems := strings.Split(name, "/") q := bstore.QueryTx[Mailbox](tx) - q.FilterFn(func(mb Mailbox) bool { - return mb.Name == elems[0] || strings.HasPrefix(mb.Name, elems[0]+"/") + q.FilterEqual("Expunged", false) + q.FilterFn(func(xmb Mailbox) bool { + t := strings.Split(xmb.Name, "/") + return len(t) <= len(elems) && slices.Equal(t, elems[:len(t)]) }) l, err := q.List() if err != nil { @@ -2085,14 +2386,16 @@ func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool, spec } p := "" - var existed bool + var exists bool + var parentID int64 for _, elem := range elems { if p != "" { p += "/" } p += elem - mb, existed = mailboxes[p] - if existed { + mb, exists = mailboxes[p] + if exists { + parentID = mb.ID continue } uidval, err := a.NextUIDValidity(tx) @@ -2106,34 +2409,41 @@ func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool, spec } } mb = Mailbox{ + CreateSeq: *modseq, + ModSeq: *modseq, + ParentID: parentID, Name: p, UIDValidity: uidval, UIDNext: 1, - ModSeq: *modseq, - CreateSeq: *modseq, HaveCounts: true, } err = tx.Insert(&mb) if err != nil { - return Mailbox{}, nil, fmt.Errorf("creating new mailbox: %v", err) + return Mailbox{}, nil, fmt.Errorf("creating new mailbox %q: %v", p, err) } + parentID = mb.ID var flags []string if subscribe { if tx.Get(&Subscription{p}) != nil { err := tx.Insert(&Subscription{p}) if err != nil { - return Mailbox{}, nil, fmt.Errorf("subscribing to mailbox: %v", err) + return Mailbox{}, nil, fmt.Errorf("subscribing to mailbox %q: %v", p, err) } } flags = []string{`\Subscribed`} + } else if err := tx.Get(&Subscription{p}); err == nil { + flags = []string{`\Subscribed`} + } else if err != bstore.ErrAbsent { + return Mailbox{}, nil, fmt.Errorf("looking up subscription for %q: %v", p, err) } + changes = append(changes, ChangeAddMailbox{mb, flags, *modseq}) } // Clear any special-use flags from existing mailboxes and assign them to this mailbox. var zeroSpecialUse SpecialUse - if !existed && specialUse != zeroSpecialUse { + if !exists && specialUse != zeroSpecialUse { var qerr error clearSpecialUse := func(b bool, fn func(*Mailbox) *bool) { if !b || qerr != nil { @@ -2182,6 +2492,7 @@ func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool, spec // Caller must hold account rlock. func (a *Account) MailboxExists(tx *bstore.Tx, name string) (bool, error) { q := bstore.QueryTx[Mailbox](tx) + q.FilterEqual("Expunged", false) q.FilterEqual("Name", name) return q.Exists() } @@ -2189,6 +2500,7 @@ func (a *Account) MailboxExists(tx *bstore.Tx, name string) (bool, error) { // MailboxFind finds a mailbox by name, returning a nil mailbox and nil error if mailbox does not exist. func (a *Account) MailboxFind(tx *bstore.Tx, name string) (*Mailbox, error) { q := bstore.QueryTx[Mailbox](tx) + q.FilterEqual("Expunged", false) q.FilterEqual("Name", name) mb, err := q.Get() if err == bstore.ErrAbsent { @@ -2213,6 +2525,7 @@ func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, erro } q := bstore.QueryTx[Mailbox](tx) + q.FilterEqual("Expunged", false) q.FilterEqual("Name", name) _, err := q.Get() if err == nil { @@ -2349,6 +2662,7 @@ func (a *Account) DeliverMailbox(log mlog.Log, mailbox string, m *Message, msgFi p := a.MessagePath(m.ID) err := os.Remove(p) log.Check(err, "remove delivered message file", slog.String("path", p)) + m.ID = 0 } }() @@ -2386,152 +2700,159 @@ func (a *Account) DeliverMailbox(log mlog.Log, mailbox string, m *Message, msgFi return nil } -// TidyRejectsMailbox removes old reject emails, and returns whether there is space for a new delivery. -// -// Caller most hold account wlock. -// Changes are broadcasted. -func (a *Account) TidyRejectsMailbox(log mlog.Log, rejectsMailbox string) (hasSpace bool, rerr error) { - var changes []Change - - var remove []Message - defer func() { - for _, m := range remove { - p := a.MessagePath(m.ID) - err := os.Remove(p) - log.Check(err, "removing rejects message file", slog.String("path", p)) - } - }() - - err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error { - mb, err := a.MailboxFind(tx, rejectsMailbox) - if err != nil { - return fmt.Errorf("finding mailbox: %w", err) - } - if mb == nil { - // No messages have been delivered yet. - hasSpace = true - return nil - } - - // Gather old messages to remove. - old := time.Now().Add(-14 * 24 * time.Hour) - qdel := bstore.QueryTx[Message](tx) - qdel.FilterNonzero(Message{MailboxID: mb.ID}) - qdel.FilterEqual("Expunged", false) - qdel.FilterLess("Received", old) - remove, err = qdel.List() - if err != nil { - return fmt.Errorf("listing old messages: %w", err) - } - - changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove) - if err != nil { - return fmt.Errorf("removing messages: %w", err) - } - - // We allow up to n messages. - qcount := bstore.QueryTx[Message](tx) - qcount.FilterNonzero(Message{MailboxID: mb.ID}) - qcount.FilterEqual("Expunged", false) - qcount.Limit(1000) - n, err := qcount.Count() - if err != nil { - return fmt.Errorf("counting rejects: %w", err) - } - hasSpace = n < 1000 - - return nil - }) - if err != nil { - remove = nil // Don't remove files on failure. - return false, err - } - - BroadcastChanges(a, changes) - - return hasSpace, nil +type RemoveOpts struct { + JunkFilter *junk.Filter // If set, this filter is used for training, instead of opening and saving the junk filter. } -func (a *Account) rejectsRemoveMessages(ctx context.Context, log mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) { +// MessageRemove markes messages as expunged, updates mailbox counts for the +// messages, sets a new modseq on the messages and mailbox, untrains the junk +// filter and queues the messages for erasing when the last reference has gone. +// +// Caller must save the modified mailbox to the database. +// +// The disk usage is not immediately updated. That will happen when the message +// is actually removed from disk. +// +// The junk filter is untrained for the messages if it was trained. +// Useful as optimization when messages are moved and the junk/nonjunk flags do not +// change (which can happen due to automatic junk/nonjunk flags for mailboxes). +// +// An empty list of messages results in an error. +// +// Caller must broadcast changes. +// +// Must be called with wlock held. +func (a *Account) MessageRemove(log mlog.Log, tx *bstore.Tx, modseq ModSeq, mb *Mailbox, opts RemoveOpts, l ...Message) (chremuids ChangeRemoveUIDs, chmbc ChangeMailboxCounts, rerr error) { if len(l) == 0 { - return nil, nil - } - ids := make([]int64, len(l)) - anyids := make([]any, len(l)) - for i, m := range l { - ids[i] = m.ID - anyids[i] = m.ID + return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("must expunge at least one message") } - // Remove any message recipients. Should not happen, but a user can move messages - // from a Sent mailbox to the rejects mailbox... - qdmr := bstore.QueryTx[Recipient](tx) - qdmr.FilterEqual("MessageID", anyids...) - if _, err := qdmr.Delete(); err != nil { - return nil, fmt.Errorf("deleting from message recipient: %w", err) - } - - // Assign new modseq. - modseq, err := a.NextModSeq(tx) - if err != nil { - return nil, fmt.Errorf("assign next modseq: %w", err) - } mb.ModSeq = modseq - // Expunge the messages. - qx := bstore.QueryTx[Message](tx) - qx.FilterIDs(ids) - var expunged []Message - qx.Gather(&expunged) - if _, err := qx.UpdateNonzero(Message{ModSeq: modseq, Expunged: true}); err != nil { - return nil, fmt.Errorf("expunging messages: %w", err) - } - - var totalSize int64 - for _, m := range expunged { - m.Expunged = false // Was set by update, but would cause wrong count. - mb.MailboxCounts.Sub(m.MailboxCounts()) - totalSize += m.Size - } - if err := tx.Update(mb); err != nil { - return nil, fmt.Errorf("updating mailbox counts: %w", err) - } - if err := a.AddMessageSize(log, tx, -totalSize); err != nil { - return nil, fmt.Errorf("updating disk usage: %w", err) - } - - // Mark as neutral and train so junk filter gets untrained with these (junk) messages. - for i := range expunged { - expunged[i].Junk = false - expunged[i].Notjunk = false - } - if err := a.RetrainMessages(ctx, log, tx, expunged); err != nil { - return nil, fmt.Errorf("retraining expunged messages: %w", err) - } - - changes := make([]Change, len(l), len(l)+1) + // Remove any message recipients. + anyIDs := make([]any, len(l)) for i, m := range l { - changes[i] = ChangeRemoveUIDs{mb.ID, []UID{m.UID}, modseq} + anyIDs[i] = m.ID } - changes = append(changes, mb.ChangeCounts()) - return changes, nil + qmr := bstore.QueryTx[Recipient](tx) + qmr.FilterEqual("MessageID", anyIDs...) + if _, err := qmr.Delete(); err != nil { + return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("deleting message recipients for messages: %w", err) + } + + // Loaded lazily. + jf := opts.JunkFilter + + // Mark messages expunged. + ids := make([]int64, 0, len(l)) + uids := make([]UID, 0, len(l)) + for _, m := range l { + ids = append(ids, m.ID) + uids = append(uids, m.UID) + + if m.Expunged { + return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("message %d is already expunged", m.ID) + } + + mb.Sub(m.MailboxCounts()) + + m.ModSeq = modseq + m.Expunged = true + m.Junk = false + m.Notjunk = false + + if err := tx.Update(&m); err != nil { + return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("marking message %d expunged: %v", m.ID, err) + } + + // Ensure message gets erased in future. + if err := tx.Insert(&MessageErase{m.ID, false}); err != nil { + return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("inserting message erase %d : %v", m.ID, err) + } + + if m.TrainedJunk == nil || !a.HasJunkFilter() { + continue + } + // Untrain, as needed by updated flags Junk/Notjunk to false. + if jf == nil { + var err error + jf, _, err = a.OpenJunkFilter(context.TODO(), log) + if err != nil { + return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("open junk filter: %v", err) + } + defer func() { + err := jf.Close() + if rerr == nil { + rerr = err + } else { + log.Check(err, "closing junk filter") + } + }() + } + if err := a.RetrainMessage(context.TODO(), log, tx, jf, &m); err != nil { + return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("retraining expunged messages: %w", err) + } + } + + return ChangeRemoveUIDs{mb.ID, uids, modseq, ids}, mb.ChangeCounts(), nil +} + +// TidyRejectsMailbox removes old reject emails, and returns whether there is space for a new delivery. +// +// The changed mailbox is saved to the database. +// +// Caller most hold account wlock. +// Caller must broadcast changes. +func (a *Account) TidyRejectsMailbox(log mlog.Log, tx *bstore.Tx, mbRej *Mailbox) (changes []Change, hasSpace bool, rerr error) { + // Gather old messages to expunge. + old := time.Now().Add(-14 * 24 * time.Hour) + qdel := bstore.QueryTx[Message](tx) + qdel.FilterNonzero(Message{MailboxID: mbRej.ID}) + qdel.FilterEqual("Expunged", false) + qdel.FilterLess("Received", old) + qdel.SortAsc("UID") + expunge, err := qdel.List() + if err != nil { + return nil, false, fmt.Errorf("listing old messages: %w", err) + } + + if len(expunge) > 0 { + modseq, err := a.NextModSeq(tx) + if err != nil { + return nil, false, fmt.Errorf("next mod seq: %v", err) + } + + chremuids, chmbcounts, err := a.MessageRemove(log, tx, modseq, mbRej, RemoveOpts{}, expunge...) + if err != nil { + return nil, false, fmt.Errorf("removing messages: %w", err) + } + if err := tx.Update(mbRej); err != nil { + return nil, false, fmt.Errorf("updating mailbox: %v", err) + } + changes = append(changes, chremuids, chmbcounts) + } + + // We allow up to n messages. + qcount := bstore.QueryTx[Message](tx) + qcount.FilterNonzero(Message{MailboxID: mbRej.ID}) + qcount.FilterEqual("Expunged", false) + qcount.Limit(1000) + n, err := qcount.Count() + if err != nil { + return nil, false, fmt.Errorf("counting rejects: %w", err) + } + hasSpace = n < 1000 + + return changes, hasSpace, nil } // RejectsRemove removes a message from the rejects mailbox if present. +// // Caller most hold account wlock. // Changes are broadcasted. func (a *Account) RejectsRemove(log mlog.Log, rejectsMailbox, messageID string) error { var changes []Change - var remove []Message - defer func() { - for _, m := range remove { - p := a.MessagePath(m.ID) - err := os.Remove(p) - log.Check(err, "removing rejects message file", slog.String("path", p)) - } - }() - err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error { mb, err := a.MailboxFind(tx, rejectsMailbox) if err != nil { @@ -2544,20 +2865,33 @@ func (a *Account) RejectsRemove(log mlog.Log, rejectsMailbox, messageID string) q := bstore.QueryTx[Message](tx) q.FilterNonzero(Message{MailboxID: mb.ID, MessageID: messageID}) q.FilterEqual("Expunged", false) - remove, err = q.List() + expunge, err := q.List() if err != nil { return fmt.Errorf("listing messages to remove: %w", err) } - changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove) + if len(expunge) == 0 { + return nil + } + + modseq, err := a.NextModSeq(tx) + if err != nil { + return fmt.Errorf("get next mod seq: %v", err) + } + + chremuids, chmbcounts, err := a.MessageRemove(log, tx, modseq, mb, RemoveOpts{}, expunge...) if err != nil { return fmt.Errorf("removing messages: %w", err) } + changes = append(changes, chremuids, chmbcounts) + + if err := tx.Update(mb); err != nil { + return fmt.Errorf("saving mailbox: %w", err) + } return nil }) if err != nil { - remove = nil // Don't remove files on failure. return err } @@ -2985,6 +3319,21 @@ func (a *Account) SendLimitReached(tx *bstore.Tx, recipients []smtp.Path) (msgli return -1, -1, nil } +var ErrMailboxExpunged = errors.New("mailbox was deleted") + +// MailboxID gets a mailbox by ID. +// +// Returns bstore.ErrAbsent if the mailbox does not exist. +// Returns ErrMailboxExpunged if the mailbox is expunged. +func MailboxID(tx *bstore.Tx, id int64) (Mailbox, error) { + mb := Mailbox{ID: id} + err := tx.Get(&mb) + if err == nil && mb.Expunged { + return Mailbox{}, ErrMailboxExpunged + } + return mb, err +} + // MailboxCreate creates a new mailbox, including any missing parent mailboxes, // the total list of created mailboxes is returned in created. On success, if // exists is false and rerr nil, the changes must be broadcasted by the caller. @@ -3023,217 +3372,200 @@ func (a *Account) MailboxCreate(tx *bstore.Tx, name string, specialUse SpecialUs return nmb, changes, created, false, nil } -// MailboxRename renames mailbox mbsrc to dst, and any missing parents for the -// destination, and any children of mbsrc and the destination. +// MailboxRename renames mailbox mbsrc to dst, including children of mbsrc, and +// adds missing parents for dst. // -// Names must be normalized and cannot be Inbox. -func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc Mailbox, dst string, modseq *ModSeq) (changes []Change, isInbox, notExists, alreadyExists bool, rerr error) { +// Names must be in normalized form and cannot be Inbox. +func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc *Mailbox, dst string, modseq *ModSeq) (changes []Change, isInbox, alreadyExists bool, rerr error) { if mbsrc.Name == "Inbox" || dst == "Inbox" { - return nil, true, false, false, fmt.Errorf("inbox cannot be renamed") + return nil, true, false, fmt.Errorf("inbox cannot be renamed") } - // We gather existing mailboxes that we need for deciding what to create/delete/update. - q := bstore.QueryTx[Mailbox](tx) - srcPrefix := mbsrc.Name + "/" - dstRoot := strings.SplitN(dst, "/", 2)[0] - dstRootPrefix := dstRoot + "/" - q.FilterFn(func(mb Mailbox) bool { - return mb.Name == mbsrc.Name || strings.HasPrefix(mb.Name, srcPrefix) || mb.Name == dstRoot || strings.HasPrefix(mb.Name, dstRootPrefix) - }) - q.SortAsc("Name") // We'll rename the parents before children. - l, err := q.List() - if err != nil { - return nil, false, false, false, fmt.Errorf("listing relevant mailboxes: %v", err) + // Check if destination mailbox already exists. + if exists, err := a.MailboxExists(tx, dst); err != nil { + return nil, false, false, fmt.Errorf("checking if destination mailbox exists: %v", err) + } else if exists { + return nil, false, true, fmt.Errorf("destination mailbox already exists") } - mailboxes := map[string]Mailbox{} - for _, mb := range l { - mailboxes[mb.Name] = mb - } - - if _, ok := mailboxes[mbsrc.Name]; !ok { - return nil, false, true, false, fmt.Errorf("mailbox does not exist") - } - - uidval, err := a.NextUIDValidity(tx) - if err != nil { - return nil, false, false, false, fmt.Errorf("next uid validity: %v", err) - } if *modseq == 0 { + var err error *modseq, err = a.NextModSeq(tx) if err != nil { - return nil, false, false, false, fmt.Errorf("get next modseq: %v", err) + return nil, false, false, fmt.Errorf("get next modseq: %v", err) } } - // Ensure parent mailboxes for the destination paths exist. - var parent string - dstElems := strings.Split(dst, "/") - for i, elem := range dstElems[:len(dstElems)-1] { - if i > 0 { - parent += "/" - } - parent += elem + origName := mbsrc.Name - mb, ok := mailboxes[parent] - if ok { + // Move children to their new name. + srcPrefix := mbsrc.Name + "/" + q := bstore.QueryTx[Mailbox](tx) + q.FilterEqual("Expunged", false) + q.FilterFn(func(mb Mailbox) bool { + return strings.HasPrefix(mb.Name, srcPrefix) + }) + q.SortDesc("Name") // From leaf towards dst. + kids, err := q.List() + if err != nil { + return nil, false, false, fmt.Errorf("listing child mailboxes") + } + + // Rename children, from leaf towards dst (because sorted reverse by name). + for _, mb := range kids { + nname := dst + "/" + mb.Name[len(mbsrc.Name)+1:] + var flags []string + if err := tx.Get(&Subscription{nname}); err == nil { + flags = []string{`\Subscribed`} + } else if err != bstore.ErrAbsent { + return nil, false, false, fmt.Errorf("look up subscription for new name of child %q: %v", nname, err) + } + // Leaf is first. + changes = append(changes, ChangeRenameMailbox{mb.ID, mb.Name, nname, flags, *modseq}) + + mb.Name = nname + mb.ModSeq = *modseq + if err := tx.Update(&mb); err != nil { + return nil, false, false, fmt.Errorf("rename child mailbox %q: %v", mb.Name, err) + } + } + + // Move name out of the way. We may have to create it again, as our new parent. + var flags []string + if err := tx.Get(&Subscription{dst}); err == nil { + flags = []string{`\Subscribed`} + } else if err != bstore.ErrAbsent { + return nil, false, false, fmt.Errorf("look up subscription for new name %q: %v", dst, err) + } + changes = append(changes, ChangeRenameMailbox{mbsrc.ID, mbsrc.Name, dst, flags, *modseq}) + mbsrc.ModSeq = *modseq + mbsrc.Name = dst + if err := tx.Update(mbsrc); err != nil { + return nil, false, false, fmt.Errorf("rename mailbox: %v", err) + } + + // Add any missing parents for the new name. A mailbox may have been renamed from + // a/b to a/b/x/y, and we'll have to add a new "a" and a/b. + t := strings.Split(dst, "/") + t = t[:len(t)-1] + var parent Mailbox + var parentChanges []Change + for i := range t { + s := strings.Join(t[:i+1], "/") + q := bstore.QueryTx[Mailbox](tx) + q.FilterEqual("Expunged", false) + q.FilterNonzero(Mailbox{Name: s}) + pmb, err := q.Get() + if err == nil { + parent = pmb continue + } else if err != bstore.ErrAbsent { + return nil, false, false, fmt.Errorf("lookup destination parent mailbox %q: %v", s, err) } - omb := mb - mb = Mailbox{ - ID: omb.ID, - Name: parent, + uidval, err := a.NextUIDValidity(tx) + if err != nil { + return nil, false, false, fmt.Errorf("next uid validity: %v", err) + } + parent = Mailbox{ + CreateSeq: *modseq, + ModSeq: *modseq, + ParentID: parent.ID, + Name: s, UIDValidity: uidval, UIDNext: 1, - ModSeq: *modseq, - CreateSeq: *modseq, HaveCounts: true, } - if err := tx.Insert(&mb); err != nil { - return nil, false, false, false, fmt.Errorf("creating parent mailbox %q: %v", mb.Name, err) + if err := tx.Insert(&parent); err != nil { + return nil, false, false, fmt.Errorf("inserting destination parent mailbox %q: %v", s, err) } - if err := tx.Get(&Subscription{Name: parent}); err != nil { - if err := tx.Insert(&Subscription{Name: parent}); err != nil { - return nil, false, false, false, fmt.Errorf("creating subscription for %q: %v", parent, err) - } + + var flags []string + if err := tx.Get(&Subscription{parent.Name}); err == nil { + flags = []string{`\Subscribed`} + } else if err != bstore.ErrAbsent { + return nil, false, false, fmt.Errorf("look up subscription for new parent %q: %v", parent.Name, err) } - changes = append(changes, ChangeAddMailbox{mb, []string{`\Subscribed`}, *modseq}) + parentChanges = append(parentChanges, ChangeAddMailbox{parent, flags, *modseq}) } - // Process src mailboxes, renaming them to dst. - for _, srcmb := range l { - if srcmb.Name != mbsrc.Name && !strings.HasPrefix(srcmb.Name, srcPrefix) { - continue - } - srcName := srcmb.Name - dstName := dst + srcmb.Name[len(mbsrc.Name):] - if _, ok := mailboxes[dstName]; ok { - return nil, false, false, true, fmt.Errorf("destination mailbox %q already exists", dstName) - } - - srcmb.Name = dstName - srcmb.UIDValidity = uidval - srcmb.ModSeq = *modseq - if err := tx.Update(&srcmb); err != nil { - return nil, false, false, false, fmt.Errorf("renaming mailbox: %v", err) - } - - var dstFlags []string - if tx.Get(&Subscription{Name: dstName}) == nil { - dstFlags = []string{`\Subscribed`} - } - changes = append(changes, ChangeRenameMailbox{srcmb.ID, srcName, dstName, dstFlags, *modseq}) + mbsrc.ParentID = parent.ID + if err := tx.Update(mbsrc); err != nil { + return nil, false, false, fmt.Errorf("set parent id on rename mailbox: %v", err) } - // If we renamed e.g. a/b to a/b/c/d, and a/b/c to a/b/c/d/c, we'll have to recreate a/b and a/b/c. - srcElems := strings.Split(mbsrc.Name, "/") - xsrc := mbsrc.Name - for i := 0; i < len(dstElems) && strings.HasPrefix(dst, xsrc+"/"); i++ { - mb := Mailbox{ - UIDValidity: uidval, - UIDNext: 1, - Name: xsrc, - ModSeq: *modseq, - CreateSeq: *modseq, - HaveCounts: true, - } - if err := tx.Insert(&mb); err != nil { - return nil, false, false, false, fmt.Errorf("creating mailbox at old path %q: %v", mb.Name, err) - } - xsrc += "/" + dstElems[len(srcElems)+i] + // If we were moved from a/b to a/b/x, we mention the creation of a/b after we mentioned the rename. + if strings.HasPrefix(dst, origName+"/") { + changes = append(changes, parentChanges) + } else { + changes = slices.Concat(parentChanges, changes) } - return changes, false, false, false, nil + + return changes, false, false, nil } -// MailboxDelete deletes a mailbox by ID, including its annotations. If it has +// MailboxDelete marks a mailbox as deleted, including its annotations. If it has // children, the return value indicates that and an error is returned. // -// Caller should broadcast the changes and remove files for the removed message IDs. -func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) { +// Caller should broadcast the changes (deleting all messages in the mailbox and +// deleting the mailbox itself). +func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx, mb *Mailbox) (changes []Change, hasChildren bool, rerr error) { // Look for existence of child mailboxes. There is a lot of text in the IMAP RFCs about // NoInferior and NoSelect. We just require only leaf mailboxes are deleted. qmb := bstore.QueryTx[Mailbox](tx) - mbprefix := mailbox.Name + "/" - qmb.FilterFn(func(mb Mailbox) bool { - return strings.HasPrefix(mb.Name, mbprefix) + qmb.FilterEqual("Expunged", false) + mbprefix := mb.Name + "/" + qmb.FilterFn(func(xmb Mailbox) bool { + return strings.HasPrefix(xmb.Name, mbprefix) }) if childExists, err := qmb.Exists(); err != nil { - return nil, nil, false, fmt.Errorf("checking if mailbox has child: %v", err) + return nil, false, fmt.Errorf("checking if mailbox has child: %v", err) } else if childExists { - return nil, nil, true, fmt.Errorf("mailbox has a child, only leaf mailboxes can be deleted") + return nil, true, fmt.Errorf("mailbox has a child, only leaf mailboxes can be deleted") } - // todo jmap: instead of completely deleting a mailbox and its messages, we need to mark them all as expunged. - modseq, err := a.NextModSeq(tx) if err != nil { - return nil, nil, false, fmt.Errorf("get next modseq: %v", err) + return nil, false, fmt.Errorf("get next modseq: %v", err) } qm := bstore.QueryTx[Message](tx) - qm.FilterNonzero(Message{MailboxID: mailbox.ID}) - remove, err := qm.List() + qm.FilterNonzero(Message{MailboxID: mb.ID}) + qm.FilterEqual("Expunged", false) + qm.SortAsc("UID") + l, err := qm.List() if err != nil { - return nil, nil, false, fmt.Errorf("listing messages to remove: %v", err) + return nil, false, fmt.Errorf("listing messages in mailbox to remove; %v", err) } - if len(remove) > 0 { - removeIDs := make([]any, len(remove)) - for i, m := range remove { - removeIDs[i] = m.ID - } - qmr := bstore.QueryTx[Recipient](tx) - qmr.FilterEqual("MessageID", removeIDs...) - if _, err = qmr.Delete(); err != nil { - return nil, nil, false, fmt.Errorf("removing message recipients for messages: %v", err) - } - - qm = bstore.QueryTx[Message](tx) - qm.FilterNonzero(Message{MailboxID: mailbox.ID}) - if _, err := qm.Delete(); err != nil { - return nil, nil, false, fmt.Errorf("removing messages: %v", err) - } - - var totalSize int64 - for _, m := range remove { - if !m.Expunged { - removeMessageIDs = append(removeMessageIDs, m.ID) - totalSize += m.Size - } - } - if err := a.AddMessageSize(log, tx, -totalSize); err != nil { - return nil, nil, false, fmt.Errorf("updating disk usage: %v", err) - } - - // Mark messages as not needing training. Then retrain them, so they are untrained if they were. - n := 0 - o := 0 - for _, m := range remove { - if !m.Expunged { - remove[o] = m - remove[o].Junk = false - remove[o].Notjunk = false - n++ - } - } - remove = remove[:n] - if err := a.RetrainMessages(ctx, log, tx, remove); err != nil { - return nil, nil, false, fmt.Errorf("untraining deleted messages: %v", err) + if len(l) > 0 { + chrem, _, err := a.MessageRemove(log, tx, modseq, mb, RemoveOpts{}, l...) + if err != nil { + return nil, false, fmt.Errorf("marking messages removed: %v", err) } + changes = append(changes, chrem) } - // Remove metadata annotations. ../rfc/5464:373 - if _, err := bstore.QueryTx[Annotation](tx).FilterNonzero(Annotation{MailboxID: mailbox.ID}).Delete(); err != nil { - return nil, nil, false, fmt.Errorf("removing annotations for mailbox: %v", err) + // Marking metadata annotations deleted. ../rfc/5464:373 + qa := bstore.QueryTx[Annotation](tx) + qa.FilterNonzero(Annotation{MailboxID: mb.ID}) + qa.FilterEqual("Expunged", false) + if _, err := qa.UpdateFields(map[string]any{"ModSeq": modseq, "Expunged": true, "IsString": false, "Value": []byte(nil)}); err != nil { + return nil, false, fmt.Errorf("removing annotations for mailbox: %v", err) } // Not sending changes about annotations on this mailbox, since the entire mailbox // is being removed. - if err := tx.Delete(&Mailbox{ID: mailbox.ID}); err != nil { - return nil, nil, false, fmt.Errorf("removing mailbox: %v", err) + mb.ModSeq = modseq + mb.Expunged = true + mb.SpecialUse = SpecialUse{} + + if err := tx.Update(mb); err != nil { + return nil, false, fmt.Errorf("updating mailbox: %v", err) } - return []Change{ChangeRemoveMailbox{mailbox.ID, mailbox.Name, modseq}}, removeMessageIDs, false, nil + + changes = append(changes, mb.ChangeRemoveMailbox()) + return changes, false, nil } // CheckMailboxName checks if name is valid, returning an INBOX-normalized name. diff --git a/store/account_test.go b/store/account_test.go index d0fa061..c5b485d 100644 --- a/store/account_test.go +++ b/store/account_test.go @@ -46,7 +46,7 @@ func TestMailbox(t *testing.T) { defer func() { err = acc.Close() tcheck(t, err, "closing account") - acc.CheckClosed() + acc.WaitClosed() }() defer Switchboard()() @@ -162,6 +162,8 @@ func TestMailbox(t *testing.T) { var modseq ModSeq acc.WithWLock(func() { + var changes []Change + err := acc.DB.Write(ctxbg, func(tx *bstore.Tx) error { _, _, err := acc.MailboxEnsure(tx, "Testbox", true, SpecialUse{}, &modseq) return err @@ -200,27 +202,33 @@ func TestMailbox(t *testing.T) { t.Fatalf("did not find Testbox2") } - changes, err := acc.SubscriptionEnsure(tx, "Testbox2") + nchanges, err := acc.SubscriptionEnsure(tx, "Testbox2") tcheck(t, err, "ensuring new subscription") - if len(changes) == 0 { + if len(nchanges) == 0 { t.Fatalf("new subscription did not result in changes") } - changes, err = acc.SubscriptionEnsure(tx, "Testbox2") + changes = append(changes, nchanges...) + nchanges, err = acc.SubscriptionEnsure(tx, "Testbox2") tcheck(t, err, "ensuring already present subscription") - if len(changes) != 0 { + if len(nchanges) != 0 { t.Fatalf("already present subscription resulted in changes") } + // todo: check that messages are removed. + mbRej, err := bstore.QueryTx[Mailbox](tx).FilterNonzero(Mailbox{Name: "Rejects"}).Get() + tcheck(t, err, "get rejects mailbox") + nchanges, hasSpace, err := acc.TidyRejectsMailbox(log, tx, &mbRej) + tcheck(t, err, "tidy rejects mailbox") + changes = append(changes, nchanges...) + if !hasSpace { + t.Fatalf("no space for more rejects") + } + return nil }) tcheck(t, err, "write tx") - // todo: check that messages are removed. - hasSpace, err := acc.TidyRejectsMailbox(log, "Rejects") - tcheck(t, err, "tidy rejects mailbox") - if !hasSpace { - t.Fatalf("no space for more rejects") - } + BroadcastChanges(acc, changes) acc.RejectsRemove(log, "Rejects", "m01@mox.example") }) diff --git a/store/export.go b/store/export.go index 447bd58..7a8075b 100644 --- a/store/export.go +++ b/store/export.go @@ -10,7 +10,6 @@ import ( "io" "log/slog" "os" - "path" "path/filepath" "strings" "time" @@ -18,6 +17,7 @@ import ( "github.com/mjl-/bstore" "github.com/mjl-/mox/mlog" + "github.com/mjl-/mox/mox-" ) // Archiver can archive multiple mailboxes and their messages. @@ -158,9 +158,10 @@ func ExportMessages(ctx context.Context, log mlog.Log, db *bstore.DB, accountDir var trimPrefix string if mailboxOpt != "" { // If exporting a specific mailbox, trim its parent path from stored file names. - trimPrefix = path.Dir(mailboxOpt) + "/" + trimPrefix = mox.ParentMailboxName(mailboxOpt) + "/" } q := bstore.QueryTx[Mailbox](tx) + q.FilterEqual("Expunged", false) q.FilterFn(func(mb Mailbox) bool { return mailboxOpt == "" || mb.Name == mailboxOpt || recursive && strings.HasPrefix(mb.Name, prefix) }) diff --git a/store/state.go b/store/state.go index 2c0b5bb..768cacb 100644 --- a/store/state.go +++ b/store/state.go @@ -1,14 +1,24 @@ package store import ( + "fmt" + "log/slog" + "os" "sync" "sync/atomic" + + "github.com/mjl-/bstore" + + "github.com/mjl-/mox/metrics" + "github.com/mjl-/mox/mlog" + "github.com/mjl-/mox/mox-" ) var ( register = make(chan *Comm) unregister = make(chan *Comm) broadcast = make(chan changeReq) + applied = make(chan removalApplied) ) type changeReq struct { @@ -18,6 +28,11 @@ type changeReq struct { done chan struct{} } +type removalApplied struct { + Account *Account + MsgIDs []int64 +} + type UID uint32 // IMAP UID. // Change to mailboxes/subscriptions/messages in an account. One of the Change* @@ -38,6 +53,7 @@ type ChangeRemoveUIDs struct { MailboxID int64 UIDs []UID // Must be in increasing UID order, for IMAP. ModSeq ModSeq + MsgIDs []int64 // Message.ID, for erasing, order does not necessarily correspond with UIDs! } // ChangeFlags is sent for an update to flags for a message, e.g. "Seen". @@ -118,61 +134,295 @@ type ChangeAnnotation struct { ModSeq ModSeq } +func messageEraser(donec chan struct{}, cleanc chan map[*Account][]int64) { + log := mlog.New("store", nil) + + for { + clean, ok := <-cleanc + if !ok { + donec <- struct{}{} + return + } + + for acc, ids := range clean { + eraseMessages(log, acc, ids) + } + } +} + +func eraseMessages(log mlog.Log, acc *Account, ids []int64) { + // We are responsible for closing the accounts. + defer func() { + err := acc.Close() + log.Check(err, "close account after erasing expunged messages", slog.String("account", acc.Name)) + }() + + acc.Lock() + defer acc.Unlock() + err := acc.DB.Write(mox.Context, func(tx *bstore.Tx) error { + du := DiskUsage{ID: 1} + if err := tx.Get(&du); err != nil { + return fmt.Errorf("get disk usage: %v", err) + } + var duchanged bool + + for _, id := range ids { + me := MessageErase{ID: id} + if err := tx.Get(&me); err != nil { + return fmt.Errorf("delete message erase record %d: %v", id, err) + } + + m := Message{ID: id} + if err := tx.Get(&m); err != nil { + return fmt.Errorf("get message %d to erase: %v", id, err) + } else if !m.Expunged { + return fmt.Errorf("message %d to erase is not marked expunged", id) + } + if !me.SkipUpdateDiskUsage { + du.MessageSize -= m.Size + duchanged = true + } + m.erase() + if err := tx.Update(&m); err != nil { + return fmt.Errorf("mark message %d erase in database: %v", id, err) + } + + if err := tx.Delete(&me); err != nil { + return fmt.Errorf("deleting message erase record %d: %v", id, err) + } + } + + if duchanged { + if err := tx.Update(&du); err != nil { + return fmt.Errorf("update disk usage after erasing: %v", err) + } + } + + return nil + }) + if err != nil { + log.Errorx("erasing expunged messages", err, + slog.String("account", acc.Name), + slog.Any("ids", ids), + ) + return + } + + // We remove the files after the database commit. It's better to have the files + // still around without being referenced from the database than references in the + // database to non-existent files. + for _, id := range ids { + p := acc.MessagePath(id) + err := os.Remove(p) + log.Check(err, "removing expunged message file from disk", slog.String("path", p)) + } +} + +func switchboard(stopc, donec chan struct{}, cleanc chan map[*Account][]int64) { + regs := map[*Account]map[*Comm]struct{}{} + + // We don't remove message files or clear fields in the Message stored in the + // database until all references, from all sessions have gone away. When we see + // an expunge of a message, we count how many comms are active (i.e. how many + // sessions reference the message). We require each of them to tell us they are no + // longer referencing that message. Once we've seen that from all Comms, we remove + // the on-disk file and the fields from the database. + // + // During the initial account open (when there are no active sessions/Comms yet, + // and we open the message database file), the message erases will also be applied. + // + // When we add an account to eraseRefs, we increase the refcount, and we decrease + // it again when removing the account. + eraseRefs := map[*Account]map[int64]int{} + + // We collect which messages can be erased per account, for sending them off to the + // eraser goroutine. When an account is added to this map, its refcount is + // increased. It is decreased again by the eraser goroutine. + eraseIDs := map[*Account][]int64{} + + addEraseIDs := func(acc *Account, ids ...int64) { + if _, ok := eraseIDs[acc]; !ok { + openAccounts.Lock() + acc.nused++ + openAccounts.Unlock() + } + eraseIDs[acc] = append(eraseIDs[acc], ids...) + } + + decreaseEraseRefs := func(acc *Account, ids ...int64) { + for _, id := range ids { + v := eraseRefs[acc][id] - 1 + if v < 0 { + metrics.PanicInc(metrics.Store) // For tests. + panic(fmt.Sprintf("negative expunged message references for account %q, message id %d", acc.Name, id)) + } + if v > 0 { + eraseRefs[acc][id] = v + continue + } + + addEraseIDs(acc, id) + delete(eraseRefs[acc], id) + if len(eraseRefs[acc]) > 0 { + continue + } + delete(eraseRefs, acc) + // Note: cannot use acc.Close, it tries to lock acc, but someone broadcasting to + // this goroutine will likely have the lock. + openAccounts.Lock() + acc.nused-- + n := acc.nused + openAccounts.Unlock() + if n < 0 { + metrics.PanicInc(metrics.Store) // For tests. + panic(fmt.Sprintf("negative reference count for account %q, after removing message id %d", acc.Name, id)) + } + } + } + + for { + // If we have messages to clean, try sending to the eraser. + cc := cleanc + if len(eraseIDs) == 0 { + cc = nil + } + + select { + case cc <- eraseIDs: + eraseIDs = map[*Account][]int64{} + + case c := <-register: + if _, ok := regs[c.acc]; !ok { + regs[c.acc] = map[*Comm]struct{}{} + } + regs[c.acc][c] = struct{}{} + + case c := <-unregister: + // Drain any ChangeRemoveUIDs references from the comm, to update our eraseRefs and + // possibly queue messages for cleaning. No need to take a lock, the caller does + // not use the comm anymore. + for _, ch := range c.changes { + rem, ok := ch.(ChangeRemoveUIDs) + if !ok { + continue + } + decreaseEraseRefs(c.acc, rem.MsgIDs...) + } + + delete(regs[c.acc], c) + if len(regs[c.acc]) == 0 { + delete(regs, c.acc) + } + + case chReq := <-broadcast: + acc := chReq.acc + + // Track references to removed messages in sessions (mostly IMAP) so we can pass + // them to the eraser. + for _, ch := range chReq.changes { + rem, ok := ch.(ChangeRemoveUIDs) + if !ok { + continue + } + + refs := len(regs[acc]) + if chReq.comm != nil { + // The sender does not get this change and doesn't have to notify us of having + // processed the removal. + refs-- + } + if refs <= 0 { + addEraseIDs(acc, rem.MsgIDs...) + continue + } + + // Comms/sessions still reference these messages, track how many. + for _, id := range rem.MsgIDs { + if _, ok := eraseRefs[acc]; !ok { + openAccounts.Lock() + acc.nused++ + openAccounts.Unlock() + + eraseRefs[acc] = map[int64]int{} + } + if _, ok := eraseRefs[acc][id]; ok { + metrics.PanicInc(metrics.Store) // For tests. + panic(fmt.Sprintf("already have eraseRef for message id %d, account %q", id, acc.Name)) + } + eraseRefs[acc][id] = refs + } + } + + for c := range regs[acc] { + // Do not send the broadcaster back their own changes. chReq.comm is nil if not + // originating from a comm, so won't match in that case. + if c == chReq.comm { + continue + } + + c.Lock() + c.changes = append(c.changes, chReq.changes...) + c.Unlock() + + select { + case c.Pending <- struct{}{}: + default: + } + } + chReq.done <- struct{}{} + + case removal := <-applied: + acc := removal.Account + + // Decrease references of messages, queueing for erasure when the last reference + // goes away. + decreaseEraseRefs(acc, removal.MsgIDs...) + + case <-stopc: + // We may still have eraseRefs, messages currently referenced in a session. Those + // messages will be erased when the database file is opened again in the future. If + // we have messages ready to erase now, we'll do that first. + + if len(eraseIDs) > 0 { + cleanc <- eraseIDs + eraseIDs = nil + } + + for acc := range eraseRefs { + err := acc.Close() + log := mlog.New("store", nil) + log.Check(err, "closing account") + } + + close(cleanc) // Tell eraser to stop. + donec <- struct{}{} // Say we are now done. + return + } + } +} + var switchboardBusy atomic.Bool // Switchboard distributes changes to accounts to interested listeners. See Comm and Change. func Switchboard() (stop func()) { - regs := map[*Account]map[*Comm]struct{}{} - done := make(chan struct{}) - if !switchboardBusy.CompareAndSwap(false, true) { panic("switchboard already busy") } - go func() { - for { - select { - case c := <-register: - if _, ok := regs[c.acc]; !ok { - regs[c.acc] = map[*Comm]struct{}{} - } - regs[c.acc][c] = struct{}{} + stopc := make(chan struct{}) + donec := make(chan struct{}) + cleanc := make(chan map[*Account][]int64) - case c := <-unregister: - delete(regs[c.acc], c) - if len(regs[c.acc]) == 0 { - delete(regs, c.acc) - } + go messageEraser(donec, cleanc) + go switchboard(stopc, donec, cleanc) - case chReq := <-broadcast: - acc := chReq.acc - for c := range regs[acc] { - // Do not send the broadcaster back their own changes. chReq.comm is nil if not - // originating from a comm, so won't match in that case. - if c == chReq.comm { - continue - } - - c.Lock() - c.changes = append(c.changes, chReq.changes...) - c.Unlock() - - select { - case c.Pending <- struct{}{}: - default: - } - } - chReq.done <- struct{}{} - - case <-done: - done <- struct{}{} - return - } - } - }() return func() { - done <- struct{}{} - <-done + stopc <- struct{}{} + + // Wait for switchboard and eraser goroutines to be ready. + <-donec + <-donec + if !switchboardBusy.CompareAndSwap(true, false) { panic("switchboard already unregistered?") } @@ -225,6 +475,13 @@ func (c *Comm) Get() []Change { return l } +// RemovalSeen must be called by consumers when they have applied the removal to +// their session. The switchboard tracks references of expunged messages, and +// removes/cleans the message up when the last reference is gone. +func (c *Comm) RemovalSeen(ch ChangeRemoveUIDs) { + applied <- removalApplied{c.acc, ch.MsgIDs} +} + // BroadcastChanges ensures changes are sent to all listeners on the accoount. func BroadcastChanges(acc *Account, ch []Change) { if len(ch) == 0 { diff --git a/store/threads.go b/store/threads.go index b2c1d25..62c3a35 100644 --- a/store/threads.go +++ b/store/threads.go @@ -357,6 +357,8 @@ func (a *Account) AssignThreads(ctx context.Context, log mlog.Log, txOpt *bstore m := Message{ID: mi.ID} if err := tx.Get(&m); err != nil { return fmt.Errorf("get message %d for resolving pending thread for message-id %s, %d: %w", mi.ID, tm.MessageID, tm.ID, err) + } else if m.Expunged { + return fmt.Errorf("message %d marked as expunged", mi.ID) } if m.ThreadID != 0 { // ThreadID already set because this is a cyclic message. If we would assign a diff --git a/store/train.go b/store/train.go index c25afa8..3188b3a 100644 --- a/store/train.go +++ b/store/train.go @@ -155,11 +155,7 @@ func (a *Account) RetrainMessage(ctx context.Context, log mlog.Log, tx *bstore.T // TrainMessage trains the junk filter based on the current m.Junk/m.Notjunk flags, // disregarding m.TrainedJunk and not updating that field. -func (a *Account) TrainMessage(ctx context.Context, log mlog.Log, jf *junk.Filter, m Message) (bool, error) { - if m.Junk == m.Notjunk { - return false, nil - } - +func (a *Account) TrainMessage(ctx context.Context, log mlog.Log, jf *junk.Filter, ham bool, m Message) (bool, error) { mr := a.MessageReader(m) defer func() { err := mr.Close() @@ -178,5 +174,5 @@ func (a *Account) TrainMessage(ctx context.Context, log mlog.Log, jf *junk.Filte return false, nil } - return true, jf.Train(ctx, m.Notjunk, words) + return true, jf.Train(ctx, ham, words) } diff --git a/verifydata.go b/verifydata.go index 9d9081d..9e60c4a 100644 --- a/verifydata.go +++ b/verifydata.go @@ -285,6 +285,9 @@ possibly making them potentially no longer readable by the previous version. if m.Expunged { return nil } + if mb.Expunged { + checkf(errors.New("mailbox is expunged but message is not"), dbpath, "message id %d is in expunged mailbox %q (id %d)", m.ID, mb.Name, mb.ID) + } totalSize += m.Size mp := store.MessagePath(m.ID) diff --git a/webaccount/account_test.go b/webaccount/account_test.go index d11b849..9283b58 100644 --- a/webaccount/account_test.go +++ b/webaccount/account_test.go @@ -111,7 +111,7 @@ func TestAccount(t *testing.T) { defer func() { err = acc.Close() tcheck(t, err, "closing account") - acc.CheckClosed() + acc.WaitClosed() }() defer store.Switchboard()() diff --git a/webaccount/import.go b/webaccount/import.go index c699470..a8fcd3f 100644 --- a/webaccount/import.go +++ b/webaccount/import.go @@ -291,7 +291,7 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store. var changes []store.Change // ID's of delivered messages. If we have to rollback, we have to remove this files. - var deliveredIDs []int64 + var newIDs []int64 sendEvent := func(kind string, v any) { buf, err := json.Marshal(v) @@ -321,7 +321,7 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store. defer func() { store.CloseRemoveTempFile(log, f, "uploaded messages") - for _, id := range deliveredIDs { + for _, id := range newIDs { p := acc.MessagePath(id) err := os.Remove(p) log.Check(err, "closing message file after import error", slog.String("path", p)) @@ -444,6 +444,7 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store. var p string var mb *store.Mailbox + var parent store.Mailbox for i, e := range strings.Split(name, "/") { if i == 0 { p = e @@ -454,10 +455,9 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store. continue } - q := bstore.QueryTx[store.Mailbox](tx) - q.FilterNonzero(store.Mailbox{Name: p}) - xmb, err := q.Get() - if err == bstore.ErrAbsent { + mb, err = acc.MailboxFind(tx, p) + ximportcheckf(err, "looking up mailbox %s to import to (aborting)", p) + if mb == nil { uidvalidity, err := acc.NextUIDValidity(tx) ximportcheckf(err, "finding next uid validity") @@ -468,26 +468,24 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store. } mb = &store.Mailbox{ + CreateSeq: modseq, + ModSeq: modseq, + ParentID: parent.ID, Name: p, UIDValidity: uidvalidity, UIDNext: 1, - ModSeq: modseq, - CreateSeq: modseq, HaveCounts: true, // Do not assign special-use flags. This existing account probably already has such mailboxes. } err = tx.Insert(mb) ximportcheckf(err, "inserting mailbox in database") + parent = *mb if tx.Get(&store.Subscription{Name: p}) != nil { err := tx.Insert(&store.Subscription{Name: p}) ximportcheckf(err, "subscribing to imported mailbox") } changes = append(changes, store.ChangeAddMailbox{Mailbox: *mb, Flags: []string{`\Subscribed`}, ModSeq: modseq}) - } else if err != nil { - ximportcheckf(err, "creating mailbox %s (aborting)", p) - } else { - mb = &xmb } if prevMailbox != "" && mb.Name != prevMailbox { sendEvent("count", importCount{prevMailbox, messages[prevMailbox]}) @@ -559,7 +557,7 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store. problemf("delivering message %s: %s (continuing)", pos, err) return } - deliveredIDs = append(deliveredIDs, m.ID) + newIDs = append(newIDs, m.ID) changes = append(changes, m.ChangeAddUID()) messages[mb.Name]++ if messages[mb.Name]%100 == 0 || prevMailbox != mb.Name { @@ -814,9 +812,9 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store. } // Match threads. - if len(deliveredIDs) > 0 { + if len(newIDs) > 0 { sendEvent("step", importStep{"matching messages with threads"}) - err = acc.AssignThreads(ctx, log, tx, deliveredIDs[0], 0, io.Discard) + err = acc.AssignThreads(ctx, log, tx, newIDs[0], 0, io.Discard) ximportcheckf(err, "assigning messages to threads") } @@ -837,7 +835,7 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store. err = tx.Commit() tx = nil ximportcheckf(err, "commit") - deliveredIDs = nil + newIDs = nil if jf != nil { if err := jf.Close(); err != nil { diff --git a/webapisrv/server.go b/webapisrv/server.go index d08b5e2..0c0b8a0 100644 --- a/webapisrv/server.go +++ b/webapisrv/server.go @@ -21,6 +21,7 @@ import ( "net" "net/http" "net/textproto" + "os" "reflect" "runtime/debug" "slices" @@ -1058,8 +1059,15 @@ func (s server) Send(ctx context.Context, req webapi.SendRequest) (resp webapi.S acc.WithRLock(func() { var changes []store.Change + var sentID int64 metricked := false defer func() { + if sentID != 0 { + p := acc.MessagePath(sentID) + err := os.Remove(p) + log.Check(err, "removing sent message file after error", slog.String("path", p)) + } + if x := recover(); x != nil { if !metricked { metricServerErrors.WithLabelValues("submit").Inc() @@ -1068,7 +1076,7 @@ func (s server) Send(ctx context.Context, req webapi.SendRequest) (resp webapi.S } }() xdbwrite(ctx, reqInfo.Account, func(tx *bstore.Tx) { - sentmb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Sent", true).Get() + sentmb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).FilterEqual("Sent", true).Get() if err == bstore.ErrAbsent { // There is no mailbox designated as Sent mailbox, so we're done. return @@ -1107,12 +1115,14 @@ func (s server) Send(ctx context.Context, req webapi.SendRequest) (resp webapi.S metricked = true } xcheckf(err, "message submitted to queue, appending message to Sent mailbox") + sentID = sentm.ID err = tx.Update(&sentmb) xcheckf(err, "updating mailbox") changes = append(changes, sentm.ChangeAddUID(), sentmb.ChangeCounts()) }) + sentID = 0 // Commit. store.BroadcastChanges(acc, changes) }) @@ -1190,8 +1200,12 @@ func xmessageGet(ctx context.Context, acc *store.Account, msgID int64) (store.Me if err := tx.Get(&m); err == bstore.ErrAbsent || err == nil && m.Expunged { panic(webapi.Error{Code: "messageNotFound", Message: "message not found"}) } - mb = store.Mailbox{ID: m.MailboxID} - return tx.Get(&mb) + var err error + mb, err = store.MailboxID(tx, m.MailboxID) + if err != nil { + return fmt.Errorf("get mailbox: %v", err) + } + return nil }) xcheckf(err, "get message") return m, mb diff --git a/webapisrv/server_test.go b/webapisrv/server_test.go index 1548ae8..b4b7564 100644 --- a/webapisrv/server_test.go +++ b/webapisrv/server_test.go @@ -82,7 +82,7 @@ func TestServer(t *testing.T) { defer func() { err := acc.Close() log.Check(err, "closing account") - acc.CheckClosed() + acc.WaitClosed() }() s := NewServer(100*1024, "/webapi/", false).(server) diff --git a/webmail/api.go b/webmail/api.go index 798e3b2..bb4d14d 100644 --- a/webmail/api.go +++ b/webmail/api.go @@ -245,7 +245,10 @@ func (Webmail) MessageFindMessageID(ctx context.Context, messageID string) (id i } xdbread(ctx, acc, func(tx *bstore.Tx) { - m, err := bstore.QueryTx[store.Message](tx).FilterNonzero(store.Message{MessageID: messageID}).Get() + q := bstore.QueryTx[store.Message](tx) + q.FilterEqual("Expunged", false) + q.FilterNonzero(store.Message{MessageID: messageID}) + m, err := q.Get() if err == bstore.ErrAbsent { return } @@ -417,24 +420,27 @@ func (w Webmail) MessageCompose(ctx context.Context, m ComposeMessage, mailboxID var nm store.Message // Remove previous draft message, append message to destination mailbox. - acc.WithRLock(func() { + acc.WithWLock(func() { var changes []store.Change + var newIDs []int64 + defer func() { + for _, id := range newIDs { + p := acc.MessagePath(id) + err := os.Remove(p) + log.Check(err, "removing added message aftr error", slog.String("path", p)) + } + }() + xdbwrite(ctx, acc, func(tx *bstore.Tx) { var modseq store.ModSeq // Only set if needed. if m.DraftMessageID > 0 { nchanges := xops.MessageDeleteTx(ctx, log, tx, acc, []int64{m.DraftMessageID}, &modseq) changes = append(changes, nchanges...) - // On-disk file is removed after lock. } - // Find mailbox to write to. - mb := store.Mailbox{ID: mailboxID} - err := tx.Get(&mb) - if err == bstore.ErrAbsent { - xcheckuserf(ctx, err, "looking up mailbox") - } + mb, err := store.MailboxID(tx, mailboxID) xcheckf(ctx, err, "looking up mailbox") if modseq == 0 { @@ -456,23 +462,18 @@ func (w Webmail) MessageCompose(ctx context.Context, m ComposeMessage, mailboxID xcheckuserf(ctx, err, "checking quota") } xcheckf(ctx, err, "storing message in mailbox") + newIDs = append(newIDs, nm.ID) err = tx.Update(&mb) xcheckf(ctx, err, "updating sent mailbox for counts") changes = append(changes, nm.ChangeAddUID(), mb.ChangeCounts()) }) + newIDs = nil store.BroadcastChanges(acc, changes) }) - // Remove on-disk file for removed draft message. - if m.DraftMessageID > 0 { - p := acc.MessagePath(m.DraftMessageID) - err := os.Remove(p) - log.Check(err, "removing draft message file") - } - return nm.ID } @@ -545,9 +546,8 @@ func xmailboxID(ctx context.Context, tx *bstore.Tx, mailboxID int64) store.Mailb if mailboxID == 0 { xcheckuserf(ctx, errors.New("invalid zero mailbox ID"), "getting mailbox") } - mb := store.Mailbox{ID: mailboxID} - err := tx.Get(&mb) - if err == bstore.ErrAbsent { + mb, err := store.MailboxID(tx, mailboxID) + if err == bstore.ErrAbsent || err == store.ErrMailboxExpunged { xcheckuserf(ctx, err, "getting mailbox") } xcheckf(ctx, err, "getting mailbox") @@ -1010,7 +1010,7 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) { // Append message to Sent mailbox, mark original messages as answered/forwarded, // remove any draft message. - acc.WithRLock(func() { + acc.WithWLock(func() { var changes []store.Change metricked := false @@ -1023,9 +1023,9 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) { } }() - var deliveredIDs []int64 + var newIDs []int64 defer func() { - for _, id := range deliveredIDs { + for _, id := range newIDs { p := acc.MessagePath(id) err := os.Remove(p) log.Check(err, "removing delivered message on error", slog.String("path", p)) @@ -1036,7 +1036,6 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) { if m.DraftMessageID > 0 { nchanges := xops.MessageDeleteTx(ctx, log, tx, acc, []int64{m.DraftMessageID}, &modseq) changes = append(changes, nchanges...) - // On-disk file is removed after lock. } if m.ResponseMessageID > 0 { @@ -1061,12 +1060,11 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) { changes = append(changes, rm.ChangeFlags(oflags)) // Update modseq of mailbox of replied/forwarded message. - rmb := store.Mailbox{ID: rm.MailboxID} - err = tx.Get(&rmb) + rmb, err := store.MailboxID(tx, rm.MailboxID) xcheckf(ctx, err, "get mailbox of replied/forwarded message for modseq update") rmb.ModSeq = modseq err = tx.Update(&rmb) - xcheckf(ctx, err, "update modseqo of mailbox of replied/forwarded message") + xcheckf(ctx, err, "update modseq of mailbox of replied/forwarded message") err = acc.RetrainMessages(ctx, log, tx, []store.Message{rm}) xcheckf(ctx, err, "retraining messages after reply/forward") @@ -1075,8 +1073,8 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) { // Move messages from this thread still in this mailbox to the designated Archive // mailbox. if m.ArchiveThread { - mbArchive, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Archive", true).Get() - if err == bstore.ErrAbsent { + mbArchive, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).FilterEqual("Archive", true).Get() + if err == bstore.ErrAbsent || err == store.ErrMailboxExpunged { xcheckuserf(ctx, errors.New("not configured"), "looking up designated archive mailbox") } xcheckf(ctx, err, "looking up designated archive mailbox") @@ -1088,14 +1086,15 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) { err = q.IDs(&msgIDs) xcheckf(ctx, err, "listing messages in thread to archive") if len(msgIDs) > 0 { - nchanges := xops.MessageMoveTx(ctx, log, acc, tx, msgIDs, mbArchive, &modseq) + ids, nchanges := xops.MessageMoveTx(ctx, log, acc, tx, msgIDs, mbArchive, &modseq) + newIDs = append(newIDs, ids...) changes = append(changes, nchanges...) } } } - sentmb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Sent", true).Get() - if err == bstore.ErrAbsent { + sentmb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).FilterEqual("Sent", true).Get() + if err == bstore.ErrAbsent || err == store.ErrMailboxExpunged { // There is no mailbox designated as Sent mailbox, so we're done. return } @@ -1135,24 +1134,17 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) { metricked = true } xcheckf(ctx, err, "message submitted to queue, appending message to Sent mailbox") - deliveredIDs = append(deliveredIDs, sentm.ID) + newIDs = append(newIDs, sentm.ID) err = tx.Update(&sentmb) xcheckf(ctx, err, "updating sent mailbox for counts") changes = append(changes, sentm.ChangeAddUID(), sentmb.ChangeCounts()) }) - deliveredIDs = nil + newIDs = nil store.BroadcastChanges(acc, changes) }) - - // Remove on-disk file for removed draft message. - if m.DraftMessageID > 0 { - p := acc.MessagePath(m.DraftMessageID) - err := os.Remove(p) - log.Check(err, "removing draft message file") - } } // MessageMove moves messages to another mailbox. If the message is already in @@ -1244,9 +1236,6 @@ func (Webmail) MailboxDelete(ctx context.Context, mailboxID int64) { acc := reqInfo.Account log := reqInfo.Log - // Messages to remove after having broadcasted the removal of messages. - var removeMessageIDs []int64 - acc.WithWLock(func() { var changes []store.Change @@ -1259,7 +1248,7 @@ func (Webmail) MailboxDelete(ctx context.Context, mailboxID int64) { var hasChildren bool var err error - changes, removeMessageIDs, hasChildren, err = acc.MailboxDelete(ctx, log, tx, mb) + changes, hasChildren, err = acc.MailboxDelete(ctx, log, tx, &mb) if hasChildren { xcheckuserf(ctx, errors.New("mailbox has children"), "deleting mailbox") } @@ -1268,12 +1257,6 @@ func (Webmail) MailboxDelete(ctx context.Context, mailboxID int64) { store.BroadcastChanges(acc, changes) }) - - for _, mID := range removeMessageIDs { - p := acc.MessagePath(mID) - err := os.Remove(p) - log.Check(err, "removing message file for mailbox delete", slog.String("path", p)) - } } // MailboxEmpty empties a mailbox, removing all messages from the mailbox, but not @@ -1283,76 +1266,36 @@ func (Webmail) MailboxEmpty(ctx context.Context, mailboxID int64) { acc := reqInfo.Account log := reqInfo.Log - var expunged []store.Message - acc.WithWLock(func() { var changes []store.Change xdbwrite(ctx, acc, func(tx *bstore.Tx) { mb := xmailboxID(ctx, tx, mailboxID) - modseq, err := acc.NextModSeq(tx) - xcheckf(ctx, err, "next modseq") - - // Mark messages as expunged. qm := bstore.QueryTx[store.Message](tx) qm.FilterNonzero(store.Message{MailboxID: mb.ID}) qm.FilterEqual("Expunged", false) qm.SortAsc("UID") - qm.Gather(&expunged) - n, err := qm.UpdateNonzero(store.Message{ModSeq: modseq, Expunged: true}) - xcheckf(ctx, err, "deleting messages") + l, err := qm.List() + xcheckf(ctx, err, "listing messages to remove") - if n == 0 { + if len(l) == 0 { xcheckuserf(ctx, errors.New("no messages in mailbox"), "emptying mailbox") } - mb.ModSeq = modseq + modseq, err := acc.NextModSeq(tx) + xcheckf(ctx, err, "next modseq") - // Remove Recipients. - anyIDs := make([]any, len(expunged)) - for i, m := range expunged { - anyIDs[i] = m.ID - } - qmr := bstore.QueryTx[store.Recipient](tx) - qmr.FilterEqual("MessageID", anyIDs...) - _, err = qmr.Delete() - xcheckf(ctx, err, "removing message recipients") - - // Adjust mailbox counts, gather UIDs for broadcasted change, prepare for untraining. - var totalSize int64 - uids := make([]store.UID, len(expunged)) - for i, m := range expunged { - m.Expunged = false // Gather returns updated values. - mb.Sub(m.MailboxCounts()) - totalSize += m.Size - uids[i] = m.UID - - expunged[i].Junk = false - expunged[i].Notjunk = false - } + chrem, chmbcounts, err := acc.MessageRemove(log, tx, modseq, &mb, store.RemoveOpts{}, l...) + xcheckf(ctx, err, "expunge messages") + changes = append(changes, chrem, chmbcounts) err = tx.Update(&mb) xcheckf(ctx, err, "updating mailbox for counts") - - err = acc.AddMessageSize(log, tx, -totalSize) - xcheckf(ctx, err, "updating disk usage") - - err = acc.RetrainMessages(ctx, log, tx, expunged) - xcheckf(ctx, err, "retraining expunged messages") - - chremove := store.ChangeRemoveUIDs{MailboxID: mb.ID, UIDs: uids, ModSeq: modseq} - changes = []store.Change{chremove, mb.ChangeCounts()} }) store.BroadcastChanges(acc, changes) }) - - for _, m := range expunged { - p := acc.MessagePath(m.ID) - err := os.Remove(p) - log.Check(err, "removing message file after emptying mailbox", slog.String("path", p)) - } } // MailboxRename renames a mailbox, possibly moving it to a new parent. The mailbox @@ -1373,10 +1316,10 @@ func (Webmail) MailboxRename(ctx context.Context, mailboxID int64, newName strin xdbwrite(ctx, acc, func(tx *bstore.Tx) { mbsrc := xmailboxID(ctx, tx, mailboxID) var err error - var isInbox, notExists, alreadyExists bool + var isInbox, alreadyExists bool var modseq store.ModSeq - changes, isInbox, notExists, alreadyExists, err = acc.MailboxRename(tx, mbsrc, newName, &modseq) - if isInbox || notExists || alreadyExists { + changes, isInbox, alreadyExists, err = acc.MailboxRename(tx, &mbsrc, newName, &modseq) + if isInbox || alreadyExists { xcheckuserf(ctx, err, "renaming mailbox") } xcheckf(ctx, err, "renaming mailbox") @@ -1555,8 +1498,8 @@ func (Webmail) ThreadCollapse(ctx context.Context, messageIDs []int64, collapse for _, id := range messageIDs { m := store.Message{ID: id} err := tx.Get(&m) - if err == bstore.ErrAbsent { - xcheckuserf(ctx, err, "get message") + if err == bstore.ErrAbsent || err == nil && m.Expunged { + xcheckuserf(ctx, bstore.ErrAbsent, "get message") } xcheckf(ctx, err, "get message") threadIDs[m.ThreadID] = struct{}{} @@ -1565,6 +1508,7 @@ func (Webmail) ThreadCollapse(ctx context.Context, messageIDs []int64, collapse var updated []store.Message q := bstore.QueryTx[store.Message](tx) + q.FilterEqual("Expunged", false) q.FilterEqual("ThreadID", slicesAny(maps.Keys(threadIDs))...) q.FilterNotEqual("ThreadCollapsed", collapse) q.FilterFn(func(tm store.Message) bool { @@ -1607,8 +1551,8 @@ func (Webmail) ThreadMute(ctx context.Context, messageIDs []int64, mute bool) { for _, id := range messageIDs { m := store.Message{ID: id} err := tx.Get(&m) - if err == bstore.ErrAbsent { - xcheckuserf(ctx, err, "get message") + if err == bstore.ErrAbsent || err == nil && m.Expunged { + xcheckuserf(ctx, bstore.ErrAbsent, "get message") } xcheckf(ctx, err, "get message") threadIDs[m.ThreadID] = struct{}{} @@ -1618,6 +1562,7 @@ func (Webmail) ThreadMute(ctx context.Context, messageIDs []int64, mute bool) { var updated []store.Message q := bstore.QueryTx[store.Message](tx) + q.FilterEqual("Expunged", false) q.FilterEqual("ThreadID", slicesAny(maps.Keys(threadIDs))...) q.FilterFn(func(tm store.Message) bool { if tm.ThreadMuted == mute && (!mute || tm.ThreadCollapsed) { diff --git a/webmail/api.json b/webmail/api.json index f657548..09ccb04 100644 --- a/webmail/api.json +++ b/webmail/api.json @@ -1601,9 +1601,37 @@ "int64" ] }, + { + "Name": "CreateSeq", + "Docs": "", + "Typewords": [ + "ModSeq" + ] + }, + { + "Name": "ModSeq", + "Docs": "Of last change, or when deleted.", + "Typewords": [ + "ModSeq" + ] + }, + { + "Name": "Expunged", + "Docs": "", + "Typewords": [ + "bool" + ] + }, + { + "Name": "ParentID", + "Docs": "Zero for top-level mailbox.", + "Typewords": [ + "int64" + ] + }, { "Name": "Name", - "Docs": "\"Inbox\" is the name for the special IMAP \"INBOX\". Slash separated for hierarchy.", + "Docs": "\"Inbox\" is the name for the special IMAP \"INBOX\". Slash separated for hierarchy. Names must be unique for mailboxes that are not expunged.", "Typewords": [ "string" ] @@ -1665,20 +1693,6 @@ "string" ] }, - { - "Name": "ModSeq", - "Docs": "ModSeq matches that of last message (including deleted), or changes to mailbox such as after metadata changes.", - "Typewords": [ - "ModSeq" - ] - }, - { - "Name": "CreateSeq", - "Docs": "", - "Typewords": [ - "ModSeq" - ] - }, { "Name": "HaveCounts", "Docs": "Whether MailboxCounts have been initialized.", @@ -2165,14 +2179,14 @@ "Fields": [ { "Name": "ID", - "Docs": "ID, unchanged over lifetime, determines path to on-disk msg file. Set during deliver.", + "Docs": "ID of the message, determines path to on-disk message file. Set when adding to a mailbox. When a message is moved to another mailbox, the mailbox ID is changed, but for synchronization purposes, a new Message record is inserted (which gets a new ID) with the Expunged field set and the MailboxID and UID copied.", "Typewords": [ "int64" ] }, { "Name": "UID", - "Docs": "UID, for IMAP. Set during deliver.", + "Docs": "UID, for IMAP. Set when adding to mailbox. Strictly increasing values, per mailbox. The UID of a message can never change (though messages can be copied), and the contents of a message/UID also never changes.", "Typewords": [ "UID" ] @@ -2435,7 +2449,7 @@ }, { "Name": "ThreadParentIDs", - "Docs": "IDs of parent messages, from closest parent to the root message. Parent messages may be in a different mailbox, or may no longer exist. ThreadParentIDs must never contain the message id itself (a cycle), and parent messages must reference the same ancestors.", + "Docs": "IDs of parent messages, from closest parent to the root message. Parent messages may be in a different mailbox, or may no longer exist. ThreadParentIDs must never contain the message id itself (a cycle), and parent messages must reference the same ancestors. Moving a message to another mailbox keeps the message ID and changes the MailboxID (and UID) of the message, leaving threading parent ids intact.", "Typewords": [ "[]", "int64" @@ -2891,6 +2905,14 @@ "Typewords": [ "ModSeq" ] + }, + { + "Name": "MsgIDs", + "Docs": "Message.ID, for erasing, order does not necessarily correspond with UIDs!", + "Typewords": [ + "[]", + "int64" + ] } ] }, @@ -3214,13 +3236,13 @@ ], "Ints": [ { - "Name": "UID", - "Docs": "IMAP UID.", + "Name": "ModSeq", + "Docs": "ModSeq represents a modseq as stored in the database. ModSeq 0 in the\ndatabase is sent to the client as 1, because modseq 0 is special in IMAP.\nModSeq coming from the client are of type int64.", "Values": null }, { - "Name": "ModSeq", - "Docs": "ModSeq represents a modseq as stored in the database. ModSeq 0 in the\ndatabase is sent to the client as 1, because modseq 0 is special in IMAP.\nModSeq coming from the client are of type int64.", + "Name": "UID", + "Docs": "IMAP UID.", "Values": null }, { diff --git a/webmail/api.ts b/webmail/api.ts index 92d5e49..5724faf 100644 --- a/webmail/api.ts +++ b/webmail/api.ts @@ -188,7 +188,11 @@ export interface ForwardAttachments { // Mailbox is collection of messages, e.g. Inbox or Sent. export interface Mailbox { ID: number - Name: string // "Inbox" is the name for the special IMAP "INBOX". Slash separated for hierarchy. + CreateSeq: ModSeq + ModSeq: ModSeq // Of last change, or when deleted. + Expunged: boolean + ParentID: number // Zero for top-level mailbox. + Name: string // "Inbox" is the name for the special IMAP "INBOX". Slash separated for hierarchy. Names must be unique for mailboxes that are not expunged. UIDValidity: number // If UIDs are invalidated, e.g. when renaming a mailbox to a previously existing name, UIDValidity must be changed. Used by IMAP for synchronization. UIDNext: UID // UID likely to be assigned to next message. Used by IMAP to detect messages delivered to a mailbox. Archive: boolean @@ -197,8 +201,6 @@ export interface Mailbox { Sent: boolean Trash: boolean Keywords?: string[] | null // Keywords as used in messages. Storing a non-system keyword for a message automatically adds it to this list. Used in the IMAP FLAGS response. Only "atoms" are allowed (IMAP syntax), keywords are case-insensitive, only stored in lower case (for JMAP), sorted. - ModSeq: ModSeq // ModSeq matches that of last message (including deleted), or changes to mailbox such as after metadata changes. - CreateSeq: ModSeq HaveCounts: boolean // Whether MailboxCounts have been initialized. Total: number // Total number of messages, excluding \Deleted. For JMAP. Deleted: number // Number of messages with \Deleted flag. Used for IMAP message count that includes messages with \Deleted. @@ -315,8 +317,8 @@ export interface MessageItem { // Messages always have a header section, even if empty. Incoming messages without // header section must get an empty header section added before inserting. export interface Message { - ID: number // ID, unchanged over lifetime, determines path to on-disk msg file. Set during deliver. - UID: UID // UID, for IMAP. Set during deliver. + ID: number // ID of the message, determines path to on-disk message file. Set when adding to a mailbox. When a message is moved to another mailbox, the mailbox ID is changed, but for synchronization purposes, a new Message record is inserted (which gets a new ID) with the Expunged field set and the MailboxID and UID copied. + UID: UID // UID, for IMAP. Set when adding to mailbox. Strictly increasing values, per mailbox. The UID of a message can never change (though messages can be copied), and the contents of a message/UID also never changes. MailboxID: number ModSeq: ModSeq // Modification sequence, for faster syncing with IMAP QRESYNC and JMAP. ModSeq is the last modification. CreateSeq is the Seq the message was inserted, always <= ModSeq. If Expunged is set, the message has been removed and should not be returned to the user. In this case, ModSeq is the Seq where the message is removed, and will never be changed again. We have an index on both ModSeq (for JMAP that synchronizes per account) and MailboxID+ModSeq (for IMAP that synchronizes per mailbox). The index on CreateSeq helps efficiently finding created messages for JMAP. The value of ModSeq is special for IMAP. Messages that existed before ModSeq was added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If we get modseq 1 from a client, the IMAP server will translate it to 0. When we return modseq to clients, we turn 0 into 1. CreateSeq: ModSeq @@ -353,7 +355,7 @@ export interface Message { SubjectBase: string // For matching threads in case there is no References/In-Reply-To header. It is lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed. MessageHash?: string | null // Hash of message. For rejects delivery in case there is no Message-ID, only set when delivered as reject. ThreadID: number // ID of message starting this thread. - ThreadParentIDs?: number[] | null // IDs of parent messages, from closest parent to the root message. Parent messages may be in a different mailbox, or may no longer exist. ThreadParentIDs must never contain the message id itself (a cycle), and parent messages must reference the same ancestors. + ThreadParentIDs?: number[] | null // IDs of parent messages, from closest parent to the root message. Parent messages may be in a different mailbox, or may no longer exist. ThreadParentIDs must never contain the message id itself (a cycle), and parent messages must reference the same ancestors. Moving a message to another mailbox keeps the message ID and changes the MailboxID (and UID) of the message, leaving threading parent ids intact. ThreadMissingLink: boolean // ThreadMissingLink is true if there is no match with a direct parent. E.g. first ID in ThreadParentIDs is not the direct ancestor (an intermediate message may have been deleted), or subject-based matching was done. ThreadMuted: boolean // If set, newly delivered child messages are automatically marked as read. This field is copied to new child messages. Changes are propagated to the webmail client. ThreadCollapsed: boolean // If set, this (sub)thread is collapsed in the webmail client, for threading mode "on" (mode "unread" ignores it). This field is copied to new child message. Changes are propagated to the webmail client. @@ -439,6 +441,7 @@ export interface ChangeMsgRemove { MailboxID: number UIDs?: UID[] | null // Must be in increasing UID order, for IMAP. ModSeq: ModSeq + MsgIDs?: number[] | null // Message.ID, for erasing, order does not necessarily correspond with UIDs! } // ChangeMsgFlags updates flags for one message. @@ -517,14 +520,14 @@ export interface ChangeMailboxKeywords { Keywords?: string[] | null } -// IMAP UID. -export type UID = number - // ModSeq represents a modseq as stored in the database. ModSeq 0 in the // database is sent to the client as 1, because modseq 0 is special in IMAP. // ModSeq coming from the client are of type int64. export type ModSeq = number +// IMAP UID. +export type UID = number + // Validation of "message From" domain. export enum Validation { ValidationUnknown = 0, @@ -613,7 +616,7 @@ export const types: TypenameMap = { "SubmitMessage": {"Name":"SubmitMessage","Docs":"","Fields":[{"Name":"From","Docs":"","Typewords":["string"]},{"Name":"To","Docs":"","Typewords":["[]","string"]},{"Name":"Cc","Docs":"","Typewords":["[]","string"]},{"Name":"Bcc","Docs":"","Typewords":["[]","string"]},{"Name":"ReplyTo","Docs":"","Typewords":["string"]},{"Name":"Subject","Docs":"","Typewords":["string"]},{"Name":"TextBody","Docs":"","Typewords":["string"]},{"Name":"Attachments","Docs":"","Typewords":["[]","File"]},{"Name":"ForwardAttachments","Docs":"","Typewords":["ForwardAttachments"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"ResponseMessageID","Docs":"","Typewords":["int64"]},{"Name":"UserAgent","Docs":"","Typewords":["string"]},{"Name":"RequireTLS","Docs":"","Typewords":["nullable","bool"]},{"Name":"FutureRelease","Docs":"","Typewords":["nullable","timestamp"]},{"Name":"ArchiveThread","Docs":"","Typewords":["bool"]},{"Name":"ArchiveReferenceMailboxID","Docs":"","Typewords":["int64"]},{"Name":"DraftMessageID","Docs":"","Typewords":["int64"]}]}, "File": {"Name":"File","Docs":"","Fields":[{"Name":"Filename","Docs":"","Typewords":["string"]},{"Name":"DataURI","Docs":"","Typewords":["string"]}]}, "ForwardAttachments": {"Name":"ForwardAttachments","Docs":"","Fields":[{"Name":"MessageID","Docs":"","Typewords":["int64"]},{"Name":"Paths","Docs":"","Typewords":["[]","[]","int32"]}]}, - "Mailbox": {"Name":"Mailbox","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"Name","Docs":"","Typewords":["string"]},{"Name":"UIDValidity","Docs":"","Typewords":["uint32"]},{"Name":"UIDNext","Docs":"","Typewords":["UID"]},{"Name":"Archive","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Sent","Docs":"","Typewords":["bool"]},{"Name":"Trash","Docs":"","Typewords":["bool"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"CreateSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"HaveCounts","Docs":"","Typewords":["bool"]},{"Name":"Total","Docs":"","Typewords":["int64"]},{"Name":"Deleted","Docs":"","Typewords":["int64"]},{"Name":"Unread","Docs":"","Typewords":["int64"]},{"Name":"Unseen","Docs":"","Typewords":["int64"]},{"Name":"Size","Docs":"","Typewords":["int64"]}]}, + "Mailbox": {"Name":"Mailbox","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"CreateSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Expunged","Docs":"","Typewords":["bool"]},{"Name":"ParentID","Docs":"","Typewords":["int64"]},{"Name":"Name","Docs":"","Typewords":["string"]},{"Name":"UIDValidity","Docs":"","Typewords":["uint32"]},{"Name":"UIDNext","Docs":"","Typewords":["UID"]},{"Name":"Archive","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Sent","Docs":"","Typewords":["bool"]},{"Name":"Trash","Docs":"","Typewords":["bool"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"HaveCounts","Docs":"","Typewords":["bool"]},{"Name":"Total","Docs":"","Typewords":["int64"]},{"Name":"Deleted","Docs":"","Typewords":["int64"]},{"Name":"Unread","Docs":"","Typewords":["int64"]},{"Name":"Unseen","Docs":"","Typewords":["int64"]},{"Name":"Size","Docs":"","Typewords":["int64"]}]}, "RecipientSecurity": {"Name":"RecipientSecurity","Docs":"","Fields":[{"Name":"STARTTLS","Docs":"","Typewords":["SecurityResult"]},{"Name":"MTASTS","Docs":"","Typewords":["SecurityResult"]},{"Name":"DNSSEC","Docs":"","Typewords":["SecurityResult"]},{"Name":"DANE","Docs":"","Typewords":["SecurityResult"]},{"Name":"RequireTLS","Docs":"","Typewords":["SecurityResult"]}]}, "Settings": {"Name":"Settings","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["uint8"]},{"Name":"Signature","Docs":"","Typewords":["string"]},{"Name":"Quoting","Docs":"","Typewords":["Quoting"]},{"Name":"ShowAddressSecurity","Docs":"","Typewords":["bool"]},{"Name":"ShowHTML","Docs":"","Typewords":["bool"]},{"Name":"NoShowShortcuts","Docs":"","Typewords":["bool"]},{"Name":"ShowHeaders","Docs":"","Typewords":["[]","string"]}]}, "Ruleset": {"Name":"Ruleset","Docs":"","Fields":[{"Name":"SMTPMailFromRegexp","Docs":"","Typewords":["string"]},{"Name":"MsgFromRegexp","Docs":"","Typewords":["string"]},{"Name":"VerifiedDomain","Docs":"","Typewords":["string"]},{"Name":"HeadersRegexp","Docs":"","Typewords":["{}","string"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"ListAllowDomain","Docs":"","Typewords":["string"]},{"Name":"AcceptRejectsToMailbox","Docs":"","Typewords":["string"]},{"Name":"Mailbox","Docs":"","Typewords":["string"]},{"Name":"Comment","Docs":"","Typewords":["string"]},{"Name":"VerifiedDNSDomain","Docs":"","Typewords":["Domain"]},{"Name":"ListAllowDNSDomain","Docs":"","Typewords":["Domain"]}]}, @@ -629,7 +632,7 @@ export const types: TypenameMap = { "EventViewChanges": {"Name":"EventViewChanges","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"Changes","Docs":"","Typewords":["[]","[]","any"]}]}, "ChangeMsgAdd": {"Name":"ChangeMsgAdd","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Flags","Docs":"","Typewords":["Flags"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"MessageItems","Docs":"","Typewords":["[]","MessageItem"]}]}, "Flags": {"Name":"Flags","Docs":"","Fields":[{"Name":"Seen","Docs":"","Typewords":["bool"]},{"Name":"Answered","Docs":"","Typewords":["bool"]},{"Name":"Flagged","Docs":"","Typewords":["bool"]},{"Name":"Forwarded","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Notjunk","Docs":"","Typewords":["bool"]},{"Name":"Deleted","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Phishing","Docs":"","Typewords":["bool"]},{"Name":"MDNSent","Docs":"","Typewords":["bool"]}]}, - "ChangeMsgRemove": {"Name":"ChangeMsgRemove","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UIDs","Docs":"","Typewords":["[]","UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]}]}, + "ChangeMsgRemove": {"Name":"ChangeMsgRemove","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UIDs","Docs":"","Typewords":["[]","UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"MsgIDs","Docs":"","Typewords":["[]","int64"]}]}, "ChangeMsgFlags": {"Name":"ChangeMsgFlags","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Mask","Docs":"","Typewords":["Flags"]},{"Name":"Flags","Docs":"","Typewords":["Flags"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]}]}, "ChangeMsgThread": {"Name":"ChangeMsgThread","Docs":"","Fields":[{"Name":"MessageIDs","Docs":"","Typewords":["[]","int64"]},{"Name":"Muted","Docs":"","Typewords":["bool"]},{"Name":"Collapsed","Docs":"","Typewords":["bool"]}]}, "ChangeMailboxRemove": {"Name":"ChangeMailboxRemove","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"Name","Docs":"","Typewords":["string"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]}]}, @@ -639,8 +642,8 @@ export const types: TypenameMap = { "ChangeMailboxSpecialUse": {"Name":"ChangeMailboxSpecialUse","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"MailboxName","Docs":"","Typewords":["string"]},{"Name":"SpecialUse","Docs":"","Typewords":["SpecialUse"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]}]}, "SpecialUse": {"Name":"SpecialUse","Docs":"","Fields":[{"Name":"Archive","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Sent","Docs":"","Typewords":["bool"]},{"Name":"Trash","Docs":"","Typewords":["bool"]}]}, "ChangeMailboxKeywords": {"Name":"ChangeMailboxKeywords","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"MailboxName","Docs":"","Typewords":["string"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]}]}, - "UID": {"Name":"UID","Docs":"","Values":null}, "ModSeq": {"Name":"ModSeq","Docs":"","Values":null}, + "UID": {"Name":"UID","Docs":"","Values":null}, "Validation": {"Name":"Validation","Docs":"","Values":[{"Name":"ValidationUnknown","Value":0,"Docs":""},{"Name":"ValidationStrict","Value":1,"Docs":""},{"Name":"ValidationDMARC","Value":2,"Docs":""},{"Name":"ValidationRelaxed","Value":3,"Docs":""},{"Name":"ValidationPass","Value":4,"Docs":""},{"Name":"ValidationNeutral","Value":5,"Docs":""},{"Name":"ValidationTemperror","Value":6,"Docs":""},{"Name":"ValidationPermerror","Value":7,"Docs":""},{"Name":"ValidationFail","Value":8,"Docs":""},{"Name":"ValidationSoftfail","Value":9,"Docs":""},{"Name":"ValidationNone","Value":10,"Docs":""}]}, "CSRFToken": {"Name":"CSRFToken","Docs":"","Values":null}, "ThreadMode": {"Name":"ThreadMode","Docs":"","Values":[{"Name":"ThreadOff","Value":"off","Docs":""},{"Name":"ThreadOn","Value":"on","Docs":""},{"Name":"ThreadUnread","Value":"unread","Docs":""}]}, @@ -694,8 +697,8 @@ export const parser = { ChangeMailboxSpecialUse: (v: any) => parse("ChangeMailboxSpecialUse", v) as ChangeMailboxSpecialUse, SpecialUse: (v: any) => parse("SpecialUse", v) as SpecialUse, ChangeMailboxKeywords: (v: any) => parse("ChangeMailboxKeywords", v) as ChangeMailboxKeywords, - UID: (v: any) => parse("UID", v) as UID, ModSeq: (v: any) => parse("ModSeq", v) as ModSeq, + UID: (v: any) => parse("UID", v) as UID, Validation: (v: any) => parse("Validation", v) as Validation, CSRFToken: (v: any) => parse("CSRFToken", v) as CSRFToken, ThreadMode: (v: any) => parse("ThreadMode", v) as ThreadMode, diff --git a/webmail/api_test.go b/webmail/api_test.go index b31986a..3e2cb3e 100644 --- a/webmail/api_test.go +++ b/webmail/api_test.go @@ -78,7 +78,7 @@ func TestAPI(t *testing.T) { tcheck(t, err, "mtastsdb close") err = acc.Close() pkglog.Check(err, "closing account") - acc.CheckClosed() + acc.WaitClosed() }() var zerom store.Message @@ -206,7 +206,7 @@ func TestAPI(t *testing.T) { var inbox, archive, sent, drafts, testbox1 store.Mailbox err = acc.DB.Read(ctx, func(tx *bstore.Tx) error { get := func(k string, v any) store.Mailbox { - mb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual(k, v).Get() + mb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).FilterEqual(k, v).Get() tcheck(t, err, "get special-use mailbox") return mb } @@ -276,7 +276,7 @@ func TestAPI(t *testing.T) { tneedError(t, func() { api.ParsedMessage(ctx, testbox1Alt.ID) }) // Message was removed and no longer exists. api.MailboxCreate(ctx, "Testbox1") - testbox1, err = bstore.QueryDB[store.Mailbox](ctx, acc.DB).FilterEqual("Name", "Testbox1").Get() + testbox1, err = bstore.QueryDB[store.Mailbox](ctx, acc.DB).FilterEqual("Expunged", false).FilterEqual("Name", "Testbox1").Get() tcheck(t, err, "get testbox1") tdeliver(t, acc, testbox1Alt) diff --git a/webmail/msg.js b/webmail/msg.js index 8ebe86a..3a165aa 100644 --- a/webmail/msg.js +++ b/webmail/msg.js @@ -309,7 +309,7 @@ var api; "SubmitMessage": { "Name": "SubmitMessage", "Docs": "", "Fields": [{ "Name": "From", "Docs": "", "Typewords": ["string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Cc", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Bcc", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "TextBody", "Docs": "", "Typewords": ["string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "File"] }, { "Name": "ForwardAttachments", "Docs": "", "Typewords": ["ForwardAttachments"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ResponseMessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UserAgent", "Docs": "", "Typewords": ["string"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "FutureRelease", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "ArchiveThread", "Docs": "", "Typewords": ["bool"] }, { "Name": "ArchiveReferenceMailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "DraftMessageID", "Docs": "", "Typewords": ["int64"] }] }, "File": { "Name": "File", "Docs": "", "Fields": [{ "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "DataURI", "Docs": "", "Typewords": ["string"] }] }, "ForwardAttachments": { "Name": "ForwardAttachments", "Docs": "", "Fields": [{ "Name": "MessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Paths", "Docs": "", "Typewords": ["[]", "[]", "int32"] }] }, - "Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] }, + "Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "ParentID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] }, "RecipientSecurity": { "Name": "RecipientSecurity", "Docs": "", "Fields": [{ "Name": "STARTTLS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "MTASTS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DNSSEC", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DANE", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["SecurityResult"] }] }, "Settings": { "Name": "Settings", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["uint8"] }, { "Name": "Signature", "Docs": "", "Typewords": ["string"] }, { "Name": "Quoting", "Docs": "", "Typewords": ["Quoting"] }, { "Name": "ShowAddressSecurity", "Docs": "", "Typewords": ["bool"] }, { "Name": "ShowHTML", "Docs": "", "Typewords": ["bool"] }, { "Name": "NoShowShortcuts", "Docs": "", "Typewords": ["bool"] }, { "Name": "ShowHeaders", "Docs": "", "Typewords": ["[]", "string"] }] }, "Ruleset": { "Name": "Ruleset", "Docs": "", "Fields": [{ "Name": "SMTPMailFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "HeadersRegexp", "Docs": "", "Typewords": ["{}", "string"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ListAllowDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "AcceptRejectsToMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Comment", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDNSDomain", "Docs": "", "Typewords": ["Domain"] }, { "Name": "ListAllowDNSDomain", "Docs": "", "Typewords": ["Domain"] }] }, @@ -325,7 +325,7 @@ var api; "EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] }, "ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] }, "Flags": { "Name": "Flags", "Docs": "", "Fields": [{ "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }] }, - "ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] }, + "ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "MsgIDs", "Docs": "", "Typewords": ["[]", "int64"] }] }, "ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] }, "ChangeMsgThread": { "Name": "ChangeMsgThread", "Docs": "", "Fields": [{ "Name": "MessageIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "Muted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Collapsed", "Docs": "", "Typewords": ["bool"] }] }, "ChangeMailboxRemove": { "Name": "ChangeMailboxRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] }, @@ -335,8 +335,8 @@ var api; "ChangeMailboxSpecialUse": { "Name": "ChangeMailboxSpecialUse", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "SpecialUse", "Docs": "", "Typewords": ["SpecialUse"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] }, "SpecialUse": { "Name": "SpecialUse", "Docs": "", "Fields": [{ "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }] }, "ChangeMailboxKeywords": { "Name": "ChangeMailboxKeywords", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] }, - "UID": { "Name": "UID", "Docs": "", "Values": null }, "ModSeq": { "Name": "ModSeq", "Docs": "", "Values": null }, + "UID": { "Name": "UID", "Docs": "", "Values": null }, "Validation": { "Name": "Validation", "Docs": "", "Values": [{ "Name": "ValidationUnknown", "Value": 0, "Docs": "" }, { "Name": "ValidationStrict", "Value": 1, "Docs": "" }, { "Name": "ValidationDMARC", "Value": 2, "Docs": "" }, { "Name": "ValidationRelaxed", "Value": 3, "Docs": "" }, { "Name": "ValidationPass", "Value": 4, "Docs": "" }, { "Name": "ValidationNeutral", "Value": 5, "Docs": "" }, { "Name": "ValidationTemperror", "Value": 6, "Docs": "" }, { "Name": "ValidationPermerror", "Value": 7, "Docs": "" }, { "Name": "ValidationFail", "Value": 8, "Docs": "" }, { "Name": "ValidationSoftfail", "Value": 9, "Docs": "" }, { "Name": "ValidationNone", "Value": 10, "Docs": "" }] }, "CSRFToken": { "Name": "CSRFToken", "Docs": "", "Values": null }, "ThreadMode": { "Name": "ThreadMode", "Docs": "", "Values": [{ "Name": "ThreadOff", "Value": "off", "Docs": "" }, { "Name": "ThreadOn", "Value": "on", "Docs": "" }, { "Name": "ThreadUnread", "Value": "unread", "Docs": "" }] }, @@ -389,8 +389,8 @@ var api; ChangeMailboxSpecialUse: (v) => api.parse("ChangeMailboxSpecialUse", v), SpecialUse: (v) => api.parse("SpecialUse", v), ChangeMailboxKeywords: (v) => api.parse("ChangeMailboxKeywords", v), - UID: (v) => api.parse("UID", v), ModSeq: (v) => api.parse("ModSeq", v), + UID: (v) => api.parse("UID", v), Validation: (v) => api.parse("Validation", v), CSRFToken: (v) => api.parse("CSRFToken", v), ThreadMode: (v) => api.parse("ThreadMode", v), diff --git a/webmail/text.js b/webmail/text.js index eb5d587..69fb530 100644 --- a/webmail/text.js +++ b/webmail/text.js @@ -309,7 +309,7 @@ var api; "SubmitMessage": { "Name": "SubmitMessage", "Docs": "", "Fields": [{ "Name": "From", "Docs": "", "Typewords": ["string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Cc", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Bcc", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "TextBody", "Docs": "", "Typewords": ["string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "File"] }, { "Name": "ForwardAttachments", "Docs": "", "Typewords": ["ForwardAttachments"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ResponseMessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UserAgent", "Docs": "", "Typewords": ["string"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "FutureRelease", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "ArchiveThread", "Docs": "", "Typewords": ["bool"] }, { "Name": "ArchiveReferenceMailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "DraftMessageID", "Docs": "", "Typewords": ["int64"] }] }, "File": { "Name": "File", "Docs": "", "Fields": [{ "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "DataURI", "Docs": "", "Typewords": ["string"] }] }, "ForwardAttachments": { "Name": "ForwardAttachments", "Docs": "", "Fields": [{ "Name": "MessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Paths", "Docs": "", "Typewords": ["[]", "[]", "int32"] }] }, - "Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] }, + "Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "ParentID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] }, "RecipientSecurity": { "Name": "RecipientSecurity", "Docs": "", "Fields": [{ "Name": "STARTTLS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "MTASTS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DNSSEC", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DANE", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["SecurityResult"] }] }, "Settings": { "Name": "Settings", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["uint8"] }, { "Name": "Signature", "Docs": "", "Typewords": ["string"] }, { "Name": "Quoting", "Docs": "", "Typewords": ["Quoting"] }, { "Name": "ShowAddressSecurity", "Docs": "", "Typewords": ["bool"] }, { "Name": "ShowHTML", "Docs": "", "Typewords": ["bool"] }, { "Name": "NoShowShortcuts", "Docs": "", "Typewords": ["bool"] }, { "Name": "ShowHeaders", "Docs": "", "Typewords": ["[]", "string"] }] }, "Ruleset": { "Name": "Ruleset", "Docs": "", "Fields": [{ "Name": "SMTPMailFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "HeadersRegexp", "Docs": "", "Typewords": ["{}", "string"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ListAllowDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "AcceptRejectsToMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Comment", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDNSDomain", "Docs": "", "Typewords": ["Domain"] }, { "Name": "ListAllowDNSDomain", "Docs": "", "Typewords": ["Domain"] }] }, @@ -325,7 +325,7 @@ var api; "EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] }, "ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] }, "Flags": { "Name": "Flags", "Docs": "", "Fields": [{ "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }] }, - "ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] }, + "ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "MsgIDs", "Docs": "", "Typewords": ["[]", "int64"] }] }, "ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] }, "ChangeMsgThread": { "Name": "ChangeMsgThread", "Docs": "", "Fields": [{ "Name": "MessageIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "Muted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Collapsed", "Docs": "", "Typewords": ["bool"] }] }, "ChangeMailboxRemove": { "Name": "ChangeMailboxRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] }, @@ -335,8 +335,8 @@ var api; "ChangeMailboxSpecialUse": { "Name": "ChangeMailboxSpecialUse", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "SpecialUse", "Docs": "", "Typewords": ["SpecialUse"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] }, "SpecialUse": { "Name": "SpecialUse", "Docs": "", "Fields": [{ "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }] }, "ChangeMailboxKeywords": { "Name": "ChangeMailboxKeywords", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] }, - "UID": { "Name": "UID", "Docs": "", "Values": null }, "ModSeq": { "Name": "ModSeq", "Docs": "", "Values": null }, + "UID": { "Name": "UID", "Docs": "", "Values": null }, "Validation": { "Name": "Validation", "Docs": "", "Values": [{ "Name": "ValidationUnknown", "Value": 0, "Docs": "" }, { "Name": "ValidationStrict", "Value": 1, "Docs": "" }, { "Name": "ValidationDMARC", "Value": 2, "Docs": "" }, { "Name": "ValidationRelaxed", "Value": 3, "Docs": "" }, { "Name": "ValidationPass", "Value": 4, "Docs": "" }, { "Name": "ValidationNeutral", "Value": 5, "Docs": "" }, { "Name": "ValidationTemperror", "Value": 6, "Docs": "" }, { "Name": "ValidationPermerror", "Value": 7, "Docs": "" }, { "Name": "ValidationFail", "Value": 8, "Docs": "" }, { "Name": "ValidationSoftfail", "Value": 9, "Docs": "" }, { "Name": "ValidationNone", "Value": 10, "Docs": "" }] }, "CSRFToken": { "Name": "CSRFToken", "Docs": "", "Values": null }, "ThreadMode": { "Name": "ThreadMode", "Docs": "", "Values": [{ "Name": "ThreadOff", "Value": "off", "Docs": "" }, { "Name": "ThreadOn", "Value": "on", "Docs": "" }, { "Name": "ThreadUnread", "Value": "unread", "Docs": "" }] }, @@ -389,8 +389,8 @@ var api; ChangeMailboxSpecialUse: (v) => api.parse("ChangeMailboxSpecialUse", v), SpecialUse: (v) => api.parse("SpecialUse", v), ChangeMailboxKeywords: (v) => api.parse("ChangeMailboxKeywords", v), - UID: (v) => api.parse("UID", v), ModSeq: (v) => api.parse("ModSeq", v), + UID: (v) => api.parse("UID", v), Validation: (v) => api.parse("Validation", v), CSRFToken: (v) => api.parse("CSRFToken", v), ThreadMode: (v) => api.parse("ThreadMode", v), diff --git a/webmail/view.go b/webmail/view.go index 04ff598..cb89025 100644 --- a/webmail/view.go +++ b/webmail/view.go @@ -708,7 +708,7 @@ func serveEvents(ctx context.Context, log mlog.Log, accountPath string, w http.R qtx, err = acc.DB.Begin(reqctx, false) xcheckf(ctx, err, "begin transaction") - mbl, err = bstore.QueryTx[store.Mailbox](qtx).List() + mbl, err = bstore.QueryTx[store.Mailbox](qtx).FilterEqual("Expunged", false).List() xcheckf(ctx, err, "list mailboxes") err = qtx.Get(&settings) @@ -916,6 +916,8 @@ func serveEvents(ctx context.Context, log mlog.Log, accountPath string, w http.R } case store.ChangeRemoveUIDs: + comm.RemovalSeen(c) + // We may send changes for uids the client doesn't know, that's fine. changedUIDs := xchangedUIDs(c.MailboxID, c.UIDs, true) if len(changedUIDs) == 0 { @@ -1126,7 +1128,7 @@ func xprepareMailboxIDs(ctx context.Context, tx *bstore.Tx, f Filter, rejectsMai if f.MailboxID == -1 { matchMailboxes = false // Add the trash, junk and account rejects mailbox. - err := bstore.QueryTx[store.Mailbox](tx).ForEach(func(mb store.Mailbox) error { + err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb store.Mailbox) error { if mb.Trash || mb.Junk || mb.Name == rejectsMailbox { mailboxPrefixes = append(mailboxPrefixes, mb.Name+"/") mailboxIDs[mb.ID] = true @@ -1135,8 +1137,7 @@ func xprepareMailboxIDs(ctx context.Context, tx *bstore.Tx, f Filter, rejectsMai }) xcheckf(ctx, err, "finding trash/junk/rejects mailbox") } else if f.MailboxID > 0 { - mb := store.Mailbox{ID: f.MailboxID} - err := tx.Get(&mb) + mb, err := store.MailboxID(tx, f.MailboxID) xcheckf(ctx, err, "get mailbox") mailboxIDs[f.MailboxID] = true mailboxPrefixes = []string{mb.Name + "/"} @@ -1151,7 +1152,7 @@ func xgatherMailboxIDs(ctx context.Context, tx *bstore.Tx, mailboxIDs map[int64] if len(mailboxPrefixes) == 0 { return } - err := bstore.QueryTx[store.Mailbox](tx).ForEach(func(mb store.Mailbox) error { + err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb store.Mailbox) error { for _, p := range mailboxPrefixes { if strings.HasPrefix(mb.Name, p) { mailboxIDs[mb.ID] = true diff --git a/webmail/view_test.go b/webmail/view_test.go index a120ad5..345c2bc 100644 --- a/webmail/view_test.go +++ b/webmail/view_test.go @@ -45,7 +45,7 @@ func TestView(t *testing.T) { defer func() { err := acc.Close() pkglog.Check(err, "closing account") - acc.CheckClosed() + acc.WaitClosed() }() api := Webmail{maxMessageSize: 1024 * 1024, cookiePath: "/"} diff --git a/webmail/webmail.go b/webmail/webmail.go index 45fca7a..71f70ee 100644 --- a/webmail/webmail.go +++ b/webmail/webmail.go @@ -407,6 +407,8 @@ func handle(apiHandler http.Handler, isForwarded bool, accountPath string, w htt err = acc.DB.Read(ctx, func(tx *bstore.Tx) error { if err := tx.Get(&m); err != nil { return err + } else if m.Expunged { + return fmt.Errorf("message was removed") } s := store.Settings{ID: 1} if err := tx.Get(&s); err != nil { diff --git a/webmail/webmail.js b/webmail/webmail.js index 1b86140..0110bac 100644 --- a/webmail/webmail.js +++ b/webmail/webmail.js @@ -309,7 +309,7 @@ var api; "SubmitMessage": { "Name": "SubmitMessage", "Docs": "", "Fields": [{ "Name": "From", "Docs": "", "Typewords": ["string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Cc", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Bcc", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "TextBody", "Docs": "", "Typewords": ["string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "File"] }, { "Name": "ForwardAttachments", "Docs": "", "Typewords": ["ForwardAttachments"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ResponseMessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UserAgent", "Docs": "", "Typewords": ["string"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "FutureRelease", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "ArchiveThread", "Docs": "", "Typewords": ["bool"] }, { "Name": "ArchiveReferenceMailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "DraftMessageID", "Docs": "", "Typewords": ["int64"] }] }, "File": { "Name": "File", "Docs": "", "Fields": [{ "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "DataURI", "Docs": "", "Typewords": ["string"] }] }, "ForwardAttachments": { "Name": "ForwardAttachments", "Docs": "", "Fields": [{ "Name": "MessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Paths", "Docs": "", "Typewords": ["[]", "[]", "int32"] }] }, - "Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] }, + "Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "ParentID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] }, "RecipientSecurity": { "Name": "RecipientSecurity", "Docs": "", "Fields": [{ "Name": "STARTTLS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "MTASTS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DNSSEC", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DANE", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["SecurityResult"] }] }, "Settings": { "Name": "Settings", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["uint8"] }, { "Name": "Signature", "Docs": "", "Typewords": ["string"] }, { "Name": "Quoting", "Docs": "", "Typewords": ["Quoting"] }, { "Name": "ShowAddressSecurity", "Docs": "", "Typewords": ["bool"] }, { "Name": "ShowHTML", "Docs": "", "Typewords": ["bool"] }, { "Name": "NoShowShortcuts", "Docs": "", "Typewords": ["bool"] }, { "Name": "ShowHeaders", "Docs": "", "Typewords": ["[]", "string"] }] }, "Ruleset": { "Name": "Ruleset", "Docs": "", "Fields": [{ "Name": "SMTPMailFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "HeadersRegexp", "Docs": "", "Typewords": ["{}", "string"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ListAllowDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "AcceptRejectsToMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Comment", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDNSDomain", "Docs": "", "Typewords": ["Domain"] }, { "Name": "ListAllowDNSDomain", "Docs": "", "Typewords": ["Domain"] }] }, @@ -325,7 +325,7 @@ var api; "EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] }, "ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] }, "Flags": { "Name": "Flags", "Docs": "", "Fields": [{ "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }] }, - "ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] }, + "ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "MsgIDs", "Docs": "", "Typewords": ["[]", "int64"] }] }, "ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] }, "ChangeMsgThread": { "Name": "ChangeMsgThread", "Docs": "", "Fields": [{ "Name": "MessageIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "Muted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Collapsed", "Docs": "", "Typewords": ["bool"] }] }, "ChangeMailboxRemove": { "Name": "ChangeMailboxRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] }, @@ -335,8 +335,8 @@ var api; "ChangeMailboxSpecialUse": { "Name": "ChangeMailboxSpecialUse", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "SpecialUse", "Docs": "", "Typewords": ["SpecialUse"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] }, "SpecialUse": { "Name": "SpecialUse", "Docs": "", "Fields": [{ "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }] }, "ChangeMailboxKeywords": { "Name": "ChangeMailboxKeywords", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] }, - "UID": { "Name": "UID", "Docs": "", "Values": null }, "ModSeq": { "Name": "ModSeq", "Docs": "", "Values": null }, + "UID": { "Name": "UID", "Docs": "", "Values": null }, "Validation": { "Name": "Validation", "Docs": "", "Values": [{ "Name": "ValidationUnknown", "Value": 0, "Docs": "" }, { "Name": "ValidationStrict", "Value": 1, "Docs": "" }, { "Name": "ValidationDMARC", "Value": 2, "Docs": "" }, { "Name": "ValidationRelaxed", "Value": 3, "Docs": "" }, { "Name": "ValidationPass", "Value": 4, "Docs": "" }, { "Name": "ValidationNeutral", "Value": 5, "Docs": "" }, { "Name": "ValidationTemperror", "Value": 6, "Docs": "" }, { "Name": "ValidationPermerror", "Value": 7, "Docs": "" }, { "Name": "ValidationFail", "Value": 8, "Docs": "" }, { "Name": "ValidationSoftfail", "Value": 9, "Docs": "" }, { "Name": "ValidationNone", "Value": 10, "Docs": "" }] }, "CSRFToken": { "Name": "CSRFToken", "Docs": "", "Values": null }, "ThreadMode": { "Name": "ThreadMode", "Docs": "", "Values": [{ "Name": "ThreadOff", "Value": "off", "Docs": "" }, { "Name": "ThreadOn", "Value": "on", "Docs": "" }, { "Name": "ThreadUnread", "Value": "unread", "Docs": "" }] }, @@ -389,8 +389,8 @@ var api; ChangeMailboxSpecialUse: (v) => api.parse("ChangeMailboxSpecialUse", v), SpecialUse: (v) => api.parse("SpecialUse", v), ChangeMailboxKeywords: (v) => api.parse("ChangeMailboxKeywords", v), - UID: (v) => api.parse("UID", v), ModSeq: (v) => api.parse("ModSeq", v), + UID: (v) => api.parse("UID", v), Validation: (v) => api.parse("Validation", v), CSRFToken: (v) => api.parse("CSRFToken", v), ThreadMode: (v) => api.parse("ThreadMode", v), diff --git a/webops/xops.go b/webops/xops.go index 9610c66..e83fe4a 100644 --- a/webops/xops.go +++ b/webops/xops.go @@ -6,16 +6,18 @@ import ( "errors" "fmt" "io" + "log/slog" "os" + "slices" "sort" "time" - "golang.org/x/exp/maps" - "github.com/mjl-/bstore" + "github.com/mjl-/mox/junk" "github.com/mjl-/mox/message" "github.com/mjl-/mox/mlog" + "github.com/mjl-/mox/moxio" "github.com/mjl-/mox/store" ) @@ -31,9 +33,8 @@ func (x XOps) mailboxID(ctx context.Context, tx *bstore.Tx, mailboxID int64) sto if mailboxID == 0 { x.Checkuserf(ctx, errors.New("invalid zero mailbox ID"), "getting mailbox") } - mb := store.Mailbox{ID: mailboxID} - err := tx.Get(&mb) - if err == bstore.ErrAbsent { + mb, err := store.MailboxID(tx, mailboxID) + if err == bstore.ErrAbsent || err == store.ErrMailboxExpunged { x.Checkuserf(ctx, err, "getting mailbox") } x.Checkf(ctx, err, "getting mailbox") @@ -67,25 +68,32 @@ func (x XOps) MessageDelete(ctx context.Context, log mlog.Log, acc *store.Accoun store.BroadcastChanges(acc, changes) }) - - for _, mID := range messageIDs { - p := acc.MessagePath(mID) - err := os.Remove(p) - log.Check(err, "removing message file for expunge") - } } func (x XOps) MessageDeleteTx(ctx context.Context, log mlog.Log, tx *bstore.Tx, acc *store.Account, messageIDs []int64, modseq *store.ModSeq) []store.Change { - removeChanges := map[int64]store.ChangeRemoveUIDs{} - changes := make([]store.Change, 0, len(messageIDs)+1) // n remove, 1 mailbox counts + changes := make([]store.Change, 0, 1+1) // 1 remove, 1 mailbox counts, optimistic that all messages are in 1 mailbox. + + var jf *junk.Filter + defer func() { + if jf != nil { + err := jf.CloseDiscard() + log.Check(err, "close junk filter") + } + }() + + conf, _ := acc.Conf() var mb store.Mailbox - remove := make([]store.Message, 0, len(messageIDs)) + var changeRemoveUIDs store.ChangeRemoveUIDs + xflushMailbox := func() { + err := tx.Update(&mb) + x.Checkf(ctx, err, "updating mailbox counts") + slices.Sort(changeRemoveUIDs.UIDs) + changes = append(changes, mb.ChangeCounts(), changeRemoveUIDs) + } - var totalSize int64 - for _, mid := range messageIDs { - m := x.messageID(ctx, tx, mid) - totalSize += m.Size + for _, id := range messageIDs { + m := x.messageID(ctx, tx, id) if *modseq == 0 { var err error @@ -95,58 +103,33 @@ func (x XOps) MessageDeleteTx(ctx context.Context, log mlog.Log, tx *bstore.Tx, if m.MailboxID != mb.ID { if mb.ID != 0 { - mb.ModSeq = *modseq - err := tx.Update(&mb) - x.Checkf(ctx, err, "updating mailbox counts") - changes = append(changes, mb.ChangeCounts()) + xflushMailbox() } mb = x.mailboxID(ctx, tx, m.MailboxID) + mb.ModSeq = *modseq + changeRemoveUIDs = store.ChangeRemoveUIDs{MailboxID: mb.ID, ModSeq: *modseq} } - qmr := bstore.QueryTx[store.Recipient](tx) - qmr.FilterEqual("MessageID", m.ID) - _, err := qmr.Delete() - x.Checkf(ctx, err, "removing message recipients") + if m.Junk != m.Notjunk && jf == nil && conf.JunkFilter != nil { + var err error + jf, _, err = acc.OpenJunkFilter(ctx, log) + x.Checkf(ctx, err, "open junk filter") + } - mb.Sub(m.MailboxCounts()) + opts := store.RemoveOpts{JunkFilter: jf} + _, _, err := acc.MessageRemove(log, tx, *modseq, &mb, opts, m) + x.Checkf(ctx, err, "expunge message") - m.Expunged = true - m.ModSeq = *modseq - err = tx.Update(&m) - x.Checkf(ctx, err, "marking message as expunged") - - ch := removeChanges[m.MailboxID] - ch.UIDs = append(ch.UIDs, m.UID) - ch.MailboxID = m.MailboxID - ch.ModSeq = *modseq - removeChanges[m.MailboxID] = ch - remove = append(remove, m) + changeRemoveUIDs.UIDs = append(changeRemoveUIDs.UIDs, m.UID) + changeRemoveUIDs.MsgIDs = append(changeRemoveUIDs.MsgIDs, m.ID) } - if mb.ID != 0 { - mb.ModSeq = *modseq - err := tx.Update(&mb) - x.Checkf(ctx, err, "updating count in mailbox") - changes = append(changes, mb.ChangeCounts()) - } + xflushMailbox() - err := acc.AddMessageSize(log, tx, -totalSize) - x.Checkf(ctx, err, "updating disk usage") - - // Mark removed messages as not needing training, then retrain them, so if they - // were trained, they get untrained. - for i := range remove { - remove[i].Junk = false - remove[i].Notjunk = false - } - err = acc.RetrainMessages(ctx, log, tx, remove) - x.Checkf(ctx, err, "untraining deleted messages") - - for _, ch := range removeChanges { - sort.Slice(ch.UIDs, func(i, j int) bool { - return ch.UIDs[i] < ch.UIDs[j] - }) - changes = append(changes, ch) + if jf != nil { + err := jf.Close() + jf = nil + x.Checkf(ctx, err, "close junk filter") } return changes @@ -361,6 +344,15 @@ func (x XOps) MessageMove(ctx context.Context, log mlog.Log, acc *store.Account, acc.WithWLock(func() { var changes []store.Change + var newIDs []int64 + defer func() { + for _, id := range newIDs { + p := acc.MessagePath(id) + err := os.Remove(p) + log.Check(err, "removing delivered message after failure", slog.String("path", p)) + } + }() + x.DBWrite(ctx, acc, func(tx *bstore.Tx) { if mailboxName != "" { mb, err := acc.MailboxFind(tx, mailboxName) @@ -379,23 +371,37 @@ func (x XOps) MessageMove(ctx context.Context, log mlog.Log, acc *store.Account, } var modseq store.ModSeq - changes = x.MessageMoveTx(ctx, log, acc, tx, messageIDs, mbDst, &modseq) + newIDs, changes = x.MessageMoveTx(ctx, log, acc, tx, messageIDs, mbDst, &modseq) }) + newIDs = nil store.BroadcastChanges(acc, changes) }) } -func (x XOps) MessageMoveTx(ctx context.Context, log mlog.Log, acc *store.Account, tx *bstore.Tx, messageIDs []int64, mbDst store.Mailbox, modseq *store.ModSeq) []store.Change { - retrain := make([]store.Message, 0, len(messageIDs)) - removeChanges := map[int64]store.ChangeRemoveUIDs{} - // n adds, 1 remove, 2 mailboxcounts, optimistic and at least for a single message. - changes := make([]store.Change, 0, len(messageIDs)+3) +// MessageMoveTx moves message to a new mailbox, which must be different than their +// current mailbox. Moving a message is done by changing the MailboxID and +// assigning an appriorate new UID, and then inserting a replacement Message record +// with new ID that is marked expunged in the original mailbox, along with a +// MessageErase record so the message gets erased when all sessions stopped +// referencing the message. +func (x XOps) MessageMoveTx(ctx context.Context, log mlog.Log, acc *store.Account, tx *bstore.Tx, messageIDs []int64, mbDst store.Mailbox, modseq *store.ModSeq) ([]int64, []store.Change) { + var newIDs []int64 + var commit bool + defer func() { + if commit { + return + } + for _, id := range newIDs { + p := acc.MessagePath(id) + err := os.Remove(p) + log.Check(err, "removing delivered message after failure", slog.String("path", p)) + } + newIDs = nil + }() - var mbSrc store.Mailbox - - keywords := map[string]struct{}{} - now := time.Now() + // n adds, 1 remove, 2 mailboxcounts, 1 mailboxkeywords, optimistic that messages are in a single source mailbox. + changes := make([]store.Change, 0, len(messageIDs)+4) var err error if *modseq == 0 { @@ -403,104 +409,135 @@ func (x XOps) MessageMoveTx(ctx context.Context, log mlog.Log, acc *store.Accoun x.Checkf(ctx, err, "assigning next modseq") } - for _, mid := range messageIDs { - m := x.messageID(ctx, tx, mid) + mbDst.ModSeq = *modseq - // We may have loaded this mailbox in the previous iteration of this loop. - if m.MailboxID != mbSrc.ID { - if mbSrc.ID != 0 { - mbSrc.ModSeq = *modseq - err := tx.Update(&mbSrc) - x.Checkf(ctx, err, "updating source mailbox counts") - changes = append(changes, mbSrc.ChangeCounts()) - } - mbSrc = x.mailboxID(ctx, tx, m.MailboxID) - } - - if mbSrc.ID == mbDst.ID { + // Get messages. group them by mailbox. + l := make([]store.Message, len(messageIDs)) + for i, id := range messageIDs { + l[i] = x.messageID(ctx, tx, id) + if l[i].MailboxID == mbDst.ID { // Client should filter out messages that are already in mailbox. - x.Checkuserf(ctx, errors.New("already in destination mailbox"), "moving message") - } - - ch := removeChanges[m.MailboxID] - ch.UIDs = append(ch.UIDs, m.UID) - ch.ModSeq = *modseq - ch.MailboxID = m.MailboxID - removeChanges[m.MailboxID] = ch - - // Copy of message record that we'll insert when UID is freed up. - om := m - om.PrepareExpunge() - om.ID = 0 // Assign new ID. - om.ModSeq = *modseq - - mbSrc.Sub(m.MailboxCounts()) - - if mbDst.Trash { - m.Seen = true - } - conf, _ := acc.Conf() - m.MailboxID = mbDst.ID - if m.IsReject && m.MailboxDestinedID != 0 { - // Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message - // is used for reputation calculation during future deliveries. - m.MailboxOrigID = m.MailboxDestinedID - m.IsReject = false - m.Seen = false - } - m.UID = mbDst.UIDNext - m.ModSeq = *modseq - mbDst.UIDNext++ - m.JunkFlagsForMailbox(mbDst, conf) - m.SaveDate = &now - err = tx.Update(&m) - x.Checkf(ctx, err, "updating moved message in database") - - // Now that UID is unused, we can insert the old record again. - err = tx.Insert(&om) - x.Checkf(ctx, err, "inserting record for expunge after moving message") - - mbDst.Add(m.MailboxCounts()) - - changes = append(changes, m.ChangeAddUID()) - retrain = append(retrain, m) - - for _, kw := range m.Keywords { - keywords[kw] = struct{}{} + x.Checkuserf(ctx, fmt.Errorf("message %d already in destination mailbox", l[i].ID), "moving message") } } - mbSrc.ModSeq = *modseq - err = tx.Update(&mbSrc) - x.Checkf(ctx, err, "updating source mailbox counts and modseq") + // Sort (group) by mailbox, sort by UID. + sort.Slice(l, func(i, j int) bool { + if l[i].MailboxID != l[j].MailboxID { + return l[i].MailboxID < l[j].MailboxID + } + return l[i].UID < l[j].UID + }) - changes = append(changes, mbSrc.ChangeCounts(), mbDst.ChangeCounts()) + var jf *junk.Filter + defer func() { + if jf != nil { + err := jf.CloseDiscard() + log.Check(err, "close junk filter") + } + }() - // Ensure destination mailbox has keywords of the moved messages. - var mbKwChanged bool - mbDst.Keywords, mbKwChanged = store.MergeKeywords(mbDst.Keywords, maps.Keys(keywords)) - if mbKwChanged { + accConf, _ := acc.Conf() + + var mbSrc store.Mailbox + var changeRemoveUIDs store.ChangeRemoveUIDs + xflushMailbox := func() { + changes = append(changes, changeRemoveUIDs, mbSrc.ChangeCounts()) + + err = tx.Update(&mbSrc) + x.Checkf(ctx, err, "updating source mailbox counts") + } + + nkeywords := len(mbDst.Keywords) + now := time.Now() + + for _, om := range l { + if om.MailboxID != mbSrc.ID { + if mbSrc.ID != 0 { + xflushMailbox() + } + mbSrc = x.mailboxID(ctx, tx, om.MailboxID) + mbSrc.ModSeq = *modseq + changeRemoveUIDs = store.ChangeRemoveUIDs{MailboxID: mbSrc.ID, ModSeq: *modseq} + } + + nm := om + nm.MailboxID = mbDst.ID + nm.UID = mbDst.UIDNext + mbDst.UIDNext++ + nm.ModSeq = *modseq + nm.CreateSeq = *modseq + nm.SaveDate = &now + if nm.IsReject && nm.MailboxDestinedID != 0 { + // Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message + // is used for reputation calculation during future deliveries. + nm.MailboxOrigID = nm.MailboxDestinedID + nm.IsReject = false + nm.Seen = false + } + if mbDst.Trash { + nm.Seen = true + } + + nm.JunkFlagsForMailbox(mbDst, accConf) + + err := tx.Update(&nm) + x.Checkf(ctx, err, "updating message with new mailbox") + + mbDst.Add(nm.MailboxCounts()) + + mbSrc.Sub(om.MailboxCounts()) + om.ID = 0 + om.Expunged = true + om.ModSeq = *modseq + om.TrainedJunk = nil + err = tx.Insert(&om) + x.Checkf(ctx, err, "inserting expunged message in old mailbox") + + err = moxio.LinkOrCopy(log, acc.MessagePath(om.ID), acc.MessagePath(nm.ID), nil, false) + x.Checkf(ctx, err, "duplicating message in old mailbox for current sessions") + newIDs = append(newIDs, nm.ID) + // We don't sync the directory. In case of a crash and files disappearing, the + // eraser will simply not find the file at next startup. + + err = tx.Insert(&store.MessageErase{ID: om.ID, SkipUpdateDiskUsage: true}) + x.Checkf(ctx, err, "insert message erase") + + mbDst.Keywords, _ = store.MergeKeywords(mbDst.Keywords, nm.Keywords) + + if accConf.JunkFilter != nil && nm.NeedsTraining() { + // Lazily open junk filter. + if jf == nil { + jf, _, err = acc.OpenJunkFilter(ctx, log) + x.Checkf(ctx, err, "open junk filter") + } + err := acc.RetrainMessage(ctx, log, tx, jf, &nm) + x.Checkf(ctx, err, "retrain message after moving") + } + + changeRemoveUIDs.UIDs = append(changeRemoveUIDs.UIDs, om.UID) + changeRemoveUIDs.MsgIDs = append(changeRemoveUIDs.MsgIDs, om.ID) + changes = append(changes, nm.ChangeAddUID()) + } + + xflushMailbox() + + changes = append(changes, mbDst.ChangeCounts()) + if nkeywords > len(mbDst.Keywords) { changes = append(changes, mbDst.ChangeKeywords()) } - mbDst.ModSeq = *modseq err = tx.Update(&mbDst) x.Checkf(ctx, err, "updating destination mailbox with uidnext and modseq") - err = acc.RetrainMessages(ctx, log, tx, retrain) - x.Checkf(ctx, err, "retraining messages after move") - - // Ensure UIDs of the removed message are in increasing order. It is quite common - // for all messages to be from a single source mailbox, meaning this is just one - // change, for which we preallocated space. - for _, ch := range removeChanges { - sort.Slice(ch.UIDs, func(i, j int) bool { - return ch.UIDs[i] < ch.UIDs[j] - }) - changes = append(changes, ch) + if jf != nil { + err := jf.Close() + x.Checkf(ctx, err, "saving junk filter") + jf = nil } - return changes + commit = true + return newIDs, changes } func isText(p message.Part) bool {