infrastructure/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2025-06-28 01:48:15 +03:00
Code Issues Packages Projects Releases Wiki Activity

Improve expunged message/UID tracking in IMAP sessions, track synchronization history for mailboxes/annotations.

Browse Source 
Keeping the message files around, and the message details in the database, is
useful for IMAP sessions that haven't seen/processed the removal of a message
yet and try to fetch it. Before, we would return errors. Similarly, a session
that has a mailbox selected that is removed can (at least in theory) still read
messages.

The mechanics to do this need keeping removed mailboxes around too. JMAP needs
that anyway, so we now keep modseq/createseq/expunged history for mailboxes
too. And while we're at it, for annotations as well.

For future JMAP support, we now also keep the mailbox parent id around for a
mailbox, with an upgrade step to set the field for existing mailboxes and
fixing up potential missing parents (which could possibly have happened in an
obscure corner case that I doubt anyone ran into).
This commit is contained in:
Mechiel Lukkien 2025-03-05 17:17:57 +01:00
parent 684c716e4d
commit 577944310c
No known key found for this signature in database
63 changed files with 1945 additions and 1249 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
backup.go
View File

@ -532,8 +532,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
}
}()
// Link/copy known message files. If a message has been removed while we read the
// database, our backup is not consistent and the backup will be marked failed.
// Link/copy known message files.
tmMsgs := time.Now()
seen := map[string]struct{}{}
var maxID int64
@ -565,6 +564,15 @@ func backupctl(ctx context.Context, ctl *ctl) {
slog.Duration("duration", time.Since(tmMsgs)))
}
eraseIDs := map[int64]struct{}{}
err = bstore.QueryDB[store.MessageErase](ctx, db).ForEach(func(me store.MessageErase) error {
eraseIDs[me.ID] = struct{}{}
return nil
})
if err != nil {
xerrx("listing erased messages", err)
}
// Read through all files in queue directory and warn about anything we haven't
// handled yet. Message files that are newer than we expect from our consistent
// database snapshot are ignored.
@ -586,9 +594,13 @@ func backupctl(ctx context.Context, ctl *ctl) {
return nil
}
// Skip any messages that were added since we started on our consistent snapshot.
// We don't want to cause spurious backup warnings.
if id, err := strconv.ParseInt(l[len(l)-1], 10, 64); err == nil && id > maxID && mp == store.MessagePath(id) {
// Skip any messages that were added since we started on our consistent snapshot,
// or messages that will be erased. We don't want to cause spurious backup
// warnings.
id, err := strconv.ParseInt(l[len(l)-1], 10, 64)
if err == nil && id > maxID && mp == store.MessagePath(id) {
return nil
} else if _, ok := eraseIDs[id]; err == nil && ok {
return nil
}
}

2
config/config.go
View File

@ -65,7 +65,7 @@ type Static struct {
ParsedLocalpart smtp.Localpart `sconf:"-"`
} `sconf:"optional" sconf-doc:"Destination for per-host TLS reports (TLSRPT). TLS reports can be per recipient domain (for MTA-STS), or per MX host (for DANE). The per-domain TLS reporting configuration is in domains.conf. This is the TLS reporting configuration for this host. If absent, no host-based TLSRPT address is configured, and no host TLSRPT DNS record is suggested."`
InitialMailboxes InitialMailboxes `sconf:"optional" sconf-doc:"Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be given a 'special-use' role, which are understood by most mail clients. If absent/empty, the following mailboxes are created: Sent, Archive, Trash, Drafts and Junk."`
InitialMailboxes InitialMailboxes `sconf:"optional" sconf-doc:"Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be given a 'special-use' role, which are understood by most mail clients. If absent/empty, the following additional mailboxes are created: Sent, Archive, Trash, Drafts and Junk."`
DefaultMailboxes []string `sconf:"optional" sconf-doc:"Deprecated in favor of InitialMailboxes. Mailboxes to create when adding an account. Inbox is always created. If no mailboxes are specified, the following are automatically created: Sent, Archive, Trash, Drafts and Junk."`
Transports map[string]Transport `sconf:"optional" sconf-doc:"Transport are mechanisms for delivering messages. Transports can be referenced from Routes in accounts, domains and the global configuration. There is always an implicit/fallback delivery transport doing direct delivery with SMTP from the outgoing message queue. Transports are typically only configured when using smarthosts, i.e. when delivering through another SMTP server. Zero or one transport methods must be set in a transport, never multiple. When using an external party to send email for a domain, keep in mind you may have to add their IP address to your domain's SPF record, and possibly additional DKIM records."`
// Awkward naming of fields to get intended default behaviour for zero values.

4
config/doc.go
View File

@ -543,8 +543,8 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
# Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be
# given a 'special-use' role, which are understood by most mail clients. If
# absent/empty, the following mailboxes are created: Sent, Archive, Trash, Drafts
# and Junk. (optional)
# absent/empty, the following additional mailboxes are created: Sent, Archive,
# Trash, Drafts and Junk. (optional)
InitialMailboxes:
# Special-use roles to mailbox to create. (optional)

27
ctl.go
View File

@ -1454,18 +1454,29 @@ func servectlcmd(ctx context.Context, ctl *ctl, cid int64, shutdown func()) {
log.Check(err, "closing junk filter during cleanup")
}()
// Read through messages with junk or nonjunk flag set, and train them.
// Read through messages with either junk or nonjunk flag set, and train them.
var total, trained int
q := bstore.QueryDB[store.Message](ctx, acc.DB)
err = acc.DB.Write(ctx, func(tx *bstore.Tx) error {
q := bstore.QueryTx[store.Message](tx)
q.FilterEqual("Expunged", false)
err = q.ForEach(func(m store.Message) error {
return q.ForEach(func(m store.Message) error {
total++
ok, err := acc.TrainMessage(ctx, log, jf, m)
if m.Junk == m.Notjunk {
return nil
}
ok, err := acc.TrainMessage(ctx, log, jf, m.Notjunk, m)
if ok {
trained++
}
if m.TrainedJunk == nil || *m.TrainedJunk != m.Junk {
m.TrainedJunk = &m.Junk
if err := tx.Update(&m); err != nil {
return fmt.Errorf("marking message as trained: %v", err)
}
}
return err
})
})
ctl.xcheck(err, "training messages")
log.Info("retrained messages", slog.Int("total", total), slog.Int("trained", trained))
@ -1509,7 +1520,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, cid int64, shutdown func()) {
var changes []store.Change
err = acc.DB.Write(ctx, func(tx *bstore.Tx) error {
var totalSize int64
err := bstore.QueryTx[store.Mailbox](tx).ForEach(func(mb store.Mailbox) error {
err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb store.Mailbox) error {
mc, err := mb.CalculateCounts(tx)
if err != nil {
return fmt.Errorf("calculating counts for mailbox %q: %w", mb.Name, err)
@ -1618,6 +1629,11 @@ func servectlcmd(ctx context.Context, ctl *ctl, cid int64, shutdown func()) {
if err := tx.Get(&mb); err != nil {
_, werr := fmt.Fprintf(w, "get mailbox id %d for message with file size mismatch: %v\n", mb.ID, err)
ctl.xcheck(werr, "write")
return nil
}
if mb.Expunged {
_, err := fmt.Fprintf(w, "message %d is in expunged mailbox %q (id %d) (continuing)\n", m.ID, mb.Name, mb.ID)
ctl.xcheck(err, "write")
}
_, err = fmt.Fprintf(w, "fixing message %d in mailbox %q (id %d) with incorrect size %d, should be %d (len msg prefix %d + on-disk file %s size %d)\n", m.ID, mb.Name, mb.ID, m.Size, correctSize, len(m.MsgPrefix), p, filesize)
ctl.xcheck(err, "write")
@ -1650,7 +1666,6 @@ func servectlcmd(ctx context.Context, ctl *ctl, cid int64, shutdown func()) {
}
return nil
})
})
ctl.xcheck(err, "find and fix wrong message sizes")

2
ctl_test.go
View File

@ -436,7 +436,7 @@ func TestCtl(t *testing.T) {
tcheck(t, err, "open account")
defer func() {
acc.Close()
acc.CheckClosed()
acc.WaitClosed()
}()
content := []byte("Subject: hi\r\n\r\nbody\r\n")

11
imapserver/append_test.go
View File

@ -13,10 +13,10 @@ func TestAppend(t *testing.T) {
defer tc.close()
tc2 := startNoSwitchboard(t) // note: without switchboard because this connection will break during tests.
defer tc2.close()
defer tc2.closeNoWait()
tc3 := startNoSwitchboard(t)
defer tc3.close()
defer tc3.closeNoWait()
tc2.client.Login("mjl@mox.example", password0)
tc2.client.Select("inbox")
@ -31,19 +31,22 @@ func TestAppend(t *testing.T) {
// Syntax error for line ending in literal causes connection abort.
tc2.transactf("bad", "append inbox (\\Badflag) {1+}\r\nx") // Unknown flag.
tc2 = startNoSwitchboard(t)
defer tc2.close()
defer tc2.closeNoWait()
tc2.client.Login("mjl@mox.example", password0)
tc2.client.Select("inbox")
tc2.transactf("bad", "append inbox () \"bad time\" {1+}\r\nx") // Bad time.
tc2 = startNoSwitchboard(t)
defer tc2.close()
defer tc2.closeNoWait()
tc2.client.Login("mjl@mox.example", password0)
tc2.client.Select("inbox")
tc2.transactf("no", "append nobox (\\Seen) \" 1-Jan-2022 10:10:00 +0100\" {1}")
tc2.xcode("TRYCREATE")
tc2.transactf("no", "append expungebox (\\Seen) {1}")
tc2.xcode("TRYCREATE")
tc2.transactf("ok", "append inbox (\\Seen Label1 $label2) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc2.xuntagged(imapclient.UntaggedExists(1))
tc2.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UIDs: xparseUIDRange("1")})

36
imapserver/condstore_test.go
View File

@ -85,6 +85,8 @@ func testCondstoreQresync(t *testing.T, qresync bool) {
// unmodified. Those messages have modseq 0 in the database. We use append for
// convenience, then adjust the records in the database.
// We have a workaround below to prevent triggering the consistency checker.
tc.account.SetSkipMessageModSeqZeroCheck(true)
defer tc.account.SetSkipMessageModSeqZeroCheck(false)
tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
@ -100,13 +102,13 @@ func testCondstoreQresync(t *testing.T, qresync bool) {
// tc2 is a client without condstore, so no modseq responses.
tc2 := startNoSwitchboard(t)
defer tc2.close()
defer tc2.closeNoWait()
tc2.client.Login("mjl@mox.example", password0)
tc2.client.Select("inbox")
// tc3 is a client with condstore, so with modseq responses.
tc3 := startNoSwitchboard(t)
defer tc3.close()
defer tc3.closeNoWait()
tc3.client.Login("mjl@mox.example", password0)
tc3.client.Enable(capability)
tc3.client.Select("inbox")
@ -124,7 +126,7 @@ func testCondstoreQresync(t *testing.T, qresync bool) {
tc.transactf("ok", "Append otherbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc.xuntagged()
tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 2, UIDs: xparseUIDRange("1")})
tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 3, UIDs: xparseUIDRange("1")})
tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc.xuntagged(imapclient.UntaggedExists(5))
@ -353,11 +355,7 @@ func testCondstoreQresync(t *testing.T, qresync bool) {
xtc := startNoSwitchboard(t)
// We have modified modseq & createseq to 0 above for testing that case. Don't
// trigger the consistency checker.
store.CheckConsistencyOnClose = false
defer func() {
xtc.close()
store.CheckConsistencyOnClose = true
}()
defer xtc.closeNoWait()
xtc.client.Login("mjl@mox.example", password0)
fn(xtc)
tagcount++
@ -444,13 +442,13 @@ func testCondstoreQresync(t *testing.T, qresync bool) {
// and untagged fetch with modseq in destination mailbox.
// tc2o is a client without condstore, so no modseq responses.
tc2o := startNoSwitchboard(t)
defer tc2o.close()
defer tc2o.closeNoWait()
tc2o.client.Login("mjl@mox.example", password0)
tc2o.client.Select("otherbox")
// tc3o is a client with condstore, so with modseq responses.
tc3o := startNoSwitchboard(t)
defer tc3o.close()
defer tc3o.closeNoWait()
tc3o.client.Login("mjl@mox.example", password0)
tc3o.client.Enable(capability)
tc3o.client.Select("otherbox")
@ -484,14 +482,9 @@ func testCondstoreQresync(t *testing.T, qresync bool) {
imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), noflags, imapclient.FetchModSeq(clientModseq)}},
)
// Restore valid modseq/createseq for the consistency checker.
_, err = bstore.QueryDB[store.Message](ctxbg, tc.account.DB).FilterEqual("CreateSeq", int64(0)).UpdateNonzero(store.Message{CreateSeq: 2})
tcheck(t, err, "updating modseq/createseq to valid values")
_, err = bstore.QueryDB[store.Message](ctxbg, tc.account.DB).FilterEqual("ModSeq", int64(0)).UpdateNonzero(store.Message{ModSeq: 2})
tcheck(t, err, "updating modseq/createseq to valid values")
tc2o.close()
tc2o.closeNoWait()
tc2o = nil
tc3o.close()
tc3o.closeNoWait()
tc3o = nil
// Then we rename inbox, which is special because it moves messages away instead of
@ -533,10 +526,7 @@ func testQresync(t *testing.T, tc *testconn, clientModseq int64) {
xtc.client.Login("mjl@mox.example", password0)
xtc.transactf("ok", "Select inbox (Condstore)")
xtc.transactf("bad", "Uid Fetch 1:* (Flags) (Changedsince 1 Vanished)")
// Prevent triggering the consistency checker, we still have modseq/createseq at 0.
store.CheckConsistencyOnClose = false
xtc.close()
store.CheckConsistencyOnClose = true
xtc.closeNoWait()
xtc = nil
// Check that we get proper vanished responses.
@ -557,9 +547,7 @@ func testQresync(t *testing.T, tc *testconn, clientModseq int64) {
xtc.client.Login("mjl@mox.example", password0)
xtc.transactf("bad", "Select inbox (Qresync 1 0)")
// Prevent triggering the consistency checker, we still have modseq/createseq at 0.
store.CheckConsistencyOnClose = false
xtc.close()
store.CheckConsistencyOnClose = true
xtc.closeNoWait()
xtc = nil
tc.transactf("bad", "Select inbox (Qresync (0 1))") // Both args must be > 0.

4
imapserver/copy_test.go
View File

@ -12,7 +12,7 @@ func TestCopy(t *testing.T) {
defer tc.close()
tc2 := startNoSwitchboard(t)
defer tc2.close()
defer tc2.closeNoWait()
tc.client.Login("mjl@mox.example", password0)
tc.client.Select("inbox")
@ -34,6 +34,8 @@ func TestCopy(t *testing.T) {
tc.transactf("no", "copy 1 nonexistent")
tc.xcode("TRYCREATE")
tc.transactf("no", "copy 1 expungebox")
tc.xcode("TRYCREATE")
tc.transactf("no", "copy 1 inbox") // Cannot copy to same mailbox.

5
imapserver/create_test.go
View File

@ -11,7 +11,7 @@ func TestCreate(t *testing.T) {
defer tc.close()
tc2 := startNoSwitchboard(t)
defer tc2.close()
defer tc2.closeNoWait()
tc.client.Login("mjl@mox.example", password0)
tc2.client.Login("mjl@mox.example", password0)
@ -84,4 +84,7 @@ func TestCreate(t *testing.T) {
tc.transactf("ok", `create "&"`) // Interpreted as UTF-8, no UTF-7.
tc2.transactf("bad", `create "&"`) // Bad UTF-7.
tc2.transactf("ok", `create "&Jjo-"`) // ☺, valid UTF-7.
tc.transactf("ok", "create expungebox") // Existed in past.
tc.transactf("ok", "delete expungebox") // Gone again.
}

5
imapserver/delete_test.go
View File

@ -11,10 +11,10 @@ func TestDelete(t *testing.T) {
defer tc.close()
tc2 := startNoSwitchboard(t)
defer tc2.close()
defer tc2.closeNoWait()
tc3 := startNoSwitchboard(t)
defer tc3.close()
defer tc3.closeNoWait()
tc.client.Login("mjl@mox.example", password0)
tc2.client.Login("mjl@mox.example", password0)
@ -24,6 +24,7 @@ func TestDelete(t *testing.T) {
tc.transactf("no", "delete inbox") // Cannot delete inbox.
tc.transactf("no", "delete nonexistent") // Cannot delete mailbox that does not exist.
tc.transactf("no", `delete "nonexistent"`) // Again, with quoted string syntax.
tc.transactf("no", `delete "expungebox"`) // Already removed.
tc.client.Subscribe("x")
tc.transactf("no", "delete x") // Subscription does not mean there is a mailbox that can be deleted.

2
imapserver/expunge_test.go
View File

@ -12,7 +12,7 @@ func TestExpunge(t *testing.T) {
defer tc.close()
tc2 := startNoSwitchboard(t)
defer tc2.close()
defer tc2.closeNoWait()
tc.client.Login("mjl@mox.example", password0)
tc.client.Select("inbox")

32
imapserver/fetch.go
View File

@ -5,7 +5,6 @@ package imapserver
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"log/slog"
@ -28,9 +27,9 @@ type fetchCmd struct {
conn *conn
isUID bool // If this is a UID FETCH command.
rtx *bstore.Tx // Read-only transaction, kept open while processing all messages.
updateSeen []int64 // IDs of messages to mark as seen, after processing all messages.
updateSeen []store.UID // To mark as seen after processing all messages. UID instead of message ID since moved messages keep their ID and insert a new ID in the original mailbox.
hasChangedSince bool // Whether CHANGEDSINCE was set. Enables MODSEQ in response.
expungeIssued bool // Set if any message cannot be read. Can happen for expunged messages.
expungeIssued bool // Set if any message has been expunged. Can happen for expunged messages.
uid store.UID // UID currently processing.
markSeen bool
@ -271,15 +270,16 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
changes := make([]store.Change, 0, len(cmd.updateSeen)+1)
c.xdbwrite(func(wtx *bstore.Tx) {
mb := store.Mailbox{ID: c.mailboxID}
err = wtx.Get(&mb)
mb, err := store.MailboxID(wtx, c.mailboxID)
if err == store.ErrMailboxExpunged {
xusercodeErrorf("NONEXISTENT", "mailbox has been expunged")
}
xcheckf(err, "get mailbox for updating counts after marking as seen")
var modseq store.ModSeq
for _, id := range cmd.updateSeen {
m := store.Message{ID: id}
err := wtx.Get(&m)
for _, uid := range cmd.updateSeen {
m, err := bstore.QueryTx[store.Message](wtx).FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: uid}).Get()
xcheckf(err, "get message")
if m.Expunged {
// Message has been deleted in the mean time.
@ -322,7 +322,8 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
if cmd.expungeIssued {
// ../rfc/2180:343
c.writeresultf("%s NO [EXPUNGEISSUED] at least one message was expunged", tag)
// ../rfc/9051:5102
c.writeresultf("%s OK [EXPUNGEISSUED] at least one message was expunged", tag)
} else {
c.ok(tag, cmdstr)
}
@ -333,12 +334,16 @@ func (cmd *fetchCmd) xensureMessage() *store.Message {
return cmd.m
}
// We do not filter by Expunged, the message may have been deleted in other
// sessions, but not in ours.
q := bstore.QueryTx[store.Message](cmd.rtx)
q.FilterNonzero(store.Message{MailboxID: cmd.conn.mailboxID, UID: cmd.uid})
q.FilterEqual("Expunged", false)
m, err := q.Get()
cmd.xcheckf(err, "get message for uid %d", cmd.uid)
cmd.m = &m
if m.Expunged {
cmd.expungeIssued = true
}
return cmd.m
}
@ -382,10 +387,6 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
if !ok {
panic(x)
}
if errors.Is(err, bstore.ErrAbsent) {
cmd.expungeIssued = true
return
}
cmd.conn.log.Infox("processing fetch attribute", err, slog.Any("uid", cmd.uid))
xuserErrorf("processing fetch attribute: %v", err)
}()
@ -401,8 +402,7 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
}
if cmd.markSeen {
m := cmd.xensureMessage()
cmd.updateSeen = append(cmd.updateSeen, m.ID)
cmd.updateSeen = append(cmd.updateSeen, cmd.uid)
}
if cmd.needFlags {

10
imapserver/fetch_test.go
View File

@ -444,6 +444,16 @@ Content-Transfer-Encoding: Quoted-printable
tc.client.Unselect()
tc.client.Examine("inbox")
// Start a second session. Use it to remove the message. First session should still
// be able to access the messages.
tc2 := startNoSwitchboard(t)
defer tc2.closeNoWait()
tc2.client.Login("mjl@mox.example", password0)
tc2.client.Select("inbox")
tc2.client.StoreFlagsSet("1", true, `\Deleted`)
tc2.client.Expunge()
tc2.client.Logout()
tc.transactf("ok", "fetch 1 binary[]")
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binary1}})

2
imapserver/fuzz_test.go
View File

@ -70,7 +70,7 @@ func FuzzServer(f *testing.F) {
}
defer func() {
acc.Close()
acc.CheckClosed()
acc.WaitClosed()
}()
err = acc.SetPassword(log, password0)
if err != nil {

2
imapserver/idle_test.go
View File

@ -13,7 +13,7 @@ func TestIdle(t *testing.T) {
defer tc1.close()
tc2 := startNoSwitchboard(t)
defer tc2.close()
defer tc2.closeNoWait()
tc1.client.Login("mjl@mox.example", password0)
tc2.client.Login("mjl@mox.example", password0)

13
imapserver/list.go
View File

@ -3,11 +3,12 @@ package imapserver
import (
"bytes"
"fmt"
"path"
"sort"
"strings"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/store"
)
@ -145,10 +146,11 @@ func (c *conn) cmdList(tag, cmd string, p *parser) {
var nameList []string
q := bstore.QueryTx[store.Mailbox](tx)
q.FilterEqual("Expunged", false)
err := q.ForEach(func(mb store.Mailbox) error {
names[mb.Name] = info{mailbox: &mb}
nameList = append(nameList, mb.Name)
for p := path.Dir(mb.Name); p != "."; p = path.Dir(p) {
for p := mox.ParentMailboxName(mb.Name); p != ""; p = mox.ParentMailboxName(p) {
hasChild[p] = true
}
return nil
@ -163,7 +165,7 @@ func (c *conn) cmdList(tag, cmd string, p *parser) {
if !ok {
nameList = append(nameList, sub.Name)
}
for p := path.Dir(sub.Name); p != "."; p = path.Dir(p) {
for p := mox.ParentMailboxName(sub.Name); p != ""; p = mox.ParentMailboxName(p) {
hasSubscribedChild[p] = true
}
return nil
@ -234,7 +236,10 @@ func (c *conn) cmdList(tag, cmd string, p *parser) {
if info.mailbox != nil && len(retMetadata) > 0 {
var meta listspace
for _, k := range retMetadata {
a, err := bstore.QueryTx[store.Annotation](tx).FilterNonzero(store.Annotation{MailboxID: info.mailbox.ID, Key: k}).Get()
q := bstore.QueryTx[store.Annotation](tx)
q.FilterNonzero(store.Annotation{MailboxID: info.mailbox.ID, Key: k})
q.FilterEqual("Expunged", false)
a, err := q.Get()
var v token
if err == bstore.ErrAbsent {
v = nilt

5
imapserver/list_test.go
View File

@ -26,6 +26,9 @@ func TestListBasic(t *testing.T) {
tc.last(tc.client.List("Inbox"))
tc.xuntagged(ulist("Inbox"))
tc.last(tc.client.List("expungebox"))
tc.xuntagged()
tc.last(tc.client.List("%"))
tc.xuntagged(ulist("Archive", `\Archive`), ulist("Drafts", `\Drafts`), ulist("Inbox"), ulist("Junk", `\Junk`), ulist("Sent", `\Sent`), ulist("Trash", `\Trash`))
@ -78,7 +81,7 @@ func TestListExtended(t *testing.T) {
for _, name := range store.DefaultInitialMailboxes.Regular {
uidvals[name] = 1
}
var uidvalnext uint32 = 2
var uidvalnext uint32 = 3
uidval := func(name string) uint32 {
v, ok := uidvals[name]
if !ok {

3
imapserver/lsub_test.go
View File

@ -19,6 +19,9 @@ func TestLsub(t *testing.T) {
tc.transactf("ok", `lsub "" x*`)
tc.xuntagged()
tc.transactf("ok", `lsub "" expungebox`)
tc.xuntagged(imapclient.UntaggedLsub{Separator: '/', Mailbox: "expungebox"})
tc.transactf("ok", "create a/b/c")
tc.transactf("ok", `lsub "" a/*`)
tc.xuntagged(imapclient.UntaggedLsub{Separator: '/', Mailbox: "a/b"}, imapclient.UntaggedLsub{Separator: '/', Mailbox: "a/b/c"})

40
imapserver/metadata.go
View File

@ -100,7 +100,7 @@ func (c *conn) cmdGetmetadata(tag, cmd string, p *parser) {
mb := c.xmailbox(tx, mailboxName, "TRYCREATE")
q.FilterNonzero(store.Annotation{MailboxID: mb.ID})
}
q.FilterEqual("Expunged", false)
q.SortAsc("MailboxID", "Key") // For tests.
err := q.ForEach(func(a store.Annotation) error {
// ../rfc/5464:516
@ -246,41 +246,35 @@ func (c *conn) cmdSetmetadata(tag, cmd string, p *parser) {
q := bstore.QueryTx[store.Annotation](tx)
q.FilterNonzero(store.Annotation{Key: a.Key})
q.FilterEqual("MailboxID", mb.ID) // Can be zero.
q.FilterEqual("Expunged", false)
oa, err := q.Get()
// Nil means remove. ../rfc/5464:579
if a.Value == nil {
var deleted []store.Annotation
q.Gather(&deleted)
_, err := q.Delete()
xcheckf(err, "deleting annotation")
for _, oa := range deleted {
changes = append(changes, oa.Change(mailboxName))
}
if err == bstore.ErrAbsent && a.Value == nil {
continue
}
if modseq == 0 {
var err error
modseq, err = c.account.NextModSeq(tx)
xcheckf(err, "get next modseq")
}
a.MailboxID = mb.ID
a.ModSeq = modseq
a.CreateSeq = modseq
oa, err := q.Get()
if err == bstore.ErrAbsent {
a.MailboxID = mb.ID
a.CreateSeq = modseq
a.ModSeq = modseq
err = tx.Insert(&a)
xcheckf(err, "inserting annotation")
changes = append(changes, a.Change(mailboxName))
continue
} else {
xcheckf(err, "get metadata")
oa.ModSeq = modseq
if a.Value == nil {
oa.Expunged = true
}
xcheckf(err, "looking up existing annotation for entry name")
changes = append(changes, a.Change(mailboxName))
oa.IsString = a.IsString
oa.Value = a.Value
err = tx.Update(&oa)
xcheckf(err, "updating metadata annotation")
xcheckf(err, "updating metdata")
}
changes = append(changes, a.Change(mailboxName))
}
c.xcheckMetadataSize(tx)
@ -304,7 +298,7 @@ func (c *conn) xcheckMetadataSize(tx *bstore.Tx) {
// ../rfc/5464:383
var n int
var size int
err := bstore.QueryTx[store.Annotation](tx).ForEach(func(a store.Annotation) error {
err := bstore.QueryTx[store.Annotation](tx).FilterEqual("Expunged", false).ForEach(func(a store.Annotation) error {
n++
if n > metadataMaxKeys {
// ../rfc/5464:590

11
imapserver/metadata_test.go
View File

@ -23,6 +23,15 @@ func TestMetadata(t *testing.T) {
tc.transactf("ok", `setmetadata "" (/PRIVATE/COMMENT "global value")`)
tc.transactf("ok", `setmetadata inbox (/private/comment "mailbox value")`)
tc.transactf("ok", `create metabox`)
tc.transactf("ok", `setmetadata metabox (/private/comment "mailbox value")`)
tc.transactf("ok", `setmetadata metabox (/shared/comment "mailbox value")`)
tc.transactf("ok", `setmetadata metabox (/shared/comment nil)`) // Remove.
tc.transactf("ok", `delete metabox`) // Delete mailbox with live and expunged metadata.
tc.transactf("no", `setmetadata expungebox (/private/comment "mailbox value")`)
tc.xcode("TRYCREATE")
tc.transactf("ok", `getmetadata "" ("/private/comment")`)
tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
Mailbox: "",
@ -176,7 +185,7 @@ func TestMetadata(t *testing.T) {
// Broadcast should not happen when metadata capability is not enabled.
tc2 := startNoSwitchboard(t)
defer tc2.close()
defer tc2.closeNoWait()
tc2.client.Login("mjl@mox.example", password0)
tc2.client.Select("inbox")

7
imapserver/move_test.go
View File

@ -12,10 +12,10 @@ func TestMove(t *testing.T) {
defer tc.close()
tc2 := startNoSwitchboard(t)
defer tc2.close()
defer tc2.closeNoWait()
tc3 := startNoSwitchboard(t)
defer tc3.close()
defer tc3.closeNoWait()
tc.client.Login("mjl@mox.example", password0)
tc.client.Select("inbox")
@ -47,6 +47,9 @@ func TestMove(t *testing.T) {
tc.transactf("no", "move 1 nonexistent")
tc.xcode("TRYCREATE")
tc.transactf("no", "move 1 expungebox")
tc.xcode("TRYCREATE")
tc.transactf("no", "move 1 inbox") // Cannot move to same mailbox.
tc2.transactf("ok", "noop") // Drain.

43
imapserver/rename_test.go
View File

@ -12,7 +12,7 @@ func TestRename(t *testing.T) {
defer tc.close()
tc2 := startNoSwitchboard(t)
defer tc2.close()
defer tc2.closeNoWait()
tc.client.Login("mjl@mox.example", password0)
tc2.client.Login("mjl@mox.example", password0)
@ -23,6 +23,8 @@ func TestRename(t *testing.T) {
tc.transactf("no", "rename doesnotexist newbox") // Does not exist.
tc.xcode("NONEXISTENT") // ../rfc/9051:5140
tc.transactf("no", "rename expungebox newbox") // No longer exists.
tc.xcode("NONEXISTENT")
tc.transactf("no", `rename "Sent" "Trash"`) // Already exists.
tc.xcode("ALREADYEXISTS")
@ -70,28 +72,51 @@ func TestRename(t *testing.T) {
tc.client.Unsubscribe("k")
tc.transactf("ok", "rename k/l k/l/m") // With "l" renamed, a new "k" will be created.
tc.transactf("ok", `list "" "k*" return (subscribed)`)
tc.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "k"}, imapclient.UntaggedList{Flags: []string{"\\Subscribed"}, Separator: '/', Mailbox: "k/l"}, imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/m"})
tc.xuntagged(
imapclient.UntaggedList{Separator: '/', Mailbox: "k"},
imapclient.UntaggedList{Flags: []string{"\\Subscribed"}, Separator: '/', Mailbox: "k/l"},
imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/m"},
)
tc.transactf("ok", "rename k/l/m k/l/x/y/m") // k/l/x and k/l/x/y will be created.
tc.transactf("ok", `list "" "k/l/x*" return (subscribed)`)
tc.xuntagged(
imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/x"},
imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/x/y"},
imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/x/y/m"},
)
// Renaming inbox keeps inbox in existence, moves messages, and does not rename children.
tc.transactf("ok", "create inbox/a")
// To check if UIDs are renumbered properly, we add UIDs 1 and 2. Expunge 1,
// keeping only 2. Then rename the inbox, which should renumber UID 2 in the old
// inbox to UID 1 in the newly created mailbox.
tc.transactf("ok", "append inbox (\\deleted) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc.transactf("ok", "append inbox (label1) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc.transactf("ok", "append inbox (\\deleted) {1+}\r\nx")
tc.transactf("ok", "append inbox (label1) {1+}\r\nx")
tc.transactf("ok", `select inbox`)
tc.transactf("ok", "expunge")
tc.transactf("ok", "rename inbox minbox")
tc.transactf("ok", `list "" (inbox inbox/a minbox)`)
tc.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"}, imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox/a"}, imapclient.UntaggedList{Separator: '/', Mailbox: "minbox"})
tc.transactf("ok", `select minbox`)
tc.transactf("ok", "rename inbox x/minbox")
tc.transactf("ok", `list "" (inbox inbox/a x/minbox)`)
tc.xuntagged(
imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"},
imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox/a"},
imapclient.UntaggedList{Separator: '/', Mailbox: "x/minbox"},
)
tc.transactf("ok", `select x/minbox`)
tc.transactf("ok", `uid fetch 1:* flags`)
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}}})
// Renaming to new hiearchy that does not have any subscribes.
tc.transactf("ok", "rename minbox w/w")
tc.transactf("ok", "rename x/minbox w/w")
tc.transactf("ok", `list "" "w*"`)
tc.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "w"}, imapclient.UntaggedList{Separator: '/', Mailbox: "w/w"})
tc.transactf("ok", "rename inbox misc/old/inbox")
tc.transactf("ok", `list "" (misc misc/old/inbox)`)
tc.xuntagged(
imapclient.UntaggedList{Separator: '/', Mailbox: "misc"},
imapclient.UntaggedList{Separator: '/', Mailbox: "misc/old/inbox"},
)
// todo: test create+delete+rename of/to a name results in a higher uidvalidity.
}

63
imapserver/replace.go
View File

@ -1,7 +1,6 @@
package imapserver
import (
"context"
"errors"
"io"
"os"
@ -116,7 +115,7 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) {
q := bstore.QueryTx[store.Message](tx)
q.FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: uidOld})
q.FilterEqual("Expunged", false)
om, err := q.Get()
_, err = q.Get()
if err == bstore.ErrAbsent {
return nil
}
@ -124,8 +123,8 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) {
return func() { xserverErrorf("get message to replace: %v", err) }
}
delta := size - om.Size
ok, maxSize, err := c.account.CanAddMessageSize(tx, delta)
// Check if we can add size bytes. We can't necessarily remove the current message yet.
ok, maxSize, err := c.account.CanAddMessageSize(tx, size)
if err != nil {
return func() { xserverErrorf("check quota: %v", err) }
}
@ -169,9 +168,7 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) {
var file *os.File
var newMsgPath string
var f io.Writer
var committed bool
var oldMsgPath string // To remove on success.
var commit bool
if errfn != nil {
// We got a non-sync literal, we will consume some data, but abort if there's too
@ -197,14 +194,10 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) {
err := file.Close()
c.xsanity(err, "close temporary file for replace")
}
if newMsgPath != "" && !committed {
if newMsgPath != "" && !commit {
err := os.Remove(newMsgPath)
c.xsanity(err, "remove temporary file for replace")
}
if committed {
err := os.Remove(oldMsgPath)
c.xsanity(err, "remove old message")
}
}()
}
@ -258,8 +251,8 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) {
return
}
// Check quota. Even if the delta is negative, the quota may have changed.
ok, maxSize, err := c.account.CanAddMessageSize(tx, mw.Size-om.Size)
// Check quota for addition of new message. We can't necessarily yet remove the old message.
ok, maxSize, err := c.account.CanAddMessageSize(tx, mw.Size)
xcheckf(err, "checking quota")
if !ok {
// ../rfc/9208:472
@ -269,31 +262,11 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) {
modseq, err := c.account.NextModSeq(tx)
xcheckf(err, "get next mod seq")
// Subtract counts for message from source mailbox.
mbSrc.Sub(om.MailboxCounts())
chremuids, _, err := c.account.MessageRemove(c.log, tx, modseq, &mbSrc, store.RemoveOpts{}, om)
xcheckf(err, "expunge old message")
changes = append(changes, chremuids)
// Note: we only add a mbSrc counts change later on, if it is not equal to mbDst.
// Remove message recipients for old message.
_, err = bstore.QueryTx[store.Recipient](tx).FilterNonzero(store.Recipient{MessageID: om.ID}).Delete()
xcheckf(err, "removing message recipients")
// Subtract size of old message from account.
err = c.account.AddMessageSize(c.log, tx, -om.Size)
xcheckf(err, "updating disk usage")
// Undo any junk filter training for the old message.
om.Junk = false
om.Notjunk = false
err = c.account.RetrainMessages(context.TODO(), c.log, tx, []store.Message{om})
xcheckf(err, "untraining expunged messages")
// Mark old message expunged.
om.ModSeq = modseq
om.PrepareExpunge()
err = tx.Update(&om)
xcheckf(err, "mark old message as expunged")
// Update source mailbox.
mbSrc.ModSeq = modseq
err = tx.Update(&mbSrc)
xcheckf(err, "updating source mailbox counts")
@ -316,22 +289,16 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) {
err = c.account.MessageAdd(c.log, tx, &mbDst, &nm, file, store.AddOpts{})
xcheckf(err, "delivering message")
// Update path to what is stored in the account. We may still have to clean it up on errors.
newMsgPath = c.account.MessagePath(nm.ID)
changes = append(changes,
store.ChangeRemoveUIDs{MailboxID: om.MailboxID, UIDs: []store.UID{om.UID}, ModSeq: om.ModSeq},
nm.ChangeAddUID(),
mbDst.ChangeCounts(),
)
changes = append(changes, nm.ChangeAddUID(), mbDst.ChangeCounts())
if nkeywords != len(mbDst.Keywords) {
changes = append(changes, mbDst.ChangeKeywords())
}
err = tx.Update(&mbDst)
xcheckf(err, "updating destination mailbox")
// Update path to what is stored in the account. We may still have to clean it up on errors.
newMsgPath = c.account.MessagePath(nm.ID)
oldMsgPath = c.account.MessagePath(om.ID)
})
// Fetch pending changes, possibly with new UIDs, so we can apply them before adding our own new UID.
@ -342,7 +309,7 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) {
}
// Success, make sure messages aren't cleaned up anymore.
committed = true
commit = true
// Broadcast the change to other connections.
if mbSrc.ID != mbDst.ID {

11
imapserver/replace_test.go
View File

@ -13,7 +13,7 @@ func TestReplace(t *testing.T) {
defer tc.close()
tc2 := startNoSwitchboard(t)
defer tc2.close()
defer tc2.closeNoWait()
tc.client.Login("mjl@mox.example", password0)
tc.client.Select("inbox")
@ -23,6 +23,9 @@ func TestReplace(t *testing.T) {
tc.client.StoreFlagsSet("1", true, `\deleted`)
tc.client.Expunge()
tc.transactf("no", "replace 2 expungebox {1}") // Mailbox no longer exists.
tc.xcode("TRYCREATE")
tc2.client.Login("mjl@mox.example", password0)
tc2.client.Select("inbox")
@ -34,7 +37,7 @@ func TestReplace(t *testing.T) {
imapclient.UntaggedExists(3),
imapclient.UntaggedExpunge(2),
)
tc.xcodeArg(imapclient.CodeHighestModSeq(6))
tc.xcodeArg(imapclient.CodeHighestModSeq(8))
// Check that other client sees Exists and Expunge.
tc2.transactf("ok", "noop")
@ -53,7 +56,7 @@ func TestReplace(t *testing.T) {
imapclient.UntaggedExists(3),
imapclient.UntaggedVanished{UIDs: xparseNumSet("2")},
)
tc.xcodeArg(imapclient.CodeHighestModSeq(7))
tc.xcodeArg(imapclient.CodeHighestModSeq(9))
// Leftover data.
tc.transactf("bad", "replace 1 inbox () {6+}\r\ntest\r\n ")
@ -125,7 +128,7 @@ func TestReplaceExpunged(t *testing.T) {
// Get in with second client and remove the message we are replacing.
tc2 := startNoSwitchboard(t)
defer tc2.close()
defer tc2.closeNoWait()
tc2.client.Login("mjl@mox.example", password0)
tc2.client.Select("inbox")
tc2.client.StoreFlagsSet("1", true, `\Deleted`)

1
imapserver/search_test.go
View File

@ -406,7 +406,6 @@ func TestSearch(t *testing.T) {
writeTextLit(1, true)
}
writeTextLit(1, false)
}
// esearchall makes an UntaggedEsearch response with All set, for comparisons.

1
imapserver/selectexamine_test.go
View File

@ -49,6 +49,7 @@ func testSelectExamine(t *testing.T, examine bool) {
// Mailbox does not exist.
tc.transactf("no", "%s bogus", cmd)
tc.transactf("no", "%s expungebox", cmd)
tc.transactf("ok", "%s inbox", cmd)
tc.xuntagged(uflags, upermflags, urecent, uexists0, uuidval1, uuidnext1, ulist)

501
imapserver/server.go
View File

@ -26,7 +26,6 @@ non-ASCII UTF-8. Until that's enabled, we do use UTF-7 for mailbox names. See
/*
- todo: do not return binary data for a fetch body. at least not for imap4rev1. we should be encoding it as base64?
- todo: on expunge we currently remove the message even if other sessions still have a reference to the uid. if they try to query the uid, they'll get an error. we could be nicer and only actually remove the message when the last reference has gone. we could add a new flag to store.Message marking the message as expunged, not give new session access to such messages, and make store remove them at startup, and clean them when the last session referencing the session goes. however, it will get much more complicated. renaming messages would need special handling. and should we do the same for removed mailboxes?
- todo: try to recover from syntax errors when the last command line ends with a }, i.e. a literal. we currently abort the entire connection. we may want to read some amount of literal data and continue with a next command.
*/
@ -69,6 +68,7 @@ import (
"github.com/mjl-/flate"
"github.com/mjl-/mox/config"
"github.com/mjl-/mox/junk"
"github.com/mjl-/mox/message"
"github.com/mjl-/mox/metrics"
"github.com/mjl-/mox/mlog"
@ -1557,10 +1557,12 @@ func (c *conn) xmailbox(tx *bstore.Tx, name string, missingErrCode string) store
// If the mailbox does not exist, panic is called with a user error.
// Must be called with account rlock held.
func (c *conn) xmailboxID(tx *bstore.Tx, id int64) store.Mailbox {
mb := store.Mailbox{ID: id}
err := tx.Get(&mb)
mb, err := store.MailboxID(tx, id)
if err == bstore.ErrAbsent {
xuserErrorf("%w", store.ErrUnknownMailbox)
} else if err == store.ErrMailboxExpunged {
// ../rfc/9051:5140
xusercodeErrorf("NONEXISTENT", "mailbox has been deleted")
}
return mb
}
@ -1589,6 +1591,7 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) {
mbID = ch.MailboxID
case store.ChangeRemoveUIDs:
mbID = ch.MailboxID
c.comm.RemovalSeen(ch)
case store.ChangeFlags:
mbID = ch.MailboxID
case store.ChangeRemoveMailbox, store.ChangeAddMailbox, store.ChangeRenameMailbox, store.ChangeAddSubscription:
@ -2881,9 +2884,6 @@ func (c *conn) cmdDelete(tag, cmd string, p *parser) {
name = xcheckmailboxname(name, false)
// Messages to remove after having broadcasted the removal of messages.
var removeMessageIDs []int64
c.account.WithWLock(func() {
var mb store.Mailbox
var changes []store.Change
@ -2893,7 +2893,7 @@ func (c *conn) cmdDelete(tag, cmd string, p *parser) {
var hasChildren bool
var err error
changes, removeMessageIDs, hasChildren, err = c.account.MailboxDelete(context.TODO(), c.log, tx, mb)
changes, hasChildren, err = c.account.MailboxDelete(context.TODO(), c.log, tx, &mb)
if hasChildren {
xusercodeErrorf("HASCHILDREN", "mailbox has a child, only leaf mailboxes can be deleted")
}
@ -2903,12 +2903,6 @@ func (c *conn) cmdDelete(tag, cmd string, p *parser) {
c.broadcast(changes)
})
for _, mID := range removeMessageIDs {
p := c.account.MessagePath(mID)
err := os.Remove(p)
c.xsanity(err, "removing message file %q for mailbox delete", p)
}
c.ok(tag, cmd)
}
@ -2934,14 +2928,21 @@ func (c *conn) cmdRename(tag, cmd string, p *parser) {
src = xcheckmailboxname(src, true)
dst = xcheckmailboxname(dst, false)
var cleanupIDs []int64
defer func() {
for _, id := range cleanupIDs {
p := c.account.MessagePath(id)
err := os.Remove(p)
c.xsanity(err, "cleaning up message")
}
}()
c.account.WithWLock(func() {
var changes []store.Change
c.xdbwrite(func(tx *bstore.Tx) {
srcMB := c.xmailbox(tx, src, "NONEXISTENT")
var modseq store.ModSeq
// Inbox is very special. Unlike other mailboxes, its children are not moved. And
// unlike a regular move, its messages are moved to a newly created mailbox. We do
// indeed create a new destination mailbox and actually move the messages.
@ -2956,111 +2957,53 @@ func (c *conn) cmdRename(tag, cmd string, p *parser) {
xuserErrorf("cannot move inbox to itself")
}
uidval, err := c.account.NextUIDValidity(tx)
xcheckf(err, "next uid validity")
var modseq store.ModSeq
dstMB, chl, err := c.account.MailboxEnsure(tx, dst, false, store.SpecialUse{}, &modseq)
xcheckf(err, "creating destination mailbox")
changes = chl
modseq, err = c.account.NextModSeq(tx)
xcheckf(err, "assigning next modseq")
dstMB := store.Mailbox{
Name: dst,
UIDValidity: uidval,
UIDNext: 1,
Keywords: srcMB.Keywords,
ModSeq: modseq,
CreateSeq: modseq,
HaveCounts: true,
// Copy mailbox annotations. ../rfc/5464:368
qa := bstore.QueryTx[store.Annotation](tx)
qa.FilterNonzero(store.Annotation{MailboxID: srcMB.ID})
qa.FilterEqual("Expunged", false)
annotations, err := qa.List()
xcheckf(err, "get annotations to copy for inbox")
for _, a := range annotations {
a.ID = 0
a.MailboxID = dstMB.ID
a.ModSeq = modseq
a.CreateSeq = modseq
err := tx.Insert(&a)
xcheckf(err, "copy annotation to destination mailbox")
changes = append(changes, a.Change(dstMB.Name))
}
err = tx.Insert(&dstMB)
xcheckf(err, "create new destination mailbox")
c.xcheckMetadataSize(tx)
changes = make([]store.Change, 2) // Placeholders filled in below.
// Move existing messages, with their ID's and on-disk files intact, to the new
// mailbox. We keep the expunged messages, the destination mailbox doesn't care
// about them.
var oldUIDs []store.UID
// Build query that selects messages to move.
q := bstore.QueryTx[store.Message](tx)
q.FilterNonzero(store.Message{MailboxID: srcMB.ID})
q.FilterEqual("Expunged", false)
q.SortAsc("UID")
err = q.ForEach(func(m store.Message) error {
om := m
om.ID = 0
om.ModSeq = modseq
om.PrepareExpunge()
oldUIDs = append(oldUIDs, om.UID)
mc := m.MailboxCounts()
srcMB.Sub(mc)
dstMB.Add(mc)
m.MailboxID = dstMB.ID
m.UID = dstMB.UIDNext
dstMB.UIDNext++
m.CreateSeq = modseq
m.ModSeq = modseq
if err := tx.Update(&m); err != nil {
return fmt.Errorf("updating message to move to new mailbox: %w", err)
}
changes = append(changes, m.ChangeAddUID())
if err := tx.Insert(&om); err != nil {
return fmt.Errorf("adding empty expunge message record to inbox: %w", err)
}
return nil
})
xcheckf(err, "moving messages from inbox to destination mailbox")
err = tx.Update(&dstMB)
xcheckf(err, "updating uidnext and counts in destination mailbox")
srcMB.ModSeq = modseq
err = tx.Update(&srcMB)
xcheckf(err, "updating counts for inbox")
var dstFlags []string
if tx.Get(&store.Subscription{Name: dstMB.Name}) == nil {
dstFlags = []string{`\Subscribed`}
}
// Copy any annotations. ../rfc/5464:368
annotations, err := bstore.QueryTx[store.Annotation](tx).FilterNonzero(store.Annotation{MailboxID: srcMB.ID}).List()
xcheckf(err, "get annotations to copy for inbox")
for i := range annotations {
annotations[i].ID = 0
annotations[i].MailboxID = dstMB.ID
annotations[i].ModSeq = modseq
annotations[i].CreateSeq = modseq
err := tx.Insert(&annotations[i])
xcheckf(err, "copy annotation to destination mailbox")
}
c.xcheckMetadataSize(tx)
changes[0] = store.ChangeRemoveUIDs{MailboxID: srcMB.ID, UIDs: oldUIDs, ModSeq: modseq}
changes[1] = store.ChangeAddMailbox{Mailbox: dstMB, Flags: dstFlags, ModSeq: modseq}
// changes[2:...] are ChangeAddUIDs
changes = append(changes, srcMB.ChangeCounts(), dstMB.ChangeCounts())
for _, a := range annotations {
changes = append(changes, a.Change(dstMB.Name))
}
newIDs, chl := c.xmoveMessages(tx, q, 0, modseq, &srcMB, &dstMB)
changes = append(changes, chl...)
cleanupIDs = newIDs
return
}
var notExists, alreadyExists bool
var modseq store.ModSeq
var alreadyExists bool
var err error
changes, _, notExists, alreadyExists, err = c.account.MailboxRename(tx, srcMB, dst, &modseq)
if notExists {
// ../rfc/9051:5140
xusercodeErrorf("NONEXISTENT", "%s", err)
} else if alreadyExists {
changes, _, alreadyExists, err = c.account.MailboxRename(tx, &srcMB, dst, &modseq)
if alreadyExists {
xusercodeErrorf("ALREADYEXISTS", "%s", err)
}
xcheckf(err, "renaming mailbox")
})
cleanupIDs = nil
c.broadcast(changes)
})
@ -3163,7 +3106,7 @@ func (c *conn) cmdLsub(tag, cmd string, p *parser) {
for _, sub := range subscriptions {
name := sub.Name
if ispercent {
for p := path.Dir(name); p != "."; p = path.Dir(p) {
for p := mox.ParentMailboxName(name); p != ""; p = mox.ParentMailboxName(p) {
subscribedKids[p] = true
}
}
@ -3181,6 +3124,7 @@ func (c *conn) cmdLsub(tag, cmd string, p *parser) {
return
}
qmb := bstore.QueryTx[store.Mailbox](tx)
qmb.FilterEqual("Expunged", false)
qmb.SortAsc("Name")
err = qmb.ForEach(func(mb store.Mailbox) error {
if have[mb.Name] || !subscribedKids[mb.Name] || !re.MatchString(mb.Name) {
@ -3544,12 +3488,11 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) {
// todo: do a single junk training
err = c.account.MessageAdd(c.log, tx, &mb, &a.m, a.file, store.AddOpts{SkipDirSync: true})
xcheckf(err, "delivering message")
changes = append(changes, a.m.ChangeAddUID())
// Update path to what is stored in the account. We may still have to clean it up on errors.
a.path = c.account.MessagePath(a.m.ID)
changes = append(changes, a.m.ChangeAddUID())
msgDirs[filepath.Dir(a.path)] = struct{}{}
}
@ -3759,29 +3702,29 @@ func (c *conn) cmdClose(tag, cmd string, p *parser) {
}
// expunge messages marked for deletion in currently selected/active mailbox.
// if uidSet is not nil, only messages matching the set are deleted.
// if uidSet is not nil, only messages matching the set are expunged.
//
// messages that have been marked expunged from the database are returned and
// have already been removed.
// Messages that have been marked expunged from the database are returned. While
// other sessions still reference the message, it is not cleared from the database
// yet, and the message file is not yet removed.
//
// the highest modseq in the mailbox is returned, typically associated with the
// The highest modseq in the mailbox is returned, typically associated with the
// removal of the messages, but if no messages were expunged the current latest max
// modseq for the mailbox is returned.
func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) (removed []store.Message, highestModSeq store.ModSeq) {
var modseq store.ModSeq
func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) (expunged []store.Message, highestModSeq store.ModSeq) {
c.account.WithWLock(func() {
var mb store.Mailbox
var changes []store.Change
c.xdbwrite(func(tx *bstore.Tx) {
mb = store.Mailbox{ID: c.mailboxID}
err := tx.Get(&mb)
if err == bstore.ErrAbsent {
mb, err := store.MailboxID(tx, c.mailboxID)
if err == bstore.ErrAbsent || err == store.ErrMailboxExpunged {
if missingMailboxOK {
return
}
xuserErrorf("%w", store.ErrUnknownMailbox)
// ../rfc/9051:5140
xusercodeErrorf("NONEXISTENT", "%w", store.ErrUnknownMailbox)
}
xcheckf(err, "get mailbox")
qm := bstore.QueryTx[store.Message](tx)
qm.FilterNonzero(store.Message{MailboxID: c.mailboxID})
@ -3792,82 +3735,32 @@ func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) (removed []store.
return uidSearch(c.uids, m.UID) > 0 && (uidSet == nil || uidSet.containsUID(m.UID, c.uids, c.searchResult))
})
qm.SortAsc("UID")
removed, err = qm.List()
xcheckf(err, "listing messages to delete")
expunged, err = qm.List()
xcheckf(err, "listing messages to expunge")
if len(removed) == 0 {
if len(expunged) == 0 {
highestModSeq = mb.ModSeq
return
}
// Assign new modseq.
modseq, err = c.account.NextModSeq(tx)
modseq, err := c.account.NextModSeq(tx)
xcheckf(err, "assigning next modseq")
highestModSeq = modseq
mb.ModSeq = modseq
removeIDs := make([]int64, len(removed))
anyIDs := make([]any, len(removed))
var totalSize int64
for i, m := range removed {
removeIDs[i] = m.ID
anyIDs[i] = m.ID
mb.Sub(m.MailboxCounts())
totalSize += m.Size
// Update "remove", because RetrainMessage below will save the message.
removed[i].Expunged = true
removed[i].ModSeq = modseq
}
qmr := bstore.QueryTx[store.Recipient](tx)
qmr.FilterEqual("MessageID", anyIDs...)
_, err = qmr.Delete()
xcheckf(err, "removing message recipients")
qm = bstore.QueryTx[store.Message](tx)
qm.FilterIDs(removeIDs)
n, err := qm.UpdateNonzero(store.Message{Expunged: true, ModSeq: modseq})
if err == nil && n != len(removeIDs) {
err = fmt.Errorf("only %d messages set to expunged, expected %d", n, len(removeIDs))
}
xcheckf(err, "marking messages marked for deleted as expunged")
chremuids, chmbcounts, err := c.account.MessageRemove(c.log, tx, modseq, &mb, store.RemoveOpts{}, expunged...)
xcheckf(err, "expunging messages")
changes = append(changes, chremuids, chmbcounts)
err = tx.Update(&mb)
xcheckf(err, "updating mailbox counts")
err = c.account.AddMessageSize(c.log, tx, -totalSize)
xcheckf(err, "updating disk usage")
// Mark expunged messages as not needing training, then retrain them, so if they
// were trained, they get untrained.
for i := range removed {
removed[i].Junk = false
removed[i].Notjunk = false
}
err = c.account.RetrainMessages(context.TODO(), c.log, tx, removed)
xcheckf(err, "untraining expunged messages")
xcheckf(err, "update mailbox")
})
// Broadcast changes to other connections. We may not have actually removed any
// messages, so take care not to send an empty update.
if len(removed) > 0 {
ouids := make([]store.UID, len(removed))
for i, m := range removed {
ouids[i] = m.UID
}
changes := []store.Change{
store.ChangeRemoveUIDs{MailboxID: c.mailboxID, UIDs: ouids, ModSeq: modseq},
mb.ChangeCounts(),
}
c.broadcast(changes)
}
for _, m := range removed {
p := c.account.MessagePath(m.ID)
err := os.Remove(p)
c.xsanity(err, "removing message file for expunge")
}
})
return removed, highestModSeq
return expunged, highestModSeq
}
// Unselect is similar to close in that it closes the currently active mailbox, but
@ -4052,9 +3945,9 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) {
uids, uidargs := c.gatherCopyMoveUIDs(isUID, nums)
// Files that were created during the copy. Remove them if the operation fails.
var createdIDs []int64
var newIDs []int64
defer func() {
for _, id := range createdIDs {
for _, id := range newIDs {
p := c.account.MessagePath(id)
err := os.Remove(p)
c.xsanity(err, "cleaning up created file")
@ -4200,7 +4093,7 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) {
}
err := moxio.LinkOrCopy(c.log, dst, src, nil, true)
xcheckf(err, "link or copy file %q to %q", src, dst)
createdIDs = append(createdIDs, newMsgIDs[i])
newIDs = append(newIDs, newMsgIDs[i])
}
for dir := range syncDirs {
@ -4212,7 +4105,7 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) {
xcheckf(err, "train copied messages")
})
createdIDs = nil
newIDs = nil
// Broadcast changes to other connections.
if len(newUIDs) > 0 {
@ -4253,127 +4146,61 @@ func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) {
uids, uidargs := c.gatherCopyMoveUIDs(isUID, nums)
var mbSrc, mbDst store.Mailbox
var changes []store.Change
var newUIDs []store.UID
var mbDst store.Mailbox
var uidFirst store.UID
var modseq store.ModSeq
var cleanupIDs []int64
defer func() {
for _, id := range cleanupIDs {
p := c.account.MessagePath(id)
err := os.Remove(p)
c.xsanity(err, "removing destination message file %v", p)
}
}()
c.account.WithWLock(func() {
var changes []store.Change
c.xdbwrite(func(tx *bstore.Tx) {
mbSrc = c.xmailboxID(tx, c.mailboxID) // Validate.
mbSrc := c.xmailboxID(tx, c.mailboxID) // Validate.
mbDst = c.xmailbox(tx, name, "TRYCREATE")
if mbDst.ID == c.mailboxID {
xuserErrorf("cannot move to currently selected mailbox")
}
if len(uidargs) == 0 {
if len(uids) == 0 {
xuserErrorf("no matching messages to move")
}
// Reserve the uids in the destination mailbox.
uidFirst := mbDst.UIDNext
uidnext := uidFirst
mbDst.UIDNext += store.UID(len(uids))
uidFirst = mbDst.UIDNext
// Assign a new modseq, for the new records and for the expunged records.
var err error
modseq, err = c.account.NextModSeq(tx)
xcheckf(err, "assigning next modseq")
mbSrc.ModSeq = modseq
mbDst.ModSeq = modseq
// Update existing record with new UID and MailboxID in database for messages. We
// add a new but expunged record again in the original/source mailbox, for qresync.
// Keeping the original ID for the live message means we don't have to move the
// on-disk message contents file.
// Make query selecting messages to move.
q := bstore.QueryTx[store.Message](tx)
q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
q.FilterNonzero(store.Message{MailboxID: mbSrc.ID})
q.FilterEqual("UID", uidargs...)
q.FilterEqual("Expunged", false)
q.SortAsc("UID")
msgs, err := q.List()
xcheckf(err, "listing messages to move")
if len(msgs) != len(uidargs) {
xserverErrorf("uid and message mismatch")
}
keywords := map[string]struct{}{}
now := time.Now()
conf, _ := c.account.Conf()
for i := range msgs {
m := &msgs[i]
if m.UID != uids[i] {
xserverErrorf("internal error: got uid %d, expected %d, for index %d", m.UID, uids[i], i)
}
mbSrc.Sub(m.MailboxCounts())
// Copy of message record that we'll insert when UID is freed up.
om := *m
om.PrepareExpunge()
om.ID = 0 // Assign new ID.
om.ModSeq = modseq
m.MailboxID = mbDst.ID
if m.IsReject && m.MailboxDestinedID != 0 {
// Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message
// is used for reputation calculation during future deliveries.
m.MailboxOrigID = m.MailboxDestinedID
m.IsReject = false
m.Seen = false
}
mbDst.Add(m.MailboxCounts())
m.UID = uidnext
m.ModSeq = modseq
m.JunkFlagsForMailbox(mbDst, conf)
m.SaveDate = &now
uidnext++
err := tx.Update(m)
xcheckf(err, "updating moved message in database")
// Now that UID is unused, we can insert the old record again.
err = tx.Insert(&om)
xcheckf(err, "inserting record for expunge after moving message")
for _, kw := range m.Keywords {
keywords[kw] = struct{}{}
}
}
// Ensure destination mailbox has keywords of the moved messages.
var mbKwChanged bool
mbDst.Keywords, mbKwChanged = store.MergeKeywords(mbDst.Keywords, maps.Keys(keywords))
if mbKwChanged {
changes = append(changes, mbDst.ChangeKeywords())
}
err = tx.Update(&mbSrc)
xcheckf(err, "updating source mailbox counts and modseq")
err = tx.Update(&mbDst)
xcheckf(err, "updating destination mailbox for uids, keywords and counts")
err = c.account.RetrainMessages(context.TODO(), c.log, tx, msgs)
xcheckf(err, "retraining messages after move")
// Prepare broadcast changes to other connections.
changes = make([]store.Change, 0, 1+len(msgs)+2)
changes = append(changes, store.ChangeRemoveUIDs{MailboxID: c.mailboxID, UIDs: uids, ModSeq: modseq})
for _, m := range msgs {
newUIDs = append(newUIDs, m.UID)
changes = append(changes, m.ChangeAddUID())
}
changes = append(changes, mbSrc.ChangeCounts(), mbDst.ChangeCounts())
newIDs, chl := c.xmoveMessages(tx, q, len(uidargs), modseq, &mbSrc, &mbDst)
changes = append(changes, chl...)
cleanupIDs = newIDs
})
cleanupIDs = nil
c.broadcast(changes)
})
// ../rfc/9051:4708 ../rfc/6851:254
// ../rfc/9051:4713
c.bwritelinef("* OK [COPYUID %d %s %s] moved", mbDst.UIDValidity, compactUIDSet(uids).String(), compactUIDSet(newUIDs).String())
newUIDs := numSet{ranges: []numRange{{setNumber{number: uint32(uidFirst)}, &setNumber{number: uint32(mbDst.UIDNext - 1)}}}}
c.bwritelinef("* OK [COPYUID %d %s %s] moved", mbDst.UIDValidity, compactUIDSet(uids).String(), newUIDs.String())
qresync := c.enabled[capQresync]
var vanishedUIDs numSet
for i := 0; i < len(uids); i++ {
@ -4400,6 +4227,132 @@ func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) {
}
}
// q must yield messages from a single mailbox.
func (c *conn) xmoveMessages(tx *bstore.Tx, q *bstore.Query[store.Message], expectCount int, modseq store.ModSeq, mbSrc, mbDst *store.Mailbox) (newIDs []int64, changes []store.Change) {
newIDs = make([]int64, 0, expectCount)
var commit bool
defer func() {
if commit {
return
}
for _, id := range newIDs {
p := c.account.MessagePath(id)
err := os.Remove(p)
c.xsanity(err, "removing added message file %v", p)
}
newIDs = nil
}()
mbSrc.ModSeq = modseq
mbDst.ModSeq = modseq
var jf *junk.Filter
defer func() {
if jf != nil {
err := jf.CloseDiscard()
c.log.Check(err, "closing junk filter after error")
}
}()
accConf, _ := c.account.Conf()
changeRemoveUIDs := store.ChangeRemoveUIDs{
MailboxID: mbSrc.ID,
ModSeq: modseq,
}
changes = make([]store.Change, 0, expectCount+4) // mbsrc removeuids, mbsrc counts, mbdst counts, mbdst keywords
nkeywords := len(mbDst.Keywords)
now := time.Now()
l, err := q.List()
xcheckf(err, "listing messages to move")
if expectCount > 0 && len(l) != expectCount {
xcheckf(fmt.Errorf("moved %d messages, expected %d", len(l), expectCount), "move messages")
}
for _, om := range l {
nm := om
nm.MailboxID = mbDst.ID
nm.UID = mbDst.UIDNext
mbDst.UIDNext++
nm.ModSeq = modseq
nm.CreateSeq = modseq
nm.SaveDate = &now
if nm.IsReject && nm.MailboxDestinedID != 0 {
// Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message
// is used for reputation calculation during future deliveries.
nm.MailboxOrigID = nm.MailboxDestinedID
nm.IsReject = false
nm.Seen = false
}
nm.JunkFlagsForMailbox(*mbDst, accConf)
err := tx.Update(&nm)
xcheckf(err, "updating message with new mailbox")
mbDst.Add(nm.MailboxCounts())
mbSrc.Sub(om.MailboxCounts())
om.ID = 0
om.Expunged = true
om.ModSeq = modseq
om.TrainedJunk = nil
err = tx.Insert(&om)
xcheckf(err, "inserting expunged message in old mailbox")
err = moxio.LinkOrCopy(c.log, c.account.MessagePath(om.ID), c.account.MessagePath(nm.ID), nil, false)
xcheckf(err, "duplicating message in old mailbox for current sessions")
newIDs = append(newIDs, nm.ID)
// We don't sync the directory. In case of a crash and files disappearing, the
// eraser will simply not find the file at next startup.
err = tx.Insert(&store.MessageErase{ID: om.ID, SkipUpdateDiskUsage: true})
xcheckf(err, "insert message erase")
mbDst.Keywords, _ = store.MergeKeywords(mbDst.Keywords, nm.Keywords)
if accConf.JunkFilter != nil && nm.NeedsTraining() {
// Lazily open junk filter.
if jf == nil {
jf, _, err = c.account.OpenJunkFilter(context.TODO(), c.log)
xcheckf(err, "open junk filter")
}
err := c.account.RetrainMessage(context.TODO(), c.log, tx, jf, &nm)
xcheckf(err, "retrain message after moving")
}
changeRemoveUIDs.UIDs = append(changeRemoveUIDs.UIDs, om.UID)
changeRemoveUIDs.MsgIDs = append(changeRemoveUIDs.MsgIDs, om.ID)
changes = append(changes, nm.ChangeAddUID())
}
xcheckf(err, "move messages")
changes = append(changes, changeRemoveUIDs, mbSrc.ChangeCounts())
err = tx.Update(mbSrc)
xcheckf(err, "updating counts for inbox")
changes = append(changes, mbDst.ChangeCounts())
if len(mbDst.Keywords) > nkeywords {
changes = append(changes, mbDst.ChangeKeywords())
}
err = tx.Update(mbDst)
xcheckf(err, "updating uidnext and counts in destination mailbox")
if jf != nil {
err := jf.Close()
jf = nil
xcheckf(err, "saving junk filter")
}
commit = true
return
}
// Store sets a full set of flags, or adds/removes specific flags.
//
// State: Selected

39
imapserver/server_test.go
View File

@ -18,6 +18,8 @@ import (
"golang.org/x/sys/unix"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/imapclient"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
@ -309,6 +311,14 @@ func (tc *testconn) waitDone() {
}
func (tc *testconn) close() {
tc.close0(true)
}
func (tc *testconn) closeNoWait() {
tc.close0(false)
}
func (tc *testconn) close0(waitclose bool) {
defer func() {
if unhandledPanics.Swap(0) > 0 {
tc.t.Fatalf("handled panic in server")
@ -319,13 +329,16 @@ func (tc *testconn) close() {
// Already closed, we are not strict about closing multiple times.
return
}
err := tc.account.Close()
tc.check(err, "close account")
// no account.CheckClosed(), the tests open accounts multiple times.
tc.account = nil
if tc.client != nil {
tc.client.Close()
}
err := tc.account.Close()
tc.check(err, "close account")
if waitclose {
tc.account.WaitClosed()
}
// no account.CheckClosed(), the tests open accounts multiple times.
tc.account = nil
tc.serverConn.Close()
tc.waitDone()
if tc.switchStop != nil {
@ -406,6 +419,22 @@ func startArgsMore(t *testing.T, first, immediateTLS bool, serverConfig, clientC
switchStop := func() {}
if first {
switchStop = store.Switchboard()
// Add a deleted mailbox, may excercise some code paths.
err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
// todo: add a message to inbox and remove it again. need to change all uids in the tests.
// todo: add tests for operating on an expunged mailbox. it should say it doesn't exist.
mb, _, _, _, err := acc.MailboxCreate(tx, "expungebox", store.SpecialUse{})
if err != nil {
return fmt.Errorf("create mailbox: %v", err)
}
if _, _, err := acc.MailboxDelete(ctxbg, pkglog, tx, &mb); err != nil {
return fmt.Errorf("delete mailbox: %v", err)
}
return nil
})
tcheck(t, err, "add expunged mailbox")
}
if afterInit != nil {
@ -698,7 +727,7 @@ func TestMailboxDeleted(t *testing.T) {
defer tc.close()
tc2 := startNoSwitchboard(t)
defer tc2.close()
defer tc2.closeNoWait()
tc.client.Login("mjl@mox.example", password0)
tc2.client.Login("mjl@mox.example", password0)

2
imapserver/status_test.go
View File

@ -19,6 +19,8 @@ func TestStatus(t *testing.T) {
tc.transactf("bad", "status inbox (uidvalidity) ") // Leftover data.
tc.transactf("bad", "status inbox (unknown)") // Unknown attribute.
tc.transactf("no", "status expungebox (messages)") // No longer exists.
tc.transactf("ok", "status inbox (messages uidnext uidvalidity unseen deleted size recent appendlimit)")
tc.xuntagged(imapclient.UntaggedStatus{
Mailbox: "Inbox",

2
imapserver/subscribe_test.go
View File

@ -11,7 +11,7 @@ func TestSubscribe(t *testing.T) {
defer tc.close()
tc2 := startNoSwitchboard(t)
defer tc2.close()
defer tc2.closeNoWait()
tc.client.Login("mjl@mox.example", password0)
tc2.client.Login("mjl@mox.example", password0)

4
imapserver/unsubscribe_test.go
View File

@ -15,9 +15,11 @@ func TestUnsubscribe(t *testing.T) {
tc.transactf("bad", "unsubscribe fine ") // Leftover data.
tc.transactf("no", "unsubscribe a/b") // Does not exist and is not subscribed.
tc.transactf("ok", "unsubscribe expungebox") // Does not exist anymore but is still subscribed.
tc.transactf("no", "unsubscribe expungebox") // Not subscribed.
tc.transactf("ok", "create a/b")
tc.transactf("ok", "unsubscribe a/b")
tc.transactf("ok", "unsubscribe a/b") // Can unsubscribe even if it does not exist.
tc.transactf("ok", "unsubscribe a/b") // Can unsubscribe even if there is no subscription.
tc.transactf("ok", "subscribe a/b")
tc.transactf("ok", "unsubscribe a/b")
}

16
import.go
View File

@ -254,7 +254,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
ctl.xwriteok()
// We will be delivering messages. If we fail halfway, we need to remove the created msg files.
var deliveredIDs []int64
var newIDs []int64
defer func() {
x := recover()
if x == nil {
@ -269,12 +269,12 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
ctl.log.Error("import error")
}
for _, id := range deliveredIDs {
for _, id := range newIDs {
p := a.MessagePath(id)
err := os.Remove(p)
ctl.log.Check(err, "closing message file after import error", slog.String("path", p))
}
deliveredIDs = nil
newIDs = nil
ctl.xerror(fmt.Sprintf("import error: %v", x))
}()
@ -371,7 +371,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
}
err = a.MessageAdd(ctl.log, tx, &mb, m, msgf, opts)
ctl.xcheck(err, "delivering message")
deliveredIDs = append(deliveredIDs, m.ID)
newIDs = append(newIDs, m.ID)
changes = append(changes, m.ChangeAddUID())
msgDirs[filepath.Dir(a.MessagePath(m.ID))] = struct{}{}
@ -393,8 +393,8 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
}
// Match threads.
if len(deliveredIDs) > 0 {
err = a.AssignThreads(ctx, ctl.log, tx, deliveredIDs[0], 0, io.Discard)
if len(newIDs) > 0 {
err = a.AssignThreads(ctx, ctl.log, tx, newIDs[0], 0, io.Discard)
ctl.xcheck(err, "assigning messages to threads")
}
@ -423,8 +423,8 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
err = tx.Commit()
ctl.xcheck(err, "commit")
tx = nil
ctl.log.Info("delivered messages through import", slog.Int("count", len(deliveredIDs)))
deliveredIDs = nil
ctl.log.Info("delivered messages through import", slog.Int("count", len(newIDs)))
newIDs = nil
store.BroadcastChanges(a, changes)
})

8
main.go
View File

@ -3340,6 +3340,7 @@ open, or is not running.
}
q := bstore.QueryTx[store.Mailbox](tx)
q.FilterEqual("Expunged", false)
if len(args) == 2 {
q.FilterEqual("Name", args[1])
}
@ -3398,7 +3399,8 @@ open, or is not running.
// Reassign UIDs, going per mailbox. We assign starting at 1, only changing the
// message if it isn't already at the intended UID. Doing it in this order ensures
// we don't get into trouble with duplicate UIDs for a mailbox. We assign a new
// modseq. Not strictly needed, but doesn't hurt.
// modseq. Not strictly needed, but doesn't hurt. It's also why we assign a UID to
// expunged messages.
modseq, err := a.NextModSeq(tx)
xcheckf(err, "assigning next modseq")
@ -3424,7 +3426,7 @@ open, or is not running.
}
// Now update the uidnext, uidvalidity and modseq for each mailbox.
err = bstore.QueryTx[store.Mailbox](tx).ForEach(func(mb store.Mailbox) error {
err = bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb store.Mailbox) error {
// Assign each mailbox a completely new uidvalidity.
uidvalidity, err := a.NextUIDValidity(tx)
if err != nil {
@ -3491,7 +3493,7 @@ open, or is not running.
err = a.DB.Write(context.Background(), func(tx *bstore.Tx) error {
// We look at each mailbox, retrieve its max UID and compare against the mailbox
// UIDNEXT.
err := bstore.QueryTx[store.Mailbox](tx).ForEach(func(mb store.Mailbox) error {
err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb store.Mailbox) error {
if mb.UIDValidity > maxUIDValidity {
maxUIDValidity = mb.UIDValidity
}

11
mox-/config.go
View File

@ -933,6 +933,10 @@ func PrepareStaticConfig(ctx context.Context, log mlog.Log, configFile string, c
// DefaultMailboxes is deprecated.
for _, mb := range c.DefaultMailboxes {
checkMailboxNormf(mb, "default mailbox")
// We don't create parent mailboxes for default mailboxes.
if ParentMailboxName(mb) != "" {
addErrorf("default mailbox cannot be a child mailbox")
}
}
checkSpecialUseMailbox := func(nameOpt string) {
if nameOpt != "" {
@ -940,6 +944,10 @@ func PrepareStaticConfig(ctx context.Context, log mlog.Log, configFile string, c
if strings.EqualFold(nameOpt, "inbox") {
addErrorf("initial mailbox cannot be set to Inbox (Inbox is always created)")
}
// We don't currently create parent mailboxes for initial mailboxes.
if ParentMailboxName(nameOpt) != "" {
addErrorf("initial mailboxes cannot be child mailboxes")
}
}
}
checkSpecialUseMailbox(c.InitialMailboxes.SpecialUse.Archive)
@ -952,6 +960,9 @@ func PrepareStaticConfig(ctx context.Context, log mlog.Log, configFile string, c
if strings.EqualFold(name, "inbox") {
addErrorf("initial regular mailbox cannot be set to Inbox (Inbox is always created)")
}
if ParentMailboxName(name) != "" {
addErrorf("initial mailboxes cannot be child mailboxes")
}
}
checkTransportSMTP := func(name string, isTLS bool, t *config.TransportSMTP) {

15
mox-/parentname.go Normal file
View File

@ -0,0 +1,15 @@
package mox
import (
"strings"
)
// ParentMailboxName returns the name of the parent mailbox, returning empty if
// there is no parent.
func ParentMailboxName(name string) string {
i := strings.LastIndex(name, "/")
if i < 0 {
return ""
}
return name[:i]
}

6
queue/hook_test.go
View File

@ -29,7 +29,7 @@ func TestHookIncoming(t *testing.T) {
tcheck(t, err, "open account for retired")
defer func() {
accret.Close()
accret.CheckClosed()
accret.WaitClosed()
}()
testIncoming := func(a *store.Account, expIn bool) {
@ -125,7 +125,7 @@ func TestFromIDIncomingDelivery(t *testing.T) {
tcheck(t, err, "open account for retired")
defer func() {
accret.Close()
accret.CheckClosed()
accret.WaitClosed()
}()
// Account that only gets webhook calls, but no retired webhooks.
@ -133,7 +133,7 @@ func TestFromIDIncomingDelivery(t *testing.T) {
tcheck(t, err, "open account for hook")
defer func() {
acchook.Close()
acchook.CheckClosed()
acchook.WaitClosed()
}()
addr, err := smtp.ParseAddress("mjl@mox.example")

2
queue/queue_test.go
View File

@ -75,7 +75,7 @@ func setup(t *testing.T) (*store.Account, func()) {
mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg)
return acc, func() {
acc.Close()
acc.CheckClosed()
acc.WaitClosed()
mox.ShutdownCancel()
mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg)
Shutdown()

2
smtpserver/fuzz_test.go
View File

@ -43,7 +43,7 @@ func FuzzServer(f *testing.F) {
}
defer func() {
acc.Close()
acc.CheckClosed()
acc.WaitClosed()
}()
err = acc.SetPassword(log, "testtest")
if err != nil {

9
smtpserver/rejects.go
View File

@ -46,14 +46,11 @@ func rejectPresent(log mlog.Log, acc *store.Account, rejectsMailbox string, m *s
var err error
acc.WithRLock(func() {
err = acc.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
mbq := bstore.QueryTx[store.Mailbox](tx)
mbq.FilterNonzero(store.Mailbox{Name: rejectsMailbox})
mb, err := mbq.Get()
if err == bstore.ErrAbsent {
return nil
}
mb, err := acc.MailboxFind(tx, rejectsMailbox)
if err != nil {
return fmt.Errorf("looking for rejects mailbox: %w", err)
} else if mb == nil {
return nil
}
q := bstore.QueryTx[store.Message](tx)

43
smtpserver/server.go
View File

@ -3272,6 +3272,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
// See if we received a non-junk message from this organizational domain.
q := bstore.QueryTx[store.Message](tx)
q.FilterNonzero(store.Message{MsgFromOrgDomain: a0.d.m.MsgFromOrgDomain})
q.FilterEqual("Expunged", false)
q.FilterEqual("Notjunk", true)
q.FilterEqual("IsReject", false)
exists, err := q.Exists()
@ -3406,25 +3407,37 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
// due to reputation for later delivery attempts.
a.d.m.MessageHash = messagehash
a.d.acc.WithWLock(func() {
hasSpace := true
var err error
if !conf.KeepRejects {
hasSpace, err = a.d.acc.TidyRejectsMailbox(c.log, conf.RejectsMailbox)
}
if err != nil {
log.Errorx("tidying rejects mailbox", err)
} else if !hasSpace {
log.Info("not storing spammy mail to full rejects mailbox")
return
}
var changes []store.Change
var stored bool
err = a.d.acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
var newID int64
defer func() {
if newID != 0 {
p := a.d.acc.MessagePath(newID)
err := os.Remove(p)
c.log.Check(err, "remove message after error delivering to rejects", slog.String("path", p))
}
}()
err := a.d.acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
mbrej, err := a.d.acc.MailboxFind(tx, conf.RejectsMailbox)
if err != nil {
return fmt.Errorf("finding rejects mailbox: %v", err)
}
hasSpace := true
if !conf.KeepRejects && mbrej != nil {
var chl []store.Change
chl, hasSpace, err = a.d.acc.TidyRejectsMailbox(c.log, tx, mbrej)
if err != nil {
return fmt.Errorf("tidying rejects mailbox: %v", err)
}
if !hasSpace {
log.Info("not storing spammy mail to full rejects mailbox")
return nil
}
changes = append(changes, chl...)
}
if mbrej == nil {
nmb, chl, _, _, err := a.d.acc.MailboxCreate(tx, conf.RejectsMailbox, store.SpecialUse{})
if err != nil {
@ -3438,6 +3451,8 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
if err := a.d.acc.MessageAdd(log, tx, mbrej, a.d.m, dataFile, store.AddOpts{}); err != nil {
return fmt.Errorf("delivering spammy mail to rejects mailbox: %v", err)
}
newID = a.d.m.ID
if err := tx.Update(mbrej); err != nil {
return fmt.Errorf("updating rejects mailbox: %v", err)
}
@ -3451,6 +3466,8 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
} else if stored {
log.Info("stored spammy mail in rejects mailbox")
}
newID = 0
store.BroadcastChanges(a.d.acc, changes)
})
}

3
smtpserver/server_test.go
View File

@ -184,7 +184,7 @@ func (ts *testserver) close() {
ts.switchStop()
err = ts.acc.Close()
tcheck(ts.t, err, "closing account")
ts.acc.CheckClosed()
ts.acc.WaitClosed()
ts.acc = nil
}
@ -193,6 +193,7 @@ func (ts *testserver) checkCount(mailboxName string, expect int) {
t.Helper()
q := bstore.QueryDB[store.Mailbox](ctxbg, ts.acc.DB)
q.FilterNonzero(store.Mailbox{Name: mailboxName})
q.FilterEqual("Expunged", false)
mb, err := q.Get()
tcheck(t, err, "get mailbox")
qm := bstore.QueryDB[store.Message](ctxbg, ts.acc.DB)

998
store/account.go
View File

File diff suppressed because it is too large Load Diff

30
store/account_test.go
View File

@ -46,7 +46,7 @@ func TestMailbox(t *testing.T) {
defer func() {
err = acc.Close()
tcheck(t, err, "closing account")
acc.CheckClosed()
acc.WaitClosed()
}()
defer Switchboard()()
@ -162,6 +162,8 @@ func TestMailbox(t *testing.T) {
var modseq ModSeq
acc.WithWLock(func() {
var changes []Change
err := acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
_, _, err := acc.MailboxEnsure(tx, "Testbox", true, SpecialUse{}, &modseq)
return err
@ -200,27 +202,33 @@ func TestMailbox(t *testing.T) {
t.Fatalf("did not find Testbox2")
}
changes, err := acc.SubscriptionEnsure(tx, "Testbox2")
nchanges, err := acc.SubscriptionEnsure(tx, "Testbox2")
tcheck(t, err, "ensuring new subscription")
if len(changes) == 0 {
if len(nchanges) == 0 {
t.Fatalf("new subscription did not result in changes")
}
changes, err = acc.SubscriptionEnsure(tx, "Testbox2")
changes = append(changes, nchanges...)
nchanges, err = acc.SubscriptionEnsure(tx, "Testbox2")
tcheck(t, err, "ensuring already present subscription")
if len(changes) != 0 {
if len(nchanges) != 0 {
t.Fatalf("already present subscription resulted in changes")
}
// todo: check that messages are removed.
mbRej, err := bstore.QueryTx[Mailbox](tx).FilterNonzero(Mailbox{Name: "Rejects"}).Get()
tcheck(t, err, "get rejects mailbox")
nchanges, hasSpace, err := acc.TidyRejectsMailbox(log, tx, &mbRej)
tcheck(t, err, "tidy rejects mailbox")
changes = append(changes, nchanges...)
if !hasSpace {
t.Fatalf("no space for more rejects")
}
return nil
})
tcheck(t, err, "write tx")
// todo: check that messages are removed.
hasSpace, err := acc.TidyRejectsMailbox(log, "Rejects")
tcheck(t, err, "tidy rejects mailbox")
if !hasSpace {
t.Fatalf("no space for more rejects")
}
BroadcastChanges(acc, changes)
acc.RejectsRemove(log, "Rejects", "m01@mox.example")
})

5
store/export.go
View File

@ -10,7 +10,6 @@ import (
"io"
"log/slog"
"os"
"path"
"path/filepath"
"strings"
"time"
@ -18,6 +17,7 @@ import (
"github.com/mjl-/bstore"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
)
// Archiver can archive multiple mailboxes and their messages.
@ -158,9 +158,10 @@ func ExportMessages(ctx context.Context, log mlog.Log, db *bstore.DB, accountDir
var trimPrefix string
if mailboxOpt != "" {
// If exporting a specific mailbox, trim its parent path from stored file names.
trimPrefix = path.Dir(mailboxOpt) + "/"
trimPrefix = mox.ParentMailboxName(mailboxOpt) + "/"
}
q := bstore.QueryTx[Mailbox](tx)
q.FilterEqual("Expunged", false)
q.FilterFn(func(mb Mailbox) bool {
return mailboxOpt == "" || mb.Name == mailboxOpt || recursive && strings.HasPrefix(mb.Name, prefix)
})

285
store/state.go
View File

@ -1,14 +1,24 @@
package store
import (
"fmt"
"log/slog"
"os"
"sync"
"sync/atomic"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/metrics"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
)
var (
register = make(chan *Comm)
unregister = make(chan *Comm)
broadcast = make(chan changeReq)
applied = make(chan removalApplied)
)
type changeReq struct {
@ -18,6 +28,11 @@ type changeReq struct {
done chan struct{}
}
type removalApplied struct {
Account *Account
MsgIDs []int64
}
type UID uint32 // IMAP UID.
// Change to mailboxes/subscriptions/messages in an account. One of the Change*
@ -38,6 +53,7 @@ type ChangeRemoveUIDs struct {
MailboxID int64
UIDs []UID // Must be in increasing UID order, for IMAP.
ModSeq ModSeq
MsgIDs []int64 // Message.ID, for erasing, order does not necessarily correspond with UIDs!
}
// ChangeFlags is sent for an update to flags for a message, e.g. "Seen".
@ -118,20 +134,163 @@ type ChangeAnnotation struct {
ModSeq ModSeq
}
var switchboardBusy atomic.Bool
func messageEraser(donec chan struct{}, cleanc chan map[*Account][]int64) {
log := mlog.New("store", nil)
// Switchboard distributes changes to accounts to interested listeners. See Comm and Change.
func Switchboard() (stop func()) {
regs := map[*Account]map[*Comm]struct{}{}
done := make(chan struct{})
if !switchboardBusy.CompareAndSwap(false, true) {
panic("switchboard already busy")
for {
clean, ok := <-cleanc
if !ok {
donec <- struct{}{}
return
}
for acc, ids := range clean {
eraseMessages(log, acc, ids)
}
}
}
func eraseMessages(log mlog.Log, acc *Account, ids []int64) {
// We are responsible for closing the accounts.
defer func() {
err := acc.Close()
log.Check(err, "close account after erasing expunged messages", slog.String("account", acc.Name))
}()
acc.Lock()
defer acc.Unlock()
err := acc.DB.Write(mox.Context, func(tx *bstore.Tx) error {
du := DiskUsage{ID: 1}
if err := tx.Get(&du); err != nil {
return fmt.Errorf("get disk usage: %v", err)
}
var duchanged bool
for _, id := range ids {
me := MessageErase{ID: id}
if err := tx.Get(&me); err != nil {
return fmt.Errorf("delete message erase record %d: %v", id, err)
}
m := Message{ID: id}
if err := tx.Get(&m); err != nil {
return fmt.Errorf("get message %d to erase: %v", id, err)
} else if !m.Expunged {
return fmt.Errorf("message %d to erase is not marked expunged", id)
}
if !me.SkipUpdateDiskUsage {
du.MessageSize -= m.Size
duchanged = true
}
m.erase()
if err := tx.Update(&m); err != nil {
return fmt.Errorf("mark message %d erase in database: %v", id, err)
}
if err := tx.Delete(&me); err != nil {
return fmt.Errorf("deleting message erase record %d: %v", id, err)
}
}
if duchanged {
if err := tx.Update(&du); err != nil {
return fmt.Errorf("update disk usage after erasing: %v", err)
}
}
return nil
})
if err != nil {
log.Errorx("erasing expunged messages", err,
slog.String("account", acc.Name),
slog.Any("ids", ids),
)
return
}
// We remove the files after the database commit. It's better to have the files
// still around without being referenced from the database than references in the
// database to non-existent files.
for _, id := range ids {
p := acc.MessagePath(id)
err := os.Remove(p)
log.Check(err, "removing expunged message file from disk", slog.String("path", p))
}
}
func switchboard(stopc, donec chan struct{}, cleanc chan map[*Account][]int64) {
regs := map[*Account]map[*Comm]struct{}{}
// We don't remove message files or clear fields in the Message stored in the
// database until all references, from all sessions have gone away. When we see
// an expunge of a message, we count how many comms are active (i.e. how many
// sessions reference the message). We require each of them to tell us they are no
// longer referencing that message. Once we've seen that from all Comms, we remove
// the on-disk file and the fields from the database.
//
// During the initial account open (when there are no active sessions/Comms yet,
// and we open the message database file), the message erases will also be applied.
//
// When we add an account to eraseRefs, we increase the refcount, and we decrease
// it again when removing the account.
eraseRefs := map[*Account]map[int64]int{}
// We collect which messages can be erased per account, for sending them off to the
// eraser goroutine. When an account is added to this map, its refcount is
// increased. It is decreased again by the eraser goroutine.
eraseIDs := map[*Account][]int64{}
addEraseIDs := func(acc *Account, ids ...int64) {
if _, ok := eraseIDs[acc]; !ok {
openAccounts.Lock()
acc.nused++
openAccounts.Unlock()
}
eraseIDs[acc] = append(eraseIDs[acc], ids...)
}
decreaseEraseRefs := func(acc *Account, ids ...int64) {
for _, id := range ids {
v := eraseRefs[acc][id] - 1
if v < 0 {
metrics.PanicInc(metrics.Store) // For tests.
panic(fmt.Sprintf("negative expunged message references for account %q, message id %d", acc.Name, id))
}
if v > 0 {
eraseRefs[acc][id] = v
continue
}
addEraseIDs(acc, id)
delete(eraseRefs[acc], id)
if len(eraseRefs[acc]) > 0 {
continue
}
delete(eraseRefs, acc)
// Note: cannot use acc.Close, it tries to lock acc, but someone broadcasting to
// this goroutine will likely have the lock.
openAccounts.Lock()
acc.nused--
n := acc.nused
openAccounts.Unlock()
if n < 0 {
metrics.PanicInc(metrics.Store) // For tests.
panic(fmt.Sprintf("negative reference count for account %q, after removing message id %d", acc.Name, id))
}
}
}
go func() {
for {
// If we have messages to clean, try sending to the eraser.
cc := cleanc
if len(eraseIDs) == 0 {
cc = nil
}
select {
case cc <- eraseIDs:
eraseIDs = map[*Account][]int64{}
case c := <-register:
if _, ok := regs[c.acc]; !ok {
regs[c.acc] = map[*Comm]struct{}{}
@ -139,6 +298,17 @@ func Switchboard() (stop func()) {
regs[c.acc][c] = struct{}{}
case c := <-unregister:
// Drain any ChangeRemoveUIDs references from the comm, to update our eraseRefs and
// possibly queue messages for cleaning. No need to take a lock, the caller does
// not use the comm anymore.
for _, ch := range c.changes {
rem, ok := ch.(ChangeRemoveUIDs)
if !ok {
continue
}
decreaseEraseRefs(c.acc, rem.MsgIDs...)
}
delete(regs[c.acc], c)
if len(regs[c.acc]) == 0 {
delete(regs, c.acc)
@ -146,6 +316,43 @@ func Switchboard() (stop func()) {
case chReq := <-broadcast:
acc := chReq.acc
// Track references to removed messages in sessions (mostly IMAP) so we can pass
// them to the eraser.
for _, ch := range chReq.changes {
rem, ok := ch.(ChangeRemoveUIDs)
if !ok {
continue
}
refs := len(regs[acc])
if chReq.comm != nil {
// The sender does not get this change and doesn't have to notify us of having
// processed the removal.
refs--
}
if refs <= 0 {
addEraseIDs(acc, rem.MsgIDs...)
continue
}
// Comms/sessions still reference these messages, track how many.
for _, id := range rem.MsgIDs {
if _, ok := eraseRefs[acc]; !ok {
openAccounts.Lock()
acc.nused++
openAccounts.Unlock()
eraseRefs[acc] = map[int64]int{}
}
if _, ok := eraseRefs[acc][id]; ok {
metrics.PanicInc(metrics.Store) // For tests.
panic(fmt.Sprintf("already have eraseRef for message id %d, account %q", id, acc.Name))
}
eraseRefs[acc][id] = refs
}
}
for c := range regs[acc] {
// Do not send the broadcaster back their own changes. chReq.comm is nil if not
// originating from a comm, so won't match in that case.
@ -164,15 +371,58 @@ func Switchboard() (stop func()) {
}
chReq.done <- struct{}{}
case <-done:
done <- struct{}{}
case removal := <-applied:
acc := removal.Account
// Decrease references of messages, queueing for erasure when the last reference
// goes away.
decreaseEraseRefs(acc, removal.MsgIDs...)
case <-stopc:
// We may still have eraseRefs, messages currently referenced in a session. Those
// messages will be erased when the database file is opened again in the future. If
// we have messages ready to erase now, we'll do that first.
if len(eraseIDs) > 0 {
cleanc <- eraseIDs
eraseIDs = nil
}
for acc := range eraseRefs {
err := acc.Close()
log := mlog.New("store", nil)
log.Check(err, "closing account")
}
close(cleanc) // Tell eraser to stop.
donec <- struct{}{} // Say we are now done.
return
}
}
}()
}
var switchboardBusy atomic.Bool
// Switchboard distributes changes to accounts to interested listeners. See Comm and Change.
func Switchboard() (stop func()) {
if !switchboardBusy.CompareAndSwap(false, true) {
panic("switchboard already busy")
}
stopc := make(chan struct{})
donec := make(chan struct{})
cleanc := make(chan map[*Account][]int64)
go messageEraser(donec, cleanc)
go switchboard(stopc, donec, cleanc)
return func() {
done <- struct{}{}
<-done
stopc <- struct{}{}
// Wait for switchboard and eraser goroutines to be ready.
<-donec
<-donec
if !switchboardBusy.CompareAndSwap(true, false) {
panic("switchboard already unregistered?")
}
@ -225,6 +475,13 @@ func (c *Comm) Get() []Change {
return l
}
// RemovalSeen must be called by consumers when they have applied the removal to
// their session. The switchboard tracks references of expunged messages, and
// removes/cleans the message up when the last reference is gone.
func (c *Comm) RemovalSeen(ch ChangeRemoveUIDs) {
applied <- removalApplied{c.acc, ch.MsgIDs}
}
// BroadcastChanges ensures changes are sent to all listeners on the accoount.
func BroadcastChanges(acc *Account, ch []Change) {
if len(ch) == 0 {

2
store/threads.go
View File

@ -357,6 +357,8 @@ func (a *Account) AssignThreads(ctx context.Context, log mlog.Log, txOpt *bstore
m := Message{ID: mi.ID}
if err := tx.Get(&m); err != nil {
return fmt.Errorf("get message %d for resolving pending thread for message-id %s, %d: %w", mi.ID, tm.MessageID, tm.ID, err)
} else if m.Expunged {
return fmt.Errorf("message %d marked as expunged", mi.ID)
}
if m.ThreadID != 0 {
// ThreadID already set because this is a cyclic message. If we would assign a

8
store/train.go
View File

@ -155,11 +155,7 @@ func (a *Account) RetrainMessage(ctx context.Context, log mlog.Log, tx *bstore.T
// TrainMessage trains the junk filter based on the current m.Junk/m.Notjunk flags,
// disregarding m.TrainedJunk and not updating that field.
func (a *Account) TrainMessage(ctx context.Context, log mlog.Log, jf *junk.Filter, m Message) (bool, error) {
if m.Junk == m.Notjunk {
return false, nil
}
func (a *Account) TrainMessage(ctx context.Context, log mlog.Log, jf *junk.Filter, ham bool, m Message) (bool, error) {
mr := a.MessageReader(m)
defer func() {
err := mr.Close()
@ -178,5 +174,5 @@ func (a *Account) TrainMessage(ctx context.Context, log mlog.Log, jf *junk.Filte
return false, nil
}
return true, jf.Train(ctx, m.Notjunk, words)
return true, jf.Train(ctx, ham, words)
}

3
verifydata.go
View File

@ -285,6 +285,9 @@ possibly making them potentially no longer readable by the previous version.
if m.Expunged {
return nil
}
if mb.Expunged {
checkf(errors.New("mailbox is expunged but message is not"), dbpath, "message id %d is in expunged mailbox %q (id %d)", m.ID, mb.Name, mb.ID)
}
totalSize += m.Size
mp := store.MessagePath(m.ID)

2
webaccount/account_test.go
View File

@ -111,7 +111,7 @@ func TestAccount(t *testing.T) {
defer func() {
err = acc.Close()
tcheck(t, err, "closing account")
acc.CheckClosed()
acc.WaitClosed()
}()
defer store.Switchboard()()

30
webaccount/import.go
View File

@ -291,7 +291,7 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store.
var changes []store.Change
// ID's of delivered messages. If we have to rollback, we have to remove this files.
var deliveredIDs []int64
var newIDs []int64
sendEvent := func(kind string, v any) {
buf, err := json.Marshal(v)
@ -321,7 +321,7 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store.
defer func() {
store.CloseRemoveTempFile(log, f, "uploaded messages")
for _, id := range deliveredIDs {
for _, id := range newIDs {
p := acc.MessagePath(id)
err := os.Remove(p)
log.Check(err, "closing message file after import error", slog.String("path", p))
@ -444,6 +444,7 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store.
var p string
var mb *store.Mailbox
var parent store.Mailbox
for i, e := range strings.Split(name, "/") {
if i == 0 {
p = e
@ -454,10 +455,9 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store.
continue
}
q := bstore.QueryTx[store.Mailbox](tx)
q.FilterNonzero(store.Mailbox{Name: p})
xmb, err := q.Get()
if err == bstore.ErrAbsent {
mb, err = acc.MailboxFind(tx, p)
ximportcheckf(err, "looking up mailbox %s to import to (aborting)", p)
if mb == nil {
uidvalidity, err := acc.NextUIDValidity(tx)
ximportcheckf(err, "finding next uid validity")
@ -468,26 +468,24 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store.
}
mb = &store.Mailbox{
CreateSeq: modseq,
ModSeq: modseq,
ParentID: parent.ID,
Name: p,
UIDValidity: uidvalidity,
UIDNext: 1,
ModSeq: modseq,
CreateSeq: modseq,
HaveCounts: true,
// Do not assign special-use flags. This existing account probably already has such mailboxes.
}
err = tx.Insert(mb)
ximportcheckf(err, "inserting mailbox in database")
parent = *mb
if tx.Get(&store.Subscription{Name: p}) != nil {
err := tx.Insert(&store.Subscription{Name: p})
ximportcheckf(err, "subscribing to imported mailbox")
}
changes = append(changes, store.ChangeAddMailbox{Mailbox: *mb, Flags: []string{`\Subscribed`}, ModSeq: modseq})
} else if err != nil {
ximportcheckf(err, "creating mailbox %s (aborting)", p)
} else {
mb = &xmb
}
if prevMailbox != "" && mb.Name != prevMailbox {
sendEvent("count", importCount{prevMailbox, messages[prevMailbox]})
@ -559,7 +557,7 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store.
problemf("delivering message %s: %s (continuing)", pos, err)
return
}
deliveredIDs = append(deliveredIDs, m.ID)
newIDs = append(newIDs, m.ID)
changes = append(changes, m.ChangeAddUID())
messages[mb.Name]++
if messages[mb.Name]%100 == 0 || prevMailbox != mb.Name {
@ -814,9 +812,9 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store.
}
// Match threads.
if len(deliveredIDs) > 0 {
if len(newIDs) > 0 {
sendEvent("step", importStep{"matching messages with threads"})
err = acc.AssignThreads(ctx, log, tx, deliveredIDs[0], 0, io.Discard)
err = acc.AssignThreads(ctx, log, tx, newIDs[0], 0, io.Discard)
ximportcheckf(err, "assigning messages to threads")
}
@ -837,7 +835,7 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store.
err = tx.Commit()
tx = nil
ximportcheckf(err, "commit")
deliveredIDs = nil
newIDs = nil
if jf != nil {
if err := jf.Close(); err != nil {

20
webapisrv/server.go
View File

@ -21,6 +21,7 @@ import (
"net"
"net/http"
"net/textproto"
"os"
"reflect"
"runtime/debug"
"slices"
@ -1058,8 +1059,15 @@ func (s server) Send(ctx context.Context, req webapi.SendRequest) (resp webapi.S
acc.WithRLock(func() {
var changes []store.Change
var sentID int64
metricked := false
defer func() {
if sentID != 0 {
p := acc.MessagePath(sentID)
err := os.Remove(p)
log.Check(err, "removing sent message file after error", slog.String("path", p))
}
if x := recover(); x != nil {
if !metricked {
metricServerErrors.WithLabelValues("submit").Inc()
@ -1068,7 +1076,7 @@ func (s server) Send(ctx context.Context, req webapi.SendRequest) (resp webapi.S
}
}()
xdbwrite(ctx, reqInfo.Account, func(tx *bstore.Tx) {
sentmb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Sent", true).Get()
sentmb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).FilterEqual("Sent", true).Get()
if err == bstore.ErrAbsent {
// There is no mailbox designated as Sent mailbox, so we're done.
return
@ -1107,12 +1115,14 @@ func (s server) Send(ctx context.Context, req webapi.SendRequest) (resp webapi.S
metricked = true
}
xcheckf(err, "message submitted to queue, appending message to Sent mailbox")
sentID = sentm.ID
err = tx.Update(&sentmb)
xcheckf(err, "updating mailbox")
changes = append(changes, sentm.ChangeAddUID(), sentmb.ChangeCounts())
})
sentID = 0 // Commit.
store.BroadcastChanges(acc, changes)
})
@ -1190,8 +1200,12 @@ func xmessageGet(ctx context.Context, acc *store.Account, msgID int64) (store.Me
if err := tx.Get(&m); err == bstore.ErrAbsent || err == nil && m.Expunged {
panic(webapi.Error{Code: "messageNotFound", Message: "message not found"})
}
mb = store.Mailbox{ID: m.MailboxID}
return tx.Get(&mb)
var err error
mb, err = store.MailboxID(tx, m.MailboxID)
if err != nil {
return fmt.Errorf("get mailbox: %v", err)
}
return nil
})
xcheckf(err, "get message")
return m, mb

2
webapisrv/server_test.go
View File

@ -82,7 +82,7 @@ func TestServer(t *testing.T) {
defer func() {
err := acc.Close()
log.Check(err, "closing account")
acc.CheckClosed()
acc.WaitClosed()
}()
s := NewServer(100*1024, "/webapi/", false).(server)

155
webmail/api.go
View File

@ -245,7 +245,10 @@ func (Webmail) MessageFindMessageID(ctx context.Context, messageID string) (id i
}
xdbread(ctx, acc, func(tx *bstore.Tx) {
m, err := bstore.QueryTx[store.Message](tx).FilterNonzero(store.Message{MessageID: messageID}).Get()
q := bstore.QueryTx[store.Message](tx)
q.FilterEqual("Expunged", false)
q.FilterNonzero(store.Message{MessageID: messageID})
m, err := q.Get()
if err == bstore.ErrAbsent {
return
}
@ -417,24 +420,27 @@ func (w Webmail) MessageCompose(ctx context.Context, m ComposeMessage, mailboxID
var nm store.Message
// Remove previous draft message, append message to destination mailbox.
acc.WithRLock(func() {
acc.WithWLock(func() {
var changes []store.Change
var newIDs []int64
defer func() {
for _, id := range newIDs {
p := acc.MessagePath(id)
err := os.Remove(p)
log.Check(err, "removing added message aftr error", slog.String("path", p))
}
}()
xdbwrite(ctx, acc, func(tx *bstore.Tx) {
var modseq store.ModSeq // Only set if needed.
if m.DraftMessageID > 0 {
nchanges := xops.MessageDeleteTx(ctx, log, tx, acc, []int64{m.DraftMessageID}, &modseq)
changes = append(changes, nchanges...)
// On-disk file is removed after lock.
}
// Find mailbox to write to.
mb := store.Mailbox{ID: mailboxID}
err := tx.Get(&mb)
if err == bstore.ErrAbsent {
xcheckuserf(ctx, err, "looking up mailbox")
}
mb, err := store.MailboxID(tx, mailboxID)
xcheckf(ctx, err, "looking up mailbox")
if modseq == 0 {
@ -456,23 +462,18 @@ func (w Webmail) MessageCompose(ctx context.Context, m ComposeMessage, mailboxID
xcheckuserf(ctx, err, "checking quota")
}
xcheckf(ctx, err, "storing message in mailbox")
newIDs = append(newIDs, nm.ID)
err = tx.Update(&mb)
xcheckf(ctx, err, "updating sent mailbox for counts")
changes = append(changes, nm.ChangeAddUID(), mb.ChangeCounts())
})
newIDs = nil
store.BroadcastChanges(acc, changes)
})
// Remove on-disk file for removed draft message.
if m.DraftMessageID > 0 {
p := acc.MessagePath(m.DraftMessageID)
err := os.Remove(p)
log.Check(err, "removing draft message file")
}
return nm.ID
}
@ -545,9 +546,8 @@ func xmailboxID(ctx context.Context, tx *bstore.Tx, mailboxID int64) store.Mailb
if mailboxID == 0 {
xcheckuserf(ctx, errors.New("invalid zero mailbox ID"), "getting mailbox")
}
mb := store.Mailbox{ID: mailboxID}
err := tx.Get(&mb)
if err == bstore.ErrAbsent {
mb, err := store.MailboxID(tx, mailboxID)
if err == bstore.ErrAbsent || err == store.ErrMailboxExpunged {
xcheckuserf(ctx, err, "getting mailbox")
}
xcheckf(ctx, err, "getting mailbox")
@ -1010,7 +1010,7 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
// Append message to Sent mailbox, mark original messages as answered/forwarded,
// remove any draft message.
acc.WithRLock(func() {
acc.WithWLock(func() {
var changes []store.Change
metricked := false
@ -1023,9 +1023,9 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
}
}()
var deliveredIDs []int64
var newIDs []int64
defer func() {
for _, id := range deliveredIDs {
for _, id := range newIDs {
p := acc.MessagePath(id)
err := os.Remove(p)
log.Check(err, "removing delivered message on error", slog.String("path", p))
@ -1036,7 +1036,6 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
if m.DraftMessageID > 0 {
nchanges := xops.MessageDeleteTx(ctx, log, tx, acc, []int64{m.DraftMessageID}, &modseq)
changes = append(changes, nchanges...)
// On-disk file is removed after lock.
}
if m.ResponseMessageID > 0 {
@ -1061,12 +1060,11 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
changes = append(changes, rm.ChangeFlags(oflags))
// Update modseq of mailbox of replied/forwarded message.
rmb := store.Mailbox{ID: rm.MailboxID}
err = tx.Get(&rmb)
rmb, err := store.MailboxID(tx, rm.MailboxID)
xcheckf(ctx, err, "get mailbox of replied/forwarded message for modseq update")
rmb.ModSeq = modseq
err = tx.Update(&rmb)
xcheckf(ctx, err, "update modseqo of mailbox of replied/forwarded message")
xcheckf(ctx, err, "update modseq of mailbox of replied/forwarded message")
err = acc.RetrainMessages(ctx, log, tx, []store.Message{rm})
xcheckf(ctx, err, "retraining messages after reply/forward")
@ -1075,8 +1073,8 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
// Move messages from this thread still in this mailbox to the designated Archive
// mailbox.
if m.ArchiveThread {
mbArchive, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Archive", true).Get()
if err == bstore.ErrAbsent {
mbArchive, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).FilterEqual("Archive", true).Get()
if err == bstore.ErrAbsent || err == store.ErrMailboxExpunged {
xcheckuserf(ctx, errors.New("not configured"), "looking up designated archive mailbox")
}
xcheckf(ctx, err, "looking up designated archive mailbox")
@ -1088,14 +1086,15 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
err = q.IDs(&msgIDs)
xcheckf(ctx, err, "listing messages in thread to archive")
if len(msgIDs) > 0 {
nchanges := xops.MessageMoveTx(ctx, log, acc, tx, msgIDs, mbArchive, &modseq)
ids, nchanges := xops.MessageMoveTx(ctx, log, acc, tx, msgIDs, mbArchive, &modseq)
newIDs = append(newIDs, ids...)
changes = append(changes, nchanges...)
}
}
}
sentmb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Sent", true).Get()
if err == bstore.ErrAbsent {
sentmb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).FilterEqual("Sent", true).Get()
if err == bstore.ErrAbsent || err == store.ErrMailboxExpunged {
// There is no mailbox designated as Sent mailbox, so we're done.
return
}
@ -1135,24 +1134,17 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
metricked = true
}
xcheckf(ctx, err, "message submitted to queue, appending message to Sent mailbox")
deliveredIDs = append(deliveredIDs, sentm.ID)
newIDs = append(newIDs, sentm.ID)
err = tx.Update(&sentmb)
xcheckf(ctx, err, "updating sent mailbox for counts")
changes = append(changes, sentm.ChangeAddUID(), sentmb.ChangeCounts())
})
deliveredIDs = nil
newIDs = nil
store.BroadcastChanges(acc, changes)
})
// Remove on-disk file for removed draft message.
if m.DraftMessageID > 0 {
p := acc.MessagePath(m.DraftMessageID)
err := os.Remove(p)
log.Check(err, "removing draft message file")
}
}
// MessageMove moves messages to another mailbox. If the message is already in
@ -1244,9 +1236,6 @@ func (Webmail) MailboxDelete(ctx context.Context, mailboxID int64) {
acc := reqInfo.Account
log := reqInfo.Log
// Messages to remove after having broadcasted the removal of messages.
var removeMessageIDs []int64
acc.WithWLock(func() {
var changes []store.Change
@ -1259,7 +1248,7 @@ func (Webmail) MailboxDelete(ctx context.Context, mailboxID int64) {
var hasChildren bool
var err error
changes, removeMessageIDs, hasChildren, err = acc.MailboxDelete(ctx, log, tx, mb)
changes, hasChildren, err = acc.MailboxDelete(ctx, log, tx, &mb)
if hasChildren {
xcheckuserf(ctx, errors.New("mailbox has children"), "deleting mailbox")
}
@ -1268,12 +1257,6 @@ func (Webmail) MailboxDelete(ctx context.Context, mailboxID int64) {
store.BroadcastChanges(acc, changes)
})
for _, mID := range removeMessageIDs {
p := acc.MessagePath(mID)
err := os.Remove(p)
log.Check(err, "removing message file for mailbox delete", slog.String("path", p))
}
}
// MailboxEmpty empties a mailbox, removing all messages from the mailbox, but not
@ -1283,76 +1266,36 @@ func (Webmail) MailboxEmpty(ctx context.Context, mailboxID int64) {
acc := reqInfo.Account
log := reqInfo.Log
var expunged []store.Message
acc.WithWLock(func() {
var changes []store.Change
xdbwrite(ctx, acc, func(tx *bstore.Tx) {
mb := xmailboxID(ctx, tx, mailboxID)
modseq, err := acc.NextModSeq(tx)
xcheckf(ctx, err, "next modseq")
// Mark messages as expunged.
qm := bstore.QueryTx[store.Message](tx)
qm.FilterNonzero(store.Message{MailboxID: mb.ID})
qm.FilterEqual("Expunged", false)
qm.SortAsc("UID")
qm.Gather(&expunged)
n, err := qm.UpdateNonzero(store.Message{ModSeq: modseq, Expunged: true})
xcheckf(ctx, err, "deleting messages")
l, err := qm.List()
xcheckf(ctx, err, "listing messages to remove")
if n == 0 {
if len(l) == 0 {
xcheckuserf(ctx, errors.New("no messages in mailbox"), "emptying mailbox")
}
mb.ModSeq = modseq
modseq, err := acc.NextModSeq(tx)
xcheckf(ctx, err, "next modseq")
// Remove Recipients.
anyIDs := make([]any, len(expunged))
for i, m := range expunged {
anyIDs[i] = m.ID
}
qmr := bstore.QueryTx[store.Recipient](tx)
qmr.FilterEqual("MessageID", anyIDs...)
_, err = qmr.Delete()
xcheckf(ctx, err, "removing message recipients")
// Adjust mailbox counts, gather UIDs for broadcasted change, prepare for untraining.
var totalSize int64
uids := make([]store.UID, len(expunged))
for i, m := range expunged {
m.Expunged = false // Gather returns updated values.
mb.Sub(m.MailboxCounts())
totalSize += m.Size
uids[i] = m.UID
expunged[i].Junk = false
expunged[i].Notjunk = false
}
chrem, chmbcounts, err := acc.MessageRemove(log, tx, modseq, &mb, store.RemoveOpts{}, l...)
xcheckf(ctx, err, "expunge messages")
changes = append(changes, chrem, chmbcounts)
err = tx.Update(&mb)
xcheckf(ctx, err, "updating mailbox for counts")
err = acc.AddMessageSize(log, tx, -totalSize)
xcheckf(ctx, err, "updating disk usage")
err = acc.RetrainMessages(ctx, log, tx, expunged)
xcheckf(ctx, err, "retraining expunged messages")
chremove := store.ChangeRemoveUIDs{MailboxID: mb.ID, UIDs: uids, ModSeq: modseq}
changes = []store.Change{chremove, mb.ChangeCounts()}
})
store.BroadcastChanges(acc, changes)
})
for _, m := range expunged {
p := acc.MessagePath(m.ID)
err := os.Remove(p)
log.Check(err, "removing message file after emptying mailbox", slog.String("path", p))
}
}
// MailboxRename renames a mailbox, possibly moving it to a new parent. The mailbox
@ -1373,10 +1316,10 @@ func (Webmail) MailboxRename(ctx context.Context, mailboxID int64, newName strin
xdbwrite(ctx, acc, func(tx *bstore.Tx) {
mbsrc := xmailboxID(ctx, tx, mailboxID)
var err error
var isInbox, notExists, alreadyExists bool
var isInbox, alreadyExists bool
var modseq store.ModSeq
changes, isInbox, notExists, alreadyExists, err = acc.MailboxRename(tx, mbsrc, newName, &modseq)
if isInbox || notExists || alreadyExists {
changes, isInbox, alreadyExists, err = acc.MailboxRename(tx, &mbsrc, newName, &modseq)
if isInbox || alreadyExists {
xcheckuserf(ctx, err, "renaming mailbox")
}
xcheckf(ctx, err, "renaming mailbox")
@ -1555,8 +1498,8 @@ func (Webmail) ThreadCollapse(ctx context.Context, messageIDs []int64, collapse
for _, id := range messageIDs {
m := store.Message{ID: id}
err := tx.Get(&m)
if err == bstore.ErrAbsent {
xcheckuserf(ctx, err, "get message")
if err == bstore.ErrAbsent || err == nil && m.Expunged {
xcheckuserf(ctx, bstore.ErrAbsent, "get message")
}
xcheckf(ctx, err, "get message")
threadIDs[m.ThreadID] = struct{}{}
@ -1565,6 +1508,7 @@ func (Webmail) ThreadCollapse(ctx context.Context, messageIDs []int64, collapse
var updated []store.Message
q := bstore.QueryTx[store.Message](tx)
q.FilterEqual("Expunged", false)
q.FilterEqual("ThreadID", slicesAny(maps.Keys(threadIDs))...)
q.FilterNotEqual("ThreadCollapsed", collapse)
q.FilterFn(func(tm store.Message) bool {
@ -1607,8 +1551,8 @@ func (Webmail) ThreadMute(ctx context.Context, messageIDs []int64, mute bool) {
for _, id := range messageIDs {
m := store.Message{ID: id}
err := tx.Get(&m)
if err == bstore.ErrAbsent {
xcheckuserf(ctx, err, "get message")
if err == bstore.ErrAbsent || err == nil && m.Expunged {
xcheckuserf(ctx, bstore.ErrAbsent, "get message")
}
xcheckf(ctx, err, "get message")
threadIDs[m.ThreadID] = struct{}{}
@ -1618,6 +1562,7 @@ func (Webmail) ThreadMute(ctx context.Context, messageIDs []int64, mute bool) {
var updated []store.Message
q := bstore.QueryTx[store.Message](tx)
q.FilterEqual("Expunged", false)
q.FilterEqual("ThreadID", slicesAny(maps.Keys(threadIDs))...)
q.FilterFn(func(tm store.Message) bool {
if tm.ThreadMuted == mute && (!mute || tm.ThreadCollapsed) {

66
webmail/api.json
View File

@ -1601,9 +1601,37 @@
"int64"
]
},
{
"Name": "CreateSeq",
"Docs": "",
"Typewords": [
"ModSeq"
]
},
{
"Name": "ModSeq",
"Docs": "Of last change, or when deleted.",
"Typewords": [
"ModSeq"
]
},
{
"Name": "Expunged",
"Docs": "",
"Typewords": [
"bool"
]
},
{
"Name": "ParentID",
"Docs": "Zero for top-level mailbox.",
"Typewords": [
"int64"
]
},
{
"Name": "Name",
"Docs": "\"Inbox\" is the name for the special IMAP \"INBOX\". Slash separated for hierarchy.",
"Docs": "\"Inbox\" is the name for the special IMAP \"INBOX\". Slash separated for hierarchy. Names must be unique for mailboxes that are not expunged.",
"Typewords": [
"string"
]
@ -1665,20 +1693,6 @@
"string"
]
},
{
"Name": "ModSeq",
"Docs": "ModSeq matches that of last message (including deleted), or changes to mailbox such as after metadata changes.",
"Typewords": [
"ModSeq"
]
},
{
"Name": "CreateSeq",
"Docs": "",
"Typewords": [
"ModSeq"
]
},
{
"Name": "HaveCounts",
"Docs": "Whether MailboxCounts have been initialized.",
@ -2165,14 +2179,14 @@
"Fields": [
{
"Name": "ID",
"Docs": "ID, unchanged over lifetime, determines path to on-disk msg file. Set during deliver.",
"Docs": "ID of the message, determines path to on-disk message file. Set when adding to a mailbox. When a message is moved to another mailbox, the mailbox ID is changed, but for synchronization purposes, a new Message record is inserted (which gets a new ID) with the Expunged field set and the MailboxID and UID copied.",
"Typewords": [
"int64"
]
},
{
"Name": "UID",
"Docs": "UID, for IMAP. Set during deliver.",
"Docs": "UID, for IMAP. Set when adding to mailbox. Strictly increasing values, per mailbox. The UID of a message can never change (though messages can be copied), and the contents of a message/UID also never changes.",
"Typewords": [
"UID"
]
@ -2435,7 +2449,7 @@
},
{
"Name": "ThreadParentIDs",
"Docs": "IDs of parent messages, from closest parent to the root message. Parent messages may be in a different mailbox, or may no longer exist. ThreadParentIDs must never contain the message id itself (a cycle), and parent messages must reference the same ancestors.",
"Docs": "IDs of parent messages, from closest parent to the root message. Parent messages may be in a different mailbox, or may no longer exist. ThreadParentIDs must never contain the message id itself (a cycle), and parent messages must reference the same ancestors. Moving a message to another mailbox keeps the message ID and changes the MailboxID (and UID) of the message, leaving threading parent ids intact.",
"Typewords": [
"[]",
"int64"
@ -2891,6 +2905,14 @@
"Typewords": [
"ModSeq"
]
},
{
"Name": "MsgIDs",
"Docs": "Message.ID, for erasing, order does not necessarily correspond with UIDs!",
"Typewords": [
"[]",
"int64"
]
}
]
},
@ -3214,13 +3236,13 @@
],
"Ints": [
{
"Name": "UID",
"Docs": "IMAP UID.",
"Name": "ModSeq",
"Docs": "ModSeq represents a modseq as stored in the database. ModSeq 0 in the\ndatabase is sent to the client as 1, because modseq 0 is special in IMAP.\nModSeq coming from the client are of type int64.",
"Values": null
},
{
"Name": "ModSeq",
"Docs": "ModSeq represents a modseq as stored in the database. ModSeq 0 in the\ndatabase is sent to the client as 1, because modseq 0 is special in IMAP.\nModSeq coming from the client are of type int64.",
"Name": "UID",
"Docs": "IMAP UID.",
"Values": null
},
{

29
webmail/api.ts
View File

@ -188,7 +188,11 @@ export interface ForwardAttachments {
// Mailbox is collection of messages, e.g. Inbox or Sent.
export interface Mailbox {
ID: number
Name: string // "Inbox" is the name for the special IMAP "INBOX". Slash separated for hierarchy.
CreateSeq: ModSeq
ModSeq: ModSeq // Of last change, or when deleted.
Expunged: boolean
ParentID: number // Zero for top-level mailbox.
Name: string // "Inbox" is the name for the special IMAP "INBOX". Slash separated for hierarchy. Names must be unique for mailboxes that are not expunged.
UIDValidity: number // If UIDs are invalidated, e.g. when renaming a mailbox to a previously existing name, UIDValidity must be changed. Used by IMAP for synchronization.
UIDNext: UID // UID likely to be assigned to next message. Used by IMAP to detect messages delivered to a mailbox.
Archive: boolean
@ -197,8 +201,6 @@ export interface Mailbox {
Sent: boolean
Trash: boolean
Keywords?: string[] | null // Keywords as used in messages. Storing a non-system keyword for a message automatically adds it to this list. Used in the IMAP FLAGS response. Only "atoms" are allowed (IMAP syntax), keywords are case-insensitive, only stored in lower case (for JMAP), sorted.
ModSeq: ModSeq // ModSeq matches that of last message (including deleted), or changes to mailbox such as after metadata changes.
CreateSeq: ModSeq
HaveCounts: boolean // Whether MailboxCounts have been initialized.
Total: number // Total number of messages, excluding \Deleted. For JMAP.
Deleted: number // Number of messages with \Deleted flag. Used for IMAP message count that includes messages with \Deleted.
@ -315,8 +317,8 @@ export interface MessageItem {
// Messages always have a header section, even if empty. Incoming messages without
// header section must get an empty header section added before inserting.
export interface Message {
ID: number // ID, unchanged over lifetime, determines path to on-disk msg file. Set during deliver.
UID: UID // UID, for IMAP. Set during deliver.
ID: number // ID of the message, determines path to on-disk message file. Set when adding to a mailbox. When a message is moved to another mailbox, the mailbox ID is changed, but for synchronization purposes, a new Message record is inserted (which gets a new ID) with the Expunged field set and the MailboxID and UID copied.
UID: UID // UID, for IMAP. Set when adding to mailbox. Strictly increasing values, per mailbox. The UID of a message can never change (though messages can be copied), and the contents of a message/UID also never changes.
MailboxID: number
ModSeq: ModSeq // Modification sequence, for faster syncing with IMAP QRESYNC and JMAP. ModSeq is the last modification. CreateSeq is the Seq the message was inserted, always <= ModSeq. If Expunged is set, the message has been removed and should not be returned to the user. In this case, ModSeq is the Seq where the message is removed, and will never be changed again. We have an index on both ModSeq (for JMAP that synchronizes per account) and MailboxID+ModSeq (for IMAP that synchronizes per mailbox). The index on CreateSeq helps efficiently finding created messages for JMAP. The value of ModSeq is special for IMAP. Messages that existed before ModSeq was added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If we get modseq 1 from a client, the IMAP server will translate it to 0. When we return modseq to clients, we turn 0 into 1.
CreateSeq: ModSeq
@ -353,7 +355,7 @@ export interface Message {
SubjectBase: string // For matching threads in case there is no References/In-Reply-To header. It is lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.
MessageHash?: string | null // Hash of message. For rejects delivery in case there is no Message-ID, only set when delivered as reject.
ThreadID: number // ID of message starting this thread.
ThreadParentIDs?: number[] | null // IDs of parent messages, from closest parent to the root message. Parent messages may be in a different mailbox, or may no longer exist. ThreadParentIDs must never contain the message id itself (a cycle), and parent messages must reference the same ancestors.
ThreadParentIDs?: number[] | null // IDs of parent messages, from closest parent to the root message. Parent messages may be in a different mailbox, or may no longer exist. ThreadParentIDs must never contain the message id itself (a cycle), and parent messages must reference the same ancestors. Moving a message to another mailbox keeps the message ID and changes the MailboxID (and UID) of the message, leaving threading parent ids intact.
ThreadMissingLink: boolean // ThreadMissingLink is true if there is no match with a direct parent. E.g. first ID in ThreadParentIDs is not the direct ancestor (an intermediate message may have been deleted), or subject-based matching was done.
ThreadMuted: boolean // If set, newly delivered child messages are automatically marked as read. This field is copied to new child messages. Changes are propagated to the webmail client.
ThreadCollapsed: boolean // If set, this (sub)thread is collapsed in the webmail client, for threading mode "on" (mode "unread" ignores it). This field is copied to new child message. Changes are propagated to the webmail client.
@ -439,6 +441,7 @@ export interface ChangeMsgRemove {
MailboxID: number
UIDs?: UID[] | null // Must be in increasing UID order, for IMAP.
ModSeq: ModSeq
MsgIDs?: number[] | null // Message.ID, for erasing, order does not necessarily correspond with UIDs!
}
// ChangeMsgFlags updates flags for one message.
@ -517,14 +520,14 @@ export interface ChangeMailboxKeywords {
Keywords?: string[] | null
}
// IMAP UID.
export type UID = number
// ModSeq represents a modseq as stored in the database. ModSeq 0 in the
// database is sent to the client as 1, because modseq 0 is special in IMAP.
// ModSeq coming from the client are of type int64.
export type ModSeq = number
// IMAP UID.
export type UID = number
// Validation of "message From" domain.
export enum Validation {
ValidationUnknown = 0,
@ -613,7 +616,7 @@ export const types: TypenameMap = {
"SubmitMessage": {"Name":"SubmitMessage","Docs":"","Fields":[{"Name":"From","Docs":"","Typewords":["string"]},{"Name":"To","Docs":"","Typewords":["[]","string"]},{"Name":"Cc","Docs":"","Typewords":["[]","string"]},{"Name":"Bcc","Docs":"","Typewords":["[]","string"]},{"Name":"ReplyTo","Docs":"","Typewords":["string"]},{"Name":"Subject","Docs":"","Typewords":["string"]},{"Name":"TextBody","Docs":"","Typewords":["string"]},{"Name":"Attachments","Docs":"","Typewords":["[]","File"]},{"Name":"ForwardAttachments","Docs":"","Typewords":["ForwardAttachments"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"ResponseMessageID","Docs":"","Typewords":["int64"]},{"Name":"UserAgent","Docs":"","Typewords":["string"]},{"Name":"RequireTLS","Docs":"","Typewords":["nullable","bool"]},{"Name":"FutureRelease","Docs":"","Typewords":["nullable","timestamp"]},{"Name":"ArchiveThread","Docs":"","Typewords":["bool"]},{"Name":"ArchiveReferenceMailboxID","Docs":"","Typewords":["int64"]},{"Name":"DraftMessageID","Docs":"","Typewords":["int64"]}]},
"File": {"Name":"File","Docs":"","Fields":[{"Name":"Filename","Docs":"","Typewords":["string"]},{"Name":"DataURI","Docs":"","Typewords":["string"]}]},
"ForwardAttachments": {"Name":"ForwardAttachments","Docs":"","Fields":[{"Name":"MessageID","Docs":"","Typewords":["int64"]},{"Name":"Paths","Docs":"","Typewords":["[]","[]","int32"]}]},
"Mailbox": {"Name":"Mailbox","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"Name","Docs":"","Typewords":["string"]},{"Name":"UIDValidity","Docs":"","Typewords":["uint32"]},{"Name":"UIDNext","Docs":"","Typewords":["UID"]},{"Name":"Archive","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Sent","Docs":"","Typewords":["bool"]},{"Name":"Trash","Docs":"","Typewords":["bool"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"CreateSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"HaveCounts","Docs":"","Typewords":["bool"]},{"Name":"Total","Docs":"","Typewords":["int64"]},{"Name":"Deleted","Docs":"","Typewords":["int64"]},{"Name":"Unread","Docs":"","Typewords":["int64"]},{"Name":"Unseen","Docs":"","Typewords":["int64"]},{"Name":"Size","Docs":"","Typewords":["int64"]}]},
"Mailbox": {"Name":"Mailbox","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"CreateSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Expunged","Docs":"","Typewords":["bool"]},{"Name":"ParentID","Docs":"","Typewords":["int64"]},{"Name":"Name","Docs":"","Typewords":["string"]},{"Name":"UIDValidity","Docs":"","Typewords":["uint32"]},{"Name":"UIDNext","Docs":"","Typewords":["UID"]},{"Name":"Archive","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Sent","Docs":"","Typewords":["bool"]},{"Name":"Trash","Docs":"","Typewords":["bool"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"HaveCounts","Docs":"","Typewords":["bool"]},{"Name":"Total","Docs":"","Typewords":["int64"]},{"Name":"Deleted","Docs":"","Typewords":["int64"]},{"Name":"Unread","Docs":"","Typewords":["int64"]},{"Name":"Unseen","Docs":"","Typewords":["int64"]},{"Name":"Size","Docs":"","Typewords":["int64"]}]},
"RecipientSecurity": {"Name":"RecipientSecurity","Docs":"","Fields":[{"Name":"STARTTLS","Docs":"","Typewords":["SecurityResult"]},{"Name":"MTASTS","Docs":"","Typewords":["SecurityResult"]},{"Name":"DNSSEC","Docs":"","Typewords":["SecurityResult"]},{"Name":"DANE","Docs":"","Typewords":["SecurityResult"]},{"Name":"RequireTLS","Docs":"","Typewords":["SecurityResult"]}]},
"Settings": {"Name":"Settings","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["uint8"]},{"Name":"Signature","Docs":"","Typewords":["string"]},{"Name":"Quoting","Docs":"","Typewords":["Quoting"]},{"Name":"ShowAddressSecurity","Docs":"","Typewords":["bool"]},{"Name":"ShowHTML","Docs":"","Typewords":["bool"]},{"Name":"NoShowShortcuts","Docs":"","Typewords":["bool"]},{"Name":"ShowHeaders","Docs":"","Typewords":["[]","string"]}]},
"Ruleset": {"Name":"Ruleset","Docs":"","Fields":[{"Name":"SMTPMailFromRegexp","Docs":"","Typewords":["string"]},{"Name":"MsgFromRegexp","Docs":"","Typewords":["string"]},{"Name":"VerifiedDomain","Docs":"","Typewords":["string"]},{"Name":"HeadersRegexp","Docs":"","Typewords":["{}","string"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"ListAllowDomain","Docs":"","Typewords":["string"]},{"Name":"AcceptRejectsToMailbox","Docs":"","Typewords":["string"]},{"Name":"Mailbox","Docs":"","Typewords":["string"]},{"Name":"Comment","Docs":"","Typewords":["string"]},{"Name":"VerifiedDNSDomain","Docs":"","Typewords":["Domain"]},{"Name":"ListAllowDNSDomain","Docs":"","Typewords":["Domain"]}]},
@ -629,7 +632,7 @@ export const types: TypenameMap = {
"EventViewChanges": {"Name":"EventViewChanges","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"Changes","Docs":"","Typewords":["[]","[]","any"]}]},
"ChangeMsgAdd": {"Name":"ChangeMsgAdd","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Flags","Docs":"","Typewords":["Flags"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"MessageItems","Docs":"","Typewords":["[]","MessageItem"]}]},
"Flags": {"Name":"Flags","Docs":"","Fields":[{"Name":"Seen","Docs":"","Typewords":["bool"]},{"Name":"Answered","Docs":"","Typewords":["bool"]},{"Name":"Flagged","Docs":"","Typewords":["bool"]},{"Name":"Forwarded","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Notjunk","Docs":"","Typewords":["bool"]},{"Name":"Deleted","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Phishing","Docs":"","Typewords":["bool"]},{"Name":"MDNSent","Docs":"","Typewords":["bool"]}]},
"ChangeMsgRemove": {"Name":"ChangeMsgRemove","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UIDs","Docs":"","Typewords":["[]","UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]}]},
"ChangeMsgRemove": {"Name":"ChangeMsgRemove","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UIDs","Docs":"","Typewords":["[]","UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"MsgIDs","Docs":"","Typewords":["[]","int64"]}]},
"ChangeMsgFlags": {"Name":"ChangeMsgFlags","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Mask","Docs":"","Typewords":["Flags"]},{"Name":"Flags","Docs":"","Typewords":["Flags"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]}]},
"ChangeMsgThread": {"Name":"ChangeMsgThread","Docs":"","Fields":[{"Name":"MessageIDs","Docs":"","Typewords":["[]","int64"]},{"Name":"Muted","Docs":"","Typewords":["bool"]},{"Name":"Collapsed","Docs":"","Typewords":["bool"]}]},
"ChangeMailboxRemove": {"Name":"ChangeMailboxRemove","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"Name","Docs":"","Typewords":["string"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]}]},
@ -639,8 +642,8 @@ export const types: TypenameMap = {
"ChangeMailboxSpecialUse": {"Name":"ChangeMailboxSpecialUse","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"MailboxName","Docs":"","Typewords":["string"]},{"Name":"SpecialUse","Docs":"","Typewords":["SpecialUse"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]}]},
"SpecialUse": {"Name":"SpecialUse","Docs":"","Fields":[{"Name":"Archive","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Sent","Docs":"","Typewords":["bool"]},{"Name":"Trash","Docs":"","Typewords":["bool"]}]},
"ChangeMailboxKeywords": {"Name":"ChangeMailboxKeywords","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"MailboxName","Docs":"","Typewords":["string"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]}]},
"UID": {"Name":"UID","Docs":"","Values":null},
"ModSeq": {"Name":"ModSeq","Docs":"","Values":null},
"UID": {"Name":"UID","Docs":"","Values":null},
"Validation": {"Name":"Validation","Docs":"","Values":[{"Name":"ValidationUnknown","Value":0,"Docs":""},{"Name":"ValidationStrict","Value":1,"Docs":""},{"Name":"ValidationDMARC","Value":2,"Docs":""},{"Name":"ValidationRelaxed","Value":3,"Docs":""},{"Name":"ValidationPass","Value":4,"Docs":""},{"Name":"ValidationNeutral","Value":5,"Docs":""},{"Name":"ValidationTemperror","Value":6,"Docs":""},{"Name":"ValidationPermerror","Value":7,"Docs":""},{"Name":"ValidationFail","Value":8,"Docs":""},{"Name":"ValidationSoftfail","Value":9,"Docs":""},{"Name":"ValidationNone","Value":10,"Docs":""}]},
"CSRFToken": {"Name":"CSRFToken","Docs":"","Values":null},
"ThreadMode": {"Name":"ThreadMode","Docs":"","Values":[{"Name":"ThreadOff","Value":"off","Docs":""},{"Name":"ThreadOn","Value":"on","Docs":""},{"Name":"ThreadUnread","Value":"unread","Docs":""}]},
@ -694,8 +697,8 @@ export const parser = {
ChangeMailboxSpecialUse: (v: any) => parse("ChangeMailboxSpecialUse", v) as ChangeMailboxSpecialUse,
SpecialUse: (v: any) => parse("SpecialUse", v) as SpecialUse,
ChangeMailboxKeywords: (v: any) => parse("ChangeMailboxKeywords", v) as ChangeMailboxKeywords,
UID: (v: any) => parse("UID", v) as UID,
ModSeq: (v: any) => parse("ModSeq", v) as ModSeq,
UID: (v: any) => parse("UID", v) as UID,
Validation: (v: any) => parse("Validation", v) as Validation,
CSRFToken: (v: any) => parse("CSRFToken", v) as CSRFToken,
ThreadMode: (v: any) => parse("ThreadMode", v) as ThreadMode,

6
webmail/api_test.go
View File

@ -78,7 +78,7 @@ func TestAPI(t *testing.T) {
tcheck(t, err, "mtastsdb close")
err = acc.Close()
pkglog.Check(err, "closing account")
acc.CheckClosed()
acc.WaitClosed()
}()
var zerom store.Message
@ -206,7 +206,7 @@ func TestAPI(t *testing.T) {
var inbox, archive, sent, drafts, testbox1 store.Mailbox
err = acc.DB.Read(ctx, func(tx *bstore.Tx) error {
get := func(k string, v any) store.Mailbox {
mb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual(k, v).Get()
mb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).FilterEqual(k, v).Get()
tcheck(t, err, "get special-use mailbox")
return mb
}
@ -276,7 +276,7 @@ func TestAPI(t *testing.T) {
tneedError(t, func() { api.ParsedMessage(ctx, testbox1Alt.ID) }) // Message was removed and no longer exists.
api.MailboxCreate(ctx, "Testbox1")
testbox1, err = bstore.QueryDB[store.Mailbox](ctx, acc.DB).FilterEqual("Name", "Testbox1").Get()
testbox1, err = bstore.QueryDB[store.Mailbox](ctx, acc.DB).FilterEqual("Expunged", false).FilterEqual("Name", "Testbox1").Get()
tcheck(t, err, "get testbox1")
tdeliver(t, acc, testbox1Alt)

8
webmail/msg.js
View File

@ -309,7 +309,7 @@ var api;
"SubmitMessage": { "Name": "SubmitMessage", "Docs": "", "Fields": [{ "Name": "From", "Docs": "", "Typewords": ["string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Cc", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Bcc", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "TextBody", "Docs": "", "Typewords": ["string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "File"] }, { "Name": "ForwardAttachments", "Docs": "", "Typewords": ["ForwardAttachments"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ResponseMessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UserAgent", "Docs": "", "Typewords": ["string"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "FutureRelease", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "ArchiveThread", "Docs": "", "Typewords": ["bool"] }, { "Name": "ArchiveReferenceMailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "DraftMessageID", "Docs": "", "Typewords": ["int64"] }] },
"File": { "Name": "File", "Docs": "", "Fields": [{ "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "DataURI", "Docs": "", "Typewords": ["string"] }] },
"ForwardAttachments": { "Name": "ForwardAttachments", "Docs": "", "Fields": [{ "Name": "MessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Paths", "Docs": "", "Typewords": ["[]", "[]", "int32"] }] },
"Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] },
"Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "ParentID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] },
"RecipientSecurity": { "Name": "RecipientSecurity", "Docs": "", "Fields": [{ "Name": "STARTTLS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "MTASTS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DNSSEC", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DANE", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["SecurityResult"] }] },
"Settings": { "Name": "Settings", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["uint8"] }, { "Name": "Signature", "Docs": "", "Typewords": ["string"] }, { "Name": "Quoting", "Docs": "", "Typewords": ["Quoting"] }, { "Name": "ShowAddressSecurity", "Docs": "", "Typewords": ["bool"] }, { "Name": "ShowHTML", "Docs": "", "Typewords": ["bool"] }, { "Name": "NoShowShortcuts", "Docs": "", "Typewords": ["bool"] }, { "Name": "ShowHeaders", "Docs": "", "Typewords": ["[]", "string"] }] },
"Ruleset": { "Name": "Ruleset", "Docs": "", "Fields": [{ "Name": "SMTPMailFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "HeadersRegexp", "Docs": "", "Typewords": ["{}", "string"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ListAllowDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "AcceptRejectsToMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Comment", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDNSDomain", "Docs": "", "Typewords": ["Domain"] }, { "Name": "ListAllowDNSDomain", "Docs": "", "Typewords": ["Domain"] }] },
@ -325,7 +325,7 @@ var api;
"EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] },
"ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] },
"Flags": { "Name": "Flags", "Docs": "", "Fields": [{ "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }] },
"ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
"ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "MsgIDs", "Docs": "", "Typewords": ["[]", "int64"] }] },
"ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] },
"ChangeMsgThread": { "Name": "ChangeMsgThread", "Docs": "", "Fields": [{ "Name": "MessageIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "Muted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Collapsed", "Docs": "", "Typewords": ["bool"] }] },
"ChangeMailboxRemove": { "Name": "ChangeMailboxRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
@ -335,8 +335,8 @@ var api;
"ChangeMailboxSpecialUse": { "Name": "ChangeMailboxSpecialUse", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "SpecialUse", "Docs": "", "Typewords": ["SpecialUse"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
"SpecialUse": { "Name": "SpecialUse", "Docs": "", "Fields": [{ "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }] },
"ChangeMailboxKeywords": { "Name": "ChangeMailboxKeywords", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] },
"UID": { "Name": "UID", "Docs": "", "Values": null },
"ModSeq": { "Name": "ModSeq", "Docs": "", "Values": null },
"UID": { "Name": "UID", "Docs": "", "Values": null },
"Validation": { "Name": "Validation", "Docs": "", "Values": [{ "Name": "ValidationUnknown", "Value": 0, "Docs": "" }, { "Name": "ValidationStrict", "Value": 1, "Docs": "" }, { "Name": "ValidationDMARC", "Value": 2, "Docs": "" }, { "Name": "ValidationRelaxed", "Value": 3, "Docs": "" }, { "Name": "ValidationPass", "Value": 4, "Docs": "" }, { "Name": "ValidationNeutral", "Value": 5, "Docs": "" }, { "Name": "ValidationTemperror", "Value": 6, "Docs": "" }, { "Name": "ValidationPermerror", "Value": 7, "Docs": "" }, { "Name": "ValidationFail", "Value": 8, "Docs": "" }, { "Name": "ValidationSoftfail", "Value": 9, "Docs": "" }, { "Name": "ValidationNone", "Value": 10, "Docs": "" }] },
"CSRFToken": { "Name": "CSRFToken", "Docs": "", "Values": null },
"ThreadMode": { "Name": "ThreadMode", "Docs": "", "Values": [{ "Name": "ThreadOff", "Value": "off", "Docs": "" }, { "Name": "ThreadOn", "Value": "on", "Docs": "" }, { "Name": "ThreadUnread", "Value": "unread", "Docs": "" }] },
@ -389,8 +389,8 @@ var api;
ChangeMailboxSpecialUse: (v) => api.parse("ChangeMailboxSpecialUse", v),
SpecialUse: (v) => api.parse("SpecialUse", v),
ChangeMailboxKeywords: (v) => api.parse("ChangeMailboxKeywords", v),
UID: (v) => api.parse("UID", v),
ModSeq: (v) => api.parse("ModSeq", v),
UID: (v) => api.parse("UID", v),
Validation: (v) => api.parse("Validation", v),
CSRFToken: (v) => api.parse("CSRFToken", v),
ThreadMode: (v) => api.parse("ThreadMode", v),

8
webmail/text.js
View File

@ -309,7 +309,7 @@ var api;
"SubmitMessage": { "Name": "SubmitMessage", "Docs": "", "Fields": [{ "Name": "From", "Docs": "", "Typewords": ["string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Cc", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Bcc", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "TextBody", "Docs": "", "Typewords": ["string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "File"] }, { "Name": "ForwardAttachments", "Docs": "", "Typewords": ["ForwardAttachments"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ResponseMessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UserAgent", "Docs": "", "Typewords": ["string"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "FutureRelease", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "ArchiveThread", "Docs": "", "Typewords": ["bool"] }, { "Name": "ArchiveReferenceMailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "DraftMessageID", "Docs": "", "Typewords": ["int64"] }] },
"File": { "Name": "File", "Docs": "", "Fields": [{ "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "DataURI", "Docs": "", "Typewords": ["string"] }] },
"ForwardAttachments": { "Name": "ForwardAttachments", "Docs": "", "Fields": [{ "Name": "MessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Paths", "Docs": "", "Typewords": ["[]", "[]", "int32"] }] },
"Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] },
"Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "ParentID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] },
"RecipientSecurity": { "Name": "RecipientSecurity", "Docs": "", "Fields": [{ "Name": "STARTTLS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "MTASTS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DNSSEC", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DANE", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["SecurityResult"] }] },
"Settings": { "Name": "Settings", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["uint8"] }, { "Name": "Signature", "Docs": "", "Typewords": ["string"] }, { "Name": "Quoting", "Docs": "", "Typewords": ["Quoting"] }, { "Name": "ShowAddressSecurity", "Docs": "", "Typewords": ["bool"] }, { "Name": "ShowHTML", "Docs": "", "Typewords": ["bool"] }, { "Name": "NoShowShortcuts", "Docs": "", "Typewords": ["bool"] }, { "Name": "ShowHeaders", "Docs": "", "Typewords": ["[]", "string"] }] },
"Ruleset": { "Name": "Ruleset", "Docs": "", "Fields": [{ "Name": "SMTPMailFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "HeadersRegexp", "Docs": "", "Typewords": ["{}", "string"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ListAllowDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "AcceptRejectsToMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Comment", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDNSDomain", "Docs": "", "Typewords": ["Domain"] }, { "Name": "ListAllowDNSDomain", "Docs": "", "Typewords": ["Domain"] }] },
@ -325,7 +325,7 @@ var api;
"EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] },
"ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] },
"Flags": { "Name": "Flags", "Docs": "", "Fields": [{ "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }] },
"ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
"ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "MsgIDs", "Docs": "", "Typewords": ["[]", "int64"] }] },
"ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] },
"ChangeMsgThread": { "Name": "ChangeMsgThread", "Docs": "", "Fields": [{ "Name": "MessageIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "Muted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Collapsed", "Docs": "", "Typewords": ["bool"] }] },
"ChangeMailboxRemove": { "Name": "ChangeMailboxRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
@ -335,8 +335,8 @@ var api;
"ChangeMailboxSpecialUse": { "Name": "ChangeMailboxSpecialUse", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "SpecialUse", "Docs": "", "Typewords": ["SpecialUse"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
"SpecialUse": { "Name": "SpecialUse", "Docs": "", "Fields": [{ "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }] },
"ChangeMailboxKeywords": { "Name": "ChangeMailboxKeywords", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] },
"UID": { "Name": "UID", "Docs": "", "Values": null },
"ModSeq": { "Name": "ModSeq", "Docs": "", "Values": null },
"UID": { "Name": "UID", "Docs": "", "Values": null },
"Validation": { "Name": "Validation", "Docs": "", "Values": [{ "Name": "ValidationUnknown", "Value": 0, "Docs": "" }, { "Name": "ValidationStrict", "Value": 1, "Docs": "" }, { "Name": "ValidationDMARC", "Value": 2, "Docs": "" }, { "Name": "ValidationRelaxed", "Value": 3, "Docs": "" }, { "Name": "ValidationPass", "Value": 4, "Docs": "" }, { "Name": "ValidationNeutral", "Value": 5, "Docs": "" }, { "Name": "ValidationTemperror", "Value": 6, "Docs": "" }, { "Name": "ValidationPermerror", "Value": 7, "Docs": "" }, { "Name": "ValidationFail", "Value": 8, "Docs": "" }, { "Name": "ValidationSoftfail", "Value": 9, "Docs": "" }, { "Name": "ValidationNone", "Value": 10, "Docs": "" }] },
"CSRFToken": { "Name": "CSRFToken", "Docs": "", "Values": null },
"ThreadMode": { "Name": "ThreadMode", "Docs": "", "Values": [{ "Name": "ThreadOff", "Value": "off", "Docs": "" }, { "Name": "ThreadOn", "Value": "on", "Docs": "" }, { "Name": "ThreadUnread", "Value": "unread", "Docs": "" }] },
@ -389,8 +389,8 @@ var api;
ChangeMailboxSpecialUse: (v) => api.parse("ChangeMailboxSpecialUse", v),
SpecialUse: (v) => api.parse("SpecialUse", v),
ChangeMailboxKeywords: (v) => api.parse("ChangeMailboxKeywords", v),
UID: (v) => api.parse("UID", v),
ModSeq: (v) => api.parse("ModSeq", v),
UID: (v) => api.parse("UID", v),
Validation: (v) => api.parse("Validation", v),
CSRFToken: (v) => api.parse("CSRFToken", v),
ThreadMode: (v) => api.parse("ThreadMode", v),

11
webmail/view.go
View File

@ -708,7 +708,7 @@ func serveEvents(ctx context.Context, log mlog.Log, accountPath string, w http.R
qtx, err = acc.DB.Begin(reqctx, false)
xcheckf(ctx, err, "begin transaction")
mbl, err = bstore.QueryTx[store.Mailbox](qtx).List()
mbl, err = bstore.QueryTx[store.Mailbox](qtx).FilterEqual("Expunged", false).List()
xcheckf(ctx, err, "list mailboxes")
err = qtx.Get(&settings)
@ -916,6 +916,8 @@ func serveEvents(ctx context.Context, log mlog.Log, accountPath string, w http.R
}
case store.ChangeRemoveUIDs:
comm.RemovalSeen(c)
// We may send changes for uids the client doesn't know, that's fine.
changedUIDs := xchangedUIDs(c.MailboxID, c.UIDs, true)
if len(changedUIDs) == 0 {
@ -1126,7 +1128,7 @@ func xprepareMailboxIDs(ctx context.Context, tx *bstore.Tx, f Filter, rejectsMai
if f.MailboxID == -1 {
matchMailboxes = false
// Add the trash, junk and account rejects mailbox.
err := bstore.QueryTx[store.Mailbox](tx).ForEach(func(mb store.Mailbox) error {
err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb store.Mailbox) error {
if mb.Trash || mb.Junk || mb.Name == rejectsMailbox {
mailboxPrefixes = append(mailboxPrefixes, mb.Name+"/")
mailboxIDs[mb.ID] = true
@ -1135,8 +1137,7 @@ func xprepareMailboxIDs(ctx context.Context, tx *bstore.Tx, f Filter, rejectsMai
})
xcheckf(ctx, err, "finding trash/junk/rejects mailbox")
} else if f.MailboxID > 0 {
mb := store.Mailbox{ID: f.MailboxID}
err := tx.Get(&mb)
mb, err := store.MailboxID(tx, f.MailboxID)
xcheckf(ctx, err, "get mailbox")
mailboxIDs[f.MailboxID] = true
mailboxPrefixes = []string{mb.Name + "/"}
@ -1151,7 +1152,7 @@ func xgatherMailboxIDs(ctx context.Context, tx *bstore.Tx, mailboxIDs map[int64]
if len(mailboxPrefixes) == 0 {
return
}
err := bstore.QueryTx[store.Mailbox](tx).ForEach(func(mb store.Mailbox) error {
err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb store.Mailbox) error {
for _, p := range mailboxPrefixes {
if strings.HasPrefix(mb.Name, p) {
mailboxIDs[mb.ID] = true

2
webmail/view_test.go
View File

@ -45,7 +45,7 @@ func TestView(t *testing.T) {
defer func() {
err := acc.Close()
pkglog.Check(err, "closing account")
acc.CheckClosed()
acc.WaitClosed()
}()
api := Webmail{maxMessageSize: 1024 * 1024, cookiePath: "/"}

2
webmail/webmail.go
View File

@ -407,6 +407,8 @@ func handle(apiHandler http.Handler, isForwarded bool, accountPath string, w htt
err = acc.DB.Read(ctx, func(tx *bstore.Tx) error {
if err := tx.Get(&m); err != nil {
return err
} else if m.Expunged {
return fmt.Errorf("message was removed")
}
s := store.Settings{ID: 1}
if err := tx.Get(&s); err != nil {

8
webmail/webmail.js
View File

@ -309,7 +309,7 @@ var api;
"SubmitMessage": { "Name": "SubmitMessage", "Docs": "", "Fields": [{ "Name": "From", "Docs": "", "Typewords": ["string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Cc", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Bcc", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "TextBody", "Docs": "", "Typewords": ["string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "File"] }, { "Name": "ForwardAttachments", "Docs": "", "Typewords": ["ForwardAttachments"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ResponseMessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UserAgent", "Docs": "", "Typewords": ["string"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "FutureRelease", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "ArchiveThread", "Docs": "", "Typewords": ["bool"] }, { "Name": "ArchiveReferenceMailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "DraftMessageID", "Docs": "", "Typewords": ["int64"] }] },
"File": { "Name": "File", "Docs": "", "Fields": [{ "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "DataURI", "Docs": "", "Typewords": ["string"] }] },
"ForwardAttachments": { "Name": "ForwardAttachments", "Docs": "", "Fields": [{ "Name": "MessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Paths", "Docs": "", "Typewords": ["[]", "[]", "int32"] }] },
"Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] },
"Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "ParentID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] },
"RecipientSecurity": { "Name": "RecipientSecurity", "Docs": "", "Fields": [{ "Name": "STARTTLS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "MTASTS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DNSSEC", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DANE", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["SecurityResult"] }] },
"Settings": { "Name": "Settings", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["uint8"] }, { "Name": "Signature", "Docs": "", "Typewords": ["string"] }, { "Name": "Quoting", "Docs": "", "Typewords": ["Quoting"] }, { "Name": "ShowAddressSecurity", "Docs": "", "Typewords": ["bool"] }, { "Name": "ShowHTML", "Docs": "", "Typewords": ["bool"] }, { "Name": "NoShowShortcuts", "Docs": "", "Typewords": ["bool"] }, { "Name": "ShowHeaders", "Docs": "", "Typewords": ["[]", "string"] }] },
"Ruleset": { "Name": "Ruleset", "Docs": "", "Fields": [{ "Name": "SMTPMailFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "HeadersRegexp", "Docs": "", "Typewords": ["{}", "string"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ListAllowDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "AcceptRejectsToMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Comment", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDNSDomain", "Docs": "", "Typewords": ["Domain"] }, { "Name": "ListAllowDNSDomain", "Docs": "", "Typewords": ["Domain"] }] },
@ -325,7 +325,7 @@ var api;
"EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] },
"ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] },
"Flags": { "Name": "Flags", "Docs": "", "Fields": [{ "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }] },
"ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
"ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "MsgIDs", "Docs": "", "Typewords": ["[]", "int64"] }] },
"ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] },
"ChangeMsgThread": { "Name": "ChangeMsgThread", "Docs": "", "Fields": [{ "Name": "MessageIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "Muted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Collapsed", "Docs": "", "Typewords": ["bool"] }] },
"ChangeMailboxRemove": { "Name": "ChangeMailboxRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
@ -335,8 +335,8 @@ var api;
"ChangeMailboxSpecialUse": { "Name": "ChangeMailboxSpecialUse", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "SpecialUse", "Docs": "", "Typewords": ["SpecialUse"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
"SpecialUse": { "Name": "SpecialUse", "Docs": "", "Fields": [{ "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }] },
"ChangeMailboxKeywords": { "Name": "ChangeMailboxKeywords", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] },
"UID": { "Name": "UID", "Docs": "", "Values": null },
"ModSeq": { "Name": "ModSeq", "Docs": "", "Values": null },
"UID": { "Name": "UID", "Docs": "", "Values": null },
"Validation": { "Name": "Validation", "Docs": "", "Values": [{ "Name": "ValidationUnknown", "Value": 0, "Docs": "" }, { "Name": "ValidationStrict", "Value": 1, "Docs": "" }, { "Name": "ValidationDMARC", "Value": 2, "Docs": "" }, { "Name": "ValidationRelaxed", "Value": 3, "Docs": "" }, { "Name": "ValidationPass", "Value": 4, "Docs": "" }, { "Name": "ValidationNeutral", "Value": 5, "Docs": "" }, { "Name": "ValidationTemperror", "Value": 6, "Docs": "" }, { "Name": "ValidationPermerror", "Value": 7, "Docs": "" }, { "Name": "ValidationFail", "Value": 8, "Docs": "" }, { "Name": "ValidationSoftfail", "Value": 9, "Docs": "" }, { "Name": "ValidationNone", "Value": 10, "Docs": "" }] },
"CSRFToken": { "Name": "CSRFToken", "Docs": "", "Values": null },
"ThreadMode": { "Name": "ThreadMode", "Docs": "", "Values": [{ "Name": "ThreadOff", "Value": "off", "Docs": "" }, { "Name": "ThreadOn", "Value": "on", "Docs": "" }, { "Name": "ThreadUnread", "Value": "unread", "Docs": "" }] },
@ -389,8 +389,8 @@ var api;
ChangeMailboxSpecialUse: (v) => api.parse("ChangeMailboxSpecialUse", v),
SpecialUse: (v) => api.parse("SpecialUse", v),
ChangeMailboxKeywords: (v) => api.parse("ChangeMailboxKeywords", v),
UID: (v) => api.parse("UID", v),
ModSeq: (v) => api.parse("ModSeq", v),
UID: (v) => api.parse("UID", v),
Validation: (v) => api.parse("Validation", v),
CSRFToken: (v) => api.parse("CSRFToken", v),
ThreadMode: (v) => api.parse("ThreadMode", v),

327
webops/xops.go
View File

@ -6,16 +6,18 @@ import (
"errors"
"fmt"
"io"
"log/slog"
"os"
"slices"
"sort"
"time"
"golang.org/x/exp/maps"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/junk"
"github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/store"
)
@ -31,9 +33,8 @@ func (x XOps) mailboxID(ctx context.Context, tx *bstore.Tx, mailboxID int64) sto
if mailboxID == 0 {
x.Checkuserf(ctx, errors.New("invalid zero mailbox ID"), "getting mailbox")
}
mb := store.Mailbox{ID: mailboxID}
err := tx.Get(&mb)
if err == bstore.ErrAbsent {
mb, err := store.MailboxID(tx, mailboxID)
if err == bstore.ErrAbsent || err == store.ErrMailboxExpunged {
x.Checkuserf(ctx, err, "getting mailbox")
}
x.Checkf(ctx, err, "getting mailbox")
@ -67,25 +68,32 @@ func (x XOps) MessageDelete(ctx context.Context, log mlog.Log, acc *store.Accoun
store.BroadcastChanges(acc, changes)
})
for _, mID := range messageIDs {
p := acc.MessagePath(mID)
err := os.Remove(p)
log.Check(err, "removing message file for expunge")
}
}
func (x XOps) MessageDeleteTx(ctx context.Context, log mlog.Log, tx *bstore.Tx, acc *store.Account, messageIDs []int64, modseq *store.ModSeq) []store.Change {
removeChanges := map[int64]store.ChangeRemoveUIDs{}
changes := make([]store.Change, 0, len(messageIDs)+1) // n remove, 1 mailbox counts
changes := make([]store.Change, 0, 1+1) // 1 remove, 1 mailbox counts, optimistic that all messages are in 1 mailbox.
var jf *junk.Filter
defer func() {
if jf != nil {
err := jf.CloseDiscard()
log.Check(err, "close junk filter")
}
}()
conf, _ := acc.Conf()
var mb store.Mailbox
remove := make([]store.Message, 0, len(messageIDs))
var changeRemoveUIDs store.ChangeRemoveUIDs
xflushMailbox := func() {
err := tx.Update(&mb)
x.Checkf(ctx, err, "updating mailbox counts")
slices.Sort(changeRemoveUIDs.UIDs)
changes = append(changes, mb.ChangeCounts(), changeRemoveUIDs)
}
var totalSize int64
for _, mid := range messageIDs {
m := x.messageID(ctx, tx, mid)
totalSize += m.Size
for _, id := range messageIDs {
m := x.messageID(ctx, tx, id)
if *modseq == 0 {
var err error
@ -95,58 +103,33 @@ func (x XOps) MessageDeleteTx(ctx context.Context, log mlog.Log, tx *bstore.Tx,
if m.MailboxID != mb.ID {
if mb.ID != 0 {
mb.ModSeq = *modseq
err := tx.Update(&mb)
x.Checkf(ctx, err, "updating mailbox counts")
changes = append(changes, mb.ChangeCounts())
xflushMailbox()
}
mb = x.mailboxID(ctx, tx, m.MailboxID)
}
qmr := bstore.QueryTx[store.Recipient](tx)
qmr.FilterEqual("MessageID", m.ID)
_, err := qmr.Delete()
x.Checkf(ctx, err, "removing message recipients")
mb.Sub(m.MailboxCounts())
m.Expunged = true
m.ModSeq = *modseq
err = tx.Update(&m)
x.Checkf(ctx, err, "marking message as expunged")
ch := removeChanges[m.MailboxID]
ch.UIDs = append(ch.UIDs, m.UID)
ch.MailboxID = m.MailboxID
ch.ModSeq = *modseq
removeChanges[m.MailboxID] = ch
remove = append(remove, m)
}
if mb.ID != 0 {
mb.ModSeq = *modseq
err := tx.Update(&mb)
x.Checkf(ctx, err, "updating count in mailbox")
changes = append(changes, mb.ChangeCounts())
changeRemoveUIDs = store.ChangeRemoveUIDs{MailboxID: mb.ID, ModSeq: *modseq}
}
err := acc.AddMessageSize(log, tx, -totalSize)
x.Checkf(ctx, err, "updating disk usage")
// Mark removed messages as not needing training, then retrain them, so if they
// were trained, they get untrained.
for i := range remove {
remove[i].Junk = false
remove[i].Notjunk = false
if m.Junk != m.Notjunk && jf == nil && conf.JunkFilter != nil {
var err error
jf, _, err = acc.OpenJunkFilter(ctx, log)
x.Checkf(ctx, err, "open junk filter")
}
err = acc.RetrainMessages(ctx, log, tx, remove)
x.Checkf(ctx, err, "untraining deleted messages")
for _, ch := range removeChanges {
sort.Slice(ch.UIDs, func(i, j int) bool {
return ch.UIDs[i] < ch.UIDs[j]
})
changes = append(changes, ch)
opts := store.RemoveOpts{JunkFilter: jf}
_, _, err := acc.MessageRemove(log, tx, *modseq, &mb, opts, m)
x.Checkf(ctx, err, "expunge message")
changeRemoveUIDs.UIDs = append(changeRemoveUIDs.UIDs, m.UID)
changeRemoveUIDs.MsgIDs = append(changeRemoveUIDs.MsgIDs, m.ID)
}
xflushMailbox()
if jf != nil {
err := jf.Close()
jf = nil
x.Checkf(ctx, err, "close junk filter")
}
return changes
@ -361,6 +344,15 @@ func (x XOps) MessageMove(ctx context.Context, log mlog.Log, acc *store.Account,
acc.WithWLock(func() {
var changes []store.Change
var newIDs []int64
defer func() {
for _, id := range newIDs {
p := acc.MessagePath(id)
err := os.Remove(p)
log.Check(err, "removing delivered message after failure", slog.String("path", p))
}
}()
x.DBWrite(ctx, acc, func(tx *bstore.Tx) {
if mailboxName != "" {
mb, err := acc.MailboxFind(tx, mailboxName)
@ -379,23 +371,37 @@ func (x XOps) MessageMove(ctx context.Context, log mlog.Log, acc *store.Account,
}
var modseq store.ModSeq
changes = x.MessageMoveTx(ctx, log, acc, tx, messageIDs, mbDst, &modseq)
newIDs, changes = x.MessageMoveTx(ctx, log, acc, tx, messageIDs, mbDst, &modseq)
})
newIDs = nil
store.BroadcastChanges(acc, changes)
})
}
func (x XOps) MessageMoveTx(ctx context.Context, log mlog.Log, acc *store.Account, tx *bstore.Tx, messageIDs []int64, mbDst store.Mailbox, modseq *store.ModSeq) []store.Change {
retrain := make([]store.Message, 0, len(messageIDs))
removeChanges := map[int64]store.ChangeRemoveUIDs{}
// n adds, 1 remove, 2 mailboxcounts, optimistic and at least for a single message.
changes := make([]store.Change, 0, len(messageIDs)+3)
// MessageMoveTx moves message to a new mailbox, which must be different than their
// current mailbox. Moving a message is done by changing the MailboxID and
// assigning an appriorate new UID, and then inserting a replacement Message record
// with new ID that is marked expunged in the original mailbox, along with a
// MessageErase record so the message gets erased when all sessions stopped
// referencing the message.
func (x XOps) MessageMoveTx(ctx context.Context, log mlog.Log, acc *store.Account, tx *bstore.Tx, messageIDs []int64, mbDst store.Mailbox, modseq *store.ModSeq) ([]int64, []store.Change) {
var newIDs []int64
var commit bool
defer func() {
if commit {
return
}
for _, id := range newIDs {
p := acc.MessagePath(id)
err := os.Remove(p)
log.Check(err, "removing delivered message after failure", slog.String("path", p))
}
newIDs = nil
}()
var mbSrc store.Mailbox
keywords := map[string]struct{}{}
now := time.Now()
// n adds, 1 remove, 2 mailboxcounts, 1 mailboxkeywords, optimistic that messages are in a single source mailbox.
changes := make([]store.Change, 0, len(messageIDs)+4)
var err error
if *modseq == 0 {
@ -403,104 +409,135 @@ func (x XOps) MessageMoveTx(ctx context.Context, log mlog.Log, acc *store.Accoun
x.Checkf(ctx, err, "assigning next modseq")
}
for _, mid := range messageIDs {
m := x.messageID(ctx, tx, mid)
mbDst.ModSeq = *modseq
// We may have loaded this mailbox in the previous iteration of this loop.
if m.MailboxID != mbSrc.ID {
if mbSrc.ID != 0 {
mbSrc.ModSeq = *modseq
err := tx.Update(&mbSrc)
x.Checkf(ctx, err, "updating source mailbox counts")
changes = append(changes, mbSrc.ChangeCounts())
}
mbSrc = x.mailboxID(ctx, tx, m.MailboxID)
}
if mbSrc.ID == mbDst.ID {
// Get messages. group them by mailbox.
l := make([]store.Message, len(messageIDs))
for i, id := range messageIDs {
l[i] = x.messageID(ctx, tx, id)
if l[i].MailboxID == mbDst.ID {
// Client should filter out messages that are already in mailbox.
x.Checkuserf(ctx, errors.New("already in destination mailbox"), "moving message")
x.Checkuserf(ctx, fmt.Errorf("message %d already in destination mailbox", l[i].ID), "moving message")
}
}
ch := removeChanges[m.MailboxID]
ch.UIDs = append(ch.UIDs, m.UID)
ch.ModSeq = *modseq
ch.MailboxID = m.MailboxID
removeChanges[m.MailboxID] = ch
// Copy of message record that we'll insert when UID is freed up.
om := m
om.PrepareExpunge()
om.ID = 0 // Assign new ID.
om.ModSeq = *modseq
mbSrc.Sub(m.MailboxCounts())
if mbDst.Trash {
m.Seen = true
// Sort (group) by mailbox, sort by UID.
sort.Slice(l, func(i, j int) bool {
if l[i].MailboxID != l[j].MailboxID {
return l[i].MailboxID < l[j].MailboxID
}
conf, _ := acc.Conf()
m.MailboxID = mbDst.ID
if m.IsReject && m.MailboxDestinedID != 0 {
return l[i].UID < l[j].UID
})
var jf *junk.Filter
defer func() {
if jf != nil {
err := jf.CloseDiscard()
log.Check(err, "close junk filter")
}
}()
accConf, _ := acc.Conf()
var mbSrc store.Mailbox
var changeRemoveUIDs store.ChangeRemoveUIDs
xflushMailbox := func() {
changes = append(changes, changeRemoveUIDs, mbSrc.ChangeCounts())
err = tx.Update(&mbSrc)
x.Checkf(ctx, err, "updating source mailbox counts")
}
nkeywords := len(mbDst.Keywords)
now := time.Now()
for _, om := range l {
if om.MailboxID != mbSrc.ID {
if mbSrc.ID != 0 {
xflushMailbox()
}
mbSrc = x.mailboxID(ctx, tx, om.MailboxID)
mbSrc.ModSeq = *modseq
changeRemoveUIDs = store.ChangeRemoveUIDs{MailboxID: mbSrc.ID, ModSeq: *modseq}
}
nm := om
nm.MailboxID = mbDst.ID
nm.UID = mbDst.UIDNext
mbDst.UIDNext++
nm.ModSeq = *modseq
nm.CreateSeq = *modseq
nm.SaveDate = &now
if nm.IsReject && nm.MailboxDestinedID != 0 {
// Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message
// is used for reputation calculation during future deliveries.
m.MailboxOrigID = m.MailboxDestinedID
m.IsReject = false
m.Seen = false
nm.MailboxOrigID = nm.MailboxDestinedID
nm.IsReject = false
nm.Seen = false
}
if mbDst.Trash {
nm.Seen = true
}
m.UID = mbDst.UIDNext
m.ModSeq = *modseq
mbDst.UIDNext++
m.JunkFlagsForMailbox(mbDst, conf)
m.SaveDate = &now
err = tx.Update(&m)
x.Checkf(ctx, err, "updating moved message in database")
// Now that UID is unused, we can insert the old record again.
nm.JunkFlagsForMailbox(mbDst, accConf)
err := tx.Update(&nm)
x.Checkf(ctx, err, "updating message with new mailbox")
mbDst.Add(nm.MailboxCounts())
mbSrc.Sub(om.MailboxCounts())
om.ID = 0
om.Expunged = true
om.ModSeq = *modseq
om.TrainedJunk = nil
err = tx.Insert(&om)
x.Checkf(ctx, err, "inserting record for expunge after moving message")
x.Checkf(ctx, err, "inserting expunged message in old mailbox")
mbDst.Add(m.MailboxCounts())
err = moxio.LinkOrCopy(log, acc.MessagePath(om.ID), acc.MessagePath(nm.ID), nil, false)
x.Checkf(ctx, err, "duplicating message in old mailbox for current sessions")
newIDs = append(newIDs, nm.ID)
// We don't sync the directory. In case of a crash and files disappearing, the
// eraser will simply not find the file at next startup.
changes = append(changes, m.ChangeAddUID())
retrain = append(retrain, m)
err = tx.Insert(&store.MessageErase{ID: om.ID, SkipUpdateDiskUsage: true})
x.Checkf(ctx, err, "insert message erase")
for _, kw := range m.Keywords {
keywords[kw] = struct{}{}
mbDst.Keywords, _ = store.MergeKeywords(mbDst.Keywords, nm.Keywords)
if accConf.JunkFilter != nil && nm.NeedsTraining() {
// Lazily open junk filter.
if jf == nil {
jf, _, err = acc.OpenJunkFilter(ctx, log)
x.Checkf(ctx, err, "open junk filter")
}
err := acc.RetrainMessage(ctx, log, tx, jf, &nm)
x.Checkf(ctx, err, "retrain message after moving")
}
mbSrc.ModSeq = *modseq
err = tx.Update(&mbSrc)
x.Checkf(ctx, err, "updating source mailbox counts and modseq")
changeRemoveUIDs.UIDs = append(changeRemoveUIDs.UIDs, om.UID)
changeRemoveUIDs.MsgIDs = append(changeRemoveUIDs.MsgIDs, om.ID)
changes = append(changes, nm.ChangeAddUID())
}
changes = append(changes, mbSrc.ChangeCounts(), mbDst.ChangeCounts())
xflushMailbox()
// Ensure destination mailbox has keywords of the moved messages.
var mbKwChanged bool
mbDst.Keywords, mbKwChanged = store.MergeKeywords(mbDst.Keywords, maps.Keys(keywords))
if mbKwChanged {
changes = append(changes, mbDst.ChangeCounts())
if nkeywords > len(mbDst.Keywords) {
changes = append(changes, mbDst.ChangeKeywords())
}
mbDst.ModSeq = *modseq
err = tx.Update(&mbDst)
x.Checkf(ctx, err, "updating destination mailbox with uidnext and modseq")
err = acc.RetrainMessages(ctx, log, tx, retrain)
x.Checkf(ctx, err, "retraining messages after move")
// Ensure UIDs of the removed message are in increasing order. It is quite common
// for all messages to be from a single source mailbox, meaning this is just one
// change, for which we preallocated space.
for _, ch := range removeChanges {
sort.Slice(ch.UIDs, func(i, j int) bool {
return ch.UIDs[i] < ch.UIDs[j]
})
changes = append(changes, ch)
if jf != nil {
err := jf.Close()
x.Checkf(ctx, err, "saving junk filter")
jf = nil
}
return changes
commit = true
return newIDs, changes
}
func isText(p message.Part) bool {