infrastructure/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2025-07-12 18:24:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

add condstore & qresync imap extensions

Browse Source 
for conditional storing and quick resynchronisation (not sure if mail clients are actually using it that).

each message now has a "modseq". it is increased for each change. with
condstore, imap clients can request changes since a certain modseq. that
already allows quickly finding changes since a previous connection. condstore
also allows storing (e.g. setting new message flags) only when the modseq of a
message hasn't changed.

qresync should make it fast for clients to get a full list of changed messages
for a mailbox, including removals.

we now also keep basic metadata of messages that have been removed (expunged).
just enough (uid, modseq) to tell client that the messages have been removed.
this does mean we have to be careful when querying messages from the database.
we must now often filter the expunged messages out.

we also keep "createseq", the modseq when a message was created. this will be
useful for the jmap implementation.
This commit is contained in:
Mechiel Lukkien
2023-07-24 21:21:05 +02:00
parent cc4ecf2927
commit 7f1b7198a8
30 changed files with 2181 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

702
imapserver/condstore_test.go Normal file
View File

@ -0,0 +1,702 @@
package imapserver
import (
"fmt"
"strings"
"testing"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/imapclient"
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/store"
)
func TestCondstore(t *testing.T) {
testCondstoreQresync(t, false)
}
func TestQresync(t *testing.T) {
testCondstoreQresync(t, true)
}
func testCondstoreQresync(t *testing.T, qresync bool) {
defer mockUIDValidity()()
tc := start(t)
defer tc.close()
// todo: check whether marking \seen will cause modseq to be returned in case of qresync.
// Check basic requirements of CONDSTORE.
capability := "Condstore"
if qresync {
capability = "Qresync"
}
tc.client.Login("mjl@mox.example", "testtest")
tc.client.Enable(capability)
tc.transactf("ok", "Select inbox")
tc.xuntaggedOpt(false, imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(1), More: "x"}})
// First some tests without any messages.
tc.transactf("ok", "Status inbox (Highestmodseq)")
tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"HIGHESTMODSEQ": 1}})
// No messages, no matches.
tc.transactf("ok", "Uid Fetch 1:* (Flags) (Changedsince 12345)")
tc.xuntagged()
// Also no messages with modseq 1, which we internally turn into modseq 0.
tc.transactf("ok", "Uid Fetch 1:* (Flags) (Changedsince 1)")
tc.xuntagged()
// Also try with modseq attribute.
tc.transactf("ok", "Uid Fetch 1:* (Flags Modseq) (Changedsince 1)")
tc.xuntagged()
// Search with modseq search criteria.
tc.transactf("ok", "Search Modseq 0") // Zero is valid, matches all.
tc.xsearch()
tc.transactf("ok", "Search Modseq 1") // Converted to zero internally.
tc.xsearch()
tc.transactf("ok", "Search Modseq 12345")
tc.xsearch()
tc.transactf("ok", `Search Modseq "/Flags/\\Draft" All 12345`)
tc.xsearch()
tc.transactf("ok", `Search Or Modseq 12345 Modseq 54321`)
tc.xsearch()
// esearch
tc.transactf("ok", "Search Return (All) Modseq 123")
tc.xesearch(imapclient.UntaggedEsearch{})
// Now we add, delete, expunge, modify some message flags and check if the
// responses are correct. We check in both a condstore-enabled and one without that
// we get the correct notifications.
// First we add 3 messages as if they were added before we implemented CONDSTORE.
// Later on, we'll update the second, and delete the third, leaving the first
// unmodified. Those messages have modseq 0 in the database. We use append for
// convenience, then adjust the records in the database.
tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
_, err := bstore.QueryDB[store.Message](ctxbg, tc.account.DB).UpdateFields(map[string]any{
"ModSeq": 0,
"CreateSeq": 0,
})
tcheck(t, err, "clearing modseq from messages")
err = tc.account.DB.Update(ctxbg, &store.SyncState{ID: 1, LastModSeq: 1})
tcheck(t, err, "resetting modseq state")
tc.client.Create("otherbox")
// tc2 is a client without condstore, so no modseq responses.
tc2 := startNoSwitchboard(t)
defer tc2.close()
tc2.client.Login("mjl@mox.example", "testtest")
tc2.client.Select("inbox")
// tc2 is a client with condstore, so with modseq responses.
tc3 := startNoSwitchboard(t)
defer tc3.close()
tc3.client.Login("mjl@mox.example", "testtest")
tc3.client.Enable(capability)
tc3.client.Select("inbox")
var clientModseq int64 = 1 // We track the client-side modseq for inbox. Not a store.ModSeq.
// Add messages to: inbox, otherbox, inbox, inbox.
// We have these messages in order of modseq: 2+1 in inbox, 1 in otherbox, 2 in inbox.
// The original two in inbox appear to have modseq 1 (with 0 stored in the database).
// The ones we insert below will start with modseq 2. So we'll have modseq 1-5.
tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc.xuntagged(imapclient.UntaggedExists(4))
tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 4})
tc.transactf("ok", "Append otherbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc.xuntagged()
tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 2, UID: 1})
tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc.xuntagged(imapclient.UntaggedExists(5))
tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 5})
tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc.xuntagged(imapclient.UntaggedExists(6))
tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 6})
tc2.transactf("ok", "Noop")
noflags := imapclient.FetchFlags(nil)
tc2.xuntagged(
imapclient.UntaggedExists(6),
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), noflags}},
imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags}},
imapclient.UntaggedFetch{Seq: 6, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags}},
)
tc3.transactf("ok", "Noop")
tc3.xuntagged(
imapclient.UntaggedExists(6),
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), noflags, imapclient.FetchModSeq(clientModseq + 1)}},
imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(clientModseq + 3)}},
imapclient.UntaggedFetch{Seq: 6, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq + 4)}},
)
moxvar.Pedantic = true
tc.transactf("bad", `Fetch 1 Flags (Changedsince 0)`) // 0 not allowed in syntax.
moxvar.Pedantic = false
tc.transactf("ok", "Uid fetch 1 (Flags) (Changedsince 0)")
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(clientModseq)}})
clientModseq += 4 // Four messages, over two mailboxes, modseq is per account.
// Check highestmodseq for mailboxes.
tc.transactf("ok", "Status inbox (highestmodseq)")
tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"HIGHESTMODSEQ": clientModseq}})
tc.transactf("ok", "Status otherbox (highestmodseq)")
tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "otherbox", Attrs: map[string]int64{"HIGHESTMODSEQ": 3}})
// Check highestmodseq when we select.
tc.transactf("ok", "Examine otherbox")
tc.xuntaggedOpt(false, imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(3), More: "x"}})
tc.transactf("ok", "Select inbox")
tc.xuntaggedOpt(false, imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(clientModseq), More: "x"}})
// Check fetch modseq response and changedsince.
tc.transactf("ok", `Fetch 1 (Modseq)`)
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchModSeq(1)}})
// Without modseq attribute, even with condseq enabled, there is no modseq response.
// For QRESYNC, we must always send MODSEQ for UID FETCH commands, but not for FETCH commands. ../rfc/7162:1427
tc.transactf("ok", `Uid Fetch 1 Flags`)
if qresync {
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(1)}})
} else {
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags}})
}
tc.transactf("ok", `Fetch 1 Flags`)
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags}})
// When CHANGEDSINCE is present, MODSEQ is automatically added to the response.
// ../rfc/7162:871
// ../rfc/7162:877
tc.transactf("ok", `Fetch 1 Flags (Changedsince 1)`)
tc.xuntagged()
tc.transactf("ok", `Fetch 1,4 Flags (Changedsince 1)`)
tc.xuntagged(imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), noflags, imapclient.FetchModSeq(2)}})
tc.transactf("ok", `Fetch 2 Flags (Changedsince 2)`)
tc.xuntagged()
// store and uid store.
// unchangedsince 0 never passes the check. ../rfc/7162:640
tc.transactf("ok", `Store 1 (Unchangedsince 0) +Flags ()`)
tc.xcodeArg(imapclient.CodeModified(xparseNumSet("1")))
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(1)}})
// Modseq is 2 for first condstore-aware-appended message, so also no match.
tc.transactf("ok", `Uid Store 4 (Unchangedsince 1) +Flags ()`)
tc.xcodeArg(imapclient.CodeModified(xparseNumSet("4")))
// Modseq is 1 for original message.
tc.transactf("ok", `Store 1 (Unchangedsince 1) +Flags (label1)`)
tc.xcode("") // No MODIFIED.
clientModseq++
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}, imapclient.FetchModSeq(clientModseq)}})
tc2.transactf("ok", "Noop")
tc2.xuntagged(
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}}},
)
tc3.transactf("ok", "Noop")
tc3.xuntagged(
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}, imapclient.FetchModSeq(clientModseq)}},
)
// Modify same message twice. Check that second application doesn't fail due to
// modseq change made in the first application. ../rfc/7162:823
tc.transactf("ok", `Uid Store 1,1 (Unchangedsince %d) -Flags (label1)`, clientModseq)
clientModseq++
tc.xcode("") // No MODIFIED.
tc.xuntagged(
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil), imapclient.FetchModSeq(clientModseq)}},
)
// We do broadcast the changes twice. Not great, but doesn't hurt. This isn't common.
tc2.transactf("ok", "Noop")
tc2.xuntagged(
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil)}},
)
tc3.transactf("ok", "Noop")
tc3.xuntagged(
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil), imapclient.FetchModSeq(clientModseq)}},
)
// Modify without actually changing flags, there will be no new modseq and no broadcast.
tc.transactf("ok", `Store 1 (Unchangedsince %d) -Flags (label1)`, clientModseq)
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil), imapclient.FetchModSeq(clientModseq)}})
tc.xcode("") // No MODIFIED.
tc2.transactf("ok", "Noop")
tc2.xuntagged()
tc3.transactf("ok", "Noop")
tc3.xuntagged()
// search with modseq criteria and modseq in response
tc.transactf("ok", "Search Modseq %d", clientModseq)
tc.xsearchmodseq(clientModseq, 1)
tc.transactf("ok", "Uid Search Or Modseq %d Modseq %d", clientModseq, clientModseq)
tc.xsearchmodseq(clientModseq, 1)
// esearch
tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq %d", clientModseq)
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1, All: esearchall0("1"), ModSeq: clientModseq})
uint32ptr := func(v uint32) *uint32 {
return &v
}
tc.transactf("ok", "Search Return (Count) 1:* Modseq 0")
tc.xesearch(imapclient.UntaggedEsearch{Count: uint32ptr(6), ModSeq: clientModseq})
tc.transactf("ok", "Search Return (Min Max) 1:* Modseq 0")
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 6, ModSeq: clientModseq})
tc.transactf("ok", "Search Return (Min) 1:* Modseq 0")
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, ModSeq: clientModseq})
// expunge, we expunge the third and fourth messages. The third was originally with
// modseq 0, the fourth was added with condstore-aware append.
tc.transactf("ok", `Store 3:4 +Flags (\Deleted)`)
clientModseq++
tc2.transactf("ok", "Noop")
tc3.transactf("ok", "Noop")
tc.transactf("ok", "Expunge")
clientModseq++
if qresync {
tc.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("3:4")})
} else {
tc.xuntagged(imapclient.UntaggedExpunge(3), imapclient.UntaggedExpunge(3))
}
tc.xcodeArg(imapclient.CodeHighestModSeq(clientModseq))
tc2.transactf("ok", "Noop")
tc2.xuntagged(imapclient.UntaggedExpunge(3), imapclient.UntaggedExpunge(3))
tc3.transactf("ok", "Noop")
if qresync {
tc3.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("3:4")})
} else {
tc3.xuntagged(imapclient.UntaggedExpunge(3), imapclient.UntaggedExpunge(3))
}
// Again after expunge: status, select, conditional store/fetch/search
tc.transactf("ok", "Status inbox (Highestmodseq Messages Unseen Deleted)")
tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"MESSAGES": 4, "UNSEEN": 4, "DELETED": 0, "HIGHESTMODSEQ": clientModseq}})
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox")
tc.xuntaggedOpt(false,
imapclient.UntaggedExists(4),
imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(clientModseq), More: "x"}},
)
tc.transactf("ok", `Fetch 1:* (Modseq)`)
tc.xuntagged(
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchModSeq(7)}},
imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), imapclient.FetchModSeq(1)}},
imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), imapclient.FetchModSeq(4)}},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), imapclient.FetchModSeq(5)}},
)
// Expunged messages, with higher modseq, should not show up.
tc.transactf("ok", "Uid Fetch 1:* (flags) (Changedsince 7)")
tc.xuntagged()
// search
tc.transactf("ok", "Search Modseq 7")
tc.xsearchmodseq(7, 1)
tc.transactf("ok", "Search Modseq 8")
tc.xsearch()
// esearch
tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq 7")
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1, All: esearchall0("1"), ModSeq: 7})
tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq 8")
tc.xuntagged(imapclient.UntaggedEsearch{Correlator: tc.client.LastTag})
// store, cannot modify expunged messages.
tc.transactf("ok", `Uid Store 3,4 (Unchangedsince %d) +Flags (label2)`, clientModseq)
tc.xuntagged()
tc.xcode("") // Not MODIFIED.
tc.transactf("ok", `Uid Store 3,4 +Flags (label2)`)
tc.xuntagged()
tc.xcode("") // Not MODIFIED.
// Check all condstore-enabling commands (and their syntax), ../rfc/7162:368
// We start a new connection, do the thing that should enable condstore, then
// change flags of a message in another connection, do a noop in the new connection
// which should result in an untagged fetch that includes modseq, the indicator
// that condstore was indeed enabled. It's a bit complicated, but i don't think
// there is a clearly specified mechanism to find out which capabilities are
// enabled at any point.
var tagcount int
checkCondstoreEnabled := func(fn func(xtc *testconn)) {
t.Helper()
xtc := startNoSwitchboard(t)
defer xtc.close()
xtc.client.Login("mjl@mox.example", "testtest")
fn(xtc)
tagcount++
label := fmt.Sprintf("l%d", tagcount)
tc.transactf("ok", "Store 4 Flags (%s)", label)
clientModseq++
xtc.transactf("ok", "Noop")
xtc.xuntagged(imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), imapclient.FetchFlags{label}, imapclient.FetchModSeq(clientModseq)}})
}
// SELECT/EXAMINE with CONDSTORE parameter, ../rfc/7162:373
checkCondstoreEnabled(func(xtc *testconn) {
t.Helper()
xtc.transactf("ok", "Select inbox (Condstore)")
})
// STATUS with HIGHESTMODSEQ attribute, ../rfc/7162:375
checkCondstoreEnabled(func(xtc *testconn) {
t.Helper()
xtc.transactf("ok", "Status otherbox (Highestmodseq)")
xtc.transactf("ok", "Select inbox")
})
// FETCH with MODSEQ ../rfc/7162:377
checkCondstoreEnabled(func(xtc *testconn) {
t.Helper()
xtc.transactf("ok", "Select inbox")
xtc.transactf("ok", "Fetch 4 (Modseq)")
})
// SEARCH with MODSEQ ../rfc/7162:377
checkCondstoreEnabled(func(xtc *testconn) {
t.Helper()
xtc.transactf("ok", "Select inbox")
xtc.transactf("ok", "Search 4 Modseq 1")
})
// FETCH with CHANGEDSINCE ../rfc/7162:380
checkCondstoreEnabled(func(xtc *testconn) {
t.Helper()
xtc.transactf("ok", "Select inbox")
xtc.transactf("ok", "Fetch 4 (Flags) (Changedsince %d)", clientModseq)
})
// STORE with UNCHANGEDSINCE ../rfc/7162:382
checkCondstoreEnabled(func(xtc *testconn) {
t.Helper()
xtc.transactf("ok", "Select inbox")
xtc.transactf("ok", "Store 4 (Unchangedsince 0) Flags ()")
})
// ENABLE CONDSTORE ../rfc/7162:384
checkCondstoreEnabled(func(xtc *testconn) {
t.Helper()
xtc.transactf("ok", "Enable Condstore")
xtc.transactf("ok", "Select inbox")
})
// ENABLE QRESYNC ../rfc/7162:1390
checkCondstoreEnabled(func(xtc *testconn) {
t.Helper()
xtc.transactf("ok", "Enable Qresync")
xtc.transactf("ok", "Select inbox")
})
tc.transactf("ok", "Store 4 Flags ()")
clientModseq++
if qresync {
testQresync(t, tc, clientModseq)
}
// Continue with some tests that further change the data.
// First we copy messages to a new mailbox, and check we get new modseq for those
// messages.
tc.transactf("ok", "Select otherbox")
tc2.transactf("ok", "Noop")
tc3.transactf("ok", "Noop")
tc.transactf("ok", "Copy 1 inbox")
clientModseq++
tc2.transactf("ok", "Noop")
tc3.transactf("ok", "Noop")
tc2.xuntagged(
imapclient.UntaggedExists(5),
imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(7), noflags}},
)
tc3.xuntagged(
imapclient.UntaggedExists(5),
imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(7), noflags, imapclient.FetchModSeq(clientModseq)}},
)
// Then we move some messages, and check if we get expunged/vanished in original
// and untagged fetch with modseq in destination mailbox.
// tc2o is a client without condstore, so no modseq responses.
tc2o := startNoSwitchboard(t)
defer tc2o.close()
tc2o.client.Login("mjl@mox.example", "testtest")
tc2o.client.Select("otherbox")
// tc3o is a client with condstore, so with modseq responses.
tc3o := startNoSwitchboard(t)
defer tc3o.close()
tc3o.client.Login("mjl@mox.example", "testtest")
tc3o.client.Enable(capability)
tc3o.client.Select("otherbox")
tc.transactf("ok", "Select inbox")
tc.transactf("ok", "Uid Move 2:4 otherbox") // Only UID 2, because UID 3 and 4 have already been expunged.
clientModseq++
if qresync {
tc.xuntaggedOpt(false, imapclient.UntaggedVanished{UIDs: xparseNumSet("2")})
tc.xcodeArg(imapclient.CodeHighestModSeq(clientModseq))
} else {
tc.xuntaggedOpt(false, imapclient.UntaggedExpunge(2))
tc.xcode("")
}
tc2.transactf("ok", "Noop")
tc2.xuntagged(imapclient.UntaggedExpunge(2))
tc3.transactf("ok", "Noop")
if qresync {
tc3.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("2")})
} else {
tc3.xuntagged(imapclient.UntaggedExpunge(2))
}
tc2o.transactf("ok", "Noop")
tc2o.xuntagged(
imapclient.UntaggedExists(2),
imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), noflags}},
)
tc3o.transactf("ok", "Noop")
tc3o.xuntagged(
imapclient.UntaggedExists(2),
imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), noflags, imapclient.FetchModSeq(clientModseq)}},
)
tc2o.close()
tc2o = nil
tc3o.close()
tc3o = nil
// Then we rename inbox, which is special because it moves messages away instead of
// actually moving the mailbox. The mailbox stays and is cleared, so we check if we
// get expunged/vanished messages.
tc.transactf("ok", "Rename inbox oldbox")
// todo spec: server doesn't respond with untagged responses, find rfc reference that says this is ok.
tc2.transactf("ok", "Noop")
tc2.xuntagged(
imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"},
imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1),
)
tc3.transactf("ok", "Noop")
if qresync {
tc3.xuntagged(
imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"},
imapclient.UntaggedVanished{UIDs: xparseNumSet("1,5:7")},
)
} else {
tc3.xuntagged(
imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"},
imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1),
)
}
// Then we delete otherbox (we cannot delete inbox). We don't keep any history for removed mailboxes, so not actually a special case.
tc.transactf("ok", "Delete otherbox")
}
func testQresync(t *testing.T, tc *testconn, clientModseq int64) {
// Vanished on non-uid fetch is not allowed. ../rfc/7162:1693
tc.transactf("bad", "fetch 1:* (Flags) (Changedsince 1 Vanished)")
// Vanished without changedsince is not allowed. ../rfc/7162:1701
tc.transactf("bad", "Uid Fetch 1:* (Flags) (Vanished)")
// Vanished not allowed without first enabling qresync. ../rfc/7162:1697
xtc := startNoSwitchboard(t)
xtc.client.Login("mjl@mox.example", "testtest")
xtc.transactf("ok", "Select inbox (Condstore)")
xtc.transactf("bad", "Uid Fetch 1:* (Flags) (Changedsince 1 Vanished)")
xtc.close()
xtc = nil
// Check that we get proper vanished responses.
tc.transactf("ok", "Uid Fetch 1:* (Flags) (Changedsince 1 Vanished)")
noflags := imapclient.FetchFlags(nil)
tc.xuntagged(
imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
)
// select/examine with qresync parameters, including the various optional fields.
tc.transactf("ok", "Close")
// Must enable qresync explicitly before using. ../rfc/7162:1446
xtc = startNoSwitchboard(t)
xtc.client.Login("mjl@mox.example", "testtest")
xtc.transactf("bad", "Select inbox (Qresync 1 0)")
xtc.close()
xtc = nil
tc.transactf("bad", "Select inbox (Qresync (0 1))") // Both args must be > 0.
tc.transactf("bad", "Select inbox (Qresync (1 0))") // Both args must be > 0.
tc.transactf("bad", "Select inbox (Qresync)") // Two args are minimum.
tc.transactf("bad", "Select inbox (Qresync (1))") // Two args are minimum.
tc.transactf("bad", "Select inbox (Qresync (1 1 1:*))") // Known UIDs, * not allowed.
tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1:* 1:6)))") // Known seqset cannot have *.
tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1:6 1:*)))") // Known uidset cannot have *.
tc.transactf("bad", "Select inbox (Qresync (1 1) qresync (1 1))") // Duplicate qresync.
flags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent label1 l1 l2 l3 l4 l5 l6 l7 l8`, " ")
permflags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent \*`, " ")
uflags := imapclient.UntaggedFlags(flags)
upermflags := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "PERMANENTFLAGS", CodeArg: imapclient.CodeList{Code: "PERMANENTFLAGS", Args: permflags}, More: "x"}}
baseUntagged := []imapclient.Untagged{
uflags,
upermflags,
imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"},
imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UIDNEXT", CodeArg: imapclient.CodeUint{Code: "UIDNEXT", Num: 7}, More: "x"}},
imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UIDVALIDITY", CodeArg: imapclient.CodeUint{Code: "UIDVALIDITY", Num: 1}, More: "x"}},
imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UNSEEN", CodeArg: imapclient.CodeUint{Code: "UNSEEN", Num: 1}, More: "x"}},
imapclient.UntaggedRecent(0),
imapclient.UntaggedExists(4),
imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(clientModseq), More: "x"}},
}
makeUntagged := func(l ...imapclient.Untagged) []imapclient.Untagged {
return append(append([]imapclient.Untagged{}, baseUntagged...), l...)
}
// uidvalidity 1, highest known modseq 1, sends full current state.
tc.transactf("ok", "Select inbox (Qresync (1 1))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
)...,
)
// Uidvalidity mismatch, server will not send any changes, so it's just a regular open.
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (2 1))")
tc.xuntagged(baseUntagged...)
// We can tell which UIDs we know. First, send broader range then exist, should work.
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (1 1 1:7))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
)...,
)
// Now send just the ones that exist. We won't get the vanished messages.
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (1 1 1,2,5:6))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
)...,
)
// We'll only get updates for UIDs we specify.
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (1 1 5))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
)...,
)
// We'll only get updates for UIDs we specify. ../rfc/7162:1523
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (1 1 3))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3")},
)...,
)
// If we specify the latest modseq, we'll get no changes.
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (1 %d))", clientModseq)
tc.xuntagged(baseUntagged...)
// We can provide our own seqs & uids, and have server determine which uids we
// know. But the seqs & uids must be of equal length. First try with a few combinations
// that aren't valid. ../rfc/7162:1579
tc.transactf("ok", "Close")
tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1 1,2)))") // Not same length.
tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1,2 1)))") // Not same length.
tc.transactf("no", "Select inbox (Qresync (1 1 1:6 (1,2 1,1)))") // Not ascending.
tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1:* 1:4)))") // Star not allowed.
// With valid parameters, based on what a client would know at this stage.
tc.transactf("ok", "Select inbox (Qresync (1 1 1:6 (1,3,6 1,3,6)))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
)...,
)
// The 3rd parameter is optional, try without.
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (1 5 (1,3,6 1,3,6)))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
)...,
)
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (1 8 (1,3,6 1,3,6)))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
)...,
)
// Client will claim a highestmodseq but then include uids that have been removed
// since that time. Server detects this, sends full vanished history and continues
// working with modseq changed to 1 before the expunged uid.
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (1 9 (1,3,6 1,3,6)))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "ALERT", More: "Synchronization inconsistency in client detected. Client tried to sync with a UID that was removed at or after the MODSEQ it sent in the request. Sending all historic message removals for selected mailbox. Full syncronization recommended."}},
imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
)...,
)
// Client will claim a highestmodseq but then include uids that have been removed
// since that time. Server detects this, sends full vanished history and continues
// working with modseq changed to 1 before the expunged uid.
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (1 18 (1,3,6 1,3,6)))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "ALERT", More: "Synchronization inconsistency in client detected. Client tried to sync with a UID that was removed at or after the MODSEQ it sent in the request. Sending all historic message removals for selected mailbox. Full syncronization recommended."}},
imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
)...,
)
}

204
imapserver/fetch.go
View File

@ -11,24 +11,31 @@ import (
"sort"
"strings"
"golang.org/x/exp/maps"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/store"
)
// functions to handle fetch attribute requests are defined on fetchCmd.
type fetchCmd struct {
conn *conn
mailboxID int64
uid store.UID
tx *bstore.Tx // Writable tx, for storing message when first parsed as mime parts.
changes []store.Change // For updated Seen flag.
markSeen bool
needFlags bool
expungeIssued bool // Set if a message cannot be read. Can happen for expunged messages.
conn *conn
mailboxID int64
uid store.UID
tx *bstore.Tx // Writable tx, for storing message when first parsed as mime parts.
changes []store.Change // For updated Seen flag.
markSeen bool
needFlags bool
needModseq bool // Whether untagged responses needs modseq.
expungeIssued bool // Set if a message cannot be read. Can happen for expunged messages.
modseq store.ModSeq // Initialized on first change, for marking messages as seen.
isUID bool // If this is a UID FETCH command.
hasChangedSince bool // Whether CHANGEDSINCE was set. Enables MODSEQ in response.
// Loaded when first needed, closed when message was processed.
m *store.Message // Message currently being processed.
@ -60,15 +67,61 @@ func (cmd *fetchCmd) xcheckf(err error, format string, args ...any) {
//
// State: Selected
func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
// Command: ../rfc/9051:4330 ../rfc/3501:2992
// Examples: ../rfc/9051:4463 ../rfc/9051:4520
// Response syntax: ../rfc/9051:6742 ../rfc/3501:4864
// Command: ../rfc/9051:4330 ../rfc/3501:2992 ../rfc/7162:864
// Examples: ../rfc/9051:4463 ../rfc/9051:4520 ../rfc/7162:880
// Response syntax: ../rfc/9051:6742 ../rfc/3501:4864 ../rfc/7162:2490
// Request syntax: ../rfc/9051:6553 ../rfc/3501:4748
// Request syntax: ../rfc/9051:6553 ../rfc/3501:4748 ../rfc/4466:535 ../rfc/7162:2475
p.xspace()
nums := p.xnumSet()
p.xspace()
atts := p.xfetchAtts()
atts := p.xfetchAtts(isUID)
var changedSince int64
var haveChangedSince bool
var vanished bool
if p.space() {
// ../rfc/4466:542
// ../rfc/7162:2479
p.xtake("(")
seen := map[string]bool{}
for {
var w string
if isUID && p.conn.enabled[capQresync] {
// Vanished only valid for uid fetch, and only for qresync. ../rfc/7162:1693
w = p.xtakelist("CHANGEDSINCE", "VANISHED")
} else {
w = p.xtakelist("CHANGEDSINCE")
}
if seen[w] {
xsyntaxErrorf("duplicate fetch modifier %s", w)
}
seen[w] = true
switch w {
case "CHANGEDSINCE":
p.xspace()
changedSince = p.xnumber64()
// workaround: ios mail (16.5.1) was seen sending changedSince 0 on an existing account that got condstore enabled.
if changedSince == 0 && moxvar.Pedantic {
// ../rfc/7162:2551
xsyntaxErrorf("changedsince modseq must be > 0")
}
// CHANGEDSINCE is a CONDSTORE-enabling parameter. ../rfc/7162:380
p.conn.xensureCondstore(nil)
haveChangedSince = true
case "VANISHED":
vanished = true
}
if p.take(")") {
break
}
p.xspace()
}
// ../rfc/7162:1701
if vanished && !haveChangedSince {
xsyntaxErrorf("VANISHED can only be used with CHANGEDSINCE")
}
}
p.xempty()
// We don't use c.account.WithRLock because we write to the client while reading messages.
@ -81,21 +134,105 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
runlock()
}()
cmd := &fetchCmd{conn: c, mailboxID: c.mailboxID}
var vanishedUIDs []store.UID
cmd := &fetchCmd{conn: c, mailboxID: c.mailboxID, isUID: isUID, hasChangedSince: haveChangedSince}
c.xdbwrite(func(tx *bstore.Tx) {
cmd.tx = tx
// Ensure the mailbox still exists.
c.xmailboxID(tx, c.mailboxID)
uids := c.xnumSetUIDs(isUID, nums)
var uids []store.UID
// With changedSince, the client is likely asking for a small set of changes. Use a
// database query to trim down the uids we need to look at.
// ../rfc/7162:871
if changedSince > 0 {
q := bstore.QueryTx[store.Message](tx)
q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
q.FilterGreater("ModSeq", store.ModSeqFromClient(changedSince))
if !vanished {
q.FilterEqual("Expunged", false)
}
err := q.ForEach(func(m store.Message) error {
if m.Expunged {
vanishedUIDs = append(vanishedUIDs, m.UID)
} else if isUID {
if nums.containsUID(m.UID, c.uids, c.searchResult) {
uids = append(uids, m.UID)
}
} else {
seq := c.sequence(m.UID)
if seq > 0 && nums.containsSeq(seq, c.uids, c.searchResult) {
uids = append(uids, m.UID)
}
}
return nil
})
xcheckf(err, "looking up messages with changedsince")
} else {
uids = c.xnumSetUIDs(isUID, nums)
}
// Send vanished for all missing requested UIDs. ../rfc/7162:1718
if vanished {
delModSeq, err := c.account.HighestDeletedModSeq(tx)
xcheckf(err, "looking up highest deleted modseq")
if changedSince < delModSeq.Client() {
// First sort the uids we already found, for fast lookup.
sort.Slice(vanishedUIDs, func(i, j int) bool {
return vanishedUIDs[i] < vanishedUIDs[j]
})
// We'll be gathering any more vanished uids in more.
more := map[store.UID]struct{}{}
checkVanished := func(uid store.UID) {
if uidSearch(c.uids, uid) <= 0 && uidSearch(vanishedUIDs, uid) <= 0 {
more[uid] = struct{}{}
}
}
// Now look through the requested uids. We may have a searchResult, handle it
// separately from a numset with potential stars, over which we can more easily
// iterate.
if nums.searchResult {
for _, uid := range c.searchResult {
checkVanished(uid)
}
} else {
iter := nums.interpretStar(c.uids).newIter()
for {
num, ok := iter.Next()
if !ok {
break
}
checkVanished(store.UID(num))
}
}
vanishedUIDs = append(vanishedUIDs, maps.Keys(more)...)
}
}
// Release the account lock.
runlock()
runlock = func() {} // Prevent defer from unlocking again.
// First report all vanished UIDs. ../rfc/7162:1714
if len(vanishedUIDs) > 0 {
// Mention all vanished UIDs in compact numset form.
// ../rfc/7162:1985
sort.Slice(vanishedUIDs, func(i, j int) bool {
return vanishedUIDs[i] < vanishedUIDs[j]
})
// No hard limit on response sizes, but clients are recommended to not send more
// than 8k. We send a more conservative max 4k.
for _, s := range compactUIDSet(vanishedUIDs).Strings(4*1024 - 32) {
c.bwritelinef("* VANISHED (EARLIER) %s", s)
}
}
for _, uid := range uids {
cmd.uid = uid
mlog.Field("processing uid", mlog.Field("uid", uid))
cmd.process(atts)
}
})
@ -113,6 +250,15 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
}
}
func (cmd *fetchCmd) xmodseq() store.ModSeq {
if cmd.modseq == 0 {
var err error
cmd.modseq, err = cmd.conn.account.NextModSeq(cmd.tx)
cmd.xcheckf(err, "assigning next modseq")
}
return cmd.modseq
}
func (cmd *fetchCmd) xensureMessage() *store.Message {
if cmd.m != nil {
return cmd.m
@ -120,6 +266,7 @@ func (cmd *fetchCmd) xensureMessage() *store.Message {
q := bstore.QueryTx[store.Message](cmd.tx)
q.FilterNonzero(store.Message{MailboxID: cmd.mailboxID, UID: cmd.uid})
q.FilterEqual("Expunged", false)
m, err := q.Get()
cmd.xcheckf(err, "get message for uid %d", cmd.uid)
cmd.m = &m
@ -178,6 +325,7 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
cmd.markSeen = false
cmd.needFlags = false
cmd.needModseq = false
for _, a := range atts {
data = append(data, cmd.xprocessAtt(a)...)
@ -186,10 +334,11 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
if cmd.markSeen {
m := cmd.xensureMessage()
m.Seen = true
m.ModSeq = cmd.xmodseq()
err := cmd.tx.Update(m)
xcheckf(err, "marking message as seen")
cmd.changes = append(cmd.changes, store.ChangeFlags{MailboxID: cmd.mailboxID, UID: cmd.uid, Mask: store.Flags{Seen: true}, Flags: m.Flags, Keywords: m.Keywords})
cmd.changes = append(cmd.changes, store.ChangeFlags{MailboxID: cmd.mailboxID, UID: cmd.uid, ModSeq: m.ModSeq, Mask: store.Flags{Seen: true}, Flags: m.Flags, Keywords: m.Keywords})
}
if cmd.needFlags {
@ -197,6 +346,26 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
data = append(data, bare("FLAGS"), flaglist(m.Flags, m.Keywords))
}
// The wording around when to include the MODSEQ attribute is hard to follow and is
// specified and refined in several places.
//
// An additional rule applies to "QRESYNC servers" (we'll assume it only applies
// when QRESYNC is enabled on a connection): setting the \Seen flag also triggers
// sending MODSEQ, and so does a UID FETCH command. ../rfc/7162:1421
//
// For example, ../rfc/7162:389 says the server must include modseq in "all
// subsequent untagged fetch responses", then lists cases, but leaves out FETCH/UID
// FETCH. That appears intentional, it is not a list of examples, it is the full
// list, and the "all subsequent untagged fetch responses" doesn't mean "all", just
// those covering the listed cases. That makes sense, because otherwise all the
// other mentioning of cases elsewhere in the RFC would be too superfluous.
//
// ../rfc/7162:877 ../rfc/7162:388 ../rfc/7162:909 ../rfc/7162:1426
if cmd.needModseq || cmd.hasChangedSince || cmd.conn.enabled[capQresync] && (cmd.isUID || cmd.markSeen) {
m := cmd.xensureMessage()
data = append(data, bare("MODSEQ"), listspace{bare(fmt.Sprintf("%d", m.ModSeq.Client()))})
}
// Write errors are turned into panics because we write through c.
fmt.Fprintf(cmd.conn.bw, "* %d FETCH ", cmd.conn.xsequence(cmd.uid))
data.writeTo(cmd.conn, cmd.conn.bw)
@ -301,6 +470,9 @@ func (cmd *fetchCmd) xprocessAtt(a fetchAtt) []token {
case "FLAGS":
cmd.needFlags = true
case "MODSEQ":
cmd.needModseq = true
default:
xserverErrorf("field %q not yet implemented", a.field)
}

3
imapserver/fuzz_test.go
View File

@ -1,7 +1,6 @@
package imapserver
import (
"context"
"encoding/base64"
"errors"
"fmt"
@ -58,7 +57,7 @@ func FuzzServer(f *testing.F) {
f.Add(tag + cmd)
}
mox.Context = context.Background()
mox.Context = ctxbg
mox.ConfigStaticPath = "../testdata/imapserverfuzz/mox.conf"
mox.MustLoadConfig(true, false)
dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir)

109
imapserver/parse.go
View File

@ -246,13 +246,21 @@ func (p *parser) xnumber64() int64 {
if s == "" {
p.xerrorf("expected number64")
}
v, err := strconv.ParseInt(s, 10, 64)
v, err := strconv.ParseInt(s, 10, 63) // ../rfc/9051:6794 ../rfc/7162:297
if err != nil {
p.xerrorf("parsing number64 %q: %v", s, err)
}
return v
}
func (p *parser) xnznumber64() int64 {
v := p.xnumber64()
if v == 0 {
p.xerrorf("expected non-zero number64")
}
return v
}
// l should be a list of uppercase words, the first match is returned
func (p *parser) takelist(l ...string) (string, bool) {
for _, w := range l {
@ -423,36 +431,43 @@ func (p *parser) xmboxOrPat() ([]string, bool) {
return l, true
}
// ../rfc/9051:7056
// RECENT only in ../rfc/3501:5047
// APPENDLIMIT is from ../rfc/7889:252
// ../rfc/9051:7056, RECENT ../rfc/3501:5047, APPENDLIMIT ../rfc/7889:252, HIGHESTMODSEQ ../rfc/7162:2452
func (p *parser) xstatusAtt() string {
return p.xtakelist("MESSAGES", "UIDNEXT", "UIDVALIDITY", "UNSEEN", "DELETED", "SIZE", "RECENT", "APPENDLIMIT")
w := p.xtakelist("MESSAGES", "UIDNEXT", "UIDVALIDITY", "UNSEEN", "DELETED", "SIZE", "RECENT", "APPENDLIMIT", "HIGHESTMODSEQ")
if w == "HIGHESTMODSEQ" {
// HIGHESTMODSEQ is a CONDSTORE-enabling parameter. ../rfc/7162:375
p.conn.enabled[capCondstore] = true
}
return w
}
// ../rfc/9051:7133 ../rfc/9051:7034
func (p *parser) xnumSet() (r numSet) {
func (p *parser) xnumSet0(allowStar, allowSearch bool) (r numSet) {
defer p.context("numSet")()
if p.take("$") {
if allowSearch && p.take("$") {
return numSet{searchResult: true}
}
r.ranges = append(r.ranges, p.xnumRange())
r.ranges = append(r.ranges, p.xnumRange0(allowStar))
for p.take(",") {
r.ranges = append(r.ranges, p.xnumRange())
r.ranges = append(r.ranges, p.xnumRange0(allowStar))
}
return r
}
func (p *parser) xnumSet() (r numSet) {
return p.xnumSet0(true, true)
}
// parse numRange, which can be just a setNumber.
func (p *parser) xnumRange() (r numRange) {
if p.take("*") {
func (p *parser) xnumRange0(allowStar bool) (r numRange) {
if allowStar && p.take("*") {
r.first.star = true
} else {
r.first.number = p.xnznumber()
}
if p.take(":") {
r.last = &setNumber{}
if p.take("*") {
if allowStar && p.take("*") {
r.last.star = true
} else {
r.last.number = p.xnznumber()
@ -550,14 +565,16 @@ func (p *parser) xsectionBinary() (r []uint32) {
return r
}
// ../rfc/9051:6557 ../rfc/3501:4751
func (p *parser) xfetchAtt() (r fetchAtt) {
var fetchAttWords = []string{
"ENVELOPE", "FLAGS", "INTERNALDATE", "RFC822.SIZE", "BODYSTRUCTURE", "UID", "BODY.PEEK", "BODY", "BINARY.PEEK", "BINARY.SIZE", "BINARY",
"RFC822.HEADER", "RFC822.TEXT", "RFC822", // older IMAP
"MODSEQ", // CONDSTORE extension.
}
// ../rfc/9051:6557 ../rfc/3501:4751 ../rfc/7162:2483
func (p *parser) xfetchAtt(isUID bool) (r fetchAtt) {
defer p.context("fetchAtt")()
words := []string{
"ENVELOPE", "FLAGS", "INTERNALDATE", "RFC822.SIZE", "BODYSTRUCTURE", "UID", "BODY.PEEK", "BODY", "BINARY.PEEK", "BINARY.SIZE", "BINARY",
"RFC822.HEADER", "RFC822.TEXT", "RFC822", // older IMAP
}
f := p.xtakelist(words...)
f := p.xtakelist(fetchAttWords...)
r.peek = strings.HasSuffix(f, ".PEEK")
r.field = strings.TrimSuffix(f, ".PEEK")
@ -576,12 +593,19 @@ func (p *parser) xfetchAtt() (r fetchAtt) {
}
case "BINARY.SIZE":
r.sectionBinary = p.xsectionBinary()
case "MODSEQ":
// The RFC text mentions MODSEQ is only for FETCH, not UID FETCH, but the ABNF adds
// the attribute to the shared syntax, so UID FETCH also implements it.
// ../rfc/7162:905
// The wording about when to respond with a MODSEQ attribute could be more clear. ../rfc/7162:923 ../rfc/7162:388
// MODSEQ attribute is a CONDSTORE-enabling parameter. ../rfc/7162:377
p.conn.xensureCondstore(nil)
}
return
}
// ../rfc/9051:6553 ../rfc/3501:4748
func (p *parser) xfetchAtts() []fetchAtt {
func (p *parser) xfetchAtts(isUID bool) []fetchAtt {
defer p.context("fetchAtts")()
fields := func(l ...string) []fetchAtt {
@ -605,13 +629,13 @@ func (p *parser) xfetchAtts() []fetchAtt {
}
if !p.hasPrefix("(") {
return []fetchAtt{p.xfetchAtt()}
return []fetchAtt{p.xfetchAtt(isUID)}
}
l := []fetchAtt{}
p.xtake("(")
for {
l = append(l, p.xfetchAtt())
l = append(l, p.xfetchAtt(isUID))
if !p.take(" ") {
break
}
@ -748,9 +772,10 @@ var searchKeyWords = []string{
"SENTBEFORE", "SENTON",
"SENTSINCE", "SMALLER",
"UID", "UNDRAFT",
"MODSEQ", // CONDSTORE extension.
}
// ../rfc/9051:6923 ../rfc/3501:4957
// ../rfc/9051:6923 ../rfc/3501:4957, MODSEQ ../rfc/7162:2492
// differences: rfc 9051 removes NEW, OLD, RECENT and makes SMALLER and LARGER number64 instead of number.
func (p *parser) xsearchKey() *searchKey {
if p.take("(") {
@ -852,12 +877,50 @@ func (p *parser) xsearchKey() *searchKey {
p.xspace()
sk.uidSet = p.xnumSet()
case "UNDRAFT":
case "MODSEQ":
// ../rfc/7162:1045 ../rfc/7162:2499
p.xspace()
if p.take(`"`) {
// We don't do anything with this field, so parse and ignore.
p.xtake(`/FLAGS/`)
if p.take(`\`) {
p.xtake(`\`) // ../rfc/7162:1072
}
p.xatom()
p.xtake(`"`)
p.xspace()
p.xtakelist("PRIV", "SHARED", "ALL")
p.xspace()
}
v := p.xnumber64()
sk.clientModseq = &v
// MODSEQ is a CONDSTORE-enabling parameter. ../rfc/7162:377
p.conn.enabled[capCondstore] = true
default:
p.xerrorf("missing case for op %q", sk.op)
}
return sk
}
// hasModseq returns whether there is a modseq filter anywhere in the searchkey.
func (sk searchKey) hasModseq() bool {
if sk.clientModseq != nil {
return true
}
for _, e := range sk.searchKeys {
if e.hasModseq() {
return true
}
}
if sk.searchKey != nil && sk.searchKey.hasModseq() {
return true
}
if sk.searchKey2 != nil && sk.searchKey2.hasModseq() {
return true
}
return false
}
// ../rfc/9051:6489 ../rfc/3501:4692
func (p *parser) xdateDay() int {
d := p.xdigit()

211
imapserver/protocol.go
View File

@ -80,15 +80,31 @@ func (ss numSet) containsUID(uid store.UID, uids []store.UID, searchResult []sto
return false
}
func (ss numSet) String() string {
if ss.searchResult {
return "$"
}
s := ""
// contains returns whether the numset contains the number.
// only allowed on basic, strictly increasing numsets.
func (ss numSet) contains(v uint32) bool {
for _, r := range ss.ranges {
if s != "" {
s += ","
if r.first.number == v || r.last != nil && v > r.first.number && v <= r.last.number {
return true
}
}
return false
}
func (ss numSet) empty() bool {
return !ss.searchResult && len(ss.ranges) == 0
}
// Strings returns the numset in zero or more strings of maxSize bytes. If
// maxSize is <= 0, a single string is returned.
func (ss numSet) Strings(maxSize int) []string {
if ss.searchResult {
return []string{"$"}
}
var l []string
var line string
for _, r := range ss.ranges {
s := ""
if r.first.star {
s += "*"
} else {
@ -98,16 +114,41 @@ func (ss numSet) String() string {
if r.first.star {
panic("invalid numSet range first star without last")
}
} else {
s += ":"
if r.last.star {
s += "*"
} else {
s += fmt.Sprintf("%d", r.last.number)
}
}
nsize := len(line) + len(s)
if line != "" {
nsize++ // comma
}
if maxSize > 0 && nsize > maxSize {
l = append(l, line)
line = s
continue
}
s += ":"
if r.last.star {
s += "*"
} else {
s += fmt.Sprintf("%d", r.last.number)
if line != "" {
line += ","
}
line += s
}
return s
if line != "" {
l = append(l, line)
}
return l
}
func (ss numSet) String() string {
l := ss.Strings(0)
if len(l) == 0 {
return ""
}
return l[0]
}
type setNumber struct {
@ -120,6 +161,127 @@ type numRange struct {
last *setNumber // if nil, this numRange is just a setNumber in "first" and first.star will be false
}
// interpretStar returns a numset that interprets stars in a numset, returning a new
// numset without stars with increasing first/last.
func (s numSet) interpretStar(uids []store.UID) numSet {
var ns numSet
for _, r := range s.ranges {
first := r.first.number
if r.first.star {
if len(uids) == 0 {
continue
}
first = uint32(uids[0])
}
last := first
if r.last != nil {
last = r.last.number
if r.last.star {
if len(uids) == 0 {
continue
}
last = uint32(uids[len(uids)-1])
if first > last {
first = last
}
} else if r.first.star && last < first {
last = first
}
}
if first > last {
first, last = last, first
}
nr := numRange{first: setNumber{number: first}}
if first != last {
nr.last = &setNumber{number: last}
}
ns.ranges = append(ns.ranges, nr)
}
return ns
}
// whether numSet only has numbers (no star/search), and is strictly increasing.
func (s *numSet) isBasicIncreasing() bool {
if s.searchResult {
return false
}
var last uint32
for _, r := range s.ranges {
if r.first.star || r.first.number <= last || r.last != nil && (r.last.star || r.last.number < r.first.number) {
return false
}
last = r.first.number
if r.last != nil {
last = r.last.number
}
}
return true
}
type numIter struct {
s numSet
i int
r *rangeIter
}
// newIter must only be called on a numSet that is basic (no star/search) and ascending.
func (s numSet) newIter() *numIter {
return &numIter{s: s, i: 0, r: s.ranges[0].newIter()}
}
func (i *numIter) Next() (uint32, bool) {
if v, ok := i.r.Next(); ok {
return v, ok
}
i.i++
if i.i >= len(i.s.ranges) {
return 0, false
}
i.r = i.s.ranges[i.i].newIter()
return i.r.Next()
}
type rangeIter struct {
r numRange
o int
}
// newIter must only be called on a range in a numSet that is basic (no star/search) and ascending.
func (r numRange) newIter() *rangeIter {
return &rangeIter{r: r, o: 0}
}
func (r *rangeIter) Next() (uint32, bool) {
if r.o == 0 {
r.o++
return r.r.first.number, true
}
if r.r.last == nil || r.r.first.number+uint32(r.o) > r.r.last.number {
return 0, false
}
v := r.r.first.number + uint32(r.o)
r.o++
return v, true
}
// append adds a new number to the set, extending a range, or starting a new one (possibly the first).
// can only be used on basic numsets, without star/searchResult.
func (s *numSet) append(v uint32) {
if len(s.ranges) == 0 {
s.ranges = []numRange{{first: setNumber{number: v}}}
return
}
ri := len(s.ranges) - 1
r := s.ranges[ri]
if v == r.first.number+1 && r.last == nil {
s.ranges[ri].last = &setNumber{number: v}
} else if r.last != nil && v == r.last.number+1 {
r.last.number++
} else {
s.ranges = append(s.ranges, numRange{first: setNumber{number: v}})
}
}
type partial struct {
offset uint32
count uint32
@ -156,17 +318,18 @@ type fetchAtt struct {
type searchKey struct {
// Only one of searchKeys, seqSet and op can be non-nil/non-empty.
searchKeys []searchKey // In case of nested/multiple keys. Also for the top-level command.
seqSet *numSet // In case of bare sequence set. For op UID, field uidSet contains the parameter.
op string // Determines which of the fields below are set.
headerField string
astring string
date time.Time
atom string
number int64
searchKey *searchKey
searchKey2 *searchKey
uidSet numSet
searchKeys []searchKey // In case of nested/multiple keys. Also for the top-level command.
seqSet *numSet // In case of bare sequence set. For op UID, field uidSet contains the parameter.
op string // Determines which of the fields below are set.
headerField string
astring string
date time.Time
atom string
number int64
searchKey *searchKey
searchKey2 *searchKey
uidSet numSet
clientModseq *int64
}
func compactUIDSet(l []store.UID) (r numSet) {

27
imapserver/protocol_test.go
View File

@ -62,3 +62,30 @@ func TestNumSetContains(t *testing.T) {
check(!ss3.containsUID(1, []store.UID{2, 3}, nil))
check(!ss3.containsUID(3, []store.UID{1, 2, 3}, nil))
}
func TestNumSetInterpret(t *testing.T) {
parseNumSet := func(s string) numSet {
p := parser{upper: s}
return p.xnumSet0(true, false)
}
checkEqual := func(uids []store.UID, a, s string) {
t.Helper()
n := parseNumSet(a).interpretStar(uids)
ns := n.String()
if ns != s {
t.Fatalf("%s != %s", ns, s)
}
}
checkEqual([]store.UID{}, "1:*", "")
checkEqual([]store.UID{1}, "1:*", "1")
checkEqual([]store.UID{1, 3}, "1:*", "1:3")
checkEqual([]store.UID{1, 3}, "4:*", "3")
checkEqual([]store.UID{2, 3}, "*:4", "2:4")
checkEqual([]store.UID{2, 3}, "*:1", "2")
checkEqual([]store.UID{1, 2, 3}, "1,2,3", "1,2,3")
checkEqual([]store.UID{}, "1,2,3", "1,2,3")
checkEqual([]store.UID{}, "1:3", "1:3")
checkEqual([]store.UID{}, "3:1", "1:3")
}

96
imapserver/search.go
View File

@ -96,6 +96,7 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
}
var expungeIssued bool
var maxModSeq store.ModSeq
var uids []store.UID
c.xdbread(func(tx *bstore.Tx) {
@ -108,8 +109,11 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
if eargs == nil || max == 0 || len(eargs) != 1 {
for i, uid := range c.uids {
lastIndex = i
if c.searchMatch(tx, msgseq(i+1), uid, *sk, &expungeIssued) {
if match, modseq := c.searchMatch(tx, msgseq(i+1), uid, *sk, &expungeIssued); match {
uids = append(uids, uid)
if modseq > maxModSeq {
maxModSeq = modseq
}
if min == 1 && min+max == len(eargs) {
break
}
@ -119,8 +123,11 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
// And reverse search for MAX if we have only MAX or MAX combined with MIN.
if max == 1 && (len(eargs) == 1 || min+max == len(eargs)) {
for i := len(c.uids) - 1; i > lastIndex; i-- {
if c.searchMatch(tx, msgseq(i+1), c.uids[i], *sk, &expungeIssued) {
if match, modseq := c.searchMatch(tx, msgseq(i+1), c.uids[i], *sk, &expungeIssued); match {
uids = append(uids, c.uids[i])
if modseq > maxModSeq {
maxModSeq = modseq
}
break
}
}
@ -147,11 +154,24 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
}
s += " " + fmt.Sprintf("%d", v)
}
// Since we don't have the max modseq for the possibly partial uid range we're
// writing here within hand reach, we conveniently interpret the ambiguous "for all
// messages being returned" in ../rfc/7162:1107 as meaning over all lines that we
// write. And that clients only commit this value after they have seen the tagged
// end of the command. Appears to be recommended behaviour, ../rfc/7162:2323.
// ../rfc/7162:1077 ../rfc/7162:1101
var modseq string
if sk.hasModseq() {
// ../rfc/7162:2557
modseq = fmt.Sprintf(" (MODSEQ %d)", maxModSeq.Client())
}
c.bwritelinef("* SEARCH%s%s", s, modseq)
uids = uids[n:]
c.bwritelinef("* SEARCH%s", s)
}
} else {
// New-style ESEARCH response. ../rfc/9051:6546 ../rfc/4466:522
// New-style ESEARCH response syntax: ../rfc/9051:6546 ../rfc/4466:522
if save {
// ../rfc/9051:3784 ../rfc/5182:13
@ -195,6 +215,13 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
if eargs["ALL"] && len(uids) > 0 {
resp += fmt.Sprintf(" ALL %s", compactUIDSet(uids).String())
}
// Interaction between ESEARCH and CONDSTORE: ../rfc/7162:1211 ../rfc/4731:273
// Summary: send the highest modseq of the returned messages.
if sk.hasModseq() && len(uids) > 0 {
resp += fmt.Sprintf(" MODSEQ %d", maxModSeq.Client())
}
c.bwritelinef("%s", resp)
}
}
@ -215,10 +242,11 @@ type search struct {
m store.Message
p *message.Part
expungeIssued *bool
hasModseq bool
}
func (c *conn) searchMatch(tx *bstore.Tx, seq msgseq, uid store.UID, sk searchKey, expungeIssued *bool) bool {
s := search{c: c, tx: tx, seq: seq, uid: uid, expungeIssued: expungeIssued}
func (c *conn) searchMatch(tx *bstore.Tx, seq msgseq, uid store.UID, sk searchKey, expungeIssued *bool) (bool, store.ModSeq) {
s := search{c: c, tx: tx, seq: seq, uid: uid, expungeIssued: expungeIssued, hasModseq: sk.hasModseq()}
defer func() {
if s.mr != nil {
err := s.mr.Close()
@ -229,12 +257,42 @@ func (c *conn) searchMatch(tx *bstore.Tx, seq msgseq, uid store.UID, sk searchKe
return s.match(sk)
}
func (s *search) match(sk searchKey) bool {
func (s *search) match(sk searchKey) (match bool, modseq store.ModSeq) {
// Instead of littering all the cases in match0 with calls to get modseq, we do it once
// here in case of a match.
defer func() {
if match && s.hasModseq {
if s.m.ID == 0 {
match = s.xloadMessage()
}
modseq = s.m.ModSeq
}
}()
match = s.match0(sk)
return
}
func (s *search) xloadMessage() bool {
q := bstore.QueryTx[store.Message](s.tx)
q.FilterNonzero(store.Message{MailboxID: s.c.mailboxID, UID: s.uid})
m, err := q.Get()
if err == bstore.ErrAbsent || err == nil && m.Expunged {
// ../rfc/2180:607
*s.expungeIssued = true
return false
}
xcheckf(err, "get message")
s.m = m
return true
}
func (s *search) match0(sk searchKey) bool {
c := s.c
if sk.searchKeys != nil {
for _, ssk := range sk.searchKeys {
if !s.match(ssk) {
if !s.match0(ssk) {
return false
}
}
@ -273,33 +331,26 @@ func (s *search) match(sk searchKey) bool {
// We do not implement the RECENT flag. All messages are not recent.
return false
case "NOT":
return !s.match(*sk.searchKey)
return !s.match0(*sk.searchKey)
case "OR":
return s.match(*sk.searchKey) || s.match(*sk.searchKey2)
return s.match0(*sk.searchKey) || s.match0(*sk.searchKey2)
case "UID":
return sk.uidSet.containsUID(s.uid, c.uids, c.searchResult)
}
// Parsed message.
if s.mr == nil {
q := bstore.QueryTx[store.Message](s.tx)
q.FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: s.uid})
m, err := q.Get()
if err == bstore.ErrAbsent {
// ../rfc/2180:607
*s.expungeIssued = true
if !s.xloadMessage() {
return false
}
xcheckf(err, "get message")
s.m = m
// Closed by searchMatch after all (recursive) search.match calls are finished.
s.mr = c.account.MessageReader(m)
s.mr = c.account.MessageReader(s.m)
if m.ParsedBuf == nil {
if s.m.ParsedBuf == nil {
c.log.Error("missing parsed message")
} else {
p, err := m.LoadPart(s.mr)
p, err := s.m.LoadPart(s.mr)
xcheckf(err, "load parsed message")
s.p = &p
}
@ -385,6 +436,9 @@ func (s *search) match(sk searchKey) bool {
return s.m.Size > sk.number
case "SMALLER":
return s.m.Size < sk.number
case "MODSEQ":
// ../rfc/7162:1045
return s.m.ModSeq.Client() >= *sk.clientModseq
}
if s.p == nil {

90
imapserver/search_test.go
View File

@ -39,6 +39,16 @@ func (tc *testconn) xsearch(nums ...uint32) {
tc.xuntagged(imapclient.UntaggedSearch(nums))
}
func (tc *testconn) xsearchmodseq(modseq int64, nums ...uint32) {
tc.t.Helper()
if len(nums) == 0 {
tc.xnountagged()
return
}
tc.xuntagged(imapclient.UntaggedSearchModSeq{Nums: nums, ModSeq: modseq})
}
func (tc *testconn) xesearch(exp imapclient.UntaggedEsearch) {
tc.t.Helper()
@ -220,43 +230,11 @@ func TestSearch(t *testing.T) {
tc.transactf("ok", `search charset utf-8 text "mox"`)
tc.xsearch(2, 3)
// esearchall makes an UntaggedEsearch response with All set, for comparisons.
esearchall0 := func(ss string) imapclient.NumSet {
seqset := imapclient.NumSet{}
for _, rs := range strings.Split(ss, ",") {
t := strings.Split(rs, ":")
if len(t) > 2 {
panic("bad seqset")
}
var first uint32
var last *uint32
if t[0] != "*" {
v, err := strconv.ParseUint(t[0], 10, 32)
if err != nil {
panic("parse first")
}
first = uint32(v)
}
if len(t) == 2 {
if t[1] != "*" {
v, err := strconv.ParseUint(t[1], 10, 32)
if err != nil {
panic("parse last")
}
u := uint32(v)
last = &u
}
}
seqset.Ranges = append(seqset.Ranges, imapclient.NumRange{First: first, Last: last})
}
return seqset
}
esearchall := func(ss string) imapclient.UntaggedEsearch {
return imapclient.UntaggedEsearch{All: esearchall0(ss)}
}
uintptr := func(v uint32) *uint32 {
uint32ptr := func(v uint32) *uint32 {
return &v
}
@ -265,10 +243,10 @@ func TestSearch(t *testing.T) {
tc.xesearch(esearchall("1:3")) // Without any options, "ALL" is implicit.
tc.transactf("ok", "search return (min max count all) all")
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uintptr(3), All: esearchall0("1:3")})
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uint32ptr(3), All: esearchall0("1:3")})
tc.transactf("ok", "UID search return (min max count all) all")
tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uintptr(3), All: esearchall0("5:7")})
tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uint32ptr(3), All: esearchall0("5:7")})
tc.transactf("ok", "search return (min) all")
tc.xesearch(imapclient.UntaggedEsearch{Min: 1})
@ -304,19 +282,19 @@ func TestSearch(t *testing.T) {
tc.xesearch(imapclient.UntaggedEsearch{})
tc.transactf("ok", "search return (min max all count) not all")
tc.xesearch(imapclient.UntaggedEsearch{Count: uintptr(0)})
tc.xesearch(imapclient.UntaggedEsearch{Count: uint32ptr(0)})
tc.transactf("ok", "search return (min max count all) 1,3")
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uintptr(2), All: esearchall0("1,3")})
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uint32ptr(2), All: esearchall0("1,3")})
tc.transactf("ok", "search return (min max count all) UID 5,7")
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uintptr(2), All: esearchall0("1,3")})
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uint32ptr(2), All: esearchall0("1,3")})
tc.transactf("ok", "uid search return (min max count all) 1,3")
tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uintptr(2), All: esearchall0("5,7")})
tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uint32ptr(2), All: esearchall0("5,7")})
tc.transactf("ok", "uid search return (min max count all) UID 5,7")
tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uintptr(2), All: esearchall0("5,7")})
tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uint32ptr(2), All: esearchall0("5,7")})
tc.transactf("no", `search return () charset unknown text "mox"`)
tc.transactf("ok", `search return () charset us-ascii text "mox"`)
@ -353,3 +331,35 @@ func TestSearch(t *testing.T) {
tc.transactf("ok", `search undraft`)
tc.xesearch(esearchall("1:2"))
}
// esearchall makes an UntaggedEsearch response with All set, for comparisons.
func esearchall0(ss string) imapclient.NumSet {
seqset := imapclient.NumSet{}
for _, rs := range strings.Split(ss, ",") {
t := strings.Split(rs, ":")
if len(t) > 2 {
panic("bad seqset")
}
var first uint32
var last *uint32
if t[0] != "*" {
v, err := strconv.ParseUint(t[0], 10, 32)
if err != nil {
panic("parse first")
}
first = uint32(v)
}
if len(t) == 2 {
if t[1] != "*" {
v, err := strconv.ParseUint(t[1], 10, 32)
if err != nil {
panic("parse last")
}
u := uint32(v)
last = &u
}
}
seqset.Ranges = append(seqset.Ranges, imapclient.NumRange{First: first, Last: last})
}
return seqset
}

613
imapserver/server.go
View File

File diff suppressed because it is too large Load Diff

32
imapserver/server_test.go
View File

@ -21,6 +21,8 @@ import (
"github.com/mjl-/mox/store"
)
var ctxbg = context.Background()
func init() {
sanityChecks = true
@ -156,6 +158,7 @@ type testconn struct {
client *imapclient.Conn
done chan struct{}
serverConn net.Conn
account *store.Account
// Result of last command.
lastUntagged []imapclient.Untagged
@ -190,14 +193,15 @@ func (tc *testconn) xcodeArg(v any) {
}
}
func (tc *testconn) xuntagged(exps ...any) {
func (tc *testconn) xuntagged(exps ...imapclient.Untagged) {
tc.t.Helper()
tc.xuntaggedCheck(true, exps...)
tc.xuntaggedOpt(true, exps...)
}
func (tc *testconn) xuntaggedCheck(all bool, exps ...any) {
func (tc *testconn) xuntaggedOpt(all bool, exps ...imapclient.Untagged) {
tc.t.Helper()
last := append([]imapclient.Untagged{}, tc.lastUntagged...)
var mismatch any
next:
for ei, exp := range exps {
for i, l := range last {
@ -205,12 +209,16 @@ next:
continue
}
if !reflect.DeepEqual(l, exp) {
tc.t.Fatalf("untagged data mismatch, got:\n\t%T %#v\nexpected:\n\t%T %#v", l, l, exp, exp)
mismatch = l
continue
}
copy(last[i:], last[i+1:])
last = last[:len(last)-1]
continue next
}
if mismatch != nil {
tc.t.Fatalf("untagged data mismatch, got:\n\t%T %#v\nexpected:\n\t%T %#v", mismatch, mismatch, exp, exp)
}
var next string
if len(tc.lastUntagged) > 0 {
next = fmt.Sprintf(", next %#v", tc.lastUntagged[0])
@ -293,11 +301,21 @@ func (tc *testconn) waitDone() {
}
func (tc *testconn) close() {
err := tc.account.Close()
tc.check(err, "close account")
tc.client.Close()
tc.serverConn.Close()
tc.waitDone()
}
func xparseNumSet(s string) imapclient.NumSet {
ns, err := imapclient.ParseNumSet(s)
if err != nil {
panic(fmt.Sprintf("parsing numset %s: %s", s, err))
}
return ns
}
var connCounter int64
func start(t *testing.T) *testconn {
@ -314,7 +332,7 @@ func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS bool) *testconn
if first {
os.RemoveAll("../testdata/imap/data")
}
mox.Context = context.Background()
mox.Context = ctxbg
mox.ConfigStaticPath = "../testdata/imap/mox.conf"
mox.MustLoadConfig(true, false)
acc, err := store.OpenAccount("mjl")
@ -323,8 +341,6 @@ func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS bool) *testconn
err = acc.SetPassword("testtest")
tcheck(t, err, "set password")
}
err = acc.Close()
tcheck(t, err, "close account")
var switchDone chan struct{}
if first {
switchDone = store.Switchboard()
@ -352,7 +368,7 @@ func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS bool) *testconn
}()
client, err := imapclient.New(clientConn, true)
tcheck(t, err, "new client")
return &testconn{t: t, conn: clientConn, client: client, done: done, serverConn: serverConn}
return &testconn{t: t, conn: clientConn, client: client, done: done, serverConn: serverConn, account: acc}
}
func fakeCert(t *testing.T) tls.Certificate {

2
imapserver/store_test.go
View File

@ -78,7 +78,7 @@ func TestStore(t *testing.T) {
// Flags are added to mailbox, not removed.
flags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent new a b c d e different`, " ")
tc.xuntaggedCheck(false, imapclient.UntaggedFlags(flags))
tc.xuntaggedOpt(false, imapclient.UntaggedFlags(flags))
tc.transactf("no", `store 1 flags ()`) // No permission to set flags.
}