From 479bf291248947db02553a963f9e9162be9665fa Mon Sep 17 00:00:00 2001 From: Mechiel Lukkien Date: Mon, 31 Mar 2025 18:33:15 +0200 Subject: [PATCH] imapserver: implement the MULTISEARCH extension, with its ESEARCH command --- README.md | 4 +- imapclient/parse.go | 49 ++- imapclient/protocol.go | 21 +- imapserver/condstore_test.go | 5 +- imapserver/parse.go | 17 + imapserver/protocol.go | 73 ++++- imapserver/search.go | 590 ++++++++++++++++++++++++++--------- imapserver/search_test.go | 373 +++++++++++++++++++++- imapserver/server.go | 10 +- rfc/index.txt | 4 +- testdata/imap/domains.conf | 11 + 11 files changed, 969 insertions(+), 188 deletions(-) diff --git a/README.md b/README.md index 979fc77..4dba9ab 100644 --- a/README.md +++ b/README.md @@ -152,9 +152,9 @@ https://nlnet.nl/project/Mox/. - External addresses in aliases/lists. - Autoresponder (out of office/vacation) - Mailing list manager -- IMAP extensions for "online"/non-syncing/webmail clients (MULTISEARCH, SORT (including +- IMAP extensions for "online"/non-syncing/webmail clients (SORT (including DISPLAYFROM, DISPLAYTO), THREAD, PARTIAL, CONTEXT=SEARCH CONTEXT=SORT ESORT, - FILTERS, PREVIEW) + FILTERS) - IMAP ACL support, for account sharing (interacts with many extensions and code) - Improve support for mobile clients with extensions: IMAP URLAUTH, SMTP CHUNKING and BINARYMIME, IMAP CATENATE diff --git a/imapclient/parse.go b/imapclient/parse.go index 93923ba..0eed343 100644 --- a/imapclient/parse.go +++ b/imapclient/parse.go @@ -1414,15 +1414,54 @@ func (c *Conn) xneedDisabled(msg string, caps ...Capability) { // ../rfc/9051:6546 // Already consumed: "ESEARCH" func (c *Conn) xesearchResponse() (r UntaggedEsearch) { - if !c.space() { return } + if c.take('(') { - // ../rfc/9051:6921 - c.xtake("TAG") - c.xspace() - r.Correlator = c.xastring() + // ../rfc/9051:6921 ../rfc/7377:465 + seen := map[string]bool{} + for { + var kind string + if c.peek('t') || c.peek('T') { + kind = "TAG" + c.xtake(kind) + c.xspace() + r.Tag = c.xastring() + } else if c.peek('m') || c.peek('M') { + kind = "MAILBOX" + c.xtake(kind) + c.xspace() + r.Mailbox = c.xastring() + if r.Mailbox == "" { + c.xerrorf("invalid empty mailbox in search correlator") + } + } else if c.peek('u') || c.peek('U') { + kind = "UIDVALIDITY" + c.xtake(kind) + c.xspace() + r.UIDValidity = c.xnzuint32() + } else { + c.xerrorf("expected tag/correlator, mailbox or uidvalidity") + } + + if seen[kind] { + c.xerrorf("duplicate search correlator %q", kind) + } + seen[kind] = true + + if !c.take(' ') { + break + } + } + + if r.Tag == "" { + c.xerrorf("missing tag search correlator") + } + if (r.Mailbox != "") != (r.UIDValidity != 0) { + c.xerrorf("mailbox and uidvalidity correlators must both be absent or both be present") + } + c.xtake(")") } if !c.space() { diff --git a/imapclient/protocol.go b/imapclient/protocol.go index 0c8f29c..3c42c27 100644 --- a/imapclient/protocol.go +++ b/imapclient/protocol.go @@ -41,6 +41,7 @@ const ( CapMultiAppend Capability = "MULTIAPPEND" // ../rfc/3502:33 CapReplace Capability = "REPLACE" // ../rfc/8508:155 CapPreview Capability = "PREVIEW" // ../rfc/8970:114 + CapMultiSearch Capability = "MULTISEARCH" // ../rfc/7377:187 ) // Status is the tagged final result of a command. @@ -314,15 +315,17 @@ type UntaggedLsub struct { // Fields are optional and zero if absent. type UntaggedEsearch struct { - // ../rfc/9051:6546 - Correlator string - UID bool - Min uint32 - Max uint32 - All NumSet - Count *uint32 - ModSeq int64 - Exts []EsearchDataExt + Tag string // ../rfc/9051:6546 + Mailbox string // For MULTISEARCH. ../rfc/7377:437 + UIDValidity uint32 // For MULTISEARCH, ../rfc/7377:438 + + UID bool + Min uint32 + Max uint32 + All NumSet + Count *uint32 + ModSeq int64 + Exts []EsearchDataExt } // UntaggedVanished is used in QRESYNC to send UIDs that have been removed. diff --git a/imapserver/condstore_test.go b/imapserver/condstore_test.go index 378725e..0353a23 100644 --- a/imapserver/condstore_test.go +++ b/imapserver/condstore_test.go @@ -264,9 +264,6 @@ func testCondstoreQresync(t *testing.T, qresync bool) { tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq %d", clientModseq) tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1, All: esearchall0("1"), ModSeq: clientModseq}) - uint32ptr := func(v uint32) *uint32 { - return &v - } tc.transactf("ok", "Search Return (Count) 1:* Modseq 0") tc.xesearch(imapclient.UntaggedEsearch{Count: uint32ptr(6), ModSeq: clientModseq}) @@ -331,7 +328,7 @@ func testCondstoreQresync(t *testing.T, qresync bool) { tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq 8") tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1, All: esearchall0("1"), ModSeq: 8}) tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq 9") - tc.xuntagged(imapclient.UntaggedEsearch{Correlator: tc.client.LastTag}) + tc.xuntagged(imapclient.UntaggedEsearch{Tag: tc.client.LastTag}) // store, cannot modify expunged messages. tc.transactf("ok", `Uid Store 3,4 (Unchangedsince %d) +Flags (label2)`, clientModseq) diff --git a/imapserver/parse.go b/imapserver/parse.go index f0d31fa..b02030a 100644 --- a/imapserver/parse.go +++ b/imapserver/parse.go @@ -965,6 +965,23 @@ func (sk searchKey) hasModseq() bool { return false } +// Whether we need message sequence numbers to evaluate. If not, we cannot optimize +// when only MAX is requested through a reverse query. +func (sk searchKey) needSeq() bool { + for _, k := range sk.searchKeys { + if k.needSeq() { + return true + } + } + if sk.searchKey != nil && sk.searchKey.needSeq() { + return true + } + if sk.searchKey2 != nil && sk.searchKey2.needSeq() { + return true + } + return sk.seqSet != nil && !sk.seqSet.searchResult +} + // ../rfc/9051:6489 ../rfc/3501:4692 func (p *parser) xdateDay() int { d := p.xdigit() diff --git a/imapserver/protocol.go b/imapserver/protocol.go index f19bc26..d91c81b 100644 --- a/imapserver/protocol.go +++ b/imapserver/protocol.go @@ -32,17 +32,25 @@ func (ss numSet) containsSeq(seq msgseq, uids []store.UID, searchResult []store. uid := uids[int(seq)-1] return uidSearch(searchResult, uid) > 0 && uidSearch(uids, uid) > 0 } + return ss.containsSeqCount(seq, uint32(len(uids))) +} + +// containsSeqCount returns whether seq is contained in ss, which must not be a searchResult, assuming the message count. +func (ss numSet) containsSeqCount(seq msgseq, msgCount uint32) bool { + if msgCount == 0 { + return false + } for _, r := range ss.ranges { first := r.first.number - if r.first.star || first > uint32(len(uids)) { - first = uint32(len(uids)) + if r.first.star || first > msgCount { + first = msgCount } last := first if r.last != nil { last = r.last.number - if r.last.star || last > uint32(len(uids)) { - last = uint32(len(uids)) + if r.last.star || last > msgCount { + last = msgCount } } if first > last { @@ -87,6 +95,56 @@ func (ss numSet) containsUID(uid store.UID, uids []store.UID, searchResult []sto return false } +// containsKnownUID returns whether uid, which is known to exist, matches the numSet. +// highestUID must return the highest/last UID in the mailbox, or an error. A last UID must +// exist, otherwise this method wouldn't have been called with a known uid. +// highestUID is needed for interpreting UID sets like ":*" where num is +// higher than the uid to check. +func (ss numSet) containsKnownUID(uid store.UID, searchResult []store.UID, highestUID func() (store.UID, error)) (bool, error) { + if ss.searchResult { + return uidSearch(searchResult, uid) > 0, nil + } + + for _, r := range ss.ranges { + a := store.UID(r.first.number) + // Num in :* can be larger than last, but it still matches the last... + // Similar for *:. ../rfc/9051:4814 + if r.first.star { + if r.last != nil && uid >= store.UID(r.last.number) { + return true, nil + } + + var err error + a, err = highestUID() + if err != nil { + return false, err + } + } + b := a + if r.last != nil { + b = store.UID(r.last.number) + if r.last.star { + if uid >= a { + return true, nil + } + + var err error + b, err = highestUID() + if err != nil { + return false, err + } + } + } + if a > b { + a, b = b, a + } + if uid >= a && uid <= b { + return true, nil + } + } + return false, nil +} + // contains returns whether the numset contains the number. // only allowed on basic, strictly increasing numsets. func (ss numSet) contains(v uint32) bool { @@ -312,9 +370,10 @@ type fetchAtt struct { type searchKey struct { // Only one of searchKeys, seqSet and op can be non-nil/non-empty. - searchKeys []searchKey // In case of nested/multiple keys. Also for the top-level command. - seqSet *numSet // In case of bare sequence set. For op UID, field uidSet contains the parameter. - op string // Determines which of the fields below are set. + searchKeys []searchKey // In case of nested/multiple keys. Also for the top-level command. + seqSet *numSet // In case of bare sequence set. For op UID, field uidSet contains the parameter. + op string // Determines which of the fields below are set. + headerField string astring string date time.Time diff --git a/imapserver/search.go b/imapserver/search.go index 9ee69c8..99381dc 100644 --- a/imapserver/search.go +++ b/imapserver/search.go @@ -1,9 +1,12 @@ package imapserver import ( + "cmp" "fmt" "log/slog" + "maps" "net/textproto" + "slices" "strings" "time" @@ -11,29 +14,76 @@ import ( "github.com/mjl-/mox/message" "github.com/mjl-/mox/store" - "slices" ) // If last search output was this long ago, we write an untagged inprogress // response. Changed during tests. ../rfc/9585:109 var inProgressPeriod = time.Duration(10 * time.Second) +// ESEARCH allows searching multiple mailboxes, referenced through mailbox filters +// borrowed from the NOTIFY extension. Unlike the regular extended SEARCH/UID +// SEARCH command that always returns an ESEARCH response, the ESEARCH command only +// returns ESEARCH responses when there were matches in a mailbox. +// +// ../rfc/7377:159 +func (c *conn) cmdEsearch(tag, cmd string, p *parser) { + c.cmdxSearch(true, true, tag, cmd, p) +} + // Search returns messages matching criteria specified in parameters. // -// State: Selected -func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) { - // Command: ../rfc/9051:3716 ../rfc/4731:31 ../rfc/4466:354 ../rfc/3501:2723 - // Examples: ../rfc/9051:3986 ../rfc/4731:153 ../rfc/3501:2975 - // Syntax: ../rfc/9051:6918 ../rfc/4466:611 ../rfc/3501:4954 +// State: Selected for SEARCH and UID SEARCH, Authenticated or selectd for ESEARCH. +func (c *conn) cmdxSearch(isUID, isE bool, tag, cmd string, p *parser) { + // Command: ../rfc/9051:3716 ../rfc/7377:159 ../rfc/6237:142 ../rfc/4731:31 ../rfc/4466:354 ../rfc/3501:2723 + // Examples: ../rfc/9051:3986 ../rfc/7377:385 ../rfc/6237:323 ../rfc/4731:153 ../rfc/3501:2975 + // Syntax: ../rfc/9051:6918 ../rfc/7377:462 ../rfc/6237:403 ../rfc/4466:611 ../rfc/3501:4954 - // We will respond with ESEARCH instead of SEARCH if "RETURN" is present or for IMAP4rev2. + // We will respond with ESEARCH instead of SEARCH if "RETURN" is present or for IMAP4rev2 or for isE (ESEARCH command). var eargs map[string]bool // Options except SAVE. Nil means old-style SEARCH response. var save bool // For SAVE option. Kept separately for easier handling of MIN/MAX later. - // IMAP4rev2 always returns ESEARCH, even with absent RETURN. - if c.enabled[capIMAP4rev2] { + if c.enabled[capIMAP4rev2] || isE { eargs = map[string]bool{} } + + // The ESEARCH command has various ways to specify which mailboxes are to be + // searched. We parse and gather the request first, and evaluate them to mailboxes + // after parsing, when we start and have a DB transaction. + type mailboxSpec struct { + Kind string + Args []string + } + var mailboxSpecs []mailboxSpec + + // ../rfc/7377:468 + if isE && p.take(" IN (") { + for { + mbs := mailboxSpec{} + mbs.Kind = p.xtakelist("SELECTED", "INBOXES", "PERSONAL", "SUBSCRIBED", "SUBTREE-ONE", "SUBTREE", "MAILBOXES") + switch mbs.Kind { + case "SUBTREE", "SUBTREE-ONE", "MAILBOXES": + p.xtake(" ") + if p.take("(") { + for { + mbs.Args = append(mbs.Args, p.xmailbox()) + if !p.take(" ") { + break + } + } + p.xtake(")") + } else { + mbs.Args = []string{p.xmailbox()} + } + } + mailboxSpecs = append(mailboxSpecs, mbs) + + if !p.take(" ") { + break + } + } + p.xtake(")") + // We are not parsing the scope-options since there aren't any defined yet. ../rfc/7377:469 + } // ../rfc/9051:6967 if p.take(" RETURN (") { eargs = map[string]bool{} @@ -131,16 +181,22 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) { // If we only have a MIN and/or MAX, we can stop processing as soon as we // have those matches. - var min, max int + var min1, max1 int if eargs["MIN"] { - min = 1 + min1 = 1 } if eargs["MAX"] { - max = 1 + max1 = 1 } - var expungeIssued bool - var maxModSeq store.ModSeq + // We'll have one Result per mailbox we are searching. For regular (UID) SEARCH + // commands, we'll have just one, for the selected mailbox. + type Result struct { + Mailbox store.Mailbox + MaxModSeq store.ModSeq + UIDs []store.UID + } + var results []Result // We periodically send an untagged OK with INPROGRESS code while searching, to let // clients doing slow searches know we're still working. @@ -151,73 +207,308 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) { inProgressTag = dquote(tag).pack(c) } - var uids []store.UID c.xdbread(func(tx *bstore.Tx) { - c.xmailboxID(tx, c.mailboxID) // Validate. + // Gather mailboxes to operate on. Usually just the selected mailbox. But with the + // ESEARCH command, we may be searching multiple. + var mailboxes []store.Mailbox + if len(mailboxSpecs) > 0 { + // While gathering, we deduplicate mailboxes. ../rfc/7377:312 + m := map[int64]store.Mailbox{} + for _, mbs := range mailboxSpecs { + switch mbs.Kind { + case "SELECTED": + // ../rfc/7377:306 + if c.state != stateSelected { + xsyntaxErrorf("cannot use ESEARCH with selected when state is not selected") + } + + mb := c.xmailboxID(tx, c.mailboxID) // Validate. + m[mb.ID] = mb + + case "INBOXES": + // Inbox and everything below. And we look at destinations and rulesets. We all + // mailboxes from the destinations, and all from the rulesets except when + // ListAllowDomain is non-empty. + // ../rfc/5465:822 + q := bstore.QueryTx[store.Mailbox](tx) + q.FilterEqual("Expunged", false) + q.FilterGreaterEqual("Name", "Inbox") + q.SortAsc("Name") + for mb, err := range q.All() { + xcheckf(err, "list mailboxes") + if mb.Name != "Inbox" && !strings.HasPrefix(mb.Name, "Inbox/") { + break + } + m[mb.ID] = mb + } + + conf, _ := c.account.Conf() + for _, dest := range conf.Destinations { + if dest.Mailbox != "" && dest.Mailbox != "Inbox" { + mb, err := c.account.MailboxFind(tx, dest.Mailbox) + xcheckf(err, "find mailbox from destination") + if mb != nil { + m[mb.ID] = *mb + } + } + + for _, rs := range dest.Rulesets { + if rs.ListAllowDomain != "" || rs.Mailbox == "" { + continue + } + + mb, err := c.account.MailboxFind(tx, rs.Mailbox) + xcheckf(err, "find mailbox from ruleset") + if mb != nil { + m[mb.ID] = *mb + } + } + } + + case "PERSONAL": + // All mailboxes in the personal namespace. Which is all mailboxes for us. + // ../rfc/5465:817 + for mb, err := range bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).All() { + xcheckf(err, "list mailboxes") + m[mb.ID] = mb + } + + case "SUBSCRIBED": + // Mailboxes that are subscribed. Will typically be same as personal, since we + // subscribe to all mailboxes. But user can manage subscriptions differently. + // ../rfc/5465:831 + for mb, err := range bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).All() { + xcheckf(err, "list mailboxes") + if err := tx.Get(&store.Subscription{Name: mb.Name}); err == nil { + m[mb.ID] = mb + } else if err != bstore.ErrAbsent { + xcheckf(err, "lookup subscription for mailbox") + } + } + + case "SUBTREE", "SUBTREE-ONE": + // The mailbox name itself, and children. ../rfc/5465:847 + // SUBTREE is arbitrarily deep, SUBTREE-ONE is one level deeper than requested + // mailbox. The mailbox itself is included too ../rfc/7377:274 + + // We don't have to worry about loops. Mailboxes are not in the file system. + // ../rfc/7377:291 + + for _, name := range mbs.Args { + name = xcheckmailboxname(name, true) + + one := mbs.Kind == "SUBTREE-ONE" + var ntoken int + if one { + ntoken = len(strings.Split(name, "/")) + } + + q := bstore.QueryTx[store.Mailbox](tx) + q.FilterEqual("Expunged", false) + q.FilterGreaterEqual("Name", name) + q.SortAsc("Name") + for mb, err := range q.All() { + xcheckf(err, "list mailboxes") + if mb.Name != name && !strings.HasPrefix(mb.Name, name+"/") { + break + } + if !one || mb.Name == name || len(strings.Split(mb.Name, "/")) == ntoken+1 { + m[mb.ID] = mb + } + } + } + + case "MAILBOXES": + // Just the specified mailboxes. ../rfc/5465:853 + for _, name := range mbs.Args { + name = xcheckmailboxname(name, true) + + // If a mailbox doesn't exist, we don't treat it as an error. Seems reasonable + // giving we are searching. Messages may not exist. And likewise for the mailbox. + // Just results in no hits. + mb, err := c.account.MailboxFind(tx, name) + xcheckf(err, "looking up mailbox") + if mb != nil { + m[mb.ID] = *mb + } + } + + default: + panic("missing case") + } + } + mailboxes = slices.Collect(maps.Values(m)) + slices.SortFunc(mailboxes, func(a, b store.Mailbox) int { + return cmp.Compare(a.Name, b.Name) + }) + + // If no source mailboxes were specified (no mailboxSpecs), the selected mailbox is + // used below. ../rfc/7377:298 + } else { + mb := c.xmailboxID(tx, c.mailboxID) // Validate. + mailboxes = []store.Mailbox{mb} + } + + if save && !(len(mailboxes) == 1 && mailboxes[0].ID == c.mailboxID) { + // ../rfc/7377:319 + xsyntaxErrorf("can only use SAVE on selected mailbox") + } + runlock() runlock = func() {} - // Normal forward search when we don't have MAX only. We only send an "inprogress" - // goal if we know how many messages we have to check. - forward := eargs == nil || max == 0 || len(eargs) != 1 - reverse := max == 1 && (len(eargs) == 1 || min+max == len(eargs)) + // Determine if search has a sequence set without search results. If so, we need + // sequence numbers for matching, and we must always go through the messages in + // forward order. No reverse search for MAX only. + needSeq := (len(mailboxes) > 1 || len(mailboxes) == 1 && mailboxes[0].ID != c.mailboxID) && sk.needSeq() + + forward := eargs == nil || max1 == 0 || len(eargs) != 1 || needSeq + reverse := max1 == 1 && (len(eargs) == 1 || min1+max1 == len(eargs)) && !needSeq + + // We set a worst-case "goal" of having gone through all messages in all mailboxes. + // Sometimes, we can be faster, when we only do a MIN and/or MAX query and we can + // stop early. We'll account for that as we go. For the selected mailbox, we'll + // only look at those the session has already seen. goal := "nil" - if len(c.uids) > 0 && forward != reverse { - goal = fmt.Sprintf("%d", len(c.uids)) + var total uint32 + for _, mb := range mailboxes { + if mb.ID == c.mailboxID { + total += uint32(len(c.uids)) + } else { + total += uint32(mb.Total + mb.Deleted) + } + } + if total > 0 { + // Goal is always non-zero. ../rfc/9585:232 + goal = fmt.Sprintf("%d", total) } - var lastIndex = -1 - if forward { - for i, uid := range c.uids { - lastIndex = i - if time.Since(inProgressLast) > inProgressPeriod { - c.writelinef("* OK [INPROGRESS (%s %d %s)] still searching", inProgressTag, i, goal) - inProgressLast = time.Now() + var progress uint32 + for _, mb := range mailboxes { + var lastUID store.UID + + result := Result{Mailbox: mb} + + msgCount := uint32(mb.MailboxCounts.Total + mb.MailboxCounts.Deleted) + if mb.ID == c.mailboxID { + msgCount = uint32(len(c.uids)) + } + + // Used for interpreting UID sets with a star, like "1:*" and "10:*". Only called + // for UIDs that are higher than the number, since "10:*" evaluates to "10:5" if 5 + // is the highest UID, and UID 5-10 would all match. + var cachedHighestUID store.UID + highestUID := func() (store.UID, error) { + if cachedHighestUID > 0 { + return cachedHighestUID, nil } - if match, modseq := c.searchMatch(tx, msgseq(i+1), uid, *sk, bodySearch, textSearch, &expungeIssued); match { - uids = append(uids, uid) - if modseq > maxModSeq { - maxModSeq = modseq + + q := bstore.QueryTx[store.Message](tx) + q.FilterNonzero(store.Message{MailboxID: mb.ID}) + q.FilterEqual("Expunged", false) + q.SortDesc("UID") + q.Limit(1) + m, err := q.Get() + cachedHighestUID = m.UID + return cachedHighestUID, err + } + + progressOrig := progress + + if forward { + // We track this for non-selected mailboxes. searchMatch will look the message + // sequence number for this session up if we are searching the selected mailbox. + var seq msgseq = 1 + + q := bstore.QueryTx[store.Message](tx) + q.FilterNonzero(store.Message{MailboxID: mb.ID}) + q.FilterEqual("Expunged", false) + q.SortAsc("UID") + for m, err := range q.All() { + xcheckf(err, "list messages in mailbox") + + // We track this for the "reverse" case, we'll stop before seeing lastUID. + lastUID = m.UID + + if time.Since(inProgressLast) > inProgressPeriod { + c.writelinef("* OK [INPROGRESS (%s %d %s)] still searching", inProgressTag, progress, goal) + inProgressLast = time.Now() } - if min == 1 && min+max == len(eargs) { + progress++ + + if c.searchMatch(tx, msgCount, seq, m, *sk, bodySearch, textSearch, highestUID) { + result.UIDs = append(result.UIDs, m.UID) + result.MaxModSeq = max(result.MaxModSeq, m.ModSeq) + if min1 == 1 && min1+max1 == len(eargs) { + if !needSeq { + break + } + // We only need a MIN and a MAX, but we also need sequence numbers so we are + // walking through and collecting all UIDs. Correct for that, keeping only the MIN + // (first) + // and MAX (second). + if len(result.UIDs) == 3 { + result.UIDs[1] = result.UIDs[2] + result.UIDs = result.UIDs[:2] + } + } + } + seq++ + } + } + // And reverse search for MAX if we have only MAX or MAX combined with MIN, and + // don't need sequence numbers. We just need a single match, then we stop. + if reverse { + q := bstore.QueryTx[store.Message](tx) + q.FilterNonzero(store.Message{MailboxID: mb.ID}) + q.FilterEqual("Expunged", false) + q.FilterGreater("UID", lastUID) + q.SortDesc("UID") + for m, err := range q.All() { + xcheckf(err, "list messages in mailbox") + + if time.Since(inProgressLast) > inProgressPeriod { + c.writelinef("* OK [INPROGRESS (%s %d %s)] still searching", inProgressTag, progress, goal) + inProgressLast = time.Now() + } + progress++ + + var seq msgseq // Filled in by searchMatch for messages in selected mailbox. + if c.searchMatch(tx, msgCount, seq, m, *sk, bodySearch, textSearch, highestUID) { + result.UIDs = append(result.UIDs, m.UID) + result.MaxModSeq = max(result.MaxModSeq, m.ModSeq) break } } } - } - // And reverse search for MAX if we have only MAX or MAX combined with MIN. - if reverse { - for i := len(c.uids) - 1; i > lastIndex; i-- { - if time.Since(inProgressLast) > inProgressPeriod { - c.writelinef("* OK [INPROGRESS (%s %d %s)] still searching", inProgressTag, len(c.uids)-1-i, goal) - inProgressLast = time.Now() - } - if match, modseq := c.searchMatch(tx, msgseq(i+1), c.uids[i], *sk, bodySearch, textSearch, &expungeIssued); match { - uids = append(uids, c.uids[i]) - if modseq > maxModSeq { - maxModSeq = modseq - } - break - } - } + + // We could have finished searching the mailbox with fewer + mailboxProcessed := progress - progressOrig + mailboxTotal := uint32(mb.MailboxCounts.Total + mb.MailboxCounts.Deleted) + progress += max(0, mailboxTotal-mailboxProcessed) + + results = append(results, result) } }) if eargs == nil { + // We'll only have a result for the one selected mailbox. + result := results[0] + // In IMAP4rev1, an untagged SEARCH response is required. ../rfc/3501:2728 - if len(uids) == 0 { + if len(result.UIDs) == 0 { c.bwritelinef("* SEARCH") } // Old-style SEARCH response. We must spell out each number. So we may be splitting // into multiple responses. ../rfc/9051:6809 ../rfc/3501:4833 - for len(uids) > 0 { - n := len(uids) + for len(result.UIDs) > 0 { + n := len(result.UIDs) if n > 100 { n = 100 } s := "" - for _, v := range uids[:n] { + for _, v := range result.UIDs[:n] { if !isUID { v = store.UID(c.xsequence(v)) } @@ -233,18 +524,18 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) { var modseq string if sk.hasModseq() { // ../rfc/7162:2557 - modseq = fmt.Sprintf(" (MODSEQ %d)", maxModSeq.Client()) + modseq = fmt.Sprintf(" (MODSEQ %d)", result.MaxModSeq.Client()) } c.bwritelinef("* SEARCH%s%s", s, modseq) - uids = uids[n:] + result.UIDs = result.UIDs[n:] } } else { // New-style ESEARCH response syntax: ../rfc/9051:6546 ../rfc/4466:522 if save { // ../rfc/9051:3784 ../rfc/5182:13 - c.searchResult = uids + c.searchResult = results[0].UIDs if sanityChecks { checkUIDs(c.searchResult) } @@ -252,72 +543,88 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) { // No untagged ESEARCH response if nothing was requested. ../rfc/9051:4160 if len(eargs) > 0 { - // The tag was originally a string, became an astring in IMAP4rev2, better stick to - // string. ../rfc/4466:707 ../rfc/5259:1163 ../rfc/9051:7087 - resp := fmt.Sprintf(`* ESEARCH (TAG "%s")`, tag) - if isUID { - resp += " UID" - } - - // NOTE: we are converting UIDs to msgseq in the uids slice (if needed) while - // keeping the "uids" name! - if !isUID { - // If searchResult is hanging on to the slice, we need to work on a copy. - if save { - nuids := make([]store.UID, len(uids)) - copy(nuids, uids) - uids = nuids + for _, result := range results { + // For the ESEARCH command, we must not return a response if there were no matching + // messages. This is unlike the later IMAP4rev2, where an ESEARCH response must be + // sent if there were no matches. ../rfc/7377:243 ../rfc/9051:3775 + if isE && len(result.UIDs) == 0 { + continue } - for i, uid := range uids { - uids[i] = store.UID(c.xsequence(uid)) + + // The tag was originally a string, became an astring in IMAP4rev2, better stick to + // string. ../rfc/4466:707 ../rfc/5259:1163 ../rfc/9051:7087 + if isE { + fmt.Fprintf(c.xbw, `* ESEARCH (TAG "%s" MAILBOX %s UIDVALIDITY %d)`, tag, result.Mailbox.Name, result.Mailbox.UIDValidity) + } else { + fmt.Fprintf(c.xbw, `* ESEARCH (TAG "%s")`, tag) + } + if isUID { + fmt.Fprintf(c.xbw, " UID") } - } - // If no matches, then no MIN/MAX response. ../rfc/4731:98 ../rfc/9051:3758 - if eargs["MIN"] && len(uids) > 0 { - resp += fmt.Sprintf(" MIN %d", uids[0]) - } - if eargs["MAX"] && len(uids) > 0 { - resp += fmt.Sprintf(" MAX %d", uids[len(uids)-1]) - } - if eargs["COUNT"] { - resp += fmt.Sprintf(" COUNT %d", len(uids)) - } - if eargs["ALL"] && len(uids) > 0 { - resp += fmt.Sprintf(" ALL %s", compactUIDSet(uids).String()) - } + // NOTE: we are potentially converting UIDs to msgseq, but keep the store.UID type + // for convenience. + nums := result.UIDs + if !isUID { + // If searchResult is hanging on to the slice, we need to work on a copy. + if save { + nums = slices.Clone(nums) + } + for i, uid := range nums { + nums[i] = store.UID(c.xsequence(uid)) + } + } - // Interaction between ESEARCH and CONDSTORE: ../rfc/7162:1211 ../rfc/4731:273 - // Summary: send the highest modseq of the returned messages. - if sk.hasModseq() && len(uids) > 0 { - resp += fmt.Sprintf(" MODSEQ %d", maxModSeq.Client()) - } + // If no matches, then no MIN/MAX response. ../rfc/4731:98 ../rfc/9051:3758 + if eargs["MIN"] && len(nums) > 0 { + fmt.Fprintf(c.xbw, " MIN %d", nums[0]) + } + if eargs["MAX"] && len(result.UIDs) > 0 { + fmt.Fprintf(c.xbw, " MAX %d", nums[len(nums)-1]) + } + if eargs["COUNT"] { + fmt.Fprintf(c.xbw, " COUNT %d", len(nums)) + } + if eargs["ALL"] && len(nums) > 0 { + fmt.Fprintf(c.xbw, " ALL %s", compactUIDSet(nums).String()) + } - c.bwritelinef("%s", resp) + // Interaction between ESEARCH and CONDSTORE: ../rfc/7162:1211 ../rfc/4731:273 + // Summary: send the highest modseq of the returned messages. + if sk.hasModseq() && len(nums) > 0 { + fmt.Fprintf(c.xbw, " MODSEQ %d", result.MaxModSeq.Client()) + } + + c.bwritelinef("") + } } } - if expungeIssued { - // ../rfc/9051:5102 - c.writeresultf("%s OK [EXPUNGEISSUED] done", tag) - } else { - c.ok(tag, cmd) - } + + c.ok(tag, cmd) } type search struct { - c *conn - tx *bstore.Tx - seq msgseq - uid store.UID - mr *store.MsgReader - m store.Message - p *message.Part - expungeIssued *bool - hasModseq bool + c *conn + tx *bstore.Tx + msgCount uint32 // Number of messages in mailbox (or session when selected). + seq msgseq // Can be 0, for other mailboxes than selected in case of MAX. + m store.Message + mr *store.MsgReader + p *message.Part + highestUID func() (store.UID, error) } -func (c *conn) searchMatch(tx *bstore.Tx, seq msgseq, uid store.UID, sk searchKey, bodySearch, textSearch *store.WordSearch, expungeIssued *bool) (bool, store.ModSeq) { - s := search{c: c, tx: tx, seq: seq, uid: uid, expungeIssued: expungeIssued, hasModseq: sk.hasModseq()} +func (c *conn) searchMatch(tx *bstore.Tx, msgCount uint32, seq msgseq, m store.Message, sk searchKey, bodySearch, textSearch *store.WordSearch, highestUID func() (store.UID, error)) bool { + if m.MailboxID == c.mailboxID { + seq = c.sequence(m.UID) + if seq == 0 { + // Session has not yet seen this message, and is not expecting to get a result that + // includes it. + return false + } + } + + s := search{c: c, tx: tx, msgCount: msgCount, seq: seq, m: m, highestUID: highestUID} defer func() { if s.mr != nil { err := s.mr.Close() @@ -328,18 +635,7 @@ func (c *conn) searchMatch(tx *bstore.Tx, seq msgseq, uid store.UID, sk searchKe return s.match(sk, bodySearch, textSearch) } -func (s *search) match(sk searchKey, bodySearch, textSearch *store.WordSearch) (match bool, modseq store.ModSeq) { - // Instead of littering all the cases in match0 with calls to get modseq, we do it once - // here in case of a match. - defer func() { - if match && s.hasModseq { - if s.m.ID == 0 { - match = s.xensureMessage() - } - modseq = s.m.ModSeq - } - }() - +func (s *search) match(sk searchKey, bodySearch, textSearch *store.WordSearch) (match bool) { match = s.match0(sk) if match && bodySearch != nil { if !s.xensurePart() { @@ -362,24 +658,6 @@ func (s *search) match(sk searchKey, bodySearch, textSearch *store.WordSearch) ( return } -func (s *search) xensureMessage() bool { - if s.m.ID > 0 { - return true - } - - q := bstore.QueryTx[store.Message](s.tx) - q.FilterNonzero(store.Message{MailboxID: s.c.mailboxID, UID: s.uid}) - m, err := q.Get() - if err == bstore.ErrAbsent || err == nil && m.Expunged { - // ../rfc/2180:607 - *s.expungeIssued = true - return false - } - xcheckf(err, "get message") - s.m = m - return true -} - // ensure message, reader and part are loaded. returns whether that was // successful. func (s *search) xensurePart() bool { @@ -387,10 +665,6 @@ func (s *search) xensurePart() bool { return s.p != nil } - if !s.xensureMessage() { - return false - } - // Closed by searchMatch after all (recursive) search.match calls are finished. s.mr = s.c.account.MessageReader(s.m) @@ -417,14 +691,23 @@ func (s *search) match0(sk searchKey) bool { } return true } else if sk.seqSet != nil { - return sk.seqSet.containsSeq(s.seq, c.uids, c.searchResult) + if sk.seqSet.searchResult { + // Interpreting search results on a mailbox that isn't selected during multisearch + // is likely a mistake. No mention about it in the RFC. ../rfc/7377:257 + if s.m.MailboxID != c.mailboxID { + xuserErrorf("can only use search result with the selected mailbox") + } + return uidSearch(c.searchResult, s.m.UID) > 0 + } + // For multisearch, we have arranged to have a seq for non-selected mailboxes too. + return sk.seqSet.containsSeqCount(s.seq, s.msgCount) } filterHeader := func(field, value string) bool { lower := strings.ToLower(value) h, err := s.p.Header() if err != nil { - c.log.Debugx("parsing message header", err, slog.Any("uid", s.uid)) + c.log.Debugx("parsing message header", err, slog.Any("uid", s.m.UID), slog.Int64("msgid", s.m.ID)) return false } for _, v := range h.Values(field) { @@ -454,7 +737,14 @@ func (s *search) match0(sk searchKey) bool { case "OR": return s.match0(*sk.searchKey) || s.match0(*sk.searchKey2) case "UID": - return sk.uidSet.containsUID(s.uid, c.uids, c.searchResult) + if sk.uidSet.searchResult && s.m.MailboxID != c.mailboxID { + // Interpreting search results on a mailbox that isn't selected during multisearch + // is likely a mistake. No mention about it in the RFC. ../rfc/7377:257 + xuserErrorf("cannot use search result from another mailbox") + } + match, err := sk.uidSet.containsKnownUID(s.m.UID, c.searchResult, s.highestUID) + xcheckf(err, "checking for presence in uid set") + return match } // Parsed part. @@ -570,7 +860,7 @@ func (s *search) match0(sk searchKey) bool { } if s.p == nil { - c.log.Info("missing parsed message, not matching", slog.Any("uid", s.uid)) + c.log.Info("missing parsed message, not matching", slog.Any("uid", s.m.UID), slog.Int64("msgid", s.m.ID)) return false } @@ -599,7 +889,7 @@ func (s *search) match0(sk searchKey) bool { lower := strings.ToLower(sk.astring) h, err := s.p.Header() if err != nil { - c.log.Errorx("parsing header for search", err, slog.Any("uid", s.uid)) + c.log.Errorx("parsing header for search", err, slog.Any("uid", s.m.UID), slog.Int64("msgid", s.m.ID)) return false } k := textproto.CanonicalMIMEHeaderKey(sk.headerField) diff --git a/imapserver/search_test.go b/imapserver/search_test.go index 9a4f46d..03acc99 100644 --- a/imapserver/search_test.go +++ b/imapserver/search_test.go @@ -34,6 +34,10 @@ this is html. --x-- `, "\n", "\r\n") +func uint32ptr(v uint32) *uint32 { + return &v +} + func (tc *testconn) xsearch(nums ...uint32) { tc.t.Helper() @@ -53,7 +57,7 @@ func (tc *testconn) xsearchmodseq(modseq int64, nums ...uint32) { func (tc *testconn) xesearch(exp imapclient.UntaggedEsearch) { tc.t.Helper() - exp.Correlator = tc.client.LastTag + exp.Tag = tc.client.LastTag tc.xuntagged(exp) } @@ -94,6 +98,11 @@ func TestSearch(t *testing.T) { // We now have sequence numbers 1,2,3 and UIDs 5,6,7. + // We need to be selected. Not the case for ESEARCH command. + tc.client.Unselect() + tc.transactf("no", "search all") + tc.client.Select("inbox") + tc.transactf("ok", "search all") tc.xsearch(1, 2, 3) @@ -289,10 +298,6 @@ func TestSearch(t *testing.T) { return imapclient.UntaggedEsearch{All: esearchall0(ss)} } - uint32ptr := func(v uint32) *uint32 { - return &v - } - // Do new-style ESEARCH requests with RETURN. We should get an ESEARCH response. tc.transactf("ok", "search return () all") tc.xesearch(esearchall("1:3")) // Without any options, "ALL" is implicit. @@ -464,3 +469,361 @@ func esearchall0(ss string) imapclient.NumSet { } return seqset } + +// Test the MULTISEARCH extension. Where we don't need to have a mailbox selected, +// operating without messag sequence numbers, and return untagged esearch responses +// that include the mailbox and uidvalidity. +func TestSearchMulti(t *testing.T) { + testSearchMulti(t, false) + testSearchMulti(t, true) +} + +// Run multisearch tests with or without a mailbox selected. +func testSearchMulti(t *testing.T, selected bool) { + defer mockUIDValidity()() + + tc := start(t) + defer tc.close() + tc.client.Login("mjl@mox.example", password0) + tc.client.Select("inbox") + + // Add 5 messages to Inbox and delete first 4 messages. So UIDs start at 5. + received := time.Date(2020, time.January, 1, 10, 0, 0, 0, time.UTC) + for range 6 { + tc.client.Append("inbox", makeAppendTime(exampleMsg, received)) + } + tc.client.StoreFlagsSet("1:4", true, `\Deleted`) + tc.client.Expunge() + + // Unselecting mailbox, esearch works in authenticated state. + if !selected { + tc.client.Unselect() + } + + received = time.Date(2022, time.January, 1, 9, 0, 0, 0, time.UTC) + tc.client.Append("inbox", makeAppendTime(searchMsg, received)) + + received = time.Date(2022, time.January, 1, 9, 0, 0, 0, time.UTC) + mostFlags := []string{ + `\Deleted`, + `\Seen`, + `\Answered`, + `\Flagged`, + `\Draft`, + `$Forwarded`, + `$Junk`, + `$Notjunk`, + `$Phishing`, + `$MDNSent`, + `custom1`, + `Custom2`, + } + tc.client.Append("Archive", imapclient.Append{Flags: mostFlags, Received: &received, Size: int64(len(searchMsg)), Data: strings.NewReader(searchMsg)}) + + // We now have sequence numbers 1,2,3 and UIDs 5,6,7 in Inbox, and UID 1 in Archive. + + // Basic esearch with mailboxes. + tc.cmdf("Tag1", `Esearch In (Personal) Return () All`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:7")}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, All: esearchall0("1")}, + ) + + // Again, but with progress information. + orig := inProgressPeriod + inProgressPeriod = 0 + inprogress := func(cur, goal uint32) imapclient.UntaggedResult { + return imapclient.UntaggedResult{ + Status: "OK", + RespText: imapclient.RespText{ + Code: "INPROGRESS", + CodeArg: imapclient.CodeInProgress{Tag: "Tag1", Current: &cur, Goal: &goal}, + More: "still searching", + }, + } + } + tc.cmdf("Tag1", `Esearch In (Personal) Return () All`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:7")}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, All: esearchall0("1")}, + inprogress(0, 4), + inprogress(1, 4), + inprogress(2, 4), + inprogress(3, 4), + ) + inProgressPeriod = orig + + // Explicit mailboxes listed, including non-existent one that is ignored, + // duplicates are ignored as well. + tc.cmdf("Tag1", `Esearch In (Mailboxes (INBOX Archive Archive)) Return (Min Max Count All) All`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5, Max: 7, Count: uint32ptr(3), All: esearchall0("5:7")}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Min: 1, Max: 1, Count: uint32ptr(1), All: esearchall0("1")}, + ) + + // No response if none of the mailboxes exist. + tc.cmdf("Tag1", `Esearch In (Mailboxes bogus Mailboxes (nonexistent)) Return (Min Max Count All) All`) + tc.response("ok") + tc.xuntagged() + + // Inboxes evaluates to just inbox on new account. We'll add more mailboxes + // matching "inboxes" later on. + tc.cmdf("Tag1", `Esearch In (Inboxes) Return () All`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:7")}, + ) + + // Subscribed is set for created mailboxes by default. + tc.cmdf("Tag1", `Esearch In (Subscribed) Return (Max) All`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Max: 7}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Max: 1}, + ) + + // Asking for max does a reverse search. + tc.cmdf("Tag1", `Esearch In (Personal) Return (Max) All`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Max: 7}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Max: 1}, + ) + + // Min stops early. + tc.cmdf("Tag1", `Esearch In (Personal) Return (Min) All`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Min: 1}, + ) + + // Min and max do forward and reverse search, stopping early. + tc.cmdf("Tag1", `Esearch In (Personal) Return (Min Max) All`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5, Max: 7}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Min: 1, Max: 1}, + ) + + if selected { + // With only 1 inbox, we can use SAVE with Inboxes. Can't anymore when we have multiple. + tc.transactf("ok", `Esearch In (Inboxes) Return (Save) All`) + tc.xuntagged() + + // Using search result ($) works with selected mailbox. + tc.cmdf("Tag1", `Esearch In (Selected) Return () $`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:7")}, + ) + } else { + // Cannot use "selected" if we are not in selected state. + tc.transactf("bad", `Esearch In (Selected) Return () All`) + } + + // Add more "inboxes", and other mailboxes for testing "subtree" and "subtree-one". + more := []string{ + "Inbox/Sub1", + "Inbox/Sub2", + "Inbox/Sub2/SubA", + "Inbox/Sub2/SubB", + "Other", + "Other/Sub1", // sub1@mox.example in config. + "Other/Sub2", + "Other/Sub2/SubA", // ruleset for sub2@mox.example in config. + "Other/Sub2/SubB", + "List", // ruleset for a mailing list + } + for _, name := range more { + tc.client.Create(name, nil) + tc.client.Append(name, makeAppendTime(exampleMsg, received)) + } + + // Cannot use SAVE with multiple mailboxes that match. + tc.transactf("bad", `Esearch In (Inboxes) Return (Save) All`) + + // "inboxes" includes everything below Inbox, and also anything that we might + // deliver to based on account addresses and rulesets, but not mailing lists. + tc.cmdf("Tag1", `Esearch In (Inboxes) Return () All`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:7")}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox/Sub1", UIDValidity: 3, UID: true, All: esearchall0("1")}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox/Sub2", UIDValidity: 4, UID: true, All: esearchall0("1")}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox/Sub2/SubA", UIDValidity: 5, UID: true, All: esearchall0("1")}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox/Sub2/SubB", UIDValidity: 6, UID: true, All: esearchall0("1")}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub1", UIDValidity: 8, UID: true, All: esearchall0("1")}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub2/SubA", UIDValidity: 10, UID: true, All: esearchall0("1")}, + ) + + // subtree + tc.cmdf("Tag1", `Esearch In (Subtree Other) Return () All`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other", UIDValidity: 7, UID: true, All: esearchall0("1")}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub1", UIDValidity: 8, UID: true, All: esearchall0("1")}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub2", UIDValidity: 9, UID: true, All: esearchall0("1")}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub2/SubA", UIDValidity: 10, UID: true, All: esearchall0("1")}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub2/SubB", UIDValidity: 11, UID: true, All: esearchall0("1")}, + ) + + // subtree-one + tc.cmdf("Tag1", `Esearch In (Subtree-One Other) Return () All`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other", UIDValidity: 7, UID: true, All: esearchall0("1")}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub1", UIDValidity: 8, UID: true, All: esearchall0("1")}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub2", UIDValidity: 9, UID: true, All: esearchall0("1")}, + ) + + // Search with sequence set also for non-selected mailboxes(!). The min/max would + // get the first and last message, but the message sequence set forces a scan. + tc.cmdf("Tag1", `Esearch In (Mailboxes Inbox) Return (Min Max) 1:*`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5, Max: 7}, + ) + + // Search with uid set with "$highnum:*" forces getting highest uid. + tc.cmdf("Tag1", `Esearch In (Mailboxes Inbox) Return (Min Max) Uid *:100`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 7, Max: 7}, + ) + tc.cmdf("Tag1", `Esearch In (Mailboxes Inbox) Return (Min Max) Uid 100:*`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 7, Max: 7}, + ) + tc.cmdf("Tag1", `Esearch In (Mailboxes Inbox) Return (Min Max) Uid 1:*`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5, Max: 7}, + ) + + // We use another session to add a new message to Inbox and to Archive. Searching + // with Inbox selected will not return the new message since it isn't available in + // the session yet. The message in Archive is returned, since there is no session + // limitation. + tc2 := startNoSwitchboard(t) + defer tc2.closeNoWait() + tc2.client.Login("mjl@mox.example", password0) + tc2.client.Append("inbox", makeAppendTime(searchMsg, received)) + tc2.client.Append("Archive", makeAppendTime(searchMsg, received)) + + tc.cmdf("Tag1", `Esearch In (Mailboxes (Inbox Archive)) Return (Count) All`) + tc.response("ok") + if selected { + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Count: uint32ptr(3)}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Count: uint32ptr(2)}, + imapclient.UntaggedExists(4), + imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(8), imapclient.FetchFlags(nil)}}, + ) + } else { + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Count: uint32ptr(4)}, + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Count: uint32ptr(2)}, + ) + } + + if selected { + // Saving a search result, and then using it with another mailbox results in error. + tc.transactf("ok", `Esearch In (Mailboxes Inbox) Return (Save) All`) + tc.transactf("no", `Esearch In (Mailboxes Archive) Return () $`) + } else { + tc.transactf("bad", `Esearch In (Inboxes) Return (Save) All`) // Need a selected mailbox with SAVE. + tc.transactf("no", `Esearch In (Inboxes) Return () $`) // Cannot use saved result with non-selected mailbox. + } + + tc.transactf("bad", `Esearch In () Return () All`) // Missing values for "IN"-list. + tc.transactf("bad", `Esearch In (Bogus) Return () All`) // Bogus word for "IN". + tc.transactf("bad", `Esearch In ("Selected") Return () All`) // IN-words can't be quoted. + tc.transactf("bad", `Esearch In (Selected-Delayed) Return () All`) // From NOTIFY, not in ESEARCH. + tc.transactf("bad", `Esearch In (Subtree-One) Return () All`) // After subtree-one we need a list. + tc.transactf("bad", `Esearch In (Subtree-One ) Return () All`) // After subtree-one we need a list. + tc.transactf("bad", `Esearch In (Subtree-One (Test) ) Return () All`) // Bogus space. + + if !selected { + return + } + // From now on, we are in selected state. + + tc.cmdf("Tag1", `Esearch In (Selected) Return () All`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:8")}, + ) + + // Testing combinations of SAVE with MIN/MAX/others ../rfc/9051:4100 + tc.transactf("ok", `Esearch In (Selected) Return (Save) All`) + tc.xuntagged() + + tc.cmdf("Tag1", `Esearch In (Selected) Return () $`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:8")}, + ) + + // Inbox happens to be the selected mailbox, so OK. + tc.cmdf("Tag1", `Esearch In (Mailboxes Inbox) Return () $`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:8")}, + ) + + // Non-selected mailboxes aren't allowed to use the saved result. + tc.transactf("no", `Esearch In (Mailboxes Archive) Return () $`) + tc.transactf("no", `Esearch In (Mailboxes Archive) Return () uid $`) + + tc.cmdf("Tag1", `Esearch In (Selected) Return (Save Min Max) All`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5, Max: 8}, + ) + tc.cmdf("Tag1", `Esearch In (Selected) Return () $`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5,8")}, + ) + + tc.cmdf("Tag1", `Esearch In (Selected) Return (Save Min) All`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5}, + ) + + tc.cmdf("Tag1", `Esearch In (Selected) Return () $`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5")}, + ) + + tc.cmdf("Tag1", `Esearch In (Selected) Return (Save Max) All`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Max: 8}, + ) + + tc.cmdf("Tag1", `Esearch In (Selected) Return () $`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("8")}, + ) + + tc.cmdf("Tag1", `Esearch In (Selected) Return (Save Min Max Count) All`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5, Max: 8, Count: uint32ptr(4)}, + ) + + tc.cmdf("Tag1", `Esearch In (Selected) Return () $`) + tc.response("ok") + tc.xuntagged( + imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:8")}, + ) +} diff --git a/imapserver/server.go b/imapserver/server.go index 18e5c39..8f85aea 100644 --- a/imapserver/server.go +++ b/imapserver/server.go @@ -182,6 +182,7 @@ var serverCapabilities = strings.Join([]string{ "REPLACE", // ../rfc/8508 "PREVIEW", // ../rfc/8970:114 "INPROGRESS", // ../rfc/9585:101 + "MULTISEARCH", // ../rfc/7377:187 // "COMPRESS=DEFLATE", // ../rfc/4978, disabled for interoperability issues: The flate reader (inflate) still blocks on partial flushes, preventing progress. }, " ") @@ -281,8 +282,8 @@ func stateCommands(cmds ...string) map[string]struct{} { var ( commandsStateAny = stateCommands("capability", "noop", "logout", "id") commandsStateNotAuthenticated = stateCommands("starttls", "authenticate", "login") - commandsStateAuthenticated = stateCommands("enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub", "getquotaroot", "getquota", "getmetadata", "setmetadata", "compress") - commandsStateSelected = stateCommands("close", "unselect", "expunge", "search", "fetch", "store", "copy", "move", "uid expunge", "uid search", "uid fetch", "uid store", "uid copy", "uid move", "replace", "uid replace") + commandsStateAuthenticated = stateCommands("enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub", "getquotaroot", "getquota", "getmetadata", "setmetadata", "compress", "esearch") + commandsStateSelected = stateCommands("close", "unselect", "expunge", "search", "fetch", "store", "copy", "move", "uid expunge", "uid search", "uid fetch", "uid store", "uid copy", "uid move", "replace", "uid replace", "esearch") ) var commands = map[string]func(c *conn, tag, cmd string, p *parser){ @@ -317,6 +318,7 @@ var commands = map[string]func(c *conn, tag, cmd string, p *parser){ "getmetadata": (*conn).cmdGetmetadata, "setmetadata": (*conn).cmdSetmetadata, "compress": (*conn).cmdCompress, + "esearch": (*conn).cmdEsearch, // Selected. "check": (*conn).cmdCheck, @@ -3847,12 +3849,12 @@ func (c *conn) cmdxExpunge(tag, cmd string, uidSet *numSet) { // State: Selected func (c *conn) cmdSearch(tag, cmd string, p *parser) { - c.cmdxSearch(false, tag, cmd, p) + c.cmdxSearch(false, false, tag, cmd, p) } // State: Selected func (c *conn) cmdUIDSearch(tag, cmd string, p *parser) { - c.cmdxSearch(true, tag, cmd, p) + c.cmdxSearch(true, false, tag, cmd, p) } // State: Selected diff --git a/rfc/index.txt b/rfc/index.txt index e6a703f..f035158 100644 --- a/rfc/index.txt +++ b/rfc/index.txt @@ -218,13 +218,13 @@ https://www.iana.org/assignments/message-headers/message-headers.xhtml 5957 Roadmap - Display-Based Address Sorting for the IMAP4 SORT Extension 6154 Yes - IMAP LIST Extension for Special-Use Mailboxes 6203 No - IMAP4 Extension for Fuzzy Search -6237 -Roadmap Obs (RFC 7377) IMAP4 Multimailbox SEARCH Extension +6237 -Yes Obs (RFC 7377) IMAP4 Multimailbox SEARCH Extension 6851 Yes - Internet Message Access Protocol (IMAP) - MOVE Extension 6855 Yes - IMAP Support for UTF-8 6858 No - Simplified POP and IMAP Downgrading for Internationalized Email 7162 Yes - IMAP Extensions: Quick Flag Changes Resynchronization (CONDSTORE) and Quick Mailbox Resynchronization (QRESYNC) 7162-eid5055 - - errata: space after untagged OK -7377 Roadmap - IMAP4 Multimailbox SEARCH Extension +7377 Yes - IMAP4 Multimailbox SEARCH Extension 7888 Yes - IMAP4 Non-synchronizing Literals 7889 Yes - The IMAP APPENDLIMIT Extension 8437 No - IMAP UNAUTHENTICATE Extension for Connection Reuse diff --git a/testdata/imap/domains.conf b/testdata/imap/domains.conf index 1c4b09c..0ea3f99 100644 --- a/testdata/imap/domains.conf +++ b/testdata/imap/domains.conf @@ -6,6 +6,17 @@ Accounts: Domain: mox.example Destinations: mjl@mox.example: nil + sub1@mox.example: + Mailbox: Other/Sub1 + sub2@mox.example: + Rulesets: + - + VerifiedDomain: test.example + Mailbox: Other/Sub2/SubA + - + VerifiedDomain: list.example + ListAllowDomain: list.example + Mailbox: List ""@mox.example: nil móx@mox.example: nil JunkFilter: