diff --git a/README.md b/README.md index fbc7148..c4f8aed 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ support: - Automate DNS management, for setup and maintenance, such as DANE/DKIM key rotation - Config options for "transactional email domains", for which mox will only send messages -- More IMAP extensions (NOTIFY, UIDONLY) +- More IMAP extensions (UIDONLY) - Encrypted storage of files (email messages, TLS keys), also with per account keys - Recognize common deliverability issues and help postmasters solve them - JMAP, IMAP OBJECTID extension, IMAP JMAPACCESS extension diff --git a/imapclient/client.go b/imapclient/client.go index f8d9cbd..969db05 100644 --- a/imapclient/client.go +++ b/imapclient/client.go @@ -185,13 +185,18 @@ func (c *Conn) xflush() { } } -func (c *Conn) xtrace(level slog.Level) func() { - c.xflush() +func (c *Conn) xtraceread(level slog.Level) func() { c.tr.SetTrace(level) + return func() { + c.tr.SetTrace(mlog.LevelTrace) + } +} + +func (c *Conn) xtracewrite(level slog.Level) func() { + c.xflush() c.xtw.SetTrace(level) return func() { c.xflush() - c.tr.SetTrace(mlog.LevelTrace) c.xtw.SetTrace(mlog.LevelTrace) } } @@ -357,9 +362,10 @@ func (c *Conn) WriteSyncLiteral(s string) (untagged []Untagged, rerr error) { _, err = c.Readline() c.xcheckf(err, "read continuation line") + defer c.xtracewrite(mlog.LevelTracedata)() _, err = c.xbw.Write([]byte(s)) c.xcheckf(err, "write literal data") - c.xflush() + c.xtracewrite(mlog.LevelTrace) return nil, nil } untagged, result, err := c.Response() diff --git a/imapclient/cmds.go b/imapclient/cmds.go index 3e948dc..809854b 100644 --- a/imapclient/cmds.go +++ b/imapclient/cmds.go @@ -59,9 +59,9 @@ func (c *Conn) Login(username, password string) (untagged []Untagged, result Res c.LastTag = c.nextTag() fmt.Fprintf(c.xbw, "%s login %s ", c.LastTag, astring(username)) - defer c.xtrace(mlog.LevelTraceauth)() + defer c.xtracewrite(mlog.LevelTraceauth)() fmt.Fprintf(c.xbw, "%s\r\n", astring(password)) - c.xtrace(mlog.LevelTrace) // Restore. + c.xtracewrite(mlog.LevelTrace) // Restore. return c.Response() } @@ -76,11 +76,11 @@ func (c *Conn) AuthenticatePlain(username, password string) (untagged []Untagged if result.Status != "" { c.xerrorf("got result status %q, expected continuation", result.Status) } - defer c.xtrace(mlog.LevelTraceauth)() + defer c.xtracewrite(mlog.LevelTraceauth)() xw := base64.NewEncoder(base64.StdEncoding, c.xbw) fmt.Fprintf(xw, "\u0000%s\u0000%s", username, password) xw.Close() - c.xtrace(mlog.LevelTrace) // Restore. + c.xtracewrite(mlog.LevelTrace) // Restore. fmt.Fprintf(c.xbw, "\r\n") c.xflush() return c.Response() @@ -317,10 +317,10 @@ func (c *Conn) Append(mailbox string, message Append, more ...Append) (untagged // todo: for larger messages, use a synchronizing literal. fmt.Fprintf(c.xbw, " (%s)%s {%d+}\r\n", strings.Join(m.Flags, " "), date, m.Size) - defer c.xtrace(mlog.LevelTracedata)() + defer c.xtracewrite(mlog.LevelTracedata)() _, err := io.Copy(c.xbw, m.Data) c.xcheckf(err, "write message data") - c.xtrace(mlog.LevelTrace) // Restore + c.xtracewrite(mlog.LevelTrace) // Restore } fmt.Fprintf(c.xbw, "\r\n") @@ -328,7 +328,8 @@ func (c *Conn) Append(mailbox string, message Append, more ...Append) (untagged return c.Response() } -// note: No idle command. Idle is better implemented by writing the request and reading and handling the responses as they come in. +// note: No Idle or Notify command. Idle/Notify is better implemented by +// writing the request and reading and handling the responses as they come in. // CloseMailbox closes the currently selected/active mailbox, permanently removing // any messages marked with \Deleted. @@ -444,10 +445,10 @@ func (c *Conn) replace(cmd string, num string, mailbox string, msg Append) (unta err := c.Commandf("", "%s %s %s (%s)%s ~{%d+}", cmd, num, astring(mailbox), strings.Join(msg.Flags, " "), date, msg.Size) c.xcheckf(err, "writing replace command") - defer c.xtrace(mlog.LevelTracedata)() + defer c.xtracewrite(mlog.LevelTracedata)() _, err = io.Copy(c.xbw, msg.Data) c.xcheckf(err, "write message data") - c.xtrace(mlog.LevelTrace) + c.xtracewrite(mlog.LevelTrace) fmt.Fprintf(c.xbw, "\r\n") c.xflush() diff --git a/imapclient/parse.go b/imapclient/parse.go index 794d94e..2b24595 100644 --- a/imapclient/parse.go +++ b/imapclient/parse.go @@ -6,6 +6,8 @@ import ( "strconv" "strings" "time" + + "github.com/mjl-/mox/mlog" ) func (c *Conn) recorded() string { @@ -131,7 +133,9 @@ var knownCodes = stringMap( // With parameters. "BADCHARSET", "CAPABILITY", "PERMANENTFLAGS", "UIDNEXT", "UIDVALIDITY", "UNSEEN", "APPENDUID", "COPYUID", "HIGHESTMODSEQ", "MODIFIED", - "INPROGRESS", // ../rfc/9585:104 + "INPROGRESS", // ../rfc/9585:104 + "BADEVENT", "NOTIFICATIONOVERFLOW", // ../rfc/5465:1023 + "SERVERBUG", ) func stringMap(l ...string) map[string]struct{} { @@ -247,6 +251,20 @@ func (c *Conn) xrespCode() (string, CodeArg) { c.xtake(")") } codeArg = CodeInProgress{tag, current, goal} + case "BADEVENT": + // ../rfc/5465:1033 + c.xspace() + c.xtake("(") + var l []string + for { + s := c.xatom() + l = append(l, s) + if !c.space() { + break + } + } + c.xtake(")") + codeArg = CodeBadEvent(l) } return W, codeArg } @@ -896,8 +914,10 @@ func (c *Conn) xliteral() []byte { c.xflush() } buf := make([]byte, int(size)) + defer c.xtraceread(mlog.LevelTracedata)() _, err := io.ReadFull(c.br, buf) c.xcheckf(err, "reading data for literal") + c.xtraceread(mlog.LevelTrace) return buf } diff --git a/imapclient/protocol.go b/imapclient/protocol.go index 3c42c27..54ab444 100644 --- a/imapclient/protocol.go +++ b/imapclient/protocol.go @@ -42,6 +42,7 @@ const ( CapReplace Capability = "REPLACE" // ../rfc/8508:155 CapPreview Capability = "PREVIEW" // ../rfc/8970:114 CapMultiSearch Capability = "MULTISEARCH" // ../rfc/7377:187 + CapNotify Capability = "NOTIFY" // ../rfc/5465:195 ) // Status is the tagged final result of a command. @@ -186,6 +187,14 @@ func (c CodeInProgress) CodeString() string { return fmt.Sprintf("INPROGRESS (%q %s %s)", c.Tag, current, goal) } +// "BADEVENT" response code, with the events that are supported, for the NOTIFY +// extension. +type CodeBadEvent []string + +func (c CodeBadEvent) CodeString() string { + return fmt.Sprintf("BADEVENT (%s)", strings.Join([]string(c), " ")) +} + // RespText represents a response line minus the leading tag. type RespText struct { Code string // The first word between [] after the status. diff --git a/imapserver/fetch.go b/imapserver/fetch.go index 5fe7588..dfda467 100644 --- a/imapserver/fetch.go +++ b/imapserver/fetch.go @@ -32,7 +32,10 @@ type fetchCmd struct { hasChangedSince bool // Whether CHANGEDSINCE was set. Enables MODSEQ in response. expungeIssued bool // Set if any message has been expunged. Can happen for expunged messages. - uid store.UID // UID currently processing. + // For message currently processing. + mailboxID int64 + uid store.UID + markSeen bool needFlags bool needModseq bool // Whether untagged responses needs modseq. @@ -76,7 +79,7 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) { p.xspace() nums := p.xnumSet() p.xspace() - atts := p.xfetchAtts(isUID) + atts := p.xfetchAtts() var changedSince int64 var haveChangedSince bool var vanished bool @@ -144,7 +147,7 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) { var uids []store.UID var vanishedUIDs []store.UID - cmd := &fetchCmd{conn: c, isUID: isUID, hasChangedSince: haveChangedSince, newPreviews: map[store.UID]string{}} + cmd := &fetchCmd{conn: c, isUID: isUID, hasChangedSince: haveChangedSince, mailboxID: c.mailboxID, newPreviews: map[store.UID]string{}} defer func() { if cmd.rtx == nil { @@ -247,9 +250,21 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) { } } + defer cmd.msgclose() // In case of panic. + for _, cmd.uid = range uids { cmd.conn.log.Debug("processing uid", slog.Any("uid", cmd.uid)) - cmd.process(atts) + data, err := cmd.process(atts) + if err != nil { + cmd.conn.log.Infox("processing fetch attribute", err, slog.Any("uid", cmd.uid)) + xuserErrorf("processing fetch attribute: %v", err) + } + + fmt.Fprintf(cmd.conn.xbw, "* %d FETCH ", cmd.conn.xsequence(cmd.uid)) + data.xwriteTo(cmd.conn, cmd.conn.xbw) + cmd.conn.xbw.Write([]byte("\r\n")) + + cmd.msgclose() } // We've returned all data. Now we mark messages as seen in one go, in a new write @@ -298,7 +313,7 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) { mb.Sub(m.MailboxCounts()) m.Seen = true mb.Add(m.MailboxCounts()) - changes = append(changes, m.ChangeFlags(oldFlags)) + changes = append(changes, m.ChangeFlags(oldFlags, mb)) m.ModSeq = modseq err = wtx.Update(&m) @@ -353,7 +368,7 @@ func (cmd *fetchCmd) xensureMessage() *store.Message { // We do not filter by Expunged, the message may have been deleted in other // sessions, but not in ours. q := bstore.QueryTx[store.Message](cmd.rtx) - q.FilterNonzero(store.Message{MailboxID: cmd.conn.mailboxID, UID: cmd.uid}) + q.FilterNonzero(store.Message{MailboxID: cmd.mailboxID, UID: cmd.uid}) m, err := q.Get() cmd.xcheckf(err, "get message for uid %d", cmd.uid) cmd.m = &m @@ -385,16 +400,20 @@ func (cmd *fetchCmd) xensureParsed() (*store.MsgReader, *message.Part) { return cmd.msgr, cmd.part } -func (cmd *fetchCmd) process(atts []fetchAtt) { - defer func() { - cmd.m = nil - cmd.part = nil - if cmd.msgr != nil { - err := cmd.msgr.Close() - cmd.conn.xsanity(err, "closing messagereader") - cmd.msgr = nil - } +// msgclose must be called after processing a message (after having written/used +// its data), even in the case of a panic. +func (cmd *fetchCmd) msgclose() { + cmd.m = nil + cmd.part = nil + if cmd.msgr != nil { + err := cmd.msgr.Close() + cmd.conn.xsanity(err, "closing messagereader") + cmd.msgr = nil + } +} +func (cmd *fetchCmd) process(atts []fetchAtt) (rdata listspace, rerr error) { + defer func() { x := recover() if x == nil { return @@ -402,9 +421,9 @@ func (cmd *fetchCmd) process(atts []fetchAtt) { err, ok := x.(attrError) if !ok { panic(x) + } else if rerr == nil { + rerr = err } - cmd.conn.log.Infox("processing fetch attribute", err, slog.Any("uid", cmd.uid)) - xuserErrorf("processing fetch attribute: %v", err) }() data := listspace{bare("UID"), number(cmd.uid)} @@ -446,10 +465,7 @@ func (cmd *fetchCmd) process(atts []fetchAtt) { data = append(data, bare("MODSEQ"), listspace{bare(fmt.Sprintf("%d", m.ModSeq.Client()))}) } - // Write errors are turned into panics because we write through c. - fmt.Fprintf(cmd.conn.xbw, "* %d FETCH ", cmd.conn.xsequence(cmd.uid)) - data.xwriteTo(cmd.conn, cmd.conn.xbw) - cmd.conn.xbw.Write([]byte("\r\n")) + return data, nil } // result for one attribute. if processing fails, e.g. because data was requested diff --git a/imapserver/notify.go b/imapserver/notify.go new file mode 100644 index 0000000..ff97a3e --- /dev/null +++ b/imapserver/notify.go @@ -0,0 +1,329 @@ +package imapserver + +import ( + "fmt" + "slices" + "strings" + + "github.com/mjl-/bstore" + + "github.com/mjl-/mox/store" +) + +// Max number of pending changes for selected-delayed mailbox before we write a +// NOTIFICATIONOVERFLOW message, flush changes and stop gathering more changes. +// Changed during tests. +var selectedDelayedChangesMax = 1000 + +// notify represents a configuration as passed to the notify command. +type notify struct { + // "NOTIFY NONE" results in an empty list, matching no events. + EventGroups []eventGroup + + // Changes for the selected mailbox in case of SELECTED-DELAYED, when we don't send + // events asynchrously. These must still be processed later on for their + // ChangeRemoveUIDs, to erase expunged message files. At the end of a command (e.g. + // NOOP) or immediately upon IDLE we will send untagged responses for these + // changes. If the connection breaks, we still process the ChangeRemoveUIDs. + Delayed []store.Change +} + +// match checks if an event for a mailbox id/name (optional depending on type) +// should be turned into a notification to the client. +func (n notify) match(c *conn, xtxfn func() *bstore.Tx, mailboxID int64, mailbox string, kind eventKind) (mailboxSpecifier, notifyEvent, bool) { + // We look through the event groups, and won't stop looking until we've found a + // confirmation the event should be notified. ../rfc/5465:756 + + // Non-message-related events are only matched by non-"selected" mailbox + // specifiers. ../rfc/5465:268 + // If you read the mailboxes matching paragraph in isolation, you would think only + // "SELECTED" and "SELECTED-DELAYED" can match events for the selected mailbox. But + // a few other places hint that that only applies to message events, not to mailbox + // events, such as subscriptions and mailbox metadata changes. With a strict + // interpretation, clients couldn't request notifications for such events for the + // selection mailbox. ../rfc/5465:752 + + for _, eg := range n.EventGroups { + switch eg.MailboxSpecifier.Kind { + case mbspecSelected, mbspecSelectedDelayed: // ../rfc/5465:800 + if mailboxID != c.mailboxID || !slices.Contains(messageEventKinds, kind) { + continue + } + for _, ev := range eg.Events { + if eventKind(ev.Kind) == kind { + return eg.MailboxSpecifier, ev, true + } + } + // We can only have a single selected for notify, so no point in continuing the search. + return mailboxSpecifier{}, notifyEvent{}, false + + default: + // The selected mailbox can only match for non-message events for specifiers other + // than "selected"/"selected-delayed". + if c.mailboxID == mailboxID && slices.Contains(messageEventKinds, kind) { + continue + } + } + + var match bool + Match: + switch eg.MailboxSpecifier.Kind { + case mbspecPersonal: // ../rfc/5465:817 + match = true + + case mbspecInboxes: // ../rfc/5465:822 + if mailbox == "Inbox" || strings.HasPrefix(mailbox, "Inbox/") { + match = true + break Match + } + + if mailbox == "" { + break Match + } + + // Include mailboxes we may deliver to based on destinations, or based on rulesets, + // not including deliveries for mailing lists. + conf, _ := c.account.Conf() + for _, dest := range conf.Destinations { + if dest.Mailbox == mailbox { + match = true + break Match + } + + for _, rs := range dest.Rulesets { + if rs.ListAllowDomain == "" && rs.Mailbox == mailbox { + match = true + break Match + } + } + } + + case mbspecSubscribed: // ../rfc/5465:831 + sub := store.Subscription{Name: mailbox} + err := xtxfn().Get(&sub) + if err != bstore.ErrAbsent { + xcheckf(err, "lookup subscription") + } + match = err == nil + + case mbspecSubtree: // ../rfc/5465:847 + for _, name := range eg.MailboxSpecifier.Mailboxes { + if mailbox == name || strings.HasPrefix(mailbox, name+"/") { + match = true + break + } + } + + case mbspecSubtreeOne: // ../rfc/7377:274 + ntoken := len(strings.Split(mailbox, "/")) + for _, name := range eg.MailboxSpecifier.Mailboxes { + if mailbox == name || (strings.HasPrefix(mailbox, name+"/") && len(strings.Split(name, "/"))+1 == ntoken) { + match = true + break + } + } + + case mbspecMailboxes: // ../rfc/5465:853 + match = slices.Contains(eg.MailboxSpecifier.Mailboxes, mailbox) + + default: + panic("missing case for " + string(eg.MailboxSpecifier.Kind)) + } + + if !match { + continue + } + + // NONE is the signal we shouldn't return events for this mailbox. ../rfc/5465:455 + if len(eg.Events) == 0 { + break + } + + // If event kind matches, we will be notifying about this change. If not, we'll + // look again at next mailbox specifiers. + for _, ev := range eg.Events { + if eventKind(ev.Kind) == kind { + return eg.MailboxSpecifier, ev, true + } + } + } + return mailboxSpecifier{}, notifyEvent{}, false +} + +// Notify enables continuous notifications from the server to the client, without +// the client issuing an IDLE command. The mailboxes and events to notify about are +// specified in the account. When notify is enabled, instead of being blocked +// waiting for a command from the client, we also wait for events from the account, +// and send events about it. +// +// State: Authenticated and selected. +func (c *conn) cmdNotify(tag, cmd string, p *parser) { + // Command: ../rfc/5465:203 + // Request syntax: ../rfc/5465:923 + + p.xspace() + + // NONE indicates client doesn't want any events, also not the "normal" events + // without notify. ../rfc/5465:234 + // ../rfc/5465:930 + if p.take("NONE") { + p.xempty() + + // If we have delayed changes for the selected mailbox, we are no longer going to + // notify about them. The client can't know anymore whether messages still exist, + // and trying to read them can cause errors if the messages have been expunged and + // erased. + var changes []store.Change + if c.notify != nil { + changes = c.notify.Delayed + } + c.notify = ¬ify{} + c.flushChanges(changes) + + c.ok(tag, cmd) + return + } + + var n notify + var status bool + + // ../rfc/5465:926 + p.xtake("SET") + p.xspace() + if p.take("STATUS") { + status = true + p.xspace() + } + for { + eg := p.xeventGroup() + n.EventGroups = append(n.EventGroups, eg) + if !p.space() { + break + } + } + p.xempty() + + for _, eg := range n.EventGroups { + var hasNew, hasExpunge, hasFlag, hasAnnotation bool + for _, ev := range eg.Events { + switch eventKind(ev.Kind) { + case eventMessageNew: + hasNew = true + case eventMessageExpunge: + hasExpunge = true + case eventFlagChange: + hasFlag = true + case eventMailboxName, eventSubscriptionChange, eventMailboxMetadataChange, eventServerMetadataChange: + // Nothing special. + default: // Including eventAnnotationChange. + hasAnnotation = true // Ineffective, we don't implement message annotations yet. + // Result must be NO instead of BAD, and we must include BADEVENT and the events we + // support. ../rfc/5465:343 + // ../rfc/5465:1033 + xusercodeErrorf("BADEVENT (MessageNew MessageExpunge FlagChange MailboxName SubscriptionChange MailboxMetadataChange ServerMetadataChange)", "unimplemented event %s", ev.Kind) + } + } + if hasNew != hasExpunge { + // ../rfc/5465:443 ../rfc/5465:987 + xsyntaxErrorf("MessageNew and MessageExpunge must be specified together") + } + if (hasFlag || hasAnnotation) && !hasNew { + // ../rfc/5465:439 + xsyntaxErrorf("FlagChange and/or AnnotationChange requires MessageNew and MessageExpunge") + } + } + + for _, eg := range n.EventGroups { + for i, name := range eg.MailboxSpecifier.Mailboxes { + eg.MailboxSpecifier.Mailboxes[i] = xcheckmailboxname(name, true) + } + } + + // Only one selected/selected-delay mailbox filter is allowed. ../rfc/5465:779 + // Only message events are allowed for selected/selected-delayed. ../rfc/5465:796 + var haveSelected bool + for _, eg := range n.EventGroups { + switch eg.MailboxSpecifier.Kind { + case mbspecSelected, mbspecSelectedDelayed: + if haveSelected { + xsyntaxErrorf("cannot have multiple selected/selected-delayed mailbox filters") + } + haveSelected = true + + // Only events from message-event are allowed with selected mailbox specifiers. + // ../rfc/5465:977 + for _, ev := range eg.Events { + if !slices.Contains(messageEventKinds, eventKind(ev.Kind)) { + xsyntaxErrorf("selected/selected-delayed is only allowed with message events, not %s", ev.Kind) + } + } + } + } + + // We must apply any changes for delayed select. ../rfc/5465:248 + if c.notify != nil { + delayed := c.notify.Delayed + c.notify.Delayed = nil + c.xapplyChangesNotify(delayed, true) + } + + if status { + var statuses []string + + // Flush new pending changes before we read the current state from the database. + // Don't allow any concurrent changes for a consistent snapshot. + c.account.WithRLock(func() { + select { + case <-c.comm.Pending: + overflow, changes := c.comm.Get() + c.xapplyChanges(overflow, changes, true, true) + default: + } + + c.xdbread(func(tx *bstore.Tx) { + // Send STATUS responses for all matching mailboxes. ../rfc/5465:271 + q := bstore.QueryTx[store.Mailbox](tx) + q.FilterEqual("Expunged", false) + q.SortAsc("Name") + for mb, err := range q.All() { + xcheckf(err, "list mailboxes for status") + + if mb.ID == c.mailboxID { + continue + } + _, _, ok := n.match(c, func() *bstore.Tx { return tx }, mb.ID, mb.Name, eventMessageNew) + if !ok { + continue + } + + list := listspace{ + bare("MESSAGES"), number(mb.MessageCountIMAP()), + bare("UIDNEXT"), number(mb.UIDNext), + bare("UIDVALIDITY"), number(mb.UIDValidity), + // Unseen is not mentioned for STATUS, but clients are able to parse it due to + // FlagChange, and it will be useful to have. + bare("UNSEEN"), number(mb.MailboxCounts.Unseen), + } + if c.enabled[capCondstore] || c.enabled[capQresync] { + list = append(list, bare("HIGHESTMODSEQ"), number(mb.ModSeq)) + } + + status := fmt.Sprintf("* STATUS %s %s", mailboxt(mb.Name).pack(c), list.pack(c)) + statuses = append(statuses, status) + } + }) + }) + + // Write outside of db transaction and lock. + for _, s := range statuses { + c.xbwritelinef("%s", s) + } + } + + // We replace the previous notify config. ../rfc/5465:245 + c.notify = &n + + // Writing OK will flush any other pending changes for the account according to the + // new filters. + c.ok(tag, cmd) +} diff --git a/imapserver/notify_test.go b/imapserver/notify_test.go new file mode 100644 index 0000000..8208608 --- /dev/null +++ b/imapserver/notify_test.go @@ -0,0 +1,570 @@ +package imapserver + +import ( + "strings" + "testing" + "time" + + "github.com/mjl-/mox/imapclient" + "github.com/mjl-/mox/store" +) + +func ptr[T any](v T) *T { + return &v +} + +func TestNotify(t *testing.T) { + defer mockUIDValidity()() + tc := start(t) + defer tc.close() + tc.client.Login("mjl@mox.example", password0) + tc.client.Select("inbox") + + // Check for some invalid syntax. + tc.transactf("bad", "Notify") + tc.transactf("bad", "Notify bogus") + tc.transactf("bad", "Notify None ") // Trailing space. + tc.transactf("bad", "Notify Set") + tc.transactf("bad", "Notify Set ") + tc.transactf("bad", "Notify Set Status") + tc.transactf("bad", "Notify Set Status ()") // Empty list. + tc.transactf("bad", "Notify Set Status (UnknownSpecifier (messageNew))") + tc.transactf("bad", "Notify Set Status (Personal messageNew)") // Missing list around events. + tc.transactf("bad", "Notify Set Status (Personal (messageNew) )") // Trailing space. + tc.transactf("bad", "Notify Set Status (Personal (messageNew)) ") // Trailing space. + + tc.transactf("bad", "Notify Set Status (Selected (mailboxName))") // MailboxName not allowed on Selected. + tc.transactf("bad", "Notify Set Status (Selected (messageNew))") // MessageNew must come with MessageExpunge. + tc.transactf("bad", "Notify Set Status (Selected (flagChange))") // flagChange must come with MessageNew and MessageExpunge. + tc.transactf("bad", "Notify Set Status (Selected (mailboxName)) (Selected-Delayed (mailboxName))") // Duplicate selected. + tc.transactf("no", "Notify Set Status (Selected (annotationChange))") // We don't implement annotation change. + tc.xcode("BADEVENT") + tc.transactf("no", "Notify Set Status (Personal (unknownEvent))") + tc.xcode("BADEVENT") + + tc2 := startNoSwitchboard(t) + defer tc2.closeNoWait() + tc2.client.Login("mjl@mox.example", password0) + tc2.client.Select("inbox") + + var modseq uint32 = 4 + + // Check that we don't get pending changes when we set "notify none". We first make + // changes that we drain with noop. Then add new pending changes and execute + // "notify none". Server should still process changes to the message sequence + // numbers of the selected mailbox. + tc2.client.Append("inbox", makeAppend(searchMsg)) // Results in exists and fetch. + modseq++ + tc2.client.Append("Junk", makeAppend(searchMsg)) // Not selected, not mentioned. + modseq++ + tc.transactf("ok", "noop") + tc.xuntagged( + imapclient.UntaggedExists(1), + imapclient.UntaggedFetch{ + Seq: 1, + Attrs: []imapclient.FetchAttr{ + imapclient.FetchUID(1), + imapclient.FetchFlags(nil), + }, + }, + ) + tc2.client.StoreFlagsAdd("1:*", true, `\Deleted`) + modseq++ + tc2.client.Expunge() + modseq++ + tc.transactf("ok", "Notify None") + tc.xuntagged() // No untagged responses for delete/expunge. + + // Enable notify, will first result in a the pending changes, then status. + tc.transactf("ok", "Notify Set Status (Selected (messageNew (Uid Modseq Bodystructure Preview) messageExpunge flagChange)) (personal (messageNew messageExpunge flagChange mailboxName subscriptionChange mailboxMetadataChange serverMetadataChange))") + tc.xuntagged( + imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(modseq), More: "after condstore-enabling command"}}, + // note: no status for Inbox since it is selected. + imapclient.UntaggedStatus{Mailbox: "Drafts", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 0, imapclient.StatusUIDNext: 1, imapclient.StatusUIDValidity: 1, imapclient.StatusUnseen: 0, imapclient.StatusHighestModSeq: 2}}, + imapclient.UntaggedStatus{Mailbox: "Sent", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 0, imapclient.StatusUIDNext: 1, imapclient.StatusUIDValidity: 1, imapclient.StatusUnseen: 0, imapclient.StatusHighestModSeq: 2}}, + imapclient.UntaggedStatus{Mailbox: "Archive", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 0, imapclient.StatusUIDNext: 1, imapclient.StatusUIDValidity: 1, imapclient.StatusUnseen: 0, imapclient.StatusHighestModSeq: 2}}, + imapclient.UntaggedStatus{Mailbox: "Trash", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 0, imapclient.StatusUIDNext: 1, imapclient.StatusUIDValidity: 1, imapclient.StatusUnseen: 0, imapclient.StatusHighestModSeq: 2}}, + imapclient.UntaggedStatus{Mailbox: "Junk", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 1, imapclient.StatusUIDNext: 2, imapclient.StatusUIDValidity: 1, imapclient.StatusUnseen: 1, imapclient.StatusHighestModSeq: int64(modseq - 2)}}, + ) + + // Selecting the mailbox again results in a refresh of the message sequence + // numbers, with the deleted message gone (it wasn't acknowledged yet due to + // "notify none"). + tc.client.Select("inbox") + + // Add message, should result in EXISTS and FETCH with the configured attributes. + tc2.client.Append("inbox", makeAppend(searchMsg)) + modseq++ + tc.readuntagged( + imapclient.UntaggedExists(1), + imapclient.UntaggedFetch{ + Seq: 1, + Attrs: []imapclient.FetchAttr{ + imapclient.FetchUID(2), + imapclient.FetchBodystructure{ + RespAttr: "BODYSTRUCTURE", + Body: imapclient.BodyTypeMpart{ + Bodies: []any{ + imapclient.BodyTypeText{ + MediaType: "TEXT", + MediaSubtype: "PLAIN", + BodyFields: imapclient.BodyFields{ + Params: [][2]string{[...]string{"CHARSET", "utf-8"}}, + Octets: 21, + }, + Lines: 1, + Ext: &imapclient.BodyExtension1Part{}, + }, + imapclient.BodyTypeText{ + MediaType: "TEXT", + MediaSubtype: "HTML", + BodyFields: imapclient.BodyFields{ + Params: [][2]string{[...]string{"CHARSET", "utf-8"}}, + Octets: 15, + }, + Lines: 1, + Ext: &imapclient.BodyExtension1Part{}, + }, + }, + MediaSubtype: "ALTERNATIVE", + Ext: &imapclient.BodyExtensionMpart{ + Params: [][2]string{{"BOUNDARY", "x"}}, + }, + }, + }, + imapclient.FetchPreview{Preview: ptr("this is plain text.")}, + imapclient.FetchModSeq(modseq), + }, + }, + ) + + // Change flags. + tc2.client.StoreFlagsAdd("1:*", true, `\Deleted`) + modseq++ + tc.readuntagged( + imapclient.UntaggedFetch{ + Seq: 1, + Attrs: []imapclient.FetchAttr{ + imapclient.FetchUID(2), + imapclient.FetchFlags{`\Deleted`}, + imapclient.FetchModSeq(modseq), + }, + }, + ) + + // Remove message. + tc2.client.Expunge() + modseq++ + tc.readuntagged( + imapclient.UntaggedExpunge(1), + ) + + // MailboxMetadataChange for mailbox annotation. + tc2.transactf("ok", `setmetadata Archive (/private/comment "test")`) + modseq++ + tc.readuntagged( + imapclient.UntaggedMetadataKeys{Mailbox: "Archive", Keys: []string{"/private/comment"}}, + ) + + // MailboxMetadataChange also for the selected Inbox. + tc2.transactf("ok", `setmetadata Inbox (/private/comment "test")`) + modseq++ + tc.readuntagged( + imapclient.UntaggedMetadataKeys{Mailbox: "Inbox", Keys: []string{"/private/comment"}}, + ) + + // ServerMetadataChange for server annotation. + tc2.transactf("ok", `setmetadata "" (/private/vendor/other/x "test")`) + modseq++ + tc.readuntagged( + imapclient.UntaggedMetadataKeys{Mailbox: "", Keys: []string{"/private/vendor/other/x"}}, + ) + + // SubscriptionChange for new subscription. + tc2.client.Subscribe("doesnotexist") + tc.readuntagged( + imapclient.UntaggedList{Mailbox: "doesnotexist", Separator: '/', Flags: []string{`\Subscribed`, `\NonExistent`}}, + ) + + // SubscriptionChange for removed subscription. + tc2.client.Unsubscribe("doesnotexist") + tc.readuntagged( + imapclient.UntaggedList{Mailbox: "doesnotexist", Separator: '/', Flags: []string{`\NonExistent`}}, + ) + + // SubscriptionChange for selected mailbox. + tc2.client.Unsubscribe("Inbox") + tc2.client.Subscribe("Inbox") + tc.readuntagged( + imapclient.UntaggedList{Mailbox: "Inbox", Separator: '/'}, + imapclient.UntaggedList{Mailbox: "Inbox", Separator: '/', Flags: []string{`\Subscribed`}}, + ) + + // MailboxName for creating mailbox. + tc2.client.Create("newbox", nil) + modseq++ + tc.readuntagged( + imapclient.UntaggedList{Mailbox: "newbox", Separator: '/', Flags: []string{`\Subscribed`}}, + ) + + // MailboxName for renaming mailbox. + tc2.client.Rename("newbox", "oldbox") + modseq++ + tc.readuntagged( + imapclient.UntaggedList{Mailbox: "oldbox", Separator: '/', OldName: "newbox"}, + ) + + // MailboxName for deleting mailbox. + tc2.client.Delete("oldbox") + modseq++ + tc.readuntagged( + imapclient.UntaggedList{Mailbox: "oldbox", Separator: '/', Flags: []string{`\NonExistent`}}, + ) + + // Add message again to check for modseq. First set notify again with fewer fetch + // attributes for simpler checking. + tc.transactf("ok", "Notify Set (personal (messageNew messageExpunge flagChange mailboxName subscriptionChange mailboxMetadataChange serverMetadataChange)) (Selected (messageNew (Uid Modseq) messageExpunge flagChange))") + tc2.client.Append("inbox", makeAppend(searchMsg)) + modseq++ + tc.readuntagged( + imapclient.UntaggedExists(1), + imapclient.UntaggedFetch{ + Seq: 1, + Attrs: []imapclient.FetchAttr{ + imapclient.FetchUID(3), + imapclient.FetchModSeq(modseq), + }, + }, + ) + + // Next round of events must be ignored. We shouldn't get anything until we add a + // message to "testbox". + tc.transactf("ok", "Notify Set (Selected None) (mailboxes testbox (messageNew messageExpunge)) (personal None)") + tc2.client.Append("inbox", makeAppend(searchMsg)) // MessageNew + modseq++ + tc2.client.StoreFlagsAdd("1:*", true, `\Deleted`) // FlagChange + modseq++ + tc2.client.Expunge() // MessageExpunge + modseq++ + tc2.transactf("ok", `setmetadata Archive (/private/comment "test2")`) // MailboxMetadataChange + modseq++ + tc2.transactf("ok", `setmetadata "" (/private/vendor/other/x "test2")`) // ServerMetadataChange + modseq++ + tc2.client.Subscribe("doesnotexist2") // SubscriptionChange + tc2.client.Unsubscribe("doesnotexist2") // SubscriptionChange + tc2.client.Create("newbox2", nil) // MailboxName + modseq++ + tc2.client.Rename("newbox2", "oldbox2") // MailboxName + modseq++ + tc2.client.Delete("oldbox2") // MailboxName + modseq++ + // Now trigger receiving a notification. + tc2.client.Create("testbox", nil) // MailboxName + modseq++ + tc2.client.Append("testbox", makeAppend(searchMsg)) // MessageNew + modseq++ + tc.readuntagged( + imapclient.UntaggedStatus{Mailbox: "testbox", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 1, imapclient.StatusUIDNext: 2, imapclient.StatusUnseen: 1, imapclient.StatusHighestModSeq: int64(modseq)}}, + ) + + // Test filtering per mailbox specifier. We create two mailboxes. + tc.client.Create("inbox/a/b", nil) + modseq++ + tc.client.Create("other/a/b", nil) + modseq++ + tc.client.Unsubscribe("other/a/b") + + // Inboxes + tc3 := startNoSwitchboard(t) + defer tc3.closeNoWait() + tc3.client.Login("mjl@mox.example", password0) + tc3.transactf("ok", "Notify Set (Inboxes (messageNew messageExpunge))") + + // Subscribed + tc4 := startNoSwitchboard(t) + defer tc4.closeNoWait() + tc4.client.Login("mjl@mox.example", password0) + tc4.transactf("ok", "Notify Set (Subscribed (messageNew messageExpunge))") + + // Subtree + tc5 := startNoSwitchboard(t) + defer tc5.closeNoWait() + tc5.client.Login("mjl@mox.example", password0) + tc5.transactf("ok", "Notify Set (Subtree (Nonexistent inbox) (messageNew messageExpunge))") + + // Subtree-One + tc6 := startNoSwitchboard(t) + defer tc6.closeNoWait() + tc6.client.Login("mjl@mox.example", password0) + tc6.transactf("ok", "Notify Set (Subtree-One (Nonexistent Inbox/a other) (messageNew messageExpunge))") + + // We append to other/a/b first. It would normally come first in the notifications, + // but we check we only get the second event. + tc2.client.Append("other/a/b", makeAppend(searchMsg)) + modseq++ + tc2.client.Append("inbox/a/b", makeAppend(searchMsg)) + modseq++ + + // No highestmodseq, these connections don't have CONDSTORE enabled. + tc3.readuntagged( + imapclient.UntaggedStatus{Mailbox: "Inbox/a/b", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 1, imapclient.StatusUIDNext: 2, imapclient.StatusUnseen: 1}}, + ) + tc4.readuntagged( + imapclient.UntaggedStatus{Mailbox: "Inbox/a/b", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 1, imapclient.StatusUIDNext: 2, imapclient.StatusUnseen: 1}}, + ) + tc5.readuntagged( + imapclient.UntaggedStatus{Mailbox: "Inbox/a/b", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 1, imapclient.StatusUIDNext: 2, imapclient.StatusUnseen: 1}}, + ) + tc6.readuntagged( + imapclient.UntaggedStatus{Mailbox: "Inbox/a/b", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 1, imapclient.StatusUIDNext: 2, imapclient.StatusUnseen: 1}}, + ) + + // Test for STATUS events on non-selected mailbox for message events. + tc.transactf("ok", "notify set (personal (messageNew messageExpunge flagChange))") + tc.client.Unselect() + tc2.client.Create("statusbox", nil) + modseq++ + tc2.client.Append("statusbox", makeAppend(searchMsg)) + modseq++ + tc.readuntagged( + imapclient.UntaggedStatus{Mailbox: "statusbox", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 1, imapclient.StatusUIDNext: 2, imapclient.StatusUnseen: 1, imapclient.StatusHighestModSeq: int64(modseq)}}, + ) + + // With Selected-Delayed, we only get the events for the selected mailbox for + // explicit commands. We still get other events. + tc.transactf("ok", "notify set (selected-delayed (messageNew messageExpunge flagChange)) (personal (messageNew messageExpunge flagChange))") + tc.client.Select("statusbox") + tc2.client.Append("inbox", makeAppend(searchMsg)) + modseq++ + tc2.client.StoreFlagsSet("1", true, `\Seen`) + modseq++ + tc2.client.Append("statusbox", imapclient.Append{Flags: []string{"newflag"}, Size: int64(len(searchMsg)), Data: strings.NewReader(searchMsg)}) + modseq++ + tc2.client.Select("statusbox") + + tc.readuntagged( + imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 1, imapclient.StatusUIDNext: 6, imapclient.StatusUnseen: 1, imapclient.StatusHighestModSeq: int64(modseq - 2)}}, + imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusUIDValidity: 1, imapclient.StatusUnseen: 0, imapclient.StatusHighestModSeq: int64(modseq - 1)}}, + ) + + tc.transactf("ok", "noop") + tc.xuntagged( + imapclient.UntaggedExists(2), + imapclient.UntaggedFetch{ + Seq: 2, + Attrs: []imapclient.FetchAttr{ + imapclient.FetchUID(2), + imapclient.FetchFlags{`newflag`}, + imapclient.FetchModSeq(modseq), + }, + }, + imapclient.UntaggedFlags{`\Seen`, `\Answered`, `\Flagged`, `\Deleted`, `\Draft`, `$Forwarded`, `$Junk`, `$NotJunk`, `$Phishing`, `$MDNSent`, `newflag`}, + ) + + tc2.client.StoreFlagsSet("2", true, `\Deleted`) + modseq++ + tc2.client.Expunge() + modseq++ + tc.transactf("ok", "noop") + tc.xuntagged( + imapclient.UntaggedFetch{ + Seq: 2, + Attrs: []imapclient.FetchAttr{ + imapclient.FetchUID(2), + imapclient.FetchFlags{`\Deleted`}, + imapclient.FetchModSeq(modseq - 1), + }, + }, + imapclient.UntaggedExpunge(2), + ) + + // With Selected-Delayed, we should get events for selected mailboxes immediately when using IDLE. + tc2.client.StoreFlagsSet("*", true, `\Answered`) + modseq++ + + tc2.client.Select("inbox") + tc2.client.StoreFlagsClear("1", true, `\Seen`) + modseq++ + tc2.client.Select("statusbox") + + tc.readuntagged( + imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusUIDValidity: 1, imapclient.StatusUnseen: 1, imapclient.StatusHighestModSeq: int64(modseq)}}, + ) + + tc.conn.SetReadDeadline(time.Now().Add(3 * time.Second)) + tc.cmdf("", "idle") + tc.readprefixline("+ ") + tc.readuntagged(imapclient.UntaggedFetch{ + Seq: 1, + Attrs: []imapclient.FetchAttr{ + imapclient.FetchUID(1), + imapclient.FetchFlags{`\Answered`}, + imapclient.FetchModSeq(modseq - 1), + }, + }) + tc.writelinef("done") + tc.response("ok") + tc.conn.SetReadDeadline(time.Now().Add(30 * time.Second)) + + // If any event matches, we normally return it. But NONE prevents looking further. + tc.client.Unselect() + tc.transactf("ok", "notify set (mailboxes statusbox NONE) (personal (mailboxName))") + tc2.client.StoreFlagsSet("*", true, `\Answered`) // Matches NONE, ignored. + //modseq++ + tc2.client.Create("eventbox", nil) + //modseq++ + tc.readuntagged( + imapclient.UntaggedList{Mailbox: "eventbox", Separator: '/', Flags: []string{`\Subscribed`}}, + ) + + // Check we can return message contents. + tc.transactf("ok", "notify set (selected (messageNew (body[header] body[text]) messageExpunge))") + tc.client.Select("statusbox") + tc2.client.Append("statusbox", makeAppend(searchMsg)) + // modseq++ + offset := strings.Index(searchMsg, "\r\n\r\n") + tc.readuntagged( + imapclient.UntaggedExists(2), + imapclient.UntaggedFetch{ + Seq: 2, + Attrs: []imapclient.FetchAttr{ + imapclient.FetchUID(3), + imapclient.FetchBody{ + RespAttr: "BODY[HEADER]", + Section: "HEADER", + Body: searchMsg[:offset+4], + }, + imapclient.FetchBody{ + RespAttr: "BODY[TEXT]", + Section: "TEXT", + Body: searchMsg[offset+4:], + }, + imapclient.FetchFlags(nil), + }, + }, + ) + + // If we encounter an error during fetch, an untagged NO is returned. + // We ask for the 2nd part of a message, and we add a message with just 1 part. + tc.transactf("ok", "notify set (selected (messageNew (body[2]) messageExpunge))") + tc2.client.Append("statusbox", makeAppend(exampleMsg)) + // modseq++ + tc.readuntagged( + imapclient.UntaggedExists(3), + imapclient.UntaggedResult{ + Status: "NO", + RespText: imapclient.RespText{ + More: "generating notify fetch response: requested part does not exist", + }, + }, + imapclient.UntaggedFetch{ + Seq: 3, + Attrs: []imapclient.FetchAttr{ + imapclient.FetchUID(4), + }, + }, + ) + + // When adding new tests, uncomment modseq++ lines above. +} + +func TestNotifyOverflow(t *testing.T) { + orig := store.CommPendingChangesMax + store.CommPendingChangesMax = 3 + defer func() { + store.CommPendingChangesMax = orig + }() + + defer mockUIDValidity()() + tc := start(t) + defer tc.close() + tc.client.Login("mjl@mox.example", password0) + tc.client.Select("inbox") + tc.transactf("ok", "noop") + + tc2 := startNoSwitchboard(t) + defer tc2.closeNoWait() + tc2.client.Login("mjl@mox.example", password0) + tc2.client.Select("inbox") + + // Generates 4 changes, crossing max 3. + tc2.client.Append("inbox", makeAppend(searchMsg)) + tc2.client.Append("inbox", makeAppend(searchMsg)) + + tc.transactf("ok", "noop") + tc.xuntagged( + imapclient.UntaggedResult{ + Status: "OK", + RespText: imapclient.RespText{ + Code: "NOTIFICATIONOVERFLOW", + More: "out of sync after too many pending changes", + }, + }, + ) + + // Won't be getting any more notifications until we enable them again with NOTIFY. + tc2.client.Append("inbox", makeAppend(searchMsg)) + tc.transactf("ok", "noop") + tc.xuntagged() + + // Enable notify again. We won't get a notification because the message isn't yet + // known in the session. + tc.transactf("ok", "notify set (selected (messageNew messageExpunge flagChange))") + tc2.client.StoreFlagsAdd("1", true, `\Seen`) + tc.transactf("ok", "noop") + tc.xuntagged() + + // Reselect to get the message visible in the session. + tc.client.Select("inbox") + tc2.client.StoreFlagsClear("1", true, `\Seen`) + tc.transactf("ok", "noop") + tc.xuntagged( + imapclient.UntaggedFetch{ + Seq: 1, + Attrs: []imapclient.FetchAttr{ + imapclient.FetchUID(1), + imapclient.FetchFlags(nil), + }, + }, + ) + + // Trigger overflow for changes for "selected-delayed". + store.CommPendingChangesMax = 10 + delayedMax := selectedDelayedChangesMax + selectedDelayedChangesMax = 1 + defer func() { + selectedDelayedChangesMax = delayedMax + }() + tc.transactf("ok", "notify set (selected-delayed (messageNew messageExpunge flagChange))") + tc2.client.StoreFlagsAdd("1", true, `\Seen`) + tc2.client.StoreFlagsClear("1", true, `\Seen`) + tc.transactf("ok", "noop") + tc.xuntagged( + imapclient.UntaggedResult{ + Status: "OK", + RespText: imapclient.RespText{ + Code: "NOTIFICATIONOVERFLOW", + More: "out of sync after too many pending changes for selected mailbox", + }, + }, + ) + + // Again, no new notifications until we select and enable again. + tc2.client.StoreFlagsAdd("1", true, `\Seen`) + tc.transactf("ok", "noop") + tc.xuntagged() + + tc.client.Select("inbox") + tc.transactf("ok", "notify set (selected-delayed (messageNew messageExpunge flagChange))") + tc2.client.StoreFlagsClear("1", true, `\Seen`) + tc.transactf("ok", "noop") + tc.xuntagged( + imapclient.UntaggedFetch{ + Seq: 1, + Attrs: []imapclient.FetchAttr{ + imapclient.FetchUID(1), + imapclient.FetchFlags(nil), + }, + }, + ) +} diff --git a/imapserver/pack.go b/imapserver/pack.go index b232f64..f6b934b 100644 --- a/imapserver/pack.go +++ b/imapserver/pack.go @@ -118,7 +118,7 @@ func (t readerSizeSyncliteral) xwriteTo(c *conn, xw io.Writer) { lit = "~" } fmt.Fprintf(xw, "%s{%d}\r\n", lit, t.size) - defer c.xtrace(mlog.LevelTracedata)() + defer c.xtracewrite(mlog.LevelTracedata)() if _, err := io.Copy(xw, io.LimitReader(t.r, t.size)); err != nil { panic(err) } @@ -143,7 +143,7 @@ func (t readerSyncliteral) xwriteTo(c *conn, xw io.Writer) { panic(err) } fmt.Fprintf(xw, "{%d}\r\n", len(buf)) - defer c.xtrace(mlog.LevelTracedata)() + defer c.xtracewrite(mlog.LevelTracedata)() xw.Write(buf) } diff --git a/imapserver/parse.go b/imapserver/parse.go index 4bef1cc..a62e235 100644 --- a/imapserver/parse.go +++ b/imapserver/parse.go @@ -581,7 +581,7 @@ var fetchAttWords = []string{ } // ../rfc/9051:6557 ../rfc/3501:4751 ../rfc/7162:2483 -func (p *parser) xfetchAtt(isUID bool) (r fetchAtt) { +func (p *parser) xfetchAtt() (r fetchAtt) { defer p.context("fetchAtt")() f := p.xtakelist(fetchAttWords...) r.peek = strings.HasSuffix(f, ".PEEK") @@ -616,7 +616,7 @@ func (p *parser) xfetchAtt(isUID bool) (r fetchAtt) { } // ../rfc/9051:6553 ../rfc/3501:4748 -func (p *parser) xfetchAtts(isUID bool) []fetchAtt { +func (p *parser) xfetchAtts() []fetchAtt { defer p.context("fetchAtts")() fields := func(l ...string) []fetchAtt { @@ -640,13 +640,13 @@ func (p *parser) xfetchAtts(isUID bool) []fetchAtt { } if !p.hasPrefix("(") { - return []fetchAtt{p.xfetchAtt(isUID)} + return []fetchAtt{p.xfetchAtt()} } l := []fetchAtt{} p.xtake("(") for { - l = append(l, p.xfetchAtt(isUID)) + l = append(l, p.xfetchAtt()) if !p.take(" ") { break } @@ -1054,3 +1054,145 @@ func (p *parser) xmetadataKeyValue() (key string, isString bool, value []byte) { return } + +type eventGroup struct { + MailboxSpecifier mailboxSpecifier + Events []notifyEvent // NONE is represented by an empty list. +} + +type mbspecKind string + +const ( + mbspecSelected mbspecKind = "SELECTED" + mbspecSelectedDelayed mbspecKind = "SELECTED-DELAYED" // Only for NOTIFY. + mbspecInboxes mbspecKind = "INBOXES" + mbspecPersonal mbspecKind = "PERSONAL" + mbspecSubscribed mbspecKind = "SUBSCRIBED" + mbspecSubtreeOne mbspecKind = "SUBTREE-ONE" // For ESEARCH, we allow it for NOTIFY too. + mbspecSubtree mbspecKind = "SUBTREE" + mbspecMailboxes mbspecKind = "MAILBOXES" +) + +// Used by both the ESEARCH and NOTIFY commands. +type mailboxSpecifier struct { + Kind mbspecKind + Mailboxes []string +} + +type notifyEvent struct { + // Kind is always upper case. Should be one of eventKind, anything else must result + // in a BADEVENT response code. + Kind string + + FetchAtt []fetchAtt // Only for MessageNew +} + +// ../rfc/5465:943 +func (p *parser) xeventGroup() (eg eventGroup) { + p.xtake("(") + eg.MailboxSpecifier = p.xfilterMailbox(mbspecsNotify) + p.xspace() + if p.take("NONE") { + p.xtake(")") + return eg + } + p.xtake("(") + for { + e := p.xnotifyEvent() + eg.Events = append(eg.Events, e) + if !p.space() { + break + } + } + p.xtake(")") + p.xtake(")") + return eg +} + +var mbspecsEsearch = []mbspecKind{ + mbspecSelected, // selected-delayed is only for NOTIFY. + mbspecInboxes, + mbspecPersonal, + mbspecSubscribed, + mbspecSubtreeOne, // Must come before Subtree due to eager parsing. + mbspecSubtree, + mbspecMailboxes, +} + +var mbspecsNotify = []mbspecKind{ + mbspecSelectedDelayed, // Must come before mbspecSelected, for eager parsing and mbspecSelected. + mbspecSelected, + mbspecInboxes, + mbspecPersonal, + mbspecSubscribed, + mbspecSubtreeOne, // From ESEARCH, we also allow it in NOTIFY. + mbspecSubtree, + mbspecMailboxes, +} + +// If not esearch with "subtree-one", then for notify with "selected-delayed". +func (p *parser) xfilterMailbox(allowed []mbspecKind) (ms mailboxSpecifier) { + var kind mbspecKind + for _, s := range allowed { + if p.take(string(s)) { + kind = s + break + } + } + if kind == mbspecKind("") { + xsyntaxErrorf("expected mailbox specifier") + } + + ms.Kind = kind + switch kind { + case "SUBTREE", "SUBTREE-ONE", "MAILBOXES": + p.xtake(" ") + // One or more mailboxes. Multiple start with a list. ../rfc/5465:937 + if p.take("(") { + for { + ms.Mailboxes = append(ms.Mailboxes, p.xmailbox()) + if !p.take(" ") { + break + } + } + p.xtake(")") + } else { + ms.Mailboxes = []string{p.xmailbox()} + } + } + return ms +} + +type eventKind string + +const ( + eventMessageNew eventKind = "MESSAGENEW" + eventMessageExpunge eventKind = "MESSAGEEXPUNGE" + eventFlagChange eventKind = "FLAGCHANGE" + eventAnnotationChange eventKind = "ANNOTATIONCHANGE" + eventMailboxName eventKind = "MAILBOXNAME" + eventSubscriptionChange eventKind = "SUBSCRIPTIONCHANGE" + eventMailboxMetadataChange eventKind = "MAILBOXMETADATACHANGE" + eventServerMetadataChange eventKind = "SERVERMETADATACHANGE" +) + +var messageEventKinds = []eventKind{eventMessageNew, eventMessageExpunge, eventFlagChange, eventAnnotationChange} + +// ../rfc/5465:974 +func (p *parser) xnotifyEvent() notifyEvent { + s := strings.ToUpper(p.xatom()) + e := notifyEvent{Kind: s} + if eventKind(e.Kind) == eventMessageNew { + if p.take(" (") { + for { + a := p.xfetchAtt() + e.FetchAtt = append(e.FetchAtt, a) + if !p.take(" ") { + break + } + } + p.xtake(")") + } + } + return e +} diff --git a/imapserver/replace.go b/imapserver/replace.go index 0d68245..c3d57ff 100644 --- a/imapserver/replace.go +++ b/imapserver/replace.go @@ -199,10 +199,10 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) { } // Read the message data. - defer c.xtrace(mlog.LevelTracedata)() + defer c.xtraceread(mlog.LevelTracedata)() mw := message.NewWriter(f) msize, err := io.Copy(mw, io.LimitReader(c.br, size)) - c.xtrace(mlog.LevelTrace) // Restore. + c.xtraceread(mlog.LevelTrace) // Restore. if err != nil { // Cannot use xcheckf due to %w handling of errIO. c.xbrokenf("reading literal message: %s (%w)", err, errIO) @@ -228,7 +228,12 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) { var om, nm store.Message var mbSrc, mbDst store.Mailbox // Src and dst mailboxes can be different. ../rfc/8508:263 + var overflow bool var pendingChanges []store.Change + defer func() { + // In case of panic. + c.flushChanges(pendingChanges) + }() c.account.WithWLock(func() { var changes []store.Change @@ -288,7 +293,7 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) { xcheckf(err, "delivering message") newID = nm.ID - changes = append(changes, nm.ChangeAddUID(), mbDst.ChangeCounts()) + changes = append(changes, nm.ChangeAddUID(mbDst), mbDst.ChangeCounts()) if nkeywords != len(mbDst.Keywords) { changes = append(changes, mbDst.ChangeKeywords()) } @@ -298,7 +303,7 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) { }) // Fetch pending changes, possibly with new UIDs, so we can apply them before adding our own new UID. - pendingChanges = c.comm.Get() + overflow, pendingChanges = c.comm.Get() if oldMsgExpunged { return @@ -315,7 +320,9 @@ func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) { }) // Must update our msgseq/uids tracking with latest pending changes. - c.applyChanges(pendingChanges, false) + l := pendingChanges + pendingChanges = nil + c.xapplyChanges(overflow, l, false, false) // If we couldn't find the message, send a NO response. We've just applied pending // changes, which should have expunged the absent message. diff --git a/imapserver/search.go b/imapserver/search.go index 1a04c5a..4141ca0 100644 --- a/imapserver/search.go +++ b/imapserver/search.go @@ -49,33 +49,13 @@ func (c *conn) cmdxSearch(isUID, isE bool, tag, cmd string, p *parser) { // The ESEARCH command has various ways to specify which mailboxes are to be // searched. We parse and gather the request first, and evaluate them to mailboxes // after parsing, when we start and have a DB transaction. - type mailboxSpec struct { - Kind string - Args []string - } - var mailboxSpecs []mailboxSpec + var mailboxSpecs []mailboxSpecifier // ../rfc/7377:468 if isE && p.take(" IN (") { for { - mbs := mailboxSpec{} - mbs.Kind = p.xtakelist("SELECTED", "INBOXES", "PERSONAL", "SUBSCRIBED", "SUBTREE-ONE", "SUBTREE", "MAILBOXES") - switch mbs.Kind { - case "SUBTREE", "SUBTREE-ONE", "MAILBOXES": - p.xtake(" ") - if p.take("(") { - for { - mbs.Args = append(mbs.Args, p.xmailbox()) - if !p.take(" ") { - break - } - } - p.xtake(")") - } else { - mbs.Args = []string{p.xmailbox()} - } - } - mailboxSpecs = append(mailboxSpecs, mbs) + ms := p.xfilterMailbox(mbspecsEsearch) + mailboxSpecs = append(mailboxSpecs, ms) if !p.take(" ") { break @@ -214,9 +194,9 @@ func (c *conn) cmdxSearch(isUID, isE bool, tag, cmd string, p *parser) { if len(mailboxSpecs) > 0 { // While gathering, we deduplicate mailboxes. ../rfc/7377:312 m := map[int64]store.Mailbox{} - for _, mbs := range mailboxSpecs { - switch mbs.Kind { - case "SELECTED": + for _, ms := range mailboxSpecs { + switch ms.Kind { + case mbspecSelected: // ../rfc/7377:306 if c.state != stateSelected { xsyntaxErrorf("cannot use ESEARCH with selected when state is not selected") @@ -225,7 +205,7 @@ func (c *conn) cmdxSearch(isUID, isE bool, tag, cmd string, p *parser) { mb := c.xmailboxID(tx, c.mailboxID) // Validate. m[mb.ID] = mb - case "INBOXES": + case mbspecInboxes: // Inbox and everything below. And we look at destinations and rulesets. We all // mailboxes from the destinations, and all from the rulesets except when // ListAllowDomain is non-empty. @@ -265,7 +245,7 @@ func (c *conn) cmdxSearch(isUID, isE bool, tag, cmd string, p *parser) { } } - case "PERSONAL": + case mbspecPersonal: // All mailboxes in the personal namespace. Which is all mailboxes for us. // ../rfc/5465:817 for mb, err := range bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).All() { @@ -273,7 +253,7 @@ func (c *conn) cmdxSearch(isUID, isE bool, tag, cmd string, p *parser) { m[mb.ID] = mb } - case "SUBSCRIBED": + case mbspecSubscribed: // Mailboxes that are subscribed. Will typically be same as personal, since we // subscribe to all mailboxes. But user can manage subscriptions differently. // ../rfc/5465:831 @@ -286,7 +266,7 @@ func (c *conn) cmdxSearch(isUID, isE bool, tag, cmd string, p *parser) { } } - case "SUBTREE", "SUBTREE-ONE": + case mbspecSubtree, mbspecSubtreeOne: // The mailbox name itself, and children. ../rfc/5465:847 // SUBTREE is arbitrarily deep, SUBTREE-ONE is one level deeper than requested // mailbox. The mailbox itself is included too ../rfc/7377:274 @@ -294,13 +274,13 @@ func (c *conn) cmdxSearch(isUID, isE bool, tag, cmd string, p *parser) { // We don't have to worry about loops. Mailboxes are not in the file system. // ../rfc/7377:291 - for _, name := range mbs.Args { + for _, name := range ms.Mailboxes { name = xcheckmailboxname(name, true) - one := mbs.Kind == "SUBTREE-ONE" + one := ms.Kind == mbspecSubtreeOne var ntoken int if one { - ntoken = len(strings.Split(name, "/")) + ntoken = len(strings.Split(name, "/")) + 1 } q := bstore.QueryTx[store.Mailbox](tx) @@ -312,15 +292,15 @@ func (c *conn) cmdxSearch(isUID, isE bool, tag, cmd string, p *parser) { if mb.Name != name && !strings.HasPrefix(mb.Name, name+"/") { break } - if !one || mb.Name == name || len(strings.Split(mb.Name, "/")) == ntoken+1 { + if !one || mb.Name == name || len(strings.Split(mb.Name, "/")) == ntoken { m[mb.ID] = mb } } } - case "MAILBOXES": + case mbspecMailboxes: // Just the specified mailboxes. ../rfc/5465:853 - for _, name := range mbs.Args { + for _, name := range ms.Mailboxes { name = xcheckmailboxname(name, true) // If a mailbox doesn't exist, we don't treat it as an error. Seems reasonable diff --git a/imapserver/server.go b/imapserver/server.go index 7759584..59977f9 100644 --- a/imapserver/server.go +++ b/imapserver/server.go @@ -183,6 +183,7 @@ var serverCapabilities = strings.Join([]string{ "PREVIEW", // ../rfc/8970:114 "INPROGRESS", // ../rfc/9585:101 "MULTISEARCH", // ../rfc/7377:187 + "NOTIFY", // ../rfc/5465:195 // "COMPRESS=DEFLATE", // ../rfc/4978, disabled for interoperability issues: The flate reader (inflate) still blocks on partial flushes, preventing progress. }, " ") @@ -213,6 +214,7 @@ type conn struct { log mlog.Log // Used for all synchronous logging on this connection, see logbg for logging in a separate goroutine. enabled map[capability]bool // All upper-case. compress bool // Whether compression is enabled, via compress command. + notify *notify // For the NOTIFY extension. Event/change filtering active if non-nil. // Set by SEARCH with SAVE. Can be used by commands accepting a sequence-set with // value "$". When used, UIDs must be verified to still exist, because they may @@ -282,7 +284,7 @@ func stateCommands(cmds ...string) map[string]struct{} { var ( commandsStateAny = stateCommands("capability", "noop", "logout", "id") commandsStateNotAuthenticated = stateCommands("starttls", "authenticate", "login") - commandsStateAuthenticated = stateCommands("enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub", "getquotaroot", "getquota", "getmetadata", "setmetadata", "compress", "esearch") + commandsStateAuthenticated = stateCommands("enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub", "getquotaroot", "getquota", "getmetadata", "setmetadata", "compress", "esearch", "notify") commandsStateSelected = stateCommands("close", "unselect", "expunge", "search", "fetch", "store", "copy", "move", "uid expunge", "uid search", "uid fetch", "uid store", "uid copy", "uid move", "replace", "uid replace", "esearch") ) @@ -319,6 +321,7 @@ var commands = map[string]func(c *conn, tag, cmd string, p *parser){ "setmetadata": (*conn).cmdSetmetadata, "compress": (*conn).cmdCompress, "esearch": (*conn).cmdEsearch, + "notify": (*conn).cmdNotify, // Connection does not have to be in selected state. ../rfc/5465:792 ../rfc/5465:921 // Selected. "check": (*conn).cmdCheck, @@ -487,6 +490,11 @@ func (c *conn) xdbread(fn func(tx *bstore.Tx)) { // Closes the currently selected/active mailbox, setting state from selected to authenticated. // Does not remove messages marked for deletion. func (c *conn) unselect() { + // Flush any pending delayed changes as if the mailbox is still selected. Probably + // better than causing STATUS responses for the mailbox being unselected but which + // is still selected. + c.flushNotifyDelayed() + if c.state == stateSelected { c.state = stateAuthenticated } @@ -494,6 +502,29 @@ func (c *conn) unselect() { c.uids = nil } +func (c *conn) flushNotifyDelayed() { + if c.notify == nil { + return + } + delayed := c.notify.Delayed + c.notify.Delayed = nil + c.flushChanges(delayed) +} + +// flushChanges is called for NOTIFY changes we shouldn't send untagged messages +// about but must process for message removals. We don't update the selected +// mailbox message sequence numbers, since the client would have no idea we +// adjusted message sequence numbers. Combined with NOTIFY NONE, this means +// messages may be erased that the client thinks still exists in its session. +func (c *conn) flushChanges(changes []store.Change) { + for _, change := range changes { + switch ch := change.(type) { + case store.ChangeRemoveUIDs: + c.comm.RemovalSeen(ch) + } + } +} + func (c *conn) setSlow(on bool) { if on && !c.slow { c.log.Debug("connection changed to slow") @@ -529,13 +560,18 @@ func (c *conn) Write(buf []byte) (int, error) { return n, nil } -func (c *conn) xtrace(level slog.Level) func() { - c.xflush() +func (c *conn) xtraceread(level slog.Level) func() { c.tr.SetTrace(level) + return func() { + c.tr.SetTrace(mlog.LevelTrace) + } +} + +func (c *conn) xtracewrite(level slog.Level) func() { + c.xflush() c.xtw.SetTrace(level) return func() { c.xflush() - c.tr.SetTrace(mlog.LevelTrace) c.xtw.SetTrace(mlog.LevelTrace) } } @@ -561,7 +597,6 @@ func (c *conn) readline0() (string, error) { if err != nil && errors.Is(err, moxio.ErrLineTooLong) { return "", fmt.Errorf("%s (%w)", err, errProtocol) } else if err != nil { - c.connBroken = true return "", fmt.Errorf("%s (%w)", err, errIO) } return line, nil @@ -591,10 +626,11 @@ func (c *conn) xreadline(readCmd bool) string { } if err != nil { if readCmd && errors.Is(err, os.ErrDeadlineExceeded) { - err := c.conn.SetWriteDeadline(time.Now().Add(10 * time.Second)) - c.log.Check(err, "setting write deadline") + err := c.conn.SetDeadline(time.Now().Add(10 * time.Second)) + c.log.Check(err, "setting deadline") c.xwritelinef("* BYE inactive") } + c.connBroken = true if !errors.Is(err, errIO) && !errors.Is(err, errProtocol) { c.xbrokenf("%s (%w)", err, errIO) } @@ -630,7 +666,8 @@ func (c *conn) xbwriteresultf(format string, args ...any) { // ../rfc/9051:5862 ../rfc/7162:2033 default: if c.comm != nil { - c.applyChanges(c.comm.Get(), false) + overflow, changes := c.comm.Get() + c.xapplyChanges(overflow, changes, false, true) } } c.xbwritelinef(format, args...) @@ -670,8 +707,7 @@ func (c *conn) xflush() { } } -func (c *conn) readCommand(tag *string) (cmd string, p *parser) { - line := c.xreadline(true) +func (c *conn) parseCommand(tag *string, line string) (cmd string, p *parser) { p = newParser(line, c) p.context("tag") *tag = p.xtag() @@ -787,6 +823,10 @@ func serve(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, x c.log.Debugx("closing connection", err) } + // If changes for NOTIFY's SELECTED-DELAYED are still pending, we'll acknowledge + // their message removals so the files can be erased. + c.flushNotifyDelayed() + if c.account != nil { c.comm.Unregister() err := c.account.Close() @@ -1225,7 +1265,53 @@ func (c *conn) command() { }() tag = "*" - cmd, p = c.readCommand(&tag) + + // If NOTIFY is enabled, we wait for either a line (with a command) from the + // client, or a change event. If we see a line, we continue below as for the + // non-NOTIFY case, parsing the command. + var line string + if c.notify != nil { + Wait: + for { + select { + case le := <-c.lineChan(): + c.line = nil + if err := le.err; err != nil { + if errors.Is(err, os.ErrDeadlineExceeded) { + err := c.conn.SetDeadline(time.Now().Add(10 * time.Second)) + c.log.Check(err, "setting write deadline") + c.xwritelinef("* BYE inactive") + } + c.connBroken = true + if !errors.Is(err, errIO) && !errors.Is(err, errProtocol) { + c.xbrokenf("%s (%w)", err, errIO) + } + panic(err) + } + line = le.line + break Wait + + case <-c.comm.Pending: + overflow, changes := c.comm.Get() + c.xapplyChanges(overflow, changes, false, false) + c.xflush() + + case <-mox.Shutdown.Done(): + // ../rfc/9051:5375 + c.xwritelinef("* BYE shutting down") + c.xbrokenf("shutting down (%w)", errIO) + } + } + + // Reset the write deadline. In case of little activity, with a command timeout of + // 30 minutes, we have likely passed it. + err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Minute)) + c.log.Check(err, "setting write deadline") + } else { + // Without NOTIFY, we just read a line. + line = c.xreadline(true) + } + cmd, p = c.parseCommand(&tag, line) cmdlow = strings.ToLower(cmd) c.cmd = cmdlow c.cmdStart = time.Now() @@ -1371,7 +1457,7 @@ func (c *conn) sequenceRemove(seq msgseq, uid store.UID) { // add uid to the session. care must be taken that pending changes are fetched // while holding the account wlock, and applied before adding this uid, because // those pending changes may contain another new uid that has to be added first. -func (c *conn) uidAppend(uid store.UID) { +func (c *conn) uidAppend(uid store.UID) msgseq { if uidSearch(c.uids, uid) > 0 { xserverErrorf("uid already present (%w)", errProtocol) } @@ -1382,6 +1468,7 @@ func (c *conn) uidAppend(uid store.UID) { if sanityChecks { checkUIDs(c.uids) } + return msgseq(len(c.uids)) } // sanity check that uids are in ascending order. @@ -1577,11 +1664,44 @@ func (c *conn) xmailboxID(tx *bstore.Tx, id int64) store.Mailbox { // If initial is true, we only apply the changes. // Should not be called while holding locks, as changes are written to client connections, which can block. // Does not flush output. -func (c *conn) applyChanges(changes []store.Change, initial bool) { +func (c *conn) xapplyChanges(overflow bool, changes []store.Change, initial, sendDelayed bool) { + // If more changes were generated than we can process, we send a + // NOTIFICATIONOVERFLOW as defined in the NOTIFY extension. ../rfc/5465:712 + if overflow { + if c.notify != nil && len(c.notify.Delayed) > 0 { + changes = append(c.notify.Delayed, changes...) + } + c.flushChanges(changes) + // We must not send any more unsolicited untagged responses to the client for + // NOTIFY, but we also follow this for IDLE. ../rfc/5465:717 + c.notify = ¬ify{} + c.xbwritelinef("* OK [NOTIFICATIONOVERFLOW] out of sync after too many pending changes") + if !initial { + return + } + changes = nil + } + + // applyChanges for IDLE and NOTIFY. When explicitly in IDLE while NOTIFY is + // enabled, we still respond with messages as for NOTIFY. ../rfc/5465:406 + if c.notify != nil { + c.xapplyChangesNotify(changes, sendDelayed) + return + } if len(changes) == 0 { return } + // Even in the case of a panic (e.g. i/o errors), we must mark removals as seen. + origChanges := changes + defer func() { + for _, change := range origChanges { + if ch, ok := change.(store.ChangeRemoveUIDs); ok { + c.comm.RemovalSeen(ch) + } + } + }() + err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Minute)) c.log.Check(err, "setting write deadline") @@ -1596,10 +1716,9 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { mbID = ch.MailboxID case store.ChangeRemoveUIDs: mbID = ch.MailboxID - c.comm.RemovalSeen(ch) case store.ChangeFlags: mbID = ch.MailboxID - case store.ChangeRemoveMailbox, store.ChangeAddMailbox, store.ChangeRenameMailbox, store.ChangeAddSubscription: + case store.ChangeRemoveMailbox, store.ChangeAddMailbox, store.ChangeRenameMailbox, store.ChangeAddSubscription, store.ChangeRemoveSubscription: n = append(n, change) continue case store.ChangeAnnotation: @@ -1653,6 +1772,7 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { if condstore { modseqStr = fmt.Sprintf(" MODSEQ (%d)", add.ModSeq.Client()) } + c.xbwritelinef("* %d FETCH (UID %d FLAGS %s%s)", seq, add.UID, flaglist(add.Flags, add.Keywords).pack(c), modseqStr) } continue @@ -1689,6 +1809,7 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { c.xbwritelinef("* VANISHED %s", s) } } + case store.ChangeFlags: // The uid can be unknown if we just expunged it while another session marked it as deleted just before. seq := c.sequence(ch.UID) @@ -1702,6 +1823,7 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { } c.xbwritelinef("* %d FETCH (UID %d FLAGS %s%s)", seq, ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c), modseqStr) } + case store.ChangeRemoveMailbox: // Only announce \NonExistent to modern clients, otherwise they may ignore the // unrecognized \NonExistent and interpret this as a newly created mailbox, while @@ -1709,8 +1831,10 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { if c.enabled[capIMAP4rev2] { c.xbwritelinef(`* LIST (\NonExistent) "/" %s`, mailboxt(ch.Name).pack(c)) } + case store.ChangeAddMailbox: c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(ch.Flags, " "), mailboxt(ch.Mailbox.Name).pack(c)) + case store.ChangeRenameMailbox: // OLDNAME only with IMAP4rev2 or NOTIFY ../rfc/9051:2726 ../rfc/5465:628 var oldname string @@ -1718,17 +1842,391 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { oldname = fmt.Sprintf(` ("OLDNAME" (%s))`, mailboxt(ch.OldName).pack(c)) } c.xbwritelinef(`* LIST (%s) "/" %s%s`, strings.Join(ch.Flags, " "), mailboxt(ch.NewName).pack(c), oldname) + case store.ChangeAddSubscription: - c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(append([]string{`\Subscribed`}, ch.Flags...), " "), mailboxt(ch.Name).pack(c)) + c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(append([]string{`\Subscribed`}, ch.ListFlags...), " "), mailboxt(ch.MailboxName).pack(c)) + + case store.ChangeRemoveSubscription: + c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(ch.ListFlags, " "), mailboxt(ch.MailboxName).pack(c)) + case store.ChangeAnnotation: // ../rfc/5464:807 ../rfc/5464:788 c.xbwritelinef(`* METADATA %s %s`, mailboxt(ch.MailboxName).pack(c), astring(ch.Key).pack(c)) + default: panic(fmt.Sprintf("internal error, missing case for %#v", change)) } } } +// Like applyChanges, but for notify, with configurable mailboxes to notify about, +// and configurable events to send, including which fetch attributes to return. +// All calls must go through applyChanges, for overflow handling. +func (c *conn) xapplyChangesNotify(changes []store.Change, sendDelayed bool) { + if sendDelayed && len(c.notify.Delayed) > 0 { + changes = append(c.notify.Delayed, changes...) + c.notify.Delayed = nil + } + + if len(changes) == 0 { + return + } + + // Even in the case of a panic (e.g. i/o errors), we must mark removals as seen. + // For selected-delayed, we may have postponed handling the message, so we call + // RemovalSeen when handling a change, and mark how far we got, so we only process + // changes that we haven't processed yet. + unhandled := changes + defer func() { + for _, change := range unhandled { + if ch, ok := change.(store.ChangeRemoveUIDs); ok { + c.comm.RemovalSeen(ch) + } + } + }() + + c.log.Debug("applying notify changes", slog.Any("changes", changes)) + + err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Minute)) + c.log.Check(err, "setting write deadline") + + qresync := c.enabled[capQresync] + condstore := c.enabled[capCondstore] + + // Prepare for providing a read-only transaction on first-use, for MessageNew fetch + // attributes. + var tx *bstore.Tx + defer func() { + if tx != nil { + err := tx.Rollback() + c.log.Check(err, "rolling back tx") + } + }() + xtx := func() *bstore.Tx { + if tx != nil { + return tx + } + + var err error + tx, err = c.account.DB.Begin(context.TODO(), false) + xcheckf(err, "tx") + return tx + } + + // On-demand mailbox lookups, with cache. + mailboxes := map[int64]store.Mailbox{} + xmailbox := func(id int64) store.Mailbox { + if mb, ok := mailboxes[id]; ok { + return mb + } + mb := store.Mailbox{ID: id} + err := xtx().Get(&mb) + xcheckf(err, "get mailbox") + mailboxes[id] = mb + return mb + } + + // Keep track of last command, to close any open message file (for fetching + // attributes) in case of a panic. + var cmd *fetchCmd + defer func() { + if cmd != nil { + cmd.msgclose() + cmd = nil + } + }() + + for index, change := range changes { + switch ch := change.(type) { + case store.ChangeAddUID: + // ../rfc/5465:511 + // todo: ../rfc/5465:525 group ChangeAddUID for the same mailbox, so we can send a single EXISTS. useful for imports. + + mb := xmailbox(ch.MailboxID) + ms, ev, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventMessageNew) + if !ok { + continue + } + + // For non-selected mailbox, send STATUS with UIDNEXT, MESSAGES. And HIGESTMODSEQ + // in case of condstore/qresync. ../rfc/5465:537 + // There is no mention of UNSEEN for MessageNew, but clients will want to show a + // new "unread messages" count, and they will have to understand it since + // FlagChange is specified as sending UNSEEN. + if mb.ID != c.mailboxID { + if condstore || qresync { + c.xbwritelinef("* STATUS %s (UIDNEXT %d MESSAGES %d HIGHESTMODSEQ %d UNSEEN %d)", mailboxt(mb.Name).pack(c), ch.UID+1, ch.MessageCountIMAP, ch.ModSeq, ch.Unseen) + } else { + c.xbwritelinef("* STATUS %s (UIDNEXT %d MESSAGES %d UNSEEN %d)", mailboxt(mb.Name).pack(c), ch.UID+1, ch.MessageCountIMAP, ch.Unseen) + } + continue + } + + // Delay sending all message events, we want to prevent synchronization issues + // around UIDNEXT and MODSEQ. ../rfc/5465:808 + if ms.Kind == mbspecSelectedDelayed && !sendDelayed { + c.notify.Delayed = append(c.notify.Delayed, change) + continue + } + + seq := c.uidAppend(ch.UID) + + // ../rfc/5465:515 + c.xbwritelinef("* %d EXISTS", len(c.uids)) + + // If client did not specify attributes, we'll send the defaults. + if len(ev.FetchAtt) == 0 { + var modseqStr string + if condstore { + modseqStr = fmt.Sprintf(" MODSEQ (%d)", ch.ModSeq.Client()) + } + // NOTIFY does not specify the default fetch attributes to return, we send UID and + // FLAGS. + c.xbwritelinef("* %d FETCH (UID %d FLAGS %s%s)", seq, ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c), modseqStr) + continue + } + + // todo: ../rfc/5465:543 mark messages as \seen after processing if client didn't use the .PEEK-variants. + cmd = &fetchCmd{conn: c, isUID: true, rtx: xtx(), mailboxID: ch.MailboxID, uid: ch.UID} + data, err := cmd.process(ev.FetchAtt) + if err != nil { + // There is no good way to notify the client about errors. We continue below to + // send a FETCH with just the UID. And we send an untagged NO in the hope a client + // developer sees the message. + c.log.Errorx("generating notify fetch response", err, slog.Int64("mailboxid", ch.MailboxID), slog.Any("uid", ch.UID)) + c.xbwritelinef("* NO generating notify fetch response: %s", err.Error()) + data = listspace{bare("UID"), number(ch.UID)} + } + fmt.Fprintf(cmd.conn.xbw, "* %d FETCH ", seq) + func() { + defer c.xtracewrite(mlog.LevelTracedata)() + data.xwriteTo(cmd.conn, cmd.conn.xbw) + c.xtracewrite(mlog.LevelTrace) // Restore. + cmd.conn.xbw.Write([]byte("\r\n")) + }() + + cmd.msgclose() + cmd = nil + + case store.ChangeRemoveUIDs: + // ../rfc/5465:567 + mb := xmailbox(ch.MailboxID) + ms, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventMessageExpunge) + if !ok { + unhandled = changes[index+1:] + c.comm.RemovalSeen(ch) + continue + } + + // For non-selected mailboxes, we send STATUS with at least UIDNEXT and MESSAGES. + // ../rfc/5465:576 + // In case of QRESYNC, we send HIGHESTMODSEQ. Also for CONDSTORE, which isn't + // required like for MessageExpunge like it is for MessageNew. ../rfc/5465:578 + // ../rfc/5465:539 + // There is no mention of UNSEEN, but clients will want to show a new "unread + // messages" count, and they can parse it since FlagChange is specified as sending + // UNSEEN. + if mb.ID != c.mailboxID { + unhandled = changes[index+1:] + c.comm.RemovalSeen(ch) + if condstore || qresync { + c.xbwritelinef("* STATUS %s (UIDNEXT %d MESSAGES %d HIGHESTMODSEQ %d UNSEEN %d)", mailboxt(mb.Name).pack(c), ch.UIDNext, ch.MessageCountIMAP, ch.ModSeq, ch.Unseen) + } else { + c.xbwritelinef("* STATUS %s (UIDNEXT %d MESSAGES %d UNSEEN %d)", mailboxt(mb.Name).pack(c), ch.UIDNext, ch.MessageCountIMAP, ch.Unseen) + } + continue + } + + // Delay sending all message events, we want to prevent synchronization issues + // around UIDNEXT and MODSEQ. ../rfc/5465:808 + if ms.Kind == mbspecSelectedDelayed && !sendDelayed { + unhandled = changes[index+1:] // We'll call RemovalSeen in the future. + c.notify.Delayed = append(c.notify.Delayed, change) + continue + } + + unhandled = changes[index+1:] + c.comm.RemovalSeen(ch) + + var vanishedUIDs numSet + for _, uid := range ch.UIDs { + + seq := c.xsequence(uid) + c.sequenceRemove(seq, uid) + if qresync { + vanishedUIDs.append(uint32(uid)) + } else { + c.xbwritelinef("* %d EXPUNGE", seq) + } + } + if qresync { + // VANISHED without EARLIER. ../rfc/7162:2004 + for _, s := range vanishedUIDs.Strings(4*1024 - 32) { + c.xbwritelinef("* VANISHED %s", s) + } + } + + case store.ChangeFlags: + // ../rfc/5465:461 + mb := xmailbox(ch.MailboxID) + ms, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventFlagChange) + if !ok { + continue + } else if mb.ID != c.mailboxID { + // ../rfc/5465:474 + // For condstore/qresync, we include HIGHESTMODSEQ. ../rfc/5465:476 + // We include UNSEEN, so clients can update the number of unread messages. ../rfc/5465:479 + if condstore || qresync { + c.xbwritelinef("* STATUS %s (HIGHESTMODSEQ %d UIDVALIDITY %d UNSEEN %d)", mailboxt(mb.Name).pack(c), ch.ModSeq, ch.UIDValidity, ch.Unseen) + } else { + c.xbwritelinef("* STATUS %s (UIDVALIDITY %d UNSEEN %d)", mailboxt(mb.Name).pack(c), ch.UIDValidity, ch.Unseen) + } + continue + } + + // Delay sending all message events, we want to prevent synchronization issues + // around UIDNEXT and MODSEQ. ../rfc/5465:808 + if ms.Kind == mbspecSelectedDelayed && !sendDelayed { + c.notify.Delayed = append(c.notify.Delayed, change) + continue + } + + // The uid can be unknown if we just expunged it while another session marked it as deleted just before. + seq := c.sequence(ch.UID) + if seq <= 0 { + continue + } + + var modseqStr string + if condstore { + modseqStr = fmt.Sprintf(" MODSEQ (%d)", ch.ModSeq.Client()) + } + // UID and FLAGS are required. ../rfc/5465:463 + c.xbwritelinef("* %d FETCH (UID %d FLAGS %s%s)", seq, ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c), modseqStr) + + case store.ChangeThread: + continue + + // ../rfc/5465:603 + case store.ChangeRemoveMailbox: + mb := xmailbox(ch.MailboxID) + _, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventMailboxName) + if !ok { + continue + } + + // ../rfc/5465:624 + c.xbwritelinef(`* LIST (\NonExistent) "/" %s`, mailboxt(ch.Name).pack(c)) + + case store.ChangeAddMailbox: + mb := xmailbox(ch.Mailbox.ID) + _, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventMailboxName) + if !ok { + continue + } + c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(ch.Flags, " "), mailboxt(ch.Mailbox.Name).pack(c)) + + case store.ChangeRenameMailbox: + mb := xmailbox(ch.MailboxID) + _, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventMailboxName) + if !ok { + continue + } + // ../rfc/5465:628 + oldname := fmt.Sprintf(` ("OLDNAME" (%s))`, mailboxt(ch.OldName).pack(c)) + c.xbwritelinef(`* LIST (%s) "/" %s%s`, strings.Join(ch.Flags, " "), mailboxt(ch.NewName).pack(c), oldname) + + // ../rfc/5465:653 + case store.ChangeAddSubscription: + _, _, ok := c.notify.match(c, xtx, 0, ch.MailboxName, eventSubscriptionChange) + if !ok { + continue + } + c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(append([]string{`\Subscribed`}, ch.ListFlags...), " "), mailboxt(ch.MailboxName).pack(c)) + + case store.ChangeRemoveSubscription: + _, _, ok := c.notify.match(c, xtx, 0, ch.MailboxName, eventSubscriptionChange) + if !ok { + continue + } + // ../rfc/5465:653 + c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(ch.ListFlags, " "), mailboxt(ch.MailboxName).pack(c)) + + case store.ChangeMailboxCounts: + continue + + case store.ChangeMailboxSpecialUse: + // todo: can we send special-use flags as part of an untagged LIST response? + continue + + case store.ChangeMailboxKeywords: + // ../rfc/5465:461 + mb := xmailbox(ch.MailboxID) + ms, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventFlagChange) + if !ok { + continue + } else if mb.ID != c.mailboxID { + continue + } + + // Delay sending all message events, we want to prevent synchronization issues + // around UIDNEXT and MODSEQ. ../rfc/5465:808 + // This change is about mailbox keywords, but it's specified under the FlagChange + // message event. ../rfc/5465:466 + + if ms.Kind == mbspecSelectedDelayed && !sendDelayed { + c.notify.Delayed = append(c.notify.Delayed, change) + continue + } + + var keywords string + if len(ch.Keywords) > 0 { + keywords = " " + strings.Join(ch.Keywords, " ") + } + c.xbwritelinef(`* FLAGS (\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent%s)`, keywords) + + case store.ChangeAnnotation: + // Client does not have to enable METADATA/METADATA-SERVER. Just asking for these + // events is enough. + // ../rfc/5465:679 + + if ch.MailboxID == 0 { + // ServerMetadataChange ../rfc/5465:695 + _, _, ok := c.notify.match(c, xtx, 0, "", eventServerMetadataChange) + if !ok { + continue + } + } else { + // MailboxMetadataChange ../rfc/5465:665 + mb := xmailbox(ch.MailboxID) + _, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventMailboxMetadataChange) + if !ok { + continue + } + } + // We don't implement message annotations. ../rfc/5465:461 + + // We must not include values. ../rfc/5465:683 ../rfc/5464:716 + // Syntax: ../rfc/5464:807 + c.xbwritelinef(`* METADATA %s %s`, mailboxt(ch.MailboxName).pack(c), astring(ch.Key).pack(c)) + + default: + panic(fmt.Sprintf("internal error, missing case for %#v", change)) + } + } + + // If we have too many delayed changes, we will warn about notification overflow, + // and not queue more changes until another NOTIFY command. ../rfc/5465:717 + if len(c.notify.Delayed) > selectedDelayedChangesMax { + l := c.notify.Delayed + c.notify.Delayed = nil + c.flushChanges(l) + + c.notify = ¬ify{} + c.xbwritelinef("* OK [NOTIFICATIONOVERFLOW] out of sync after too many pending changes for selected mailbox") + } +} + // Capability returns the capabilities this server implements and currently has // available given the connection state. // @@ -2039,9 +2537,9 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) { } // Plain text passwords, mark as traceauth. - defer c.xtrace(mlog.LevelTraceauth)() + defer c.xtraceread(mlog.LevelTraceauth)() buf := xreadInitial() - c.xtrace(mlog.LevelTrace) // Restore. + c.xtraceread(mlog.LevelTrace) // Restore. plain := bytes.Split(buf, []byte{0}) if len(plain) != 3 { xsyntaxErrorf("bad plain auth data, expected 3 nul-separated tokens, got %d tokens", len(plain)) @@ -2618,7 +3116,9 @@ func (c *conn) cmdSelectExamine(isselect bool, tag, cmd string, p *parser) { } }) }) - c.applyChanges(c.comm.Get(), true) + + overflow, changes := c.comm.Get() + c.xapplyChanges(overflow, changes, true, false) var flags string if len(mb.Keywords) > 0 { @@ -3062,6 +3562,8 @@ func (c *conn) cmdUnsubscribe(tag, cmd string, p *parser) { name = xcheckmailboxname(name, true) c.account.WithWLock(func() { + var changes []store.Change + c.xdbwrite(func(tx *bstore.Tx) { // It's OK if not currently subscribed, ../rfc/9051:2215 err := tx.Delete(&store.Subscription{Name: name}) @@ -3074,8 +3576,19 @@ func (c *conn) cmdUnsubscribe(tag, cmd string, p *parser) { return } xcheckf(err, "removing subscription") + + var flags []string + exists, err := c.account.MailboxExists(tx, name) + xcheckf(err, "looking up mailbox existence") + if !exists { + flags = []string{`\NonExistent`} + } + + changes = []store.Change{store.ChangeRemoveSubscription{MailboxName: name, ListFlags: flags}} }) + c.broadcast(changes) + // todo: can we send untagged message about a mailbox no longer being subscribed? }) @@ -3409,10 +3922,10 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) { } } - defer c.xtrace(mlog.LevelTracedata)() + defer c.xtracewrite(mlog.LevelTracedata)() a.mw = message.NewWriter(f) msize, err := io.Copy(a.mw, io.LimitReader(c.br, size)) - c.xtrace(mlog.LevelTrace) // Restore. + c.xtracewrite(mlog.LevelTrace) // Restore. if err != nil { // Cannot use xcheckf due to %w handling of errIO. c.xbrokenf("reading literal message: %s (%w)", err, errIO) @@ -3448,7 +3961,12 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) { } var mb store.Mailbox + var overflow bool var pendingChanges []store.Change + defer func() { + // In case of panic. + c.flushChanges(pendingChanges) + }() // Append all messages in a single atomic transaction. ../rfc/3502:143 @@ -3490,7 +4008,7 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) { err = c.account.MessageAdd(c.log, tx, &mb, &a.m, a.file, store.AddOpts{SkipDirSync: true}) xcheckf(err, "delivering message") - changes = append(changes, a.m.ChangeAddUID()) + changes = append(changes, a.m.ChangeAddUID(mb)) msgDirs[filepath.Dir(c.account.MessagePath(a.m.ID))] = struct{}{} } @@ -3512,14 +4030,16 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) { commit = true // Fetch pending changes, possibly with new UIDs, so we can apply them before adding our own new UID. - pendingChanges = c.comm.Get() + overflow, pendingChanges = c.comm.Get() // Broadcast the change to other connections. c.broadcast(changes) }) if c.mailboxID == mb.ID { - c.applyChanges(pendingChanges, false) + l := pendingChanges + pendingChanges = nil + c.xapplyChanges(overflow, l, false, true) for _, a := range appends { c.uidAppend(a.m.UID) } @@ -3552,17 +4072,35 @@ func (c *conn) cmdIdle(tag, cmd string, p *parser) { c.xwritelinef("+ waiting") + // With NOTIFY enabled, flush all pending changes. + if c.notify != nil && len(c.notify.Delayed) > 0 { + c.xapplyChanges(false, nil, false, true) + c.xflush() + } + var line string -wait: +Wait: for { select { case le := <-c.lineChan(): c.line = nil - xcheckf(le.err, "get line") + if err := le.err; err != nil { + if errors.Is(le.err, os.ErrDeadlineExceeded) { + err := c.conn.SetDeadline(time.Now().Add(10 * time.Second)) + c.log.Check(err, "setting deadline") + c.xwritelinef("* BYE inactive") + } + c.connBroken = true + if !errors.Is(err, errIO) && !errors.Is(err, errProtocol) { + c.xbrokenf("%s (%w)", err, errIO) + } + panic(err) + } line = le.line - break wait + break Wait case <-c.comm.Pending: - c.applyChanges(c.comm.Get(), false) + overflow, changes := c.comm.Get() + c.xapplyChanges(overflow, changes, false, true) c.xflush() case <-mox.Shutdown.Done(): // ../rfc/9051:5375 @@ -4108,7 +4646,16 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) { if len(newUIDs) > 0 { changes := make([]store.Change, 0, len(newUIDs)+2) for i, uid := range newUIDs { - changes = append(changes, store.ChangeAddUID{MailboxID: mbDst.ID, UID: uid, ModSeq: modseq, Flags: flags[i], Keywords: keywords[i]}) + add := store.ChangeAddUID{ + MailboxID: mbDst.ID, + UID: uid, + ModSeq: modseq, + Flags: flags[i], + Keywords: keywords[i], + MessageCountIMAP: mbDst.MessageCountIMAP(), + Unseen: uint32(mbDst.MailboxCounts.Unseen), + } + changes = append(changes, add) } changes = append(changes, mbDst.ChangeCounts()) if nkeywords != len(mbDst.Keywords) { @@ -4333,7 +4880,7 @@ func (c *conn) xmoveMessages(tx *bstore.Tx, q *bstore.Query[store.Message], expe changeRemoveUIDs.UIDs = append(changeRemoveUIDs.UIDs, om.UID) changeRemoveUIDs.MsgIDs = append(changeRemoveUIDs.MsgIDs, om.ID) - changes = append(changes, nm.ChangeAddUID()) + changes = append(changes, nm.ChangeAddUID(*mbDst)) } xcheckf(err, "move messages") @@ -4342,6 +4889,9 @@ func (c *conn) xmoveMessages(tx *bstore.Tx, q *bstore.Query[store.Message], expe xcheckf(err, "sync directory") } + changeRemoveUIDs.UIDNext = mbDst.UIDNext + changeRemoveUIDs.MessageCountIMAP = mbDst.MessageCountIMAP() + changeRemoveUIDs.Unseen = uint32(mbDst.MailboxCounts.Unseen) changes = append(changes, changeRemoveUIDs, mbSrc.ChangeCounts()) err = tx.Update(mbSrc) @@ -4524,7 +5074,7 @@ func (c *conn) cmdxStore(isUID bool, tag, cmd string, p *parser) { modified[m.ID] = true updated = append(updated, m) - changes = append(changes, m.ChangeFlags(origFlags)) + changes = append(changes, m.ChangeFlags(origFlags, mb)) return tx.Update(&m) }) diff --git a/imapserver/server_test.go b/imapserver/server_test.go index 951d656..55217b3 100644 --- a/imapserver/server_test.go +++ b/imapserver/server_test.go @@ -250,7 +250,7 @@ func tuntagged(t *testing.T, got imapclient.Untagged, dst any) { gotv := reflect.ValueOf(got) dstv := reflect.ValueOf(dst) if gotv.Type() != dstv.Type().Elem() { - t.Fatalf("got %v, expected %v", gotv.Type(), dstv.Type().Elem()) + t.Fatalf("got %#v, expected %#v", gotv.Type(), dstv.Type().Elem()) } dstv.Elem().Set(gotv) } @@ -262,6 +262,18 @@ func (tc *testconn) xnountagged() { } } +func (tc *testconn) readuntagged(exps ...imapclient.Untagged) { + tc.t.Helper() + for i, exp := range exps { + tc.conn.SetReadDeadline(time.Now().Add(3 * time.Second)) + v, err := tc.client.ReadUntagged() + tcheck(tc.t, err, "reading untagged") + if !reflect.DeepEqual(v, exp) { + tc.t.Fatalf("got %#v, expected %#v, response %d/%d", v, exp, i+1, len(exps)) + } + } +} + func (tc *testconn) transactf(status, format string, args ...any) { tc.t.Helper() tc.cmdf("", format, args...) diff --git a/import.go b/import.go index d62ec41..4e91b72 100644 --- a/import.go +++ b/import.go @@ -380,7 +380,7 @@ func ximportctl(ctx context.Context, xctl *ctl, mbox bool) { err = a.MessageAdd(xctl.log, tx, &mb, m, msgf, opts) xctl.xcheck(err, "delivering message") newIDs = append(newIDs, m.ID) - changes = append(changes, m.ChangeAddUID()) + changes = append(changes, m.ChangeAddUID(mb)) msgDirs[filepath.Dir(a.MessagePath(m.ID))] = struct{}{} diff --git a/rfc/index.txt b/rfc/index.txt index c6d88e4..3cb0e58 100644 --- a/rfc/index.txt +++ b/rfc/index.txt @@ -212,7 +212,7 @@ https://www.iana.org/assignments/message-headers/message-headers.xhtml 5464-eid2785 - - errata: fix GETMETADATA example 5464-eid2786 - - errata: fix GETMETADATA example 5464-eid3868 - - errata: fix GETMETADATA example -5465 Roadmap - The IMAP NOTIFY Extension +5465 Yes - The IMAP NOTIFY Extension 5466 Roadmap - IMAP4 Extension for Named Searches (Filters) 5524 No - Extended URLFETCH for Binary and Converted Parts 5530 Yes - IMAP Response Codes diff --git a/smtpserver/server.go b/smtpserver/server.go index 7531ebc..721ba6f 100644 --- a/smtpserver/server.go +++ b/smtpserver/server.go @@ -3454,7 +3454,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW if err := tx.Update(mbrej); err != nil { return fmt.Errorf("updating rejects mailbox: %v", err) } - changes = append(changes, a.d.m.ChangeAddUID(), mbrej.ChangeCounts()) + changes = append(changes, a.d.m.ChangeAddUID(*mbrej), mbrej.ChangeCounts()) stored = true return nil }) diff --git a/smtpserver/server_test.go b/smtpserver/server_test.go index 89393d0..9aee6b6 100644 --- a/smtpserver/server_test.go +++ b/smtpserver/server_test.go @@ -624,7 +624,8 @@ func TestDelivery(t *testing.T) { changes := make(chan []store.Change) go func() { - changes <- ts.comm.Get() + _, l := ts.comm.Get() + changes <- l }() timer := time.NewTimer(time.Second) diff --git a/store/account.go b/store/account.go index 307e049..f80f598 100644 --- a/store/account.go +++ b/store/account.go @@ -259,6 +259,13 @@ type MailboxCounts struct { Size int64 // Number of bytes for all messages. } +// MessageCountIMAP returns the total message count for use in IMAP. In IMAP, +// message marked \Deleted are included, in JMAP they those messages are not +// visible at all. +func (mc MailboxCounts) MessageCountIMAP() uint32 { + return uint32(mc.Total + mc.Deleted) +} + func (mc MailboxCounts) String() string { return fmt.Sprintf("%d total, %d deleted, %d unread, %d unseen, size %d bytes", mc.Total, mc.Deleted, mc.Unread, mc.Unseen, mc.Size) } @@ -601,13 +608,13 @@ func (m Message) MailboxCounts() (mc MailboxCounts) { return } -func (m Message) ChangeAddUID() ChangeAddUID { - return ChangeAddUID{m.MailboxID, m.UID, m.ModSeq, m.Flags, m.Keywords} +func (m Message) ChangeAddUID(mb Mailbox) ChangeAddUID { + return ChangeAddUID{m.MailboxID, m.UID, m.ModSeq, m.Flags, m.Keywords, mb.MessageCountIMAP(), uint32(mb.MailboxCounts.Unseen)} } -func (m Message) ChangeFlags(orig Flags) ChangeFlags { +func (m Message) ChangeFlags(orig Flags, mb Mailbox) ChangeFlags { mask := m.Flags.Changed(orig) - return ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, ModSeq: m.ModSeq, Mask: mask, Flags: m.Flags, Keywords: m.Keywords} + return ChangeFlags{m.MailboxID, m.UID, m.ModSeq, mask, m.Flags, m.Keywords, mb.UIDValidity, uint32(mb.MailboxCounts.Unseen)} } func (m Message) ChangeThread() ChangeThread { @@ -2884,7 +2891,7 @@ func (a *Account) DeliverMailbox(log mlog.Log, mailbox string, m *Message, msgFi } changes = append(changes, chl...) - changes = append(changes, m.ChangeAddUID(), mb.ChangeCounts()) + changes = append(changes, m.ChangeAddUID(mb), mb.ChangeCounts()) if nmbkeywords != len(mb.Keywords) { changes = append(changes, mb.ChangeKeywords()) } @@ -2992,7 +2999,7 @@ func (a *Account) MessageRemove(log mlog.Log, tx *bstore.Tx, modseq ModSeq, mb * } } - return ChangeRemoveUIDs{mb.ID, uids, modseq, ids}, mb.ChangeCounts(), nil + return ChangeRemoveUIDs{mb.ID, uids, modseq, ids, mb.UIDNext, mb.MessageCountIMAP(), uint32(mb.MailboxCounts.Unseen)}, mb.ChangeCounts(), nil } // TidyRejectsMailbox removes old reject emails, and returns whether there is space for a new delivery. diff --git a/store/state.go b/store/state.go index ede4d7b..348c7c3 100644 --- a/store/state.go +++ b/store/state.go @@ -14,6 +14,11 @@ import ( "github.com/mjl-/mox/mox-" ) +// CommPendingChangesMax is the maximum number of changes kept for a Comm before +// registering a notification overflow and flushing changes. Variable because set +// to low value during tests. +var CommPendingChangesMax = 10000 + var ( register = make(chan *Comm) unregister = make(chan *Comm) @@ -48,6 +53,10 @@ type ChangeAddUID struct { ModSeq ModSeq Flags Flags // System flags. Keywords []string // Other flags. + + // For IMAP NOTIFY. + MessageCountIMAP uint32 + Unseen uint32 } func (c ChangeAddUID) ChangeModSeq() ModSeq { return c.ModSeq } @@ -58,6 +67,11 @@ type ChangeRemoveUIDs struct { UIDs []UID // Must be in increasing UID order, for IMAP. ModSeq ModSeq MsgIDs []int64 // Message.ID, for erasing, order does not necessarily correspond with UIDs! + + // For IMAP NOTIFY. + UIDNext UID + MessageCountIMAP uint32 + Unseen uint32 } func (c ChangeRemoveUIDs) ChangeModSeq() ModSeq { return c.ModSeq } @@ -70,6 +84,10 @@ type ChangeFlags struct { Mask Flags // Which flags are actually modified. Flags Flags // New flag values. All are set, not just mask. Keywords []string // Non-system/well-known flags/keywords/labels. + + // For IMAP NOTIFY. + UIDValidity uint32 + Unseen uint32 } func (c ChangeFlags) ChangeModSeq() ModSeq { return c.ModSeq } @@ -113,12 +131,20 @@ func (c ChangeRenameMailbox) ChangeModSeq() ModSeq { return c.ModSeq } // ChangeAddSubscription is sent for an added subscription to a mailbox. type ChangeAddSubscription struct { - Name string - Flags []string // For additional IMAP flags like \NonExistent. + MailboxName string + ListFlags []string // For additional IMAP flags like \NonExistent. } func (c ChangeAddSubscription) ChangeModSeq() ModSeq { return -1 } +// ChangeRemoveSubscription is sent for a removed subscription of a mailbox. +type ChangeRemoveSubscription struct { + MailboxName string + ListFlags []string // For additional IMAP flags like \NonExistent. +} + +func (c ChangeRemoveSubscription) ChangeModSeq() ModSeq { return -1 } + // ChangeMailboxCounts is sent when the number of total/deleted/unseen/unread messages changes. type ChangeMailboxCounts struct { MailboxID int64 @@ -327,11 +353,9 @@ func switchboard(stopc, donec chan struct{}, cleanc chan map[*Account][]int64) { // possibly queue messages for cleaning. No need to take a lock, the caller does // not use the comm anymore. for _, ch := range c.changes { - rem, ok := ch.(ChangeRemoveUIDs) - if !ok { - continue + if rem, ok := ch.(ChangeRemoveUIDs); ok { + decreaseEraseRefs(c.acc, rem.MsgIDs...) } - decreaseEraseRefs(c.acc, rem.MsgIDs...) } delete(regs[c.acc], c) @@ -381,14 +405,31 @@ func switchboard(stopc, donec chan struct{}, cleanc chan map[*Account][]int64) { for c := range regs[acc] { // Do not send the broadcaster back their own changes. chReq.comm is nil if not // originating from a comm, so won't match in that case. + // Relevant for IMAP IDLE, and NOTIFY ../rfc/5465:428 if c == chReq.comm { continue } + var overflow bool c.Lock() - c.changes = append(c.changes, chReq.changes...) + if len(c.changes)+len(chReq.changes) > CommPendingChangesMax { + c.overflow = true + overflow = true + } else { + c.changes = append(c.changes, chReq.changes...) + } c.Unlock() + // In case of overflow, we didn't add the pending changes to the comm, so we must + // decrease references again. + if overflow { + for _, ch := range chReq.changes { + if rem, ok := ch.(ChangeRemoveUIDs); ok { + decreaseEraseRefs(acc, rem.MsgIDs...) + } + } + } + select { case c.Pending <- struct{}{}: default: @@ -463,6 +504,9 @@ type Comm struct { sync.Mutex changes []Change + // Set if too many changes were queued, cleared when changes are retrieved. While + // in overflow, no new changes are added. + overflow bool } // Register starts a Comm for the account. Unregister must be called. @@ -491,13 +535,16 @@ func (c *Comm) Broadcast(ch []Change) { } // Get retrieves all pending changes. If no changes are pending a nil or empty list -// is returned. -func (c *Comm) Get() []Change { +// is returned. If too many changes were pending, overflow is true, and this Comm +// stopped getting new changes. The caller should usually return an error to its +// connection. Even with overflow, changes may still be non-empty. On +// ChangeRemoveUIDs, the RemovalSeen must still be called by the caller. +func (c *Comm) Get() (overflow bool, changes []Change) { c.Lock() defer c.Unlock() - l := c.changes - c.changes = nil - return l + overflow, changes = c.overflow, c.changes + c.overflow, c.changes = false, nil + return } // RemovalSeen must be called by consumers when they have applied the removal to diff --git a/webaccount/import.go b/webaccount/import.go index efc88fb..3d2ae06 100644 --- a/webaccount/import.go +++ b/webaccount/import.go @@ -558,7 +558,7 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store. return } newIDs = append(newIDs, m.ID) - changes = append(changes, m.ChangeAddUID()) + changes = append(changes, m.ChangeAddUID(*mb)) messages[mb.Name]++ if messages[mb.Name]%100 == 0 || prevMailbox != mb.Name { prevMailbox = mb.Name @@ -767,7 +767,7 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store. } err = tx.Update(&m) ximportcheckf(err, "updating message after flag update") - changes = append(changes, m.ChangeFlags(oflags)) + changes = append(changes, m.ChangeFlags(oflags, *mb)) } delete(mailboxMissingKeywordMessages, mailbox) } diff --git a/webapisrv/server.go b/webapisrv/server.go index c0bef0f..95860e7 100644 --- a/webapisrv/server.go +++ b/webapisrv/server.go @@ -1118,7 +1118,7 @@ func (s server) Send(ctx context.Context, req webapi.SendRequest) (resp webapi.S err = tx.Update(&sentmb) xcheckf(err, "updating mailbox") - changes = append(changes, sentm.ChangeAddUID(), sentmb.ChangeCounts()) + changes = append(changes, sentm.ChangeAddUID(sentmb), sentmb.ChangeCounts()) }) sentID = 0 // Commit. diff --git a/webmail/api.go b/webmail/api.go index 5f089e3..71976b8 100644 --- a/webmail/api.go +++ b/webmail/api.go @@ -472,7 +472,7 @@ func (w Webmail) MessageCompose(ctx context.Context, m ComposeMessage, mailboxID err = tx.Update(&mb) xcheckf(ctx, err, "updating sent mailbox for counts") - changes = append(changes, nm.ChangeAddUID(), mb.ChangeCounts()) + changes = append(changes, nm.ChangeAddUID(mb), mb.ChangeCounts()) }) newIDs = nil @@ -1063,7 +1063,6 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) { rm.ModSeq = modseq err := tx.Update(&rm) xcheckf(ctx, err, "updating flags of replied/forwarded message") - changes = append(changes, rm.ChangeFlags(oflags)) // Update modseq of mailbox of replied/forwarded message. rmb, err := store.MailboxID(tx, rm.MailboxID) @@ -1072,6 +1071,8 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) { err = tx.Update(&rmb) xcheckf(ctx, err, "update modseq of mailbox of replied/forwarded message") + changes = append(changes, rm.ChangeFlags(oflags, rmb)) + err = acc.RetrainMessages(ctx, log, tx, []store.Message{rm}) xcheckf(ctx, err, "retraining messages after reply/forward") } @@ -1145,7 +1146,7 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) { err = tx.Update(&sentmb) xcheckf(ctx, err, "updating sent mailbox for counts") - changes = append(changes, sentm.ChangeAddUID(), sentmb.ChangeCounts()) + changes = append(changes, sentm.ChangeAddUID(sentmb), sentmb.ChangeCounts()) }) newIDs = nil diff --git a/webmail/api.json b/webmail/api.json index 9712ee1..e20e724 100644 --- a/webmail/api.json +++ b/webmail/api.json @@ -2849,6 +2849,20 @@ "string" ] }, + { + "Name": "MessageCountIMAP", + "Docs": "For IMAP NOTIFY.", + "Typewords": [ + "uint32" + ] + }, + { + "Name": "Unseen", + "Docs": "", + "Typewords": [ + "uint32" + ] + }, { "Name": "MessageItems", "Docs": "", @@ -2968,6 +2982,27 @@ "[]", "int64" ] + }, + { + "Name": "UIDNext", + "Docs": "For IMAP NOTIFY.", + "Typewords": [ + "UID" + ] + }, + { + "Name": "MessageCountIMAP", + "Docs": "", + "Typewords": [ + "uint32" + ] + }, + { + "Name": "Unseen", + "Docs": "", + "Typewords": [ + "uint32" + ] } ] }, @@ -3017,6 +3052,20 @@ "[]", "string" ] + }, + { + "Name": "UIDValidity", + "Docs": "For IMAP NOTIFY.", + "Typewords": [ + "uint32" + ] + }, + { + "Name": "Unseen", + "Docs": "", + "Typewords": [ + "uint32" + ] } ] }, diff --git a/webmail/api.ts b/webmail/api.ts index 2bc5d71..82f5958 100644 --- a/webmail/api.ts +++ b/webmail/api.ts @@ -423,6 +423,8 @@ export interface ChangeMsgAdd { ModSeq: ModSeq Flags: Flags // System flags. Keywords?: string[] | null // Other flags. + MessageCountIMAP: number // For IMAP NOTIFY. + Unseen: number MessageItems?: MessageItem[] | null } @@ -446,6 +448,9 @@ export interface ChangeMsgRemove { UIDs?: UID[] | null // Must be in increasing UID order, for IMAP. ModSeq: ModSeq MsgIDs?: number[] | null // Message.ID, for erasing, order does not necessarily correspond with UIDs! + UIDNext: UID // For IMAP NOTIFY. + MessageCountIMAP: number + Unseen: number } // ChangeMsgFlags updates flags for one message. @@ -456,6 +461,8 @@ export interface ChangeMsgFlags { Mask: Flags // Which flags are actually modified. Flags: Flags // New flag values. All are set, not just mask. Keywords?: string[] | null // Non-system/well-known flags/keywords/labels. + UIDValidity: number // For IMAP NOTIFY. + Unseen: number } // ChangeMsgThread updates muted/collapsed fields for one message. @@ -634,10 +641,10 @@ export const types: TypenameMap = { "MessageEnvelope": {"Name":"MessageEnvelope","Docs":"","Fields":[{"Name":"Date","Docs":"","Typewords":["timestamp"]},{"Name":"Subject","Docs":"","Typewords":["string"]},{"Name":"From","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"Sender","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"ReplyTo","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"To","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"CC","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"BCC","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"InReplyTo","Docs":"","Typewords":["string"]},{"Name":"MessageID","Docs":"","Typewords":["string"]}]}, "Attachment": {"Name":"Attachment","Docs":"","Fields":[{"Name":"Path","Docs":"","Typewords":["[]","int32"]},{"Name":"Filename","Docs":"","Typewords":["string"]},{"Name":"Part","Docs":"","Typewords":["Part"]}]}, "EventViewChanges": {"Name":"EventViewChanges","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"Changes","Docs":"","Typewords":["[]","[]","any"]}]}, - "ChangeMsgAdd": {"Name":"ChangeMsgAdd","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Flags","Docs":"","Typewords":["Flags"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"MessageItems","Docs":"","Typewords":["[]","MessageItem"]}]}, + "ChangeMsgAdd": {"Name":"ChangeMsgAdd","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Flags","Docs":"","Typewords":["Flags"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"MessageCountIMAP","Docs":"","Typewords":["uint32"]},{"Name":"Unseen","Docs":"","Typewords":["uint32"]},{"Name":"MessageItems","Docs":"","Typewords":["[]","MessageItem"]}]}, "Flags": {"Name":"Flags","Docs":"","Fields":[{"Name":"Seen","Docs":"","Typewords":["bool"]},{"Name":"Answered","Docs":"","Typewords":["bool"]},{"Name":"Flagged","Docs":"","Typewords":["bool"]},{"Name":"Forwarded","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Notjunk","Docs":"","Typewords":["bool"]},{"Name":"Deleted","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Phishing","Docs":"","Typewords":["bool"]},{"Name":"MDNSent","Docs":"","Typewords":["bool"]}]}, - "ChangeMsgRemove": {"Name":"ChangeMsgRemove","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UIDs","Docs":"","Typewords":["[]","UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"MsgIDs","Docs":"","Typewords":["[]","int64"]}]}, - "ChangeMsgFlags": {"Name":"ChangeMsgFlags","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Mask","Docs":"","Typewords":["Flags"]},{"Name":"Flags","Docs":"","Typewords":["Flags"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]}]}, + "ChangeMsgRemove": {"Name":"ChangeMsgRemove","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UIDs","Docs":"","Typewords":["[]","UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"MsgIDs","Docs":"","Typewords":["[]","int64"]},{"Name":"UIDNext","Docs":"","Typewords":["UID"]},{"Name":"MessageCountIMAP","Docs":"","Typewords":["uint32"]},{"Name":"Unseen","Docs":"","Typewords":["uint32"]}]}, + "ChangeMsgFlags": {"Name":"ChangeMsgFlags","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Mask","Docs":"","Typewords":["Flags"]},{"Name":"Flags","Docs":"","Typewords":["Flags"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"UIDValidity","Docs":"","Typewords":["uint32"]},{"Name":"Unseen","Docs":"","Typewords":["uint32"]}]}, "ChangeMsgThread": {"Name":"ChangeMsgThread","Docs":"","Fields":[{"Name":"MessageIDs","Docs":"","Typewords":["[]","int64"]},{"Name":"Muted","Docs":"","Typewords":["bool"]},{"Name":"Collapsed","Docs":"","Typewords":["bool"]}]}, "ChangeMailboxRemove": {"Name":"ChangeMailboxRemove","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"Name","Docs":"","Typewords":["string"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]}]}, "ChangeMailboxAdd": {"Name":"ChangeMailboxAdd","Docs":"","Fields":[{"Name":"Mailbox","Docs":"","Typewords":["Mailbox"]}]}, diff --git a/webmail/msg.js b/webmail/msg.js index dfe29bc..241b3b0 100644 --- a/webmail/msg.js +++ b/webmail/msg.js @@ -323,10 +323,10 @@ var api; "MessageEnvelope": { "Name": "MessageEnvelope", "Docs": "", "Fields": [{ "Name": "Date", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "Sender", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "CC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "BCC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "InReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }] }, "Attachment": { "Name": "Attachment", "Docs": "", "Fields": [{ "Name": "Path", "Docs": "", "Typewords": ["[]", "int32"] }, { "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "Part", "Docs": "", "Typewords": ["Part"] }] }, "EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] }, - "ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] }, + "ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageCountIMAP", "Docs": "", "Typewords": ["uint32"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["uint32"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] }, "Flags": { "Name": "Flags", "Docs": "", "Fields": [{ "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }] }, - "ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "MsgIDs", "Docs": "", "Typewords": ["[]", "int64"] }] }, - "ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] }, + "ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "MsgIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "MessageCountIMAP", "Docs": "", "Typewords": ["uint32"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["uint32"] }] }, + "ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["uint32"] }] }, "ChangeMsgThread": { "Name": "ChangeMsgThread", "Docs": "", "Fields": [{ "Name": "MessageIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "Muted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Collapsed", "Docs": "", "Typewords": ["bool"] }] }, "ChangeMailboxRemove": { "Name": "ChangeMailboxRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] }, "ChangeMailboxAdd": { "Name": "ChangeMailboxAdd", "Docs": "", "Fields": [{ "Name": "Mailbox", "Docs": "", "Typewords": ["Mailbox"] }] }, diff --git a/webmail/text.js b/webmail/text.js index e02bda2..30f0056 100644 --- a/webmail/text.js +++ b/webmail/text.js @@ -323,10 +323,10 @@ var api; "MessageEnvelope": { "Name": "MessageEnvelope", "Docs": "", "Fields": [{ "Name": "Date", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "Sender", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "CC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "BCC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "InReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }] }, "Attachment": { "Name": "Attachment", "Docs": "", "Fields": [{ "Name": "Path", "Docs": "", "Typewords": ["[]", "int32"] }, { "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "Part", "Docs": "", "Typewords": ["Part"] }] }, "EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] }, - "ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] }, + "ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageCountIMAP", "Docs": "", "Typewords": ["uint32"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["uint32"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] }, "Flags": { "Name": "Flags", "Docs": "", "Fields": [{ "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }] }, - "ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "MsgIDs", "Docs": "", "Typewords": ["[]", "int64"] }] }, - "ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] }, + "ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "MsgIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "MessageCountIMAP", "Docs": "", "Typewords": ["uint32"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["uint32"] }] }, + "ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["uint32"] }] }, "ChangeMsgThread": { "Name": "ChangeMsgThread", "Docs": "", "Fields": [{ "Name": "MessageIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "Muted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Collapsed", "Docs": "", "Typewords": ["bool"] }] }, "ChangeMailboxRemove": { "Name": "ChangeMailboxRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] }, "ChangeMailboxAdd": { "Name": "ChangeMailboxAdd", "Docs": "", "Fields": [{ "Name": "Mailbox", "Docs": "", "Typewords": ["Mailbox"] }] }, diff --git a/webmail/view.go b/webmail/view.go index d25a908..53aaf3c 100644 --- a/webmail/view.go +++ b/webmail/view.go @@ -959,7 +959,7 @@ func serveEvents(ctx context.Context, log mlog.Log, accountPath string, w http.R case store.ChangeMailboxKeywords: taggedChanges = append(taggedChanges, [2]any{"ChangeMailboxKeywords", ChangeMailboxKeywords{c}}) - case store.ChangeAddSubscription: + case store.ChangeAddSubscription, store.ChangeRemoveSubscription: // Webmail does not care about subscriptions. case store.ChangeAnnotation: @@ -1111,7 +1111,12 @@ func serveEvents(ctx context.Context, log mlog.Log, accountPath string, w http.R } case <-pending: - xprocessChanges(comm.Get()) + overflow, changes := comm.Get() + if overflow { + writer.xsendEvent(ctx, log, "fatalErr", "out of sync, too many pending changes") + return + } + xprocessChanges(changes) case <-ctx.Done(): // Work around go vet, it doesn't see defer cancelDrain. diff --git a/webmail/webmail.js b/webmail/webmail.js index 0aebb28..4bf9b40 100644 --- a/webmail/webmail.js +++ b/webmail/webmail.js @@ -323,10 +323,10 @@ var api; "MessageEnvelope": { "Name": "MessageEnvelope", "Docs": "", "Fields": [{ "Name": "Date", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "Sender", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "CC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "BCC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "InReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }] }, "Attachment": { "Name": "Attachment", "Docs": "", "Fields": [{ "Name": "Path", "Docs": "", "Typewords": ["[]", "int32"] }, { "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "Part", "Docs": "", "Typewords": ["Part"] }] }, "EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] }, - "ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] }, + "ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageCountIMAP", "Docs": "", "Typewords": ["uint32"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["uint32"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] }, "Flags": { "Name": "Flags", "Docs": "", "Fields": [{ "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }] }, - "ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "MsgIDs", "Docs": "", "Typewords": ["[]", "int64"] }] }, - "ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] }, + "ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "MsgIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "MessageCountIMAP", "Docs": "", "Typewords": ["uint32"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["uint32"] }] }, + "ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["uint32"] }] }, "ChangeMsgThread": { "Name": "ChangeMsgThread", "Docs": "", "Fields": [{ "Name": "MessageIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "Muted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Collapsed", "Docs": "", "Typewords": ["bool"] }] }, "ChangeMailboxRemove": { "Name": "ChangeMailboxRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] }, "ChangeMailboxAdd": { "Name": "ChangeMailboxAdd", "Docs": "", "Fields": [{ "Name": "Mailbox", "Docs": "", "Typewords": ["Mailbox"] }] }, diff --git a/webops/xops.go b/webops/xops.go index e0af97c..b089eab 100644 --- a/webops/xops.go +++ b/webops/xops.go @@ -90,6 +90,9 @@ func (x XOps) MessageDeleteTx(ctx context.Context, log mlog.Log, tx *bstore.Tx, err := tx.Update(&mb) x.Checkf(ctx, err, "updating mailbox counts") slices.Sort(changeRemoveUIDs.UIDs) + changeRemoveUIDs.UIDNext = mb.UIDNext + changeRemoveUIDs.MessageCountIMAP = mb.MessageCountIMAP() + changeRemoveUIDs.Unseen = uint32(mb.MailboxCounts.Unseen) changes = append(changes, mb.ChangeCounts(), changeRemoveUIDs) } @@ -188,7 +191,7 @@ func (x XOps) MessageFlagsAdd(ctx context.Context, log mlog.Log, acc *store.Acco err = tx.Update(&m) x.Checkf(ctx, err, "updating message") - changes = append(changes, m.ChangeFlags(oflags)) + changes = append(changes, m.ChangeFlags(oflags, mb)) retrain = append(retrain, m) } @@ -261,7 +264,7 @@ func (x XOps) MessageFlagsClear(ctx context.Context, log mlog.Log, acc *store.Ac err = tx.Update(&m) x.Checkf(ctx, err, "updating message") - changes = append(changes, m.ChangeFlags(oflags)) + changes = append(changes, m.ChangeFlags(oflags, mb)) retrain = append(retrain, m) } @@ -322,7 +325,7 @@ func (x XOps) MailboxesMarkRead(ctx context.Context, log mlog.Log, acc *store.Ac err := tx.Update(&m) x.Checkf(ctx, err, "updating message") - changes = append(changes, m.ChangeFlags(oflags)) + changes = append(changes, m.ChangeFlags(oflags, mb)) return nil }) x.Checkf(ctx, err, "listing messages to mark as read") @@ -443,6 +446,9 @@ func (x XOps) MessageMoveTx(ctx context.Context, log mlog.Log, acc *store.Accoun var mbSrc store.Mailbox var changeRemoveUIDs store.ChangeRemoveUIDs xflushMailbox := func() { + changeRemoveUIDs.UIDNext = mbSrc.UIDNext + changeRemoveUIDs.MessageCountIMAP = mbSrc.MessageCountIMAP() + changeRemoveUIDs.Unseen = uint32(mbSrc.MailboxCounts.Unseen) changes = append(changes, changeRemoveUIDs, mbSrc.ChangeCounts()) err = tx.Update(&mbSrc) @@ -527,7 +533,7 @@ func (x XOps) MessageMoveTx(ctx context.Context, log mlog.Log, acc *store.Accoun changeRemoveUIDs.UIDs = append(changeRemoveUIDs.UIDs, om.UID) changeRemoveUIDs.MsgIDs = append(changeRemoveUIDs.MsgIDs, om.ID) - changes = append(changes, nm.ChangeAddUID()) + changes = append(changes, nm.ChangeAddUID(mbDst)) } for dir := range syncDirs {