mirror of
https://github.com/mjl-/mox.git
synced 2025-07-10 10:34:40 +03:00
add condstore & qresync imap extensions
for conditional storing and quick resynchronisation (not sure if mail clients are actually using it that). each message now has a "modseq". it is increased for each change. with condstore, imap clients can request changes since a certain modseq. that already allows quickly finding changes since a previous connection. condstore also allows storing (e.g. setting new message flags) only when the modseq of a message hasn't changed. qresync should make it fast for clients to get a full list of changed messages for a mailbox, including removals. we now also keep basic metadata of messages that have been removed (expunged). just enough (uid, modseq) to tell client that the messages have been removed. this does mean we have to be careful when querying messages from the database. we must now often filter the expunged messages out. we also keep "createseq", the modseq when a message was created. this will be useful for the jmap implementation.
This commit is contained in:
@ -11,24 +11,31 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/exp/maps"
|
||||
|
||||
"github.com/mjl-/bstore"
|
||||
|
||||
"github.com/mjl-/mox/message"
|
||||
"github.com/mjl-/mox/mlog"
|
||||
"github.com/mjl-/mox/moxio"
|
||||
"github.com/mjl-/mox/moxvar"
|
||||
"github.com/mjl-/mox/store"
|
||||
)
|
||||
|
||||
// functions to handle fetch attribute requests are defined on fetchCmd.
|
||||
type fetchCmd struct {
|
||||
conn *conn
|
||||
mailboxID int64
|
||||
uid store.UID
|
||||
tx *bstore.Tx // Writable tx, for storing message when first parsed as mime parts.
|
||||
changes []store.Change // For updated Seen flag.
|
||||
markSeen bool
|
||||
needFlags bool
|
||||
expungeIssued bool // Set if a message cannot be read. Can happen for expunged messages.
|
||||
conn *conn
|
||||
mailboxID int64
|
||||
uid store.UID
|
||||
tx *bstore.Tx // Writable tx, for storing message when first parsed as mime parts.
|
||||
changes []store.Change // For updated Seen flag.
|
||||
markSeen bool
|
||||
needFlags bool
|
||||
needModseq bool // Whether untagged responses needs modseq.
|
||||
expungeIssued bool // Set if a message cannot be read. Can happen for expunged messages.
|
||||
modseq store.ModSeq // Initialized on first change, for marking messages as seen.
|
||||
isUID bool // If this is a UID FETCH command.
|
||||
hasChangedSince bool // Whether CHANGEDSINCE was set. Enables MODSEQ in response.
|
||||
|
||||
// Loaded when first needed, closed when message was processed.
|
||||
m *store.Message // Message currently being processed.
|
||||
@ -60,15 +67,61 @@ func (cmd *fetchCmd) xcheckf(err error, format string, args ...any) {
|
||||
//
|
||||
// State: Selected
|
||||
func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
|
||||
// Command: ../rfc/9051:4330 ../rfc/3501:2992
|
||||
// Examples: ../rfc/9051:4463 ../rfc/9051:4520
|
||||
// Response syntax: ../rfc/9051:6742 ../rfc/3501:4864
|
||||
// Command: ../rfc/9051:4330 ../rfc/3501:2992 ../rfc/7162:864
|
||||
// Examples: ../rfc/9051:4463 ../rfc/9051:4520 ../rfc/7162:880
|
||||
// Response syntax: ../rfc/9051:6742 ../rfc/3501:4864 ../rfc/7162:2490
|
||||
|
||||
// Request syntax: ../rfc/9051:6553 ../rfc/3501:4748
|
||||
// Request syntax: ../rfc/9051:6553 ../rfc/3501:4748 ../rfc/4466:535 ../rfc/7162:2475
|
||||
p.xspace()
|
||||
nums := p.xnumSet()
|
||||
p.xspace()
|
||||
atts := p.xfetchAtts()
|
||||
atts := p.xfetchAtts(isUID)
|
||||
var changedSince int64
|
||||
var haveChangedSince bool
|
||||
var vanished bool
|
||||
if p.space() {
|
||||
// ../rfc/4466:542
|
||||
// ../rfc/7162:2479
|
||||
p.xtake("(")
|
||||
seen := map[string]bool{}
|
||||
for {
|
||||
var w string
|
||||
if isUID && p.conn.enabled[capQresync] {
|
||||
// Vanished only valid for uid fetch, and only for qresync. ../rfc/7162:1693
|
||||
w = p.xtakelist("CHANGEDSINCE", "VANISHED")
|
||||
} else {
|
||||
w = p.xtakelist("CHANGEDSINCE")
|
||||
}
|
||||
if seen[w] {
|
||||
xsyntaxErrorf("duplicate fetch modifier %s", w)
|
||||
}
|
||||
seen[w] = true
|
||||
switch w {
|
||||
case "CHANGEDSINCE":
|
||||
p.xspace()
|
||||
changedSince = p.xnumber64()
|
||||
// workaround: ios mail (16.5.1) was seen sending changedSince 0 on an existing account that got condstore enabled.
|
||||
if changedSince == 0 && moxvar.Pedantic {
|
||||
// ../rfc/7162:2551
|
||||
xsyntaxErrorf("changedsince modseq must be > 0")
|
||||
}
|
||||
// CHANGEDSINCE is a CONDSTORE-enabling parameter. ../rfc/7162:380
|
||||
p.conn.xensureCondstore(nil)
|
||||
haveChangedSince = true
|
||||
case "VANISHED":
|
||||
vanished = true
|
||||
}
|
||||
if p.take(")") {
|
||||
break
|
||||
}
|
||||
p.xspace()
|
||||
}
|
||||
|
||||
// ../rfc/7162:1701
|
||||
if vanished && !haveChangedSince {
|
||||
xsyntaxErrorf("VANISHED can only be used with CHANGEDSINCE")
|
||||
}
|
||||
}
|
||||
p.xempty()
|
||||
|
||||
// We don't use c.account.WithRLock because we write to the client while reading messages.
|
||||
@ -81,21 +134,105 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
|
||||
runlock()
|
||||
}()
|
||||
|
||||
cmd := &fetchCmd{conn: c, mailboxID: c.mailboxID}
|
||||
var vanishedUIDs []store.UID
|
||||
cmd := &fetchCmd{conn: c, mailboxID: c.mailboxID, isUID: isUID, hasChangedSince: haveChangedSince}
|
||||
c.xdbwrite(func(tx *bstore.Tx) {
|
||||
cmd.tx = tx
|
||||
|
||||
// Ensure the mailbox still exists.
|
||||
c.xmailboxID(tx, c.mailboxID)
|
||||
|
||||
uids := c.xnumSetUIDs(isUID, nums)
|
||||
var uids []store.UID
|
||||
|
||||
// With changedSince, the client is likely asking for a small set of changes. Use a
|
||||
// database query to trim down the uids we need to look at.
|
||||
// ../rfc/7162:871
|
||||
if changedSince > 0 {
|
||||
q := bstore.QueryTx[store.Message](tx)
|
||||
q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
|
||||
q.FilterGreater("ModSeq", store.ModSeqFromClient(changedSince))
|
||||
if !vanished {
|
||||
q.FilterEqual("Expunged", false)
|
||||
}
|
||||
err := q.ForEach(func(m store.Message) error {
|
||||
if m.Expunged {
|
||||
vanishedUIDs = append(vanishedUIDs, m.UID)
|
||||
} else if isUID {
|
||||
if nums.containsUID(m.UID, c.uids, c.searchResult) {
|
||||
uids = append(uids, m.UID)
|
||||
}
|
||||
} else {
|
||||
seq := c.sequence(m.UID)
|
||||
if seq > 0 && nums.containsSeq(seq, c.uids, c.searchResult) {
|
||||
uids = append(uids, m.UID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
xcheckf(err, "looking up messages with changedsince")
|
||||
} else {
|
||||
uids = c.xnumSetUIDs(isUID, nums)
|
||||
}
|
||||
|
||||
// Send vanished for all missing requested UIDs. ../rfc/7162:1718
|
||||
if vanished {
|
||||
delModSeq, err := c.account.HighestDeletedModSeq(tx)
|
||||
xcheckf(err, "looking up highest deleted modseq")
|
||||
if changedSince < delModSeq.Client() {
|
||||
// First sort the uids we already found, for fast lookup.
|
||||
sort.Slice(vanishedUIDs, func(i, j int) bool {
|
||||
return vanishedUIDs[i] < vanishedUIDs[j]
|
||||
})
|
||||
|
||||
// We'll be gathering any more vanished uids in more.
|
||||
more := map[store.UID]struct{}{}
|
||||
checkVanished := func(uid store.UID) {
|
||||
if uidSearch(c.uids, uid) <= 0 && uidSearch(vanishedUIDs, uid) <= 0 {
|
||||
more[uid] = struct{}{}
|
||||
}
|
||||
}
|
||||
// Now look through the requested uids. We may have a searchResult, handle it
|
||||
// separately from a numset with potential stars, over which we can more easily
|
||||
// iterate.
|
||||
if nums.searchResult {
|
||||
for _, uid := range c.searchResult {
|
||||
checkVanished(uid)
|
||||
}
|
||||
} else {
|
||||
iter := nums.interpretStar(c.uids).newIter()
|
||||
for {
|
||||
num, ok := iter.Next()
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
checkVanished(store.UID(num))
|
||||
}
|
||||
}
|
||||
vanishedUIDs = append(vanishedUIDs, maps.Keys(more)...)
|
||||
}
|
||||
}
|
||||
|
||||
// Release the account lock.
|
||||
runlock()
|
||||
runlock = func() {} // Prevent defer from unlocking again.
|
||||
|
||||
// First report all vanished UIDs. ../rfc/7162:1714
|
||||
if len(vanishedUIDs) > 0 {
|
||||
// Mention all vanished UIDs in compact numset form.
|
||||
// ../rfc/7162:1985
|
||||
sort.Slice(vanishedUIDs, func(i, j int) bool {
|
||||
return vanishedUIDs[i] < vanishedUIDs[j]
|
||||
})
|
||||
// No hard limit on response sizes, but clients are recommended to not send more
|
||||
// than 8k. We send a more conservative max 4k.
|
||||
for _, s := range compactUIDSet(vanishedUIDs).Strings(4*1024 - 32) {
|
||||
c.bwritelinef("* VANISHED (EARLIER) %s", s)
|
||||
}
|
||||
}
|
||||
|
||||
for _, uid := range uids {
|
||||
cmd.uid = uid
|
||||
mlog.Field("processing uid", mlog.Field("uid", uid))
|
||||
cmd.process(atts)
|
||||
}
|
||||
})
|
||||
@ -113,6 +250,15 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
|
||||
}
|
||||
}
|
||||
|
||||
func (cmd *fetchCmd) xmodseq() store.ModSeq {
|
||||
if cmd.modseq == 0 {
|
||||
var err error
|
||||
cmd.modseq, err = cmd.conn.account.NextModSeq(cmd.tx)
|
||||
cmd.xcheckf(err, "assigning next modseq")
|
||||
}
|
||||
return cmd.modseq
|
||||
}
|
||||
|
||||
func (cmd *fetchCmd) xensureMessage() *store.Message {
|
||||
if cmd.m != nil {
|
||||
return cmd.m
|
||||
@ -120,6 +266,7 @@ func (cmd *fetchCmd) xensureMessage() *store.Message {
|
||||
|
||||
q := bstore.QueryTx[store.Message](cmd.tx)
|
||||
q.FilterNonzero(store.Message{MailboxID: cmd.mailboxID, UID: cmd.uid})
|
||||
q.FilterEqual("Expunged", false)
|
||||
m, err := q.Get()
|
||||
cmd.xcheckf(err, "get message for uid %d", cmd.uid)
|
||||
cmd.m = &m
|
||||
@ -178,6 +325,7 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
|
||||
|
||||
cmd.markSeen = false
|
||||
cmd.needFlags = false
|
||||
cmd.needModseq = false
|
||||
|
||||
for _, a := range atts {
|
||||
data = append(data, cmd.xprocessAtt(a)...)
|
||||
@ -186,10 +334,11 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
|
||||
if cmd.markSeen {
|
||||
m := cmd.xensureMessage()
|
||||
m.Seen = true
|
||||
m.ModSeq = cmd.xmodseq()
|
||||
err := cmd.tx.Update(m)
|
||||
xcheckf(err, "marking message as seen")
|
||||
|
||||
cmd.changes = append(cmd.changes, store.ChangeFlags{MailboxID: cmd.mailboxID, UID: cmd.uid, Mask: store.Flags{Seen: true}, Flags: m.Flags, Keywords: m.Keywords})
|
||||
cmd.changes = append(cmd.changes, store.ChangeFlags{MailboxID: cmd.mailboxID, UID: cmd.uid, ModSeq: m.ModSeq, Mask: store.Flags{Seen: true}, Flags: m.Flags, Keywords: m.Keywords})
|
||||
}
|
||||
|
||||
if cmd.needFlags {
|
||||
@ -197,6 +346,26 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
|
||||
data = append(data, bare("FLAGS"), flaglist(m.Flags, m.Keywords))
|
||||
}
|
||||
|
||||
// The wording around when to include the MODSEQ attribute is hard to follow and is
|
||||
// specified and refined in several places.
|
||||
//
|
||||
// An additional rule applies to "QRESYNC servers" (we'll assume it only applies
|
||||
// when QRESYNC is enabled on a connection): setting the \Seen flag also triggers
|
||||
// sending MODSEQ, and so does a UID FETCH command. ../rfc/7162:1421
|
||||
//
|
||||
// For example, ../rfc/7162:389 says the server must include modseq in "all
|
||||
// subsequent untagged fetch responses", then lists cases, but leaves out FETCH/UID
|
||||
// FETCH. That appears intentional, it is not a list of examples, it is the full
|
||||
// list, and the "all subsequent untagged fetch responses" doesn't mean "all", just
|
||||
// those covering the listed cases. That makes sense, because otherwise all the
|
||||
// other mentioning of cases elsewhere in the RFC would be too superfluous.
|
||||
//
|
||||
// ../rfc/7162:877 ../rfc/7162:388 ../rfc/7162:909 ../rfc/7162:1426
|
||||
if cmd.needModseq || cmd.hasChangedSince || cmd.conn.enabled[capQresync] && (cmd.isUID || cmd.markSeen) {
|
||||
m := cmd.xensureMessage()
|
||||
data = append(data, bare("MODSEQ"), listspace{bare(fmt.Sprintf("%d", m.ModSeq.Client()))})
|
||||
}
|
||||
|
||||
// Write errors are turned into panics because we write through c.
|
||||
fmt.Fprintf(cmd.conn.bw, "* %d FETCH ", cmd.conn.xsequence(cmd.uid))
|
||||
data.writeTo(cmd.conn, cmd.conn.bw)
|
||||
@ -301,6 +470,9 @@ func (cmd *fetchCmd) xprocessAtt(a fetchAtt) []token {
|
||||
case "FLAGS":
|
||||
cmd.needFlags = true
|
||||
|
||||
case "MODSEQ":
|
||||
cmd.needModseq = true
|
||||
|
||||
default:
|
||||
xserverErrorf("field %q not yet implemented", a.field)
|
||||
}
|
||||
|
Reference in New Issue
Block a user