infrastructure/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2025-07-10 07:54:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

add condstore & qresync imap extensions

Browse Source 
for conditional storing and quick resynchronisation (not sure if mail clients are actually using it that).

each message now has a "modseq". it is increased for each change. with
condstore, imap clients can request changes since a certain modseq. that
already allows quickly finding changes since a previous connection. condstore
also allows storing (e.g. setting new message flags) only when the modseq of a
message hasn't changed.

qresync should make it fast for clients to get a full list of changed messages
for a mailbox, including removals.

we now also keep basic metadata of messages that have been removed (expunged).
just enough (uid, modseq) to tell client that the messages have been removed.
this does mean we have to be careful when querying messages from the database.
we must now often filter the expunged messages out.

we also keep "createseq", the modseq when a message was created. this will be
useful for the jmap implementation.
This commit is contained in:
Mechiel Lukkien
2023-07-24 21:21:05 +02:00
parent cc4ecf2927
commit 7f1b7198a8
30 changed files with 2181 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

204
imapserver/fetch.go
View File

@ -11,24 +11,31 @@ import (
"sort"
"strings"
"golang.org/x/exp/maps"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/store"
)
// functions to handle fetch attribute requests are defined on fetchCmd.
type fetchCmd struct {
conn *conn
mailboxID int64
uid store.UID
tx *bstore.Tx // Writable tx, for storing message when first parsed as mime parts.
changes []store.Change // For updated Seen flag.
markSeen bool
needFlags bool
expungeIssued bool // Set if a message cannot be read. Can happen for expunged messages.
conn *conn
mailboxID int64
uid store.UID
tx *bstore.Tx // Writable tx, for storing message when first parsed as mime parts.
changes []store.Change // For updated Seen flag.
markSeen bool
needFlags bool
needModseq bool // Whether untagged responses needs modseq.
expungeIssued bool // Set if a message cannot be read. Can happen for expunged messages.
modseq store.ModSeq // Initialized on first change, for marking messages as seen.
isUID bool // If this is a UID FETCH command.
hasChangedSince bool // Whether CHANGEDSINCE was set. Enables MODSEQ in response.
// Loaded when first needed, closed when message was processed.
m *store.Message // Message currently being processed.
@ -60,15 +67,61 @@ func (cmd *fetchCmd) xcheckf(err error, format string, args ...any) {
//
// State: Selected
func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
// Command: ../rfc/9051:4330 ../rfc/3501:2992
// Examples: ../rfc/9051:4463 ../rfc/9051:4520
// Response syntax: ../rfc/9051:6742 ../rfc/3501:4864
// Command: ../rfc/9051:4330 ../rfc/3501:2992 ../rfc/7162:864
// Examples: ../rfc/9051:4463 ../rfc/9051:4520 ../rfc/7162:880
// Response syntax: ../rfc/9051:6742 ../rfc/3501:4864 ../rfc/7162:2490
// Request syntax: ../rfc/9051:6553 ../rfc/3501:4748
// Request syntax: ../rfc/9051:6553 ../rfc/3501:4748 ../rfc/4466:535 ../rfc/7162:2475
p.xspace()
nums := p.xnumSet()
p.xspace()
atts := p.xfetchAtts()
atts := p.xfetchAtts(isUID)
var changedSince int64
var haveChangedSince bool
var vanished bool
if p.space() {
// ../rfc/4466:542
// ../rfc/7162:2479
p.xtake("(")
seen := map[string]bool{}
for {
var w string
if isUID && p.conn.enabled[capQresync] {
// Vanished only valid for uid fetch, and only for qresync. ../rfc/7162:1693
w = p.xtakelist("CHANGEDSINCE", "VANISHED")
} else {
w = p.xtakelist("CHANGEDSINCE")
}
if seen[w] {
xsyntaxErrorf("duplicate fetch modifier %s", w)
}
seen[w] = true
switch w {
case "CHANGEDSINCE":
p.xspace()
changedSince = p.xnumber64()
// workaround: ios mail (16.5.1) was seen sending changedSince 0 on an existing account that got condstore enabled.
if changedSince == 0 && moxvar.Pedantic {
// ../rfc/7162:2551
xsyntaxErrorf("changedsince modseq must be > 0")
}
// CHANGEDSINCE is a CONDSTORE-enabling parameter. ../rfc/7162:380
p.conn.xensureCondstore(nil)
haveChangedSince = true
case "VANISHED":
vanished = true
}
if p.take(")") {
break
}
p.xspace()
}
// ../rfc/7162:1701
if vanished && !haveChangedSince {
xsyntaxErrorf("VANISHED can only be used with CHANGEDSINCE")
}
}
p.xempty()
// We don't use c.account.WithRLock because we write to the client while reading messages.
@ -81,21 +134,105 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
runlock()
}()
cmd := &fetchCmd{conn: c, mailboxID: c.mailboxID}
var vanishedUIDs []store.UID
cmd := &fetchCmd{conn: c, mailboxID: c.mailboxID, isUID: isUID, hasChangedSince: haveChangedSince}
c.xdbwrite(func(tx *bstore.Tx) {
cmd.tx = tx
// Ensure the mailbox still exists.
c.xmailboxID(tx, c.mailboxID)
uids := c.xnumSetUIDs(isUID, nums)
var uids []store.UID
// With changedSince, the client is likely asking for a small set of changes. Use a
// database query to trim down the uids we need to look at.
// ../rfc/7162:871
if changedSince > 0 {
q := bstore.QueryTx[store.Message](tx)
q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
q.FilterGreater("ModSeq", store.ModSeqFromClient(changedSince))
if !vanished {
q.FilterEqual("Expunged", false)
}
err := q.ForEach(func(m store.Message) error {
if m.Expunged {
vanishedUIDs = append(vanishedUIDs, m.UID)
} else if isUID {
if nums.containsUID(m.UID, c.uids, c.searchResult) {
uids = append(uids, m.UID)
}
} else {
seq := c.sequence(m.UID)
if seq > 0 && nums.containsSeq(seq, c.uids, c.searchResult) {
uids = append(uids, m.UID)
}
}
return nil
})
xcheckf(err, "looking up messages with changedsince")
} else {
uids = c.xnumSetUIDs(isUID, nums)
}
// Send vanished for all missing requested UIDs. ../rfc/7162:1718
if vanished {
delModSeq, err := c.account.HighestDeletedModSeq(tx)
xcheckf(err, "looking up highest deleted modseq")
if changedSince < delModSeq.Client() {
// First sort the uids we already found, for fast lookup.
sort.Slice(vanishedUIDs, func(i, j int) bool {
return vanishedUIDs[i] < vanishedUIDs[j]
})
// We'll be gathering any more vanished uids in more.
more := map[store.UID]struct{}{}
checkVanished := func(uid store.UID) {
if uidSearch(c.uids, uid) <= 0 && uidSearch(vanishedUIDs, uid) <= 0 {
more[uid] = struct{}{}
}
}
// Now look through the requested uids. We may have a searchResult, handle it
// separately from a numset with potential stars, over which we can more easily
// iterate.
if nums.searchResult {
for _, uid := range c.searchResult {
checkVanished(uid)
}
} else {
iter := nums.interpretStar(c.uids).newIter()
for {
num, ok := iter.Next()
if !ok {
break
}
checkVanished(store.UID(num))
}
}
vanishedUIDs = append(vanishedUIDs, maps.Keys(more)...)
}
}
// Release the account lock.
runlock()
runlock = func() {} // Prevent defer from unlocking again.
// First report all vanished UIDs. ../rfc/7162:1714
if len(vanishedUIDs) > 0 {
// Mention all vanished UIDs in compact numset form.
// ../rfc/7162:1985
sort.Slice(vanishedUIDs, func(i, j int) bool {
return vanishedUIDs[i] < vanishedUIDs[j]
})
// No hard limit on response sizes, but clients are recommended to not send more
// than 8k. We send a more conservative max 4k.
for _, s := range compactUIDSet(vanishedUIDs).Strings(4*1024 - 32) {
c.bwritelinef("* VANISHED (EARLIER) %s", s)
}
}
for _, uid := range uids {
cmd.uid = uid
mlog.Field("processing uid", mlog.Field("uid", uid))
cmd.process(atts)
}
})
@ -113,6 +250,15 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
}
}
func (cmd *fetchCmd) xmodseq() store.ModSeq {
if cmd.modseq == 0 {
var err error
cmd.modseq, err = cmd.conn.account.NextModSeq(cmd.tx)
cmd.xcheckf(err, "assigning next modseq")
}
return cmd.modseq
}
func (cmd *fetchCmd) xensureMessage() *store.Message {
if cmd.m != nil {
return cmd.m
@ -120,6 +266,7 @@ func (cmd *fetchCmd) xensureMessage() *store.Message {
q := bstore.QueryTx[store.Message](cmd.tx)
q.FilterNonzero(store.Message{MailboxID: cmd.mailboxID, UID: cmd.uid})
q.FilterEqual("Expunged", false)
m, err := q.Get()
cmd.xcheckf(err, "get message for uid %d", cmd.uid)
cmd.m = &m
@ -178,6 +325,7 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
cmd.markSeen = false
cmd.needFlags = false
cmd.needModseq = false
for _, a := range atts {
data = append(data, cmd.xprocessAtt(a)...)
@ -186,10 +334,11 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
if cmd.markSeen {
m := cmd.xensureMessage()
m.Seen = true
m.ModSeq = cmd.xmodseq()
err := cmd.tx.Update(m)
xcheckf(err, "marking message as seen")
cmd.changes = append(cmd.changes, store.ChangeFlags{MailboxID: cmd.mailboxID, UID: cmd.uid, Mask: store.Flags{Seen: true}, Flags: m.Flags, Keywords: m.Keywords})
cmd.changes = append(cmd.changes, store.ChangeFlags{MailboxID: cmd.mailboxID, UID: cmd.uid, ModSeq: m.ModSeq, Mask: store.Flags{Seen: true}, Flags: m.Flags, Keywords: m.Keywords})
}
if cmd.needFlags {
@ -197,6 +346,26 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
data = append(data, bare("FLAGS"), flaglist(m.Flags, m.Keywords))
}
// The wording around when to include the MODSEQ attribute is hard to follow and is
// specified and refined in several places.
//
// An additional rule applies to "QRESYNC servers" (we'll assume it only applies
// when QRESYNC is enabled on a connection): setting the \Seen flag also triggers
// sending MODSEQ, and so does a UID FETCH command. ../rfc/7162:1421
//
// For example, ../rfc/7162:389 says the server must include modseq in "all
// subsequent untagged fetch responses", then lists cases, but leaves out FETCH/UID
// FETCH. That appears intentional, it is not a list of examples, it is the full
// list, and the "all subsequent untagged fetch responses" doesn't mean "all", just
// those covering the listed cases. That makes sense, because otherwise all the
// other mentioning of cases elsewhere in the RFC would be too superfluous.
//
// ../rfc/7162:877 ../rfc/7162:388 ../rfc/7162:909 ../rfc/7162:1426
if cmd.needModseq || cmd.hasChangedSince || cmd.conn.enabled[capQresync] && (cmd.isUID || cmd.markSeen) {
m := cmd.xensureMessage()
data = append(data, bare("MODSEQ"), listspace{bare(fmt.Sprintf("%d", m.ModSeq.Client()))})
}
// Write errors are turned into panics because we write through c.
fmt.Fprintf(cmd.conn.bw, "* %d FETCH ", cmd.conn.xsequence(cmd.uid))
data.writeTo(cmd.conn, cmd.conn.bw)
@ -301,6 +470,9 @@ func (cmd *fetchCmd) xprocessAtt(a fetchAtt) []token {
case "FLAGS":
cmd.needFlags = true
case "MODSEQ":
cmd.needModseq = true
default:
xserverErrorf("field %q not yet implemented", a.field)
}