From 0ed820e3b084e1d1cbd4e95a5457692519694108 Mon Sep 17 00:00:00 2001 From: Mechiel Lukkien Date: Sun, 23 Feb 2025 22:12:18 +0100 Subject: [PATCH] imapserver: implement rfc 9590, returning metadata in the extended list command only with "return" including "metadata". so clients can quickly get certain metadata (eg for display, such as a color) for mailboxes. this also adds a protocol token type "mailboxt" that properly encodes to utf7 if required. --- imapclient/parse.go | 5 ++++- imapclient/protocol.go | 3 +++ imapserver/list.go | 43 ++++++++++++++++++++++++++++++++++++++++- imapserver/list_test.go | 17 ++++++++++++++++ imapserver/metadata.go | 2 +- imapserver/pack.go | 38 ++++++++++++++++++++++++++++++++++++ imapserver/server.go | 34 ++++++++++++++------------------ rfc/index.txt | 2 +- 8 files changed, 120 insertions(+), 24 deletions(-) diff --git a/imapclient/parse.go b/imapclient/parse.go index 957e4ff..312e284 100644 --- a/imapclient/parse.go +++ b/imapclient/parse.go @@ -460,11 +460,14 @@ func (c *Conn) xuntagged() Untagged { var isString bool if c.take('~') { value = c.xliteral() - } else { + } else if c.peek('"') { value = []byte(c.xstring()) isString = true // note: the abnf also allows nstring, but that only makes sense when the // production rule is used in the setmetadata command. ../rfc/5464:831 + } else { + // For response to extended list. + c.xtake("nil") } r.Annotations = append(r.Annotations, Annotation{key, isString, value}) diff --git a/imapclient/protocol.go b/imapclient/protocol.go index c0f5b82..3bd56fc 100644 --- a/imapclient/protocol.go +++ b/imapclient/protocol.go @@ -37,6 +37,7 @@ const ( CapSaveDate Capability = "SAVEDATE" // ../rfc/8514 CapCreateSpecialUse Capability = "CREATE-SPECIAL-USE" // ../rfc/6154:296 CapCompressDeflate Capability = "COMPRESS=DEFLATE" // ../rfc/4978:65 + CapListMetadata Capability = "LIST-METADTA" // ../rfc/9590:73 ) // Status is the tagged final result of a command. @@ -242,8 +243,10 @@ type UntaggedMetadataKeys struct { Keys []string } +// Annotation is a metadata server of mailbox annotation. type Annotation struct { Key string + // Nil is represented by IsString false and a nil Value. IsString bool Value []byte } diff --git a/imapserver/list.go b/imapserver/list.go index 3754419..d65cb72 100644 --- a/imapserver/list.go +++ b/imapserver/list.go @@ -1,6 +1,7 @@ package imapserver import ( + "bytes" "fmt" "path" "sort" @@ -60,6 +61,7 @@ func (c *conn) cmdList(tag, cmd string, p *parser) { isExtended = isExtended || isList var retSubscribed, retChildren bool var retStatusAttrs []string + var retMetadata []string if p.take(" RETURN (") { isExtended = true // ../rfc/9051:6613 ../rfc/9051:6915 ../rfc/9051:7072 ../rfc/9051:6821 ../rfc/5819:95 @@ -90,6 +92,18 @@ func (c *conn) cmdList(tag, cmd string, p *parser) { retStatusAttrs = append(retStatusAttrs, p.xstatusAtt()) } p.xtake(")") + case "METADATA": + // ../rfc/9590:167 + p.xspace() + p.xtake("(") + for { + s := p.xmetadataKey() + retMetadata = append(retMetadata, s) + if !p.space() { + break + } + } + p.xtake(")") default: // ../rfc/9051:2398 xsyntaxErrorf("bad list return option %q", w) @@ -117,6 +131,7 @@ func (c *conn) cmdList(tag, cmd string, p *parser) { } re := xmailboxPatternMatcher(reference, patterns) var responseLines []string + var respMetadata []concatspace c.account.WithRLock(func() { c.xdbread(func(tx *bstore.Tx) { @@ -208,12 +223,34 @@ func (c *conn) cmdList(tag, cmd string, p *parser) { if extended != nil { extStr = " " + extended.pack(c) } - line := fmt.Sprintf(`* LIST %s "/" %s%s`, flags.pack(c), astring(c.encodeMailbox(name)).pack(c), extStr) + line := fmt.Sprintf(`* LIST %s "/" %s%s`, flags.pack(c), mailboxt(name).pack(c), extStr) responseLines = append(responseLines, line) if retStatusAttrs != nil && info.mailbox != nil { responseLines = append(responseLines, c.xstatusLine(tx, *info.mailbox, retStatusAttrs)) } + + // ../rfc/9590:101 + if info.mailbox != nil && len(retMetadata) > 0 { + var meta listspace + for _, k := range retMetadata { + a, err := bstore.QueryTx[store.Annotation](tx).FilterNonzero(store.Annotation{MailboxID: info.mailbox.ID, Key: k}).Get() + var v token + if err == bstore.ErrAbsent { + v = nilt + } else { + xcheckf(err, "get annotation") + if a.IsString { + v = string0(string(a.Value)) + } else { + v = readerSizeSyncliteral{bytes.NewReader(a.Value), int64(len(a.Value)), true} + } + } + meta = append(meta, astring(k), v) + } + line := concatspace{bare("*"), bare("METADATA"), mailboxt(info.mailbox.Name), meta} + respMetadata = append(respMetadata, line) + } } }) }) @@ -221,5 +258,9 @@ func (c *conn) cmdList(tag, cmd string, p *parser) { for _, line := range responseLines { c.bwritelinef("%s", line) } + for _, meta := range respMetadata { + meta.writeTo(c, c.bw) + c.bwritelinef("") + } c.ok(tag, cmd) } diff --git a/imapserver/list_test.go b/imapserver/list_test.go index ffe65f2..d7aa29f 100644 --- a/imapserver/list_test.go +++ b/imapserver/list_test.go @@ -216,4 +216,21 @@ func TestListExtended(t *testing.T) { tc.transactf("bad", `list (recursivematch remote) "" "*"`) // "remote" is not a base selection option. tc.transactf("bad", `list (unknown) "" "*"`) // Unknown selection options must result in BAD. tc.transactf("bad", `list () "" "*" return (unknown)`) // Unknown return options must result in BAD. + + // Return metadata. + tc.transactf("ok", `setmetadata inbox (/private/comment "y")`) + tc.transactf("ok", `list () "" ("inbox") return (metadata (/private/comment /shared/comment))`) + tc.xuntagged( + ulist("Inbox"), + imapclient.UntaggedMetadataAnnotations{ + Mailbox: "Inbox", + Annotations: []imapclient.Annotation{ + {Key: "/private/comment", IsString: true, Value: []byte("y")}, + {Key: "/shared/comment"}, + }, + }, + ) + + tc.transactf("bad", `list () "" ("inbox") return (metadata ())`) // Metadata list must be non-empty. + tc.transactf("bad", `list () "" ("inbox") return (metadata (/shared/comment "/private/comment" ))`) // Extra space. } diff --git a/imapserver/metadata.go b/imapserver/metadata.go index dbfbe42..98f96a4 100644 --- a/imapserver/metadata.go +++ b/imapserver/metadata.go @@ -155,7 +155,7 @@ func (c *conn) cmdGetmetadata(tag, cmd string, p *parser) { // Response syntax: ../rfc/5464:807 ../rfc/5464:778 // We can only send untagged responses when we have any matches. if len(annotations) > 0 { - fmt.Fprintf(c.bw, "* METADATA %s (", astring(mailboxName).pack(c)) + fmt.Fprintf(c.bw, "* METADATA %s (", mailboxt(mailboxName).pack(c)) for i, a := range annotations { if i > 0 { fmt.Fprint(c.bw, " ") diff --git a/imapserver/pack.go b/imapserver/pack.go index ef9c59f..f03b4f4 100644 --- a/imapserver/pack.go +++ b/imapserver/pack.go @@ -176,6 +176,29 @@ func (t listspace) writeTo(c *conn, w io.Writer) { fmt.Fprint(w, ")") } +// concatenate tokens space-separated +type concatspace []token + +func (t concatspace) pack(c *conn) string { + var s string + for i, e := range t { + if i > 0 { + s += " " + } + s += e.pack(c) + } + return s +} + +func (t concatspace) writeTo(c *conn, w io.Writer) { + for i, e := range t { + if i > 0 { + fmt.Fprint(w, " ") + } + e.writeTo(c, w) + } +} + // Concatenated tokens, no spaces or list syntax. type concat []token @@ -215,6 +238,21 @@ func (t astring) writeTo(c *conn, w io.Writer) { w.Write([]byte(t.pack(c))) } +// mailbox with utf7 encoding if connection requires it, or utf8 otherwise. +type mailboxt string + +func (t mailboxt) pack(c *conn) string { + s := string(t) + if !c.utf8strings() { + s = utf7encode(s) + } + return astring(s).pack(c) +} + +func (t mailboxt) writeTo(c *conn, w io.Writer) { + w.Write([]byte(t.pack(c))) +} + type number uint32 func (t number) pack(c *conn) string { diff --git a/imapserver/server.go b/imapserver/server.go index a1a0633..5f33707 100644 --- a/imapserver/server.go +++ b/imapserver/server.go @@ -164,12 +164,13 @@ var authFailDelay = time.Second // After authentication failure. // WITHIN: ../rfc/5032 // NAMESPACE: ../rfc/2342 // COMPRESS=DEFLATE: ../rfc/4978 +// LIST-METADATA: ../rfc/9590 // // We always announce support for SCRAM PLUS-variants, also on connections without // TLS. The client should not be selecting PLUS variants on non-TLS connections, // instead opting to do the bare SCRAM variant without indicating the server claims // to support the PLUS variant (skipping the server downgrade detection check). -const serverCapabilities = "IMAP4rev2 IMAP4rev1 ENABLE LITERAL+ IDLE SASL-IR BINARY UNSELECT UIDPLUS ESEARCH SEARCHRES MOVE UTF8=ACCEPT LIST-EXTENDED SPECIAL-USE CREATE-SPECIAL-USE LIST-STATUS AUTH=SCRAM-SHA-256-PLUS AUTH=SCRAM-SHA-256 AUTH=SCRAM-SHA-1-PLUS AUTH=SCRAM-SHA-1 AUTH=CRAM-MD5 ID APPENDLIMIT=9223372036854775807 CONDSTORE QRESYNC STATUS=SIZE QUOTA QUOTA=RES-STORAGE METADATA SAVEDATE WITHIN NAMESPACE COMPRESS=DEFLATE" +const serverCapabilities = "IMAP4rev2 IMAP4rev1 ENABLE LITERAL+ IDLE SASL-IR BINARY UNSELECT UIDPLUS ESEARCH SEARCHRES MOVE UTF8=ACCEPT LIST-EXTENDED SPECIAL-USE CREATE-SPECIAL-USE LIST-STATUS AUTH=SCRAM-SHA-256-PLUS AUTH=SCRAM-SHA-256 AUTH=SCRAM-SHA-1-PLUS AUTH=SCRAM-SHA-1 AUTH=CRAM-MD5 ID APPENDLIMIT=9223372036854775807 CONDSTORE QRESYNC STATUS=SIZE QUOTA QUOTA=RES-STORAGE METADATA SAVEDATE WITHIN NAMESPACE COMPRESS=DEFLATE LIST-METADATA" type conn struct { cid int64 @@ -441,13 +442,6 @@ func (c *conn) utf8strings() bool { return c.enabled[capIMAP4rev2] || c.enabled[capUTF8Accept] } -func (c *conn) encodeMailbox(s string) string { - if c.utf8strings() { - return s - } - return utf7encode(s) -} - func (c *conn) xdbwrite(fn func(tx *bstore.Tx)) { err := c.account.DB.Write(context.TODO(), func(tx *bstore.Tx) error { fn(tx) @@ -1686,22 +1680,22 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { // unrecognized \NonExistent and interpret this as a newly created mailbox, while // the goal was to remove it... if c.enabled[capIMAP4rev2] { - c.bwritelinef(`* LIST (\NonExistent) "/" %s`, astring(c.encodeMailbox(ch.Name)).pack(c)) + c.bwritelinef(`* LIST (\NonExistent) "/" %s`, mailboxt(ch.Name).pack(c)) } case store.ChangeAddMailbox: - c.bwritelinef(`* LIST (%s) "/" %s`, strings.Join(ch.Flags, " "), astring(c.encodeMailbox(ch.Mailbox.Name)).pack(c)) + c.bwritelinef(`* LIST (%s) "/" %s`, strings.Join(ch.Flags, " "), mailboxt(ch.Mailbox.Name).pack(c)) case store.ChangeRenameMailbox: // OLDNAME only with IMAP4rev2 or NOTIFY ../rfc/9051:2726 ../rfc/5465:628 var oldname string if c.enabled[capIMAP4rev2] { - oldname = fmt.Sprintf(` ("OLDNAME" (%s))`, string0(c.encodeMailbox(ch.OldName)).pack(c)) + oldname = fmt.Sprintf(` ("OLDNAME" (%s))`, mailboxt(ch.OldName).pack(c)) } - c.bwritelinef(`* LIST (%s) "/" %s%s`, strings.Join(ch.Flags, " "), astring(c.encodeMailbox(ch.NewName)).pack(c), oldname) + c.bwritelinef(`* LIST (%s) "/" %s%s`, strings.Join(ch.Flags, " "), mailboxt(ch.NewName).pack(c), oldname) case store.ChangeAddSubscription: - c.bwritelinef(`* LIST (%s) "/" %s`, strings.Join(append([]string{`\Subscribed`}, ch.Flags...), " "), astring(c.encodeMailbox(ch.Name)).pack(c)) + c.bwritelinef(`* LIST (%s) "/" %s`, strings.Join(append([]string{`\Subscribed`}, ch.Flags...), " "), mailboxt(ch.Name).pack(c)) case store.ChangeAnnotation: // ../rfc/5464:807 ../rfc/5464:788 - c.bwritelinef(`* METADATA %s %s`, astring(c.encodeMailbox(ch.MailboxName)).pack(c), astring(ch.Key).pack(c)) + c.bwritelinef(`* METADATA %s %s`, mailboxt(ch.MailboxName).pack(c), astring(ch.Key).pack(c)) default: panic(fmt.Sprintf("internal error, missing case for %#v", change)) } @@ -2607,7 +2601,7 @@ func (c *conn) cmdSelectExamine(isselect bool, tag, cmd string, p *parser) { } c.bwritelinef(`* OK [UIDVALIDITY %d] x`, mb.UIDValidity) c.bwritelinef(`* OK [UIDNEXT %d] x`, mb.UIDNext) - c.bwritelinef(`* LIST () "/" %s`, astring(c.encodeMailbox(mb.Name)).pack(c)) + c.bwritelinef(`* LIST () "/" %s`, mailboxt(mb.Name).pack(c)) if c.enabled[capCondstore] { // ../rfc/7162:417 // ../rfc/7162-eid5055 ../rfc/7162:484 ../rfc/7162:1167 @@ -2842,9 +2836,9 @@ func (c *conn) cmdCreate(tag, cmd string, p *parser) { var oldname string // OLDNAME only with IMAP4rev2 or NOTIFY ../rfc/9051:2726 ../rfc/5465:628 if c.enabled[capIMAP4rev2] && n == name && name != origName && !(name == "Inbox" || strings.HasPrefix(name, "Inbox/")) { - oldname = fmt.Sprintf(` ("OLDNAME" (%s))`, string0(c.encodeMailbox(origName)).pack(c)) + oldname = fmt.Sprintf(` ("OLDNAME" (%s))`, mailboxt(origName).pack(c)) } - c.bwritelinef(`* LIST (\Subscribed) "/" %s%s`, astring(c.encodeMailbox(n)).pack(c), oldname) + c.bwritelinef(`* LIST (\Subscribed) "/" %s%s`, mailboxt(n).pack(c), oldname) } c.ok(tag, cmd) } @@ -3152,7 +3146,7 @@ func (c *conn) cmdLsub(tag, cmd string, p *parser) { continue } have[name] = true - line := fmt.Sprintf(`* LSUB () "/" %s`, astring(c.encodeMailbox(name)).pack(c)) + line := fmt.Sprintf(`* LSUB () "/" %s`, mailboxt(name).pack(c)) lines = append(lines, line) } @@ -3167,7 +3161,7 @@ func (c *conn) cmdLsub(tag, cmd string, p *parser) { if have[mb.Name] || !subscribedKids[mb.Name] || !re.MatchString(mb.Name) { return nil } - line := fmt.Sprintf(`* LSUB (\NoSelect) "/" %s`, astring(c.encodeMailbox(mb.Name)).pack(c)) + line := fmt.Sprintf(`* LSUB (\NoSelect) "/" %s`, mailboxt(mb.Name).pack(c)) lines = append(lines, line) return nil }) @@ -3274,7 +3268,7 @@ func (c *conn) xstatusLine(tx *bstore.Tx, mb store.Mailbox, attrs []string) stri xsyntaxErrorf("unknown attribute %q", a) } } - return fmt.Sprintf("* STATUS %s (%s)", astring(c.encodeMailbox(mb.Name)).pack(c), strings.Join(status, " ")) + return fmt.Sprintf("* STATUS %s (%s)", mailboxt(mb.Name).pack(c), strings.Join(status, " ")) } func flaglist(fl store.Flags, keywords []string) listspace { diff --git a/rfc/index.txt b/rfc/index.txt index 8aa2d6d..75905d5 100644 --- a/rfc/index.txt +++ b/rfc/index.txt @@ -238,7 +238,7 @@ https://www.iana.org/assignments/message-headers/message-headers.xhtml 9394 Roadmap - IMAP PARTIAL Extension for Paged SEARCH and FETCH 9585 ? - IMAP Response Code for Command Progress Notifications 9586 Roadmap - IMAP Extension for Using and Returning Unique Identifiers (UIDs) Only -9590 Roadmap - IMAP Extension for Returning Mailbox METADATA in Extended LIST +9590 Yes - IMAP Extension for Returning Mailbox METADATA in Extended LIST 9698 ? - The JMAPACCESS Extension for IMAP 5198 -? - Unicode Format for Network Interchange