imapserver: implement rfc 9590, returning metadata in the extended list command

only with "return" including "metadata". so clients can quickly get certain
metadata (eg for display, such as a color) for mailboxes.

this also adds a protocol token type "mailboxt" that properly encodes to utf7
if required.

imapclient/parse.go

@@ -460,11 +460,14 @@ func (c *Conn) xuntagged() Untagged {
 			var isString bool
 			if c.take('~') {
 				value = c.xliteral()
-			} else {
+			} else if c.peek('"') {
 				value = []byte(c.xstring())
 				isString = true
 				// note: the abnf also allows nstring, but that only makes sense when the
 				// production rule is used in the setmetadata command. ../rfc/5464:831
+			} else {
+				// For response to extended list.
+				c.xtake("nil")
 			}
 			r.Annotations = append(r.Annotations, Annotation{key, isString, value})
 

imapclient/protocol.go

@@ -37,6 +37,7 @@ const (
 	CapSaveDate         Capability = "SAVEDATE"           // ../rfc/8514
 	CapCreateSpecialUse Capability = "CREATE-SPECIAL-USE" // ../rfc/6154:296
 	CapCompressDeflate  Capability = "COMPRESS=DEFLATE"   // ../rfc/4978:65
+	CapListMetadata     Capability = "LIST-METADTA"       // ../rfc/9590:73
 )
 
 // Status is the tagged final result of a command.
@@ -242,8 +243,10 @@ type UntaggedMetadataKeys struct {
 	Keys []string
 }
 
+// Annotation is a metadata server of mailbox annotation.
 type Annotation struct {
 	Key      string
+	// Nil is represented by IsString false and a nil Value.
 	IsString bool
 	Value    []byte
 }

imapserver/list.go

@@ -1,6 +1,7 @@
 package imapserver
 
 import (
+	"bytes"
 	"fmt"
 	"path"
 	"sort"
@@ -60,6 +61,7 @@ func (c *conn) cmdList(tag, cmd string, p *parser) {
 	isExtended = isExtended || isList
 	var retSubscribed, retChildren bool
 	var retStatusAttrs []string
+	var retMetadata []string
 	if p.take(" RETURN (") {
 		isExtended = true
 		// ../rfc/9051:6613 ../rfc/9051:6915 ../rfc/9051:7072 ../rfc/9051:6821 ../rfc/5819:95
@@ -90,6 +92,18 @@ func (c *conn) cmdList(tag, cmd string, p *parser) {
 					retStatusAttrs = append(retStatusAttrs, p.xstatusAtt())
 				}
 				p.xtake(")")
+			case "METADATA":
+				// ../rfc/9590:167
+				p.xspace()
+				p.xtake("(")
+				for {
+					s := p.xmetadataKey()
+					retMetadata = append(retMetadata, s)
+					if !p.space() {
+						break
+					}
+				}
+				p.xtake(")")
 			default:
 				// ../rfc/9051:2398
 				xsyntaxErrorf("bad list return option %q", w)
@@ -117,6 +131,7 @@ func (c *conn) cmdList(tag, cmd string, p *parser) {
 	}
 	re := xmailboxPatternMatcher(reference, patterns)
 	var responseLines []string
+	var respMetadata []concatspace
 
 	c.account.WithRLock(func() {
 		c.xdbread(func(tx *bstore.Tx) {
@@ -208,12 +223,34 @@ func (c *conn) cmdList(tag, cmd string, p *parser) {
 				if extended != nil {
 					extStr = " " + extended.pack(c)
 				}
-				line := fmt.Sprintf(`* LIST %s "/" %s%s`, flags.pack(c), astring(c.encodeMailbox(name)).pack(c), extStr)
+				line := fmt.Sprintf(`* LIST %s "/" %s%s`, flags.pack(c), mailboxt(name).pack(c), extStr)
 				responseLines = append(responseLines, line)
 
 				if retStatusAttrs != nil && info.mailbox != nil {
 					responseLines = append(responseLines, c.xstatusLine(tx, *info.mailbox, retStatusAttrs))
 				}
+
+				// ../rfc/9590:101
+				if info.mailbox != nil && len(retMetadata) > 0 {
+					var meta listspace
+					for _, k := range retMetadata {
+						a, err := bstore.QueryTx[store.Annotation](tx).FilterNonzero(store.Annotation{MailboxID: info.mailbox.ID, Key: k}).Get()
+						var v token
+						if err == bstore.ErrAbsent {
+							v = nilt
+						} else {
+							xcheckf(err, "get annotation")
+							if a.IsString {
+								v = string0(string(a.Value))
+							} else {
+								v = readerSizeSyncliteral{bytes.NewReader(a.Value), int64(len(a.Value)), true}
+							}
+						}
+						meta = append(meta, astring(k), v)
+					}
+					line := concatspace{bare("*"), bare("METADATA"), mailboxt(info.mailbox.Name), meta}
+					respMetadata = append(respMetadata, line)
+				}
 			}
 		})
 	})
@@ -221,5 +258,9 @@ func (c *conn) cmdList(tag, cmd string, p *parser) {
 	for _, line := range responseLines {
 		c.bwritelinef("%s", line)
 	}
+	for _, meta := range respMetadata {
+		meta.writeTo(c, c.bw)
+		c.bwritelinef("")
+	}
 	c.ok(tag, cmd)
 }

imapserver/list_test.go

@@ -216,4 +216,21 @@ func TestListExtended(t *testing.T) {
 	tc.transactf("bad", `list (recursivematch remote) "" "*"`) // "remote" is not a base selection option.
 	tc.transactf("bad", `list (unknown) "" "*"`)               // Unknown selection options must result in BAD.
 	tc.transactf("bad", `list () "" "*" return (unknown)`)     // Unknown return options must result in BAD.
+
+	// Return metadata.
+	tc.transactf("ok", `setmetadata inbox (/private/comment "y")`)
+	tc.transactf("ok", `list () "" ("inbox") return (metadata (/private/comment /shared/comment))`)
+	tc.xuntagged(
+		ulist("Inbox"),
+		imapclient.UntaggedMetadataAnnotations{
+			Mailbox: "Inbox",
+			Annotations: []imapclient.Annotation{
+				{Key: "/private/comment", IsString: true, Value: []byte("y")},
+				{Key: "/shared/comment"},
+			},
+		},
+	)
+
+	tc.transactf("bad", `list () "" ("inbox") return (metadata ())`)                                    // Metadata list must be non-empty.
+	tc.transactf("bad", `list () "" ("inbox") return (metadata (/shared/comment "/private/comment" ))`) // Extra space.
 }

imapserver/metadata.go

@@ -155,7 +155,7 @@ func (c *conn) cmdGetmetadata(tag, cmd string, p *parser) {
 	// Response syntax: ../rfc/5464:807 ../rfc/5464:778
 	// We can only send untagged responses when we have any matches.
 	if len(annotations) > 0 {
-		fmt.Fprintf(c.bw, "* METADATA %s (", astring(mailboxName).pack(c))
+		fmt.Fprintf(c.bw, "* METADATA %s (", mailboxt(mailboxName).pack(c))
 		for i, a := range annotations {
 			if i > 0 {
 				fmt.Fprint(c.bw, " ")

imapserver/pack.go

@@ -176,6 +176,29 @@ func (t listspace) writeTo(c *conn, w io.Writer) {
 	fmt.Fprint(w, ")")
 }
 
+// concatenate tokens space-separated
+type concatspace []token
+
+func (t concatspace) pack(c *conn) string {
+	var s string
+	for i, e := range t {
+		if i > 0 {
+			s += " "
+		}
+		s += e.pack(c)
+	}
+	return s
+}
+
+func (t concatspace) writeTo(c *conn, w io.Writer) {
+	for i, e := range t {
+		if i > 0 {
+			fmt.Fprint(w, " ")
+		}
+		e.writeTo(c, w)
+	}
+}
+
 // Concatenated tokens, no spaces or list syntax.
 type concat []token
 
@@ -215,6 +238,21 @@ func (t astring) writeTo(c *conn, w io.Writer) {
 	w.Write([]byte(t.pack(c)))
 }
 
+// mailbox with utf7 encoding if connection requires it, or utf8 otherwise.
+type mailboxt string
+
+func (t mailboxt) pack(c *conn) string {
+	s := string(t)
+	if !c.utf8strings() {
+		s = utf7encode(s)
+	}
+	return astring(s).pack(c)
+}
+
+func (t mailboxt) writeTo(c *conn, w io.Writer) {
+	w.Write([]byte(t.pack(c)))
+}
+
 type number uint32
 
 func (t number) pack(c *conn) string {

imapserver/server.go

@@ -164,12 +164,13 @@ var authFailDelay = time.Second  // After authentication failure.
 // WITHIN: ../rfc/5032
 // NAMESPACE: ../rfc/2342
 // COMPRESS=DEFLATE: ../rfc/4978
+// LIST-METADATA: ../rfc/9590
 //
 // We always announce support for SCRAM PLUS-variants, also on connections without
 // TLS. The client should not be selecting PLUS variants on non-TLS connections,
 // instead opting to do the bare SCRAM variant without indicating the server claims
 // to support the PLUS variant (skipping the server downgrade detection check).
-const serverCapabilities = "IMAP4rev2 IMAP4rev1 ENABLE LITERAL+ IDLE SASL-IR BINARY UNSELECT UIDPLUS ESEARCH SEARCHRES MOVE UTF8=ACCEPT LIST-EXTENDED SPECIAL-USE CREATE-SPECIAL-USE LIST-STATUS AUTH=SCRAM-SHA-256-PLUS AUTH=SCRAM-SHA-256 AUTH=SCRAM-SHA-1-PLUS AUTH=SCRAM-SHA-1 AUTH=CRAM-MD5 ID APPENDLIMIT=9223372036854775807 CONDSTORE QRESYNC STATUS=SIZE QUOTA QUOTA=RES-STORAGE METADATA SAVEDATE WITHIN NAMESPACE COMPRESS=DEFLATE"
+const serverCapabilities = "IMAP4rev2 IMAP4rev1 ENABLE LITERAL+ IDLE SASL-IR BINARY UNSELECT UIDPLUS ESEARCH SEARCHRES MOVE UTF8=ACCEPT LIST-EXTENDED SPECIAL-USE CREATE-SPECIAL-USE LIST-STATUS AUTH=SCRAM-SHA-256-PLUS AUTH=SCRAM-SHA-256 AUTH=SCRAM-SHA-1-PLUS AUTH=SCRAM-SHA-1 AUTH=CRAM-MD5 ID APPENDLIMIT=9223372036854775807 CONDSTORE QRESYNC STATUS=SIZE QUOTA QUOTA=RES-STORAGE METADATA SAVEDATE WITHIN NAMESPACE COMPRESS=DEFLATE LIST-METADATA"
 
 type conn struct {
 	cid               int64
@@ -441,13 +442,6 @@ func (c *conn) utf8strings() bool {
 	return c.enabled[capIMAP4rev2] || c.enabled[capUTF8Accept]
 }
 
-func (c *conn) encodeMailbox(s string) string {
-	if c.utf8strings() {
-		return s
-	}
-	return utf7encode(s)
-}
-
 func (c *conn) xdbwrite(fn func(tx *bstore.Tx)) {
 	err := c.account.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
 		fn(tx)
@@ -1686,22 +1680,22 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) {
 			// unrecognized \NonExistent and interpret this as a newly created mailbox, while
 			// the goal was to remove it...
 			if c.enabled[capIMAP4rev2] {
-				c.bwritelinef(`* LIST (\NonExistent) "/" %s`, astring(c.encodeMailbox(ch.Name)).pack(c))
+				c.bwritelinef(`* LIST (\NonExistent) "/" %s`, mailboxt(ch.Name).pack(c))
 			}
 		case store.ChangeAddMailbox:
-			c.bwritelinef(`* LIST (%s) "/" %s`, strings.Join(ch.Flags, " "), astring(c.encodeMailbox(ch.Mailbox.Name)).pack(c))
+			c.bwritelinef(`* LIST (%s) "/" %s`, strings.Join(ch.Flags, " "), mailboxt(ch.Mailbox.Name).pack(c))
 		case store.ChangeRenameMailbox:
 			// OLDNAME only with IMAP4rev2 or NOTIFY ../rfc/9051:2726 ../rfc/5465:628
 			var oldname string
 			if c.enabled[capIMAP4rev2] {
-				oldname = fmt.Sprintf(` ("OLDNAME" (%s))`, string0(c.encodeMailbox(ch.OldName)).pack(c))
+				oldname = fmt.Sprintf(` ("OLDNAME" (%s))`, mailboxt(ch.OldName).pack(c))
 			}
-			c.bwritelinef(`* LIST (%s) "/" %s%s`, strings.Join(ch.Flags, " "), astring(c.encodeMailbox(ch.NewName)).pack(c), oldname)
+			c.bwritelinef(`* LIST (%s) "/" %s%s`, strings.Join(ch.Flags, " "), mailboxt(ch.NewName).pack(c), oldname)
 		case store.ChangeAddSubscription:
-			c.bwritelinef(`* LIST (%s) "/" %s`, strings.Join(append([]string{`\Subscribed`}, ch.Flags...), " "), astring(c.encodeMailbox(ch.Name)).pack(c))
+			c.bwritelinef(`* LIST (%s) "/" %s`, strings.Join(append([]string{`\Subscribed`}, ch.Flags...), " "), mailboxt(ch.Name).pack(c))
 		case store.ChangeAnnotation:
 			// ../rfc/5464:807 ../rfc/5464:788
-			c.bwritelinef(`* METADATA %s %s`, astring(c.encodeMailbox(ch.MailboxName)).pack(c), astring(ch.Key).pack(c))
+			c.bwritelinef(`* METADATA %s %s`, mailboxt(ch.MailboxName).pack(c), astring(ch.Key).pack(c))
 		default:
 			panic(fmt.Sprintf("internal error, missing case for %#v", change))
 		}
@@ -2607,7 +2601,7 @@ func (c *conn) cmdSelectExamine(isselect bool, tag, cmd string, p *parser) {
 	}
 	c.bwritelinef(`* OK [UIDVALIDITY %d] x`, mb.UIDValidity)
 	c.bwritelinef(`* OK [UIDNEXT %d] x`, mb.UIDNext)
-	c.bwritelinef(`* LIST () "/" %s`, astring(c.encodeMailbox(mb.Name)).pack(c))
+	c.bwritelinef(`* LIST () "/" %s`, mailboxt(mb.Name).pack(c))
 	if c.enabled[capCondstore] {
 		// ../rfc/7162:417
 		// ../rfc/7162-eid5055 ../rfc/7162:484 ../rfc/7162:1167
@@ -2842,9 +2836,9 @@ func (c *conn) cmdCreate(tag, cmd string, p *parser) {
 		var oldname string
 		// OLDNAME only with IMAP4rev2 or NOTIFY ../rfc/9051:2726 ../rfc/5465:628
 		if c.enabled[capIMAP4rev2] && n == name && name != origName && !(name == "Inbox" || strings.HasPrefix(name, "Inbox/")) {
-			oldname = fmt.Sprintf(` ("OLDNAME" (%s))`, string0(c.encodeMailbox(origName)).pack(c))
+			oldname = fmt.Sprintf(` ("OLDNAME" (%s))`, mailboxt(origName).pack(c))
 		}
-		c.bwritelinef(`* LIST (\Subscribed) "/" %s%s`, astring(c.encodeMailbox(n)).pack(c), oldname)
+		c.bwritelinef(`* LIST (\Subscribed) "/" %s%s`, mailboxt(n).pack(c), oldname)
 	}
 	c.ok(tag, cmd)
 }
@@ -3152,7 +3146,7 @@ func (c *conn) cmdLsub(tag, cmd string, p *parser) {
 				continue
 			}
 			have[name] = true
-			line := fmt.Sprintf(`* LSUB () "/" %s`, astring(c.encodeMailbox(name)).pack(c))
+			line := fmt.Sprintf(`* LSUB () "/" %s`, mailboxt(name).pack(c))
 			lines = append(lines, line)
 
 		}
@@ -3167,7 +3161,7 @@ func (c *conn) cmdLsub(tag, cmd string, p *parser) {
 			if have[mb.Name] || !subscribedKids[mb.Name] || !re.MatchString(mb.Name) {
 				return nil
 			}
-			line := fmt.Sprintf(`* LSUB (\NoSelect) "/" %s`, astring(c.encodeMailbox(mb.Name)).pack(c))
+			line := fmt.Sprintf(`* LSUB (\NoSelect) "/" %s`, mailboxt(mb.Name).pack(c))
 			lines = append(lines, line)
 			return nil
 		})
@@ -3274,7 +3268,7 @@ func (c *conn) xstatusLine(tx *bstore.Tx, mb store.Mailbox, attrs []string) stri
 			xsyntaxErrorf("unknown attribute %q", a)
 		}
 	}
-	return fmt.Sprintf("* STATUS %s (%s)", astring(c.encodeMailbox(mb.Name)).pack(c), strings.Join(status, " "))
+	return fmt.Sprintf("* STATUS %s (%s)", mailboxt(mb.Name).pack(c), strings.Join(status, " "))
 }
 
 func flaglist(fl store.Flags, keywords []string) listspace {

rfc/index.txt

@@ -238,7 +238,7 @@ https://www.iana.org/assignments/message-headers/message-headers.xhtml
 9394	Roadmap	-	IMAP PARTIAL Extension for Paged SEARCH and FETCH
 9585	?	-	IMAP Response Code for Command Progress Notifications
 9586	Roadmap	-	IMAP Extension for Using and Returning Unique Identifiers (UIDs) Only
-9590	Roadmap	-	IMAP Extension for Returning Mailbox METADATA in Extended LIST
+9590	Yes	-	IMAP Extension for Returning Mailbox METADATA in Extended LIST
 9698	?	-	The JMAPACCESS Extension for IMAP
 
 5198	-?	-	Unicode Format for Network Interchange