implement the imap metadata extension, rfc 5464

this allows setting per-mailbox and per-server annotations (metadata). we have
a fixed maximum for total number of annotations (1000) and their total size
(1000000 bytes). this size isn't held against the regular quota for simplicity.
we send unsolicited metadata responses when a connection is in the idle
command and a change to a metadata item is made.

we currently only implement the /private/ namespace.  we should implement the
/shared/ namespace, for mox-global metadata annotations.  only the admin should
be able to configure those, probably through the config file, cli, or admin web
interface.

for issue #290

imapclient/parse.go

@@ -427,6 +427,50 @@ func (c *Conn) xuntagged() Untagged {
 		c.xcrlf()
 		return r
 
+	case "METADATA":
+		// ../rfc/5464:807
+		c.xspace()
+		mailbox := c.xastring()
+		c.xspace()
+		if !c.take('(') {
+			// Unsolicited form, with only annotation keys, not values.
+			var keys []string
+			for {
+				key := c.xastring()
+				keys = append(keys, key)
+				if !c.space() {
+					break
+				}
+			}
+			c.xcrlf()
+			return UntaggedMetadataKeys{mailbox, keys}
+		}
+
+		// Form with values, in response to GETMETADATA command.
+		r := UntaggedMetadataAnnotations{Mailbox: mailbox}
+		for {
+			key := c.xastring()
+			c.xspace()
+			var value []byte
+			var isString bool
+			if c.take('~') {
+				value = c.xliteral()
+			} else {
+				value = []byte(c.xstring())
+				isString = true
+				// note: the abnf also allows nstring, but that only makes sense when the
+				// production rule is used in the setmetadata command. ../rfc/5464:831
+			}
+			r.Annotations = append(r.Annotations, Annotation{key, isString, value})
+
+			if c.take(')') {
+				break
+			}
+			c.xspace()
+		}
+		c.xcrlf()
+		return r
+
 	case "NAMESPACE":
 		// ../rfc/9051:6778
 		c.xspace()

imapclient/protocol.go

@@ -31,6 +31,8 @@ const (
 	CapUTF8Only       Capability = "UTF8=ONLY"
 	CapUTF8Accept     Capability = "UTF8=ACCEPT"
 	CapID             Capability = "ID"              // ../rfc/2971:80
+	CapMetadata       Capability = "METADATA"        // ../rfc/5464:124
+	CapMetadataServer Capability = "METADATA-SERVER" // ../rfc/5464:124
 )
 
 // Status is the tagged final result of a command.
@@ -227,6 +229,27 @@ type UntaggedStatus struct {
 	Attrs   map[StatusAttr]int64 // Upper case status attributes.
 }
 
+// ../rfc/5464:716 Unsolicited response, indicating an annotation has changed.
+type UntaggedMetadataKeys struct {
+	Mailbox string // Empty means not specific to mailbox.
+
+	// Keys that have changed. To get values (or determine absence), the server must be
+	// queried.
+	Keys []string
+}
+
+type Annotation struct {
+	Key      string
+	IsString bool
+	Value    []byte
+}
+
+// ../rfc/5464:683
+type UntaggedMetadataAnnotations struct {
+	Mailbox     string // Empty means not specific to mailbox.
+	Annotations []Annotation
+}
+
 // ../rfc/9051:7059 ../9208:712
 type StatusAttr string
 

imapserver/fetch.go

@@ -681,7 +681,7 @@ func (cmd *fetchCmd) xbody(a fetchAtt) (string, token) {
 				count = m.Size - offset
 			}
 		}
-		return respField, readerSizeSyncliteral{&moxio.AtReader{R: msgr, Offset: offset}, count}
+		return respField, readerSizeSyncliteral{&moxio.AtReader{R: msgr, Offset: offset}, count, false}
 	}
 
 	sr := cmd.xsection(a.section, part)

imapserver/metadata.go

@@ -0,0 +1,301 @@
+package imapserver
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+
+	"github.com/mjl-/bstore"
+
+	"github.com/mjl-/mox/store"
+)
+
+// Changed during tests.
+var metadataMaxKeys = 1000
+var metadataMaxSize = 1000 * 1000
+
+// Metadata errata:
+// ../rfc/5464:183 ../rfc/5464-eid1691
+// ../rfc/5464:564 ../rfc/5464-eid1692
+// ../rfc/5464:494 ../rfc/5464-eid2785 ../rfc/5464-eid2786
+// ../rfc/5464:698 ../rfc/5464-eid3868
+
+// Note: We do not tie the special-use mailbox flags to a (synthetic) private
+// per-mailbox annotation. ../rfc/6154:303
+
+// For registration of names, see https://www.iana.org/assignments/imap-metadata/imap-metadata.xhtml
+
+// Get metadata annotations, per mailbox or globally.
+//
+// State: Authenticated and selected.
+func (c *conn) cmdGetmetadata(tag, cmd string, p *parser) {
+	// Command: ../rfc/5464:412
+
+	// Request syntax: ../rfc/5464:792
+
+	p.xspace()
+	var optMaxSize int64 = -1
+	var optDepth string
+	if p.take("(") {
+		for {
+			if p.take("MAXSIZE") {
+				// ../rfc/5464:804
+				p.xspace()
+				v := p.xnumber()
+				if optMaxSize >= 0 {
+					p.xerrorf("only a single maxsize option accepted")
+				}
+				optMaxSize = int64(v)
+			} else if p.take("DEPTH") {
+				// ../rfc/5464:823
+				p.xspace()
+				s := p.xtakelist("0", "1", "INFINITY")
+				if optDepth != "" {
+					p.xerrorf("only single depth option accepted")
+				}
+				optDepth = s
+			} else {
+				// ../rfc/5464:800 We are not doing anything further parsing for future extensions.
+				p.xerrorf("unknown option for getmetadata, expected maxsize or depth")
+			}
+
+			if p.take(")") {
+				break
+			}
+			p.xspace()
+		}
+		p.xspace()
+	}
+	mailboxName := p.xmailbox()
+	if mailboxName != "" {
+		mailboxName = xcheckmailboxname(mailboxName, true)
+	}
+	p.xspace()
+	// Entries ../rfc/5464:768
+	entryNames := map[string]struct{}{}
+	if p.take("(") {
+		for {
+			s := p.xmetadataKey()
+			entryNames[s] = struct{}{}
+			if p.take(")") {
+				break
+			}
+			p.xtake(" ")
+		}
+	} else {
+		s := p.xmetadataKey()
+		entryNames[s] = struct{}{}
+	}
+	p.xempty()
+
+	var annotations []store.Annotation
+	longentries := -1 // Size of largest value skipped due to optMaxSize. ../rfc/5464:482
+
+	c.account.WithRLock(func() {
+		c.xdbread(func(tx *bstore.Tx) {
+			q := bstore.QueryTx[store.Annotation](tx)
+			if mailboxName == "" {
+				q.FilterEqual("MailboxID", 0)
+			} else {
+				mb := c.xmailbox(tx, mailboxName, "TRYCREATE")
+				q.FilterNonzero(store.Annotation{MailboxID: mb.ID})
+			}
+
+			err := q.ForEach(func(a store.Annotation) error {
+				// ../rfc/5464:516
+				switch optDepth {
+				case "", "0":
+					if _, ok := entryNames[a.Key]; !ok {
+						return nil
+					}
+				case "1", "INFINITY":
+					// Go through all keys, matching depth.
+					if _, ok := entryNames[a.Key]; ok {
+						break
+					}
+					var match bool
+					for s := range entryNames {
+						prefix := s
+						if s != "/" {
+							prefix += "/"
+						}
+						if !strings.HasPrefix(a.Key, prefix) {
+							continue
+						}
+						if optDepth == "INFINITY" {
+							match = true
+							break
+						}
+						suffix := a.Key[len(prefix):]
+						t := strings.SplitN(suffix, "/", 2)
+						if len(t) == 1 {
+							match = true
+							break
+						}
+					}
+					if !match {
+						return nil
+					}
+				default:
+					xcheckf(fmt.Errorf("%q", optDepth), "missing case for depth")
+				}
+
+				if optMaxSize >= 0 && int64(len(a.Value)) > optMaxSize {
+					longentries = max(longentries, len(a.Value))
+				} else {
+					annotations = append(annotations, a)
+				}
+				return nil
+			})
+			xcheckf(err, "looking up annotations")
+		})
+	})
+
+	// Response syntax: ../rfc/5464:807 ../rfc/5464:778
+	// We can only send untagged responses when we have any matches.
+	if len(annotations) > 0 {
+		fmt.Fprintf(c.bw, "* METADATA %s (", astring(mailboxName).pack(c))
+		for i, a := range annotations {
+			if i > 0 {
+				fmt.Fprint(c.bw, " ")
+			}
+			astring(a.Key).writeTo(c, c.bw)
+			fmt.Fprint(c.bw, " ")
+			if a.IsString {
+				string0(string(a.Value)).writeTo(c, c.bw)
+			} else {
+				v := readerSizeSyncliteral{bytes.NewReader(a.Value), int64(len(a.Value)), true}
+				v.writeTo(c, c.bw)
+			}
+		}
+		c.bwritelinef(")")
+	}
+
+	if longentries >= 0 {
+		c.bwritelinef("%s OK [METADATA LONGENTRIES %d] getmetadata done", tag, longentries)
+	} else {
+		c.ok(tag, cmd)
+	}
+}
+
+// Set metadata annotation, per mailbox or globally.
+//
+// We only implement private annotations, not shared annotations. We don't
+// currently have a mechanism for determining if the user should have access.
+//
+// State: Authenticated and selected.
+func (c *conn) cmdSetmetadata(tag, cmd string, p *parser) {
+	// Command: ../rfc/5464:547
+
+	// Request syntax: ../rfc/5464:826
+
+	p.xspace()
+	mailboxName := p.xmailbox()
+	// Empty name means a global (per-account) annotation, not for a mailbox.
+	if mailboxName != "" {
+		mailboxName = xcheckmailboxname(mailboxName, true)
+	}
+	p.xspace()
+	p.xtake("(")
+	var l []store.Annotation
+	for {
+		key, isString, value := p.xmetadataKeyValue()
+		l = append(l, store.Annotation{Key: key, IsString: isString, Value: value})
+		if p.take(")") {
+			break
+		}
+		p.xspace()
+	}
+	p.xempty()
+
+	// Additional checks on entry names.
+	for _, a := range l {
+		// We only allow /private/* entry names, so check early and fail if we see anything
+		// else (the only other option is /shared/* at this moment).
+		// ../rfc/5464:217
+		if !strings.HasPrefix(a.Key, "/private/") {
+			// ../rfc/5464:346
+			xuserErrorf("only /private/* entry names allowed")
+		}
+
+		// We also enforce that /private/vendor/ is followed by at least 2 elements.
+		// ../rfc/5464:234
+		if a.Key == "/private/vendor" || strings.HasPrefix(a.Key, "/private/vendor/") {
+			t := strings.SplitN(a.Key[1:], "/", 4)
+			if len(t) < 4 {
+				xuserErrorf("entry names starting with /private/vendor must have at least 4 components")
+			}
+		}
+	}
+
+	// Store the annotations, possibly removing/inserting/updating them.
+	c.account.WithWLock(func() {
+		var changes []store.Change
+
+		c.xdbwrite(func(tx *bstore.Tx) {
+			var mb store.Mailbox // mb.ID as 0 is used in query below.
+			if mailboxName != "" {
+				mb = c.xmailbox(tx, mailboxName, "TRYCREATE")
+			}
+
+			for _, a := range l {
+				q := bstore.QueryTx[store.Annotation](tx)
+				q.FilterNonzero(store.Annotation{Key: a.Key})
+				q.FilterEqual("MailboxID", mb.ID) // Can be zero.
+
+				// Nil means remove. ../rfc/5464:579
+				if a.Value == nil {
+					var deleted []store.Annotation
+					q.Gather(&deleted)
+					_, err := q.Delete()
+					xcheckf(err, "deleting annotation")
+					for _, oa := range deleted {
+						changes = append(changes, oa.Change(mailboxName))
+					}
+					continue
+				}
+
+				a.MailboxID = mb.ID
+
+				oa, err := q.Get()
+				if err == bstore.ErrAbsent {
+					err = tx.Insert(&a)
+					xcheckf(err, "inserting annotation")
+					changes = append(changes, a.Change(mailboxName))
+					continue
+				}
+				xcheckf(err, "looking up existing annotation for entry name")
+				if oa.IsString != a.IsString || (oa.Value == nil) != (a.Value == nil) || !bytes.Equal(oa.Value, a.Value) {
+					changes = append(changes, a.Change(mailboxName))
+				}
+				oa.Value = a.Value
+				err = tx.Update(&oa)
+				xcheckf(err, "updating metadata annotation")
+			}
+
+			// Check for total size. We allow a total of 1000 entries, with total capacity of 1MB.
+			// ../rfc/5464:383
+			var n int
+			var size int
+			err := bstore.QueryTx[store.Annotation](tx).ForEach(func(a store.Annotation) error {
+				n++
+				if n > metadataMaxKeys {
+					// ../rfc/5464:590
+					xusercodeErrorf("METADATA TOOMANY", "too many metadata entries, 1000 allowed in total")
+				}
+				size += len(a.Key) + len(a.Value)
+				if size > metadataMaxSize {
+					// ../rfc/5464:585 We only have a max total size limit, not per entry. We'll
+					// mention the max total size.
+					xusercodeErrorf(fmt.Sprintf("METADATA MAXSIZE %d", metadataMaxSize), "metadata entry values too large, total maximum size is 1MB")
+				}
+				return nil
+			})
+			xcheckf(err, "checking metadata annotation size")
+		})
+
+		c.broadcast(changes)
+	})
+
+	c.ok(tag, cmd)
+}

imapserver/metadata_test.go

@@ -0,0 +1,276 @@
+package imapserver
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/mjl-/mox/imapclient"
+)
+
+func TestMetadata(t *testing.T) {
+	tc := start(t)
+	defer tc.close()
+
+	tc.client.Login("mjl@mox.example", password0)
+
+	tc.transactf("ok", `getmetadata "" /private/comment`)
+	tc.xuntagged()
+
+	tc.transactf("ok", `getmetadata inbox (/private/comment)`)
+	tc.xuntagged()
+
+	tc.transactf("ok", `setmetadata "" (/PRIVATE/COMMENT "global value")`)
+	tc.transactf("ok", `setmetadata inbox (/private/comment "mailbox value")`)
+
+	tc.transactf("ok", `getmetadata "" ("/private/comment")`)
+	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
+		Mailbox: "",
+		Annotations: []imapclient.Annotation{
+			{Key: "/private/comment", IsString: true, Value: []byte("global value")},
+		},
+	})
+
+	tc.transactf("ok", `getmetadata inbox (/private/comment /private/unknown /shared/comment)`)
+	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
+		Mailbox: "Inbox",
+		Annotations: []imapclient.Annotation{
+			{Key: "/private/comment", IsString: true, Value: []byte("mailbox value")},
+		},
+	})
+
+	tc.transactf("no", `setmetadata doesnotexist (/private/comment "test")`) // Bad mailbox.
+	tc.transactf("no", `setmetadata Inbox (/shared/comment "")`)             // /shared/ not implemented.
+	tc.transactf("no", `setmetadata Inbox (/badprefix/comment "")`)
+	tc.transactf("no", `setmetadata Inbox (/private/vendor "")`)          // /*/vendor must have more components.
+	tc.transactf("no", `setmetadata Inbox (/private/vendor/stillbad "")`) // /*/vendor must have more components.
+	tc.transactf("ok", `setmetadata Inbox (/private/vendor/a/b "")`)
+	tc.transactf("bad", `setmetadata Inbox (/private/no* "")`)
+	tc.transactf("bad", `setmetadata Inbox (/private/no%% "")`)
+	tc.transactf("bad", `setmetadata Inbox (/private/notrailingslash/ "")`)
+	tc.transactf("bad", `setmetadata Inbox (/private//nodupslash "")`)
+	tc.transactf("bad", "setmetadata Inbox (/private/\001 \"\")")
+	tc.transactf("bad", "setmetadata Inbox (/private/\u007f \"\")")
+	tc.transactf("bad", `getmetadata (depth 0 depth 0) inbox (/private/a)`) // Duplicate option.
+	tc.transactf("bad", `getmetadata (depth badvalue) inbox (/private/a)`)
+	tc.transactf("bad", `getmetadata (maxsize invalid) inbox (/private/a)`)
+	tc.transactf("bad", `getmetadata (badoption) inbox (/private/a)`)
+
+	// Update existing annotation by key.
+	tc.transactf("ok", `setmetadata "" (/PRIVATE/COMMENT "global updated")`)
+	tc.transactf("ok", `setmetadata inbox (/private/comment "mailbox updated")`)
+	tc.transactf("ok", `getmetadata "" (/private/comment)`)
+	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
+		Mailbox: "",
+		Annotations: []imapclient.Annotation{
+			{Key: "/private/comment", IsString: true, Value: []byte("global updated")},
+		},
+	})
+	tc.transactf("ok", `getmetadata inbox (/private/comment)`)
+	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
+		Mailbox: "Inbox",
+		Annotations: []imapclient.Annotation{
+			{Key: "/private/comment", IsString: true, Value: []byte("mailbox updated")},
+		},
+	})
+
+	// Delete annotation with nil value.
+	tc.transactf("ok", `setmetadata "" (/private/comment nil)`)
+	tc.transactf("ok", `setmetadata inbox (/private/comment nil)`)
+	tc.transactf("ok", `getmetadata "" (/private/comment)`)
+	tc.xuntagged()
+	tc.transactf("ok", `getmetadata inbox (/private/comment)`)
+	tc.xuntagged()
+
+	// Create a literal8 value, not a string.
+	tc.transactf("ok", "setmetadata inbox (/private/comment ~{4+}\r\ntest)")
+	tc.transactf("ok", `getmetadata inbox (/private/comment)`)
+	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
+		Mailbox: "Inbox",
+		Annotations: []imapclient.Annotation{
+			{Key: "/private/comment", IsString: false, Value: []byte("test")},
+		},
+	})
+
+	// Request with a maximum size, we don't get anything larger.
+	tc.transactf("ok", `setmetadata inbox (/private/another "longer")`)
+	tc.transactf("ok", `getmetadata (maxsize 4) inbox (/private/comment /private/another)`)
+	tc.xcodeArg(imapclient.CodeOther{Code: "METADATA", Args: []string{"LONGENTRIES", "6"}})
+	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
+		Mailbox: "Inbox",
+		Annotations: []imapclient.Annotation{
+			{Key: "/private/comment", IsString: false, Value: []byte("test")},
+		},
+	})
+
+	// Request with various depth values.
+	tc.transactf("ok", `setmetadata inbox (/private/a "x" /private/a/b "x" /private/a/b/c "x" /private/a/b/c/d "x")`)
+	tc.transactf("ok", `getmetadata (depth 0) inbox (/private/a)`)
+	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
+		Mailbox: "Inbox",
+		Annotations: []imapclient.Annotation{
+			{Key: "/private/a", IsString: true, Value: []byte("x")},
+		},
+	})
+	tc.transactf("ok", `getmetadata (depth 1) inbox (/private/a)`)
+	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
+		Mailbox: "Inbox",
+		Annotations: []imapclient.Annotation{
+			{Key: "/private/a", IsString: true, Value: []byte("x")},
+			{Key: "/private/a/b", IsString: true, Value: []byte("x")},
+		},
+	})
+	tc.transactf("ok", `getmetadata (depth infinity) inbox (/private/a)`)
+	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
+		Mailbox: "Inbox",
+		Annotations: []imapclient.Annotation{
+			{Key: "/private/a", IsString: true, Value: []byte("x")},
+			{Key: "/private/a/b", IsString: true, Value: []byte("x")},
+			{Key: "/private/a/b/c", IsString: true, Value: []byte("x")},
+			{Key: "/private/a/b/c/d", IsString: true, Value: []byte("x")},
+		},
+	})
+	// Same as previous, but ask for everything below /.
+	tc.transactf("ok", `getmetadata (depth infinity) inbox (/)`)
+	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
+		Mailbox: "Inbox",
+		Annotations: []imapclient.Annotation{
+			{Key: "/private/a", IsString: true, Value: []byte("x")},
+			{Key: "/private/a/b", IsString: true, Value: []byte("x")},
+			{Key: "/private/a/b/c", IsString: true, Value: []byte("x")},
+			{Key: "/private/a/b/c/d", IsString: true, Value: []byte("x")},
+			{Key: "/private/another", IsString: true, Value: []byte("longer")},
+			{Key: "/private/comment", IsString: false, Value: []byte("test")},
+			{Key: "/private/vendor/a/b", IsString: true, Value: []byte("")},
+		},
+	})
+
+	// Deleting a mailbox with an annotation should work and annotations should not
+	// come back when recreating mailbox.
+	tc.transactf("ok", "create testbox")
+	tc.transactf("ok", `setmetadata testbox (/private/a "x")`)
+	tc.transactf("ok", "delete testbox")
+	tc.transactf("ok", "create testbox")
+	tc.transactf("ok", `getmetadata testbox (/private/a)`)
+	tc.xuntagged()
+
+	// When renaming mailbox, annotations must be copied to destination mailbox.
+	tc.transactf("ok", "rename inbox newbox")
+	tc.transactf("ok", `getmetadata newbox (/private/a)`)
+	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
+		Mailbox: "newbox",
+		Annotations: []imapclient.Annotation{
+			{Key: "/private/a", IsString: true, Value: []byte("x")},
+		},
+	})
+	tc.transactf("ok", `getmetadata inbox (/private/a)`)
+	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
+		Mailbox: "Inbox",
+		Annotations: []imapclient.Annotation{
+			{Key: "/private/a", IsString: true, Value: []byte("x")},
+		},
+	})
+
+	// Broadcast should not happen when metadata capability is not enabled.
+	tc2 := startNoSwitchboard(t)
+	defer tc2.close()
+	tc2.client.Login("mjl@mox.example", password0)
+	tc2.client.Select("inbox")
+
+	tc2.cmdf("", "idle")
+	tc2.readprefixline("+ ")
+	done := make(chan error)
+	go func() {
+		defer func() {
+			x := recover()
+			if x != nil {
+				done <- fmt.Errorf("%v", x)
+			}
+		}()
+		untagged, _ := tc2.client.ReadUntagged()
+		var exists imapclient.UntaggedExists
+		tuntagged(tc2.t, untagged, &exists)
+		tc2.writelinef("done")
+		tc2.response("ok")
+		done <- nil
+	}()
+
+	// Should not cause idle to return.
+	tc.transactf("ok", `setmetadata inbox (/private/a "y")`)
+	// Cause to return.
+	tc.transactf("ok", "append inbox {4+}\r\ntest")
+
+	timer := time.NewTimer(time.Second)
+	defer timer.Stop()
+	select {
+	case err := <-done:
+		tc.check(err, "idle")
+	case <-timer.C:
+		t.Fatalf("idle did not finish")
+	}
+
+	// Broadcast should happen when metadata capability is enabled.
+	tc2.client.Enable(string(imapclient.CapMetadata))
+	tc2.cmdf("", "idle")
+	tc2.readprefixline("+ ")
+	done = make(chan error)
+	go func() {
+		defer func() {
+			x := recover()
+			if x != nil {
+				done <- fmt.Errorf("%v", x)
+			}
+		}()
+		untagged, _ := tc2.client.ReadUntagged()
+		var metadataKeys imapclient.UntaggedMetadataKeys
+		tuntagged(tc2.t, untagged, &metadataKeys)
+		tc2.writelinef("done")
+		tc2.response("ok")
+		done <- nil
+	}()
+
+	// Should cause idle to return.
+	tc.transactf("ok", `setmetadata inbox (/private/a "z")`)
+
+	timer = time.NewTimer(time.Second)
+	defer timer.Stop()
+	select {
+	case err := <-done:
+		tc.check(err, "idle")
+	case <-timer.C:
+		t.Fatalf("idle did not finish")
+	}
+}
+
+func TestMetadataLimit(t *testing.T) {
+	tc := start(t)
+	defer tc.close()
+
+	tc.client.Login("mjl@mox.example", password0)
+
+	maxKeys, maxSize := metadataMaxKeys, metadataMaxSize
+	defer func() {
+		metadataMaxKeys = maxKeys
+		metadataMaxSize = maxSize
+	}()
+	metadataMaxKeys = 10
+	metadataMaxSize = 1000
+
+	// Reach max total size limit.
+	buf := make([]byte, metadataMaxSize+1)
+	for i := range buf {
+		buf[i] = 'x'
+	}
+	tc.cmdf("", "setmetadata inbox (/private/large ~{%d+}", len(buf))
+	tc.client.Write(buf)
+	tc.client.Writelinef(")")
+	tc.response("no")
+	tc.xcodeArg(imapclient.CodeOther{Code: "METADATA", Args: []string{"MAXSIZE", fmt.Sprintf("%d", metadataMaxSize)}})
+
+	// Reach limit for max number.
+	for i := 1; i <= metadataMaxKeys; i++ {
+		tc.transactf("ok", `setmetadata inbox (/private/key%d "test")`, i)
+	}
+	tc.transactf("no", `setmetadata inbox (/private/toomany "test")`)
+	tc.xcodeArg(imapclient.CodeOther{Code: "METADATA", Args: []string{"TOOMANY"}})
+}

imapserver/pack.go

@@ -97,6 +97,7 @@ func (t syncliteral) writeTo(c *conn, w io.Writer) {
 type readerSizeSyncliteral struct {
 	r    io.Reader
 	size int64
+	lit8 bool
 }
 
 func (t readerSizeSyncliteral) pack(c *conn) string {
@@ -104,11 +105,19 @@ func (t readerSizeSyncliteral) pack(c *conn) string {
 	if err != nil {
 		panic(err)
 	}
-	return fmt.Sprintf("{%d}\r\n", t.size) + string(buf)
+	var lit string
+	if t.lit8 {
+		lit = "~"
+	}
+	return fmt.Sprintf("%s{%d}\r\n", lit, t.size) + string(buf)
 }
 
 func (t readerSizeSyncliteral) writeTo(c *conn, w io.Writer) {
-	fmt.Fprintf(w, "{%d}\r\n", t.size)
+	var lit string
+	if t.lit8 {
+		lit = "~"
+	}
+	fmt.Fprintf(w, "%s{%d}\r\n", lit, t.size)
 	defer c.xtrace(mlog.LevelTracedata)()
 	if _, err := io.Copy(w, io.LimitReader(t.r, t.size)); err != nil {
 		panic(err)

imapserver/parse.go

@@ -305,10 +305,10 @@ func (p *parser) xstring() (r string) {
 		p.xerrorf("missing closing dquote in string")
 	}
 	size, sync := p.xliteralSize(false, true)
-	s := p.conn.xreadliteral(size, sync)
+	buf := p.conn.xreadliteral(size, sync)
 	line := p.conn.readline(false)
 	p.orig, p.upper, p.o = line, toUpper(line), 0
-	return s
+	return string(buf)
 }
 
 func (p *parser) xnil() {
@@ -974,3 +974,53 @@ func (p *parser) xdate() time.Time {
 	}
 	return time.Date(year, mon, day, 0, 0, 0, 0, time.UTC)
 }
+
+// Parse and validate a metadata key (entry name), returned as lower-case.
+//
+// ../rfc/5464:190
+func (p *parser) xmetadataKey() string {
+	// ../rfc/5464:772
+	s := p.xastring()
+
+	// ../rfc/5464:192
+	if strings.Contains(s, "//") {
+		p.xerrorf("entry name must not contain two slashes")
+	}
+	// We allow a single slash, so it can be used with option "(depth infinity)" to get
+	// all annotations.
+	if s != "/" && strings.HasSuffix(s, "/") {
+		p.xerrorf("entry name must not end with slash")
+	}
+	// ../rfc/5464:202
+	if strings.Contains(s, "*") || strings.Contains(s, "%") {
+		p.xerrorf("entry name must not contain * or %%")
+	}
+	for _, c := range s {
+		if c < ' ' || c >= 0x7f {
+			p.xerrorf("entry name must only contain non-control ascii characters")
+		}
+	}
+	return strings.ToLower(s)
+}
+
+// ../rfc/5464:776
+func (p *parser) xmetadataKeyValue() (key string, isString bool, value []byte) {
+	key = p.xmetadataKey()
+	p.xspace()
+
+	if p.hasPrefix("~{") {
+		size, sync := p.xliteralSize(true, true)
+		value = p.conn.xreadliteral(size, sync)
+		line := p.conn.readline(false)
+		p.orig, p.upper, p.o = line, toUpper(line), 0
+	} else if p.hasPrefix(`"`) {
+		value = []byte(p.xstring())
+		isString = true
+	} else if p.take("NIL") {
+		value = nil
+	} else {
+		p.xerrorf("expected metadata value")
+	}
+
+	return
+}

imapserver/server.go

@@ -158,12 +158,13 @@ var authFailDelay = time.Second  // After authentication failure.
 // QRESYNC: ../rfc/7162:1323
 // STATUS=SIZE: ../rfc/8438 ../rfc/9051:8024
 // QUOTA QUOTA=RES-STORAGE: ../rfc/9208:111
+// METADATA: ../rfc/5464
 //
 // We always announce support for SCRAM PLUS-variants, also on connections without
 // TLS. The client should not be selecting PLUS variants on non-TLS connections,
 // instead opting to do the bare SCRAM variant without indicating the server claims
 // to support the PLUS variant (skipping the server downgrade detection check).
-const serverCapabilities = "IMAP4rev2 IMAP4rev1 ENABLE LITERAL+ IDLE SASL-IR BINARY UNSELECT UIDPLUS ESEARCH SEARCHRES MOVE UTF8=ACCEPT LIST-EXTENDED SPECIAL-USE LIST-STATUS AUTH=SCRAM-SHA-256-PLUS AUTH=SCRAM-SHA-256 AUTH=SCRAM-SHA-1-PLUS AUTH=SCRAM-SHA-1 AUTH=CRAM-MD5 ID APPENDLIMIT=9223372036854775807 CONDSTORE QRESYNC STATUS=SIZE QUOTA QUOTA=RES-STORAGE"
+const serverCapabilities = "IMAP4rev2 IMAP4rev1 ENABLE LITERAL+ IDLE SASL-IR BINARY UNSELECT UIDPLUS ESEARCH SEARCHRES MOVE UTF8=ACCEPT LIST-EXTENDED SPECIAL-USE LIST-STATUS AUTH=SCRAM-SHA-256-PLUS AUTH=SCRAM-SHA-256 AUTH=SCRAM-SHA-1-PLUS AUTH=SCRAM-SHA-1 AUTH=CRAM-MD5 ID APPENDLIMIT=9223372036854775807 CONDSTORE QRESYNC STATUS=SIZE QUOTA QUOTA=RES-STORAGE METADATA"
 
 type conn struct {
 	cid               int64
@@ -227,6 +228,7 @@ const (
 	capUTF8Accept capability = "UTF8=ACCEPT"
 	capCondstore  capability = "CONDSTORE"
 	capQresync    capability = "QRESYNC"
+	capMetadata   capability = "METADATA"
 )
 
 type lineErr struct {
@@ -253,7 +255,7 @@ func stateCommands(cmds ...string) map[string]struct{} {
 var (
 	commandsStateAny              = stateCommands("capability", "noop", "logout", "id")
 	commandsStateNotAuthenticated = stateCommands("starttls", "authenticate", "login")
-	commandsStateAuthenticated    = stateCommands("enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub", "getquotaroot", "getquota")
+	commandsStateAuthenticated    = stateCommands("enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub", "getquotaroot", "getquota", "getmetadata", "setmetadata")
 	commandsStateSelected         = stateCommands("close", "unselect", "expunge", "search", "fetch", "store", "copy", "move", "uid expunge", "uid search", "uid fetch", "uid store", "uid copy", "uid move")
 )
 
@@ -286,6 +288,8 @@ var commands = map[string]func(c *conn, tag, cmd string, p *parser){
 	"idle":         (*conn).cmdIdle,
 	"getquotaroot": (*conn).cmdGetquotaroot,
 	"getquota":     (*conn).cmdGetquota,
+	"getmetadata":  (*conn).cmdGetmetadata,
+	"setmetadata":  (*conn).cmdSetmetadata,
 
 	// Selected.
 	"check":       (*conn).cmdCheck,
@@ -617,7 +621,7 @@ func (c *conn) readCommand(tag *string) (cmd string, p *parser) {
 	return cmd, newParser(p.remainder(), c)
 }
 
-func (c *conn) xreadliteral(size int64, sync bool) string {
+func (c *conn) xreadliteral(size int64, sync bool) []byte {
 	if sync {
 		c.writelinef("+ ")
 	}
@@ -633,7 +637,7 @@ func (c *conn) xreadliteral(size int64, sync bool) string {
 			panic(fmt.Errorf("reading literal: %s (%w)", err, errIO))
 		}
 	}
-	return string(buf)
+	return buf
 }
 
 func (c *conn) xhighestModSeq(tx *bstore.Tx, mailboxID int64) store.ModSeq {
@@ -1541,6 +1545,14 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) {
 		case store.ChangeRemoveMailbox, store.ChangeAddMailbox, store.ChangeRenameMailbox, store.ChangeAddSubscription:
 			n = append(n, change)
 			continue
+		case store.ChangeAnnotation:
+			// note: annotations may have a mailbox associated with them, but we pass all
+			// changes on.
+			// Only when the metadata capability was enabled. ../rfc/5464:660
+			if c.enabled[capMetadata] {
+				n = append(n, change)
+				continue
+			}
 		case store.ChangeMailboxCounts, store.ChangeMailboxSpecialUse, store.ChangeMailboxKeywords, store.ChangeThread:
 		default:
 			panic(fmt.Errorf("missing case for %#v", change))
@@ -1651,6 +1663,9 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) {
 			c.bwritelinef(`* LIST (%s) "/" %s%s`, strings.Join(ch.Flags, " "), astring(c.encodeMailbox(ch.NewName)).pack(c), oldname)
 		case store.ChangeAddSubscription:
 			c.bwritelinef(`* LIST (%s) "/" %s`, strings.Join(append([]string{`\Subscribed`}, ch.Flags...), " "), astring(c.encodeMailbox(ch.Name)).pack(c))
+		case store.ChangeAnnotation:
+			// ../rfc/5464:807 ../rfc/5464:788
+			c.bwritelinef(`* METADATA %s %s`, astring(c.encodeMailbox(ch.MailboxName)).pack(c), astring(ch.Key).pack(c))
 		default:
 			panic(fmt.Sprintf("internal error, missing case for %#v", change))
 		}
@@ -2325,6 +2340,9 @@ func (c *conn) cmdEnable(tag, cmd string, p *parser) {
 			c.enabled[cap] = true
 			enabled += " " + s
 			qresync = true
+		case capMetadata:
+			c.enabled[cap] = true
+			enabled += " " + s
 		}
 	}
 	// QRESYNC enabled CONDSTORE too ../rfc/7162:1391
@@ -2713,7 +2731,7 @@ func (c *conn) cmdCreate(tag, cmd string, p *parser) {
 	c.ok(tag, cmd)
 }
 
-// Delete removes a mailbox and all its messages.
+// Delete removes a mailbox and all its messages and annotations.
 // Inbox cannot be removed.
 //
 // State: Authenticated and selected.
@@ -2760,7 +2778,8 @@ func (c *conn) cmdDelete(tag, cmd string, p *parser) {
 }
 
 // Rename changes the name of a mailbox.
-// Renaming INBOX is special, it moves the inbox messages to a new mailbox, leaving inbox empty.
+// Renaming INBOX is special, it moves the inbox messages to a new mailbox, leaving
+// inbox empty, but copying metadata annotations.
 // Renaming a mailbox with submailboxes also renames all submailboxes.
 // Subscriptions stay with the old name, though newly created missing parent
 // mailboxes for the destination name are automatically subscribed.
@@ -2865,10 +2884,25 @@ func (c *conn) cmdRename(tag, cmd string, p *parser) {
 				if tx.Get(&store.Subscription{Name: dstMB.Name}) == nil {
 					dstFlags = []string{`\Subscribed`}
 				}
+
+				// Copy any annotations. ../rfc/5464:368
+				annotations, err := bstore.QueryTx[store.Annotation](tx).FilterNonzero(store.Annotation{MailboxID: srcMB.ID}).List()
+				xcheckf(err, "get annotations to copy for inbox")
+				for i := range annotations {
+					annotations[i].ID = 0
+					annotations[i].MailboxID = dstMB.ID
+					err := tx.Insert(&annotations[i])
+					xcheckf(err, "copy annotation to destination mailbox")
+				}
+
 				changes[0] = store.ChangeRemoveUIDs{MailboxID: srcMB.ID, UIDs: oldUIDs, ModSeq: modseq}
 				changes[1] = store.ChangeAddMailbox{Mailbox: dstMB, Flags: dstFlags}
 				// changes[2:...] are ChangeAddUIDs
 				changes = append(changes, srcMB.ChangeCounts(), dstMB.ChangeCounts())
+				for _, a := range annotations {
+					changes = append(changes, a.Change(dstMB.Name))
+				}
+
 				return
 			}
 

rfc/index.txt

@@ -201,7 +201,12 @@ https://www.iana.org/assignments/message-headers/message-headers.xhtml
 5258	Yes	-	Internet Message Access Protocol version 4 - LIST Command Extensions
 5259	No	-	Internet Message Access Protocol - CONVERT Extension
 5267	Roadmap	-	Contexts for IMAP4
-5464	Roadmap	-	The IMAP METADATA Extension
+5464	Yes	-	The IMAP METADATA Extension
+5464-eid1691	-	-	errata: fix example entry name
+5464-eid1692	-	-	errata: make text match abnf
+5464-eid2785	-	-	errata: fix GETMETADATA example
+5464-eid2786	-	-	errata: fix GETMETADATA example
+5464-eid3868	-	-	errata: fix GETMETADATA example
 5465	Roadmap	-	The IMAP NOTIFY Extension
 5466	Roadmap	-	IMAP4 Extension for Named Searches (Filters)
 5524	No	-	Extended URLFETCH for Binary and Converted Parts

store/account.go

@@ -216,6 +216,27 @@ type Mailbox struct {
 	MailboxCounts      // Statistics about messages, kept up to date whenever a change happens.
 }
 
+// Annotation is a per-mailbox or global (per-account) annotation for the IMAP
+// metadata extension, currently always a private annotation.
+type Annotation struct {
+	ID int64
+
+	// Can be zero, indicates global (per-account) annotation.
+	MailboxID int64 `bstore:"ref Mailbox,unique MailboxID+Key"`
+
+	// "Entry name", always starts with "/private/". Stored lower-case, comparisons
+	// must be done case-insensitively.
+	Key string `bstore:"nonzero"`
+
+	IsString bool // If true, the value is a string instead of bytes.
+	Value    []byte
+}
+
+// Change returns a broadcastable change for the annotation.
+func (a Annotation) Change(mailboxName string) ChangeAnnotation {
+	return ChangeAnnotation{a.MailboxID, mailboxName, a.Key}
+}
+
 // MailboxCounts tracks statistics about messages for a mailbox.
 type MailboxCounts struct {
 	Total   int64 // Total number of messages, excluding \Deleted. For JMAP.
@@ -818,6 +839,7 @@ var DBTypes = []any{
 	RulesetNoListID{},
 	RulesetNoMsgFrom{},
 	RulesetNoMailbox{},
+	Annotation{},
 }
 
 // Account holds the information about a user, includings mailboxes, messages, imap subscriptions.
@@ -2713,8 +2735,8 @@ func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc Mailbox, dst string) (chang
 	return changes, false, false, false, nil
 }
 
-// MailboxDelete deletes a mailbox by ID. If it has children, the return value
+// MailboxDelete deletes a mailbox by ID, including its annotations. If it has
-// indicates that and an error is returned.
+// children, the return value indicates that and an error is returned.
 //
 // Caller should broadcast the changes and remove files for the removed message IDs.
 func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) {
@@ -2785,6 +2807,13 @@ func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx
 		}
 	}
 
+	// Remove metadata annotations. ../rfc/5464:373
+	if _, err := bstore.QueryTx[Annotation](tx).FilterNonzero(Annotation{MailboxID: mailbox.ID}).Delete(); err != nil {
+		return nil, nil, false, fmt.Errorf("removing annotations for mailbox: %v", err)
+	}
+	// Not sending changes about annotations on this mailbox, since the entire mailbox
+	// is being removed.
+
 	if err := tx.Delete(&Mailbox{ID: mailbox.ID}); err != nil {
 		return nil, nil, false, fmt.Errorf("removing mailbox: %v", err)
 	}

store/state.go

@@ -105,6 +105,14 @@ type ChangeMailboxKeywords struct {
 	Keywords    []string
 }
 
+// ChangeAnnotation is sent when an annotation is added/updated/removed, either for
+// a mailbox or a global per-account annotation. The value is not included.
+type ChangeAnnotation struct {
+	MailboxID   int64  // Can be zero, meaning global (per-account) annotation.
+	MailboxName string // Empty for global (per-account) annotation.
+	Key         string // Also called "entry name", e.g. "/private/comment".
+}
+
 var switchboardBusy atomic.Bool
 
 // Switchboard distributes changes to accounts to interested listeners. See Comm and Change.

webmail/api.go

@@ -1230,7 +1230,7 @@ func (Webmail) MailboxCreate(ctx context.Context, name string) {
 	})
 }
 
-// MailboxDelete deletes a mailbox and all its messages.
+// MailboxDelete deletes a mailbox and all its messages and annotations.
 func (Webmail) MailboxDelete(ctx context.Context, mailboxID int64) {
 	reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)
 	acc := reqInfo.Account

webmail/api.json

@@ -276,7 +276,7 @@
 		},
 		{
 			"Name": "MailboxDelete",
-			"Docs": "MailboxDelete deletes a mailbox and all its messages.",
+			"Docs": "MailboxDelete deletes a mailbox and all its messages and annotations.",
 			"Params": [
 				{
 					"Name": "mailboxID",

webmail/api.ts

@@ -894,7 +894,7 @@ export class Client {
 		return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params) as void
 	}
 
-	// MailboxDelete deletes a mailbox and all its messages.
+	// MailboxDelete deletes a mailbox and all its messages and annotations.
 	async MailboxDelete(mailboxID: number): Promise<void> {
 		const fn: string = "MailboxDelete"
 		const paramTypes: string[][] = [["int64"]]

webmail/msg.js

@@ -573,7 +573,7 @@ var api;
 			const params = [name];
 			return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params);
 		}
-		// MailboxDelete deletes a mailbox and all its messages.
+		// MailboxDelete deletes a mailbox and all its messages and annotations.
 		async MailboxDelete(mailboxID) {
 			const fn = "MailboxDelete";
 			const paramTypes = [["int64"]];

webmail/text.js

@@ -573,7 +573,7 @@ var api;
 			const params = [name];
 			return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params);
 		}
-		// MailboxDelete deletes a mailbox and all its messages.
+		// MailboxDelete deletes a mailbox and all its messages and annotations.
 		async MailboxDelete(mailboxID) {
 			const fn = "MailboxDelete";
 			const paramTypes = [["int64"]];

webmail/webmail.js

@@ -573,7 +573,7 @@ var api;
 			const params = [name];
 			return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params);
 		}
-		// MailboxDelete deletes a mailbox and all its messages.
+		// MailboxDelete deletes a mailbox and all its messages and annotations.
 		async MailboxDelete(mailboxID) {
 			const fn = "MailboxDelete";
 			const paramTypes = [["int64"]];