diff --git a/imapclient/parse.go b/imapclient/parse.go index f755e7c..9f11ed8 100644 --- a/imapclient/parse.go +++ b/imapclient/parse.go @@ -427,6 +427,50 @@ func (c *Conn) xuntagged() Untagged { c.xcrlf() return r + case "METADATA": + // ../rfc/5464:807 + c.xspace() + mailbox := c.xastring() + c.xspace() + if !c.take('(') { + // Unsolicited form, with only annotation keys, not values. + var keys []string + for { + key := c.xastring() + keys = append(keys, key) + if !c.space() { + break + } + } + c.xcrlf() + return UntaggedMetadataKeys{mailbox, keys} + } + + // Form with values, in response to GETMETADATA command. + r := UntaggedMetadataAnnotations{Mailbox: mailbox} + for { + key := c.xastring() + c.xspace() + var value []byte + var isString bool + if c.take('~') { + value = c.xliteral() + } else { + value = []byte(c.xstring()) + isString = true + // note: the abnf also allows nstring, but that only makes sense when the + // production rule is used in the setmetadata command. ../rfc/5464:831 + } + r.Annotations = append(r.Annotations, Annotation{key, isString, value}) + + if c.take(')') { + break + } + c.xspace() + } + c.xcrlf() + return r + case "NAMESPACE": // ../rfc/9051:6778 c.xspace() diff --git a/imapclient/protocol.go b/imapclient/protocol.go index 1bf819b..56a5942 100644 --- a/imapclient/protocol.go +++ b/imapclient/protocol.go @@ -10,27 +10,29 @@ import ( type Capability string const ( - CapIMAP4rev1 Capability = "IMAP4rev1" - CapIMAP4rev2 Capability = "IMAP4rev2" - CapLoginDisabled Capability = "LOGINDISABLED" - CapStarttls Capability = "STARTTLS" - CapAuthPlain Capability = "AUTH=PLAIN" - CapLiteralPlus Capability = "LITERAL+" - CapLiteralMinus Capability = "LITERAL-" - CapIdle Capability = "IDLE" - CapNamespace Capability = "NAMESPACE" - CapBinary Capability = "BINARY" - CapUnselect Capability = "UNSELECT" - CapUidplus Capability = "UIDPLUS" - CapEsearch Capability = "ESEARCH" - CapEnable Capability = "ENABLE" - CapSave Capability = "SAVE" - CapListExtended Capability = "LIST-EXTENDED" - CapSpecialUse Capability = "SPECIAL-USE" - CapMove Capability = "MOVE" - CapUTF8Only Capability = "UTF8=ONLY" - CapUTF8Accept Capability = "UTF8=ACCEPT" - CapID Capability = "ID" // ../rfc/2971:80 + CapIMAP4rev1 Capability = "IMAP4rev1" + CapIMAP4rev2 Capability = "IMAP4rev2" + CapLoginDisabled Capability = "LOGINDISABLED" + CapStarttls Capability = "STARTTLS" + CapAuthPlain Capability = "AUTH=PLAIN" + CapLiteralPlus Capability = "LITERAL+" + CapLiteralMinus Capability = "LITERAL-" + CapIdle Capability = "IDLE" + CapNamespace Capability = "NAMESPACE" + CapBinary Capability = "BINARY" + CapUnselect Capability = "UNSELECT" + CapUidplus Capability = "UIDPLUS" + CapEsearch Capability = "ESEARCH" + CapEnable Capability = "ENABLE" + CapSave Capability = "SAVE" + CapListExtended Capability = "LIST-EXTENDED" + CapSpecialUse Capability = "SPECIAL-USE" + CapMove Capability = "MOVE" + CapUTF8Only Capability = "UTF8=ONLY" + CapUTF8Accept Capability = "UTF8=ACCEPT" + CapID Capability = "ID" // ../rfc/2971:80 + CapMetadata Capability = "METADATA" // ../rfc/5464:124 + CapMetadataServer Capability = "METADATA-SERVER" // ../rfc/5464:124 ) // Status is the tagged final result of a command. @@ -227,6 +229,27 @@ type UntaggedStatus struct { Attrs map[StatusAttr]int64 // Upper case status attributes. } +// ../rfc/5464:716 Unsolicited response, indicating an annotation has changed. +type UntaggedMetadataKeys struct { + Mailbox string // Empty means not specific to mailbox. + + // Keys that have changed. To get values (or determine absence), the server must be + // queried. + Keys []string +} + +type Annotation struct { + Key string + IsString bool + Value []byte +} + +// ../rfc/5464:683 +type UntaggedMetadataAnnotations struct { + Mailbox string // Empty means not specific to mailbox. + Annotations []Annotation +} + // ../rfc/9051:7059 ../9208:712 type StatusAttr string diff --git a/imapserver/fetch.go b/imapserver/fetch.go index 250b7af..7113707 100644 --- a/imapserver/fetch.go +++ b/imapserver/fetch.go @@ -681,7 +681,7 @@ func (cmd *fetchCmd) xbody(a fetchAtt) (string, token) { count = m.Size - offset } } - return respField, readerSizeSyncliteral{&moxio.AtReader{R: msgr, Offset: offset}, count} + return respField, readerSizeSyncliteral{&moxio.AtReader{R: msgr, Offset: offset}, count, false} } sr := cmd.xsection(a.section, part) diff --git a/imapserver/metadata.go b/imapserver/metadata.go new file mode 100644 index 0000000..21bef86 --- /dev/null +++ b/imapserver/metadata.go @@ -0,0 +1,301 @@ +package imapserver + +import ( + "bytes" + "fmt" + "strings" + + "github.com/mjl-/bstore" + + "github.com/mjl-/mox/store" +) + +// Changed during tests. +var metadataMaxKeys = 1000 +var metadataMaxSize = 1000 * 1000 + +// Metadata errata: +// ../rfc/5464:183 ../rfc/5464-eid1691 +// ../rfc/5464:564 ../rfc/5464-eid1692 +// ../rfc/5464:494 ../rfc/5464-eid2785 ../rfc/5464-eid2786 +// ../rfc/5464:698 ../rfc/5464-eid3868 + +// Note: We do not tie the special-use mailbox flags to a (synthetic) private +// per-mailbox annotation. ../rfc/6154:303 + +// For registration of names, see https://www.iana.org/assignments/imap-metadata/imap-metadata.xhtml + +// Get metadata annotations, per mailbox or globally. +// +// State: Authenticated and selected. +func (c *conn) cmdGetmetadata(tag, cmd string, p *parser) { + // Command: ../rfc/5464:412 + + // Request syntax: ../rfc/5464:792 + + p.xspace() + var optMaxSize int64 = -1 + var optDepth string + if p.take("(") { + for { + if p.take("MAXSIZE") { + // ../rfc/5464:804 + p.xspace() + v := p.xnumber() + if optMaxSize >= 0 { + p.xerrorf("only a single maxsize option accepted") + } + optMaxSize = int64(v) + } else if p.take("DEPTH") { + // ../rfc/5464:823 + p.xspace() + s := p.xtakelist("0", "1", "INFINITY") + if optDepth != "" { + p.xerrorf("only single depth option accepted") + } + optDepth = s + } else { + // ../rfc/5464:800 We are not doing anything further parsing for future extensions. + p.xerrorf("unknown option for getmetadata, expected maxsize or depth") + } + + if p.take(")") { + break + } + p.xspace() + } + p.xspace() + } + mailboxName := p.xmailbox() + if mailboxName != "" { + mailboxName = xcheckmailboxname(mailboxName, true) + } + p.xspace() + // Entries ../rfc/5464:768 + entryNames := map[string]struct{}{} + if p.take("(") { + for { + s := p.xmetadataKey() + entryNames[s] = struct{}{} + if p.take(")") { + break + } + p.xtake(" ") + } + } else { + s := p.xmetadataKey() + entryNames[s] = struct{}{} + } + p.xempty() + + var annotations []store.Annotation + longentries := -1 // Size of largest value skipped due to optMaxSize. ../rfc/5464:482 + + c.account.WithRLock(func() { + c.xdbread(func(tx *bstore.Tx) { + q := bstore.QueryTx[store.Annotation](tx) + if mailboxName == "" { + q.FilterEqual("MailboxID", 0) + } else { + mb := c.xmailbox(tx, mailboxName, "TRYCREATE") + q.FilterNonzero(store.Annotation{MailboxID: mb.ID}) + } + + err := q.ForEach(func(a store.Annotation) error { + // ../rfc/5464:516 + switch optDepth { + case "", "0": + if _, ok := entryNames[a.Key]; !ok { + return nil + } + case "1", "INFINITY": + // Go through all keys, matching depth. + if _, ok := entryNames[a.Key]; ok { + break + } + var match bool + for s := range entryNames { + prefix := s + if s != "/" { + prefix += "/" + } + if !strings.HasPrefix(a.Key, prefix) { + continue + } + if optDepth == "INFINITY" { + match = true + break + } + suffix := a.Key[len(prefix):] + t := strings.SplitN(suffix, "/", 2) + if len(t) == 1 { + match = true + break + } + } + if !match { + return nil + } + default: + xcheckf(fmt.Errorf("%q", optDepth), "missing case for depth") + } + + if optMaxSize >= 0 && int64(len(a.Value)) > optMaxSize { + longentries = max(longentries, len(a.Value)) + } else { + annotations = append(annotations, a) + } + return nil + }) + xcheckf(err, "looking up annotations") + }) + }) + + // Response syntax: ../rfc/5464:807 ../rfc/5464:778 + // We can only send untagged responses when we have any matches. + if len(annotations) > 0 { + fmt.Fprintf(c.bw, "* METADATA %s (", astring(mailboxName).pack(c)) + for i, a := range annotations { + if i > 0 { + fmt.Fprint(c.bw, " ") + } + astring(a.Key).writeTo(c, c.bw) + fmt.Fprint(c.bw, " ") + if a.IsString { + string0(string(a.Value)).writeTo(c, c.bw) + } else { + v := readerSizeSyncliteral{bytes.NewReader(a.Value), int64(len(a.Value)), true} + v.writeTo(c, c.bw) + } + } + c.bwritelinef(")") + } + + if longentries >= 0 { + c.bwritelinef("%s OK [METADATA LONGENTRIES %d] getmetadata done", tag, longentries) + } else { + c.ok(tag, cmd) + } +} + +// Set metadata annotation, per mailbox or globally. +// +// We only implement private annotations, not shared annotations. We don't +// currently have a mechanism for determining if the user should have access. +// +// State: Authenticated and selected. +func (c *conn) cmdSetmetadata(tag, cmd string, p *parser) { + // Command: ../rfc/5464:547 + + // Request syntax: ../rfc/5464:826 + + p.xspace() + mailboxName := p.xmailbox() + // Empty name means a global (per-account) annotation, not for a mailbox. + if mailboxName != "" { + mailboxName = xcheckmailboxname(mailboxName, true) + } + p.xspace() + p.xtake("(") + var l []store.Annotation + for { + key, isString, value := p.xmetadataKeyValue() + l = append(l, store.Annotation{Key: key, IsString: isString, Value: value}) + if p.take(")") { + break + } + p.xspace() + } + p.xempty() + + // Additional checks on entry names. + for _, a := range l { + // We only allow /private/* entry names, so check early and fail if we see anything + // else (the only other option is /shared/* at this moment). + // ../rfc/5464:217 + if !strings.HasPrefix(a.Key, "/private/") { + // ../rfc/5464:346 + xuserErrorf("only /private/* entry names allowed") + } + + // We also enforce that /private/vendor/ is followed by at least 2 elements. + // ../rfc/5464:234 + if a.Key == "/private/vendor" || strings.HasPrefix(a.Key, "/private/vendor/") { + t := strings.SplitN(a.Key[1:], "/", 4) + if len(t) < 4 { + xuserErrorf("entry names starting with /private/vendor must have at least 4 components") + } + } + } + + // Store the annotations, possibly removing/inserting/updating them. + c.account.WithWLock(func() { + var changes []store.Change + + c.xdbwrite(func(tx *bstore.Tx) { + var mb store.Mailbox // mb.ID as 0 is used in query below. + if mailboxName != "" { + mb = c.xmailbox(tx, mailboxName, "TRYCREATE") + } + + for _, a := range l { + q := bstore.QueryTx[store.Annotation](tx) + q.FilterNonzero(store.Annotation{Key: a.Key}) + q.FilterEqual("MailboxID", mb.ID) // Can be zero. + + // Nil means remove. ../rfc/5464:579 + if a.Value == nil { + var deleted []store.Annotation + q.Gather(&deleted) + _, err := q.Delete() + xcheckf(err, "deleting annotation") + for _, oa := range deleted { + changes = append(changes, oa.Change(mailboxName)) + } + continue + } + + a.MailboxID = mb.ID + + oa, err := q.Get() + if err == bstore.ErrAbsent { + err = tx.Insert(&a) + xcheckf(err, "inserting annotation") + changes = append(changes, a.Change(mailboxName)) + continue + } + xcheckf(err, "looking up existing annotation for entry name") + if oa.IsString != a.IsString || (oa.Value == nil) != (a.Value == nil) || !bytes.Equal(oa.Value, a.Value) { + changes = append(changes, a.Change(mailboxName)) + } + oa.Value = a.Value + err = tx.Update(&oa) + xcheckf(err, "updating metadata annotation") + } + + // Check for total size. We allow a total of 1000 entries, with total capacity of 1MB. + // ../rfc/5464:383 + var n int + var size int + err := bstore.QueryTx[store.Annotation](tx).ForEach(func(a store.Annotation) error { + n++ + if n > metadataMaxKeys { + // ../rfc/5464:590 + xusercodeErrorf("METADATA TOOMANY", "too many metadata entries, 1000 allowed in total") + } + size += len(a.Key) + len(a.Value) + if size > metadataMaxSize { + // ../rfc/5464:585 We only have a max total size limit, not per entry. We'll + // mention the max total size. + xusercodeErrorf(fmt.Sprintf("METADATA MAXSIZE %d", metadataMaxSize), "metadata entry values too large, total maximum size is 1MB") + } + return nil + }) + xcheckf(err, "checking metadata annotation size") + }) + + c.broadcast(changes) + }) + + c.ok(tag, cmd) +} diff --git a/imapserver/metadata_test.go b/imapserver/metadata_test.go new file mode 100644 index 0000000..87c0de8 --- /dev/null +++ b/imapserver/metadata_test.go @@ -0,0 +1,276 @@ +package imapserver + +import ( + "fmt" + "testing" + "time" + + "github.com/mjl-/mox/imapclient" +) + +func TestMetadata(t *testing.T) { + tc := start(t) + defer tc.close() + + tc.client.Login("mjl@mox.example", password0) + + tc.transactf("ok", `getmetadata "" /private/comment`) + tc.xuntagged() + + tc.transactf("ok", `getmetadata inbox (/private/comment)`) + tc.xuntagged() + + tc.transactf("ok", `setmetadata "" (/PRIVATE/COMMENT "global value")`) + tc.transactf("ok", `setmetadata inbox (/private/comment "mailbox value")`) + + tc.transactf("ok", `getmetadata "" ("/private/comment")`) + tc.xuntagged(imapclient.UntaggedMetadataAnnotations{ + Mailbox: "", + Annotations: []imapclient.Annotation{ + {Key: "/private/comment", IsString: true, Value: []byte("global value")}, + }, + }) + + tc.transactf("ok", `getmetadata inbox (/private/comment /private/unknown /shared/comment)`) + tc.xuntagged(imapclient.UntaggedMetadataAnnotations{ + Mailbox: "Inbox", + Annotations: []imapclient.Annotation{ + {Key: "/private/comment", IsString: true, Value: []byte("mailbox value")}, + }, + }) + + tc.transactf("no", `setmetadata doesnotexist (/private/comment "test")`) // Bad mailbox. + tc.transactf("no", `setmetadata Inbox (/shared/comment "")`) // /shared/ not implemented. + tc.transactf("no", `setmetadata Inbox (/badprefix/comment "")`) + tc.transactf("no", `setmetadata Inbox (/private/vendor "")`) // /*/vendor must have more components. + tc.transactf("no", `setmetadata Inbox (/private/vendor/stillbad "")`) // /*/vendor must have more components. + tc.transactf("ok", `setmetadata Inbox (/private/vendor/a/b "")`) + tc.transactf("bad", `setmetadata Inbox (/private/no* "")`) + tc.transactf("bad", `setmetadata Inbox (/private/no%% "")`) + tc.transactf("bad", `setmetadata Inbox (/private/notrailingslash/ "")`) + tc.transactf("bad", `setmetadata Inbox (/private//nodupslash "")`) + tc.transactf("bad", "setmetadata Inbox (/private/\001 \"\")") + tc.transactf("bad", "setmetadata Inbox (/private/\u007f \"\")") + tc.transactf("bad", `getmetadata (depth 0 depth 0) inbox (/private/a)`) // Duplicate option. + tc.transactf("bad", `getmetadata (depth badvalue) inbox (/private/a)`) + tc.transactf("bad", `getmetadata (maxsize invalid) inbox (/private/a)`) + tc.transactf("bad", `getmetadata (badoption) inbox (/private/a)`) + + // Update existing annotation by key. + tc.transactf("ok", `setmetadata "" (/PRIVATE/COMMENT "global updated")`) + tc.transactf("ok", `setmetadata inbox (/private/comment "mailbox updated")`) + tc.transactf("ok", `getmetadata "" (/private/comment)`) + tc.xuntagged(imapclient.UntaggedMetadataAnnotations{ + Mailbox: "", + Annotations: []imapclient.Annotation{ + {Key: "/private/comment", IsString: true, Value: []byte("global updated")}, + }, + }) + tc.transactf("ok", `getmetadata inbox (/private/comment)`) + tc.xuntagged(imapclient.UntaggedMetadataAnnotations{ + Mailbox: "Inbox", + Annotations: []imapclient.Annotation{ + {Key: "/private/comment", IsString: true, Value: []byte("mailbox updated")}, + }, + }) + + // Delete annotation with nil value. + tc.transactf("ok", `setmetadata "" (/private/comment nil)`) + tc.transactf("ok", `setmetadata inbox (/private/comment nil)`) + tc.transactf("ok", `getmetadata "" (/private/comment)`) + tc.xuntagged() + tc.transactf("ok", `getmetadata inbox (/private/comment)`) + tc.xuntagged() + + // Create a literal8 value, not a string. + tc.transactf("ok", "setmetadata inbox (/private/comment ~{4+}\r\ntest)") + tc.transactf("ok", `getmetadata inbox (/private/comment)`) + tc.xuntagged(imapclient.UntaggedMetadataAnnotations{ + Mailbox: "Inbox", + Annotations: []imapclient.Annotation{ + {Key: "/private/comment", IsString: false, Value: []byte("test")}, + }, + }) + + // Request with a maximum size, we don't get anything larger. + tc.transactf("ok", `setmetadata inbox (/private/another "longer")`) + tc.transactf("ok", `getmetadata (maxsize 4) inbox (/private/comment /private/another)`) + tc.xcodeArg(imapclient.CodeOther{Code: "METADATA", Args: []string{"LONGENTRIES", "6"}}) + tc.xuntagged(imapclient.UntaggedMetadataAnnotations{ + Mailbox: "Inbox", + Annotations: []imapclient.Annotation{ + {Key: "/private/comment", IsString: false, Value: []byte("test")}, + }, + }) + + // Request with various depth values. + tc.transactf("ok", `setmetadata inbox (/private/a "x" /private/a/b "x" /private/a/b/c "x" /private/a/b/c/d "x")`) + tc.transactf("ok", `getmetadata (depth 0) inbox (/private/a)`) + tc.xuntagged(imapclient.UntaggedMetadataAnnotations{ + Mailbox: "Inbox", + Annotations: []imapclient.Annotation{ + {Key: "/private/a", IsString: true, Value: []byte("x")}, + }, + }) + tc.transactf("ok", `getmetadata (depth 1) inbox (/private/a)`) + tc.xuntagged(imapclient.UntaggedMetadataAnnotations{ + Mailbox: "Inbox", + Annotations: []imapclient.Annotation{ + {Key: "/private/a", IsString: true, Value: []byte("x")}, + {Key: "/private/a/b", IsString: true, Value: []byte("x")}, + }, + }) + tc.transactf("ok", `getmetadata (depth infinity) inbox (/private/a)`) + tc.xuntagged(imapclient.UntaggedMetadataAnnotations{ + Mailbox: "Inbox", + Annotations: []imapclient.Annotation{ + {Key: "/private/a", IsString: true, Value: []byte("x")}, + {Key: "/private/a/b", IsString: true, Value: []byte("x")}, + {Key: "/private/a/b/c", IsString: true, Value: []byte("x")}, + {Key: "/private/a/b/c/d", IsString: true, Value: []byte("x")}, + }, + }) + // Same as previous, but ask for everything below /. + tc.transactf("ok", `getmetadata (depth infinity) inbox (/)`) + tc.xuntagged(imapclient.UntaggedMetadataAnnotations{ + Mailbox: "Inbox", + Annotations: []imapclient.Annotation{ + {Key: "/private/a", IsString: true, Value: []byte("x")}, + {Key: "/private/a/b", IsString: true, Value: []byte("x")}, + {Key: "/private/a/b/c", IsString: true, Value: []byte("x")}, + {Key: "/private/a/b/c/d", IsString: true, Value: []byte("x")}, + {Key: "/private/another", IsString: true, Value: []byte("longer")}, + {Key: "/private/comment", IsString: false, Value: []byte("test")}, + {Key: "/private/vendor/a/b", IsString: true, Value: []byte("")}, + }, + }) + + // Deleting a mailbox with an annotation should work and annotations should not + // come back when recreating mailbox. + tc.transactf("ok", "create testbox") + tc.transactf("ok", `setmetadata testbox (/private/a "x")`) + tc.transactf("ok", "delete testbox") + tc.transactf("ok", "create testbox") + tc.transactf("ok", `getmetadata testbox (/private/a)`) + tc.xuntagged() + + // When renaming mailbox, annotations must be copied to destination mailbox. + tc.transactf("ok", "rename inbox newbox") + tc.transactf("ok", `getmetadata newbox (/private/a)`) + tc.xuntagged(imapclient.UntaggedMetadataAnnotations{ + Mailbox: "newbox", + Annotations: []imapclient.Annotation{ + {Key: "/private/a", IsString: true, Value: []byte("x")}, + }, + }) + tc.transactf("ok", `getmetadata inbox (/private/a)`) + tc.xuntagged(imapclient.UntaggedMetadataAnnotations{ + Mailbox: "Inbox", + Annotations: []imapclient.Annotation{ + {Key: "/private/a", IsString: true, Value: []byte("x")}, + }, + }) + + // Broadcast should not happen when metadata capability is not enabled. + tc2 := startNoSwitchboard(t) + defer tc2.close() + tc2.client.Login("mjl@mox.example", password0) + tc2.client.Select("inbox") + + tc2.cmdf("", "idle") + tc2.readprefixline("+ ") + done := make(chan error) + go func() { + defer func() { + x := recover() + if x != nil { + done <- fmt.Errorf("%v", x) + } + }() + untagged, _ := tc2.client.ReadUntagged() + var exists imapclient.UntaggedExists + tuntagged(tc2.t, untagged, &exists) + tc2.writelinef("done") + tc2.response("ok") + done <- nil + }() + + // Should not cause idle to return. + tc.transactf("ok", `setmetadata inbox (/private/a "y")`) + // Cause to return. + tc.transactf("ok", "append inbox {4+}\r\ntest") + + timer := time.NewTimer(time.Second) + defer timer.Stop() + select { + case err := <-done: + tc.check(err, "idle") + case <-timer.C: + t.Fatalf("idle did not finish") + } + + // Broadcast should happen when metadata capability is enabled. + tc2.client.Enable(string(imapclient.CapMetadata)) + tc2.cmdf("", "idle") + tc2.readprefixline("+ ") + done = make(chan error) + go func() { + defer func() { + x := recover() + if x != nil { + done <- fmt.Errorf("%v", x) + } + }() + untagged, _ := tc2.client.ReadUntagged() + var metadataKeys imapclient.UntaggedMetadataKeys + tuntagged(tc2.t, untagged, &metadataKeys) + tc2.writelinef("done") + tc2.response("ok") + done <- nil + }() + + // Should cause idle to return. + tc.transactf("ok", `setmetadata inbox (/private/a "z")`) + + timer = time.NewTimer(time.Second) + defer timer.Stop() + select { + case err := <-done: + tc.check(err, "idle") + case <-timer.C: + t.Fatalf("idle did not finish") + } +} + +func TestMetadataLimit(t *testing.T) { + tc := start(t) + defer tc.close() + + tc.client.Login("mjl@mox.example", password0) + + maxKeys, maxSize := metadataMaxKeys, metadataMaxSize + defer func() { + metadataMaxKeys = maxKeys + metadataMaxSize = maxSize + }() + metadataMaxKeys = 10 + metadataMaxSize = 1000 + + // Reach max total size limit. + buf := make([]byte, metadataMaxSize+1) + for i := range buf { + buf[i] = 'x' + } + tc.cmdf("", "setmetadata inbox (/private/large ~{%d+}", len(buf)) + tc.client.Write(buf) + tc.client.Writelinef(")") + tc.response("no") + tc.xcodeArg(imapclient.CodeOther{Code: "METADATA", Args: []string{"MAXSIZE", fmt.Sprintf("%d", metadataMaxSize)}}) + + // Reach limit for max number. + for i := 1; i <= metadataMaxKeys; i++ { + tc.transactf("ok", `setmetadata inbox (/private/key%d "test")`, i) + } + tc.transactf("no", `setmetadata inbox (/private/toomany "test")`) + tc.xcodeArg(imapclient.CodeOther{Code: "METADATA", Args: []string{"TOOMANY"}}) +} diff --git a/imapserver/pack.go b/imapserver/pack.go index bdcd77e..ef9c59f 100644 --- a/imapserver/pack.go +++ b/imapserver/pack.go @@ -97,6 +97,7 @@ func (t syncliteral) writeTo(c *conn, w io.Writer) { type readerSizeSyncliteral struct { r io.Reader size int64 + lit8 bool } func (t readerSizeSyncliteral) pack(c *conn) string { @@ -104,11 +105,19 @@ func (t readerSizeSyncliteral) pack(c *conn) string { if err != nil { panic(err) } - return fmt.Sprintf("{%d}\r\n", t.size) + string(buf) + var lit string + if t.lit8 { + lit = "~" + } + return fmt.Sprintf("%s{%d}\r\n", lit, t.size) + string(buf) } func (t readerSizeSyncliteral) writeTo(c *conn, w io.Writer) { - fmt.Fprintf(w, "{%d}\r\n", t.size) + var lit string + if t.lit8 { + lit = "~" + } + fmt.Fprintf(w, "%s{%d}\r\n", lit, t.size) defer c.xtrace(mlog.LevelTracedata)() if _, err := io.Copy(w, io.LimitReader(t.r, t.size)); err != nil { panic(err) diff --git a/imapserver/parse.go b/imapserver/parse.go index 22a2877..227b79c 100644 --- a/imapserver/parse.go +++ b/imapserver/parse.go @@ -305,10 +305,10 @@ func (p *parser) xstring() (r string) { p.xerrorf("missing closing dquote in string") } size, sync := p.xliteralSize(false, true) - s := p.conn.xreadliteral(size, sync) + buf := p.conn.xreadliteral(size, sync) line := p.conn.readline(false) p.orig, p.upper, p.o = line, toUpper(line), 0 - return s + return string(buf) } func (p *parser) xnil() { @@ -974,3 +974,53 @@ func (p *parser) xdate() time.Time { } return time.Date(year, mon, day, 0, 0, 0, 0, time.UTC) } + +// Parse and validate a metadata key (entry name), returned as lower-case. +// +// ../rfc/5464:190 +func (p *parser) xmetadataKey() string { + // ../rfc/5464:772 + s := p.xastring() + + // ../rfc/5464:192 + if strings.Contains(s, "//") { + p.xerrorf("entry name must not contain two slashes") + } + // We allow a single slash, so it can be used with option "(depth infinity)" to get + // all annotations. + if s != "/" && strings.HasSuffix(s, "/") { + p.xerrorf("entry name must not end with slash") + } + // ../rfc/5464:202 + if strings.Contains(s, "*") || strings.Contains(s, "%") { + p.xerrorf("entry name must not contain * or %%") + } + for _, c := range s { + if c < ' ' || c >= 0x7f { + p.xerrorf("entry name must only contain non-control ascii characters") + } + } + return strings.ToLower(s) +} + +// ../rfc/5464:776 +func (p *parser) xmetadataKeyValue() (key string, isString bool, value []byte) { + key = p.xmetadataKey() + p.xspace() + + if p.hasPrefix("~{") { + size, sync := p.xliteralSize(true, true) + value = p.conn.xreadliteral(size, sync) + line := p.conn.readline(false) + p.orig, p.upper, p.o = line, toUpper(line), 0 + } else if p.hasPrefix(`"`) { + value = []byte(p.xstring()) + isString = true + } else if p.take("NIL") { + value = nil + } else { + p.xerrorf("expected metadata value") + } + + return +} diff --git a/imapserver/server.go b/imapserver/server.go index 5ef6a1f..24190c4 100644 --- a/imapserver/server.go +++ b/imapserver/server.go @@ -158,12 +158,13 @@ var authFailDelay = time.Second // After authentication failure. // QRESYNC: ../rfc/7162:1323 // STATUS=SIZE: ../rfc/8438 ../rfc/9051:8024 // QUOTA QUOTA=RES-STORAGE: ../rfc/9208:111 +// METADATA: ../rfc/5464 // // We always announce support for SCRAM PLUS-variants, also on connections without // TLS. The client should not be selecting PLUS variants on non-TLS connections, // instead opting to do the bare SCRAM variant without indicating the server claims // to support the PLUS variant (skipping the server downgrade detection check). -const serverCapabilities = "IMAP4rev2 IMAP4rev1 ENABLE LITERAL+ IDLE SASL-IR BINARY UNSELECT UIDPLUS ESEARCH SEARCHRES MOVE UTF8=ACCEPT LIST-EXTENDED SPECIAL-USE LIST-STATUS AUTH=SCRAM-SHA-256-PLUS AUTH=SCRAM-SHA-256 AUTH=SCRAM-SHA-1-PLUS AUTH=SCRAM-SHA-1 AUTH=CRAM-MD5 ID APPENDLIMIT=9223372036854775807 CONDSTORE QRESYNC STATUS=SIZE QUOTA QUOTA=RES-STORAGE" +const serverCapabilities = "IMAP4rev2 IMAP4rev1 ENABLE LITERAL+ IDLE SASL-IR BINARY UNSELECT UIDPLUS ESEARCH SEARCHRES MOVE UTF8=ACCEPT LIST-EXTENDED SPECIAL-USE LIST-STATUS AUTH=SCRAM-SHA-256-PLUS AUTH=SCRAM-SHA-256 AUTH=SCRAM-SHA-1-PLUS AUTH=SCRAM-SHA-1 AUTH=CRAM-MD5 ID APPENDLIMIT=9223372036854775807 CONDSTORE QRESYNC STATUS=SIZE QUOTA QUOTA=RES-STORAGE METADATA" type conn struct { cid int64 @@ -227,6 +228,7 @@ const ( capUTF8Accept capability = "UTF8=ACCEPT" capCondstore capability = "CONDSTORE" capQresync capability = "QRESYNC" + capMetadata capability = "METADATA" ) type lineErr struct { @@ -253,7 +255,7 @@ func stateCommands(cmds ...string) map[string]struct{} { var ( commandsStateAny = stateCommands("capability", "noop", "logout", "id") commandsStateNotAuthenticated = stateCommands("starttls", "authenticate", "login") - commandsStateAuthenticated = stateCommands("enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub", "getquotaroot", "getquota") + commandsStateAuthenticated = stateCommands("enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub", "getquotaroot", "getquota", "getmetadata", "setmetadata") commandsStateSelected = stateCommands("close", "unselect", "expunge", "search", "fetch", "store", "copy", "move", "uid expunge", "uid search", "uid fetch", "uid store", "uid copy", "uid move") ) @@ -286,6 +288,8 @@ var commands = map[string]func(c *conn, tag, cmd string, p *parser){ "idle": (*conn).cmdIdle, "getquotaroot": (*conn).cmdGetquotaroot, "getquota": (*conn).cmdGetquota, + "getmetadata": (*conn).cmdGetmetadata, + "setmetadata": (*conn).cmdSetmetadata, // Selected. "check": (*conn).cmdCheck, @@ -617,7 +621,7 @@ func (c *conn) readCommand(tag *string) (cmd string, p *parser) { return cmd, newParser(p.remainder(), c) } -func (c *conn) xreadliteral(size int64, sync bool) string { +func (c *conn) xreadliteral(size int64, sync bool) []byte { if sync { c.writelinef("+ ") } @@ -633,7 +637,7 @@ func (c *conn) xreadliteral(size int64, sync bool) string { panic(fmt.Errorf("reading literal: %s (%w)", err, errIO)) } } - return string(buf) + return buf } func (c *conn) xhighestModSeq(tx *bstore.Tx, mailboxID int64) store.ModSeq { @@ -1541,6 +1545,14 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { case store.ChangeRemoveMailbox, store.ChangeAddMailbox, store.ChangeRenameMailbox, store.ChangeAddSubscription: n = append(n, change) continue + case store.ChangeAnnotation: + // note: annotations may have a mailbox associated with them, but we pass all + // changes on. + // Only when the metadata capability was enabled. ../rfc/5464:660 + if c.enabled[capMetadata] { + n = append(n, change) + continue + } case store.ChangeMailboxCounts, store.ChangeMailboxSpecialUse, store.ChangeMailboxKeywords, store.ChangeThread: default: panic(fmt.Errorf("missing case for %#v", change)) @@ -1651,6 +1663,9 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { c.bwritelinef(`* LIST (%s) "/" %s%s`, strings.Join(ch.Flags, " "), astring(c.encodeMailbox(ch.NewName)).pack(c), oldname) case store.ChangeAddSubscription: c.bwritelinef(`* LIST (%s) "/" %s`, strings.Join(append([]string{`\Subscribed`}, ch.Flags...), " "), astring(c.encodeMailbox(ch.Name)).pack(c)) + case store.ChangeAnnotation: + // ../rfc/5464:807 ../rfc/5464:788 + c.bwritelinef(`* METADATA %s %s`, astring(c.encodeMailbox(ch.MailboxName)).pack(c), astring(ch.Key).pack(c)) default: panic(fmt.Sprintf("internal error, missing case for %#v", change)) } @@ -2325,6 +2340,9 @@ func (c *conn) cmdEnable(tag, cmd string, p *parser) { c.enabled[cap] = true enabled += " " + s qresync = true + case capMetadata: + c.enabled[cap] = true + enabled += " " + s } } // QRESYNC enabled CONDSTORE too ../rfc/7162:1391 @@ -2713,7 +2731,7 @@ func (c *conn) cmdCreate(tag, cmd string, p *parser) { c.ok(tag, cmd) } -// Delete removes a mailbox and all its messages. +// Delete removes a mailbox and all its messages and annotations. // Inbox cannot be removed. // // State: Authenticated and selected. @@ -2760,7 +2778,8 @@ func (c *conn) cmdDelete(tag, cmd string, p *parser) { } // Rename changes the name of a mailbox. -// Renaming INBOX is special, it moves the inbox messages to a new mailbox, leaving inbox empty. +// Renaming INBOX is special, it moves the inbox messages to a new mailbox, leaving +// inbox empty, but copying metadata annotations. // Renaming a mailbox with submailboxes also renames all submailboxes. // Subscriptions stay with the old name, though newly created missing parent // mailboxes for the destination name are automatically subscribed. @@ -2865,10 +2884,25 @@ func (c *conn) cmdRename(tag, cmd string, p *parser) { if tx.Get(&store.Subscription{Name: dstMB.Name}) == nil { dstFlags = []string{`\Subscribed`} } + + // Copy any annotations. ../rfc/5464:368 + annotations, err := bstore.QueryTx[store.Annotation](tx).FilterNonzero(store.Annotation{MailboxID: srcMB.ID}).List() + xcheckf(err, "get annotations to copy for inbox") + for i := range annotations { + annotations[i].ID = 0 + annotations[i].MailboxID = dstMB.ID + err := tx.Insert(&annotations[i]) + xcheckf(err, "copy annotation to destination mailbox") + } + changes[0] = store.ChangeRemoveUIDs{MailboxID: srcMB.ID, UIDs: oldUIDs, ModSeq: modseq} changes[1] = store.ChangeAddMailbox{Mailbox: dstMB, Flags: dstFlags} // changes[2:...] are ChangeAddUIDs changes = append(changes, srcMB.ChangeCounts(), dstMB.ChangeCounts()) + for _, a := range annotations { + changes = append(changes, a.Change(dstMB.Name)) + } + return } diff --git a/rfc/index.txt b/rfc/index.txt index 06193c3..8e0ceba 100644 --- a/rfc/index.txt +++ b/rfc/index.txt @@ -201,7 +201,12 @@ https://www.iana.org/assignments/message-headers/message-headers.xhtml 5258 Yes - Internet Message Access Protocol version 4 - LIST Command Extensions 5259 No - Internet Message Access Protocol - CONVERT Extension 5267 Roadmap - Contexts for IMAP4 -5464 Roadmap - The IMAP METADATA Extension +5464 Yes - The IMAP METADATA Extension +5464-eid1691 - - errata: fix example entry name +5464-eid1692 - - errata: make text match abnf +5464-eid2785 - - errata: fix GETMETADATA example +5464-eid2786 - - errata: fix GETMETADATA example +5464-eid3868 - - errata: fix GETMETADATA example 5465 Roadmap - The IMAP NOTIFY Extension 5466 Roadmap - IMAP4 Extension for Named Searches (Filters) 5524 No - Extended URLFETCH for Binary and Converted Parts diff --git a/store/account.go b/store/account.go index c6d04c7..b2aeeeb 100644 --- a/store/account.go +++ b/store/account.go @@ -216,6 +216,27 @@ type Mailbox struct { MailboxCounts // Statistics about messages, kept up to date whenever a change happens. } +// Annotation is a per-mailbox or global (per-account) annotation for the IMAP +// metadata extension, currently always a private annotation. +type Annotation struct { + ID int64 + + // Can be zero, indicates global (per-account) annotation. + MailboxID int64 `bstore:"ref Mailbox,unique MailboxID+Key"` + + // "Entry name", always starts with "/private/". Stored lower-case, comparisons + // must be done case-insensitively. + Key string `bstore:"nonzero"` + + IsString bool // If true, the value is a string instead of bytes. + Value []byte +} + +// Change returns a broadcastable change for the annotation. +func (a Annotation) Change(mailboxName string) ChangeAnnotation { + return ChangeAnnotation{a.MailboxID, mailboxName, a.Key} +} + // MailboxCounts tracks statistics about messages for a mailbox. type MailboxCounts struct { Total int64 // Total number of messages, excluding \Deleted. For JMAP. @@ -818,6 +839,7 @@ var DBTypes = []any{ RulesetNoListID{}, RulesetNoMsgFrom{}, RulesetNoMailbox{}, + Annotation{}, } // Account holds the information about a user, includings mailboxes, messages, imap subscriptions. @@ -2713,8 +2735,8 @@ func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc Mailbox, dst string) (chang return changes, false, false, false, nil } -// MailboxDelete deletes a mailbox by ID. If it has children, the return value -// indicates that and an error is returned. +// MailboxDelete deletes a mailbox by ID, including its annotations. If it has +// children, the return value indicates that and an error is returned. // // Caller should broadcast the changes and remove files for the removed message IDs. func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) { @@ -2785,6 +2807,13 @@ func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx } } + // Remove metadata annotations. ../rfc/5464:373 + if _, err := bstore.QueryTx[Annotation](tx).FilterNonzero(Annotation{MailboxID: mailbox.ID}).Delete(); err != nil { + return nil, nil, false, fmt.Errorf("removing annotations for mailbox: %v", err) + } + // Not sending changes about annotations on this mailbox, since the entire mailbox + // is being removed. + if err := tx.Delete(&Mailbox{ID: mailbox.ID}); err != nil { return nil, nil, false, fmt.Errorf("removing mailbox: %v", err) } diff --git a/store/state.go b/store/state.go index dac92be..ff3a02a 100644 --- a/store/state.go +++ b/store/state.go @@ -105,6 +105,14 @@ type ChangeMailboxKeywords struct { Keywords []string } +// ChangeAnnotation is sent when an annotation is added/updated/removed, either for +// a mailbox or a global per-account annotation. The value is not included. +type ChangeAnnotation struct { + MailboxID int64 // Can be zero, meaning global (per-account) annotation. + MailboxName string // Empty for global (per-account) annotation. + Key string // Also called "entry name", e.g. "/private/comment". +} + var switchboardBusy atomic.Bool // Switchboard distributes changes to accounts to interested listeners. See Comm and Change. diff --git a/webmail/api.go b/webmail/api.go index 6cbff87..9191f73 100644 --- a/webmail/api.go +++ b/webmail/api.go @@ -1230,7 +1230,7 @@ func (Webmail) MailboxCreate(ctx context.Context, name string) { }) } -// MailboxDelete deletes a mailbox and all its messages. +// MailboxDelete deletes a mailbox and all its messages and annotations. func (Webmail) MailboxDelete(ctx context.Context, mailboxID int64) { reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) acc := reqInfo.Account diff --git a/webmail/api.json b/webmail/api.json index 31ff804..12d9eb5 100644 --- a/webmail/api.json +++ b/webmail/api.json @@ -276,7 +276,7 @@ }, { "Name": "MailboxDelete", - "Docs": "MailboxDelete deletes a mailbox and all its messages.", + "Docs": "MailboxDelete deletes a mailbox and all its messages and annotations.", "Params": [ { "Name": "mailboxID", diff --git a/webmail/api.ts b/webmail/api.ts index 93b5d9b..01d75cf 100644 --- a/webmail/api.ts +++ b/webmail/api.ts @@ -894,7 +894,7 @@ export class Client { return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params) as void } - // MailboxDelete deletes a mailbox and all its messages. + // MailboxDelete deletes a mailbox and all its messages and annotations. async MailboxDelete(mailboxID: number): Promise { const fn: string = "MailboxDelete" const paramTypes: string[][] = [["int64"]] diff --git a/webmail/msg.js b/webmail/msg.js index 44b7ffa..2861108 100644 --- a/webmail/msg.js +++ b/webmail/msg.js @@ -573,7 +573,7 @@ var api; const params = [name]; return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); } - // MailboxDelete deletes a mailbox and all its messages. + // MailboxDelete deletes a mailbox and all its messages and annotations. async MailboxDelete(mailboxID) { const fn = "MailboxDelete"; const paramTypes = [["int64"]]; diff --git a/webmail/text.js b/webmail/text.js index 82f0d25..3f799c5 100644 --- a/webmail/text.js +++ b/webmail/text.js @@ -573,7 +573,7 @@ var api; const params = [name]; return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); } - // MailboxDelete deletes a mailbox and all its messages. + // MailboxDelete deletes a mailbox and all its messages and annotations. async MailboxDelete(mailboxID) { const fn = "MailboxDelete"; const paramTypes = [["int64"]]; diff --git a/webmail/webmail.js b/webmail/webmail.js index 2282248..bfe1641 100644 --- a/webmail/webmail.js +++ b/webmail/webmail.js @@ -573,7 +573,7 @@ var api; const params = [name]; return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); } - // MailboxDelete deletes a mailbox and all its messages. + // MailboxDelete deletes a mailbox and all its messages and annotations. async MailboxDelete(mailboxID) { const fn = "MailboxDelete"; const paramTypes = [["int64"]];