implement the imap metadata extension, rfc 5464

this allows setting per-mailbox and per-server annotations (metadata). we have
a fixed maximum for total number of annotations (1000) and their total size
(1000000 bytes). this size isn't held against the regular quota for simplicity.
we send unsolicited metadata responses when a connection is in the idle
command and a change to a metadata item is made.

we currently only implement the /private/ namespace.  we should implement the
/shared/ namespace, for mox-global metadata annotations.  only the admin should
be able to configure those, probably through the config file, cli, or admin web
interface.

for issue #290
This commit is contained in:
Mechiel Lukkien 2025-02-17 22:44:51 +01:00
parent 9dff879164
commit f30c44eddb
No known key found for this signature in database
17 changed files with 820 additions and 41 deletions

View File

@ -427,6 +427,50 @@ func (c *Conn) xuntagged() Untagged {
c.xcrlf()
return r
case "METADATA":
// ../rfc/5464:807
c.xspace()
mailbox := c.xastring()
c.xspace()
if !c.take('(') {
// Unsolicited form, with only annotation keys, not values.
var keys []string
for {
key := c.xastring()
keys = append(keys, key)
if !c.space() {
break
}
}
c.xcrlf()
return UntaggedMetadataKeys{mailbox, keys}
}
// Form with values, in response to GETMETADATA command.
r := UntaggedMetadataAnnotations{Mailbox: mailbox}
for {
key := c.xastring()
c.xspace()
var value []byte
var isString bool
if c.take('~') {
value = c.xliteral()
} else {
value = []byte(c.xstring())
isString = true
// note: the abnf also allows nstring, but that only makes sense when the
// production rule is used in the setmetadata command. ../rfc/5464:831
}
r.Annotations = append(r.Annotations, Annotation{key, isString, value})
if c.take(')') {
break
}
c.xspace()
}
c.xcrlf()
return r
case "NAMESPACE":
// ../rfc/9051:6778
c.xspace()

View File

@ -10,27 +10,29 @@ import (
type Capability string
const (
CapIMAP4rev1 Capability = "IMAP4rev1"
CapIMAP4rev2 Capability = "IMAP4rev2"
CapLoginDisabled Capability = "LOGINDISABLED"
CapStarttls Capability = "STARTTLS"
CapAuthPlain Capability = "AUTH=PLAIN"
CapLiteralPlus Capability = "LITERAL+"
CapLiteralMinus Capability = "LITERAL-"
CapIdle Capability = "IDLE"
CapNamespace Capability = "NAMESPACE"
CapBinary Capability = "BINARY"
CapUnselect Capability = "UNSELECT"
CapUidplus Capability = "UIDPLUS"
CapEsearch Capability = "ESEARCH"
CapEnable Capability = "ENABLE"
CapSave Capability = "SAVE"
CapListExtended Capability = "LIST-EXTENDED"
CapSpecialUse Capability = "SPECIAL-USE"
CapMove Capability = "MOVE"
CapUTF8Only Capability = "UTF8=ONLY"
CapUTF8Accept Capability = "UTF8=ACCEPT"
CapID Capability = "ID" // ../rfc/2971:80
CapIMAP4rev1 Capability = "IMAP4rev1"
CapIMAP4rev2 Capability = "IMAP4rev2"
CapLoginDisabled Capability = "LOGINDISABLED"
CapStarttls Capability = "STARTTLS"
CapAuthPlain Capability = "AUTH=PLAIN"
CapLiteralPlus Capability = "LITERAL+"
CapLiteralMinus Capability = "LITERAL-"
CapIdle Capability = "IDLE"
CapNamespace Capability = "NAMESPACE"
CapBinary Capability = "BINARY"
CapUnselect Capability = "UNSELECT"
CapUidplus Capability = "UIDPLUS"
CapEsearch Capability = "ESEARCH"
CapEnable Capability = "ENABLE"
CapSave Capability = "SAVE"
CapListExtended Capability = "LIST-EXTENDED"
CapSpecialUse Capability = "SPECIAL-USE"
CapMove Capability = "MOVE"
CapUTF8Only Capability = "UTF8=ONLY"
CapUTF8Accept Capability = "UTF8=ACCEPT"
CapID Capability = "ID" // ../rfc/2971:80
CapMetadata Capability = "METADATA" // ../rfc/5464:124
CapMetadataServer Capability = "METADATA-SERVER" // ../rfc/5464:124
)
// Status is the tagged final result of a command.
@ -227,6 +229,27 @@ type UntaggedStatus struct {
Attrs map[StatusAttr]int64 // Upper case status attributes.
}
// ../rfc/5464:716 Unsolicited response, indicating an annotation has changed.
type UntaggedMetadataKeys struct {
Mailbox string // Empty means not specific to mailbox.
// Keys that have changed. To get values (or determine absence), the server must be
// queried.
Keys []string
}
type Annotation struct {
Key string
IsString bool
Value []byte
}
// ../rfc/5464:683
type UntaggedMetadataAnnotations struct {
Mailbox string // Empty means not specific to mailbox.
Annotations []Annotation
}
// ../rfc/9051:7059 ../9208:712
type StatusAttr string

View File

@ -681,7 +681,7 @@ func (cmd *fetchCmd) xbody(a fetchAtt) (string, token) {
count = m.Size - offset
}
}
return respField, readerSizeSyncliteral{&moxio.AtReader{R: msgr, Offset: offset}, count}
return respField, readerSizeSyncliteral{&moxio.AtReader{R: msgr, Offset: offset}, count, false}
}
sr := cmd.xsection(a.section, part)

301
imapserver/metadata.go Normal file
View File

@ -0,0 +1,301 @@
package imapserver
import (
"bytes"
"fmt"
"strings"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/store"
)
// Changed during tests.
var metadataMaxKeys = 1000
var metadataMaxSize = 1000 * 1000
// Metadata errata:
// ../rfc/5464:183 ../rfc/5464-eid1691
// ../rfc/5464:564 ../rfc/5464-eid1692
// ../rfc/5464:494 ../rfc/5464-eid2785 ../rfc/5464-eid2786
// ../rfc/5464:698 ../rfc/5464-eid3868
// Note: We do not tie the special-use mailbox flags to a (synthetic) private
// per-mailbox annotation. ../rfc/6154:303
// For registration of names, see https://www.iana.org/assignments/imap-metadata/imap-metadata.xhtml
// Get metadata annotations, per mailbox or globally.
//
// State: Authenticated and selected.
func (c *conn) cmdGetmetadata(tag, cmd string, p *parser) {
// Command: ../rfc/5464:412
// Request syntax: ../rfc/5464:792
p.xspace()
var optMaxSize int64 = -1
var optDepth string
if p.take("(") {
for {
if p.take("MAXSIZE") {
// ../rfc/5464:804
p.xspace()
v := p.xnumber()
if optMaxSize >= 0 {
p.xerrorf("only a single maxsize option accepted")
}
optMaxSize = int64(v)
} else if p.take("DEPTH") {
// ../rfc/5464:823
p.xspace()
s := p.xtakelist("0", "1", "INFINITY")
if optDepth != "" {
p.xerrorf("only single depth option accepted")
}
optDepth = s
} else {
// ../rfc/5464:800 We are not doing anything further parsing for future extensions.
p.xerrorf("unknown option for getmetadata, expected maxsize or depth")
}
if p.take(")") {
break
}
p.xspace()
}
p.xspace()
}
mailboxName := p.xmailbox()
if mailboxName != "" {
mailboxName = xcheckmailboxname(mailboxName, true)
}
p.xspace()
// Entries ../rfc/5464:768
entryNames := map[string]struct{}{}
if p.take("(") {
for {
s := p.xmetadataKey()
entryNames[s] = struct{}{}
if p.take(")") {
break
}
p.xtake(" ")
}
} else {
s := p.xmetadataKey()
entryNames[s] = struct{}{}
}
p.xempty()
var annotations []store.Annotation
longentries := -1 // Size of largest value skipped due to optMaxSize. ../rfc/5464:482
c.account.WithRLock(func() {
c.xdbread(func(tx *bstore.Tx) {
q := bstore.QueryTx[store.Annotation](tx)
if mailboxName == "" {
q.FilterEqual("MailboxID", 0)
} else {
mb := c.xmailbox(tx, mailboxName, "TRYCREATE")
q.FilterNonzero(store.Annotation{MailboxID: mb.ID})
}
err := q.ForEach(func(a store.Annotation) error {
// ../rfc/5464:516
switch optDepth {
case "", "0":
if _, ok := entryNames[a.Key]; !ok {
return nil
}
case "1", "INFINITY":
// Go through all keys, matching depth.
if _, ok := entryNames[a.Key]; ok {
break
}
var match bool
for s := range entryNames {
prefix := s
if s != "/" {
prefix += "/"
}
if !strings.HasPrefix(a.Key, prefix) {
continue
}
if optDepth == "INFINITY" {
match = true
break
}
suffix := a.Key[len(prefix):]
t := strings.SplitN(suffix, "/", 2)
if len(t) == 1 {
match = true
break
}
}
if !match {
return nil
}
default:
xcheckf(fmt.Errorf("%q", optDepth), "missing case for depth")
}
if optMaxSize >= 0 && int64(len(a.Value)) > optMaxSize {
longentries = max(longentries, len(a.Value))
} else {
annotations = append(annotations, a)
}
return nil
})
xcheckf(err, "looking up annotations")
})
})
// Response syntax: ../rfc/5464:807 ../rfc/5464:778
// We can only send untagged responses when we have any matches.
if len(annotations) > 0 {
fmt.Fprintf(c.bw, "* METADATA %s (", astring(mailboxName).pack(c))
for i, a := range annotations {
if i > 0 {
fmt.Fprint(c.bw, " ")
}
astring(a.Key).writeTo(c, c.bw)
fmt.Fprint(c.bw, " ")
if a.IsString {
string0(string(a.Value)).writeTo(c, c.bw)
} else {
v := readerSizeSyncliteral{bytes.NewReader(a.Value), int64(len(a.Value)), true}
v.writeTo(c, c.bw)
}
}
c.bwritelinef(")")
}
if longentries >= 0 {
c.bwritelinef("%s OK [METADATA LONGENTRIES %d] getmetadata done", tag, longentries)
} else {
c.ok(tag, cmd)
}
}
// Set metadata annotation, per mailbox or globally.
//
// We only implement private annotations, not shared annotations. We don't
// currently have a mechanism for determining if the user should have access.
//
// State: Authenticated and selected.
func (c *conn) cmdSetmetadata(tag, cmd string, p *parser) {
// Command: ../rfc/5464:547
// Request syntax: ../rfc/5464:826
p.xspace()
mailboxName := p.xmailbox()
// Empty name means a global (per-account) annotation, not for a mailbox.
if mailboxName != "" {
mailboxName = xcheckmailboxname(mailboxName, true)
}
p.xspace()
p.xtake("(")
var l []store.Annotation
for {
key, isString, value := p.xmetadataKeyValue()
l = append(l, store.Annotation{Key: key, IsString: isString, Value: value})
if p.take(")") {
break
}
p.xspace()
}
p.xempty()
// Additional checks on entry names.
for _, a := range l {
// We only allow /private/* entry names, so check early and fail if we see anything
// else (the only other option is /shared/* at this moment).
// ../rfc/5464:217
if !strings.HasPrefix(a.Key, "/private/") {
// ../rfc/5464:346
xuserErrorf("only /private/* entry names allowed")
}
// We also enforce that /private/vendor/ is followed by at least 2 elements.
// ../rfc/5464:234
if a.Key == "/private/vendor" || strings.HasPrefix(a.Key, "/private/vendor/") {
t := strings.SplitN(a.Key[1:], "/", 4)
if len(t) < 4 {
xuserErrorf("entry names starting with /private/vendor must have at least 4 components")
}
}
}
// Store the annotations, possibly removing/inserting/updating them.
c.account.WithWLock(func() {
var changes []store.Change
c.xdbwrite(func(tx *bstore.Tx) {
var mb store.Mailbox // mb.ID as 0 is used in query below.
if mailboxName != "" {
mb = c.xmailbox(tx, mailboxName, "TRYCREATE")
}
for _, a := range l {
q := bstore.QueryTx[store.Annotation](tx)
q.FilterNonzero(store.Annotation{Key: a.Key})
q.FilterEqual("MailboxID", mb.ID) // Can be zero.
// Nil means remove. ../rfc/5464:579
if a.Value == nil {
var deleted []store.Annotation
q.Gather(&deleted)
_, err := q.Delete()
xcheckf(err, "deleting annotation")
for _, oa := range deleted {
changes = append(changes, oa.Change(mailboxName))
}
continue
}
a.MailboxID = mb.ID
oa, err := q.Get()
if err == bstore.ErrAbsent {
err = tx.Insert(&a)
xcheckf(err, "inserting annotation")
changes = append(changes, a.Change(mailboxName))
continue
}
xcheckf(err, "looking up existing annotation for entry name")
if oa.IsString != a.IsString || (oa.Value == nil) != (a.Value == nil) || !bytes.Equal(oa.Value, a.Value) {
changes = append(changes, a.Change(mailboxName))
}
oa.Value = a.Value
err = tx.Update(&oa)
xcheckf(err, "updating metadata annotation")
}
// Check for total size. We allow a total of 1000 entries, with total capacity of 1MB.
// ../rfc/5464:383
var n int
var size int
err := bstore.QueryTx[store.Annotation](tx).ForEach(func(a store.Annotation) error {
n++
if n > metadataMaxKeys {
// ../rfc/5464:590
xusercodeErrorf("METADATA TOOMANY", "too many metadata entries, 1000 allowed in total")
}
size += len(a.Key) + len(a.Value)
if size > metadataMaxSize {
// ../rfc/5464:585 We only have a max total size limit, not per entry. We'll
// mention the max total size.
xusercodeErrorf(fmt.Sprintf("METADATA MAXSIZE %d", metadataMaxSize), "metadata entry values too large, total maximum size is 1MB")
}
return nil
})
xcheckf(err, "checking metadata annotation size")
})
c.broadcast(changes)
})
c.ok(tag, cmd)
}

276
imapserver/metadata_test.go Normal file
View File

@ -0,0 +1,276 @@
package imapserver
import (
"fmt"
"testing"
"time"
"github.com/mjl-/mox/imapclient"
)
func TestMetadata(t *testing.T) {
tc := start(t)
defer tc.close()
tc.client.Login("mjl@mox.example", password0)
tc.transactf("ok", `getmetadata "" /private/comment`)
tc.xuntagged()
tc.transactf("ok", `getmetadata inbox (/private/comment)`)
tc.xuntagged()
tc.transactf("ok", `setmetadata "" (/PRIVATE/COMMENT "global value")`)
tc.transactf("ok", `setmetadata inbox (/private/comment "mailbox value")`)
tc.transactf("ok", `getmetadata "" ("/private/comment")`)
tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
Mailbox: "",
Annotations: []imapclient.Annotation{
{Key: "/private/comment", IsString: true, Value: []byte("global value")},
},
})
tc.transactf("ok", `getmetadata inbox (/private/comment /private/unknown /shared/comment)`)
tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
Mailbox: "Inbox",
Annotations: []imapclient.Annotation{
{Key: "/private/comment", IsString: true, Value: []byte("mailbox value")},
},
})
tc.transactf("no", `setmetadata doesnotexist (/private/comment "test")`) // Bad mailbox.
tc.transactf("no", `setmetadata Inbox (/shared/comment "")`) // /shared/ not implemented.
tc.transactf("no", `setmetadata Inbox (/badprefix/comment "")`)
tc.transactf("no", `setmetadata Inbox (/private/vendor "")`) // /*/vendor must have more components.
tc.transactf("no", `setmetadata Inbox (/private/vendor/stillbad "")`) // /*/vendor must have more components.
tc.transactf("ok", `setmetadata Inbox (/private/vendor/a/b "")`)
tc.transactf("bad", `setmetadata Inbox (/private/no* "")`)
tc.transactf("bad", `setmetadata Inbox (/private/no%% "")`)
tc.transactf("bad", `setmetadata Inbox (/private/notrailingslash/ "")`)
tc.transactf("bad", `setmetadata Inbox (/private//nodupslash "")`)
tc.transactf("bad", "setmetadata Inbox (/private/\001 \"\")")
tc.transactf("bad", "setmetadata Inbox (/private/\u007f \"\")")
tc.transactf("bad", `getmetadata (depth 0 depth 0) inbox (/private/a)`) // Duplicate option.
tc.transactf("bad", `getmetadata (depth badvalue) inbox (/private/a)`)
tc.transactf("bad", `getmetadata (maxsize invalid) inbox (/private/a)`)
tc.transactf("bad", `getmetadata (badoption) inbox (/private/a)`)
// Update existing annotation by key.
tc.transactf("ok", `setmetadata "" (/PRIVATE/COMMENT "global updated")`)
tc.transactf("ok", `setmetadata inbox (/private/comment "mailbox updated")`)
tc.transactf("ok", `getmetadata "" (/private/comment)`)
tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
Mailbox: "",
Annotations: []imapclient.Annotation{
{Key: "/private/comment", IsString: true, Value: []byte("global updated")},
},
})
tc.transactf("ok", `getmetadata inbox (/private/comment)`)
tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
Mailbox: "Inbox",
Annotations: []imapclient.Annotation{
{Key: "/private/comment", IsString: true, Value: []byte("mailbox updated")},
},
})
// Delete annotation with nil value.
tc.transactf("ok", `setmetadata "" (/private/comment nil)`)
tc.transactf("ok", `setmetadata inbox (/private/comment nil)`)
tc.transactf("ok", `getmetadata "" (/private/comment)`)
tc.xuntagged()
tc.transactf("ok", `getmetadata inbox (/private/comment)`)
tc.xuntagged()
// Create a literal8 value, not a string.
tc.transactf("ok", "setmetadata inbox (/private/comment ~{4+}\r\ntest)")
tc.transactf("ok", `getmetadata inbox (/private/comment)`)
tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
Mailbox: "Inbox",
Annotations: []imapclient.Annotation{
{Key: "/private/comment", IsString: false, Value: []byte("test")},
},
})
// Request with a maximum size, we don't get anything larger.
tc.transactf("ok", `setmetadata inbox (/private/another "longer")`)
tc.transactf("ok", `getmetadata (maxsize 4) inbox (/private/comment /private/another)`)
tc.xcodeArg(imapclient.CodeOther{Code: "METADATA", Args: []string{"LONGENTRIES", "6"}})
tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
Mailbox: "Inbox",
Annotations: []imapclient.Annotation{
{Key: "/private/comment", IsString: false, Value: []byte("test")},
},
})
// Request with various depth values.
tc.transactf("ok", `setmetadata inbox (/private/a "x" /private/a/b "x" /private/a/b/c "x" /private/a/b/c/d "x")`)
tc.transactf("ok", `getmetadata (depth 0) inbox (/private/a)`)
tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
Mailbox: "Inbox",
Annotations: []imapclient.Annotation{
{Key: "/private/a", IsString: true, Value: []byte("x")},
},
})
tc.transactf("ok", `getmetadata (depth 1) inbox (/private/a)`)
tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
Mailbox: "Inbox",
Annotations: []imapclient.Annotation{
{Key: "/private/a", IsString: true, Value: []byte("x")},
{Key: "/private/a/b", IsString: true, Value: []byte("x")},
},
})
tc.transactf("ok", `getmetadata (depth infinity) inbox (/private/a)`)
tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
Mailbox: "Inbox",
Annotations: []imapclient.Annotation{
{Key: "/private/a", IsString: true, Value: []byte("x")},
{Key: "/private/a/b", IsString: true, Value: []byte("x")},
{Key: "/private/a/b/c", IsString: true, Value: []byte("x")},
{Key: "/private/a/b/c/d", IsString: true, Value: []byte("x")},
},
})
// Same as previous, but ask for everything below /.
tc.transactf("ok", `getmetadata (depth infinity) inbox (/)`)
tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
Mailbox: "Inbox",
Annotations: []imapclient.Annotation{
{Key: "/private/a", IsString: true, Value: []byte("x")},
{Key: "/private/a/b", IsString: true, Value: []byte("x")},
{Key: "/private/a/b/c", IsString: true, Value: []byte("x")},
{Key: "/private/a/b/c/d", IsString: true, Value: []byte("x")},
{Key: "/private/another", IsString: true, Value: []byte("longer")},
{Key: "/private/comment", IsString: false, Value: []byte("test")},
{Key: "/private/vendor/a/b", IsString: true, Value: []byte("")},
},
})
// Deleting a mailbox with an annotation should work and annotations should not
// come back when recreating mailbox.
tc.transactf("ok", "create testbox")
tc.transactf("ok", `setmetadata testbox (/private/a "x")`)
tc.transactf("ok", "delete testbox")
tc.transactf("ok", "create testbox")
tc.transactf("ok", `getmetadata testbox (/private/a)`)
tc.xuntagged()
// When renaming mailbox, annotations must be copied to destination mailbox.
tc.transactf("ok", "rename inbox newbox")
tc.transactf("ok", `getmetadata newbox (/private/a)`)
tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
Mailbox: "newbox",
Annotations: []imapclient.Annotation{
{Key: "/private/a", IsString: true, Value: []byte("x")},
},
})
tc.transactf("ok", `getmetadata inbox (/private/a)`)
tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
Mailbox: "Inbox",
Annotations: []imapclient.Annotation{
{Key: "/private/a", IsString: true, Value: []byte("x")},
},
})
// Broadcast should not happen when metadata capability is not enabled.
tc2 := startNoSwitchboard(t)
defer tc2.close()
tc2.client.Login("mjl@mox.example", password0)
tc2.client.Select("inbox")
tc2.cmdf("", "idle")
tc2.readprefixline("+ ")
done := make(chan error)
go func() {
defer func() {
x := recover()
if x != nil {
done <- fmt.Errorf("%v", x)
}
}()
untagged, _ := tc2.client.ReadUntagged()
var exists imapclient.UntaggedExists
tuntagged(tc2.t, untagged, &exists)
tc2.writelinef("done")
tc2.response("ok")
done <- nil
}()
// Should not cause idle to return.
tc.transactf("ok", `setmetadata inbox (/private/a "y")`)
// Cause to return.
tc.transactf("ok", "append inbox {4+}\r\ntest")
timer := time.NewTimer(time.Second)
defer timer.Stop()
select {
case err := <-done:
tc.check(err, "idle")
case <-timer.C:
t.Fatalf("idle did not finish")
}
// Broadcast should happen when metadata capability is enabled.
tc2.client.Enable(string(imapclient.CapMetadata))
tc2.cmdf("", "idle")
tc2.readprefixline("+ ")
done = make(chan error)
go func() {
defer func() {
x := recover()
if x != nil {
done <- fmt.Errorf("%v", x)
}
}()
untagged, _ := tc2.client.ReadUntagged()
var metadataKeys imapclient.UntaggedMetadataKeys
tuntagged(tc2.t, untagged, &metadataKeys)
tc2.writelinef("done")
tc2.response("ok")
done <- nil
}()
// Should cause idle to return.
tc.transactf("ok", `setmetadata inbox (/private/a "z")`)
timer = time.NewTimer(time.Second)
defer timer.Stop()
select {
case err := <-done:
tc.check(err, "idle")
case <-timer.C:
t.Fatalf("idle did not finish")
}
}
func TestMetadataLimit(t *testing.T) {
tc := start(t)
defer tc.close()
tc.client.Login("mjl@mox.example", password0)
maxKeys, maxSize := metadataMaxKeys, metadataMaxSize
defer func() {
metadataMaxKeys = maxKeys
metadataMaxSize = maxSize
}()
metadataMaxKeys = 10
metadataMaxSize = 1000
// Reach max total size limit.
buf := make([]byte, metadataMaxSize+1)
for i := range buf {
buf[i] = 'x'
}
tc.cmdf("", "setmetadata inbox (/private/large ~{%d+}", len(buf))
tc.client.Write(buf)
tc.client.Writelinef(")")
tc.response("no")
tc.xcodeArg(imapclient.CodeOther{Code: "METADATA", Args: []string{"MAXSIZE", fmt.Sprintf("%d", metadataMaxSize)}})
// Reach limit for max number.
for i := 1; i <= metadataMaxKeys; i++ {
tc.transactf("ok", `setmetadata inbox (/private/key%d "test")`, i)
}
tc.transactf("no", `setmetadata inbox (/private/toomany "test")`)
tc.xcodeArg(imapclient.CodeOther{Code: "METADATA", Args: []string{"TOOMANY"}})
}

View File

@ -97,6 +97,7 @@ func (t syncliteral) writeTo(c *conn, w io.Writer) {
type readerSizeSyncliteral struct {
r io.Reader
size int64
lit8 bool
}
func (t readerSizeSyncliteral) pack(c *conn) string {
@ -104,11 +105,19 @@ func (t readerSizeSyncliteral) pack(c *conn) string {
if err != nil {
panic(err)
}
return fmt.Sprintf("{%d}\r\n", t.size) + string(buf)
var lit string
if t.lit8 {
lit = "~"
}
return fmt.Sprintf("%s{%d}\r\n", lit, t.size) + string(buf)
}
func (t readerSizeSyncliteral) writeTo(c *conn, w io.Writer) {
fmt.Fprintf(w, "{%d}\r\n", t.size)
var lit string
if t.lit8 {
lit = "~"
}
fmt.Fprintf(w, "%s{%d}\r\n", lit, t.size)
defer c.xtrace(mlog.LevelTracedata)()
if _, err := io.Copy(w, io.LimitReader(t.r, t.size)); err != nil {
panic(err)

View File

@ -305,10 +305,10 @@ func (p *parser) xstring() (r string) {
p.xerrorf("missing closing dquote in string")
}
size, sync := p.xliteralSize(false, true)
s := p.conn.xreadliteral(size, sync)
buf := p.conn.xreadliteral(size, sync)
line := p.conn.readline(false)
p.orig, p.upper, p.o = line, toUpper(line), 0
return s
return string(buf)
}
func (p *parser) xnil() {
@ -974,3 +974,53 @@ func (p *parser) xdate() time.Time {
}
return time.Date(year, mon, day, 0, 0, 0, 0, time.UTC)
}
// Parse and validate a metadata key (entry name), returned as lower-case.
//
// ../rfc/5464:190
func (p *parser) xmetadataKey() string {
// ../rfc/5464:772
s := p.xastring()
// ../rfc/5464:192
if strings.Contains(s, "//") {
p.xerrorf("entry name must not contain two slashes")
}
// We allow a single slash, so it can be used with option "(depth infinity)" to get
// all annotations.
if s != "/" && strings.HasSuffix(s, "/") {
p.xerrorf("entry name must not end with slash")
}
// ../rfc/5464:202
if strings.Contains(s, "*") || strings.Contains(s, "%") {
p.xerrorf("entry name must not contain * or %%")
}
for _, c := range s {
if c < ' ' || c >= 0x7f {
p.xerrorf("entry name must only contain non-control ascii characters")
}
}
return strings.ToLower(s)
}
// ../rfc/5464:776
func (p *parser) xmetadataKeyValue() (key string, isString bool, value []byte) {
key = p.xmetadataKey()
p.xspace()
if p.hasPrefix("~{") {
size, sync := p.xliteralSize(true, true)
value = p.conn.xreadliteral(size, sync)
line := p.conn.readline(false)
p.orig, p.upper, p.o = line, toUpper(line), 0
} else if p.hasPrefix(`"`) {
value = []byte(p.xstring())
isString = true
} else if p.take("NIL") {
value = nil
} else {
p.xerrorf("expected metadata value")
}
return
}

View File

@ -158,12 +158,13 @@ var authFailDelay = time.Second // After authentication failure.
// QRESYNC: ../rfc/7162:1323
// STATUS=SIZE: ../rfc/8438 ../rfc/9051:8024
// QUOTA QUOTA=RES-STORAGE: ../rfc/9208:111
// METADATA: ../rfc/5464
//
// We always announce support for SCRAM PLUS-variants, also on connections without
// TLS. The client should not be selecting PLUS variants on non-TLS connections,
// instead opting to do the bare SCRAM variant without indicating the server claims
// to support the PLUS variant (skipping the server downgrade detection check).
const serverCapabilities = "IMAP4rev2 IMAP4rev1 ENABLE LITERAL+ IDLE SASL-IR BINARY UNSELECT UIDPLUS ESEARCH SEARCHRES MOVE UTF8=ACCEPT LIST-EXTENDED SPECIAL-USE LIST-STATUS AUTH=SCRAM-SHA-256-PLUS AUTH=SCRAM-SHA-256 AUTH=SCRAM-SHA-1-PLUS AUTH=SCRAM-SHA-1 AUTH=CRAM-MD5 ID APPENDLIMIT=9223372036854775807 CONDSTORE QRESYNC STATUS=SIZE QUOTA QUOTA=RES-STORAGE"
const serverCapabilities = "IMAP4rev2 IMAP4rev1 ENABLE LITERAL+ IDLE SASL-IR BINARY UNSELECT UIDPLUS ESEARCH SEARCHRES MOVE UTF8=ACCEPT LIST-EXTENDED SPECIAL-USE LIST-STATUS AUTH=SCRAM-SHA-256-PLUS AUTH=SCRAM-SHA-256 AUTH=SCRAM-SHA-1-PLUS AUTH=SCRAM-SHA-1 AUTH=CRAM-MD5 ID APPENDLIMIT=9223372036854775807 CONDSTORE QRESYNC STATUS=SIZE QUOTA QUOTA=RES-STORAGE METADATA"
type conn struct {
cid int64
@ -227,6 +228,7 @@ const (
capUTF8Accept capability = "UTF8=ACCEPT"
capCondstore capability = "CONDSTORE"
capQresync capability = "QRESYNC"
capMetadata capability = "METADATA"
)
type lineErr struct {
@ -253,7 +255,7 @@ func stateCommands(cmds ...string) map[string]struct{} {
var (
commandsStateAny = stateCommands("capability", "noop", "logout", "id")
commandsStateNotAuthenticated = stateCommands("starttls", "authenticate", "login")
commandsStateAuthenticated = stateCommands("enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub", "getquotaroot", "getquota")
commandsStateAuthenticated = stateCommands("enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub", "getquotaroot", "getquota", "getmetadata", "setmetadata")
commandsStateSelected = stateCommands("close", "unselect", "expunge", "search", "fetch", "store", "copy", "move", "uid expunge", "uid search", "uid fetch", "uid store", "uid copy", "uid move")
)
@ -286,6 +288,8 @@ var commands = map[string]func(c *conn, tag, cmd string, p *parser){
"idle": (*conn).cmdIdle,
"getquotaroot": (*conn).cmdGetquotaroot,
"getquota": (*conn).cmdGetquota,
"getmetadata": (*conn).cmdGetmetadata,
"setmetadata": (*conn).cmdSetmetadata,
// Selected.
"check": (*conn).cmdCheck,
@ -617,7 +621,7 @@ func (c *conn) readCommand(tag *string) (cmd string, p *parser) {
return cmd, newParser(p.remainder(), c)
}
func (c *conn) xreadliteral(size int64, sync bool) string {
func (c *conn) xreadliteral(size int64, sync bool) []byte {
if sync {
c.writelinef("+ ")
}
@ -633,7 +637,7 @@ func (c *conn) xreadliteral(size int64, sync bool) string {
panic(fmt.Errorf("reading literal: %s (%w)", err, errIO))
}
}
return string(buf)
return buf
}
func (c *conn) xhighestModSeq(tx *bstore.Tx, mailboxID int64) store.ModSeq {
@ -1541,6 +1545,14 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) {
case store.ChangeRemoveMailbox, store.ChangeAddMailbox, store.ChangeRenameMailbox, store.ChangeAddSubscription:
n = append(n, change)
continue
case store.ChangeAnnotation:
// note: annotations may have a mailbox associated with them, but we pass all
// changes on.
// Only when the metadata capability was enabled. ../rfc/5464:660
if c.enabled[capMetadata] {
n = append(n, change)
continue
}
case store.ChangeMailboxCounts, store.ChangeMailboxSpecialUse, store.ChangeMailboxKeywords, store.ChangeThread:
default:
panic(fmt.Errorf("missing case for %#v", change))
@ -1651,6 +1663,9 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) {
c.bwritelinef(`* LIST (%s) "/" %s%s`, strings.Join(ch.Flags, " "), astring(c.encodeMailbox(ch.NewName)).pack(c), oldname)
case store.ChangeAddSubscription:
c.bwritelinef(`* LIST (%s) "/" %s`, strings.Join(append([]string{`\Subscribed`}, ch.Flags...), " "), astring(c.encodeMailbox(ch.Name)).pack(c))
case store.ChangeAnnotation:
// ../rfc/5464:807 ../rfc/5464:788
c.bwritelinef(`* METADATA %s %s`, astring(c.encodeMailbox(ch.MailboxName)).pack(c), astring(ch.Key).pack(c))
default:
panic(fmt.Sprintf("internal error, missing case for %#v", change))
}
@ -2325,6 +2340,9 @@ func (c *conn) cmdEnable(tag, cmd string, p *parser) {
c.enabled[cap] = true
enabled += " " + s
qresync = true
case capMetadata:
c.enabled[cap] = true
enabled += " " + s
}
}
// QRESYNC enabled CONDSTORE too ../rfc/7162:1391
@ -2713,7 +2731,7 @@ func (c *conn) cmdCreate(tag, cmd string, p *parser) {
c.ok(tag, cmd)
}
// Delete removes a mailbox and all its messages.
// Delete removes a mailbox and all its messages and annotations.
// Inbox cannot be removed.
//
// State: Authenticated and selected.
@ -2760,7 +2778,8 @@ func (c *conn) cmdDelete(tag, cmd string, p *parser) {
}
// Rename changes the name of a mailbox.
// Renaming INBOX is special, it moves the inbox messages to a new mailbox, leaving inbox empty.
// Renaming INBOX is special, it moves the inbox messages to a new mailbox, leaving
// inbox empty, but copying metadata annotations.
// Renaming a mailbox with submailboxes also renames all submailboxes.
// Subscriptions stay with the old name, though newly created missing parent
// mailboxes for the destination name are automatically subscribed.
@ -2865,10 +2884,25 @@ func (c *conn) cmdRename(tag, cmd string, p *parser) {
if tx.Get(&store.Subscription{Name: dstMB.Name}) == nil {
dstFlags = []string{`\Subscribed`}
}
// Copy any annotations. ../rfc/5464:368
annotations, err := bstore.QueryTx[store.Annotation](tx).FilterNonzero(store.Annotation{MailboxID: srcMB.ID}).List()
xcheckf(err, "get annotations to copy for inbox")
for i := range annotations {
annotations[i].ID = 0
annotations[i].MailboxID = dstMB.ID
err := tx.Insert(&annotations[i])
xcheckf(err, "copy annotation to destination mailbox")
}
changes[0] = store.ChangeRemoveUIDs{MailboxID: srcMB.ID, UIDs: oldUIDs, ModSeq: modseq}
changes[1] = store.ChangeAddMailbox{Mailbox: dstMB, Flags: dstFlags}
// changes[2:...] are ChangeAddUIDs
changes = append(changes, srcMB.ChangeCounts(), dstMB.ChangeCounts())
for _, a := range annotations {
changes = append(changes, a.Change(dstMB.Name))
}
return
}

View File

@ -201,7 +201,12 @@ https://www.iana.org/assignments/message-headers/message-headers.xhtml
5258 Yes - Internet Message Access Protocol version 4 - LIST Command Extensions
5259 No - Internet Message Access Protocol - CONVERT Extension
5267 Roadmap - Contexts for IMAP4
5464 Roadmap - The IMAP METADATA Extension
5464 Yes - The IMAP METADATA Extension
5464-eid1691 - - errata: fix example entry name
5464-eid1692 - - errata: make text match abnf
5464-eid2785 - - errata: fix GETMETADATA example
5464-eid2786 - - errata: fix GETMETADATA example
5464-eid3868 - - errata: fix GETMETADATA example
5465 Roadmap - The IMAP NOTIFY Extension
5466 Roadmap - IMAP4 Extension for Named Searches (Filters)
5524 No - Extended URLFETCH for Binary and Converted Parts

View File

@ -216,6 +216,27 @@ type Mailbox struct {
MailboxCounts // Statistics about messages, kept up to date whenever a change happens.
}
// Annotation is a per-mailbox or global (per-account) annotation for the IMAP
// metadata extension, currently always a private annotation.
type Annotation struct {
ID int64
// Can be zero, indicates global (per-account) annotation.
MailboxID int64 `bstore:"ref Mailbox,unique MailboxID+Key"`
// "Entry name", always starts with "/private/". Stored lower-case, comparisons
// must be done case-insensitively.
Key string `bstore:"nonzero"`
IsString bool // If true, the value is a string instead of bytes.
Value []byte
}
// Change returns a broadcastable change for the annotation.
func (a Annotation) Change(mailboxName string) ChangeAnnotation {
return ChangeAnnotation{a.MailboxID, mailboxName, a.Key}
}
// MailboxCounts tracks statistics about messages for a mailbox.
type MailboxCounts struct {
Total int64 // Total number of messages, excluding \Deleted. For JMAP.
@ -818,6 +839,7 @@ var DBTypes = []any{
RulesetNoListID{},
RulesetNoMsgFrom{},
RulesetNoMailbox{},
Annotation{},
}
// Account holds the information about a user, includings mailboxes, messages, imap subscriptions.
@ -2713,8 +2735,8 @@ func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc Mailbox, dst string) (chang
return changes, false, false, false, nil
}
// MailboxDelete deletes a mailbox by ID. If it has children, the return value
// indicates that and an error is returned.
// MailboxDelete deletes a mailbox by ID, including its annotations. If it has
// children, the return value indicates that and an error is returned.
//
// Caller should broadcast the changes and remove files for the removed message IDs.
func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) {
@ -2785,6 +2807,13 @@ func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx
}
}
// Remove metadata annotations. ../rfc/5464:373
if _, err := bstore.QueryTx[Annotation](tx).FilterNonzero(Annotation{MailboxID: mailbox.ID}).Delete(); err != nil {
return nil, nil, false, fmt.Errorf("removing annotations for mailbox: %v", err)
}
// Not sending changes about annotations on this mailbox, since the entire mailbox
// is being removed.
if err := tx.Delete(&Mailbox{ID: mailbox.ID}); err != nil {
return nil, nil, false, fmt.Errorf("removing mailbox: %v", err)
}

View File

@ -105,6 +105,14 @@ type ChangeMailboxKeywords struct {
Keywords []string
}
// ChangeAnnotation is sent when an annotation is added/updated/removed, either for
// a mailbox or a global per-account annotation. The value is not included.
type ChangeAnnotation struct {
MailboxID int64 // Can be zero, meaning global (per-account) annotation.
MailboxName string // Empty for global (per-account) annotation.
Key string // Also called "entry name", e.g. "/private/comment".
}
var switchboardBusy atomic.Bool
// Switchboard distributes changes to accounts to interested listeners. See Comm and Change.

View File

@ -1230,7 +1230,7 @@ func (Webmail) MailboxCreate(ctx context.Context, name string) {
})
}
// MailboxDelete deletes a mailbox and all its messages.
// MailboxDelete deletes a mailbox and all its messages and annotations.
func (Webmail) MailboxDelete(ctx context.Context, mailboxID int64) {
reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)
acc := reqInfo.Account

View File

@ -276,7 +276,7 @@
},
{
"Name": "MailboxDelete",
"Docs": "MailboxDelete deletes a mailbox and all its messages.",
"Docs": "MailboxDelete deletes a mailbox and all its messages and annotations.",
"Params": [
{
"Name": "mailboxID",

View File

@ -894,7 +894,7 @@ export class Client {
return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params) as void
}
// MailboxDelete deletes a mailbox and all its messages.
// MailboxDelete deletes a mailbox and all its messages and annotations.
async MailboxDelete(mailboxID: number): Promise<void> {
const fn: string = "MailboxDelete"
const paramTypes: string[][] = [["int64"]]

View File

@ -573,7 +573,7 @@ var api;
const params = [name];
return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params);
}
// MailboxDelete deletes a mailbox and all its messages.
// MailboxDelete deletes a mailbox and all its messages and annotations.
async MailboxDelete(mailboxID) {
const fn = "MailboxDelete";
const paramTypes = [["int64"]];

View File

@ -573,7 +573,7 @@ var api;
const params = [name];
return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params);
}
// MailboxDelete deletes a mailbox and all its messages.
// MailboxDelete deletes a mailbox and all its messages and annotations.
async MailboxDelete(mailboxID) {
const fn = "MailboxDelete";
const paramTypes = [["int64"]];

View File

@ -573,7 +573,7 @@ var api;
const params = [name];
return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params);
}
// MailboxDelete deletes a mailbox and all its messages.
// MailboxDelete deletes a mailbox and all its messages and annotations.
async MailboxDelete(mailboxID) {
const fn = "MailboxDelete";
const paramTypes = [["int64"]];