implement IMAP CREATE-SPECIAL-USE extension for the mailbox create command, part of rfc 6154

we already supported special-use flags. settable through the webmail interface,
and new accounts already got standard mailboxes with special-use flags
predefined. but now the IMAP "CREATE" command implements creating mailboxes
with special-use flags.

imapclient/client.go

@@ -34,8 +34,8 @@ type Conn struct {
 
 	Preauth      bool
 	LastTag      string
-	CapAvailable map[Capability]struct{} // Capabilities available at server, from CAPABILITY command or response code.
+	CapAvailable map[Capability]struct{} // Capabilities available at server, from CAPABILITY command or response code. All uppercase.
-	CapEnabled   map[Capability]struct{} // Capabilities enabled through ENABLE command.
+	CapEnabled   map[Capability]struct{} // Capabilities enabled through ENABLE command. All uppercase.
 }
 
 // Error is a parse or other protocol error.

imapclient/cmds.go

@@ -149,9 +149,18 @@ func (c *Conn) Examine(mailbox string) (untagged []Untagged, result Result, rerr
 }
 
 // Create makes a new mailbox on the server.
-func (c *Conn) Create(mailbox string) (untagged []Untagged, result Result, rerr error) {
+// SpecialUse can only be used on servers that announced the CREATE-SPECIAL-USE
+// capability. Specify flags like \Archive, \Draft, \Junk, \Sent, \Trash, \All.
+func (c *Conn) Create(mailbox string, specialUse []string) (untagged []Untagged, result Result, rerr error) {
 	defer c.recover(&rerr)
-	return c.Transactf("create %s", astring(mailbox))
+	if _, ok := c.CapAvailable[CapCreateSpecialUse]; !ok && len(specialUse) > 0 {
+		c.xerrorf("server does not implement create-special-use extension")
+	}
+	var useStr string
+	if len(specialUse) > 0 {
+		useStr = fmt.Sprintf(" USE (%s)", strings.Join(specialUse, " "))
+	}
+	return c.Transactf("create %s%s", astring(mailbox), useStr)
 }
 
 // Delete removes an entire mailbox and its messages.

imapclient/parse.go

@@ -181,6 +181,7 @@ func (c *Conn) xrespCode() (string, CodeArg) {
 		}
 		c.CapAvailable = map[Capability]struct{}{}
 		for _, cap := range caps {
+			cap = strings.ToUpper(cap)
 			c.CapAvailable[Capability(cap)] = struct{}{}
 		}
 		codeArg = CodeWords{W, caps}
@@ -343,6 +344,7 @@ func (c *Conn) xuntagged() Untagged {
 		}
 		c.CapAvailable = map[Capability]struct{}{}
 		for _, cap := range caps {
+			cap = strings.ToUpper(cap)
 			c.CapAvailable[Capability(cap)] = struct{}{}
 		}
 		r := UntaggedCapability(caps)
@@ -356,6 +358,7 @@ func (c *Conn) xuntagged() Untagged {
 			caps = append(caps, c.xnonspace())
 		}
 		for _, cap := range caps {
+			cap = strings.ToUpper(cap)
 			c.CapEnabled[Capability(cap)] = struct{}{}
 		}
 		r := UntaggedEnabled(caps)

imapclient/protocol.go

@@ -34,6 +34,8 @@ const (
 	CapID               Capability = "ID"                 // ../rfc/2971:80
 	CapMetadata         Capability = "METADATA"           // ../rfc/5464:124
 	CapMetadataServer   Capability = "METADATA-SERVER"    // ../rfc/5464:124
+	CapSaveDate         Capability = "SAVEDATE"           // ../rfc/8514
+	CapCreateSpecialUse Capability = "CREATE-SPECIAL-USE" // ../rfc/6154:296
 )
 
 // Status is the tagged final result of a command.

imapserver/condstore_test.go

@@ -96,7 +96,7 @@ func testCondstoreQresync(t *testing.T, qresync bool) {
 	err = tc.account.DB.Update(ctxbg, &store.SyncState{ID: 1, LastModSeq: 1})
 	tcheck(t, err, "resetting modseq state")
 
-	tc.client.Create("otherbox")
+	tc.client.Create("otherbox", nil)
 
 	// tc2 is a client without condstore, so no modseq responses.
 	tc2 := startNoSwitchboard(t)

imapserver/create_test.go

@@ -72,6 +72,14 @@ func TestCreate(t *testing.T) {
 	tc.transactf("no", `create "#"`) // Leading hash not allowed.
 	tc.transactf("ok", `create "test#"`)
 
+	// Create with flags.
+	tc.transactf("no", `create "newwithflags" (use (\unknown))`)
+	tc.transactf("no", `create "newwithflags" (use (\all))`)
+	tc.transactf("ok", `create "newwithflags" (use (\archive))`)
+	tc.transactf("ok", "noop")
+	tc.xuntagged()
+	tc.transactf("ok", `create "newwithflags2" (use (\archive) use (\drafts \sent))`)
+
 	// UTF-7 checks are only for IMAP4 before rev2 and without UTF8=ACCEPT.
 	tc.transactf("ok", `create "&"`)      // Interpreted as UTF-8, no UTF-7.
 	tc2.transactf("bad", `create "&"`)    // Bad UTF-7.

imapserver/delete_test.go

@@ -28,7 +28,7 @@ func TestDelete(t *testing.T) {
 	tc.client.Subscribe("x")
 	tc.transactf("no", "delete x") // Subscription does not mean there is a mailbox that can be deleted.
 
-	tc.client.Create("a/b")
+	tc.client.Create("a/b", nil)
 	tc2.transactf("ok", "noop") // Drain changes.
 	tc3.transactf("ok", "noop")
 
@@ -53,12 +53,12 @@ func TestDelete(t *testing.T) {
 	)
 
 	// Let's try again with a message present.
-	tc.client.Create("msgs")
+	tc.client.Create("msgs", nil)
 	tc.client.Append("msgs", nil, nil, []byte(exampleMsg))
 	tc.transactf("ok", "delete msgs")
 
 	// Delete for inbox/* is allowed.
-	tc.client.Create("inbox/a")
+	tc.client.Create("inbox/a", nil)
 	tc.transactf("ok", "delete inbox/a")
 
 }

imapserver/list_test.go

@@ -35,7 +35,7 @@ func TestListBasic(t *testing.T) {
 	tc.last(tc.client.List("A*"))
 	tc.xuntagged(ulist("Archive", `\Archive`))
 
-	tc.client.Create("Inbox/todo")
+	tc.client.Create("Inbox/todo", nil)
 
 	tc.last(tc.client.List("Inbox*"))
 	tc.xuntagged(ulist("Inbox"), ulist("Inbox/todo"))
@@ -146,7 +146,7 @@ func TestListExtended(t *testing.T) {
 	tc.last(tc.client.ListFull(false, "A*", "Junk"))
 	tc.xuntagged(xlist("Archive", Farchive), ustatus("Archive"), xlist("Junk", Fjunk), ustatus("Junk"))
 
-	tc.client.Create("Inbox/todo")
+	tc.client.Create("Inbox/todo", nil)
 
 	tc.last(tc.client.ListFull(false, "Inbox*"))
 	tc.xuntagged(ulist("Inbox", Fhaschildren, Fsubscribed), ustatus("Inbox"), xlist("Inbox/todo"), ustatus("Inbox/todo"))
@@ -204,7 +204,7 @@ func TestListExtended(t *testing.T) {
 	tc.transactf("ok", `list (remote) "inbox" "a"`)
 	tc.xuntagged()
 
-	tc.client.Create("inbox/a")
+	tc.client.Create("inbox/a", nil)
 	tc.transactf("ok", `list (remote) "inbox" "a"`)
 	tc.xuntagged(ulist("Inbox/a"))
 

imapserver/rename_test.go

@@ -26,9 +26,9 @@ func TestRename(t *testing.T) {
 	tc.transactf("no", `rename "Sent" "Trash"`)      // Already exists.
 	tc.xcode("ALREADYEXISTS")
 
-	tc.client.Create("x")
+	tc.client.Create("x", nil)
 	tc.client.Subscribe("sub")
-	tc.client.Create("a/b/c")
+	tc.client.Create("a/b/c", nil)
 	tc.client.Subscribe("x/y/c") // For later rename, but not affected by rename of x.
 	tc2.transactf("ok", "noop")  // Drain.
 
@@ -58,7 +58,7 @@ func TestRename(t *testing.T) {
 	tc2.transactf("ok", "noop")
 	tc2.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "x"}, imapclient.UntaggedList{Separator: '/', Mailbox: "x/y", OldName: "a/b"}, imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "x/y/c", OldName: "a/b/c"})
 
-	tc.client.Create("k/l")
+	tc.client.Create("k/l", nil)
 	tc.transactf("ok", "rename k/l k/l/m") // With "l" renamed, a new "k" will be created.
 	tc.transactf("ok", `list "" "k*" return (subscribed)`)
 	tc.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "k"}, imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "k/l"}, imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/m"})

imapserver/server.go

@@ -28,7 +28,7 @@ non-ASCII UTF-8. Until that's enabled, we do use UTF-7 for mailbox names. See
 - todo: do not return binary data for a fetch body. at least not for imap4rev1. we should be encoding it as base64?
 - todo: on expunge we currently remove the message even if other sessions still have a reference to the uid. if they try to query the uid, they'll get an error. we could be nicer and only actually remove the message when the last reference has gone. we could add a new flag to store.Message marking the message as expunged, not give new session access to such messages, and make store remove them at startup, and clean them when the last session referencing the session goes. however, it will get much more complicated. renaming messages would need special handling. and should we do the same for removed mailboxes?
 - todo: try to recover from syntax errors when the last command line ends with a }, i.e. a literal. we currently abort the entire connection. we may want to read some amount of literal data and continue with a next command.
-- todo future: more extensions: OBJECTID, MULTISEARCH, REPLACE, NOTIFY, CATENATE, MULTIAPPEND, SORT, THREAD, CREATE-SPECIAL-USE.
+- todo future: more extensions: OBJECTID, MULTISEARCH, REPLACE, NOTIFY, CATENATE, MULTIAPPEND, SORT, THREAD.
 */
 
 import (
@@ -146,7 +146,7 @@ var authFailDelay = time.Second  // After authentication failure.
 // MOVE: ../rfc/6851
 // UTF8=ONLY: ../rfc/6855
 // LIST-EXTENDED: ../rfc/5258
-// SPECIAL-USE: ../rfc/6154
+// SPECIAL-USE CREATE-SPECIAL-USE: ../rfc/6154
 // LIST-STATUS: ../rfc/5819
 // ID: ../rfc/2971
 // AUTH=EXTERNAL: ../rfc/4422:1575
@@ -165,7 +165,7 @@ var authFailDelay = time.Second  // After authentication failure.
 // TLS. The client should not be selecting PLUS variants on non-TLS connections,
 // instead opting to do the bare SCRAM variant without indicating the server claims
 // to support the PLUS variant (skipping the server downgrade detection check).
-const serverCapabilities = "IMAP4rev2 IMAP4rev1 ENABLE LITERAL+ IDLE SASL-IR BINARY UNSELECT UIDPLUS ESEARCH SEARCHRES MOVE UTF8=ACCEPT LIST-EXTENDED SPECIAL-USE LIST-STATUS AUTH=SCRAM-SHA-256-PLUS AUTH=SCRAM-SHA-256 AUTH=SCRAM-SHA-1-PLUS AUTH=SCRAM-SHA-1 AUTH=CRAM-MD5 ID APPENDLIMIT=9223372036854775807 CONDSTORE QRESYNC STATUS=SIZE QUOTA QUOTA=RES-STORAGE METADATA SAVEDATE"
+const serverCapabilities = "IMAP4rev2 IMAP4rev1 ENABLE LITERAL+ IDLE SASL-IR BINARY UNSELECT UIDPLUS ESEARCH SEARCHRES MOVE UTF8=ACCEPT LIST-EXTENDED SPECIAL-USE CREATE-SPECIAL-USE LIST-STATUS AUTH=SCRAM-SHA-256-PLUS AUTH=SCRAM-SHA-256 AUTH=SCRAM-SHA-1-PLUS AUTH=SCRAM-SHA-1 AUTH=CRAM-MD5 ID APPENDLIMIT=9223372036854775807 CONDSTORE QRESYNC STATUS=SIZE QUOTA QUOTA=RES-STORAGE METADATA SAVEDATE"
 
 type conn struct {
 	cid               int64
@@ -2710,13 +2710,50 @@ func (c *conn) cmdCreate(tag, cmd string, p *parser) {
 	// Request syntax: ../rfc/9051:6484 ../rfc/6154:468 ../rfc/4466:500 ../rfc/3501:4687
 	p.xspace()
 	name := p.xmailbox()
-	// todo: support CREATE-SPECIAL-USE ../rfc/6154:296
+	// Optional parameters. ../rfc/4466:501 ../rfc/4466:511
+	var useAttrs []string // Special-use attributes without leading \.
+	if p.space() {
+		p.xtake("(")
+		// We only support "USE", and there don't appear to be more types of parameters.
+		for {
+			p.xtake("USE (")
+			for {
+				p.xtake(`\`)
+				useAttrs = append(useAttrs, p.xatom())
+				if !p.space() {
+					break
+				}
+			}
+			p.xtake(")")
+			if !p.space() {
+				break
+			}
+		}
+		p.xtake(")")
+	}
 	p.xempty()
 
 	origName := name
 	name = strings.TrimRight(name, "/") // ../rfc/9051:1930
 	name = xcheckmailboxname(name, false)
 
+	var specialUse store.SpecialUse
+	specialUseBools := map[string]*bool{
+		"archive": &specialUse.Archive,
+		"drafts":  &specialUse.Draft,
+		"junk":    &specialUse.Junk,
+		"sent":    &specialUse.Sent,
+		"trash":   &specialUse.Trash,
+	}
+	for _, s := range useAttrs {
+		p, ok := specialUseBools[strings.ToLower(s)]
+		if !ok {
+			// ../rfc/6154:287
+			xusercodeErrorf("USEATTR", `cannot create mailbox with special-use attribute \%s`, s)
+		}
+		*p = true
+	}
+
 	var changes []store.Change
 	var created []string // Created mailbox names.
 
@@ -2724,7 +2761,7 @@ func (c *conn) cmdCreate(tag, cmd string, p *parser) {
 		c.xdbwrite(func(tx *bstore.Tx) {
 			var exists bool
 			var err error
-			changes, created, exists, err = c.account.MailboxCreate(tx, name)
+			changes, created, exists, err = c.account.MailboxCreate(tx, name, specialUse)
 			if exists {
 				// ../rfc/9051:1914
 				xuserErrorf("mailbox already exists")

imapserver/server_test.go

@@ -513,7 +513,7 @@ func TestLiterals(t *testing.T) {
 	defer tc.close()
 
 	tc.client.Login("mjl@mox.example", password0)
-	tc.client.Create("tmpbox")
+	tc.client.Create("tmpbox", nil)
 
 	tc.transactf("ok", "rename {6+}\r\ntmpbox {7+}\r\nntmpbox")
 
@@ -642,7 +642,7 @@ func TestMailboxDeleted(t *testing.T) {
 	tc.client.Login("mjl@mox.example", password0)
 	tc2.client.Login("mjl@mox.example", password0)
 
-	tc.client.Create("testbox")
+	tc.client.Create("testbox", nil)
 	tc2.client.Select("testbox")
 	tc.client.Delete("testbox")
 
@@ -663,7 +663,7 @@ func TestMailboxDeleted(t *testing.T) {
 
 	tc2.transactf("ok", "unselect")
 
-	tc.client.Create("testbox")
+	tc.client.Create("testbox", nil)
 	tc2.client.Select("testbox")
 	tc.client.Delete("testbox")
 	tc2.transactf("ok", "close")

import.go

@@ -298,7 +298,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
 	a.WithWLock(func() {
 		// Ensure mailbox exists.
 		var mb store.Mailbox
-		mb, changes, err = a.MailboxEnsure(tx, mailbox, true)
+		mb, changes, err = a.MailboxEnsure(tx, mailbox, true, store.SpecialUse{})
 		ctl.xcheck(err, "ensuring mailbox exists")
 
 		// We ensure keywords in messages make it to the mailbox as well.

store/account.go

@@ -1729,9 +1729,13 @@ func (a *Account) Subjectpass(email string) (key string, err error) {
 // parents if they aren't present.
 //
 // If subscribe is true, any mailboxes that were created will also be subscribed to.
+//
+// The leaf mailbox is created with special-use flags, taking the flags away from
+// other mailboxes, and reflecting that in the returned changes.
+//
 // Caller must hold account wlock.
 // Caller must propagate changes if any.
-func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool) (mb Mailbox, changes []Change, rerr error) {
+func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool, specialUse SpecialUse) (mb Mailbox, changes []Change, rerr error) {
 	if norm.NFC.String(name) != name {
 		return Mailbox{}, nil, fmt.Errorf("mailbox name not normalized")
 	}
@@ -1757,14 +1761,14 @@ func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool) (mb
 	}
 
 	p := ""
+	var existed bool
 	for _, elem := range elems {
 		if p != "" {
 			p += "/"
 		}
 		p += elem
-		var ok bool
+		mb, existed = mailboxes[p]
-		mb, ok = mailboxes[p]
+		if existed {
-		if ok {
 			continue
 		}
 		uidval, err := a.NextUIDValidity(tx)
@@ -1794,6 +1798,49 @@ func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool) (mb
 		}
 		changes = append(changes, ChangeAddMailbox{mb, flags})
 	}
+
+	// Clear any special-use flags from existing mailboxes and assign them to this mailbox.
+	var zeroSpecialUse SpecialUse
+	if !existed && specialUse != zeroSpecialUse {
+		var qerr error
+		clearSpecialUse := func(b bool, fn func(*Mailbox) *bool) {
+			if !b || qerr != nil {
+				return
+			}
+			qs := bstore.QueryTx[Mailbox](tx)
+			qs.FilterFn(func(xmb Mailbox) bool {
+				return *fn(&xmb)
+			})
+			xmb, err := qs.Get()
+			if err == bstore.ErrAbsent {
+				return
+			} else if err != nil {
+				qerr = fmt.Errorf("looking up mailbox with special-use flag: %v", err)
+				return
+			}
+			p := fn(&xmb)
+			*p = false
+			if err := tx.Update(&xmb); err != nil {
+				qerr = fmt.Errorf("clearing special-use flag: %v", err)
+			} else {
+				changes = append(changes, ChangeMailboxSpecialUse{xmb.ID, xmb.Name, xmb.SpecialUse})
+			}
+		}
+		clearSpecialUse(specialUse.Archive, func(xmb *Mailbox) *bool { return &xmb.Archive })
+		clearSpecialUse(specialUse.Draft, func(xmb *Mailbox) *bool { return &xmb.Draft })
+		clearSpecialUse(specialUse.Junk, func(xmb *Mailbox) *bool { return &xmb.Junk })
+		clearSpecialUse(specialUse.Sent, func(xmb *Mailbox) *bool { return &xmb.Sent })
+		clearSpecialUse(specialUse.Trash, func(xmb *Mailbox) *bool { return &xmb.Trash })
+		if qerr != nil {
+			return Mailbox{}, nil, qerr
+		}
+
+		mb.SpecialUse = specialUse
+		if err := tx.Update(&mb); err != nil {
+			return Mailbox{}, nil, fmt.Errorf("setting special-use flag for new mailbox: %v", err)
+		}
+		changes = append(changes, ChangeMailboxSpecialUse{mb.ID, mb.Name, mb.SpecialUse})
+	}
 	return mb, changes, nil
 }
 
@@ -1968,7 +2015,7 @@ func (a *Account) DeliverMailbox(log mlog.Log, mailbox string, m *Message, msgFi
 			return ErrOverQuota
 		}
 
-		mb, chl, err := a.MailboxEnsure(tx, mailbox, true)
+		mb, chl, err := a.MailboxEnsure(tx, mailbox, true, SpecialUse{})
 		if err != nil {
 			return fmt.Errorf("ensuring mailbox: %w", err)
 		}
@@ -2601,8 +2648,11 @@ func (a *Account) SendLimitReached(tx *bstore.Tx, recipients []smtp.Path) (msgli
 // the total list of created mailboxes is returned in created. On success, if
 // exists is false and rerr nil, the changes must be broadcasted by the caller.
 //
+// The mailbox is created with special-use flags, with those flags taken away from
+// other mailboxes if they have them, reflected in the returned changes.
+//
 // Name must be in normalized form.
-func (a *Account) MailboxCreate(tx *bstore.Tx, name string) (changes []Change, created []string, exists bool, rerr error) {
+func (a *Account) MailboxCreate(tx *bstore.Tx, name string, specialUse SpecialUse) (changes []Change, created []string, exists bool, rerr error) {
 	elems := strings.Split(name, "/")
 	var p string
 	for i, elem := range elems {
@@ -2620,9 +2670,9 @@ func (a *Account) MailboxCreate(tx *bstore.Tx, name string) (changes []Change, c
 			}
 			continue
 		}
-		_, nchanges, err := a.MailboxEnsure(tx, p, true)
+		_, nchanges, err := a.MailboxEnsure(tx, p, true, specialUse)
 		if err != nil {
-			return nil, nil, false, fmt.Errorf("ensuring mailbox exists")
+			return nil, nil, false, fmt.Errorf("ensuring mailbox exists: %v", err)
 		}
 		changes = append(changes, nchanges...)
 		created = append(created, p)

store/account_test.go

@@ -168,18 +168,18 @@ func TestMailbox(t *testing.T) {
 
 	acc.WithWLock(func() {
 		err := acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
-			_, _, err := acc.MailboxEnsure(tx, "Testbox", true)
+			_, _, err := acc.MailboxEnsure(tx, "Testbox", true, SpecialUse{})
 			return err
 		})
 		tcheck(t, err, "ensure mailbox exists")
 		err = acc.DB.Read(ctxbg, func(tx *bstore.Tx) error {
-			_, _, err := acc.MailboxEnsure(tx, "Testbox", true)
+			_, _, err := acc.MailboxEnsure(tx, "Testbox", true, SpecialUse{})
 			return err
 		})
 		tcheck(t, err, "ensure mailbox exists")
 
 		err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
-			_, _, err := acc.MailboxEnsure(tx, "Testbox2", false)
+			_, _, err := acc.MailboxEnsure(tx, "Testbox2", false, SpecialUse{})
 			tcheck(t, err, "create mailbox")
 
 			exists, err := acc.MailboxExists(tx, "Testbox2")

webmail/api.go

@@ -1219,7 +1219,7 @@ func (Webmail) MailboxCreate(ctx context.Context, name string) {
 		xdbwrite(ctx, acc, func(tx *bstore.Tx) {
 			var exists bool
 			var err error
-			changes, _, exists, err = acc.MailboxCreate(tx, name)
+			changes, _, exists, err = acc.MailboxCreate(tx, name, store.SpecialUse{})
 			if exists {
 				xcheckuserf(ctx, errors.New("mailbox already exists"), "creating mailbox")
 			}