imapserver: allow creating mailboxes with characters &#*%, and encode mailbox names in imap with imaputf7 when needed

the imapserver started with imap4rev2-only and utf8=only.  to prevent potential
issues with imaputf7, which makes "&" special, we refused any mailbox with an
"&" in the name. we already tried decoding utf7, falling back to using a
mailbox name verbatim. that behaviour wasn't great. we now treat the enabled
extensions IMAP4rev2 and/or UTF8=ACCEPT as indication whether mailbox names are
in imaputf7. if they are, the encoding must be correct.

we now also send mailbox names in imaputf7 when imap4rev2/utf8=accept isn't
enabled.

and we now allow "*" and "%" (wildcard characters for matching) in mailbox
names. not ideal for IMAP LIST with patterns, but not enough reason to refuse
them in mailbox names. people that migrate may run into this, possibly as
blocker.

we also allow "#" in mailbox names, but not as first character, to prevent
potential clashes with IMAP namespaces in the future.

based on report from Damian Poddebniak using
https://github.com/duesee/imap-flow and issue #110, thanks for reporting!

imapserver/utf7.go

@@ -1,11 +1,17 @@
 package imapserver
 
 import (
+	"bytes"
 	"encoding/base64"
 	"errors"
 	"fmt"
+	"unicode/utf16"
 )
 
+// IMAP4rev1 uses a modified version of UTF-7.
+// ../rfc/3501:1050
+// ../rfc/2152:69
+
 const utf7chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,"
 
 var utf7encoding = base64.NewEncoding(utf7chars).WithPadding(base64.NoPadding)
@@ -16,6 +22,7 @@ var (
 	errUTF7OddSized         = errors.New("utf7: odd-sized data")
 	errUTF7UnneededShift    = errors.New("utf7: unneeded shift")
 	errUTF7UnfinishedShift  = errors.New("utf7: unfinished shift")
+	errUTF7BadSurrogate     = errors.New("utf7: bad utf16 surrogates")
 )
 
 func utf7decode(s string) (string, error) {
@@ -60,20 +67,36 @@ func utf7decode(s string) (string, error) {
 
 		x := make([]rune, len(buf)/2)
 		j := 0
+		trymerge := false
 		for i := 0; i < len(buf); i += 2 {
 			x[j] = rune(buf[i])<<8 | rune(buf[i+1])
+			if trymerge {
+				s0 := utf16.IsSurrogate(x[j-1])
+				s1 := utf16.IsSurrogate(x[j])
+				if s0 && s1 {
+					c := utf16.DecodeRune(x[j-1], x[j])
+					if c == 0xfffd {
+						return "", fmt.Errorf("%w: decoding %x %x", errUTF7BadSurrogate, x[j-1], x[j])
+					}
+					x[j-1] = c
+					trymerge = false
+					continue
+				} else if s0 != s1 {
+					return "", fmt.Errorf("%w: not both surrogate: %x %x", errUTF7BadSurrogate, x[j-1], x[j])
+				}
+			}
 			j++
+			trymerge = true
 		}
+		x = x[:j]
 
-		need := false
 		for _, c := range x {
 			if c < 0x20 || c > 0x7e || c == '&' {
-				need = true
+				r += string(c)
+			} else {
+				// ../rfc/3501:1057
+				return "", errUTF7UnneededShift
 			}
-			r += string(c)
-		}
-		if !need {
-			return "", errUTF7UnneededShift
 		}
 	}
 	if shifted {
@@ -81,3 +104,43 @@ func utf7decode(s string) (string, error) {
 	}
 	return r, nil
 }
+
+func utf7encode(s string) string {
+	var r string
+	var code string
+
+	flushcode := func() {
+		if code == "" {
+			return
+		}
+		var b bytes.Buffer
+		for _, c := range code {
+			high, low := utf16.EncodeRune(c)
+			if high == 0xfffd && low == 0xfffd {
+				b.WriteByte(byte(c >> 8))
+				b.WriteByte(byte(c >> 0))
+			} else {
+				b.WriteByte(byte(high >> 8))
+				b.WriteByte(byte(high >> 0))
+				b.WriteByte(byte(low >> 8))
+				b.WriteByte(byte(low >> 0))
+			}
+		}
+		r += "&" + utf7encoding.EncodeToString(b.Bytes()) + "-"
+		code = ""
+	}
+
+	for _, c := range s {
+		if c == '&' {
+			flushcode()
+			r += "&-"
+		} else if c >= ' ' && c < 0x7f {
+			flushcode()
+			r += string(c)
+		} else {
+			code += string(c)
+		}
+	}
+	flushcode()
+	return r
+}