implement IMAP extension COMPRESS=DEFLATE, rfc 4978

to compress the entire IMAP connection. tested with thunderbird, meli, k9, ios
mail. the initial implementation had interoperability issues with some of these
clients: if they write the deflate stream and flush in "partial mode", the go
stdlib flate reader does not return any data (until there is an explicit
zero-length "sync flush" block, or until the history/sliding window is full),
blocking progress, resulting in clients closing the seemingly stuck connection
after considering the connection timed out. this includes a coy of the flate
package with a new reader that returns partially flushed blocks earlier.

this also adds imap trace logging to imapclient.Conn, which was useful for
debugging.

imapclient/client.go

@@ -21,12 +21,26 @@ import (
 	"net"
 	"reflect"
 	"strings"
+
+	"github.com/mjl-/flate"
+
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/moxio"
 )
 
 // Conn is an IMAP connection to a server.
 type Conn struct {
-	conn      net.Conn
-	r         *bufio.Reader
+	// Connection, may be original TCP or TLS connection. Reads go through c.br, and
+	// writes through c.bw. It wraps a tracing reading/writer and may wrap flate
+	// compression.
+	conn        net.Conn
+	br          *bufio.Reader
+	bw          *bufio.Writer
+	compress    bool // If compression is enabled, we must flush flateWriter and its target original bufio writer.
+	flateWriter *flate.Writer
+	flateBW     *bufio.Writer
+
+	log       mlog.Log
 	panic     bool
 	tagGen    int
 	record    bool // If true, bytes read are added to recordBuf. recorded() resets.
@@ -57,14 +71,18 @@ func (e Error) Unwrap() error {
 // The initial untagged greeting response is read and must be "OK" or
 // "PREAUTH". If preauth, the connection is already in authenticated state,
 // typically through TLS client certificate. This is indicated in Conn.Preauth.
-func New(conn net.Conn, xpanic bool) (client *Conn, rerr error) {
+func New(cid int64, conn net.Conn, xpanic bool) (client *Conn, rerr error) {
+	log := mlog.New("imapclient", nil).WithCid(cid)
 	c := Conn{
 		conn:         conn,
-		r:            bufio.NewReader(conn),
+		br:           bufio.NewReader(moxio.NewTraceReader(log, "CR: ", conn)),
+		log:          log,
 		panic:        xpanic,
 		CapAvailable: map[Capability]struct{}{},
 		CapEnabled:   map[Capability]struct{}{},
 	}
+	// Writes are buffered and write to Conn, which may panic.
+	c.bw = bufio.NewWriter(moxio.NewTraceWriter(log, "CW: ", &c))
 
 	defer c.recover(&rerr)
 	tag := c.xnonspace()
@@ -122,6 +140,53 @@ func (c *Conn) xcheck(err error) {
 	}
 }
 
+// Write writes directly to the connection. Write errors do take the connection's
+// panic mode into account, i.e. Write can panic.
+func (c *Conn) Write(buf []byte) (n int, rerr error) {
+	defer c.recover(&rerr)
+
+	n, rerr = c.conn.Write(buf)
+	c.xcheckf(rerr, "write")
+	return n, nil
+}
+
+func (c *Conn) xflush() {
+	err := c.bw.Flush()
+	c.xcheckf(err, "flush")
+
+	// If compression is active, we need to flush the deflate stream.
+	if c.compress {
+		err := c.flateWriter.Flush()
+		c.xcheckf(err, "flush deflate")
+		err = c.flateBW.Flush()
+		c.xcheckf(err, "flush deflate buffer")
+	}
+}
+
+// Close closes the connection, flushing and closing any compression and TLS layer.
+//
+// You may want to call Logout first. Closing a connection with a mailbox with
+// deleted messages not yet expunged will not expunge those messages.
+func (c *Conn) Close() (rerr error) {
+	defer c.recover(&rerr)
+
+	if c.conn == nil {
+		return nil
+	}
+	if c.flateWriter != nil {
+		err := c.flateWriter.Close()
+		c.xcheckf(err, "close deflate writer")
+		err = c.flateBW.Flush()
+		c.xcheckf(err, "flush deflate buffer")
+		c.flateWriter = nil
+		c.flateBW = nil
+	}
+	err := c.conn.Close()
+	c.xcheckf(err, "close connection")
+	c.conn = nil
+	return
+}
+
 // TLSConnectionState returns the TLS connection state if the connection uses TLS.
 func (c *Conn) TLSConnectionState() *tls.ConnectionState {
 	if conn, ok := c.conn.(*tls.Conn); ok {
@@ -141,8 +206,9 @@ func (c *Conn) Commandf(tag string, format string, args ...any) (rerr error) {
 	}
 	c.LastTag = tag
 
-	_, err := fmt.Fprintf(c.conn, "%s %s\r\n", tag, fmt.Sprintf(format, args...))
+	_, err := fmt.Fprintf(c.bw, "%s %s\r\n", tag, fmt.Sprintf(format, args...))
 	c.xcheckf(err, "write command")
+	c.xflush()
 	return
 }
 
@@ -196,7 +262,7 @@ func (c *Conn) ReadUntagged() (untagged Untagged, rerr error) {
 func (c *Conn) Readline() (line string, rerr error) {
 	defer c.recover(&rerr)
 
-	line, err := c.r.ReadString('\n')
+	line, err := c.br.ReadString('\n')
 	c.xcheckf(err, "read line")
 	return line, nil
 }
@@ -225,37 +291,30 @@ func (c *Conn) Writelinef(format string, args ...any) (rerr error) {
 	defer c.recover(&rerr)
 
 	s := fmt.Sprintf(format, args...)
-	_, err := fmt.Fprintf(c.conn, "%s\r\n", s)
+	_, err := fmt.Fprintf(c.bw, "%s\r\n", s)
 	c.xcheckf(err, "writeline")
+	c.xflush()
 	return nil
 }
 
-// Write writes directly to the connection. Write errors do take the connections
-// panic mode into account, i.e. Write can panic.
-func (c *Conn) Write(buf []byte) (n int, rerr error) {
-	defer c.recover(&rerr)
-
-	n, rerr = c.conn.Write(buf)
-	c.xcheckf(rerr, "write")
-	return n, nil
-}
-
-// WriteSyncLiteral first writes the synchronous literal size, then read the
+// WriteSyncLiteral first writes the synchronous literal size, then reads the
 // continuation "+" and finally writes the data.
 func (c *Conn) WriteSyncLiteral(s string) (untagged []Untagged, rerr error) {
 	defer c.recover(&rerr)
 
-	_, err := fmt.Fprintf(c.conn, "{%d}\r\n", len(s))
+	_, err := fmt.Fprintf(c.bw, "{%d}\r\n", len(s))
 	c.xcheckf(err, "write sync literal size")
+	c.xflush()
 
-	plus, err := c.r.Peek(1)
+	plus, err := c.br.Peek(1)
 	c.xcheckf(err, "read continuation")
 	if plus[0] == '+' {
 		_, err = c.Readline()
 		c.xcheckf(err, "read continuation line")
 
-		_, err = c.conn.Write([]byte(s))
+		_, err = c.bw.Write([]byte(s))
 		c.xcheckf(err, "write literal data")
+		c.xflush()
 		return nil, nil
 	}
 	untagged, result, err := c.Response()
@@ -301,15 +360,3 @@ func (c *Conn) xgetUntagged(l []Untagged, dst any) {
 	}
 	dstv.Elem().Set(gotv)
 }
-
-// Close closes the connection without writing anything to the server.
-// You may want to call Logout. Closing a connection with a mailbox with deleted
-// message not yet expunged will not expunge those messages.
-func (c *Conn) Close() error {
-	var err error
-	if c.conn != nil {
-		err = c.conn.Close()
-		c.conn = nil
-	}
-	return err
-}