mox!

moxio/atreader.go

@@ -0,0 +1,20 @@
+package moxio
+
+import (
+	"io"
+)
+
+// AtReader is turns an io.ReaderAt into a io.Reader by keeping track of the
+// offset.
+type AtReader struct {
+	R      io.ReaderAt
+	Offset int64
+}
+
+func (r *AtReader) Read(buf []byte) (int, error) {
+	n, err := r.R.ReadAt(buf, r.Offset)
+	if n > 0 {
+		r.Offset += int64(n)
+	}
+	return n, err
+}

moxio/bufpool.go

@@ -0,0 +1,103 @@
+package moxio
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/mjl-/mox/mlog"
+)
+
+var xlog = mlog.New("moxio")
+
+// todo: instead of a bufpool, should maybe just make an alternative to bufio.Reader with a big enough buffer that we can fully use to read a line.
+
+var ErrLineTooLong = errors.New("line from remote too long") // Returned by Bufpool.Readline.
+
+// Bufpool caches byte slices for reuse during parsing of line-terminated commands.
+type Bufpool struct {
+	c    chan []byte
+	size int
+}
+
+// NewBufpool makes a new pool, initially empty, but holding at most "max" buffers of "size" bytes each.
+func NewBufpool(max, size int) *Bufpool {
+	return &Bufpool{
+		c:    make(chan []byte, max),
+		size: size,
+	}
+}
+
+// get returns a buffer from the pool if available, otherwise allocates a new buffer.
+// The buffer should be returned with a call to put.
+func (b *Bufpool) get() []byte {
+	var buf []byte
+
+	// Attempt to get buffer from pool. Otherwise create new buffer.
+	select {
+	case buf = <-b.c:
+	default:
+	}
+	if buf == nil {
+		buf = make([]byte, b.size)
+	}
+	return buf
+}
+
+// put puts a "buf" back in the pool. Put clears the first "n" bytes, which should
+// be all the bytes that have been read in the buffer. If the pool is full, the
+// buffer is discarded, and will be cleaned up by the garbage collector.
+// The caller should no longer reference "buf" after a call to put.
+func (b *Bufpool) put(buf []byte, n int) {
+	if len(buf) != b.size {
+		xlog.Error("buffer with bad size returned, ignoring", mlog.Field("badsize", len(buf)), mlog.Field("expsize", b.size))
+		return
+	}
+
+	for i := 0; i < n; i++ {
+		buf[i] = 0
+	}
+	select {
+	case b.c <- buf:
+	default:
+	}
+}
+
+// Readline reads a \n- or \r\n-terminated line. Line is returned without \n or \r\n.
+// If the line was too long, ErrLineTooLong is returned.
+// If an EOF is encountered before a \n, io.ErrUnexpectedEOF is returned.
+func (b *Bufpool) Readline(r *bufio.Reader) (line string, rerr error) {
+	var nread int
+	buf := b.get()
+	defer func() {
+		b.put(buf, nread)
+	}()
+
+	// Read until newline. If we reach the end of the buffer first, we write back an
+	// error and abort the connection because our protocols cannot be recovered. We
+	// don't want to consume data until we finally see a newline, which may be never.
+	for {
+		if nread >= len(buf) {
+			return "", fmt.Errorf("%w: no newline after all %d bytes", ErrLineTooLong, nread)
+		}
+		c, err := r.ReadByte()
+		if err == io.EOF {
+			return "", io.ErrUnexpectedEOF
+		} else if err != nil {
+			return "", fmt.Errorf("reading line from remote: %w", err)
+		}
+		if c == '\n' {
+			var s string
+			if nread > 0 && buf[nread-1] == '\r' {
+				s = string(buf[:nread-1])
+			} else {
+				s = string(buf[:nread])
+			}
+			nread++
+			return s, nil
+		}
+		buf[nread] = c
+		nread++
+	}
+}

moxio/bufpool_test.go

@@ -0,0 +1,57 @@
+package moxio
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"testing"
+)
+
+func TestBufpool(t *testing.T) {
+	bp := NewBufpool(1, 8)
+	a := bp.get()
+	b := bp.get()
+	for i := 0; i < len(a); i++ {
+		a[i] = 1
+	}
+	bp.put(a, len(a)) // Will be stored.
+	bp.put(b, 0)      // Will be discarded.
+	na := bp.get()
+	if fmt.Sprintf("%p", a) != fmt.Sprintf("%p", na) {
+		t.Fatalf("received unexpected new buf %p != %p", a, na)
+	}
+	for _, c := range na {
+		if c != 0 {
+			t.Fatalf("reused buf not cleared")
+		}
+	}
+
+	if _, err := bp.Readline(bufio.NewReader(strings.NewReader("this is too long"))); !errors.Is(err, ErrLineTooLong) {
+		t.Fatalf("expected ErrLineTooLong, got error %v", err)
+	}
+	if _, err := bp.Readline(bufio.NewReader(strings.NewReader("short"))); !errors.Is(err, io.ErrUnexpectedEOF) {
+		t.Fatalf("expected ErrLineTooLong, got error %v", err)
+	}
+
+	er := errReader{fmt.Errorf("bad")}
+	if _, err := bp.Readline(bufio.NewReader(er)); err == nil || !errors.Is(err, er.err) {
+		t.Fatalf("got unexpected error %s", err)
+	}
+
+	if line, err := bp.Readline(bufio.NewReader(strings.NewReader("ok\r\n"))); line != "ok" {
+		t.Fatalf(`got %q, err %v, expected line "ok"`, line, err)
+	}
+	if line, err := bp.Readline(bufio.NewReader(strings.NewReader("ok\n"))); line != "ok" {
+		t.Fatalf(`got %q, err %v, expected line "ok"`, line, err)
+	}
+}
+
+type errReader struct {
+	err error
+}
+
+func (r errReader) Read(buf []byte) (int, error) {
+	return 0, r.err
+}

moxio/doc.go

@@ -0,0 +1,2 @@
+// Package moxio has common i/o functions.
+package moxio

moxio/isclosed.go

@@ -0,0 +1,24 @@
+package moxio
+
+import (
+	"errors"
+	"net"
+	"syscall"
+)
+
+// In separate file because of import of syscall.
+
+// IsClosed returns whether i/o failed, typically because the connection is closed
+// or otherwise cannot be used for further i/o.
+//
+// Used to prevent error logging for connections that are closed.
+func IsClosed(err error) bool {
+	return errors.Is(err, net.ErrClosed) || errors.Is(err, syscall.EPIPE) || errors.Is(err, syscall.ECONNRESET) || isRemoteTLSError(err)
+}
+
+// A remote TLS client can send a message indicating failure, this makes it back to
+// us as a write error.
+func isRemoteTLSError(err error) bool {
+	var netErr *net.OpError
+	return errors.As(err, &netErr) && netErr.Op == "remote error"
+}

moxio/limitatreader.go

@@ -0,0 +1,20 @@
+package moxio
+
+import (
+	"io"
+)
+
+// LimitAtReader is a reader at that returns ErrLimit if reads would extend
+// beyond Limit.
+type LimitAtReader struct {
+	R     io.ReaderAt
+	Limit int64
+}
+
+// ReadAt passes the read on to R, but returns an error if the read data would extend beyond Limit.
+func (r *LimitAtReader) ReadAt(buf []byte, offset int64) (int, error) {
+	if offset+int64(len(buf)) > r.Limit {
+		return 0, ErrLimit
+	}
+	return r.R.ReadAt(buf, offset)
+}

moxio/limitreader.go

@@ -0,0 +1,27 @@
+package moxio
+
+import (
+	"errors"
+	"io"
+)
+
+var ErrLimit = errors.New("input exceeds maximum size") // Returned by LimitReader.
+
+// LimitReader reads up to Limit bytes, returning an error if more bytes are
+// read. LimitReader can be used to enforce a maximum input length.
+type LimitReader struct {
+	R     io.Reader
+	Limit int64
+}
+
+// Read reads bytes from the underlying reader.
+func (r *LimitReader) Read(buf []byte) (int, error) {
+	n, err := r.R.Read(buf)
+	if n > 0 {
+		r.Limit -= int64(n)
+		if r.Limit < 0 {
+			return 0, ErrLimit
+		}
+	}
+	return n, err
+}

moxio/prefixconn.go

@@ -0,0 +1,25 @@
+package moxio
+
+import (
+	"io"
+	"net"
+)
+
+// PrefixConn is a net.Conn prefixed with a reader that is first drained.
+// Used for STARTTLS where already did a buffered read of initial TLS data.
+type PrefixConn struct {
+	PrefixReader io.Reader // If not nil, reads are fulfilled from here. It is cleared when a read returns io.EOF.
+	net.Conn
+}
+
+// Read returns data when PrefixReader when not nil, and net.Conn otherwise.
+func (c *PrefixConn) Read(buf []byte) (int, error) {
+	if c.PrefixReader != nil {
+		n, err := c.PrefixReader.Read(buf)
+		if err == io.EOF {
+			c.PrefixReader = nil
+		}
+		return n, err
+	}
+	return c.Conn.Read(buf)
+}

moxio/storagespace.go

@@ -0,0 +1,14 @@
+package moxio
+
+import (
+	"errors"
+	"syscall"
+)
+
+// In separate file because of syscall import.
+
+// IsStorageSpace returns whether the error is for storage space issue.
+// Like disk full, no inodes, quota reached.
+func IsStorageSpace(err error) bool {
+	return errors.Is(err, syscall.ENOSPC) || errors.Is(err, syscall.EDQUOT)
+}

moxio/syncdir.go

@@ -0,0 +1,17 @@
+package moxio
+
+import (
+	"fmt"
+	"os"
+)
+
+// SyncDir opens a directory and syncs its contents to disk.
+func SyncDir(dir string) error {
+	d, err := os.Open(dir)
+	if err != nil {
+		return fmt.Errorf("open directory: %v", err)
+	}
+	xerr := d.Sync()
+	d.Close()
+	return xerr
+}

moxio/trace.go

@@ -0,0 +1,48 @@
+package moxio
+
+import (
+	"io"
+
+	"github.com/mjl-/mox/mlog"
+)
+
+type writer struct {
+	log    *mlog.Log
+	prefix string
+	w      io.Writer
+}
+
+// NewTraceWriter wraps "w" into a writer that logs all writes to "log" with
+// log level trace, prefixed with "prefix".
+func NewTraceWriter(log *mlog.Log, prefix string, w io.Writer) io.Writer {
+	return writer{log, prefix, w}
+}
+
+// Write logs a trace line for writing buf to the client, then writes to the
+// client.
+func (w writer) Write(buf []byte) (int, error) {
+	w.log.Trace(w.prefix + string(buf))
+	return w.w.Write(buf)
+}
+
+type reader struct {
+	log    *mlog.Log
+	prefix string
+	r      io.Reader
+}
+
+// NewTraceReader wraps reader "r" into a reader that logs all reads to "log"
+// with log level trace, prefixed with "prefix".
+func NewTraceReader(log *mlog.Log, prefix string, r io.Reader) io.Reader {
+	return reader{log, prefix, r}
+}
+
+// Read does a single Read on its underlying reader, logs data of successful
+// reads, and returns the data read.
+func (r reader) Read(buf []byte) (int, error) {
+	n, err := r.r.Read(buf)
+	if n > 0 {
+		r.log.Trace(r.prefix + string(buf[:n]))
+	}
+	return n, err
+}

moxio/umask.go

@@ -0,0 +1,18 @@
+package moxio
+
+import (
+	"fmt"
+	"syscall"
+)
+
+// CheckUmask checks that the umask is 7 for "other". Because files written
+// should not be world-accessible. E.g. database files, and the control unix
+// domain socket.
+func CheckUmask() error {
+	old := syscall.Umask(007)
+	syscall.Umask(old)
+	if old&7 != 7 {
+		return fmt.Errorf(`umask must have "7" for world/other, e.g. 007, not current %o`, old)
+	}
+	return nil
+}