add more documentation, examples with tests to illustrate reusable components

message/examples_test.go

@@ -0,0 +1,196 @@
+package message_test
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"strings"
+	"time"
+
+	"golang.org/x/exp/slog"
+
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/message"
+	"github.com/mjl-/mox/smtp"
+)
+
+func ExampleDecodeReader() {
+	// Convert from iso-8859-1 to utf-8.
+	input := []byte{'t', 0xe9, 's', 't'}
+	output, err := io.ReadAll(message.DecodeReader("iso-8859-1", bytes.NewReader(input)))
+	if err != nil {
+		log.Fatalf("read from decoder: %v", err)
+	}
+	fmt.Printf("%s\n", string(output))
+	// Output: tést
+}
+
+func ExampleMessageIDCanonical() {
+	// Valid message-id.
+	msgid, invalidAddress, err := message.MessageIDCanonical("<ok@localhost>")
+	if err != nil {
+		fmt.Printf("invalid message-id: %v\n", err)
+	} else {
+		fmt.Printf("canonical: %s %v\n", msgid, invalidAddress)
+	}
+
+	// Missing <>.
+	msgid, invalidAddress, err = message.MessageIDCanonical("bogus@localhost")
+	if err != nil {
+		fmt.Printf("invalid message-id: %v\n", err)
+	} else {
+		fmt.Printf("canonical: %s %v\n", msgid, invalidAddress)
+	}
+
+	// Invalid address, but returned as not being in error.
+	msgid, invalidAddress, err = message.MessageIDCanonical("<invalid>")
+	if err != nil {
+		fmt.Printf("invalid message-id: %v\n", err)
+	} else {
+		fmt.Printf("canonical: %s %v\n", msgid, invalidAddress)
+	}
+
+	// Output:
+	// canonical: ok@localhost false
+	// invalid message-id: not a message-id: missing <
+	// canonical: invalid true
+}
+
+func ExampleThreadSubject() {
+	// Basic subject.
+	s, isResp := message.ThreadSubject("nothing special", false)
+	fmt.Printf("%s, response: %v\n", s, isResp)
+
+	// List tags and "re:" are stripped.
+	s, isResp = message.ThreadSubject("[list1] [list2] Re: test", false)
+	fmt.Printf("%s, response: %v\n", s, isResp)
+
+	// "fwd:" is stripped.
+	s, isResp = message.ThreadSubject("fwd: a forward", false)
+	fmt.Printf("%s, response: %v\n", s, isResp)
+
+	// Trailing "(fwd)" is also a forward.
+	s, isResp = message.ThreadSubject("another forward (fwd)", false)
+	fmt.Printf("%s, response: %v\n", s, isResp)
+
+	// [fwd: ...] is stripped.
+	s, isResp = message.ThreadSubject("[fwd: [list] fwd: re: it's complicated]", false)
+	fmt.Printf("%s, response: %v\n", s, isResp)
+
+	// Output:
+	// nothing special, response: false
+	// test, response: true
+	// a forward, response: true
+	// another forward, response: true
+	// it's complicated, response: true
+}
+
+func ExampleComposer() {
+	// We store in a buffer. We could also write to a file.
+	var b bytes.Buffer
+
+	// NewComposer. Keep in mind that operations on a Composer will panic on error.
+	xc := message.NewComposer(&b, 10*1024*1024)
+
+	// Catch and handle errors when composing.
+	defer func() {
+		x := recover()
+		if x == nil {
+			return
+		}
+		if err, ok := x.(error); ok && errors.Is(err, message.ErrCompose) {
+			log.Printf("compose: %v", err)
+		}
+		panic(x)
+	}()
+
+	// Add an address header.
+	xc.HeaderAddrs("From", []message.NameAddress{{DisplayName: "Charlie", Address: smtp.Address{Localpart: "root", Domain: dns.Domain{ASCII: "localhost"}}}})
+
+	// Add subject header, with encoding
+	xc.Subject("hi ☺")
+
+	// Add Date and Message-ID headers, required.
+	tm, _ := time.Parse(time.RFC3339, "2006-01-02T15:04:05+07:00")
+	xc.Header("Date", tm.Format(message.RFC5322Z))
+	xc.Header("Message-ID", "<unique@host>") // Should generate unique id for each message.
+
+	xc.Header("MIME-Version", "1.0")
+
+	// Write content-* headers for the text body.
+	body, ct, cte := xc.TextPart("this is the body")
+	xc.Header("Content-Type", ct)
+	xc.Header("Content-Transfer-Encoding", cte)
+
+	// Header/Body separator
+	xc.Line()
+
+	// The part body. Use mime/multipart to make messages with multiple parts.
+	xc.Write(body)
+
+	// Flush any buffered writes to the original writer.
+	xc.Flush()
+
+	fmt.Println(strings.ReplaceAll(b.String(), "\r\n", "\n"))
+	// Output:
+	// From: "Charlie" <root@localhost>
+	// Subject: hi =?utf-8?q?=E2=98=BA?=
+	// Date: 2 Jan 2006 15:04:05 +0700
+	// Message-ID: <unique@host>
+	// MIME-Version: 1.0
+	// Content-Type: text/plain; charset=us-ascii
+	// Content-Transfer-Encoding: 7bit
+	//
+	// this is the body
+}
+
+func ExamplePart() {
+	// Parse a message from an io.ReaderAt, which could be a file.
+	strict := false
+	r := strings.NewReader("header: value\r\nanother: value\r\n\r\nbody ...\r\n")
+	part, err := message.Parse(slog.Default(), strict, r)
+	if err != nil {
+		log.Fatalf("parsing message: %v", err)
+	}
+
+	// The headers of the first part have been parsed, i.e. the message headers.
+	// A message can be multipart (e.g. alternative, related, mixed), and possibly
+	// nested.
+
+	// By walking the entire message, all part metadata (like offsets into the file
+	// where a part starts) is recorded.
+	err = part.Walk(slog.Default(), nil)
+	if err != nil {
+		log.Fatalf("walking message: %v", err)
+	}
+
+	// Messages can have a recursive multipart structure. Print the structure.
+	var printPart func(indent string, p message.Part)
+	printPart = func(indent string, p message.Part) {
+		log.Printf("%s- part: %v", indent, part)
+		for _, pp := range p.Parts {
+			printPart("  "+indent, pp)
+		}
+	}
+	printPart("", part)
+}
+
+func ExampleWriter() {
+	// NewWriter on a string builder.
+	var b strings.Builder
+	w := message.NewWriter(&b)
+
+	// Write some lines, some with proper CRLF line ending, others without.
+	fmt.Fprint(w, "header: value\r\n")
+	fmt.Fprint(w, "another: value\n") // missing \r
+	fmt.Fprint(w, "\r\n")
+	fmt.Fprint(w, "hi ☺\n") // missing \r
+
+	fmt.Printf("%q\n", b.String())
+	fmt.Printf("%v %v", w.HaveBody, w.Has8bit)
+	// Output:
+	// "header: value\r\nanother: value\r\n\r\nhi ☺\r\n"
+	// true true
+}

message/parseheaderfields.go

@@ -8,8 +8,9 @@ import (
 )
 
 // ParseHeaderFields parses only the header fields in "fields" from the complete
-// header buffer "header", while using "scratch" as temporary space, prevent lots
-// of unneeded allocations when only a few headers are needed.
+// header buffer "header". It uses "scratch" as temporary space, which can be
+// reused across calls, potentially saving lots of unneeded allocations when only a
+// few headers are needed and/or many messages are parsed.
 func ParseHeaderFields(header []byte, scratch []byte, fields [][]byte) (textproto.MIMEHeader, error) {
 	// todo: should not use mail.ReadMessage, it allocates a bufio.Reader. should implement header parsing ourselves.
 

message/part.go

@@ -580,7 +580,7 @@ func (p *Part) Reader() io.Reader {
 	return p.bodyReader(p.RawReader())
 }
 
-// ReaderUTF8OrBinary returns a reader for the decode body content, transformed to
+// ReaderUTF8OrBinary returns a reader for the decoded body content, transformed to
 // utf-8 for known mime/iana encodings (only if they aren't us-ascii or utf-8
 // already). For unknown or missing character sets/encodings, the original reader
 // is returned.

message/qp.go

@@ -4,8 +4,10 @@ import (
 	"strings"
 )
 
-// NeedsQuotedPrintable returns whether text should be encoded with
-// quoted-printable. If not, it can be included as 7bit or 8bit encoding.
+// NeedsQuotedPrintable returns whether text, with crlf-separated lines, should be
+// encoded with quoted-printable, based on line lengths and any bare carriage
+// return or bare newline. If not, it can be included as 7bit or 8bit encoding in a
+// new message.
 func NeedsQuotedPrintable(text string) bool {
 	// ../rfc/2045:1025
 	for _, line := range strings.Split(text, "\r\n") {

message/threadsubject.go

@@ -9,6 +9,9 @@ import (
 // always required to match to an existing thread, both if
 // References/In-Reply-To header(s) are present, and if not.
 //
+// isResponse indicates if this message is a response, such as a reply or a
+// forward.
+//
 // Subject should already be q/b-word-decoded.
 //
 // If allowNull is true, base subjects with a \0 can be returned. If not set,

message/writer.go

@@ -9,9 +9,9 @@ import (
 type Writer struct {
 	writer io.Writer
 
-	HaveBody bool // Body is optional. ../rfc/5322:343
-	Has8bit  bool // Whether a byte with the high/8bit has been read. So whether this is 8BITMIME instead of 7BIT.
-	Size     int64
+	HaveBody bool  // Body is optional in a message. ../rfc/5322:343
+	Has8bit  bool  // Whether a byte with the high/8bit has been read. So whether this needs SMTP 8BITMIME instead of 7BIT.
+	Size     int64 // Number of bytes written, may be different from bytes read due to LF to CRLF conversion.
 
 	tail [3]byte // For detecting header/body-separating crlf.
 	// todo: should be parsing headers here, as we go
@@ -22,7 +22,9 @@ func NewWriter(w io.Writer) *Writer {
 	return &Writer{writer: w, tail: [3]byte{0, '\r', '\n'}}
 }
 
-// Write implements io.Writer.
+// Write implements io.Writer, and writes buf as message to the Writer's underlying
+// io.Writer. It converts bare new lines (LF) to carriage returns with new lines
+// (CRLF).
 func (w *Writer) Write(buf []byte) (int, error) {
 	origtail := w.tail