switch to slog.Logger for logging, for easier reuse of packages by external software

we don't want external software to include internal details like mlog.
slog.Logger is/will be the standard.

we still have mlog for its helper functions, and its handler that logs in
concise logfmt used by mox.

packages that are not meant for reuse still pass around mlog.Log for
convenience.

we use golang.org/x/exp/slog because we also support the previous Go toolchain
version. with the next Go release, we'll switch to the builtin slog.
This commit is contained in:
Mechiel Lukkien
2023-12-05 13:35:58 +01:00
parent 56b2a9d980
commit 5b20cba50a
150 changed files with 5176 additions and 1898 deletions

View File

@ -5,6 +5,8 @@ import (
"io"
"net/textproto"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/smtp"
@ -17,12 +19,14 @@ import (
// From headers may be present. From returns an error if there is not exactly
// one address. This address can be used for evaluating a DMARC policy against
// SPF and DKIM results.
func From(log *mlog.Log, strict bool, r io.ReaderAt) (raddr smtp.Address, envelope *Envelope, header textproto.MIMEHeader, rerr error) {
func From(elog *slog.Logger, strict bool, r io.ReaderAt) (raddr smtp.Address, envelope *Envelope, header textproto.MIMEHeader, rerr error) {
log := mlog.New("message", elog)
// ../rfc/7489:1243
// todo: only allow utf8 if enabled in session/message?
p, err := Parse(log, strict, r)
p, err := Parse(log.Logger, strict, r)
if err != nil {
// todo: should we continue with p, perhaps headers can be parsed?
return raddr, nil, nil, fmt.Errorf("parsing message: %v", err)

View File

@ -21,6 +21,7 @@ import (
"strings"
"time"
"golang.org/x/exp/slog"
"golang.org/x/text/encoding/ianaindex"
"github.com/mjl-/mox/mlog"
@ -111,7 +112,8 @@ type Address struct {
//
// If strict is set, fewer attempts are made to continue parsing when errors are
// encountered, such as with invalid content-type headers or bare carriage returns.
func Parse(log *mlog.Log, strict bool, r io.ReaderAt) (Part, error) {
func Parse(elog *slog.Logger, strict bool, r io.ReaderAt) (Part, error) {
log := mlog.New("message", elog)
return newPart(log, strict, r, 0, nil)
}
@ -122,10 +124,11 @@ func Parse(log *mlog.Log, strict bool, r io.ReaderAt) (Part, error) {
//
// If strict is set, fewer attempts are made to continue parsing when errors are
// encountered, such as with invalid content-type headers or bare carriage returns.
func EnsurePart(log *mlog.Log, strict bool, r io.ReaderAt, size int64) (Part, error) {
p, err := Parse(log, strict, r)
func EnsurePart(elog *slog.Logger, strict bool, r io.ReaderAt, size int64) (Part, error) {
log := mlog.New("message", elog)
p, err := Parse(log.Logger, strict, r)
if err == nil {
err = p.Walk(log, nil)
err = p.Walk(log.Logger, nil)
}
if err != nil {
np, err2 := fallbackPart(p, r, size)
@ -185,7 +188,9 @@ func (p *Part) SetMessageReaderAt() error {
}
// Walk through message, decoding along the way, and collecting mime part offsets and sizes, and line counts.
func (p *Part) Walk(log *mlog.Log, parent *Part) error {
func (p *Part) Walk(elog *slog.Logger, parent *Part) error {
log := mlog.New("message", elog)
if len(p.bound) == 0 {
if p.MediaType == "MESSAGE" && (p.MediaSubType == "RFC822" || p.MediaSubType == "GLOBAL") {
// todo: don't read whole submessage in memory...
@ -194,11 +199,11 @@ func (p *Part) Walk(log *mlog.Log, parent *Part) error {
return err
}
br := bytes.NewReader(buf)
mp, err := Parse(log, p.strict, br)
mp, err := Parse(log.Logger, p.strict, br)
if err != nil {
return fmt.Errorf("parsing embedded message: %w", err)
}
if err := mp.Walk(log, nil); err != nil {
if err := mp.Walk(log.Logger, nil); err != nil {
// If this is a DSN and we are not in pedantic mode, accept unexpected end of
// message. This is quite common because MTA's sometimes just truncate the original
// message in a place that makes the message invalid.
@ -220,14 +225,14 @@ func (p *Part) Walk(log *mlog.Log, parent *Part) error {
}
for {
pp, err := p.ParseNextPart(log)
pp, err := p.ParseNextPart(log.Logger)
if err == io.EOF {
return nil
}
if err != nil {
return err
}
if err := pp.Walk(log, p); err != nil {
if err := pp.Walk(log.Logger, p); err != nil {
return err
}
}
@ -241,7 +246,7 @@ func (p *Part) String() string {
// newPart parses a new part, which can be the top-level message.
// offset is the bound offset for parts, and the start of message for top-level messages. parent indicates if this is a top-level message or sub-part.
// If an error occurs, p's exported values can still be relevant. EnsurePart uses these values.
func newPart(log *mlog.Log, strict bool, r io.ReaderAt, offset int64, parent *Part) (p Part, rerr error) {
func newPart(log mlog.Log, strict bool, r io.ReaderAt, offset int64, parent *Part) (p Part, rerr error) {
if r == nil {
panic("nil reader")
}
@ -325,14 +330,14 @@ func newPart(log *mlog.Log, strict bool, r io.ReaderAt, offset int64, parent *Pa
p.MediaType = "APPLICATION"
p.MediaSubType = "OCTET-STREAM"
}
log.Debugx("malformed content-type, attempting to recover and continuing", err, mlog.Field("contenttype", p.header.Get("Content-Type")), mlog.Field("mediatype", p.MediaType), mlog.Field("mediasubtype", p.MediaSubType))
log.Debugx("malformed content-type, attempting to recover and continuing", err, slog.String("contenttype", p.header.Get("Content-Type")), slog.String("mediatype", p.MediaType), slog.String("mediasubtype", p.MediaSubType))
} else if mt != "" {
t := strings.SplitN(strings.ToUpper(mt), "/", 2)
if len(t) != 2 {
if moxvar.Pedantic || strict {
return p, fmt.Errorf("bad content-type: %q (content-type %q)", mt, ct)
}
log.Debug("malformed media-type, ignoring and continuing", mlog.Field("type", mt))
log.Debug("malformed media-type, ignoring and continuing", slog.String("type", mt))
p.MediaType = "APPLICATION"
p.MediaSubType = "OCTET-STREAM"
} else {
@ -444,7 +449,7 @@ var wordDecoder = mime.WordDecoder{
},
}
func parseEnvelope(log *mlog.Log, h mail.Header) (*Envelope, error) {
func parseEnvelope(log mlog.Log, h mail.Header) (*Envelope, error) {
date, _ := h.Date()
// We currently marshal this field to JSON. But JSON cannot represent all
@ -478,7 +483,7 @@ func parseEnvelope(log *mlog.Log, h mail.Header) (*Envelope, error) {
return env, nil
}
func parseAddressList(log *mlog.Log, h mail.Header, k string) []Address {
func parseAddressList(log mlog.Log, h mail.Header, k string) []Address {
// todo: possibly work around ios mail generating incorrect q-encoded "phrases" with unencoded double quotes? ../rfc/2047:382
l, err := h.AddressList(k)
if err != nil {
@ -490,7 +495,7 @@ func parseAddressList(log *mlog.Log, h mail.Header, k string) []Address {
var user, host string
addr, err := smtp.ParseAddress(a.Address)
if err != nil {
log.Infox("parsing address (continuing)", err, mlog.Field("address", a.Address))
log.Infox("parsing address (continuing)", err, slog.Any("address", a.Address))
} else {
user = addr.Localpart.String()
host = addr.Domain.ASCII
@ -503,7 +508,9 @@ func parseAddressList(log *mlog.Log, h mail.Header, k string) []Address {
// ParseNextPart parses the next (sub)part of this multipart message.
// ParseNextPart returns io.EOF and a nil part when there are no more parts.
// Only used for initial parsing of message. Once parsed, use p.Parts.
func (p *Part) ParseNextPart(log *mlog.Log) (*Part, error) {
func (p *Part) ParseNextPart(elog *slog.Logger) (*Part, error) {
log := mlog.New("message", elog)
if len(p.bound) == 0 {
return nil, errNotMultipart
}

View File

@ -15,7 +15,7 @@ import (
"github.com/mjl-/mox/moxvar"
)
var xlog = mlog.New("message")
var pkglog = mlog.New("message", nil)
func tcheck(t *testing.T, err error, msg string) {
t.Helper()
@ -40,7 +40,7 @@ func tfail(t *testing.T, err, expErr error) {
func TestEmptyHeader(t *testing.T) {
s := "\r\nx"
p, err := EnsurePart(xlog, true, strings.NewReader(s), int64(len(s)))
p, err := EnsurePart(pkglog.Logger, true, strings.NewReader(s), int64(len(s)))
tcheck(t, err, "parse empty headers")
buf, err := io.ReadAll(p.Reader())
tcheck(t, err, "read")
@ -56,7 +56,7 @@ func TestBadContentType(t *testing.T) {
// Pedantic is like strict.
moxvar.Pedantic = true
s := "content-type: text/html;;\r\n\r\ntest"
p, err := EnsurePart(xlog, false, strings.NewReader(s), int64(len(s)))
p, err := EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tfail(t, err, ErrBadContentType)
buf, err := io.ReadAll(p.Reader())
tcheck(t, err, "read")
@ -67,7 +67,7 @@ func TestBadContentType(t *testing.T) {
// Strict
s = "content-type: text/html;;\r\n\r\ntest"
p, err = EnsurePart(xlog, true, strings.NewReader(s), int64(len(s)))
p, err = EnsurePart(pkglog.Logger, true, strings.NewReader(s), int64(len(s)))
tfail(t, err, ErrBadContentType)
buf, err = io.ReadAll(p.Reader())
tcheck(t, err, "read")
@ -77,7 +77,7 @@ func TestBadContentType(t *testing.T) {
// Non-strict but unrecoverable content-type.
s = "content-type: not a content type;;\r\n\r\ntest"
p, err = EnsurePart(xlog, false, strings.NewReader(s), int64(len(s)))
p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tcheck(t, err, "parsing message with bad but recoverable content-type")
buf, err = io.ReadAll(p.Reader())
tcheck(t, err, "read")
@ -87,7 +87,7 @@ func TestBadContentType(t *testing.T) {
// We try to use only the content-type, typically better than application/octet-stream.
s = "content-type: text/html;;\r\n\r\ntest"
p, err = EnsurePart(xlog, false, strings.NewReader(s), int64(len(s)))
p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tcheck(t, err, "parsing message with bad but recoverable content-type")
buf, err = io.ReadAll(p.Reader())
tcheck(t, err, "read")
@ -97,7 +97,7 @@ func TestBadContentType(t *testing.T) {
// Not recovering multipart, we won't have a boundary.
s = "content-type: multipart/mixed;;\r\n\r\ntest"
p, err = EnsurePart(xlog, false, strings.NewReader(s), int64(len(s)))
p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tcheck(t, err, "parsing message with bad but recoverable content-type")
buf, err = io.ReadAll(p.Reader())
tcheck(t, err, "read")
@ -112,20 +112,20 @@ func TestBareCR(t *testing.T) {
// Pedantic is like strict.
moxvar.Pedantic = true
p, err := EnsurePart(xlog, false, strings.NewReader(s), int64(len(s)))
p, err := EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tfail(t, err, errBareCR)
_, err = io.ReadAll(p.Reader())
tfail(t, err, errBareCR)
moxvar.Pedantic = false
// Strict.
p, err = EnsurePart(xlog, true, strings.NewReader(s), int64(len(s)))
p, err = EnsurePart(pkglog.Logger, true, strings.NewReader(s), int64(len(s)))
tfail(t, err, errBareCR)
_, err = io.ReadAll(p.Reader())
tcheck(t, err, "read fallback part without error")
// Non-strict allows bare cr.
p, err = EnsurePart(xlog, false, strings.NewReader(s), int64(len(s)))
p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tcheck(t, err, "parse")
buf, err := io.ReadAll(p.Reader())
tcheck(t, err, "read")
@ -141,7 +141,7 @@ aGkK
func TestBasic(t *testing.T) {
r := strings.NewReader(basicMsg)
p, err := Parse(xlog, true, r)
p, err := Parse(pkglog.Logger, true, r)
tcheck(t, err, "new reader")
buf, err := io.ReadAll(p.RawReader())
@ -176,7 +176,7 @@ Hello Joe, do you think we can meet at 3:30 tomorrow?
func TestBasic2(t *testing.T) {
r := strings.NewReader(basicMsg2)
p, err := Parse(xlog, true, r)
p, err := Parse(pkglog.Logger, true, r)
tcheck(t, err, "new reader")
buf, err := io.ReadAll(p.RawReader())
@ -196,9 +196,9 @@ func TestBasic2(t *testing.T) {
}
r = strings.NewReader(basicMsg2)
p, err = Parse(xlog, true, r)
p, err = Parse(pkglog.Logger, true, r)
tcheck(t, err, "new reader")
err = p.Walk(xlog, nil)
err = p.Walk(pkglog.Logger, nil)
tcheck(t, err, "walk")
if p.RawLineCount != 2 {
t.Fatalf("basic message, got %d lines, expected 2", p.RawLineCount)
@ -237,25 +237,25 @@ This is the epilogue. It is also to be ignored.
func TestMime(t *testing.T) {
// from ../rfc/2046:1148
r := strings.NewReader(mimeMsg)
p, err := Parse(xlog, true, r)
p, err := Parse(pkglog.Logger, true, r)
tcheck(t, err, "new reader")
if len(p.bound) == 0 {
t.Fatalf("got no bound, expected bound for mime message")
}
pp, err := p.ParseNextPart(xlog)
pp, err := p.ParseNextPart(pkglog.Logger)
tcheck(t, err, "next part")
buf, err := io.ReadAll(pp.Reader())
tcheck(t, err, "read all")
tcompare(t, string(buf), "This is implicitly typed plain US-ASCII text.\r\nIt does NOT end with a linebreak.")
pp, err = p.ParseNextPart(xlog)
pp, err = p.ParseNextPart(pkglog.Logger)
tcheck(t, err, "next part")
buf, err = io.ReadAll(pp.Reader())
tcheck(t, err, "read all")
tcompare(t, string(buf), "This is explicitly typed plain US-ASCII text.\r\nIt DOES end with a linebreak.\r\n")
_, err = p.ParseNextPart(xlog)
_, err = p.ParseNextPart(pkglog.Logger)
tcompare(t, err, io.EOF)
if len(p.Parts) != 2 {
@ -274,17 +274,17 @@ func TestLongLine(t *testing.T) {
for i := range line {
line[i] = 'a'
}
_, err := Parse(xlog, true, bytes.NewReader(line))
_, err := Parse(pkglog.Logger, true, bytes.NewReader(line))
tfail(t, err, errLineTooLong)
}
func TestBareCrLf(t *testing.T) {
parse := func(strict bool, s string) error {
p, err := Parse(xlog, strict, strings.NewReader(s))
p, err := Parse(pkglog.Logger, strict, strings.NewReader(s))
if err != nil {
return err
}
return p.Walk(xlog, nil)
return p.Walk(pkglog.Logger, nil)
}
err := parse(false, "subject: test\ntest\r\n")
tfail(t, err, errBareLF)
@ -316,25 +316,25 @@ func TestMissingClosingBoundary(t *testing.T) {
test
`, "\n", "\r\n")
msg, err := Parse(xlog, false, strings.NewReader(message))
msg, err := Parse(pkglog.Logger, false, strings.NewReader(message))
tcheck(t, err, "new reader")
err = walkmsg(&msg)
tfail(t, err, errMissingClosingBoundary)
msg, _ = Parse(xlog, false, strings.NewReader(message))
err = msg.Walk(xlog, nil)
msg, _ = Parse(pkglog.Logger, false, strings.NewReader(message))
err = msg.Walk(pkglog.Logger, nil)
tfail(t, err, errMissingClosingBoundary)
}
func TestHeaderEOF(t *testing.T) {
message := "header: test"
_, err := Parse(xlog, false, strings.NewReader(message))
_, err := Parse(pkglog.Logger, false, strings.NewReader(message))
tfail(t, err, errUnexpectedEOF)
}
func TestBodyEOF(t *testing.T) {
message := "header: test\r\n\r\ntest"
msg, err := Parse(xlog, true, strings.NewReader(message))
msg, err := Parse(pkglog.Logger, true, strings.NewReader(message))
tcheck(t, err, "new reader")
buf, err := io.ReadAll(msg.Reader())
tcheck(t, err, "read body")
@ -365,7 +365,7 @@ test
`, "\n", "\r\n")
msg, err := Parse(xlog, false, strings.NewReader(message))
msg, err := Parse(pkglog.Logger, false, strings.NewReader(message))
tcheck(t, err, "new reader")
enforceSequential = true
defer func() {
@ -374,8 +374,8 @@ test
err = walkmsg(&msg)
tcheck(t, err, "walkmsg")
msg, _ = Parse(xlog, false, strings.NewReader(message))
err = msg.Walk(xlog, nil)
msg, _ = Parse(pkglog.Logger, false, strings.NewReader(message))
err = msg.Walk(pkglog.Logger, nil)
tcheck(t, err, "msg.Walk")
}
@ -452,7 +452,7 @@ Content-Transfer-Encoding: Quoted-printable
--unique-boundary-1--
`, "\n", "\r\n")
msg, err := Parse(xlog, true, strings.NewReader(nestedMessage))
msg, err := Parse(pkglog.Logger, true, strings.NewReader(nestedMessage))
tcheck(t, err, "new reader")
enforceSequential = true
defer func() {
@ -477,8 +477,8 @@ Content-Transfer-Encoding: Quoted-printable
t.Fatalf("got %q, expected %q", buf, exp)
}
msg, _ = Parse(xlog, false, strings.NewReader(nestedMessage))
err = msg.Walk(xlog, nil)
msg, _ = Parse(pkglog.Logger, false, strings.NewReader(nestedMessage))
err = msg.Walk(pkglog.Logger, nil)
tcheck(t, err, "msg.Walk")
}
@ -518,7 +518,7 @@ func walk(path string) error {
return err
}
defer r.Close()
msg, err := Parse(xlog, false, r)
msg, err := Parse(pkglog.Logger, false, r)
if err != nil {
return err
}
@ -538,7 +538,7 @@ func walkmsg(msg *Part) error {
}
if msg.MediaType == "MESSAGE" && (msg.MediaSubType == "RFC822" || msg.MediaSubType == "GLOBAL") {
mp, err := Parse(xlog, false, bytes.NewReader(buf))
mp, err := Parse(pkglog.Logger, false, bytes.NewReader(buf))
if err != nil {
return err
}
@ -566,7 +566,7 @@ func walkmsg(msg *Part) error {
}
for {
pp, err := msg.ParseNextPart(xlog)
pp, err := msg.ParseNextPart(pkglog.Logger)
if err == io.EOF {
return nil
}
@ -585,7 +585,7 @@ func TestEmbedded(t *testing.T) {
tcheck(t, err, "open")
fi, err := f.Stat()
tcheck(t, err, "stat")
_, err = EnsurePart(xlog, false, f, fi.Size())
_, err = EnsurePart(pkglog.Logger, false, f, fi.Size())
tcheck(t, err, "parse")
}
@ -594,6 +594,6 @@ func TestEmbedded2(t *testing.T) {
tcheck(t, err, "readfile")
buf = bytes.ReplaceAll(buf, []byte("\n"), []byte("\r\n"))
_, err = EnsurePart(xlog, false, bytes.NewReader(buf), int64(len(buf)))
_, err = EnsurePart(pkglog.Logger, false, bytes.NewReader(buf), int64(len(buf)))
tfail(t, err, nil)
}

View File

@ -9,7 +9,7 @@ func TestReferencedIDs(t *testing.T) {
check := func(msg string, expRefs []string) {
t.Helper()
p, err := Parse(xlog, true, strings.NewReader(msg))
p, err := Parse(pkglog.Logger, true, strings.NewReader(msg))
tcheck(t, err, "parsing message")
h, err := p.Header()

View File

@ -1,46 +0,0 @@
package message
import (
"crypto/tls"
"fmt"
"github.com/mjl-/mox/mlog"
)
// TLSReceivedComment returns a comment about TLS of the connection for use in a Receive header.
func TLSReceivedComment(log *mlog.Log, cs tls.ConnectionState) []string {
// todo future: we could use the "tls" clause for the Received header as specified in ../rfc/8314:496. however, the text implies it is only for submission, not regular smtp. and it cannot specify the tls version. for now, not worth the trouble.
// Comments from other mail servers:
// gmail.com: (version=TLS1_3 cipher=TLS_AES_128_GCM_SHA256 bits=128/128)
// yahoo.com: (version=TLS1_3 cipher=TLS_AES_128_GCM_SHA256)
// proton.me: (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested)
// outlook.com: (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384)
var l []string
add := func(s string) {
l = append(l, s)
}
versions := map[uint16]string{
tls.VersionTLS10: "TLS1.0",
tls.VersionTLS11: "TLS1.1",
tls.VersionTLS12: "TLS1.2",
tls.VersionTLS13: "TLS1.3",
}
if version, ok := versions[cs.Version]; ok {
add(version)
} else {
log.Info("unknown tls version identifier", mlog.Field("version", cs.Version))
add(fmt.Sprintf("TLS identifier %x", cs.Version))
}
add(tls.CipherSuiteName(cs.CipherSuite))
// Make it a comment.
l[0] = "(" + l[0]
l[len(l)-1] = l[len(l)-1] + ")"
return l
}