infrastructure/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2025-07-10 10:34:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

queue: deliver to multiple recipients in a single smtp transaction

Browse Source 
transferring the data only once. we only do this when the recipient domains
are the same. when queuing, we now take care to set the same NextAttempt
timestamp, so queued messages are actually eligable for combined delivery.

this adds a DeliverMultiple to the smtp client. for pipelined requests, it will
send all RCPT TO (and MAIL and DATA) in one go, and handles the various
responses and error conditions, returning either an overal error, or per
recipient smtp responses. the results of the smtp LIMITS extension are also
available in the smtp client now.

this also takes the "LIMITS RCPTMAX" smtp extension into account: if the server
only accepts a single recipient, we won't send multiple.
if a server doesn't announce a RCPTMAX limit, but still has one (like mox does
for non-spf-verified transactions), we'll recognize code 452 and 552 (for
historic reasons) as temporary error, and try again in a separate transaction
immediately after. we don't yet implement "LIMITS MAILMAX", doesn't seem likely
in practice.
This commit is contained in:
Mechiel Lukkien
2024-03-07 10:07:53 +01:00
parent 8550a5af45
commit 9e7d6b85b7
19 changed files with 1158 additions and 379 deletions
Download Patch File Download Diff File Expand all files Collapse all files

160
queue/submit.go
View File

@ -28,9 +28,12 @@ import (
// deliver via another SMTP server, e.g. relaying to a smart host, possibly
// with authentication (submission).
func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, m Msg, backoff time.Duration, transportName string, transport *config.TransportSMTP, dialTLS bool, defaultPort int) {
func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, msgs []*Msg, backoff time.Duration, transportName string, transport *config.TransportSMTP, dialTLS bool, defaultPort int) {
// todo: configurable timeouts
// For convenience, all messages share the same relevant values.
m0 := msgs[0]
port := transport.Port
if port == 0 {
port = defaultPort
@ -47,22 +50,27 @@ func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
tlsMode = smtpclient.TLSSkip
tlsPKIX = false
}
// Prepare values for logging/metrics. They are updated for various error
// conditions later on.
start := time.Now()
var deliveryResult string
var permanent bool
var secodeOpt string
var errmsg string
var success bool
var submiterr error // Of whole operation, nil for partial failure/success.
var delivered int
failed := len(msgs) // Reset and updated after smtp transaction.
defer func() {
metricDelivery.WithLabelValues(fmt.Sprintf("%d", m.Attempts), transportName, string(tlsMode), deliveryResult).Observe(float64(time.Since(start)) / float64(time.Second))
qlog.Debug("queue deliversubmit result",
r := deliveryResult(submiterr, delivered, failed)
d := float64(time.Since(start)) / float64(time.Second)
metricDelivery.WithLabelValues(fmt.Sprintf("%d", m0.Attempts), transportName, string(tlsMode), r).Observe(d)
qlog.Debugx("queue deliversubmit result", submiterr,
slog.Any("host", transport.DNSHost),
slog.Int("port", port),
slog.Int("attempt", m.Attempts),
slog.Bool("permanent", permanent),
slog.String("secodeopt", secodeOpt),
slog.String("errmsg", errmsg),
slog.Bool("ok", success),
slog.Int("attempt", m0.Attempts),
slog.String("result", r),
slog.Int("delivered", delivered),
slog.Int("failed", failed),
slog.Any("tlsmode", tlsMode),
slog.Bool("tlspkix", tlsPKIX),
slog.Duration("duration", time.Since(start)))
}()
@ -75,25 +83,28 @@ func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
// If submit was done with REQUIRETLS extension for SMTP, we must verify TLS
// certificates. If our submission connection is not configured that way, abort.
requireTLS := m.RequireTLS != nil && *m.RequireTLS
requireTLS := m0.RequireTLS != nil && *m0.RequireTLS
if requireTLS && (tlsMode != smtpclient.TLSRequiredStartTLS && tlsMode != smtpclient.TLSImmediate || !tlsPKIX) {
errmsg = fmt.Sprintf("transport %s: message requires verified tls but transport does not verify tls", transportName)
fail(ctx, qlog, m, backoff, true, dsn.NameIP{}, smtp.SePol7MissingReqTLS, errmsg, "", nil)
submiterr = smtpclient.Error{
Permanent: true,
Code: smtp.C554TransactionFailed,
Secode: smtp.SePol7MissingReqTLS30,
Err: fmt.Errorf("transport %s: message requires verified tls but transport does not verify tls", transportName),
}
fail(ctx, qlog, msgs, m0.DialedIPs, backoff, dsn.NameIP{}, submiterr)
return
}
dialctx, dialcancel := context.WithTimeout(ctx, 30*time.Second)
defer dialcancel()
if m.DialedIPs == nil {
m.DialedIPs = map[string][]net.IP{}
if msgs[0].DialedIPs == nil {
msgs[0].DialedIPs = map[string][]net.IP{}
m0 = msgs[0]
}
_, _, _, ips, _, err := smtpclient.GatherIPs(dialctx, qlog.Logger, resolver, dns.IPDomain{Domain: transport.DNSHost}, m.DialedIPs)
_, _, _, ips, _, err := smtpclient.GatherIPs(dialctx, qlog.Logger, resolver, dns.IPDomain{Domain: transport.DNSHost}, m0.DialedIPs)
var conn net.Conn
if err == nil {
if m.DialedIPs == nil {
m.DialedIPs = map[string][]net.IP{}
}
conn, _, err = smtpclient.Dial(dialctx, qlog.Logger, dialer, dns.IPDomain{Domain: transport.DNSHost}, ips, port, m.DialedIPs, mox.Conf.Static.SpecifiedSMTPListenIPs)
conn, _, err = smtpclient.Dial(dialctx, qlog.Logger, dialer, dns.IPDomain{Domain: transport.DNSHost}, ips, port, m0.DialedIPs, mox.Conf.Static.SpecifiedSMTPListenIPs)
}
addr := net.JoinHostPort(transport.Host, fmt.Sprintf("%d", port))
var result string
@ -114,8 +125,8 @@ func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
qlog.Check(err, "closing connection")
}
qlog.Errorx("dialing for submission", err, slog.String("remote", addr))
errmsg = fmt.Sprintf("transport %s: dialing %s for submission: %v", transportName, addr, err)
fail(ctx, qlog, m, backoff, false, dsn.NameIP{}, "", errmsg, "", nil)
submiterr = fmt.Errorf("transport %s: dialing %s for submission: %w", transportName, addr, err)
fail(ctx, qlog, msgs, m0.DialedIPs, backoff, dsn.NameIP{}, submiterr)
return
}
dialcancel()
@ -165,13 +176,14 @@ func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
if err != nil {
smtperr, ok := err.(smtpclient.Error)
var remoteMTA dsn.NameIP
submiterr = fmt.Errorf("transport %s: establishing smtp session with %s for submission: %w", transportName, addr, err)
if ok {
remoteMTA.Name = transport.Host
smtperr.Err = submiterr
submiterr = smtperr
}
qlog.Errorx("establishing smtp session for submission", err, slog.String("remote", addr))
errmsg = fmt.Sprintf("transport %s: establishing smtp session with %s for submission: %v", transportName, addr, err)
secodeOpt = smtperr.Secode
fail(ctx, qlog, m, backoff, false, remoteMTA, secodeOpt, errmsg, smtperr.Line, smtperr.MoreLines)
qlog.Errorx("establishing smtp session for submission", submiterr, slog.String("remote", addr))
fail(ctx, qlog, msgs, m0.DialedIPs, backoff, remoteMTA, submiterr)
return
}
defer func() {
@ -183,23 +195,23 @@ func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
var msgr io.ReadCloser
var size int64
var req8bit, reqsmtputf8 bool
if len(m.DSNUTF8) > 0 && client.SupportsSMTPUTF8() {
msgr = io.NopCloser(bytes.NewReader(m.DSNUTF8))
if len(m0.DSNUTF8) > 0 && client.SupportsSMTPUTF8() {
msgr = io.NopCloser(bytes.NewReader(m0.DSNUTF8))
reqsmtputf8 = true
size = int64(len(m.DSNUTF8))
size = int64(len(m0.DSNUTF8))
} else {
req8bit = m.Has8bit // todo: not require this, but just try to submit?
size = m.Size
req8bit = m0.Has8bit // todo: not require this, but just try to submit?
size = m0.Size
p := m.MessagePath()
p := m0.MessagePath()
f, err := os.Open(p)
if err != nil {
qlog.Errorx("opening message for delivery", err, slog.String("remote", addr), slog.String("path", p))
errmsg = fmt.Sprintf("transport %s: opening message file for submission: %v", transportName, err)
fail(ctx, qlog, m, backoff, false, dsn.NameIP{}, "", errmsg, "", nil)
submiterr = fmt.Errorf("transport %s: opening message file for submission: %w", transportName, err)
fail(ctx, qlog, msgs, m0.DialedIPs, backoff, dsn.NameIP{}, submiterr)
return
}
msgr = store.FileMsgReader(m.MsgPrefix, f)
msgr = store.FileMsgReader(m0.MsgPrefix, f)
defer func() {
err := msgr.Close()
qlog.Check(err, "closing message after delivery attempt")
@ -208,42 +220,48 @@ func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
deliverctx, delivercancel := context.WithTimeout(context.Background(), time.Duration(60+size/(1024*1024))*time.Second)
defer delivercancel()
err = client.Deliver(deliverctx, m.Sender().String(), m.Recipient().String(), size, msgr, req8bit, reqsmtputf8, requireTLS)
if err != nil {
qlog.Infox("delivery failed", err)
rcpts := make([]string, len(msgs))
for i, m := range msgs {
rcpts[i] = m.Recipient().String()
}
var cerr smtpclient.Error
switch {
case err == nil:
deliveryResult = "ok"
success = true
case errors.Is(err, os.ErrDeadlineExceeded), errors.Is(err, context.DeadlineExceeded):
deliveryResult = "timeout"
case errors.Is(err, context.Canceled):
deliveryResult = "canceled"
case errors.As(err, &cerr):
deliveryResult = "temperror"
if cerr.Permanent {
deliveryResult = "permerror"
rcptErrs, submiterr := client.DeliverMultiple(deliverctx, m0.Sender().String(), rcpts, size, msgr, req8bit, reqsmtputf8, requireTLS)
if submiterr != nil {
qlog.Infox("smtp transaction for delivery failed", submiterr)
}
failed = 0 // Reset, we are looking at the SMTP results below.
var delIDs []int64
for i, m := range msgs {
qmlog := qlog.With(
slog.Int64("msgid", m.ID),
slog.Any("recipient", m.Recipient()))
err := submiterr
if err == nil && len(rcptErrs) > i {
if rcptErrs[i].Code != smtp.C250Completed {
err = smtpclient.Error(rcptErrs[i])
}
}
default:
deliveryResult = "error"
}
if err != nil {
smtperr, ok := err.(smtpclient.Error)
var remoteMTA dsn.NameIP
if ok {
remoteMTA.Name = transport.Host
if err != nil {
smtperr, ok := err.(smtpclient.Error)
err = fmt.Errorf("transport %s: submitting message to %s: %w", transportName, addr, err)
var remoteMTA dsn.NameIP
if ok {
remoteMTA.Name = transport.Host
smtperr.Err = err
err = smtperr
}
qmlog.Errorx("submitting message", err, slog.String("remote", addr))
fail(ctx, qmlog, []*Msg{m}, m0.DialedIPs, backoff, remoteMTA, err)
failed++
} else {
delIDs = append(delIDs, m.ID)
qmlog.Info("delivered from queue with transport")
delivered++
}
qlog.Errorx("submitting email", err, slog.String("remote", addr))
permanent = smtperr.Permanent
secodeOpt = smtperr.Secode
errmsg = fmt.Sprintf("transport %s: submitting email to %s: %v", transportName, addr, err)
fail(ctx, qlog, m, backoff, permanent, remoteMTA, secodeOpt, errmsg, smtperr.Line, smtperr.MoreLines)
return
}
qlog.Info("delivered from queue with transport")
if err := queueDelete(context.Background(), m.ID); err != nil {
qlog.Errorx("deleting message from queue after delivery", err)
if len(delIDs) > 0 {
if err := queueDelete(context.Background(), delIDs...); err != nil {
qlog.Errorx("deleting message from queue after delivery", err)
}
}
}