infrastructure/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2025-07-12 17:44:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

implement dnssec-awareness throughout code, and dane for incoming/outgoing mail delivery

Browse Source 
the vendored dns resolver code is a copy of the go stdlib dns resolver, with
awareness of the "authentic data" (i.e. dnssec secure) added, as well as support
for enhanced dns errors, and looking up tlsa records (for dane). ideally it
would be upstreamed, but the chances seem slim.

dnssec-awareness is added to all packages, e.g. spf, dkim, dmarc, iprev. their
dnssec status is added to the Received message headers for incoming email.

but the main reason to add dnssec was for implementing dane. with dane, the
verification of tls certificates can be done through certificates/public keys
published in dns (in the tlsa records). this only makes sense (is trustworthy)
if those dns records can be verified to be authentic.

mox now applies dane to delivering messages over smtp. mox already implemented
mta-sts for webpki/pkix-verification of certificates against the (large) pool
of CA's, and still enforces those policies when present. but it now also checks
for dane records, and will verify those if present. if dane and mta-sts are
both absent, the regular opportunistic tls with starttls is still done. and the
fallback to plaintext is also still done.

mox also makes it easy to setup dane for incoming deliveries, so other servers
can deliver with dane tls certificate verification. the quickstart now
generates private keys that are used when requesting certificates with acme.
the private keys are pre-generated because they must be static and known during
setup, because their public keys must be published in tlsa records in dns.
autocert would generate private keys on its own, so had to be forked to add the
option to provide the private key when requesting a new certificate. hopefully
upstream will accept the change and we can drop the fork.

with this change, using the quickstart to setup a new mox instance, the checks
at internet.nl result in a 100% score, provided the domain is dnssec-signed and
the network doesn't have any issues.
This commit is contained in:
Mechiel Lukkien
2023-10-10 12:09:35 +02:00
parent c4324fdaa1
commit daa908e9f4
177 changed files with 12907 additions and 3131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

369
queue/direct.go
View File

@ -11,6 +11,10 @@ import (
"strings"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/mjl-/adns"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/dns"
@ -23,6 +27,39 @@ import (
"github.com/mjl-/mox/store"
)
var (
metricDestinations = promauto.NewCounter(
prometheus.CounterOpts{
Name: "mox_queue_destinations_total",
Help: "Total destination (e.g. MX) lookups for delivery attempts, including those in mox_smtpclient_destinations_authentic_total.",
},
)
metricDestinationsAuthentic = promauto.NewCounter(
prometheus.CounterOpts{
Name: "mox_queue_destinations_authentic_total",
Help: "Destination (e.g. MX) lookups for delivery attempts authenticated with DNSSEC so they are candidates for DANE verification.",
},
)
metricDestinationDANERequired = promauto.NewCounter(
prometheus.CounterOpts{
Name: "mox_queue_destination_dane_required_total",
Help: "Total number of connections to hosts with valid TLSA records making DANE required.",
},
)
metricDestinationDANESTARTTLSUnverified = promauto.NewCounter(
prometheus.CounterOpts{
Name: "mox_queue_destination_dane_starttlsunverified_total",
Help: "Total number of connections with required DANE where all TLSA records were unusable.",
},
)
metricDestinationDANEGatherTLSAErrors = promauto.NewCounter(
prometheus.CounterOpts{
Name: "mox_queue_destination_dane_gathertlsa_errors_total",
Help: "Total number of connections where looking up TLSA records resulted in an error.",
},
)
)
// todo: rename function, perhaps put some of the params in a delivery struct so we don't pass all the params all the time?
func fail(qlog *mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMTA dsn.NameIP, secodeOpt, errmsg string) {
if permanent || m.Attempts >= 8 {
@ -53,9 +90,27 @@ func fail(qlog *mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMT
}
}
// Delivery by directly dialing MX hosts for destination domain.
func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer contextDialer, ourHostname dns.Domain, transportName string, m Msg, backoff time.Duration) {
hosts, effectiveDomain, permanent, err := gatherHosts(resolver, m, cid, qlog)
// Delivery by directly dialing (MX) hosts for destination domain of message.
func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, ourHostname dns.Domain, transportName string, m Msg, backoff time.Duration) {
// High-level approach:
// - Resolve domain to deliver to (CNAME), and determine hosts to try to deliver to (MX)
// - Get MTA-STS policy for domain (optional). If present, only deliver to its
// allowlisted hosts and verify TLS against CA pool.
// - For each host, attempt delivery. If the attempt results in a permanent failure
// (as claimed by remote with a 5xx SMTP response, or perhaps decided by us), the
// attempt can be aborted. Other errors are often temporary and may result in later
// successful delivery. But hopefully the delivery just succeeds. For each host:
// - If there is an MTA-STS policy, we only connect to allow-listed hosts.
// - We try to lookup DANE records (optional) and verify them if present.
// Resolve domain and hosts to attempt delivery to.
// These next-hop names are often the name under which we find MX records. The
// expanded name is different from the original if the original was a CNAME,
// possibly a chain. If there are no MX records, it can be an IP or the host
// directly.
origNextHop := m.RecipientDomain.Domain
ctx := context.WithValue(mox.Context, mlog.CidKey, cid)
haveMX, origNextHopAuthentic, expandedNextHopAuthentic, expandedNextHop, hosts, permanent, err := smtpclient.GatherDestinations(ctx, qlog, resolver, m.RecipientDomain)
if err != nil {
fail(qlog, m, backoff, permanent, dsn.NameIP{}, "", err.Error())
return
@ -63,65 +118,87 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer cont
// Check for MTA-STS policy and enforce it if needed. We have to check the
// effective domain (found after following CNAME record(s)): there will certainly
// not be an mtasts record for the original recipient domain, because that is not
// not be an MTA-STS record for the original recipient domain, because that is not
// allowed when a CNAME record is present.
var policyFresh bool
var policy *mtasts.Policy
tlsModeDefault := smtpclient.TLSOpportunistic
if !effectiveDomain.IsZero() {
if !expandedNextHop.IsZero() {
cidctx := context.WithValue(mox.Shutdown, mlog.CidKey, cid)
policy, policyFresh, err = mtastsdb.Get(cidctx, resolver, effectiveDomain)
policy, _, err = mtastsdb.Get(cidctx, resolver, expandedNextHop)
if err != nil {
// No need to refuse to deliver if we have some mtasts error.
qlog.Infox("mtasts failed, continuing with strict tls requirement", err, mlog.Field("domain", effectiveDomain))
qlog.Infox("mtasts failed, continuing with strict tls requirement", err, mlog.Field("domain", expandedNextHop))
tlsModeDefault = smtpclient.TLSStrictStartTLS
}
// note: policy can be nil, if a domain does not implement MTA-STS or its the first
// time we fetch the policy and if we encountered an error.
// note: policy can be nil, if a domain does not implement MTA-STS or it's the
// first time we fetch the policy and if we encountered an error.
}
// We try delivery to each record until we have success or a permanent failure. So
// for transient errors, we'll try the next MX record. For MX records pointing to a
// We try delivery to each host until we have success or a permanent failure. So
// for transient errors, we'll try the next host. For MX records pointing to a
// dual stack host, we turn a permanent failure due to policy on the first delivery
// attempt into a temporary failure and make sure to try the other address family
// the next attempt. This should reduce issues due to one of our IPs being on a
// block list. We won't try multiple IPs of the same address family. Surprisingly,
// RFC 5321 does not specify a clear algorithm, but common practicie is probably
// RFC 5321 does not specify a clear algorithm, but common practice is probably
// ../rfc/3974:268.
var remoteMTA dsn.NameIP
var secodeOpt, errmsg string
permanent = false
mtastsFailure := true
// todo: should make distinction between host permanently not accepting the message, and the message not being deliverable permanently. e.g. a mx host may have a size limit, or not accept 8bitmime, while another host in the list does accept the message. same for smtputf8, ../rfc/6531:555
for _, h := range hosts {
var badTLS, ok bool
// ../rfc/8461:913
if policy != nil && policy.Mode == mtasts.ModeEnforce && !policy.Matches(h.Domain) {
if policy != nil && !policy.Matches(h.Domain) {
var policyHosts []string
for _, mx := range policy.MX {
policyHosts = append(policyHosts, mx.LogString())
}
errmsg = fmt.Sprintf("mx host %s does not match enforced mta-sts policy with hosts %s", h.Domain, strings.Join(policyHosts, ","))
qlog.Error("mx host does not match enforce mta-sts policy, skipping", mlog.Field("host", h.Domain), mlog.Field("policyhosts", policyHosts))
continue
if policy.Mode == mtasts.ModeEnforce {
errmsg = fmt.Sprintf("mx host %s does not match enforced mta-sts policy with hosts %s", h.Domain, strings.Join(policyHosts, ","))
qlog.Error("mx host does not match mta-sts policy in mode enforce, skipping", mlog.Field("host", h.Domain), mlog.Field("policyhosts", policyHosts))
continue
}
qlog.Error("mx host does not match mta-sts policy, but it is not enforced, continuing", mlog.Field("host", h.Domain), mlog.Field("policyhosts", policyHosts))
}
qlog.Info("delivering to remote", mlog.Field("remote", h), mlog.Field("queuecid", cid))
cid := mox.Cid()
nqlog := qlog.WithCid(cid)
var remoteIP net.IP
tlsMode := tlsModeDefault
if policy != nil && policy.Mode == mtasts.ModeEnforce {
tlsMode = smtpclient.TLSStrictStartTLS
}
permanent, badTLS, secodeOpt, remoteIP, errmsg, ok = deliverHost(nqlog, resolver, dialer, cid, ourHostname, transportName, h, &m, tlsMode)
if !ok && badTLS && tlsMode == smtpclient.TLSOpportunistic {
// Try to deliver to host. We can get various errors back. Like permanent failure
// response codes, TCP, DNSSEC, TLS (opportunistic, i.e. optional with fallback to
// without), etc. It's a balancing act to handle these situations correctly. We
// don't want to bounce unnecessarily. But also not keep trying if there is no
// chance of success.
// Set if there TLSA records were found. Means TLS is required for this host,
// usually with verification of the certificate.
var daneRequired bool
enforceMTASTS := policy != nil && policy.Mode == mtasts.ModeEnforce
permanent, daneRequired, badTLS, secodeOpt, remoteIP, errmsg, ok = deliverHost(nqlog, resolver, dialer, cid, ourHostname, transportName, h, enforceMTASTS, haveMX, origNextHopAuthentic, origNextHop, expandedNextHopAuthentic, expandedNextHop, &m, tlsMode)
// If we had a TLS-related failure when doing opportunistic (optional) TLS, and no
// DANE records were not found, we should try again without TLS. This could be an
// old server that only does ancient TLS versions, or has a misconfiguration. Note
// that opportunistic TLS does not do regular certificate verification, so that can't
// be the problem.
if !ok && badTLS && !enforceMTASTS && tlsMode == smtpclient.TLSOpportunistic && !daneRequired {
// In case of failure with opportunistic TLS, try again without TLS. ../rfc/7435:459
// todo future: revisit this decision. perhaps it should be a configuration option that defaults to not doing this?
// todo future: add a configuration option to not fall back?
nqlog.Info("connecting again for delivery attempt without tls")
permanent, badTLS, secodeOpt, remoteIP, errmsg, ok = deliverHost(nqlog, resolver, dialer, cid, ourHostname, transportName, h, &m, smtpclient.TLSSkip)
tlsMode = smtpclient.TLSSkip
permanent, _, _, secodeOpt, remoteIP, errmsg, ok = deliverHost(nqlog, resolver, dialer, cid, ourHostname, transportName, h, enforceMTASTS, haveMX, origNextHopAuthentic, origNextHop, expandedNextHopAuthentic, expandedNextHop, &m, tlsMode)
}
if ok {
nqlog.Info("delivered from queue")
if err := queueDelete(context.Background(), m.ID); err != nil {
@ -130,141 +207,54 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer cont
return
}
remoteMTA = dsn.NameIP{Name: h.XString(false), IP: remoteIP}
if !badTLS {
mtastsFailure = false
}
if permanent {
break
}
}
if mtastsFailure && policyFresh {
permanent = true
}
// In theory, we could make a failure permanent if we didn't find any mx host
// matching the mta-sts policy AND the policy is fresh AND all DNS records leading
// to the MX targets (including CNAME) have a TTL that is beyond the latest
// possible delivery attempt. Until that time, configuration problems can be
// corrected through DNS or policy update. Not sure if worth it in practice, there
// is a good chance the MX records can still change, at least on initial delivery
// failures.
// todo: possibly detect that future deliveries will fail due to long ttl's of cached records that are preventing delivery.
fail(qlog, m, backoff, permanent, remoteMTA, secodeOpt, errmsg)
}
var (
errCNAMELoop = errors.New("cname loop")
errCNAMELimit = errors.New("too many cname records")
errNoRecord = errors.New("no dns record")
errDNS = errors.New("dns lookup error")
errNoMail = errors.New("domain does not accept email as indicated with single dot for mx record")
)
// Gather hosts to try to deliver to. We start with the straight-forward MX record.
// If that does not exist, we'll look for CNAME of the entire domain (following
// chains if needed). If a CNAME does not exist, but the domain name has an A or
// AAAA record, we'll try delivery directly to that host.
// ../rfc/5321:3824
func gatherHosts(resolver dns.Resolver, m Msg, cid int64, qlog *mlog.Log) (hosts []dns.IPDomain, effectiveDomain dns.Domain, permanent bool, err error) {
if len(m.RecipientDomain.IP) > 0 {
return []dns.IPDomain{m.RecipientDomain}, effectiveDomain, false, nil
}
// We start out delivering to the recipient domain. We follow CNAMEs a few times.
rcptDomain := m.RecipientDomain.Domain
// Domain we are actually delivering to, after following CNAME record(s).
effectiveDomain = rcptDomain
domainsSeen := map[string]bool{}
for i := 0; ; i++ {
if domainsSeen[effectiveDomain.ASCII] {
return nil, effectiveDomain, true, fmt.Errorf("%w: recipient domain %s: already saw %s", errCNAMELoop, rcptDomain, effectiveDomain)
}
domainsSeen[effectiveDomain.ASCII] = true
// note: The Go resolver returns the requested name if the domain has no CNAME record but has a host record.
if i == 16 {
// We have a maximum number of CNAME records we follow. There is no hard limit for
// DNS, and you might think folks wouldn't configure CNAME chains at all, but for
// (non-mail) domains, CNAME chains of 10 records have been encountered according
// to the internet.
return nil, effectiveDomain, true, fmt.Errorf("%w: recipient domain %s, last resolved domain %s", errCNAMELimit, rcptDomain, effectiveDomain)
}
cidctx := context.WithValue(mox.Context, mlog.CidKey, cid)
ctx, cancel := context.WithTimeout(cidctx, 30*time.Second)
defer cancel()
// Note: LookupMX can return an error and still return records: Invalid records are
// filtered out and an error returned. We must process any records that are valid.
// Only if all are unusable will we return an error. ../rfc/5321:3851
mxl, err := resolver.LookupMX(ctx, effectiveDomain.ASCII+".")
cancel()
if err != nil && len(mxl) == 0 {
if !dns.IsNotFound(err) {
return nil, effectiveDomain, false, fmt.Errorf("%w: mx lookup for %s: %v", errDNS, effectiveDomain, err)
}
// No MX record. First attempt CNAME lookup. ../rfc/5321:3838 ../rfc/3974:197
ctx, cancel = context.WithTimeout(cidctx, 30*time.Second)
defer cancel()
cname, err := resolver.LookupCNAME(ctx, effectiveDomain.ASCII+".")
cancel()
if err != nil && !dns.IsNotFound(err) {
return nil, effectiveDomain, false, fmt.Errorf("%w: cname lookup for %s: %v", errDNS, effectiveDomain, err)
}
if err == nil && cname != effectiveDomain.ASCII+"." {
d, err := dns.ParseDomain(strings.TrimSuffix(cname, "."))
if err != nil {
return nil, effectiveDomain, true, fmt.Errorf("%w: parsing cname domain %s: %v", errDNS, effectiveDomain, err)
}
effectiveDomain = d
// Start again with new domain.
continue
}
// See if the host exists. If so, attempt delivery directly to host. ../rfc/5321:3842
ctx, cancel = context.WithTimeout(cidctx, 30*time.Second)
defer cancel()
_, err = resolver.LookupHost(ctx, effectiveDomain.ASCII+".")
cancel()
if dns.IsNotFound(err) {
return nil, effectiveDomain, true, fmt.Errorf("%w: recipient domain/host %s", errNoRecord, effectiveDomain)
} else if err != nil {
return nil, effectiveDomain, false, fmt.Errorf("%w: looking up host %s because of no mx record: %v", errDNS, effectiveDomain, err)
}
hosts = []dns.IPDomain{{Domain: effectiveDomain}}
} else if err != nil {
qlog.Infox("partial mx failure, attempting delivery to valid mx records", err)
}
// ../rfc/7505:122
if err == nil && len(mxl) == 1 && mxl[0].Host == "." {
return nil, effectiveDomain, true, errNoMail
}
// The Go resolver already sorts by preference, randomizing records of same
// preference. ../rfc/5321:3885
for _, mx := range mxl {
host, err := dns.ParseDomain(strings.TrimSuffix(mx.Host, "."))
if err != nil {
// note: should not happen because Go resolver already filters these out.
return nil, effectiveDomain, true, fmt.Errorf("%w: invalid host name in mx record %q: %v", errDNS, mx.Host, err)
}
hosts = append(hosts, dns.IPDomain{Domain: host})
}
if len(hosts) > 0 {
err = nil
}
return hosts, effectiveDomain, false, err
}
}
// deliverHost attempts to deliver m to host.
// deliverHost updated m.DialedIPs, which must be saved in case of failure to deliver.
func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer contextDialer, cid int64, ourHostname dns.Domain, transportName string, host dns.IPDomain, m *Msg, tlsMode smtpclient.TLSMode) (permanent, badTLS bool, secodeOpt string, remoteIP net.IP, errmsg string, ok bool) {
// deliverHost attempts to deliver m to host. Depending on tlsMode, we'll do
// required TLS with WebPKI verification (with MTA-STS), opportunistic DANE TLS
// (opportunistic TLS) or non-verifying TLS (opportunistic TLS) deliverHost updates
// m.DialedIPs, which must be saved in case of failure to deliver.
//
// The haveMX and next-hop-authentic fields are used to determine if DANE is
// applicable. The next-hop fields themselves are used to determine valid names
// during DANE TLS certificate verification.
func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, cid int64, ourHostname dns.Domain, transportName string, host dns.IPDomain, enforceMTASTS, haveMX, origNextHopAuthentic bool, origNextHop dns.Domain, expandedNextHopAuthentic bool, expandedNextHop dns.Domain, m *Msg, tlsMode smtpclient.TLSMode) (permanent, daneRequired, badTLS bool, secodeOpt string, remoteIP net.IP, errmsg string, ok bool) {
// About attempting delivery to multiple addresses of a host: ../rfc/5321:3898
start := time.Now()
var deliveryResult string
defer func() {
metricDelivery.WithLabelValues(fmt.Sprintf("%d", m.Attempts), transportName, string(tlsMode), deliveryResult).Observe(float64(time.Since(start)) / float64(time.Second))
log.Debug("queue deliverhost result", mlog.Field("host", host), mlog.Field("attempt", m.Attempts), mlog.Field("tlsmode", tlsMode), mlog.Field("permanent", permanent), mlog.Field("badtls", badTLS), mlog.Field("secodeopt", secodeOpt), mlog.Field("errmsg", errmsg), mlog.Field("ok", ok), mlog.Field("duration", time.Since(start)))
log.Debug("queue deliverhost result",
mlog.Field("host", host),
mlog.Field("attempt", m.Attempts),
mlog.Field("tlsmode", tlsMode),
mlog.Field("permanent", permanent),
mlog.Field("badtls", badTLS),
mlog.Field("secodeopt", secodeOpt),
mlog.Field("errmsg", errmsg),
mlog.Field("ok", ok),
mlog.Field("duration", time.Since(start)))
}()
// Open message to deliver.
f, err := os.Open(m.MessagePath())
if err != nil {
return false, false, "", nil, fmt.Sprintf("open message file: %s", err), false
return false, false, false, "", nil, fmt.Sprintf("open message file: %s", err), false
}
msgr := store.FileMsgReader(m.MsgPrefix, f)
defer func() {
@ -276,9 +266,83 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer contextDialer, cid
ctx, cancel := context.WithTimeout(cidctx, 30*time.Second)
defer cancel()
conn, ip, dualstack, err := dialHost(ctx, log, resolver, dialer, host, 25, m)
remoteIP = ip
// We must lookup the IPs for the host name before checking DANE TLSA records. And
// only check TLSA records for secure responses. This prevents problems with old
// name servers returning an error for TLSA requests or letting it timeout (not
// sending a response). ../rfc/7672:879
var daneRecords []adns.TLSA
var tlsRemoteHostnames []dns.Domain
if host.IsDomain() {
tlsRemoteHostnames = []dns.Domain{host.Domain}
}
if m.DialedIPs == nil {
m.DialedIPs = map[string][]net.IP{}
}
metricDestinations.Inc()
authentic, expandedAuthentic, expandedHost, ips, dualstack, err := smtpclient.GatherIPs(ctx, log, resolver, host, m.DialedIPs)
if err == nil && authentic && origNextHopAuthentic && (!haveMX || expandedNextHopAuthentic) && host.IsDomain() {
metricDestinationsAuthentic.Inc()
// Modes to skip and not verify aren't normally set when we get here. But in the
// future may perhaps be set on a message manually after delivery failures. We can
// handle them here.
switch tlsMode {
case smtpclient.TLSSkip:
// No TLS, so clearly no DANE.
case smtpclient.TLSUnverifiedStartTLS:
// Fallback mode for DANE without usable records, so skip DANE.
default:
// Look for TLSA records in either the expandedHost, or otherwise the original
// host. ../rfc/7672:912
var tlsaBaseDomain dns.Domain
daneRequired, daneRecords, tlsaBaseDomain, err = smtpclient.GatherTLSA(ctx, log, resolver, host.Domain, expandedNextHopAuthentic && expandedAuthentic, expandedHost)
if daneRequired {
metricDestinationDANERequired.Inc()
}
if err != nil {
metricDestinationDANEGatherTLSAErrors.Inc()
}
if err == nil && daneRequired {
tlsMode = smtpclient.TLSStrictStartTLS
if len(daneRecords) == 0 {
// If there are no usable DANE records, we still have to use TLS, but without
// verifying its certificate. At least when there is no MTA-STS. Why? Perhaps to
// prevent ossification? The SMTP TLSA specification has different behaviour than
// the generic TLSA. "Usable" means different things in different places.
// ../rfc/7672:718 ../rfc/6698:1845 ../rfc/6698:660
if !enforceMTASTS {
tlsMode = smtpclient.TLSUnverifiedStartTLS
log.Debug("no usable dane records, not verifying dane records, but doing required non-verifying opportunistic tls")
metricDestinationDANESTARTTLSUnverified.Inc()
}
daneRecords = nil
} else {
// Based on CNAMEs followed and DNSSEC-secure status, we must allow up to 4 host
// names.
tlsRemoteHostnames = smtpclient.GatherTLSANames(haveMX, expandedNextHopAuthentic, expandedAuthentic, origNextHop, expandedNextHop, host.Domain, tlsaBaseDomain)
log.Debug("delivery with required starttls with dane verification", mlog.Field("allowedtlshostnames", tlsRemoteHostnames))
}
} else if !daneRequired {
log.Debugx("not doing opportunistic dane after gathering tlsa records", err)
err = nil
}
// else, err is propagated below.
}
} else {
log.Debugx("not attempting verification with dane", err, mlog.Field("authentic", authentic), mlog.Field("expandedauthentic", expandedAuthentic))
}
// Dial the remote host given the IPs if no error yet.
var conn net.Conn
if err == nil {
if m.DialedIPs == nil {
m.DialedIPs = map[string][]net.IP{}
}
conn, remoteIP, err = smtpclient.Dial(ctx, log, dialer, host, ips, 25, m.DialedIPs)
}
cancel()
// Set error for metrics.
var result string
switch {
case err == nil:
@ -293,7 +357,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer contextDialer, cid
metricConnection.WithLabelValues(result).Inc()
if err != nil {
log.Debugx("connecting to remote smtp", err, mlog.Field("host", host))
return false, false, "", ip, fmt.Sprintf("dialing smtp server: %v", err), false
return false, daneRequired, false, "", remoteIP, fmt.Sprintf("dialing smtp server: %v", err), false
}
var mailFrom string
@ -303,11 +367,21 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer contextDialer, cid
rcptTo := m.Recipient().XString(m.SMTPUTF8)
// todo future: get closer to timeouts specified in rfc? ../rfc/5321:3610
log = log.Fields(mlog.Field("remoteip", ip))
log = log.Fields(mlog.Field("remoteip", remoteIP))
ctx, cancel = context.WithTimeout(cidctx, 30*time.Minute)
defer cancel()
mox.Connections.Register(conn, "smtpclient", "queue")
sc, err := smtpclient.New(ctx, log, conn, tlsMode, ourHostname, host.Domain, nil)
// Initialize SMTP session, sending EHLO/HELO and STARTTLS with specified tls mode.
var firstHost dns.Domain
var moreHosts []dns.Domain
if len(tlsRemoteHostnames) > 0 {
// For use with DANE-TA.
firstHost = tlsRemoteHostnames[0]
moreHosts = tlsRemoteHostnames[1:]
}
var verifiedRecord adns.TLSA
sc, err := smtpclient.New(ctx, log, conn, tlsMode, ourHostname, firstHost, nil, daneRecords, moreHosts, &verifiedRecord)
defer func() {
if sc == nil {
conn.Close()
@ -317,6 +391,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer contextDialer, cid
mox.Connections.Unregister(conn)
}()
if err == nil {
// SMTP session is ready. Finally try to actually deliver.
has8bit := m.Has8bit
smtputf8 := m.SMTPUTF8
var msg io.Reader = msgr
@ -349,7 +424,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer contextDialer, cid
deliveryResult = "error"
}
if err == nil {
return false, false, "", ip, "", true
return false, daneRequired, false, "", remoteIP, "", true
} else if cerr, ok := err.(smtpclient.Error); ok {
// If we are being rejected due to policy reasons on the first
// attempt and remote has both IPv4 and IPv6, we'll give it
@ -359,8 +434,8 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer contextDialer, cid
if permanent && m.Attempts == 1 && dualstack && strings.HasPrefix(cerr.Secode, "7.") {
permanent = false
}
return permanent, errors.Is(cerr, smtpclient.ErrTLS), cerr.Secode, ip, cerr.Error(), false
return permanent, daneRequired, errors.Is(cerr, smtpclient.ErrTLS), cerr.Secode, remoteIP, cerr.Error(), false
} else {
return false, errors.Is(cerr, smtpclient.ErrTLS), "", ip, err.Error(), false
return false, daneRequired, errors.Is(cerr, smtpclient.ErrTLS), "", remoteIP, err.Error(), false
}
}

118
queue/queue.go
View File

@ -30,6 +30,7 @@ import (
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/smtp"
"github.com/mjl-/mox/smtpclient"
"github.com/mjl-/mox/store"
)
@ -60,24 +61,6 @@ var (
)
)
type contextDialer interface {
DialContext(ctx context.Context, network, addr string) (c net.Conn, err error)
}
// Used to dial remote SMTP servers.
// Overridden for tests.
var dial = func(ctx context.Context, dialer contextDialer, timeout time.Duration, addr string, laddr net.Addr) (net.Conn, error) {
// If this is a net.Dialer, use its settings and add the timeout and localaddr.
// This is the typical case, but SOCKS5 support can use a different dialer.
if d, ok := dialer.(*net.Dialer); ok {
nd := *d
nd.Timeout = timeout
nd.LocalAddr = laddr
return nd.DialContext(ctx, "tcp", addr)
}
return dialer.DialContext(ctx, "tcp", addr)
}
var jitter = mox.NewRand()
var DBTypes = []any{Msg{}} // Types stored in DB.
@ -547,7 +530,7 @@ func deliver(resolver dns.Resolver, m Msg) {
qlog.Debug("delivering with transport", mlog.Field("transport", transportName))
}
var dialer contextDialer = &net.Dialer{}
var dialer smtpclient.Dialer = &net.Dialer{}
if transport.Submissions != nil {
deliverSubmit(cid, qlog, resolver, dialer, m, backoff, transportName, transport.Submissions, true, 465)
} else if transport.Submission != nil {
@ -561,7 +544,7 @@ func deliver(resolver dns.Resolver, m Msg) {
if err != nil {
fail(qlog, m, backoff, false, dsn.NameIP{}, "", fmt.Sprintf("socks dialer: %v", err))
return
} else if d, ok := socksdialer.(contextDialer); !ok {
} else if d, ok := socksdialer.(smtpclient.Dialer); !ok {
fail(qlog, m, backoff, false, dsn.NameIP{}, "", "socks dialer is not a contextdialer")
return
} else {
@ -611,98 +594,3 @@ func routeMatchDomain(l []string, d dns.Domain) bool {
}
return false
}
// dialHost dials host for delivering Msg, taking previous attempts into accounts.
// If the previous attempt used IPv4, this attempt will use IPv6 (in case one of the IPs is in a DNSBL).
// The second attempt for an address family we prefer the same IP as earlier, to increase our chances if remote is doing greylisting.
// dialHost updates m with the dialed IP and m should be saved in case of failure.
// If we have fully specified local smtp listen IPs, we set those for the outgoing
// connection. The admin probably configured these same IPs in SPF, but others
// possibly not.
func dialHost(ctx context.Context, log *mlog.Log, resolver dns.Resolver, dialer contextDialer, host dns.IPDomain, port int, m *Msg) (conn net.Conn, ip net.IP, dualstack bool, rerr error) {
var ips []net.IP
if len(host.IP) > 0 {
ips = []net.IP{host.IP}
} else {
// todo: The Go resolver automatically follows CNAMEs, which is not allowed for
// host names in MX records. ../rfc/5321:3861 ../rfc/2181:661
name := host.Domain.ASCII + "."
ipaddrs, err := resolver.LookupIPAddr(ctx, name)
if err != nil || len(ipaddrs) == 0 {
return nil, nil, false, fmt.Errorf("looking up %q: %v", name, err)
}
var have4, have6 bool
for _, ipaddr := range ipaddrs {
ips = append(ips, ipaddr.IP)
if ipaddr.IP.To4() == nil {
have6 = true
} else {
have4 = true
}
}
dualstack = have4 && have6
prevIPs := m.DialedIPs[host.String()]
if len(prevIPs) > 0 {
prevIP := prevIPs[len(prevIPs)-1]
prevIs4 := prevIP.To4() != nil
sameFamily := 0
for _, ip := range prevIPs {
is4 := ip.To4() != nil
if prevIs4 == is4 {
sameFamily++
}
}
preferPrev := sameFamily == 1
// We use stable sort so any preferred/randomized listing from DNS is kept intact.
sort.SliceStable(ips, func(i, j int) bool {
aIs4 := ips[i].To4() != nil
bIs4 := ips[j].To4() != nil
if aIs4 != bIs4 {
// Prefer "i" if it is not same address family.
return aIs4 != prevIs4
}
// Prefer "i" if it is the same as last and we should be preferring it.
return preferPrev && ips[i].Equal(prevIP)
})
log.Debug("ordered ips for dialing", mlog.Field("ips", ips))
}
}
var timeout time.Duration
deadline, ok := ctx.Deadline()
if !ok {
timeout = 30 * time.Second
} else {
timeout = time.Until(deadline) / time.Duration(len(ips))
}
var lastErr error
var lastIP net.IP
for _, ip := range ips {
addr := net.JoinHostPort(ip.String(), fmt.Sprintf("%d", port))
log.Debug("dialing remote host for delivery", mlog.Field("addr", addr))
var laddr net.Addr
for _, lip := range mox.Conf.Static.SpecifiedSMTPListenIPs {
ipIs4 := ip.To4() != nil
lipIs4 := lip.To4() != nil
if ipIs4 == lipIs4 {
laddr = &net.TCPAddr{IP: lip}
break
}
}
conn, err := dial(ctx, dialer, timeout, addr, laddr)
if err == nil {
log.Debug("connected for smtp delivery", mlog.Field("host", host), mlog.Field("addr", addr), mlog.Field("laddr", laddr))
if m.DialedIPs == nil {
m.DialedIPs = map[string][]net.IP{}
}
name := host.String()
m.DialedIPs[name] = append(m.DialedIPs[name], ip)
return conn, ip, dualstack, nil
}
log.Debugx("connection attempt for smtp delivery", err, mlog.Field("host", host), mlog.Field("addr", addr), mlog.Field("laddr", laddr))
lastErr = err
lastIP = ip
}
return nil, lastIP, dualstack, lastErr
}

380
queue/queue_test.go
View File

@ -5,24 +5,25 @@ import (
"context"
"crypto/ed25519"
cryptorand "crypto/rand"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"io"
"math/big"
"net"
"os"
"reflect"
"strings"
"testing"
"time"
"github.com/mjl-/adns"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/smtp"
"github.com/mjl-/mox/smtpclient"
"github.com/mjl-/mox/store"
)
@ -132,13 +133,18 @@ func TestQueue(t *testing.T) {
MX: map[string][]*net.MX{"mox.example.": {{Host: "mox.example", Pref: 10}}},
}
dialed := make(chan struct{}, 1)
dial = func(ctx context.Context, dialer contextDialer, timeout time.Duration, addr string, laddr net.Addr) (net.Conn, error) {
smtpclient.DialHook = func(ctx context.Context, dialer smtpclient.Dialer, timeout time.Duration, addr string, laddr net.Addr) (net.Conn, error) {
dialed <- struct{}{}
return nil, fmt.Errorf("failure from test")
}
defer func() {
smtpclient.DialHook = nil
}()
launchWork(resolver, map[string]struct{}{})
moxCert := fakeCert(t, "mox.example", false)
// Wait until we see the dial and the failed attempt.
timer := time.NewTimer(time.Second)
defer timer.Stop()
@ -192,23 +198,92 @@ func TestQueue(t *testing.T) {
// We do a minimal fake smtp server. We cannot import smtpserver.Serve due to cyclic dependencies.
fmt.Fprintf(server, "220 mox.example\r\n")
br := bufio.NewReader(server)
br.ReadString('\n') // Should be EHLO.
fmt.Fprintf(server, "250 ok\r\n")
br.ReadString('\n') // Should be MAIL FROM.
fmt.Fprintf(server, "250 ok\r\n")
br.ReadString('\n') // Should be RCPT TO.
fmt.Fprintf(server, "250 ok\r\n")
br.ReadString('\n') // Should be DATA.
fmt.Fprintf(server, "354 continue\r\n")
readline := func(cmd string) {
line, err := br.ReadString('\n')
if err == nil && !strings.HasPrefix(strings.ToLower(line), cmd) {
panic(fmt.Sprintf("unexpected line %q, expected %q", line, cmd))
}
}
writeline := func(s string) {
fmt.Fprintf(server, "%s\r\n", s)
}
readline("ehlo")
writeline("250 mox.example")
readline("mail")
writeline("250 ok")
readline("rcpt")
writeline("250 ok")
readline("data")
writeline("354 continue")
reader := smtp.NewDataReader(br)
io.Copy(io.Discard, reader)
fmt.Fprintf(server, "250 ok\r\n")
br.ReadString('\n') // Should be QUIT.
fmt.Fprintf(server, "221 ok\r\n")
writeline("250 ok")
readline("quit")
writeline("221 ok")
smtpdone <- struct{}{}
}
goodTLSConfig := tls.Config{Certificates: []tls.Certificate{moxCert}}
makeFakeSMTPSTARTTLSServer := func(tlsConfig *tls.Config, nstarttls int) func(server net.Conn) {
attempt := 0
return func(server net.Conn) {
attempt++
// We do a minimal fake smtp server. We cannot import smtpserver.Serve due to cyclic dependencies.
fmt.Fprintf(server, "220 mox.example\r\n")
br := bufio.NewReader(server)
readline := func(cmd string) {
line, err := br.ReadString('\n')
if err == nil && !strings.HasPrefix(strings.ToLower(line), cmd) {
panic(fmt.Sprintf("unexpected line %q, expected %q", line, cmd))
}
}
writeline := func(s string) {
fmt.Fprintf(server, "%s\r\n", s)
}
readline("ehlo")
writeline("250-mox.example")
writeline("250 starttls")
if nstarttls == 0 || attempt <= nstarttls {
readline("starttls")
writeline("220 ok")
tlsConn := tls.Server(server, tlsConfig)
err := tlsConn.Handshake()
if err != nil {
return
}
server = tlsConn
br = bufio.NewReader(server)
readline("ehlo")
writeline("250 mox.example")
}
readline("mail")
writeline("250 ok")
readline("rcpt")
writeline("250 ok")
readline("data")
writeline("354 continue")
reader := smtp.NewDataReader(br)
io.Copy(io.Discard, reader)
writeline("250 ok")
readline("quit")
writeline("221 ok")
smtpdone <- struct{}{}
}
}
fakeSMTPSTARTTLSServer := makeFakeSMTPSTARTTLSServer(&goodTLSConfig, 0)
makeBadFakeSMTPSTARTTLSServer := func() func(server net.Conn) {
return makeFakeSMTPSTARTTLSServer(&tls.Config{MaxVersion: tls.VersionTLS10, Certificates: []tls.Certificate{moxCert}}, 1)
}
fakeSubmitServer := func(server net.Conn) {
// We do a minimal fake smtp server. We cannot import smtpserver.Serve due to cyclic dependencies.
fmt.Fprintf(server, "220 mox.example\r\n")
@ -236,19 +311,37 @@ func TestQueue(t *testing.T) {
testDeliver := func(fakeServer func(conn net.Conn)) bool {
t.Helper()
// Setting up a pipe. We'll start a fake smtp server on the server-side. And return the
// client-side to the invocation dial, for the attempted delivery from the queue.
// The delivery should succeed.
server, client := net.Pipe()
defer server.Close()
defer client.Close()
var pipes []net.Conn
defer func() {
for _, conn := range pipes {
conn.Close()
}
}()
var wasNetDialer bool
dial = func(ctx context.Context, dialer contextDialer, timeout time.Duration, addr string, laddr net.Addr) (net.Conn, error) {
smtpclient.DialHook = func(ctx context.Context, dialer smtpclient.Dialer, timeout time.Duration, addr string, laddr net.Addr) (net.Conn, error) {
// Setting up a pipe. We'll start a fake smtp server on the server-side. And return the
// client-side to the invocation dial, for the attempted delivery from the queue.
server, client := net.Pipe()
for _, c := range pipes {
c.Close()
}
pipes = []net.Conn{server, client}
go fakeServer(server)
_, wasNetDialer = dialer.(*net.Dialer)
dialed <- struct{}{}
// For reconnects, we are already waiting for delivery below.
select {
case dialed <- struct{}{}:
default:
}
return client, nil
}
defer func() {
smtpclient.DialHook = nil
}()
waitDeliver := func() {
t.Helper()
@ -279,7 +372,6 @@ func TestQueue(t *testing.T) {
<-deliveryResult // Deliver sends here.
}
go fakeServer(server)
launchWork(resolver, map[string]struct{}{})
waitDeliver()
return wasNetDialer
@ -325,7 +417,7 @@ func TestQueue(t *testing.T) {
}
// Add a message to be delivered with socks.
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, prepareFile(t), nil, true)
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<socks@localhost>", nil, prepareFile(t), nil, true)
tcheck(t, err, "add message to queue for delivery")
transportSocks := "socks"
n, err = Kick(ctxbg, msgID, "", "", &transportSocks)
@ -338,6 +430,79 @@ func TestQueue(t *testing.T) {
t.Fatalf("expected non-net.Dialer as dialer") // SOCKS5 dialer is a private type, we cannot check for it.
}
// Add message to be delivered with opportunistic TLS verification.
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, prepareFile(t), nil, true)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
tcheck(t, err, "kick queue")
if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n)
}
testDeliver(fakeSMTPSTARTTLSServer)
// Test fallback to plain text with TLS handshake fails.
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<badtls@localhost>", nil, prepareFile(t), nil, true)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
tcheck(t, err, "kick queue")
if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n)
}
testDeliver(makeBadFakeSMTPSTARTTLSServer())
// Add message to be delivered with DANE verification.
resolver.AllAuthentic = true
resolver.TLSA = map[string][]adns.TLSA{
"_25._tcp.mox.example.": {
{Usage: adns.TLSAUsageDANEEE, Selector: adns.TLSASelectorSPKI, MatchType: adns.TLSAMatchTypeFull, CertAssoc: moxCert.Leaf.RawSubjectPublicKeyInfo},
},
}
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<dane@localhost>", nil, prepareFile(t), nil, true)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
tcheck(t, err, "kick queue")
if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n)
}
testDeliver(fakeSMTPSTARTTLSServer)
// Check that message is delivered with all unusable DANE records.
resolver.TLSA = map[string][]adns.TLSA{
"_25._tcp.mox.example.": {
{},
},
}
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<daneunusable@localhost>", nil, prepareFile(t), nil, true)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
tcheck(t, err, "kick queue")
if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n)
}
testDeliver(fakeSMTPSTARTTLSServer)
// Check that message is delivered with insecure TLSA records. They should be
// ignored and regular STARTTLS tried.
resolver.Inauthentic = []string{"tlsa _25._tcp.mox.example."}
resolver.TLSA = map[string][]adns.TLSA{
"_25._tcp.mox.example.": {
{Usage: adns.TLSAUsageDANEEE, Selector: adns.TLSASelectorSPKI, MatchType: adns.TLSAMatchTypeFull, CertAssoc: make([]byte, sha256.Size)},
},
}
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<daneinsecure@localhost>", nil, prepareFile(t), nil, true)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
tcheck(t, err, "kick queue")
if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n)
}
testDeliver(makeBadFakeSMTPSTARTTLSServer())
resolver.Inauthentic = nil
// Restore pre-DANE behaviour.
resolver.AllAuthentic = false
resolver.TLSA = nil
// Add another message that we'll fail to deliver entirely.
_, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, prepareFile(t), nil, true)
tcheck(t, err, "add message to queue for delivery")
@ -349,10 +514,10 @@ func TestQueue(t *testing.T) {
}
msg = msgs[0]
prepServer := func(code string) (net.Conn, func()) {
prepServer := func(fn func(c net.Conn)) (net.Conn, func()) {
server, client := net.Pipe()
go func() {
fmt.Fprintf(server, "%s mox.example\r\n", code)
fn(server)
server.Close()
}()
return client, func() {
@ -361,15 +526,17 @@ func TestQueue(t *testing.T) {
}
}
conn2, cleanup2 := prepServer("220")
conn3, cleanup3 := prepServer("451")
conn2, cleanup2 := prepServer(func(conn net.Conn) { fmt.Fprintf(conn, "220 mox.example\r\n") })
conn3, cleanup3 := prepServer(func(conn net.Conn) { fmt.Fprintf(conn, "451 mox.example\r\n") })
conn4, cleanup4 := prepServer(fakeSMTPSTARTTLSServer)
defer func() {
cleanup2()
cleanup3()
cleanup4()
}()
seq := 0
dial = func(ctx context.Context, dialer contextDialer, timeout time.Duration, addr string, laddr net.Addr) (net.Conn, error) {
smtpclient.DialHook = func(ctx context.Context, dialer smtpclient.Dialer, timeout time.Duration, addr string, laddr net.Addr) (net.Conn, error) {
seq++
switch seq {
default:
@ -378,14 +545,31 @@ func TestQueue(t *testing.T) {
return conn2, nil
case 3:
return conn3, nil
case 4:
return conn4, nil
}
}
defer func() {
smtpclient.DialHook = nil
}()
comm := store.RegisterComm(acc)
defer comm.Unregister()
for i := 1; i < 8; i++ {
go func() { <-deliveryResult }() // Deliver sends here.
if i == 4 {
resolver.AllAuthentic = true
resolver.TLSA = map[string][]adns.TLSA{
"_25._tcp.mox.example.": {
// Non-matching zero CertAssoc, should cause failure.
{Usage: adns.TLSAUsageDANEEE, Selector: adns.TLSASelectorSPKI, MatchType: adns.TLSAMatchTypeSHA256, CertAssoc: make([]byte, sha256.Size)},
},
}
} else {
resolver.AllAuthentic = false
resolver.TLSA = nil
}
deliver(resolver, msg)
err = DB.Get(ctxbg, &msg)
tcheck(t, err, "get msg")
@ -436,10 +620,13 @@ func TestQueueStart(t *testing.T) {
MX: map[string][]*net.MX{"mox.example.": {{Host: "mox.example", Pref: 10}}},
}
dialed := make(chan struct{}, 1)
dial = func(ctx context.Context, dialer contextDialer, timeout time.Duration, addr string, laddr net.Addr) (net.Conn, error) {
smtpclient.DialHook = func(ctx context.Context, dialer smtpclient.Dialer, timeout time.Duration, addr string, laddr net.Addr) (net.Conn, error) {
dialed <- struct{}{}
return nil, fmt.Errorf("failure from test")
}
defer func() {
smtpclient.DialHook = nil
}()
_, cleanup := setup(t)
defer cleanup()
@ -491,139 +678,6 @@ func TestQueueStart(t *testing.T) {
time.Sleep(100 * time.Millisecond) // Racy... we won't get notified when work is done...
}
func TestGatherHosts(t *testing.T) {
mox.Context = ctxbg
// Test basic MX lookup case, but also following CNAME, detecting CNAME loops and
// having a CNAME limit, connecting directly to a host, and domain that does not
// exist or has temporary error.
resolver := dns.MockResolver{
MX: map[string][]*net.MX{
"basic.example.": {{Host: "mail.basic.example.", Pref: 10}},
"multimx.example.": {{Host: "mail1.multimx.example.", Pref: 10}, {Host: "mail2.multimx.example.", Pref: 10}},
"nullmx.example.": {{Host: ".", Pref: 10}},
"temperror-mx.example.": {{Host: "absent.example.", Pref: 10}},
},
A: map[string][]string{
"mail.basic.example": {"10.0.0.1"},
"justhost.example.": {"10.0.0.1"}, // No MX record for domain, only an A record.
"temperror-a.example.": {"10.0.0.1"},
},
AAAA: map[string][]string{
"justhost6.example.": {"2001:db8::1"}, // No MX record for domain, only an AAAA record.
},
CNAME: map[string]string{
"cname.example.": "basic.example.",
"cnameloop.example.": "cnameloop2.example.",
"cnameloop2.example.": "cnameloop.example.",
"danglingcname.example.": "absent.example.", // Points to missing name.
"temperror-cname.example.": "absent.example.",
},
Fail: map[dns.Mockreq]struct{}{
{Type: "mx", Name: "temperror-mx.example."}: {},
{Type: "host", Name: "temperror-a.example."}: {},
{Type: "cname", Name: "temperror-cname.example."}: {},
},
}
for i := 0; i <= 16; i++ {
s := fmt.Sprintf("cnamelimit%d.example.", i)
next := fmt.Sprintf("cnamelimit%d.example.", i+1)
resolver.CNAME[s] = next
}
test := func(ipd dns.IPDomain, expHosts []dns.IPDomain, expDomain dns.Domain, expPerm bool, expErr error) {
t.Helper()
m := Msg{RecipientDomain: ipd}
hosts, ed, perm, err := gatherHosts(resolver, m, 1, xlog)
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
// todo: could also check the individual errors? code currently does not have structured errors.
t.Fatalf("gather hosts: %v", err)
}
if err != nil {
return
}
if !reflect.DeepEqual(hosts, expHosts) || ed != expDomain || perm != expPerm {
t.Fatalf("got hosts %#v, effectiveDomain %#v, permanent %#v, expected %#v %#v %#v", hosts, ed, perm, expHosts, expDomain, expPerm)
}
}
domain := func(s string) dns.Domain {
d, err := dns.ParseDomain(s)
if err != nil {
t.Fatalf("parse domain: %v", err)
}
return d
}
ipdomain := func(s string) dns.IPDomain {
ip := net.ParseIP(s)
if ip != nil {
return dns.IPDomain{IP: ip}
}
d, err := dns.ParseDomain(s)
if err != nil {
t.Fatalf("parse domain %q: %v", s, err)
}
return dns.IPDomain{Domain: d}
}
ipdomains := func(s ...string) (l []dns.IPDomain) {
for _, e := range s {
l = append(l, ipdomain(e))
}
return
}
var zerodom dns.Domain
test(ipdomain("10.0.0.1"), ipdomains("10.0.0.1"), zerodom, false, nil)
test(ipdomain("basic.example"), ipdomains("mail.basic.example"), domain("basic.example"), false, nil) // Basic with simple MX.
test(ipdomain("multimx.example"), ipdomains("mail1.multimx.example", "mail2.multimx.example"), domain("multimx.example"), false, nil) // Basic with simple MX.
test(ipdomain("justhost.example"), ipdomains("justhost.example"), domain("justhost.example"), false, nil) // Only an A record.
test(ipdomain("justhost6.example"), ipdomains("justhost6.example"), domain("justhost6.example"), false, nil) // Only an AAAA record.
test(ipdomain("cname.example"), ipdomains("mail.basic.example"), domain("basic.example"), false, nil) // Follow CNAME.
test(ipdomain("cnamelimit1.example"), nil, zerodom, true, errCNAMELimit)
test(ipdomain("cnameloop.example"), nil, zerodom, true, errCNAMELoop)
test(ipdomain("absent.example"), nil, zerodom, true, errNoRecord)
test(ipdomain("danglingcname.example"), nil, zerodom, true, errNoRecord)
test(ipdomain("nullmx.example"), nil, zerodom, true, errNoMail)
test(ipdomain("temperror-mx.example"), nil, zerodom, false, errDNS)
test(ipdomain("temperror-cname.example"), nil, zerodom, false, errDNS)
test(ipdomain("temperror-a.example"), nil, zerodom, false, errDNS)
}
func TestDialHost(t *testing.T) {
// We mostly want to test that dialing a second time switches to the other address family.
resolver := dns.MockResolver{
A: map[string][]string{
"dualstack.example.": {"10.0.0.1"},
},
AAAA: map[string][]string{
"dualstack.example.": {"2001:db8::1"},
},
}
dial = func(ctx context.Context, dialer contextDialer, timeout time.Duration, addr string, laddr net.Addr) (net.Conn, error) {
return nil, nil // No error, nil connection isn't used.
}
ipdomain := func(s string) dns.IPDomain {
return dns.IPDomain{Domain: dns.Domain{ASCII: s}}
}
m := Msg{DialedIPs: map[string][]net.IP{}}
_, ip, dualstack, err := dialHost(ctxbg, xlog, resolver, nil, ipdomain("dualstack.example"), 25, &m)
if err != nil || ip.String() != "10.0.0.1" || !dualstack {
t.Fatalf("expected err nil, address 10.0.0.1, dualstack true, got %v %v %v", err, ip, dualstack)
}
_, ip, dualstack, err = dialHost(ctxbg, xlog, resolver, nil, ipdomain("dualstack.example"), 25, &m)
if err != nil || ip.String() != "2001:db8::1" || !dualstack {
t.Fatalf("expected err nil, address 2001:db8::1, dualstack true, got %v %v %v", err, ip, dualstack)
}
}
// Just a cert that appears valid.
func fakeCert(t *testing.T, name string, expired bool) tls.Certificate {
notAfter := time.Now()

21
queue/submit.go
View File

@ -24,7 +24,7 @@ import (
// deliver via another SMTP server, e.g. relaying to a smart host, possibly
// with authentication (submission).
func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer contextDialer, m Msg, backoff time.Duration, transportName string, transport *config.TransportSMTP, dialTLS bool, defaultPort int) {
func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, m Msg, backoff time.Duration, transportName string, transport *config.TransportSMTP, dialTLS bool, defaultPort int) {
// todo: configurable timeouts
port := transport.Port
@ -51,10 +51,25 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer cont
qlog.Debug("queue deliversubmit result", mlog.Field("host", transport.DNSHost), mlog.Field("port", port), mlog.Field("attempt", m.Attempts), mlog.Field("permanent", permanent), mlog.Field("secodeopt", secodeOpt), mlog.Field("errmsg", errmsg), mlog.Field("ok", success), mlog.Field("duration", time.Since(start)))
}()
// We don't have to attempt SMTP-DANE for submission, since it only applies to SMTP
// relaying on port 25. ../rfc/7672:1261
// todo: for submission, understand SRV records, and even DANE.
dialctx, dialcancel := context.WithTimeout(context.Background(), 30*time.Second)
defer dialcancel()
if m.DialedIPs == nil {
m.DialedIPs = map[string][]net.IP{}
}
_, _, _, ips, _, err := smtpclient.GatherIPs(dialctx, qlog, resolver, dns.IPDomain{Domain: transport.DNSHost}, m.DialedIPs)
var conn net.Conn
if err == nil {
if m.DialedIPs == nil {
m.DialedIPs = map[string][]net.IP{}
}
conn, _, err = smtpclient.Dial(dialctx, qlog, dialer, dns.IPDomain{Domain: transport.DNSHost}, ips, port, m.DialedIPs)
}
addr := net.JoinHostPort(transport.Host, fmt.Sprintf("%d", port))
conn, _, _, err := dialHost(dialctx, qlog, resolver, dialer, dns.IPDomain{Domain: transport.DNSHost}, port, &m)
var result string
switch {
case err == nil:
@ -103,7 +118,7 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer cont
}
clientctx, clientcancel := context.WithTimeout(context.Background(), 60*time.Second)
defer clientcancel()
client, err := smtpclient.New(clientctx, qlog, conn, tlsMode, mox.Conf.Static.HostnameDomain, transport.DNSHost, auth)
client, err := smtpclient.New(clientctx, qlog, conn, tlsMode, mox.Conf.Static.HostnameDomain, transport.DNSHost, auth, nil, nil, nil)
if err != nil {
smtperr, ok := err.(smtpclient.Error)
var remoteMTA dsn.NameIP