new feature: when delivering messages from the queue, make it possible to use a "transport"

the default transport is still just "direct delivery", where we connect to the destination domain's MX servers. other transports are: - regular smtp without authentication, this is relaying to a smarthost. - submission with authentication, e.g. to a third party email sending service. - direct delivery, but with with connections going through a socks proxy. this can be helpful if your ip is blocked, you need to get email out, and you have another IP that isn't blocked. keep in mind that for all of the above, appropriate SPF/DKIM settings have to be configured. the "dnscheck" for a domain does a check for any SOCKS IP in the SPF record. SPF for smtp/submission (ranges? includes?) and any DKIM requirements cannot really be checked. which transport is used can be configured through routes. routes can be set on an account, a domain, or globally. the routes are evaluated in that order, with the first match selecting the transport. these routes are evaluated for each delivery attempt. common selection criteria are recipient domain and sender domain, but also which delivery attempt this is. you could configured mox to attempt sending through a 3rd party from the 4th attempt onwards. routes and transports are optional. if no route matches, or an empty/zero transport is selected, normal direct delivery is done. we could already "submit" emails with 3rd party accounts with "sendmail". but we now support more SASL authentication mechanisms with SMTP (not only PLAIN, but also SCRAM-SHA-256, SCRAM-SHA-1 and CRAM-MD5), which sendmail now also supports. sendmail will use the most secure mechanism supported by the server, or the explicitly configured mechanism. for issue #36 by dmikushin. also based on earlier discussion on hackernews.
2025-07-12 17:44:35 +03:00 · 2023-06-16 18:38:28 +02:00
parent 2eecf38842
commit 8096441f67
36 changed files with 3125 additions and 650 deletions
--- a/http/admin.go
+++ b/http/admin.go
@ -339,8 +339,15 @@ func logPanic(ctx context.Context) {
 }

 // return IPs we may be listening on.
-func xlistenIPs(ctx context.Context) []net.IP {
-	ips, err := mox.IPs(ctx)
+func xlistenIPs(ctx context.Context, receiveOnly bool) []net.IP {
+	ips, err := mox.IPs(ctx, receiveOnly)
+	xcheckf(ctx, err, "listing ips")
+	return ips
+}
+
+// return IPs from which we may be sending.
+func xsendingIPs(ctx context.Context) []net.IP {
+	ips, err := mox.IPs(ctx, false)
 	xcheckf(ctx, err, "listing ips")
 	return ips
 }
@ -366,7 +373,7 @@ func checkDomain(ctx context.Context, resolver dns.Resolver, dialer *net.Dialer,
 		panic(&sherpa.Error{Code: "user:notFound", Message: "domain not found"})
 	}

-	listenIPs := xlistenIPs(ctx)
+	listenIPs := xlistenIPs(ctx, true)
 	isListenIP := func(ip net.IP) bool {
 		for _, lip := range listenIPs {
 			if ip.Equal(lip) {
@ -442,7 +449,7 @@ func checkDomain(ctx context.Context, resolver dns.Resolver, dialer *net.Dialer,

 		// For each mox.Conf.SpecifiedSMTPListenIPs, and each address for
 		// mox.Conf.HostnameDomain, check if they resolve back to the host name.
-		var ips []net.IP
+		hostIPs := map[dns.Domain][]net.IP{}
 		ips, err := resolver.LookupIP(ctx, "ip", mox.Conf.Static.HostnameDomain.ASCII+".")
 		if err != nil {
 			addf(&r.IPRev.Errors, "Looking up IPs for hostname: %s", err)
@ -458,32 +465,59 @@ func checkDomain(ctx context.Context, resolver dns.Resolver, dialer *net.Dialer,
 				ips = append(ips, ip)
 			}
 		}
+		hostIPs[mox.Conf.Static.HostnameDomain] = ips
+
+		iplist := func(ips []net.IP) string {
+			var ipstrs []string
+			for _, ip := range ips {
+				ipstrs = append(ipstrs, ip.String())
+			}
+			return strings.Join(ipstrs, ", ")
+		}
+
+		r.IPRev.Hostname = mox.Conf.Static.HostnameDomain
+		r.IPRev.Instructions = []string{
+			fmt.Sprintf("Ensure IPs %s have reverse address %s.", iplist(ips), mox.Conf.Static.HostnameDomain.ASCII),
+		}
+
+		// If we have a socks transport, also check its host and IP.
+		for tname, t := range mox.Conf.Static.Transports {
+			if t.Socks != nil {
+				hostIPs[t.Socks.Hostname] = append(hostIPs[t.Socks.Hostname], t.Socks.IPs...)
+				instr := fmt.Sprintf("For SOCKS transport %s, ensure IPs %s have reverse address %s.", tname, iplist(t.Socks.IPs), t.Socks.Hostname)
+				r.IPRev.Instructions = append(r.IPRev.Instructions, instr)
+			}
+		}

 		type result struct {
+			Host  dns.Domain
 			IP    string
 			Addrs []string
 			Err   error
 		}
 		results := make(chan result)
-		var ipstrs []string
-		for _, ip := range ips {
-			s := ip.String()
-			go func() {
-				addrs, err := resolver.LookupAddr(ctx, s)
-				results <- result{s, addrs, err}
-			}()
-			ipstrs = append(ipstrs, s)
+		n := 0
+		for host, ips := range hostIPs {
+			for _, ip := range ips {
+				n++
+				s := ip.String()
+				host := host
+				go func() {
+					addrs, err := resolver.LookupAddr(ctx, s)
+					results <- result{host, s, addrs, err}
+				}()
+			}
 		}
 		r.IPRev.IPNames = map[string][]string{}
-		for i := 0; i < len(ips); i++ {
+		for i := 0; i < n; i++ {
 			lr := <-results
-			addrs, ip, err := lr.Addrs, lr.IP, lr.Err
+			host, addrs, ip, err := lr.Host, lr.Addrs, lr.IP, lr.Err
 			if err != nil {
-				addf(&r.IPRev.Errors, "Looking up reverse name for %s: %v", ip, err)
+				addf(&r.IPRev.Errors, "Looking up reverse name for %s of %s: %v", ip, host, err)
 				continue
 			}
 			if len(addrs) != 1 {
-				addf(&r.IPRev.Errors, "Expected exactly 1 name for %s, got %d (%v)", ip, len(addrs), addrs)
+				addf(&r.IPRev.Errors, "Expected exactly 1 name for %s of %s, got %d (%v)", ip, host, len(addrs), addrs)
 			}
 			var match bool
 			for i, a := range addrs {
@ -493,20 +527,15 @@ func checkDomain(ctx context.Context, resolver dns.Resolver, dialer *net.Dialer,
 				if err != nil {
 					addf(&r.IPRev.Errors, "Parsing reverse name %q for %s: %v", a, ip, err)
 				}
-				if ad == mox.Conf.Static.HostnameDomain {
+				if ad == host {
 					match = true
 				}
 			}
 			if !match {
-				addf(&r.IPRev.Errors, "Reverse name(s) %s for ip %s do not match hostname %s, which will cause other mail servers to reject incoming messages from this IP.", strings.Join(addrs, ","), ip, mox.Conf.Static.HostnameDomain)
+				addf(&r.IPRev.Errors, "Reverse name(s) %s for ip %s do not match hostname %s, which will cause other mail servers to reject incoming messages from this IP.", strings.Join(addrs, ","), ip, host)
 			}
 			r.IPRev.IPNames[ip] = addrs
 		}
-
-		r.IPRev.Hostname = mox.Conf.Static.HostnameDomain
-		r.IPRev.Instructions = []string{
-			fmt.Sprintf("Ensure IPs %s have reverse address %s.", strings.Join(ipstrs, ", "), mox.Conf.Static.HostnameDomain.ASCII),
-		}
 	}()

 	// MX
@ -648,6 +677,7 @@ func checkDomain(ctx context.Context, resolver dns.Resolver, dialer *net.Dialer,
 	}()

 	// SPF
+	// todo: add warnings if we have Transports with submission? admin should ensure their IPs are in the SPF record. it may be an IP(net), or an include. that means we cannot easily check for it. and should we first check the transport can be used from this domain (or an account that has this domain?). also see DKIM.
 	wg.Add(1)
 	go func() {
 		defer logPanic(ctx)
@ -667,38 +697,51 @@ func checkDomain(ctx context.Context, resolver dns.Resolver, dialer *net.Dialer,
 			spfr := spf.Record{
 				Version: "spf1",
 			}
+
+			checkSPFIP := func(ip net.IP) {
+				mechanism := "ip4"
+				if ip.To4() == nil {
+					mechanism = "ip6"
+				}
+				spfr.Directives = append(spfr.Directives, spf.Directive{Mechanism: mechanism, IP: ip})
+
+				if record == nil {
+					return
+				}
+
+				args := spf.Args{
+					RemoteIP:          ip,
+					MailFromLocalpart: "postmaster",
+					MailFromDomain:    domain,
+					HelloDomain:       dns.IPDomain{Domain: domain},
+					LocalIP:           net.ParseIP("127.0.0.1"),
+					LocalHostname:     dns.Domain{ASCII: "localhost"},
+				}
+				status, mechanism, expl, err := spf.Evaluate(ctx, record, resolver, args)
+				if err != nil {
+					addf(&r.SPF.Errors, "Evaluating IP %q against %s SPF record: %s", ip, kind, err)
+				} else if status != spf.StatusPass {
+					addf(&r.SPF.Errors, "IP %q does not pass %s SPF evaluation, status not \"pass\" but %q (mechanism %q, explanation %q)", ip, kind, status, mechanism, expl)
+				}
+			}
+
 			for _, l := range mox.Conf.Static.Listeners {
 				if !l.SMTP.Enabled || l.IPsNATed {
 					continue
 				}
 				for _, ipstr := range l.IPs {
 					ip := net.ParseIP(ipstr)
-					mechanism := "ip4"
-					if ip.To4() == nil {
-						mechanism = "ip6"
-					}
-					spfr.Directives = append(spfr.Directives, spf.Directive{Mechanism: mechanism, IP: ip})
-
-					if record == nil {
-						continue
-					}
-
-					args := spf.Args{
-						RemoteIP:          ip,
-						MailFromLocalpart: "postmaster",
-						MailFromDomain:    domain,
-						HelloDomain:       dns.IPDomain{Domain: domain},
-						LocalIP:           net.ParseIP("127.0.0.1"),
-						LocalHostname:     dns.Domain{ASCII: "localhost"},
-					}
-					status, mechanism, expl, err := spf.Evaluate(ctx, record, resolver, args)
-					if err != nil {
-						addf(&r.SPF.Errors, "Evaluating IP %q against %s SPF record: %s", ip, kind, err)
-					} else if status != spf.StatusPass {
-						addf(&r.SPF.Errors, "IP %q does not pass %s SPF evaluation, status not \"pass\" but %q (mechanism %q, explanation %q)", ip, kind, status, mechanism, expl)
+					checkSPFIP(ip)
+				}
+			}
+			for _, t := range mox.Conf.Static.Transports {
+				if t.Socks != nil {
+					for _, ip := range t.Socks.IPs {
+						checkSPFIP(ip)
 					}
 				}
 			}
+
 			spfr.Directives = append(spfr.Directives, spf.Directive{Qualifier: "-", Mechanism: "all"})
 			return txt, xrecord, spfr
 		}
@ -722,6 +765,7 @@ func checkDomain(ctx context.Context, resolver dns.Resolver, dialer *net.Dialer,
 	}()

 	// DKIM
+	// todo: add warnings if we have Transports with submission? admin should ensure DKIM records exist. we cannot easily check if they actually exist though. and should we first check the transport can be used from this domain (or an account that has this domain?). also see SPF.
 	wg.Add(1)
 	go func() {
 		defer logPanic(ctx)
@ -1392,7 +1436,7 @@ func dnsblsStatus(ctx context.Context, resolver dns.Resolver) map[string]map[str
 	}

 	r := map[string]map[string]string{}
-	for _, ip := range xlistenIPs(ctx) {
+	for _, ip := range xsendingIPs(ctx) {
 		if ip.IsLoopback() || ip.IsPrivate() {
 			continue
 		}
@ -1517,9 +1561,10 @@ func (Admin) QueueSize(ctx context.Context) int {
 	return n
 }

-// QueueKick initiates delivery of a message from the queue.
-func (Admin) QueueKick(ctx context.Context, id int64) {
-	n, err := queue.Kick(ctx, id, "", "")
+// QueueKick initiates delivery of a message from the queue and sets the transport
+// to use for delivery.
+func (Admin) QueueKick(ctx context.Context, id int64, transport string) {
+	n, err := queue.Kick(ctx, id, "", "", &transport)
 	if err == nil && n == 0 {
 		err = errors.New("message not found")
 	}
@ -1632,3 +1677,8 @@ func (Admin) WebserverConfigSave(ctx context.Context, oldConf, newConf Webserver
 	savedConf.WebDomainRedirects = nil
 	return savedConf
 }
+
+// Transports returns the configured transports, for sending email.
+func (Admin) Transports(ctx context.Context) map[string]config.Transport {
+	return mox.Conf.Static.Transports
+}
--- a/http/admin.html
+++ b/http/admin.html
@ -195,9 +195,9 @@ const formatSize = n => {

 const index = async () => {
 	const [domains, queueSize, checkUpdatesEnabled] = await Promise.all([
-		await api.Domains(),
-		await api.QueueSize(),
-		await api.CheckUpdatesEnabled(),
+		api.Domains(),
+		api.QueueSize(),
+		api.CheckUpdatesEnabled(),
 	])

 	let fieldset, domain, account, localpart
@ -265,6 +265,7 @@ const index = async () => {
 		dom.div(dom.a('MTA-STS policies', attr({href: '#mtasts'}))),
 		// todo: outgoing DMARC findings
 		// todo: outgoing TLSRPT findings
+		// todo: routing, globally, per domain and per account
 		dom.br(),
 		dom.h2('DNS blocklist status'),
 		dom.div(dom.a('DNSBL status', attr({href: '#dnsbl'}))),
@ -1525,9 +1526,13 @@ const dnsbl = async () => {
 }

 const queueList = async () => {
-	const msgs = await api.QueueList()
+	const [msgs, transports] = await Promise.all([
+		api.QueueList(),
+		api.Transports(),
+	])

 	const nowSecs = new Date().getTime()/1000
+	let transport

 	const page = document.getElementById('page')
 	dom._kids(page,
@ -1550,7 +1555,8 @@ const queueList = async () => {
 						dom.th('Next attempt'),
 						dom.th('Last attempt'),
 						dom.th('Last error'),
-						dom.th('Action'),
+						dom.th('Transport/Retry'),
+						dom.th('Remove'),
 					),
 				),
 				dom.tbody(
@ -1565,11 +1571,17 @@ const queueList = async () => {
 						dom.td(m.LastAttempt ? age(new Date(m.LastAttempt), false, nowSecs) : '-'),
 						dom.td(m.LastError || '-'),
 						dom.td(
-							dom.button('Try now', async function click(e) {
+							transport=dom.select(
+								attr({title: 'Transport to use for delivery attempts. The default is direct delivery, connecting to the MX hosts of the domain.'}),
+								dom.option('(default)', attr({value: ''})),
+								Object.keys(transports).sort().map(t => dom.option(t, m.Transport === t ? attr({checked: ''}) : [])),
+							),
+							' ',
+							dom.button('Retry now', async function click(e) {
 								e.preventDefault()
 								try {
 									e.target.disabled = true
-									await api.QueueKick(m.ID)
+									await api.QueueKick(m.ID, transport.value)
 								} catch (err) {
 									console.log({err})
 									window.alert('Error: ' + err.message)
@ -1579,7 +1591,8 @@ const queueList = async () => {
 								}
 								window.location.reload() // todo: only refresh the list
 							}),
-							' ',
+						),
+						dom.td(
 							dom.button('Remove', async function click(e) {
 								e.preventDefault()
 								if (!window.confirm('Are you sure you want to remove this message? It will be removed completely.')) {
--- a/http/adminapi.json
+++ b/http/adminapi.json
@ -592,13 +592,19 @@
 		},
 		{
 			"Name": "QueueKick",
-			"Docs": "QueueKick initiates delivery of a message from the queue.",
+			"Docs": "QueueKick initiates delivery of a message from the queue and sets the transport\nto use for delivery.",
 			"Params": [
 				{
 					"Name": "id",
 					"Typewords": [
 						"int64"
 					]
+				},
+				{
+					"Name": "transport",
+					"Typewords": [
+						"string"
+					]
 				}
 			],
 			"Returns": []
@ -713,6 +719,20 @@
 					]
 				}
 			]
+		},
+		{
+			"Name": "Transports",
+			"Docs": "Transports returns the configured transports, for sending email.",
+			"Params": [],
+			"Returns": [
+				{
+					"Name": "r0",
+					"Typewords": [
+						"{}",
+						"Transport"
+					]
+				}
+			]
 		}
 	],
 	"Sections": [],
@ -2826,7 +2846,7 @@
 				},
 				{
 					"Name": "SenderAccount",
-					"Docs": "Failures are delivered back to this local account.",
+					"Docs": "Failures are delivered back to this local account. Also used for routing.",
 					"Typewords": [
 						"string"
 					]
@ -2940,6 +2960,13 @@
 						"[]",
 						"uint8"
 					]
+				},
+				{
+					"Name": "Transport",
+					"Docs": "If non-empty, the transport to use for this message. Can be set through cli or admin interface. If empty (the default for a submitted message), regular routing rules apply.",
+					"Typewords": [
+						"string"
+					]
 				}
 			]
 		},
@ -3170,6 +3197,142 @@
 					]
 				}
 			]
+		},
+		{
+			"Name": "Transport",
+			"Docs": "Transport is a method to delivery a message. At most one of the fields can\nbe non-nil. The non-nil field represents the type of transport. For a\ntransport with all fields nil, regular email delivery is done.",
+			"Fields": [
+				{
+					"Name": "Submissions",
+					"Docs": "",
+					"Typewords": [
+						"nullable",
+						"TransportSMTP"
+					]
+				},
+				{
+					"Name": "Submission",
+					"Docs": "",
+					"Typewords": [
+						"nullable",
+						"TransportSMTP"
+					]
+				},
+				{
+					"Name": "SMTP",
+					"Docs": "",
+					"Typewords": [
+						"nullable",
+						"TransportSMTP"
+					]
+				},
+				{
+					"Name": "Socks",
+					"Docs": "",
+					"Typewords": [
+						"nullable",
+						"TransportSocks"
+					]
+				}
+			]
+		},
+		{
+			"Name": "TransportSMTP",
+			"Docs": "TransportSMTP delivers messages by \"submission\" (SMTP, typically\nauthenticated) to the queue of a remote host (smarthost), or by relaying\n(SMTP, typically unauthenticated).",
+			"Fields": [
+				{
+					"Name": "Host",
+					"Docs": "",
+					"Typewords": [
+						"string"
+					]
+				},
+				{
+					"Name": "Port",
+					"Docs": "",
+					"Typewords": [
+						"int32"
+					]
+				},
+				{
+					"Name": "STARTTLSInsecureSkipVerify",
+					"Docs": "",
+					"Typewords": [
+						"bool"
+					]
+				},
+				{
+					"Name": "NoSTARTTLS",
+					"Docs": "",
+					"Typewords": [
+						"bool"
+					]
+				},
+				{
+					"Name": "Auth",
+					"Docs": "",
+					"Typewords": [
+						"nullable",
+						"SMTPAuth"
+					]
+				}
+			]
+		},
+		{
+			"Name": "SMTPAuth",
+			"Docs": "SMTPAuth hold authentication credentials used when delivering messages\nthrough a smarthost.",
+			"Fields": [
+				{
+					"Name": "Username",
+					"Docs": "",
+					"Typewords": [
+						"string"
+					]
+				},
+				{
+					"Name": "Password",
+					"Docs": "",
+					"Typewords": [
+						"string"
+					]
+				},
+				{
+					"Name": "Mechanisms",
+					"Docs": "",
+					"Typewords": [
+						"[]",
+						"string"
+					]
+				}
+			]
+		},
+		{
+			"Name": "TransportSocks",
+			"Docs": "",
+			"Fields": [
+				{
+					"Name": "Address",
+					"Docs": "",
+					"Typewords": [
+						"string"
+					]
+				},
+				{
+					"Name": "RemoteIPs",
+					"Docs": "",
+					"Typewords": [
+						"[]",
+						"string"
+					]
+				},
+				{
+					"Name": "RemoteHostname",
+					"Docs": "",
+					"Typewords": [
+						"string"
+					]
+				}
+			]
 		}
 	],
 	"Ints": [],