mirror of
https://github.com/mjl-/mox.git
synced 2025-07-12 17:04:39 +03:00
localserve: change queue to deliver to localserve smtp server
instead of skipping on any smtp and delivering messages to accounts. we dial the ip of the smtp listener, which is localhost:1025 by default. the smtp server now uses a mock dns resolver during spf & dkim verification for hosted domains (localhost by default), so they should pass. the advantage is that we get regular full smtp server behaviour for delivering in localserve, including webhooks, and potential first-time sender delays (though this is disabled by default now). incoming deliveries now go through normal address resolution, where before we would always deliver to mox@localhost. we still accept email for unknown recipients to mox@localhost. this will be useful upcoming alias/list functionality. localserve will now generate a dkim key when creating a new config. existing users may wish to reset (remove) their localserve directory, or add a dkim key.
This commit is contained in:
Mechiel Lukkien
103
queue/localserve.go
Normal file
103
queue/localserve.go
Normal file
@ -0,0 +1,103 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/mjl-/mox/dns"
|
||||
"github.com/mjl-/mox/dsn"
|
||||
"github.com/mjl-/mox/mlog"
|
||||
"github.com/mjl-/mox/mox-"
|
||||
"github.com/mjl-/mox/smtpclient"
|
||||
"github.com/mjl-/mox/store"
|
||||
)
|
||||
|
||||
// We won't be dialing remote servers. We just connect the smtp port of the first
|
||||
// ip in the "local" listener, with fallback to localhost:1025 for any destination
|
||||
// address and try to deliver. Our smtpserver uses a mocked dns resolver to give
|
||||
// spf/dkim a chance to pass.
|
||||
func deliverLocalserve(ctx context.Context, log mlog.Log, msgs []*Msg, backoff time.Duration) {
|
||||
m0 := msgs[0]
|
||||
|
||||
addr := "localhost:1025"
|
||||
l, ok := mox.Conf.Static.Listeners["local"]
|
||||
if ok && l.SMTP.Enabled {
|
||||
port := 1025
|
||||
if l.SMTP.Port != 0 {
|
||||
port = l.SMTP.Port
|
||||
}
|
||||
addr = net.JoinHostPort(l.IPs[0], fmt.Sprintf("%d", port))
|
||||
}
|
||||
var d net.Dialer
|
||||
dialctx, dialcancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer dialcancel()
|
||||
conn, err := d.DialContext(dialctx, "tcp", addr)
|
||||
dialcancel()
|
||||
if err != nil {
|
||||
failMsgsDB(log, msgs, m0.DialedIPs, backoff, dsn.NameIP{}, err)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if conn != nil {
|
||||
err = conn.Close()
|
||||
log.Check(err, "closing connection")
|
||||
}
|
||||
}()
|
||||
|
||||
clientctx, clientcancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer clientcancel()
|
||||
localhost := dns.Domain{ASCII: "localhost"}
|
||||
client, err := smtpclient.New(clientctx, log.Logger, conn, smtpclient.TLSOpportunistic, false, localhost, localhost, smtpclient.Opts{})
|
||||
clientcancel()
|
||||
if err != nil {
|
||||
failMsgsDB(log, msgs, m0.DialedIPs, backoff, dsn.NameIP{}, err)
|
||||
return
|
||||
}
|
||||
conn = nil // Will be closed when closing client.
|
||||
defer func() {
|
||||
err := client.Close()
|
||||
log.Check(err, "closing smtp client")
|
||||
}()
|
||||
|
||||
var msgr io.ReadCloser
|
||||
var size int64
|
||||
if len(m0.DSNUTF8) > 0 {
|
||||
msgr = io.NopCloser(bytes.NewReader(m0.DSNUTF8))
|
||||
size = int64(len(m0.DSNUTF8))
|
||||
} else {
|
||||
size = m0.Size
|
||||
p := m0.MessagePath()
|
||||
f, err := os.Open(p)
|
||||
if err != nil {
|
||||
log.Errorx("opening message for delivery", err, slog.String("remote", addr), slog.String("path", p))
|
||||
err = fmt.Errorf("opening message file: %v", err)
|
||||
failMsgsDB(log, msgs, m0.DialedIPs, backoff, dsn.NameIP{}, err)
|
||||
return
|
||||
}
|
||||
msgr = store.FileMsgReader(m0.MsgPrefix, f)
|
||||
defer func() {
|
||||
err := msgr.Close()
|
||||
log.Check(err, "closing message after delivery attempt")
|
||||
}()
|
||||
}
|
||||
|
||||
deliverctx, delivercancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer delivercancel()
|
||||
requireTLS := m0.RequireTLS != nil && *m0.RequireTLS
|
||||
rcpts := make([]string, len(msgs))
|
||||
for i, m := range msgs {
|
||||
rcpts[i] = m.Recipient().String()
|
||||
}
|
||||
rcptErrs, err := client.DeliverMultiple(deliverctx, m0.Sender().String(), rcpts, size, msgr, m0.Has8bit, m0.SMTPUTF8, requireTLS)
|
||||
delivercancel()
|
||||
if err != nil {
|
||||
log.Infox("smtp transaction for delivery failed", err)
|
||||
}
|
||||
processDeliveries(log, m0, msgs, addr, "localhost", backoff, rcptErrs, err)
|
||||
}
|
||||
114
queue/queue.go
114
queue/queue.go
@ -28,12 +28,10 @@ import (
|
||||
"github.com/mjl-/mox/config"
|
||||
"github.com/mjl-/mox/dns"
|
||||
"github.com/mjl-/mox/dsn"
|
||||
"github.com/mjl-/mox/message"
|
||||
"github.com/mjl-/mox/metrics"
|
||||
"github.com/mjl-/mox/mlog"
|
||||
"github.com/mjl-/mox/mox-"
|
||||
"github.com/mjl-/mox/moxio"
|
||||
"github.com/mjl-/mox/publicsuffix"
|
||||
"github.com/mjl-/mox/smtp"
|
||||
"github.com/mjl-/mox/smtpclient"
|
||||
"github.com/mjl-/mox/store"
|
||||
@ -1484,117 +1482,7 @@ func deliver(log mlog.Log, resolver dns.Resolver, m0 Msg) {
|
||||
}
|
||||
|
||||
if Localserve {
|
||||
// We are not actually going to deliver. We'll deliver to the sender account.
|
||||
// Unless recipients match certain special patterns, in which case we can pretend
|
||||
// to cause delivery failures. Useful for testing.
|
||||
|
||||
acc, err := store.OpenAccount(log, m0.SenderAccount)
|
||||
if err != nil {
|
||||
log.Errorx("opening sender account for immediate delivery with localserve, skipping", err)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
err := acc.Close()
|
||||
log.Check(err, "closing account")
|
||||
}()
|
||||
conf, _ := acc.Conf()
|
||||
|
||||
p := m0.MessagePath()
|
||||
msgFile, err := os.Open(p)
|
||||
if err != nil {
|
||||
xerr := fmt.Errorf("open message for delivery: %v", err)
|
||||
failMsgsDB(qlog, msgs, m0.DialedIPs, backoff, dsn.NameIP{}, xerr)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
err := msgFile.Close()
|
||||
qlog.Check(err, "closing message after delivery attempt")
|
||||
}()
|
||||
|
||||
// Parse the message for a From-address, but continue on error.
|
||||
fromAddr, _, _, fromErr := message.From(qlog.Logger, false, store.FileMsgReader(m0.MsgPrefix, msgFile), nil)
|
||||
log.Check(fromErr, "parsing message From header")
|
||||
|
||||
for _, qm := range msgs {
|
||||
code, timeout := mox.LocalserveNeedsError(qm.RecipientLocalpart)
|
||||
if timeout || code != 0 {
|
||||
err := errors.New("simulated error due to localserve mode and special recipient localpart")
|
||||
if timeout {
|
||||
err = fmt.Errorf("%s: timeout", err)
|
||||
} else {
|
||||
err = smtpclient.Error{Permanent: code/100 == 5, Code: code, Err: err}
|
||||
}
|
||||
failMsgsDB(qlog, []*Msg{qm}, m0.DialedIPs, backoff, remoteMTA, err)
|
||||
continue
|
||||
}
|
||||
|
||||
msgFromOrgDomain := publicsuffix.Lookup(ctx, qlog.Logger, fromAddr.Domain)
|
||||
|
||||
dm := store.Message{
|
||||
RemoteIP: "::1",
|
||||
RemoteIPMasked1: "::",
|
||||
RemoteIPMasked2: "::",
|
||||
RemoteIPMasked3: "::",
|
||||
MailFrom: qm.Sender().XString(true),
|
||||
MailFromLocalpart: qm.SenderLocalpart,
|
||||
MailFromDomain: qm.SenderDomainStr,
|
||||
RcptToLocalpart: qm.RecipientLocalpart,
|
||||
RcptToDomain: qm.RecipientDomainStr,
|
||||
MsgFromLocalpart: fromAddr.Localpart,
|
||||
MsgFromDomain: fromAddr.Domain.Name(),
|
||||
MsgFromOrgDomain: msgFromOrgDomain.Name(),
|
||||
EHLOValidated: true,
|
||||
MailFromValidated: true,
|
||||
MsgFromValidated: true,
|
||||
EHLOValidation: store.ValidationPass,
|
||||
MailFromValidation: store.ValidationPass,
|
||||
MsgFromValidation: store.ValidationDMARC,
|
||||
DKIMDomains: []string{"localhost"},
|
||||
ReceivedRequireTLS: qm.RequireTLS != nil && *qm.RequireTLS,
|
||||
Size: qm.Size,
|
||||
MsgPrefix: qm.MsgPrefix,
|
||||
}
|
||||
var err error
|
||||
var mb store.Mailbox
|
||||
acc.WithWLock(func() {
|
||||
dest := conf.Destinations[qm.Recipient().String()]
|
||||
err = acc.DeliverDestination(log, dest, &dm, msgFile)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("delivering message: %v", err)
|
||||
return // Returned again outside WithWLock.
|
||||
}
|
||||
|
||||
mb = store.Mailbox{ID: dm.MailboxID}
|
||||
if err = acc.DB.Get(context.Background(), &mb); err != nil {
|
||||
err = fmt.Errorf("getting mailbox for message after delivery: %v", err)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorx("delivering from queue to original sender account failed, skipping", err)
|
||||
continue
|
||||
}
|
||||
log.Debug("delivered from queue to original sender account")
|
||||
qm.markResult(0, "", "", true)
|
||||
err = DB.Write(context.Background(), func(tx *bstore.Tx) error {
|
||||
return retireMsgs(qlog, tx, webhook.EventDelivered, smtp.C250Completed, "", nil, *qm)
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorx("removing queue message from database after local delivery to sender account", err)
|
||||
} else if err := removeMsgsFS(qlog, *qm); err != nil {
|
||||
log.Errorx("removing queue messages from file system after local delivery to sender account", err)
|
||||
}
|
||||
kick()
|
||||
|
||||
// Process incoming message for incoming webhook.
|
||||
mr := store.FileMsgReader(dm.MsgPrefix, msgFile)
|
||||
part, err := dm.LoadPart(mr)
|
||||
if err != nil {
|
||||
log.Errorx("loading parsed part for evaluating webhook", err)
|
||||
} else {
|
||||
err = Incoming(context.Background(), log, acc, m0.MessageID, dm, part, mb.Name)
|
||||
log.Check(err, "queueing webhook for incoming delivery")
|
||||
}
|
||||
}
|
||||
deliverLocalserve(ctx, qlog, msgs, backoff)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@ -1284,84 +1284,6 @@ func TestQueueStart(t *testing.T) {
|
||||
time.Sleep(100 * time.Millisecond) // Racy... give time to finish.
|
||||
}
|
||||
|
||||
// Localserve should cause deliveries to go to sender account, with failure (DSN)
|
||||
// for recipient addresses that start with "queue" and end with
|
||||
// "temperror"/"permerror"/"timeout".
|
||||
func TestLocalserve(t *testing.T) {
|
||||
Localserve = true
|
||||
defer func() {
|
||||
Localserve = false
|
||||
}()
|
||||
|
||||
path := smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "mox.example"}}}
|
||||
|
||||
testDeliver := func(to smtp.Path, expSuccess bool) {
|
||||
t.Helper()
|
||||
|
||||
_, cleanup := setup(t)
|
||||
defer cleanup()
|
||||
err := Init()
|
||||
tcheck(t, err, "queue init")
|
||||
|
||||
accret, err := store.OpenAccount(pkglog, "retired")
|
||||
tcheck(t, err, "open account")
|
||||
defer func() {
|
||||
err := accret.Close()
|
||||
tcheck(t, err, "closing account")
|
||||
accret.CheckClosed()
|
||||
}()
|
||||
|
||||
mf := prepareFile(t)
|
||||
defer os.Remove(mf.Name())
|
||||
defer mf.Close()
|
||||
|
||||
// Regular message.
|
||||
qm := MakeMsg(path, to, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil, time.Now(), "test")
|
||||
qml := []Msg{qm}
|
||||
err = Add(ctxbg, pkglog, accret.Name, mf, qml...)
|
||||
tcheck(t, err, "add message to queue")
|
||||
qm = qml[0]
|
||||
|
||||
deliver(pkglog, nil, qm)
|
||||
<-deliveryResults
|
||||
// Message should be delivered to account.
|
||||
n, err := bstore.QueryDB[store.Message](ctxbg, accret.DB).Count()
|
||||
tcheck(t, err, "count messages in account")
|
||||
tcompare(t, n, 1)
|
||||
|
||||
n, err = Count(ctxbg)
|
||||
tcheck(t, err, "count message queue")
|
||||
tcompare(t, n, 0)
|
||||
|
||||
_, err = bstore.QueryDB[MsgRetired](ctxbg, DB).Count()
|
||||
tcheck(t, err, "get retired message")
|
||||
|
||||
hl, err := bstore.QueryDB[Hook](ctxbg, DB).List()
|
||||
tcheck(t, err, "get webhooks")
|
||||
if expSuccess {
|
||||
tcompare(t, len(hl), 2)
|
||||
tcompare(t, hl[0].IsIncoming, false)
|
||||
tcompare(t, hl[1].IsIncoming, true)
|
||||
} else {
|
||||
tcompare(t, len(hl), 1)
|
||||
tcompare(t, hl[0].IsIncoming, false)
|
||||
}
|
||||
var out webhook.Outgoing
|
||||
err = json.Unmarshal([]byte(hl[0].Payload), &out)
|
||||
tcheck(t, err, "unmarshal outgoing webhook payload")
|
||||
if expSuccess {
|
||||
tcompare(t, out.Event, webhook.EventDelivered)
|
||||
} else {
|
||||
tcompare(t, out.Event, webhook.EventFailed)
|
||||
}
|
||||
}
|
||||
|
||||
testDeliver(path, true)
|
||||
badpath := path
|
||||
badpath.Localpart = smtp.Localpart("queuepermerror")
|
||||
testDeliver(badpath, false)
|
||||
}
|
||||
|
||||
func TestListFilterSort(t *testing.T) {
|
||||
_, cleanup := setup(t)
|
||||
defer cleanup()
|
||||
|
||||
@ -231,7 +231,14 @@ func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
|
||||
if submiterr != nil {
|
||||
qlog.Infox("smtp transaction for delivery failed", submiterr)
|
||||
}
|
||||
failed = 0 // Reset, we are looking at the SMTP results below.
|
||||
failed, delivered = processDeliveries(qlog, m0, msgs, addr, transport.Host, backoff, rcptErrs, submiterr)
|
||||
}
|
||||
|
||||
// Process failures and successful deliveries, retiring/removing messages from
|
||||
// queue, queueing webhooks.
|
||||
//
|
||||
// Also used by deliverLocalserve.
|
||||
func processDeliveries(qlog mlog.Log, m0 *Msg, msgs []*Msg, remoteAddr string, remoteHost string, backoff time.Duration, rcptErrs []smtpclient.Response, submiterr error) (failed, delivered int) {
|
||||
var delMsgs []Msg
|
||||
for i, m := range msgs {
|
||||
qmlog := qlog.With(
|
||||
@ -246,14 +253,14 @@ func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
|
||||
}
|
||||
if err != nil {
|
||||
smtperr, ok := err.(smtpclient.Error)
|
||||
err = fmt.Errorf("transport %s: submitting message to %s: %w", transportName, addr, err)
|
||||
err = fmt.Errorf("delivering message to %s: %w", remoteAddr, err)
|
||||
var remoteMTA dsn.NameIP
|
||||
if ok {
|
||||
remoteMTA.Name = transport.Host
|
||||
remoteMTA.Name = remoteHost
|
||||
smtperr.Err = err
|
||||
err = smtperr
|
||||
}
|
||||
qmlog.Errorx("submitting message", err, slog.String("remote", addr))
|
||||
qmlog.Errorx("submitting message", err, slog.String("remote", remoteAddr))
|
||||
failMsgsDB(qmlog, []*Msg{m}, m0.DialedIPs, backoff, remoteMTA, err)
|
||||
failed++
|
||||
} else {
|
||||
@ -274,4 +281,5 @@ func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
|
||||
}
|
||||
kick()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user