infrastructure/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2025-07-12 12:24:38 +03:00
Code Issues Packages Projects Releases Wiki Activity

add a webapi and webhooks for a simple http/json-based api

Browse Source 
for applications to compose/send messages, receive delivery feedback, and
maintain suppression lists.

this is an alternative to applications using a library to compose messages,
submitting those messages using smtp, and monitoring a mailbox with imap for
DSNs, which can be processed into the equivalent of suppression lists. but you
need to know about all these standards/protocols and find libraries. by using
the webapi & webhooks, you just need a http & json library.

unfortunately, there is no standard for these kinds of api, so mox has made up
yet another one...

matching incoming DSNs about deliveries to original outgoing messages requires
keeping history of "retired" messages (delivered from the queue, either
successfully or failed). this can be enabled per account. history is also
useful for debugging deliveries. we now also keep history of each delivery
attempt, accessible while still in the queue, and kept when a message is
retired. the queue webadmin pages now also have pagination, to show potentially
large history.

a queue of webhook calls is now managed too. failures are retried similar to
message deliveries. webhooks can also be saved to the retired list after
completing. also configurable per account.

messages can be sent with a "unique smtp mail from" address. this can only be
used if the domain is configured with a localpart catchall separator such as
"+". when enabled, a queued message gets assigned a random "fromid", which is
added after the separator when sending. when DSNs are returned, they can be
related to previously sent messages based on this fromid. in the future, we can
implement matching on the "envid" used in the smtp dsn extension, or on the
"message-id" of the message. using a fromid can be triggered by authenticating
with a login email address that is configured as enabling fromid.

suppression lists are automatically managed per account. if a delivery attempt
results in certain smtp errors, the destination address is added to the
suppression list. future messages queued for that recipient will immediately
fail without a delivery attempt. suppression lists protect your mail server
reputation.

submitted messages can carry "extra" data through the queue and webhooks for
outgoing deliveries. through webapi as a json object, through smtp submission
as message headers of the form "x-mox-extra-<key>: value".

to make it easy to test webapi/webhooks locally, the "localserve" mode actually
puts messages in the queue. when it's time to deliver, it still won't do a full
delivery attempt, but just delivers to the sender account. unless the recipient
address has a special form, simulating a failure to deliver.

admins now have more control over the queue. "hold rules" can be added to mark
newly queued messages as "on hold", pausing delivery. rules can be about
certain sender or recipient domains/addresses, or apply to all messages pausing
the entire queue. also useful for (local) testing.

new config options have been introduced. they are editable through the admin
and/or account web interfaces.

the webapi http endpoints are enabled for newly generated configs with the
quickstart, and in localserve. existing configurations must explicitly enable
the webapi in mox.conf.

gopherwatch.org was created to dogfood this code. it initially used just the
compose/smtpclient/imapclient mox packages to send messages and process
delivery feedback. it will get a config option to use the mox webapi/webhooks
instead. the gopherwatch code to use webapi/webhook is smaller and simpler, and
developing that shaped development of the mox webapi/webhooks.

for issue #31 by cuu508
This commit is contained in:
Mechiel Lukkien
2024-04-15 21:49:02 +02:00
parent 8bec5ef7d4
commit 09fcc49223
87 changed files with 15556 additions and 1306 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
smtpserver/server.go
View File

@ -6,6 +6,7 @@ import (
"bytes"
"context"
"crypto/md5"
cryptorand "crypto/rand"
"crypto/rsa"
"crypto/sha1"
"crypto/sha256"
@ -21,8 +22,8 @@ import (
"net/textproto"
"os"
"runtime/debug"
"slices"
"sort"
"strconv"
"strings"
"sync"
"time"
@ -150,7 +151,7 @@ var (
"reason",
},
)
// Similar between ../webmail/webmail.go:/metricSubmission and ../smtpserver/server.go:/metricSubmission
// Similar between ../webmail/webmail.go:/metricSubmission and ../smtpserver/server.go:/metricSubmission and ../webapisrv/server.go:/metricSubmission
metricSubmission = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "mox_smtpserver_submission_total",
@ -1944,7 +1945,7 @@ func hasTLSRequiredNo(h textproto.MIMEHeader) bool {
// submit is used for mail from authenticated users that we will try to deliver.
func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWriter *message.Writer, dataFile *os.File, part *message.Part) {
// Similar between ../smtpserver/server.go:/submit\( and ../webmail/webmail.go:/MessageSubmit\(
// Similar between ../smtpserver/server.go:/submit\( and ../webmail/api.go:/MessageSubmit\( and ../webapisrv/server.go:/Send\(
var msgPrefix []byte
@ -2017,6 +2018,26 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
})
xcheckf(err, "read-only transaction")
// We gather any X-Mox-Extra-* headers into the "extra" data during queueing, which
// will make it into any webhook we deliver.
// todo: remove the X-Mox-Extra-* headers from the message. we don't currently rewrite the message...
// todo: should we not canonicalize keys?
var extra map[string]string
for k, vl := range header {
if !strings.HasPrefix(k, "X-Mox-Extra-") {
continue
}
if extra == nil {
extra = map[string]string{}
}
xk := k[len("X-Mox-Extra-"):]
// We don't allow duplicate keys.
if _, ok := extra[xk]; ok || len(vl) > 1 {
xsmtpUserErrorf(smtp.C554TransactionFailed, smtp.SeMsg6Other0, "duplicate x-mox-extra- key %q", xk)
}
extra[xk] = vl[len(vl)-1]
}
// todo future: in a pedantic mode, we can parse the headers, and return an error if rcpt is only in To or Cc header, and not in the non-empty Bcc header. indicates a client that doesn't blind those bcc's.
// Add DKIM signatures.
@ -2054,13 +2075,23 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
msgPrefix = append(msgPrefix, []byte(authResults.Header())...)
// We always deliver through the queue. It would be more efficient to deliver
// directly, but we don't want to circumvent all the anti-spam measures. Accounts
// on a single mox instance should be allowed to block each other.
// directly for local accounts, but we don't want to circumvent all the anti-spam
// measures. Accounts on a single mox instance should be allowed to block each
// other.
accConf, _ := c.account.Conf()
loginAddr, err := smtp.ParseAddress(c.username)
xcheckf(err, "parsing login address")
useFromID := slices.Contains(accConf.ParsedFromIDLoginAddresses, loginAddr)
var localpartBase string
if useFromID {
localpartBase = strings.SplitN(string(c.mailFrom.Localpart), confDom.LocalpartCatchallSeparator, 2)[0]
}
now := time.Now()
qml := make([]queue.Msg, len(c.recipients))
for i, rcptAcc := range c.recipients {
if Localserve {
code, timeout := localserveNeedsError(rcptAcc.rcptTo.Localpart)
code, timeout := mox.LocalserveNeedsError(rcptAcc.rcptTo.Localpart)
if timeout {
c.log.Info("timing out submission due to special localpart")
mox.Sleep(mox.Context, time.Hour)
@ -2071,6 +2102,13 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
}
}
fp := *c.mailFrom
var fromID string
if useFromID {
fromID = xrandomID(16)
fp.Localpart = smtp.Localpart(localpartBase + confDom.LocalpartCatchallSeparator + fromID)
}
// For multiple recipients, we don't make each message prefix unique, leaving out
// the "for" clause in the Received header. This allows the queue to deliver the
// messages in a single smtp transaction.
@ -2080,11 +2118,13 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
}
xmsgPrefix := append([]byte(recvHdrFor(rcptTo)), msgPrefix...)
msgSize := int64(len(xmsgPrefix)) + msgWriter.Size
qm := queue.MakeMsg(*c.mailFrom, rcptAcc.rcptTo, msgWriter.Has8bit, c.msgsmtputf8, msgSize, messageID, xmsgPrefix, c.requireTLS, now)
qm := queue.MakeMsg(fp, rcptAcc.rcptTo, msgWriter.Has8bit, c.msgsmtputf8, msgSize, messageID, xmsgPrefix, c.requireTLS, now, header.Get("Subject"))
if !c.futureRelease.IsZero() {
qm.NextAttempt = c.futureRelease
qm.FutureReleaseRequest = c.futureReleaseRequest
}
qm.FromID = fromID
qm.Extra = extra
qml[i] = qm
}
@ -2124,6 +2164,20 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
c.writecodeline(smtp.C250Completed, smtp.SeMailbox2Other0, "it is done", nil)
}
func xrandomID(n int) string {
return base64.RawURLEncoding.EncodeToString(xrandom(n))
}
func xrandom(n int) []byte {
buf := make([]byte, n)
x, err := cryptorand.Read(buf)
xcheckf(err, "read random")
if x != n {
xcheckf(errors.New("short random read"), "read random")
}
return buf
}
func ipmasked(ip net.IP) (string, string, string) {
if ip.To4() != nil {
m1 := ip.String()
@ -2137,31 +2191,8 @@ func ipmasked(ip net.IP) (string, string, string) {
return m1, m2, m3
}
func localserveNeedsError(lp smtp.Localpart) (code int, timeout bool) {
s := string(lp)
if strings.HasSuffix(s, "temperror") {
return smtp.C451LocalErr, false
} else if strings.HasSuffix(s, "permerror") {
return smtp.C550MailboxUnavail, false
} else if strings.HasSuffix(s, "timeout") {
return 0, true
}
if len(s) < 3 {
return 0, false
}
s = s[len(s)-3:]
v, err := strconv.ParseInt(s, 10, 32)
if err != nil {
return 0, false
}
if v < 400 || v > 600 {
return 0, false
}
return int(v), false
}
func (c *conn) xlocalserveError(lp smtp.Localpart) {
code, timeout := localserveNeedsError(lp)
code, timeout := mox.LocalserveNeedsError(lp)
if timeout {
c.log.Info("timing out due to special localpart")
mox.Sleep(mox.Context, time.Hour)
@ -2178,7 +2209,16 @@ func (c *conn) xlocalserveError(lp smtp.Localpart) {
func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgWriter *message.Writer, iprevStatus iprev.Status, iprevAuthentic bool, dataFile *os.File) {
// todo: in decision making process, if we run into (some) temporary errors, attempt to continue. if we decide to accept, all good. if we decide to reject, we'll make it a temporary reject.
msgFrom, envelope, headers, err := message.From(c.log.Logger, false, dataFile, nil)
var msgFrom smtp.Address
var envelope *message.Envelope
var headers textproto.MIMEHeader
var isDSN bool
part, err := message.Parse(c.log.Logger, false, dataFile)
if err == nil {
// todo: is it enough to check only the the content-type header? in other places we look at the content-types of the parts before considering a message a dsn. should we change other places to this simpler check?
isDSN = part.MediaType == "MULTIPART" && part.MediaSubType == "REPORT" && strings.EqualFold(part.ContentTypeParams["report-type"], "delivery-status")
msgFrom, envelope, headers, err = message.From(c.log.Logger, false, dataFile, &part)
}
if err != nil {
c.log.Infox("parsing message for From address", err)
}
@ -2676,6 +2716,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
MailFromValidation: mailFromValidation,
MsgFromValidation: msgFromValidation,
DKIMDomains: verifiedDKIMDomains,
DSN: isDSN,
Size: msgWriter.Size,
}
if c.tls {
@ -2960,8 +3001,8 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
}
}
if Localserve {
code, timeout := localserveNeedsError(rcptAcc.rcptTo.Localpart)
if Localserve && !strings.HasPrefix(string(rcptAcc.rcptTo.Localpart), "queue") {
code, timeout := mox.LocalserveNeedsError(rcptAcc.rcptTo.Localpart)
if timeout {
log.Info("timing out due to special localpart")
mox.Sleep(mox.Context, time.Hour)
@ -2972,6 +3013,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
addError(rcptAcc, code, smtp.SeOther00, false, fmt.Sprintf("failure with code %d due to special localpart", code))
}
}
var delivered bool
acc.WithWLock(func() {
if err := acc.DeliverMailbox(log, a.mailbox, &m, dataFile); err != nil {
log.Errorx("delivering", err)
@ -2983,6 +3025,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
}
return
}
delivered = true
metricDelivery.WithLabelValues("delivered", a.reason).Inc()
log.Info("incoming message delivered", slog.String("reason", a.reason), slog.Any("msgfrom", msgFrom))
@ -2994,6 +3037,18 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
}
})
// Pass delivered messages to queue for DSN processing and/or hooks.
if delivered {
mr := store.FileMsgReader(m.MsgPrefix, dataFile)
part, err := m.LoadPart(mr)
if err != nil {
log.Errorx("loading parsed part for evaluating webhook", err)
} else {
err = queue.Incoming(context.Background(), log, acc, messageID, m, part, a.mailbox)
log.Check(err, "queueing webhook for incoming delivery")
}
}
err = acc.Close()
log.Check(err, "closing account after delivering")
acc = nil