mirror of
https://github.com/mjl-/mox.git
synced 2025-07-10 09:54:40 +03:00
add a webapi and webhooks for a simple http/json-based api
for applications to compose/send messages, receive delivery feedback, and maintain suppression lists. this is an alternative to applications using a library to compose messages, submitting those messages using smtp, and monitoring a mailbox with imap for DSNs, which can be processed into the equivalent of suppression lists. but you need to know about all these standards/protocols and find libraries. by using the webapi & webhooks, you just need a http & json library. unfortunately, there is no standard for these kinds of api, so mox has made up yet another one... matching incoming DSNs about deliveries to original outgoing messages requires keeping history of "retired" messages (delivered from the queue, either successfully or failed). this can be enabled per account. history is also useful for debugging deliveries. we now also keep history of each delivery attempt, accessible while still in the queue, and kept when a message is retired. the queue webadmin pages now also have pagination, to show potentially large history. a queue of webhook calls is now managed too. failures are retried similar to message deliveries. webhooks can also be saved to the retired list after completing. also configurable per account. messages can be sent with a "unique smtp mail from" address. this can only be used if the domain is configured with a localpart catchall separator such as "+". when enabled, a queued message gets assigned a random "fromid", which is added after the separator when sending. when DSNs are returned, they can be related to previously sent messages based on this fromid. in the future, we can implement matching on the "envid" used in the smtp dsn extension, or on the "message-id" of the message. using a fromid can be triggered by authenticating with a login email address that is configured as enabling fromid. suppression lists are automatically managed per account. if a delivery attempt results in certain smtp errors, the destination address is added to the suppression list. future messages queued for that recipient will immediately fail without a delivery attempt. suppression lists protect your mail server reputation. submitted messages can carry "extra" data through the queue and webhooks for outgoing deliveries. through webapi as a json object, through smtp submission as message headers of the form "x-mox-extra-<key>: value". to make it easy to test webapi/webhooks locally, the "localserve" mode actually puts messages in the queue. when it's time to deliver, it still won't do a full delivery attempt, but just delivers to the sender account. unless the recipient address has a special form, simulating a failure to deliver. admins now have more control over the queue. "hold rules" can be added to mark newly queued messages as "on hold", pausing delivery. rules can be about certain sender or recipient domains/addresses, or apply to all messages pausing the entire queue. also useful for (local) testing. new config options have been introduced. they are editable through the admin and/or account web interfaces. the webapi http endpoints are enabled for newly generated configs with the quickstart, and in localserve. existing configurations must explicitly enable the webapi in mox.conf. gopherwatch.org was created to dogfood this code. it initially used just the compose/smtpclient/imapclient mox packages to send messages and process delivery feedback. it will get a config option to use the mox webapi/webhooks instead. the gopherwatch code to use webapi/webhook is smaller and simpler, and developing that shaped development of the mox webapi/webhooks. for issue #31 by cuu508
This commit is contained in:
Mechiel Lukkien
367
ctl.go
367
ctl.go
@ -4,6 +4,7 @@ import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@ -27,6 +28,7 @@ import (
|
||||
"github.com/mjl-/mox/queue"
|
||||
"github.com/mjl-/mox/smtp"
|
||||
"github.com/mjl-/mox/store"
|
||||
"github.com/mjl-/mox/webapi"
|
||||
)
|
||||
|
||||
// ctl represents a connection to the ctl unix domain socket of a running mox instance.
|
||||
@ -294,12 +296,11 @@ func servectl(ctx context.Context, log mlog.Log, conn net.Conn, shutdown func())
|
||||
}
|
||||
}
|
||||
|
||||
func xparseFilters(ctl *ctl, s string) (f queue.Filter) {
|
||||
func xparseJSON(ctl *ctl, s string, v any) {
|
||||
dec := json.NewDecoder(strings.NewReader(s))
|
||||
dec.DisallowUnknownFields()
|
||||
err := dec.Decode(&f)
|
||||
ctl.xcheck(err, "parsing filters")
|
||||
return f
|
||||
err := dec.Decode(v)
|
||||
ctl.xcheck(err, "parsing from ctl as json")
|
||||
}
|
||||
|
||||
func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
|
||||
@ -447,14 +448,17 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
|
||||
|
||||
case "queuelist":
|
||||
/* protocol:
|
||||
> "queue"
|
||||
> queuefilters as json
|
||||
> "queuelist"
|
||||
> filters as json
|
||||
> sort as json
|
||||
< "ok"
|
||||
< stream
|
||||
*/
|
||||
fs := ctl.xread()
|
||||
f := xparseFilters(ctl, fs)
|
||||
qmsgs, err := queue.List(ctx, f)
|
||||
var f queue.Filter
|
||||
xparseJSON(ctl, ctl.xread(), &f)
|
||||
var s queue.Sort
|
||||
xparseJSON(ctl, ctl.xread(), &s)
|
||||
qmsgs, err := queue.List(ctx, f, s)
|
||||
ctl.xcheck(err, "listing queue")
|
||||
ctl.xwriteok()
|
||||
|
||||
@ -465,7 +469,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
|
||||
if qm.LastAttempt != nil {
|
||||
lastAttempt = time.Since(*qm.LastAttempt).Round(time.Second).String()
|
||||
}
|
||||
fmt.Fprintf(xw, "%5d %s from:%s to:%s next %s last %s error %q\n", qm.ID, qm.Queued.Format(time.RFC3339), qm.Sender().LogString(), qm.Recipient().LogString(), -time.Since(qm.NextAttempt).Round(time.Second), lastAttempt, qm.LastError)
|
||||
fmt.Fprintf(xw, "%5d %s from:%s to:%s next %s last %s error %q\n", qm.ID, qm.Queued.Format(time.RFC3339), qm.Sender().LogString(), qm.Recipient().LogString(), -time.Since(qm.NextAttempt).Round(time.Second), lastAttempt, qm.LastResult().Error)
|
||||
}
|
||||
if len(qmsgs) == 0 {
|
||||
fmt.Fprint(xw, "(none)\n")
|
||||
@ -481,8 +485,8 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
|
||||
< count
|
||||
*/
|
||||
|
||||
fs := ctl.xread()
|
||||
f := xparseFilters(ctl, fs)
|
||||
var f queue.Filter
|
||||
xparseJSON(ctl, ctl.xread(), &f)
|
||||
hold := ctl.xread() == "true"
|
||||
count, err := queue.HoldSet(ctx, f, hold)
|
||||
ctl.xcheck(err, "setting on hold status for messages")
|
||||
@ -499,8 +503,8 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
|
||||
< count
|
||||
*/
|
||||
|
||||
fs := ctl.xread()
|
||||
f := xparseFilters(ctl, fs)
|
||||
var f queue.Filter
|
||||
xparseJSON(ctl, ctl.xread(), &f)
|
||||
relnow := ctl.xread()
|
||||
d, err := time.ParseDuration(ctl.xread())
|
||||
ctl.xcheck(err, "parsing duration for next delivery attempt")
|
||||
@ -523,8 +527,8 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
|
||||
< count
|
||||
*/
|
||||
|
||||
fs := ctl.xread()
|
||||
f := xparseFilters(ctl, fs)
|
||||
var f queue.Filter
|
||||
xparseJSON(ctl, ctl.xread(), &f)
|
||||
transport := ctl.xread()
|
||||
count, err := queue.TransportSet(ctx, f, transport)
|
||||
ctl.xcheck(err, "adding to next delivery attempts in queue")
|
||||
@ -540,8 +544,8 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
|
||||
< count
|
||||
*/
|
||||
|
||||
fs := ctl.xread()
|
||||
f := xparseFilters(ctl, fs)
|
||||
var f queue.Filter
|
||||
xparseJSON(ctl, ctl.xread(), &f)
|
||||
reqtls := ctl.xread()
|
||||
var req *bool
|
||||
switch reqtls {
|
||||
@ -568,8 +572,8 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
|
||||
< count
|
||||
*/
|
||||
|
||||
fs := ctl.xread()
|
||||
f := xparseFilters(ctl, fs)
|
||||
var f queue.Filter
|
||||
xparseJSON(ctl, ctl.xread(), &f)
|
||||
count, err := queue.Fail(ctx, log, f)
|
||||
ctl.xcheck(err, "marking messages from queue as failed")
|
||||
ctl.xwriteok()
|
||||
@ -583,8 +587,8 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
|
||||
< count
|
||||
*/
|
||||
|
||||
fs := ctl.xread()
|
||||
f := xparseFilters(ctl, fs)
|
||||
var f queue.Filter
|
||||
xparseJSON(ctl, ctl.xread(), &f)
|
||||
count, err := queue.Drop(ctx, log, f)
|
||||
ctl.xcheck(err, "dropping messages from queue")
|
||||
ctl.xwriteok()
|
||||
@ -612,6 +616,325 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
|
||||
ctl.xwriteok()
|
||||
ctl.xstreamfrom(mr)
|
||||
|
||||
case "queueretiredlist":
|
||||
/* protocol:
|
||||
> "queueretiredlist"
|
||||
> filters as json
|
||||
> sort as json
|
||||
< "ok"
|
||||
< stream
|
||||
*/
|
||||
var f queue.RetiredFilter
|
||||
xparseJSON(ctl, ctl.xread(), &f)
|
||||
var s queue.RetiredSort
|
||||
xparseJSON(ctl, ctl.xread(), &s)
|
||||
qmsgs, err := queue.RetiredList(ctx, f, s)
|
||||
ctl.xcheck(err, "listing retired queue")
|
||||
ctl.xwriteok()
|
||||
|
||||
xw := ctl.writer()
|
||||
fmt.Fprintln(xw, "retired messages:")
|
||||
for _, qm := range qmsgs {
|
||||
var lastAttempt string
|
||||
if qm.LastAttempt != nil {
|
||||
lastAttempt = time.Since(*qm.LastAttempt).Round(time.Second).String()
|
||||
}
|
||||
result := "failure"
|
||||
if qm.Success {
|
||||
result = "success"
|
||||
}
|
||||
sender, err := qm.Sender()
|
||||
xcheckf(err, "parsing sender")
|
||||
fmt.Fprintf(xw, "%5d %s %s from:%s to:%s last %s error %q\n", qm.ID, qm.Queued.Format(time.RFC3339), result, sender.LogString(), qm.Recipient().LogString(), lastAttempt, qm.LastResult().Error)
|
||||
}
|
||||
if len(qmsgs) == 0 {
|
||||
fmt.Fprint(xw, "(none)\n")
|
||||
}
|
||||
xw.xclose()
|
||||
|
||||
case "queueretiredprint":
|
||||
/* protocol:
|
||||
> "queueretiredprint"
|
||||
> id
|
||||
< "ok"
|
||||
< stream
|
||||
*/
|
||||
idstr := ctl.xread()
|
||||
id, err := strconv.ParseInt(idstr, 10, 64)
|
||||
if err != nil {
|
||||
ctl.xcheck(err, "parsing id")
|
||||
}
|
||||
l, err := queue.RetiredList(ctx, queue.RetiredFilter{IDs: []int64{id}}, queue.RetiredSort{})
|
||||
ctl.xcheck(err, "getting retired messages")
|
||||
if len(l) == 0 {
|
||||
ctl.xcheck(errors.New("not found"), "getting retired message")
|
||||
}
|
||||
m := l[0]
|
||||
ctl.xwriteok()
|
||||
xw := ctl.writer()
|
||||
enc := json.NewEncoder(xw)
|
||||
enc.SetIndent("", "\t")
|
||||
err = enc.Encode(m)
|
||||
ctl.xcheck(err, "encode retired message")
|
||||
xw.xclose()
|
||||
|
||||
case "queuehooklist":
|
||||
/* protocol:
|
||||
> "queuehooklist"
|
||||
> filters as json
|
||||
> sort as json
|
||||
< "ok"
|
||||
< stream
|
||||
*/
|
||||
var f queue.HookFilter
|
||||
xparseJSON(ctl, ctl.xread(), &f)
|
||||
var s queue.HookSort
|
||||
xparseJSON(ctl, ctl.xread(), &s)
|
||||
hooks, err := queue.HookList(ctx, f, s)
|
||||
ctl.xcheck(err, "listing webhooks")
|
||||
ctl.xwriteok()
|
||||
|
||||
xw := ctl.writer()
|
||||
fmt.Fprintln(xw, "webhooks:")
|
||||
for _, h := range hooks {
|
||||
var lastAttempt string
|
||||
if len(h.Results) > 0 {
|
||||
lastAttempt = time.Since(h.LastResult().Start).Round(time.Second).String()
|
||||
}
|
||||
fmt.Fprintf(xw, "%5d %s account:%s next %s last %s error %q url %s\n", h.ID, h.Submitted.Format(time.RFC3339), h.Account, time.Until(h.NextAttempt).Round(time.Second), lastAttempt, h.LastResult().Error, h.URL)
|
||||
}
|
||||
if len(hooks) == 0 {
|
||||
fmt.Fprint(xw, "(none)\n")
|
||||
}
|
||||
xw.xclose()
|
||||
|
||||
case "queuehookschedule":
|
||||
/* protocol:
|
||||
> "queuehookschedule"
|
||||
> hookfilters as json
|
||||
> relative to now
|
||||
> duration
|
||||
< "ok" or error
|
||||
< count
|
||||
*/
|
||||
|
||||
var f queue.HookFilter
|
||||
xparseJSON(ctl, ctl.xread(), &f)
|
||||
relnow := ctl.xread()
|
||||
d, err := time.ParseDuration(ctl.xread())
|
||||
ctl.xcheck(err, "parsing duration for next delivery attempt")
|
||||
var count int
|
||||
if relnow == "" {
|
||||
count, err = queue.HookNextAttemptAdd(ctx, f, d)
|
||||
} else {
|
||||
count, err = queue.HookNextAttemptSet(ctx, f, time.Now().Add(d))
|
||||
}
|
||||
ctl.xcheck(err, "setting next delivery attempts in queue")
|
||||
ctl.xwriteok()
|
||||
ctl.xwrite(fmt.Sprintf("%d", count))
|
||||
|
||||
case "queuehookcancel":
|
||||
/* protocol:
|
||||
> "queuehookcancel"
|
||||
> hookfilters as json
|
||||
< "ok" or error
|
||||
< count
|
||||
*/
|
||||
|
||||
var f queue.HookFilter
|
||||
xparseJSON(ctl, ctl.xread(), &f)
|
||||
count, err := queue.HookCancel(ctx, log, f)
|
||||
ctl.xcheck(err, "canceling webhooks in queue")
|
||||
ctl.xwriteok()
|
||||
ctl.xwrite(fmt.Sprintf("%d", count))
|
||||
|
||||
case "queuehookprint":
|
||||
/* protocol:
|
||||
> "queuehookprint"
|
||||
> id
|
||||
< "ok"
|
||||
< stream
|
||||
*/
|
||||
idstr := ctl.xread()
|
||||
id, err := strconv.ParseInt(idstr, 10, 64)
|
||||
if err != nil {
|
||||
ctl.xcheck(err, "parsing id")
|
||||
}
|
||||
l, err := queue.HookList(ctx, queue.HookFilter{IDs: []int64{id}}, queue.HookSort{})
|
||||
ctl.xcheck(err, "getting webhooks")
|
||||
if len(l) == 0 {
|
||||
ctl.xcheck(errors.New("not found"), "getting webhook")
|
||||
}
|
||||
h := l[0]
|
||||
ctl.xwriteok()
|
||||
xw := ctl.writer()
|
||||
enc := json.NewEncoder(xw)
|
||||
enc.SetIndent("", "\t")
|
||||
err = enc.Encode(h)
|
||||
ctl.xcheck(err, "encode webhook")
|
||||
xw.xclose()
|
||||
|
||||
case "queuehookretiredlist":
|
||||
/* protocol:
|
||||
> "queuehookretiredlist"
|
||||
> filters as json
|
||||
> sort as json
|
||||
< "ok"
|
||||
< stream
|
||||
*/
|
||||
var f queue.HookRetiredFilter
|
||||
xparseJSON(ctl, ctl.xread(), &f)
|
||||
var s queue.HookRetiredSort
|
||||
xparseJSON(ctl, ctl.xread(), &s)
|
||||
l, err := queue.HookRetiredList(ctx, f, s)
|
||||
ctl.xcheck(err, "listing retired webhooks")
|
||||
ctl.xwriteok()
|
||||
|
||||
xw := ctl.writer()
|
||||
fmt.Fprintln(xw, "retired webhooks:")
|
||||
for _, h := range l {
|
||||
var lastAttempt string
|
||||
if len(h.Results) > 0 {
|
||||
lastAttempt = time.Since(h.LastResult().Start).Round(time.Second).String()
|
||||
}
|
||||
result := "success"
|
||||
if !h.Success {
|
||||
result = "failure"
|
||||
}
|
||||
fmt.Fprintf(xw, "%5d %s %s account:%s last %s error %q url %s\n", h.ID, h.Submitted.Format(time.RFC3339), result, h.Account, lastAttempt, h.LastResult().Error, h.URL)
|
||||
}
|
||||
if len(l) == 0 {
|
||||
fmt.Fprint(xw, "(none)\n")
|
||||
}
|
||||
xw.xclose()
|
||||
|
||||
case "queuehookretiredprint":
|
||||
/* protocol:
|
||||
> "queuehookretiredprint"
|
||||
> id
|
||||
< "ok"
|
||||
< stream
|
||||
*/
|
||||
idstr := ctl.xread()
|
||||
id, err := strconv.ParseInt(idstr, 10, 64)
|
||||
if err != nil {
|
||||
ctl.xcheck(err, "parsing id")
|
||||
}
|
||||
l, err := queue.HookRetiredList(ctx, queue.HookRetiredFilter{IDs: []int64{id}}, queue.HookRetiredSort{})
|
||||
ctl.xcheck(err, "getting retired webhooks")
|
||||
if len(l) == 0 {
|
||||
ctl.xcheck(errors.New("not found"), "getting retired webhook")
|
||||
}
|
||||
h := l[0]
|
||||
ctl.xwriteok()
|
||||
xw := ctl.writer()
|
||||
enc := json.NewEncoder(xw)
|
||||
enc.SetIndent("", "\t")
|
||||
err = enc.Encode(h)
|
||||
ctl.xcheck(err, "encode retired webhook")
|
||||
xw.xclose()
|
||||
|
||||
case "queuesuppresslist":
|
||||
/* protocol:
|
||||
> "queuesuppresslist"
|
||||
> account (or empty)
|
||||
< "ok" or error
|
||||
< stream
|
||||
*/
|
||||
|
||||
account := ctl.xread()
|
||||
l, err := queue.SuppressionList(ctx, account)
|
||||
ctl.xcheck(err, "listing suppressions")
|
||||
ctl.xwriteok()
|
||||
xw := ctl.writer()
|
||||
fmt.Fprintln(xw, "suppressions (account, address, manual, time added, base adddress, reason):")
|
||||
for _, sup := range l {
|
||||
manual := "No"
|
||||
if sup.Manual {
|
||||
manual = "Yes"
|
||||
}
|
||||
fmt.Fprintf(xw, "%q\t%q\t%s\t%s\t%q\t%q\n", sup.Account, sup.OriginalAddress, manual, sup.Created.Round(time.Second), sup.BaseAddress, sup.Reason)
|
||||
}
|
||||
if len(l) == 0 {
|
||||
fmt.Fprintln(xw, "(none)")
|
||||
}
|
||||
xw.xclose()
|
||||
|
||||
case "queuesuppressadd":
|
||||
/* protocol:
|
||||
> "queuesuppressadd"
|
||||
> account
|
||||
> address
|
||||
< "ok" or error
|
||||
*/
|
||||
|
||||
account := ctl.xread()
|
||||
address := ctl.xread()
|
||||
_, ok := mox.Conf.Account(account)
|
||||
if !ok {
|
||||
ctl.xcheck(errors.New("unknown account"), "looking up account")
|
||||
}
|
||||
addr, err := smtp.ParseAddress(address)
|
||||
ctl.xcheck(err, "parsing address")
|
||||
sup := webapi.Suppression{
|
||||
Account: account,
|
||||
Manual: true,
|
||||
Reason: "added through mox cli",
|
||||
}
|
||||
err = queue.SuppressionAdd(ctx, addr.Path(), &sup)
|
||||
ctl.xcheck(err, "adding suppression")
|
||||
ctl.xwriteok()
|
||||
|
||||
case "queuesuppressremove":
|
||||
/* protocol:
|
||||
> "queuesuppressremove"
|
||||
> account
|
||||
> address
|
||||
< "ok" or error
|
||||
*/
|
||||
|
||||
account := ctl.xread()
|
||||
address := ctl.xread()
|
||||
addr, err := smtp.ParseAddress(address)
|
||||
ctl.xcheck(err, "parsing address")
|
||||
err = queue.SuppressionRemove(ctx, account, addr.Path())
|
||||
ctl.xcheck(err, "removing suppression")
|
||||
ctl.xwriteok()
|
||||
|
||||
case "queuesuppresslookup":
|
||||
/* protocol:
|
||||
> "queuesuppresslookup"
|
||||
> account or empty
|
||||
> address
|
||||
< "ok" or error
|
||||
< stream
|
||||
*/
|
||||
|
||||
account := ctl.xread()
|
||||
address := ctl.xread()
|
||||
if account != "" {
|
||||
_, ok := mox.Conf.Account(account)
|
||||
if !ok {
|
||||
ctl.xcheck(errors.New("unknown account"), "looking up account")
|
||||
}
|
||||
}
|
||||
addr, err := smtp.ParseAddress(address)
|
||||
ctl.xcheck(err, "parsing address")
|
||||
sup, err := queue.SuppressionLookup(ctx, account, addr.Path())
|
||||
ctl.xcheck(err, "looking up suppression")
|
||||
ctl.xwriteok()
|
||||
xw := ctl.writer()
|
||||
if sup == nil {
|
||||
fmt.Fprintln(xw, "not present")
|
||||
} else {
|
||||
manual := "no"
|
||||
if sup.Manual {
|
||||
manual = "yes"
|
||||
}
|
||||
fmt.Fprintf(xw, "present\nadded: %s\nmanual: %s\nbase address: %s\nreason: %q\n", sup.Created.Round(time.Second), manual, sup.BaseAddress, sup.Reason)
|
||||
}
|
||||
xw.xclose()
|
||||
|
||||
case "importmaildir", "importmbox":
|
||||
mbox := cmd == "importmbox"
|
||||
importctl(ctx, ctl, mbox)
|
||||
|
||||
Reference in New Issue
Block a user