infrastructure/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2025-07-12 17:04:39 +03:00
Code Issues Packages Projects Releases Wiki Activity

update to latest bstore (with support for an index on a []string: Message.DKIMDomains), and cyclic data types (to be used for Message.Part soon); also adds a context.Context to database operations.

Browse Source
This commit is contained in:
Mechiel Lukkien
2023-05-22 14:40:36 +02:00
parent f6ed860ccb
commit e81930ba20
58 changed files with 1970 additions and 1035 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
queue/queue.go
View File

@ -122,7 +122,7 @@ func Init() error {
}
var err error
queueDB, err = bstore.Open(qpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, Msg{})
queueDB, err = bstore.Open(mox.Shutdown, qpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, Msg{})
if err != nil {
if isNew {
os.Remove(qpath)
@ -141,8 +141,8 @@ func Shutdown() {
// List returns all messages in the delivery queue.
// Ordered by earliest delivery attempt first.
func List() ([]Msg, error) {
qmsgs, err := bstore.QueryDB[Msg](queueDB).List()
func List(ctx context.Context) ([]Msg, error) {
qmsgs, err := bstore.QueryDB[Msg](ctx, queueDB).List()
if err != nil {
return nil, err
}
@ -165,8 +165,8 @@ func List() ([]Msg, error) {
}
// Count returns the number of messages in the delivery queue.
func Count() (int, error) {
return bstore.QueryDB[Msg](queueDB).Count()
func Count(ctx context.Context) (int, error) {
return bstore.QueryDB[Msg](ctx, queueDB).Count()
}
// Add a new message to the queue. The queue is kicked immediately to start a
@ -179,7 +179,7 @@ func Count() (int, error) {
// this data is used as the message when delivering the DSN and the remote SMTP
// server supports SMTPUTF8. If the remote SMTP server does not support SMTPUTF8,
// the regular non-utf8 message is delivered.
func Add(log *mlog.Log, senderAccount string, mailFrom, rcptTo smtp.Path, has8bit, smtputf8 bool, size int64, msgPrefix []byte, msgFile *os.File, dsnutf8Opt []byte, consumeFile bool) error {
func Add(ctx context.Context, log *mlog.Log, senderAccount string, mailFrom, rcptTo smtp.Path, has8bit, smtputf8 bool, size int64, msgPrefix []byte, msgFile *os.File, dsnutf8Opt []byte, consumeFile bool) error {
// todo: Add should accept multiple rcptTo if they are for the same domain. so we can queue them for delivery in one (or just a few) session(s), transferring the data only once. ../rfc/5321:3759
if Localserve {
@ -187,7 +187,7 @@ func Add(log *mlog.Log, senderAccount string, mailFrom, rcptTo smtp.Path, has8bi
return fmt.Errorf("no queuing with localserve")
}
tx, err := queueDB.Begin(true)
tx, err := queueDB.Begin(ctx, true)
if err != nil {
return fmt.Errorf("begin transaction: %w", err)
}
@ -287,8 +287,8 @@ func queuekick() {
// and kicks the queue, attempting delivery of those messages. If all parameters
// are zero, all messages are kicked.
// Returns number of messages queued for immediate delivery.
func Kick(ID int64, toDomain string, recipient string) (int, error) {
q := bstore.QueryDB[Msg](queueDB)
func Kick(ctx context.Context, ID int64, toDomain string, recipient string) (int, error) {
q := bstore.QueryDB[Msg](ctx, queueDB)
if ID > 0 {
q.FilterID(ID)
}
@ -311,8 +311,8 @@ func Kick(ID int64, toDomain string, recipient string) (int, error) {
// Drop removes messages from the queue that match all nonzero parameters.
// If all parameters are zero, all messages are removed.
// Returns number of messages removed.
func Drop(ID int64, toDomain string, recipient string) (int, error) {
q := bstore.QueryDB[Msg](queueDB)
func Drop(ctx context.Context, ID int64, toDomain string, recipient string) (int, error) {
q := bstore.QueryDB[Msg](ctx, queueDB)
if ID > 0 {
q.FilterID(ID)
}
@ -337,9 +337,9 @@ type ReadReaderAtCloser interface {
}
// OpenMessage opens a message present in the queue.
func OpenMessage(id int64) (ReadReaderAtCloser, error) {
func OpenMessage(ctx context.Context, id int64) (ReadReaderAtCloser, error) {
qm := Msg{ID: id}
err := queueDB.Get(&qm)
err := queueDB.Get(ctx, &qm)
if err != nil {
return nil, err
}
@ -382,14 +382,14 @@ func Start(resolver dns.Resolver, done chan struct{}) error {
}
launchWork(resolver, busyDomains)
timer.Reset(nextWork(busyDomains))
timer.Reset(nextWork(mox.Shutdown, busyDomains))
}
}()
return nil
}
func nextWork(busyDomains map[string]struct{}) time.Duration {
q := bstore.QueryDB[Msg](queueDB)
func nextWork(ctx context.Context, busyDomains map[string]struct{}) time.Duration {
q := bstore.QueryDB[Msg](ctx, queueDB)
if len(busyDomains) > 0 {
var doms []any
for d := range busyDomains {
@ -410,7 +410,7 @@ func nextWork(busyDomains map[string]struct{}) time.Duration {
}
func launchWork(resolver dns.Resolver, busyDomains map[string]struct{}) int {
q := bstore.QueryDB[Msg](queueDB)
q := bstore.QueryDB[Msg](mox.Shutdown, queueDB)
q.FilterLessEqual("NextAttempt", time.Now())
q.SortAsc("NextAttempt")
q.Limit(maxConcurrentDeliveries)
@ -424,7 +424,7 @@ func launchWork(resolver dns.Resolver, busyDomains map[string]struct{}) int {
msgs, err := q.List()
if err != nil {
xlog.Errorx("querying for work in queue", err)
mox.Sleep(mox.Context, 1*time.Second)
mox.Sleep(mox.Shutdown, 1*time.Second)
return -1
}
@ -436,8 +436,8 @@ func launchWork(resolver dns.Resolver, busyDomains map[string]struct{}) int {
}
// Remove message from queue in database and file system.
func queueDelete(msgID int64) error {
if err := queueDB.Delete(&Msg{ID: msgID}); err != nil {
func queueDelete(ctx context.Context, msgID int64) error {
if err := queueDB.Delete(ctx, &Msg{ID: msgID}); err != nil {
return err
}
// If removing from database fails, we'll also leave the file in the file system.
@ -483,7 +483,7 @@ func deliver(resolver dns.Resolver, m Msg) {
now := time.Now()
m.LastAttempt = &now
m.NextAttempt = now.Add(backoff)
qup := bstore.QueryDB[Msg](queueDB)
qup := bstore.QueryDB[Msg](mox.Shutdown, queueDB)
qup.FilterID(m.ID)
update := Msg{Attempts: m.Attempts, NextAttempt: m.NextAttempt, LastAttempt: m.LastAttempt}
if _, err := qup.UpdateNonzero(update); err != nil {
@ -496,13 +496,13 @@ func deliver(resolver dns.Resolver, m Msg) {
qlog.Errorx("permanent failure delivering from queue", errors.New(errmsg))
queueDSNFailure(qlog, m, remoteMTA, secodeOpt, errmsg)
if err := queueDelete(m.ID); err != nil {
if err := queueDelete(context.Background(), m.ID); err != nil {
qlog.Errorx("deleting message from queue after permanent failure", err)
}
return
}
qup := bstore.QueryDB[Msg](queueDB)
qup := bstore.QueryDB[Msg](context.Background(), queueDB)
qup.FilterID(m.ID)
if _, err := qup.UpdateNonzero(Msg{LastError: errmsg, DialedIPs: m.DialedIPs}); err != nil {
qlog.Errorx("storing delivery error", err, mlog.Field("deliveryerror", errmsg))
@ -534,7 +534,7 @@ func deliver(resolver dns.Resolver, m Msg) {
var policy *mtasts.Policy
tlsModeDefault := smtpclient.TLSOpportunistic
if !effectiveDomain.IsZero() {
cidctx := context.WithValue(mox.Context, mlog.CidKey, cid)
cidctx := context.WithValue(mox.Shutdown, mlog.CidKey, cid)
policy, policyFresh, err = mtastsdb.Get(cidctx, resolver, effectiveDomain)
if err != nil {
// No need to refuse to deliver if we have some mtasts error.
@ -586,7 +586,7 @@ func deliver(resolver dns.Resolver, m Msg) {
}
if ok {
nqlog.Info("delivered from queue")
if err := queueDelete(m.ID); err != nil {
if err := queueDelete(context.Background(), m.ID); err != nil {
nqlog.Errorx("deleting message from queue after delivery", err)
}
return

54
queue/queue_test.go
View File

@ -21,6 +21,8 @@ import (
"github.com/mjl-/mox/store"
)
var ctxbg = context.Background()
func tcheck(t *testing.T, err error, msg string) {
if err != nil {
t.Helper()
@ -31,7 +33,7 @@ func tcheck(t *testing.T, err error, msg string) {
func setup(t *testing.T) (*store.Account, func()) {
// Prepare config so email can be delivered to mjl@mox.example.
os.RemoveAll("../testdata/queue/data")
mox.Context = context.Background()
mox.Context = ctxbg
mox.ConfigStaticPath = "../testdata/queue/mox.conf"
mox.MustLoadConfig(false)
acc, err := store.OpenAccount("mjl")
@ -39,11 +41,11 @@ func setup(t *testing.T) (*store.Account, func()) {
err = acc.SetPassword("testtest")
tcheck(t, err, "set password")
switchDone := store.Switchboard()
mox.Shutdown, mox.ShutdownCancel = context.WithCancel(context.Background())
mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg)
return acc, func() {
acc.Close()
mox.ShutdownCancel()
mox.Shutdown, mox.ShutdownCancel = context.WithCancel(context.Background())
mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg)
Shutdown()
close(switchDone)
}
@ -71,22 +73,22 @@ func TestQueue(t *testing.T) {
err := Init()
tcheck(t, err, "queue init")
msgs, err := List()
msgs, err := List(ctxbg)
tcheck(t, err, "listing messages in queue")
if len(msgs) != 0 {
t.Fatalf("got %d messages in queue, expected 0", len(msgs))
}
path := smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "mox.example"}}}
err = Add(xlog, "mjl", path, path, false, false, int64(len(testmsg)), nil, prepareFile(t), nil, true)
err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), nil, prepareFile(t), nil, true)
tcheck(t, err, "add message to queue for delivery")
mf2 := prepareFile(t)
err = Add(xlog, "mjl", path, path, false, false, int64(len(testmsg)), nil, mf2, nil, false)
err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), nil, mf2, nil, false)
tcheck(t, err, "add message to queue for delivery")
os.Remove(mf2.Name())
msgs, err = List()
msgs, err = List(ctxbg)
tcheck(t, err, "listing queue")
if len(msgs) != 2 {
t.Fatalf("got msgs %v, expected 1", msgs)
@ -95,18 +97,18 @@ func TestQueue(t *testing.T) {
if msg.Attempts != 0 {
t.Fatalf("msg attempts %d, expected 0", msg.Attempts)
}
n, err := Drop(msgs[1].ID, "", "")
n, err := Drop(ctxbg, msgs[1].ID, "", "")
tcheck(t, err, "drop")
if n != 1 {
t.Fatalf("dropped %d, expected 1", n)
}
next := nextWork(nil)
next := nextWork(ctxbg, nil)
if next > 0 {
t.Fatalf("nextWork in %s, should be now", next)
}
busy := map[string]struct{}{"mox.example": {}}
if x := nextWork(busy); x != 24*time.Hour {
if x := nextWork(ctxbg, busy); x != 24*time.Hour {
t.Fatalf("nextWork in %s for busy domain, should be in 24 hours", x)
}
if nn := launchWork(nil, busy); nn != 0 {
@ -133,7 +135,7 @@ func TestQueue(t *testing.T) {
case <-dialed:
i := 0
for {
m, err := bstore.QueryDB[Msg](queueDB).Get()
m, err := bstore.QueryDB[Msg](ctxbg, queueDB).Get()
tcheck(t, err, "get")
if m.Attempts == 1 {
break
@ -149,11 +151,11 @@ func TestQueue(t *testing.T) {
}
<-deliveryResult // Deliver sends here.
_, err = OpenMessage(msg.ID + 1)
_, err = OpenMessage(ctxbg, msg.ID+1)
if err != bstore.ErrAbsent {
t.Fatalf("OpenMessage, got %v, expected ErrAbsent", err)
}
reader, err := OpenMessage(msg.ID)
reader, err := OpenMessage(ctxbg, msg.ID)
tcheck(t, err, "open message")
defer reader.Close()
msgbuf, err := io.ReadAll(reader)
@ -162,12 +164,12 @@ func TestQueue(t *testing.T) {
t.Fatalf("message mismatch, got %q, expected %q", string(msgbuf), testmsg)
}
n, err = Kick(msg.ID+1, "", "")
n, err = Kick(ctxbg, msg.ID+1, "", "")
tcheck(t, err, "kick")
if n != 0 {
t.Fatalf("kick %d, expected 0", n)
}
n, err = Kick(msg.ID, "", "")
n, err = Kick(ctxbg, msg.ID, "", "")
tcheck(t, err, "kick")
if n != 1 {
t.Fatalf("kicked %d, expected 1", n)
@ -215,7 +217,7 @@ func TestQueue(t *testing.T) {
case <-smtpdone:
i := 0
for {
xmsgs, err := List()
xmsgs, err := List(ctxbg)
tcheck(t, err, "list queue")
if len(xmsgs) == 0 {
break
@ -235,10 +237,10 @@ func TestQueue(t *testing.T) {
<-deliveryResult // Deliver sends here.
// Add another message that we'll fail to deliver entirely.
err = Add(xlog, "mjl", path, path, false, false, int64(len(testmsg)), nil, prepareFile(t), nil, true)
err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), nil, prepareFile(t), nil, true)
tcheck(t, err, "add message to queue for delivery")
msgs, err = List()
msgs, err = List(ctxbg)
tcheck(t, err, "list queue")
if len(msgs) != 1 {
t.Fatalf("queue has %d messages, expected 1", len(msgs))
@ -283,7 +285,7 @@ func TestQueue(t *testing.T) {
for i := 1; i < 8; i++ {
go func() { <-deliveryResult }() // Deliver sends here.
deliver(resolver, msg)
err = queueDB.Get(&msg)
err = queueDB.Get(ctxbg, &msg)
tcheck(t, err, "get msg")
if msg.Attempts != i {
t.Fatalf("got attempt %d, expected %d", msg.Attempts, i)
@ -306,7 +308,7 @@ func TestQueue(t *testing.T) {
// Trigger final failure.
go func() { <-deliveryResult }() // Deliver sends here.
deliver(resolver, msg)
err = queueDB.Get(&msg)
err = queueDB.Get(ctxbg, &msg)
if err != bstore.ErrAbsent {
t.Fatalf("attempt to fetch delivered and removed message from queue, got err %v, expected ErrAbsent", err)
}
@ -343,7 +345,7 @@ func TestQueueStart(t *testing.T) {
defer func() {
mox.ShutdownCancel()
<-done
mox.Shutdown, mox.ShutdownCancel = context.WithCancel(context.Background())
mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg)
}()
err := Start(resolver, done)
tcheck(t, err, "queue start")
@ -369,7 +371,7 @@ func TestQueueStart(t *testing.T) {
}
path := smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "mox.example"}}}
err = Add(xlog, "mjl", path, path, false, false, int64(len(testmsg)), nil, prepareFile(t), nil, true)
err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), nil, prepareFile(t), nil, true)
tcheck(t, err, "add message to queue for delivery")
checkDialed(true)
@ -378,7 +380,7 @@ func TestQueueStart(t *testing.T) {
checkDialed(false)
// Kick for real, should see another attempt.
n, err := Kick(0, "mox.example", "")
n, err := Kick(ctxbg, 0, "mox.example", "")
tcheck(t, err, "kick queue")
if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n)
@ -402,7 +404,7 @@ func TestWriteFile(t *testing.T) {
}
func TestGatherHosts(t *testing.T) {
mox.Context = context.Background()
mox.Context = ctxbg
// Test basic MX lookup case, but also following CNAME, detecting CNAME loops and
// having a CNAME limit, connecting directly to a host, and domain that does not
@ -524,11 +526,11 @@ func TestDialHost(t *testing.T) {
}
m := Msg{DialedIPs: map[string][]net.IP{}}
_, ip, dualstack, err := dialHost(context.Background(), xlog, resolver, ipdomain("dualstack.example"), &m)
_, ip, dualstack, err := dialHost(ctxbg, xlog, resolver, ipdomain("dualstack.example"), &m)
if err != nil || ip.String() != "10.0.0.1" || !dualstack {
t.Fatalf("expected err nil, address 10.0.0.1, dualstack true, got %v %v %v", err, ip, dualstack)
}
_, ip, dualstack, err = dialHost(context.Background(), xlog, resolver, ipdomain("dualstack.example"), &m)
_, ip, dualstack, err = dialHost(ctxbg, xlog, resolver, ipdomain("dualstack.example"), &m)
if err != nil || ip.String() != "2001:db8::1" || !dualstack {
t.Fatalf("expected err nil, address 2001:db8::1, dualstack true, got %v %v %v", err, ip, dualstack)
}