improve queue management

- add option to put messages in the queue "on hold", preventing delivery
  attempts until taken off hold again.
- add "hold rules", to automatically mark some/all submitted messages as "on
  hold", e.g. from a specific account or to a specific domain.
- add operation to "fail" a message, causing a DSN to be delivered to the
  sender. previously we could only drop a message from the queue.
- update admin page & add new cli tools for these operations, with new
  filtering rules for selecting the messages to operate on. in the admin
  interface, add filtering and checkboxes to select a set of messages to operate
  on.

queue/queue.go

@@ -61,12 +61,18 @@ var (
 			"result",    // ok, timeout, canceled, temperror, permerror, error
 		},
 	)
+	metricHold = promauto.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "mox_queue_hold",
+			Help: "Messages in queue that are on hold.",
+		},
+	)
 )
 
 var jitter = mox.NewPseudoRand()
 
-var DBTypes = []any{Msg{}} // Types stored in DB.
-var DB *bstore.DB          // Exported for making backups.
+var DBTypes = []any{Msg{}, HoldRule{}} // Types stored in DB.
+var DB *bstore.DB                      // Exported for making backups.
 
 // Allow requesting delivery starting from up to this interval from time of submission.
 const FutureReleaseIntervalMax = 60 * 24 * time.Hour
@@ -74,6 +80,27 @@ const FutureReleaseIntervalMax = 60 * 24 * time.Hour
 // Set for mox localserve, to prevent queueing.
 var Localserve bool
 
+// HoldRule is a set of conditions that cause a matching message to be marked as on
+// hold when it is queued. All-empty conditions matches all messages, effectively
+// pausing the entire queue.
+type HoldRule struct {
+	ID                 int64
+	Account            string
+	SenderDomain       dns.Domain
+	RecipientDomain    dns.Domain
+	SenderDomainStr    string // Unicode.
+	RecipientDomainStr string // Unicode.
+}
+
+func (pr HoldRule) All() bool {
+	pr.ID = 0
+	return pr == HoldRule{}
+}
+
+func (pr HoldRule) matches(m Msg) bool {
+	return pr.All() || pr.Account == m.SenderAccount || pr.SenderDomainStr == m.SenderDomainStr || pr.RecipientDomainStr == m.RecipientDomainStr
+}
+
 // Msg is a message in the queue.
 //
 // Use MakeMsg to make a message with fields that Add needs. Add will further set
@@ -89,12 +116,14 @@ type Msg struct {
 	BaseID int64 `bstore:"index"`
 
 	Queued             time.Time      `bstore:"default now"`
+	Hold               bool           // If set, delivery won't be attempted.
 	SenderAccount      string         // Failures are delivered back to this local account. Also used for routing.
 	SenderLocalpart    smtp.Localpart // Should be a local user and domain.
 	SenderDomain       dns.IPDomain
+	SenderDomainStr    string         // For filtering, unicode.
 	RecipientLocalpart smtp.Localpart // Typically a remote user and domain.
 	RecipientDomain    dns.IPDomain
-	RecipientDomainStr string              // For filtering.
+	RecipientDomainStr string              // For filtering, unicode.
 	Attempts           int                 // Next attempt is based on last attempt and exponential back off based on attempts.
 	MaxAttempts        int                 // Max number of attempts before giving up. If 0, then the default of 8 attempts is used instead.
 	DialedIPs          map[string][]net.IP // For each host, the IPs that were dialed. Used for IP selection for later attempts.
@@ -174,9 +203,20 @@ func Init() error {
 		}
 		return fmt.Errorf("open queue database: %s", err)
 	}
+	metricHoldUpdate()
 	return nil
 }
 
+// When we update the gauge, we just get the full current value, not try to account
+// for adds/removes.
+func metricHoldUpdate() {
+	count, err := bstore.QueryDB[Msg](context.Background(), DB).FilterNonzero(Msg{Hold: true}).Count()
+	if err != nil {
+		mlog.New("queue", nil).Errorx("querying number of queued messages that are on hold", err)
+	}
+	metricHold.Set(float64(count))
+}
+
 // Shutdown closes the queue database. The delivery process isn't stopped. For tests only.
 func Shutdown() {
 	err := DB.Close()
@@ -186,10 +226,85 @@ func Shutdown() {
 	DB = nil
 }
 
+// Filter filters messages to list or operate on. Used by admin web interface
+// and cli.
+//
+// Only non-empty/non-zero values are applied to the filter. Leaving all fields
+// empty/zero matches all messages.
+type Filter struct {
+	IDs         []int64
+	Account     string
+	From        string
+	To          string
+	Hold        *bool
+	Submitted   string // Whether submitted before/after a time relative to now. ">$duration" or "<$duration", also with "now" for duration.
+	NextAttempt string // ">$duration" or "<$duration", also with "now" for duration.
+	Transport   *string
+}
+
+func (f Filter) apply(q *bstore.Query[Msg]) error {
+	if len(f.IDs) > 0 {
+		q.FilterIDs(f.IDs)
+	}
+	applyTime := func(field string, s string) error {
+		orig := s
+		var before bool
+		if strings.HasPrefix(s, "<") {
+			before = true
+		} else if !strings.HasPrefix(s, ">") {
+			return fmt.Errorf(`must start with "<" for before or ">" for after a duration`)
+		}
+		s = s[1:]
+		var t time.Time
+		if s == "now" {
+			t = time.Now()
+		} else if d, err := time.ParseDuration(s); err != nil {
+			return fmt.Errorf("parsing duration %q: %v", orig, err)
+		} else {
+			t = time.Now().Add(d)
+		}
+		if before {
+			q.FilterLess(field, t)
+		} else {
+			q.FilterGreater(field, t)
+		}
+		return nil
+	}
+	if f.Hold != nil {
+		q.FilterEqual("Hold", *f.Hold)
+	}
+	if f.Submitted != "" {
+		if err := applyTime("Queued", f.Submitted); err != nil {
+			return fmt.Errorf("applying filter for submitted: %v", err)
+		}
+	}
+	if f.NextAttempt != "" {
+		if err := applyTime("NextAttempt", f.NextAttempt); err != nil {
+			return fmt.Errorf("applying filter for next attempt: %v", err)
+		}
+	}
+	if f.Account != "" {
+		q.FilterNonzero(Msg{SenderAccount: f.Account})
+	}
+	if f.Transport != nil {
+		q.FilterEqual("Transport", *f.Transport)
+	}
+	if f.From != "" || f.To != "" {
+		q.FilterFn(func(m Msg) bool {
+			return f.From != "" && strings.Contains(m.Sender().XString(true), f.From) || f.To != "" && strings.Contains(m.Recipient().XString(true), f.To)
+		})
+	}
+	return nil
+}
+
 // List returns all messages in the delivery queue.
 // Ordered by earliest delivery attempt first.
-func List(ctx context.Context) ([]Msg, error) {
-	qmsgs, err := bstore.QueryDB[Msg](ctx, DB).List()
+func List(ctx context.Context, f Filter) ([]Msg, error) {
+	q := bstore.QueryDB[Msg](ctx, DB)
+	if err := f.apply(q); err != nil {
+		return nil, err
+	}
+	qmsgs, err := q.List()
 	if err != nil {
 		return nil, err
 	}
@@ -216,6 +331,59 @@ func Count(ctx context.Context) (int, error) {
 	return bstore.QueryDB[Msg](ctx, DB).Count()
 }
 
+// HoldRuleList returns all hold rules.
+func HoldRuleList(ctx context.Context) ([]HoldRule, error) {
+	return bstore.QueryDB[HoldRule](ctx, DB).List()
+}
+
+// HoldRuleAdd adds a new hold rule causing newly submitted messages to be marked
+// as "on hold", and existing matching messages too.
+func HoldRuleAdd(ctx context.Context, log mlog.Log, hr HoldRule) (HoldRule, error) {
+	err := DB.Write(ctx, func(tx *bstore.Tx) error {
+		hr.ID = 0
+		hr.SenderDomainStr = hr.SenderDomain.Name()
+		hr.RecipientDomainStr = hr.RecipientDomain.Name()
+		if err := tx.Insert(&hr); err != nil {
+			return err
+		}
+		log.Info("adding hold rule", slog.Any("holdrule", hr))
+
+		q := bstore.QueryTx[Msg](tx)
+		if !hr.All() {
+			q.FilterNonzero(Msg{
+				SenderAccount:      hr.Account,
+				SenderDomainStr:    hr.SenderDomainStr,
+				RecipientDomainStr: hr.RecipientDomainStr,
+			})
+		}
+		n, err := q.UpdateField("Hold", true)
+		if err != nil {
+			return fmt.Errorf("marking existing matching messages in queue on hold: %v", err)
+		}
+		log.Info("marked messages in queue as on hold", slog.Int("messages", n))
+		return nil
+	})
+	if err != nil {
+		return HoldRule{}, err
+	}
+	queuekick()
+	metricHoldUpdate()
+	return hr, nil
+}
+
+// HoldRuleRemove removes a hold rule. The Hold field of existing messages are not
+// changed.
+func HoldRuleRemove(ctx context.Context, log mlog.Log, holdRuleID int64) error {
+	return DB.Write(ctx, func(tx *bstore.Tx) error {
+		hr := HoldRule{ID: holdRuleID}
+		if err := tx.Get(&hr); err != nil {
+			return err
+		}
+		log.Info("removing hold rule", slog.Any("holdrule", hr))
+		return tx.Delete(HoldRule{ID: holdRuleID})
+	})
+}
+
 // MakeMsg is a convenience function that sets the commonly used fields for a Msg.
 func MakeMsg(sender, recipient smtp.Path, has8bit, smtputf8 bool, size int64, messageID string, prefix []byte, requireTLS *bool, next time.Time) Msg {
 	return Msg{
@@ -223,7 +391,6 @@ func MakeMsg(sender, recipient smtp.Path, has8bit, smtputf8 bool, size int64, me
 		SenderDomain:       sender.IPDomain,
 		RecipientLocalpart: recipient.Localpart,
 		RecipientDomain:    recipient.IPDomain,
-		RecipientDomainStr: formatIPDomain(recipient.IPDomain),
 		Has8bit:            has8bit,
 		SMTPUTF8:           smtputf8,
 		Size:               size,
@@ -242,25 +409,20 @@ func MakeMsg(sender, recipient smtp.Path, has8bit, smtputf8 bool, size int64, me
 //
 // ID of the messagse must be 0 and will be set after inserting in the queue.
 //
-// Add sets derived fields like RecipientDomainStr, and fields related to queueing,
-// such as Queued, NextAttempt, LastAttempt, LastError.
+// Add sets derived fields like SenderDomainStr and RecipientDomainStr, and fields
+// related to queueing, such as Queued, NextAttempt, LastAttempt, LastError.
 func Add(ctx context.Context, log mlog.Log, senderAccount string, msgFile *os.File, qml ...Msg) error {
 	if len(qml) == 0 {
 		return fmt.Errorf("must queue at least one message")
 	}
 
-	for _, qm := range qml {
+	for i, qm := range qml {
 		if qm.ID != 0 {
 			return fmt.Errorf("id of queued messages must be 0")
 		}
-		if qm.RecipientDomainStr == "" {
-			return fmt.Errorf("recipient domain cannot be empty")
-		}
 		// Sanity check, internal consistency.
-		rcptDom := formatIPDomain(qm.RecipientDomain)
-		if qm.RecipientDomainStr != rcptDom {
-			return fmt.Errorf("mismatch between recipient domain and string form of domain")
-		}
+		qml[i].SenderDomainStr = formatIPDomain(qm.SenderDomain)
+		qml[i].RecipientDomainStr = formatIPDomain(qm.RecipientDomain)
 	}
 
 	if Localserve {
@@ -307,12 +469,24 @@ func Add(ctx context.Context, log mlog.Log, senderAccount string, msgFile *os.Fi
 		}
 	}()
 
+	// Mark messages Hold if they match a hold rule.
+	holdRules, err := bstore.QueryTx[HoldRule](tx).List()
+	if err != nil {
+		return fmt.Errorf("getting queue hold rules")
+	}
+
 	// Insert messages into queue. If there are multiple messages, they all get a
 	// non-zero BaseID that is the Msg.ID of the first message inserted.
 	var baseID int64
 	for i := range qml {
 		qml[i].SenderAccount = senderAccount
 		qml[i].BaseID = baseID
+		for _, hr := range holdRules {
+			if hr.matches(qml[i]) {
+				qml[i].Hold = true
+				break
+			}
+		}
 		if err := tx.Insert(&qml[i]); err != nil {
 			return err
 		}
@@ -351,7 +525,15 @@ func Add(ctx context.Context, log mlog.Log, senderAccount string, msgFile *os.Fi
 	tx = nil
 	paths = nil
 
+	for _, m := range qml {
+		if m.Hold {
+			metricHoldUpdate()
+			break
+		}
+	}
+
 	queuekick()
+
 	return nil
 }
 
@@ -374,36 +556,43 @@ func queuekick() {
 	}
 }
 
-// Kick sets the NextAttempt for messages matching all filter parameters (ID,
-// toDomain, recipient) that are nonzero, and kicks the queue, attempting delivery
-// of those messages. If all parameters are zero, all messages are kicked. If
-// transport is set, the delivery attempts for the matching messages will use the
-// transport. An empty string is the default transport, i.e. direct delivery.
-// Returns number of messages queued for immediate delivery.
-func Kick(ctx context.Context, ID int64, toDomain, recipient string, transport *string) (int, error) {
-	q := bstore.QueryDB[Msg](ctx, DB)
-	if ID > 0 {
-		q.FilterID(ID)
-	}
-	if toDomain != "" {
-		q.FilterEqual("RecipientDomainStr", toDomain)
-	}
-	if recipient != "" {
-		q.FilterFn(func(qm Msg) bool {
-			return qm.Recipient().XString(true) == recipient
-		})
-	}
-	up := map[string]any{"NextAttempt": time.Now()}
-	if transport != nil {
-		if *transport != "" {
-			_, ok := mox.Conf.Static.Transports[*transport]
-			if !ok {
-				return 0, fmt.Errorf("unknown transport %q", *transport)
+// NextAttemptAdd adds a duration to the NextAttempt for all matching messages, and
+// kicks the queue.
+func NextAttemptAdd(ctx context.Context, f Filter, d time.Duration) (affected int, err error) {
+	err = DB.Write(ctx, func(tx *bstore.Tx) error {
+		q := bstore.QueryDB[Msg](ctx, DB)
+		if err := f.apply(q); err != nil {
+			return err
+		}
+		var msgs []Msg
+		msgs, err := q.List()
+		if err != nil {
+			return fmt.Errorf("listing matching messages: %v", err)
+		}
+		for _, m := range msgs {
+			m.NextAttempt = m.NextAttempt.Add(d)
+			if err := tx.Update(&m); err != nil {
+				return err
 			}
 		}
-		up["Transport"] = *transport
+		affected = len(msgs)
+		return nil
+	})
+	if err != nil {
+		return 0, err
 	}
-	n, err := q.UpdateFields(up)
+	queuekick()
+	return affected, nil
+}
+
+// NextAttemptSet sets NextAttempt for all matching messages to a new time, and
+// kicks the queue.
+func NextAttemptSet(ctx context.Context, f Filter, t time.Time) (affected int, err error) {
+	q := bstore.QueryDB[Msg](ctx, DB)
+	if err := f.apply(q); err != nil {
+		return 0, err
+	}
+	n, err := q.UpdateNonzero(Msg{NextAttempt: t})
 	if err != nil {
 		return 0, fmt.Errorf("selecting and updating messages in queue: %v", err)
 	}
@@ -411,21 +600,74 @@ func Kick(ctx context.Context, ID int64, toDomain, recipient string, transport *
 	return n, nil
 }
 
-// Drop removes messages from the queue that match all nonzero parameters.
-// If all parameters are zero, all messages are removed.
-// Returns number of messages removed.
-func Drop(ctx context.Context, log mlog.Log, ID int64, toDomain string, recipient string) (int, error) {
+// HoldSet sets Hold for all matching messages and kicks the queue.
+func HoldSet(ctx context.Context, f Filter, hold bool) (affected int, err error) {
 	q := bstore.QueryDB[Msg](ctx, DB)
-	if ID > 0 {
-		q.FilterID(ID)
+	if err := f.apply(q); err != nil {
+		return 0, err
 	}
-	if toDomain != "" {
-		q.FilterEqual("RecipientDomainStr", toDomain)
+	n, err := q.UpdateFields(map[string]any{"Hold": hold})
+	if err != nil {
+		return 0, fmt.Errorf("selecting and updating messages in queue: %v", err)
 	}
-	if recipient != "" {
-		q.FilterFn(func(qm Msg) bool {
-			return qm.Recipient().XString(true) == recipient
-		})
+	queuekick()
+	metricHoldUpdate()
+	return n, nil
+}
+
+// TransportSet changes the transport to use for the matching messages.
+func TransportSet(ctx context.Context, f Filter, transport string) (affected int, err error) {
+	q := bstore.QueryDB[Msg](ctx, DB)
+	if err := f.apply(q); err != nil {
+		return 0, err
+	}
+	n, err := q.UpdateFields(map[string]any{"Transport": transport})
+	if err != nil {
+		return 0, fmt.Errorf("selecting and updating messages in queue: %v", err)
+	}
+	queuekick()
+	return n, nil
+}
+
+// Fail marks matching messages as failed for delivery and delivers DSNs to the sender.
+func Fail(ctx context.Context, log mlog.Log, f Filter) (affected int, err error) {
+	err = DB.Write(ctx, func(tx *bstore.Tx) error {
+		q := bstore.QueryTx[Msg](tx)
+		if err := f.apply(q); err != nil {
+			return err
+		}
+		var msgs []Msg
+		q.Gather(&msgs)
+		n, err := q.Delete()
+		if err != nil {
+			return fmt.Errorf("selecting and deleting messages from queue: %v", err)
+		}
+
+		var remoteMTA dsn.NameIP
+		for _, m := range msgs {
+			if m.LastAttempt == nil {
+				now := time.Now()
+				m.LastAttempt = &now
+			}
+			deliverDSNFailure(ctx, log, m, remoteMTA, "", "delivery canceled by admin", nil)
+		}
+		affected = n
+		return nil
+	})
+	if err != nil {
+		return 0, fmt.Errorf("selecting and updating messages in queue: %v", err)
+	}
+	queuekick()
+	metricHoldUpdate()
+	return affected, nil
+}
+
+// Drop removes matching messages from the queue.
+// Returns number of messages removed.
+func Drop(ctx context.Context, log mlog.Log, f Filter) (affected int, err error) {
+	q := bstore.QueryDB[Msg](ctx, DB)
+	if err := f.apply(q); err != nil {
+		return 0, err
 	}
 	var msgs []Msg
 	q.Gather(&msgs)
@@ -439,19 +681,20 @@ func Drop(ctx context.Context, log mlog.Log, ID int64, toDomain string, recipien
 			log.Errorx("removing queue message from file system", err, slog.Int64("queuemsgid", m.ID), slog.String("path", p))
 		}
 	}
+	queuekick()
+	metricHoldUpdate()
 	return n, nil
 }
 
-// SaveRequireTLS updates the RequireTLS field of the message with id.
-func SaveRequireTLS(ctx context.Context, id int64, requireTLS *bool) error {
-	return DB.Write(ctx, func(tx *bstore.Tx) error {
-		m := Msg{ID: id}
-		if err := tx.Get(&m); err != nil {
-			return fmt.Errorf("get message: %w", err)
-		}
-		m.RequireTLS = requireTLS
-		return tx.Update(&m)
-	})
+// RequireTLSSet updates the RequireTLS field of matching messages.
+func RequireTLSSet(ctx context.Context, f Filter, requireTLS *bool) (affected int, err error) {
+	q := bstore.QueryDB[Msg](ctx, DB)
+	if err := f.apply(q); err != nil {
+		return 0, err
+	}
+	n, err := q.UpdateFields(map[string]any{"RequireTLS": requireTLS})
+	queuekick()
+	return n, err
 }
 
 type ReadReaderAtCloser interface {
@@ -522,6 +765,7 @@ func nextWork(ctx context.Context, log mlog.Log, busyDomains map[string]struct{}
 		}
 		q.FilterNotEqual("RecipientDomainStr", doms...)
 	}
+	q.FilterEqual("Hold", false)
 	q.SortAsc("NextAttempt")
 	q.Limit(1)
 	qm, err := q.Get()
@@ -537,6 +781,7 @@ func nextWork(ctx context.Context, log mlog.Log, busyDomains map[string]struct{}
 func launchWork(log mlog.Log, resolver dns.Resolver, busyDomains map[string]struct{}) int {
 	q := bstore.QueryDB[Msg](mox.Shutdown, DB)
 	q.FilterLessEqual("NextAttempt", time.Now())
+	q.FilterEqual("Hold", false)
 	q.SortAsc("NextAttempt")
 	q.Limit(maxConcurrentDeliveries)
 	if len(busyDomains) > 0 {
@@ -679,6 +924,7 @@ func deliver(log mlog.Log, resolver dns.Resolver, m Msg) {
 			q.FilterNonzero(Msg{BaseID: m.BaseID, RecipientDomainStr: m.RecipientDomainStr, Attempts: m.Attempts - 1})
 			q.FilterNotEqual("ID", m.ID)
 			q.FilterLessEqual("NextAttempt", origNextAttempt)
+			q.FilterEqual("Hold", false)
 			err := q.ForEach(func(xm Msg) error {
 				mrtls := m.RequireTLS != nil
 				xmrtls := xm.RequireTLS != nil

queue/queue_test.go

@@ -104,7 +104,20 @@ func TestQueue(t *testing.T) {
 	err := Init()
 	tcheck(t, err, "queue init")
 
-	msgs, err := List(ctxbg)
+	idfilter := func(msgID int64) Filter {
+		return Filter{IDs: []int64{msgID}}
+	}
+
+	kick := func(expn int, id int64) {
+		t.Helper()
+		n, err := NextAttemptSet(ctxbg, idfilter(id), time.Now())
+		tcheck(t, err, "kick queue")
+		if n != expn {
+			t.Fatalf("kick changed %d messages, expected %d", n, expn)
+		}
+	}
+
+	msgs, err := List(ctxbg, Filter{})
 	tcheck(t, err, "listing messages in queue")
 	if len(msgs) != 0 {
 		t.Fatalf("got %d messages in queue, expected 0", len(msgs))
@@ -125,16 +138,26 @@ func TestQueue(t *testing.T) {
 	err = Add(ctxbg, pkglog, "mjl", mf, qm)
 	tcheck(t, err, "add message to queue for delivery")
 
-	msgs, err = List(ctxbg)
+	qm = MakeMsg(path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil, time.Now())
+	err = Add(ctxbg, pkglog, "mjl", mf, qm)
+	tcheck(t, err, "add message to queue for delivery")
+
+	msgs, err = List(ctxbg, Filter{})
 	tcheck(t, err, "listing queue")
-	if len(msgs) != 2 {
+	if len(msgs) != 3 {
 		t.Fatalf("got msgs %v, expected 1", msgs)
 	}
+
+	yes := true
+	n, err := RequireTLSSet(ctxbg, Filter{IDs: []int64{msgs[2].ID}}, &yes)
+	tcheck(t, err, "requiretlsset")
+	tcompare(t, n, 1)
+
 	msg := msgs[0]
 	if msg.Attempts != 0 {
 		t.Fatalf("msg attempts %d, expected 0", msg.Attempts)
 	}
-	n, err := Drop(ctxbg, pkglog, msgs[1].ID, "", "")
+	n, err = Drop(ctxbg, pkglog, Filter{IDs: []int64{msgs[1].ID}})
 	tcheck(t, err, "drop")
 	if n != 1 {
 		t.Fatalf("dropped %d, expected 1", n)
@@ -143,6 +166,48 @@ func TestQueue(t *testing.T) {
 		t.Fatalf("dropped message not removed from file system")
 	}
 
+	// Fail a message, check the account has a message afterwards, the DSN.
+	n, err = bstore.QueryDB[store.Message](ctxbg, acc.DB).Count()
+	tcheck(t, err, "count messages in account")
+	tcompare(t, n, 0)
+	n, err = Fail(ctxbg, pkglog, Filter{IDs: []int64{msgs[2].ID}})
+	tcheck(t, err, "fail")
+	if n != 1 {
+		t.Fatalf("failed %d, expected 1", n)
+	}
+	n, err = bstore.QueryDB[store.Message](ctxbg, acc.DB).Count()
+	tcheck(t, err, "count messages in account")
+	tcompare(t, n, 1)
+
+	// Check filter through various List calls. Other code uses the same filtering function.
+	filter := func(f Filter, expn int) {
+		t.Helper()
+		l, err := List(ctxbg, f)
+		tcheck(t, err, "list messages")
+		tcompare(t, len(l), expn)
+	}
+	filter(Filter{}, 1)
+	filter(Filter{Account: "mjl"}, 1)
+	filter(Filter{Account: "bogus"}, 0)
+	filter(Filter{IDs: []int64{msgs[0].ID}}, 1)
+	filter(Filter{IDs: []int64{msgs[2].ID}}, 0)     // Removed.
+	filter(Filter{IDs: []int64{msgs[2].ID + 1}}, 0) // Never existed.
+	filter(Filter{From: "mjl@"}, 1)
+	filter(Filter{From: "bogus@"}, 0)
+	filter(Filter{To: "mjl@"}, 1)
+	filter(Filter{To: "bogus@"}, 0)
+	filter(Filter{Hold: &yes}, 0)
+	no := false
+	filter(Filter{Hold: &no}, 1)
+	filter(Filter{Submitted: "<now"}, 1)
+	filter(Filter{Submitted: ">now"}, 0)
+	filter(Filter{NextAttempt: "<1m"}, 1)
+	filter(Filter{NextAttempt: ">1m"}, 0)
+	var empty string
+	bogus := "bogus"
+	filter(Filter{Transport: &empty}, 1)
+	filter(Filter{Transport: &bogus}, 0)
+
 	next := nextWork(ctxbg, pkglog, nil)
 	if next > 0 {
 		t.Fatalf("nextWork in %s, should be now", next)
@@ -217,12 +282,13 @@ func TestQueue(t *testing.T) {
 		t.Fatalf("message mismatch, got %q, expected %q", string(msgbuf), testmsg)
 	}
 
-	n, err = Kick(ctxbg, msg.ID+1, "", "", nil)
+	// Reduce by more than first attempt interval of 7.5 minutes.
+	n, err = NextAttemptAdd(ctxbg, idfilter(msg.ID+1), -10*time.Minute)
 	tcheck(t, err, "kick")
 	if n != 0 {
 		t.Fatalf("kick %d, expected 0", n)
 	}
-	n, err = Kick(ctxbg, msg.ID, "", "", nil)
+	n, err = NextAttemptAdd(ctxbg, idfilter(msg.ID), -10*time.Minute)
 	tcheck(t, err, "kick")
 	if n != 1 {
 		t.Fatalf("kicked %d, expected 1", n)
@@ -485,7 +551,7 @@ func TestQueue(t *testing.T) {
 				case <-smtpdone:
 					i := 0
 					for {
-						xmsgs, err := List(ctxbg)
+						xmsgs, err := List(ctxbg, Filter{})
 						tcheck(t, err, "list queue")
 						if len(xmsgs) == 0 {
 							ninbox, err := bstore.QueryDB[store.Message](ctxbg, acc.DB).FilterNonzero(store.Message{MailboxID: inbox.ID}).Count()
@@ -595,10 +661,10 @@ func TestQueue(t *testing.T) {
 	err = Add(ctxbg, pkglog, "mjl", mf, qml...)
 	tcheck(t, err, "add message to queue for delivery")
 	transportSubmitTLS := "submittls"
-	n, err = Kick(ctxbg, qml[0].ID, "", "", &transportSubmitTLS)
-	tcheck(t, err, "kick queue")
+	n, err = TransportSet(ctxbg, Filter{IDs: []int64{qml[0].ID}}, transportSubmitTLS)
+	tcheck(t, err, "set transport")
 	if n != 1 {
-		t.Fatalf("kick changed %d messages, expected 1", n)
+		t.Fatalf("TransportSet changed %d messages, expected 1", n)
 	}
 	// Make fake cert, and make it trusted.
 	cert := fakeCert(t, "submission.example", false)
@@ -643,12 +709,12 @@ func TestQueue(t *testing.T) {
 	qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<socks@localhost>", nil, nil, time.Now())}
 	err = Add(ctxbg, pkglog, "mjl", mf, qml...)
 	tcheck(t, err, "add message to queue for delivery")
-	transportSocks := "socks"
-	n, err = Kick(ctxbg, qml[0].ID, "", "", &transportSocks)
-	tcheck(t, err, "kick queue")
+	n, err = TransportSet(ctxbg, idfilter(qml[0].ID), "socks")
+	tcheck(t, err, "TransportSet")
 	if n != 1 {
-		t.Fatalf("kick changed %d messages, expected 1", n)
+		t.Fatalf("TransportSet changed %d messages, expected 1", n)
 	}
+	kick(1, qml[0].ID)
 	wasNetDialer = testDeliver(fakeSMTPServer)
 	if wasNetDialer {
 		t.Fatalf("expected non-net.Dialer as dialer") // SOCKS5 dialer is a private type, we cannot check for it.
@@ -659,11 +725,7 @@ func TestQueue(t *testing.T) {
 	qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, nil, time.Now())}
 	err = Add(ctxbg, pkglog, "mjl", mf, qml...)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
-	tcheck(t, err, "kick queue")
-	if n != 1 {
-		t.Fatalf("kick changed %d messages, expected 1", n)
-	}
+	kick(1, qml[0].ID)
 	testDeliver(fakeSMTPSTARTTLSServer)
 	checkTLSResults(t, "mox.example", "mox.example", false, addCounts(1, 0, tlsrpt.MakeResult(tlsrpt.NoPolicyFound, mailDomain, fdNotTrusted)))
 	checkTLSResults(t, "mail.mox.example", "mox.example", true, addCounts(1, 0, tlsrpt.MakeResult(tlsrpt.NoPolicyFound, mailHost)))
@@ -673,11 +735,7 @@ func TestQueue(t *testing.T) {
 	qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<badtls@localhost>", nil, nil, time.Now())}
 	err = Add(ctxbg, pkglog, "mjl", mf, qml...)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
-	tcheck(t, err, "kick queue")
-	if n != 1 {
-		t.Fatalf("kick changed %d messages, expected 1", n)
-	}
+	kick(1, qml[0].ID)
 	testDeliver(makeBadFakeSMTPSTARTTLSServer(true))
 	checkTLSResults(t, "mox.example", "mox.example", false, addCounts(0, 1, tlsrpt.MakeResult(tlsrpt.NoPolicyFound, mailDomain, fdBadProtocol)))
 	checkTLSResults(t, "mail.mox.example", "mox.example", true, addCounts(0, 1, tlsrpt.MakeResult(tlsrpt.NoPolicyFound, mailHost, fdBadProtocol)))
@@ -693,11 +751,7 @@ func TestQueue(t *testing.T) {
 	qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<dane@localhost>", nil, nil, time.Now())}
 	err = Add(ctxbg, pkglog, "mjl", mf, qml...)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
-	tcheck(t, err, "kick queue")
-	if n != 1 {
-		t.Fatalf("kick changed %d messages, expected 1", n)
-	}
+	kick(1, qml[0].ID)
 	testDeliver(fakeSMTPSTARTTLSServer)
 	checkTLSResults(t, "mox.example", "mox.example", false, addCounts(1, 0, tlsrpt.MakeResult(tlsrpt.NoPolicyFound, mailDomain, fdNotTrusted)))
 	checkTLSResults(t, "mail.mox.example", "mox.example", true, addCounts(1, 0, tlsrpt.Result{Policy: tlsrpt.TLSAPolicy(resolver.TLSA["_25._tcp.mail.mox.example."], mailHost), FailureDetails: []tlsrpt.FailureDetails{}}))
@@ -710,15 +764,10 @@ func TestQueue(t *testing.T) {
 	tcompare(t, rdt.RequireTLS, true)
 
 	// Add message to be delivered with verified TLS and REQUIRETLS.
-	yes := true
 	qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, &yes, time.Now())}
 	err = Add(ctxbg, pkglog, "mjl", mf, qml...)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
-	tcheck(t, err, "kick queue")
-	if n != 1 {
-		t.Fatalf("kick changed %d messages, expected 1", n)
-	}
+	kick(1, qml[0].ID)
 	testDeliver(fakeSMTPSTARTTLSServer)
 
 	// Check that message is delivered with all unusable DANE records.
@@ -731,11 +780,7 @@ func TestQueue(t *testing.T) {
 	qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<daneunusable@localhost>", nil, nil, time.Now())}
 	err = Add(ctxbg, pkglog, "mjl", mf, qml...)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
-	tcheck(t, err, "kick queue")
-	if n != 1 {
-		t.Fatalf("kick changed %d messages, expected 1", n)
-	}
+	kick(1, qml[0].ID)
 	testDeliver(fakeSMTPSTARTTLSServer)
 	checkTLSResults(t, "mox.example", "mox.example", false, addCounts(1, 0, tlsrpt.MakeResult(tlsrpt.NoPolicyFound, mailDomain, fdNotTrusted)))
 	checkTLSResults(t, "mail.mox.example", "mox.example", true, addCounts(1, 0, tlsrpt.Result{Policy: tlsrpt.TLSAPolicy([]adns.TLSA{}, mailHost), FailureDetails: []tlsrpt.FailureDetails{fdTLSAUnusable}}))
@@ -752,11 +797,7 @@ func TestQueue(t *testing.T) {
 	qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<daneinsecure@localhost>", nil, nil, time.Now())}
 	err = Add(ctxbg, pkglog, "mjl", mf, qml...)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
-	tcheck(t, err, "kick queue")
-	if n != 1 {
-		t.Fatalf("kick changed %d messages, expected 1", n)
-	}
+	kick(1, qml[0].ID)
 	testDeliver(makeBadFakeSMTPSTARTTLSServer(true))
 	resolver.Inauthentic = nil
 	checkTLSResults(t, "mox.example", "mox.example", false, addCounts(0, 1, tlsrpt.MakeResult(tlsrpt.NoPolicyFound, mailDomain, fdBadProtocol)))
@@ -770,37 +811,24 @@ func TestQueue(t *testing.T) {
 	tcompare(t, rdt.RequireTLS, false)
 
 	// Check that message is delivered with TLS-Required: No and non-matching DANE record.
-	no := false
 	qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<tlsrequirednostarttls@localhost>", nil, &no, time.Now())}
 	err = Add(ctxbg, pkglog, "mjl", mf, qml...)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
-	tcheck(t, err, "kick queue")
-	if n != 1 {
-		t.Fatalf("kick changed %d messages, expected 1", n)
-	}
+	kick(1, qml[0].ID)
 	testDeliver(fakeSMTPSTARTTLSServer)
 
 	// Check that message is delivered with TLS-Required: No and bad TLS, falling back to plain text.
 	qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<tlsrequirednoplaintext@localhost>", nil, &no, time.Now())}
 	err = Add(ctxbg, pkglog, "mjl", mf, qml...)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
-	tcheck(t, err, "kick queue")
-	if n != 1 {
-		t.Fatalf("kick changed %d messages, expected 1", n)
-	}
+	kick(1, qml[0].ID)
 	testDeliver(makeBadFakeSMTPSTARTTLSServer(true))
 
 	// Add message with requiretls that fails immediately due to no REQUIRETLS support in all servers.
 	qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<tlsrequiredunsupported@localhost>", nil, &yes, time.Now())}
 	err = Add(ctxbg, pkglog, "mjl", mf, qml...)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
-	tcheck(t, err, "kick queue")
-	if n != 1 {
-		t.Fatalf("kick changed %d messages, expected 1", n)
-	}
+	kick(1, qml[0].ID)
 	testDSN(makeBadFakeSMTPSTARTTLSServer(false))
 
 	// Restore pre-DANE behaviour.
@@ -811,11 +839,7 @@ func TestQueue(t *testing.T) {
 	qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<tlsrequirednopolicy@localhost>", nil, &yes, time.Now())}
 	err = Add(ctxbg, pkglog, "mjl", mf, qml...)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
-	tcheck(t, err, "kick queue")
-	if n != 1 {
-		t.Fatalf("kick changed %d messages, expected 1", n)
-	}
+	kick(1, qml[0].ID)
 	// Based on DNS lookups, there won't be any dialing or SMTP connection.
 	dialed <- struct{}{}
 	testDSN(func(conn net.Conn) {
@@ -827,7 +851,7 @@ func TestQueue(t *testing.T) {
 	err = Add(ctxbg, pkglog, "mjl", mf, qm)
 	tcheck(t, err, "add message to queue for delivery")
 
-	msgs, err = List(ctxbg)
+	msgs, err = List(ctxbg, Filter{})
 	tcheck(t, err, "list queue")
 	if len(msgs) != 1 {
 		t.Fatalf("queue has %d messages, expected 1", len(msgs))
@@ -1010,6 +1034,14 @@ func TestQueueStart(t *testing.T) {
 		}
 	}
 
+	// HoldRule prevents delivery.
+	hr, err := HoldRuleAdd(ctxbg, pkglog, HoldRule{})
+	tcheck(t, err, "add hold rule")
+
+	hrl, err := HoldRuleList(ctxbg)
+	tcheck(t, err, "listing hold rules")
+	tcompare(t, hrl, []HoldRule{hr})
+
 	path := smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "mox.example"}}}
 	mf := prepareFile(t)
 	defer os.Remove(mf.Name())
@@ -1017,20 +1049,50 @@ func TestQueueStart(t *testing.T) {
 	qm := MakeMsg(path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil, time.Now())
 	err = Add(ctxbg, pkglog, "mjl", mf, qm)
 	tcheck(t, err, "add message to queue for delivery")
+	checkDialed(false) // No delivery attempt yet.
+
+	n, err := Count(ctxbg)
+	tcheck(t, err, "count messages in queue")
+	tcompare(t, n, 1)
+
+	// Take message off hold.
+	n, err = HoldSet(ctxbg, Filter{}, false)
+	tcheck(t, err, "taking message off hold")
+	tcompare(t, n, 1)
 	checkDialed(true)
 
+	// Remove hold rule.
+	err = HoldRuleRemove(ctxbg, pkglog, hr.ID)
+	tcheck(t, err, "removing hold rule")
+	// Check it is gone.
+	hrl, err = HoldRuleList(ctxbg)
+	tcheck(t, err, "listing hold rules")
+	tcompare(t, len(hrl), 0)
+
 	// Don't change message nextattempt time, but kick queue. Message should not be delivered.
 	queuekick()
 	checkDialed(false)
 
-	// Kick for real, should see another attempt.
-	n, err := Kick(ctxbg, 0, "mox.example", "", nil)
+	// Set new next attempt, should see another attempt.
+	n, err = NextAttemptSet(ctxbg, Filter{From: "@mox.example"}, time.Now())
 	tcheck(t, err, "kick queue")
 	if n != 1 {
 		t.Fatalf("kick changed %d messages, expected 1", n)
 	}
 	checkDialed(true)
 	time.Sleep(100 * time.Millisecond) // Racy... we won't get notified when work is done...
+
+	// Submit another, should be delivered immediately without HoldRule.
+	path = smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "mox.example"}}}
+	mf = prepareFile(t)
+	defer os.Remove(mf.Name())
+	defer mf.Close()
+	qm = MakeMsg(path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil, time.Now())
+	err = Add(ctxbg, pkglog, "mjl", mf, qm)
+	tcheck(t, err, "add message to queue for delivery")
+	checkDialed(true) // Immediate.
+
+	time.Sleep(100 * time.Millisecond) // Racy... give time to finish.
 }
 
 // Just a cert that appears valid.