infrastructure/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2025-07-12 17:44:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

update to latest bstore (with support for an index on a []string: Message.DKIMDomains), and cyclic data types (to be used for Message.Part soon); also adds a context.Context to database operations.

Browse Source
This commit is contained in:
Mechiel Lukkien
2023-05-22 14:40:36 +02:00
parent f6ed860ccb
commit e81930ba20
58 changed files with 1970 additions and 1035 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
store/account.go
View File

@ -278,7 +278,7 @@ type Message struct {
MsgFromValidation Validation // Desirable validations: Strict, DMARC, Relaxed. Will not be just Pass.
// todo: needs an "in" index, which bstore does not yet support. for performance while checking reputation.
DKIMDomains []string // Domains with verified DKIM signatures. Unicode string.
DKIMDomains []string `bstore:"index DKIMDomains+Received"` // Domains with verified DKIM signatures. Unicode string.
// Value of Message-Id header. Only set for messages that were
// delivered to the rejects mailbox. For ensuring such messages are
@ -455,7 +455,7 @@ func openAccount(name string) (a *Account, rerr error) {
os.MkdirAll(dir, 0770)
}
db, err := bstore.Open(dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{})
db, err := bstore.Open(context.TODO(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{})
if err != nil {
return nil, err
}
@ -484,7 +484,7 @@ func openAccount(name string) (a *Account, rerr error) {
}
func initAccount(db *bstore.DB) error {
return db.Write(func(tx *bstore.Tx) error {
return db.Write(context.TODO(), func(tx *bstore.Tx) error {
uidvalidity := InitialUIDValidity()
mailboxes := InitialMailboxes
@ -700,7 +700,7 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
if !notrain && m.NeedsTraining() {
l := []Message{*m}
if err := a.RetrainMessages(log, tx, l, false); err != nil {
if err := a.RetrainMessages(context.TODO(), log, tx, l, false); err != nil {
return fmt.Errorf("training junkfilter: %w", err)
}
*m = l[0]
@ -739,7 +739,7 @@ func (a *Account) SetPassword(password string) error {
return fmt.Errorf("generating password hash: %w", err)
}
err = a.DB.Write(func(tx *bstore.Tx) error {
err = a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
if _, err := bstore.QueryTx[Password](tx).Delete(); err != nil {
return fmt.Errorf("deleting existing password: %v", err)
}
@ -793,7 +793,7 @@ func (a *Account) SetPassword(password string) error {
// Subjectpass returns the signing key for use with subjectpass for the given
// email address with canonical localpart.
func (a *Account) Subjectpass(email string) (key string, err error) {
return key, a.DB.Write(func(tx *bstore.Tx) error {
return key, a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
v := Subjectpass{Email: email}
err := tx.Get(&v)
if err == nil {
@ -1036,7 +1036,7 @@ func (a *Account) Deliver(log *mlog.Log, dest config.Destination, m *Message, ms
// Message delivery and possible mailbox creation are broadcasted.
func (a *Account) DeliverMailbox(log *mlog.Log, mailbox string, m *Message, msgFile *os.File, consumeFile bool) error {
var changes []Change
err := a.DB.Write(func(tx *bstore.Tx) error {
err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
mb, chl, err := a.MailboxEnsure(tx, mailbox, true)
if err != nil {
return fmt.Errorf("ensuring mailbox: %w", err)
@ -1075,7 +1075,7 @@ func (a *Account) TidyRejectsMailbox(log *mlog.Log, rejectsMailbox string) (hasS
}
}()
err := a.DB.Write(func(tx *bstore.Tx) error {
err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
mb, err := a.MailboxFind(tx, rejectsMailbox)
if err != nil {
return fmt.Errorf("finding mailbox: %w", err)
@ -1096,7 +1096,7 @@ func (a *Account) TidyRejectsMailbox(log *mlog.Log, rejectsMailbox string) (hasS
return fmt.Errorf("listing old messages: %w", err)
}
changes, err = a.removeMessages(log, tx, mb, remove)
changes, err = a.removeMessages(context.TODO(), log, tx, mb, remove)
if err != nil {
return fmt.Errorf("removing messages: %w", err)
}
@ -1125,7 +1125,7 @@ func (a *Account) TidyRejectsMailbox(log *mlog.Log, rejectsMailbox string) (hasS
return hasSpace, nil
}
func (a *Account) removeMessages(log *mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) {
func (a *Account) removeMessages(ctx context.Context, log *mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) {
if len(l) == 0 {
return nil, nil
}
@ -1158,7 +1158,7 @@ func (a *Account) removeMessages(log *mlog.Log, tx *bstore.Tx, mb *Mailbox, l []
deleted[i].Junk = false
deleted[i].Notjunk = false
}
if err := a.RetrainMessages(log, tx, deleted, true); err != nil {
if err := a.RetrainMessages(ctx, log, tx, deleted, true); err != nil {
return nil, fmt.Errorf("training deleted messages: %w", err)
}
@ -1184,7 +1184,7 @@ func (a *Account) RejectsRemove(log *mlog.Log, rejectsMailbox, messageID string)
}
}()
err := a.DB.Write(func(tx *bstore.Tx) error {
err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
mb, err := a.MailboxFind(tx, rejectsMailbox)
if err != nil {
return fmt.Errorf("finding mailbox: %w", err)
@ -1200,7 +1200,7 @@ func (a *Account) RejectsRemove(log *mlog.Log, rejectsMailbox, messageID string)
return fmt.Errorf("listing messages to remove: %w", err)
}
changes, err = a.removeMessages(log, tx, mb, remove)
changes, err = a.removeMessages(context.TODO(), log, tx, mb, remove)
if err != nil {
return fmt.Errorf("removing messages: %w", err)
}
@ -1262,7 +1262,7 @@ func OpenEmailAuth(email string, password string) (acc *Account, rerr error) {
}
}()
pw, err := bstore.QueryDB[Password](acc.DB).Get()
pw, err := bstore.QueryDB[Password](context.TODO(), acc.DB).Get()
if err != nil {
if err == bstore.ErrAbsent {
return acc, ErrUnknownCredentials

25
store/account_test.go
View File

@ -1,6 +1,7 @@
package store
import (
"context"
"os"
"regexp"
"strings"
@ -16,6 +17,8 @@ import (
"github.com/mjl-/mox/mox-"
)
var ctxbg = context.Background()
func tcheck(t *testing.T, err error, msg string) {
t.Helper()
if err != nil {
@ -66,7 +69,7 @@ func TestMailbox(t *testing.T) {
err := acc.Deliver(xlog, conf.Destinations["mjl"], &m, msgFile, false)
tcheck(t, err, "deliver without consume")
err = acc.DB.Write(func(tx *bstore.Tx) error {
err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
var err error
mbsent, err = bstore.QueryTx[Mailbox](tx).FilterNonzero(Mailbox{Name: "Sent"}).Get()
tcheck(t, err, "sent mailbox")
@ -89,10 +92,10 @@ func TestMailbox(t *testing.T) {
err = acc.Deliver(xlog, conf.Destinations["mjl"], &mconsumed, msgFile, true)
tcheck(t, err, "deliver with consume")
err = acc.DB.Write(func(tx *bstore.Tx) error {
err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
m.Junk = true
l := []Message{m}
err = acc.RetrainMessages(log, tx, l, false)
err = acc.RetrainMessages(ctxbg, log, tx, l, false)
tcheck(t, err, "train as junk")
m = l[0]
return nil
@ -102,18 +105,18 @@ func TestMailbox(t *testing.T) {
m.Junk = false
m.Notjunk = true
jf, _, err := acc.OpenJunkFilter(log)
jf, _, err := acc.OpenJunkFilter(ctxbg, log)
tcheck(t, err, "open junk filter")
err = acc.DB.Write(func(tx *bstore.Tx) error {
return acc.RetrainMessage(log, tx, jf, &m, false)
err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
return acc.RetrainMessage(ctxbg, log, tx, jf, &m, false)
})
tcheck(t, err, "retraining as non-junk")
err = jf.Close()
tcheck(t, err, "close junk filter")
m.Notjunk = false
err = acc.DB.Write(func(tx *bstore.Tx) error {
return acc.RetrainMessages(log, tx, []Message{m}, false)
err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
return acc.RetrainMessages(ctxbg, log, tx, []Message{m}, false)
})
tcheck(t, err, "untraining non-junk")
@ -134,18 +137,18 @@ func TestMailbox(t *testing.T) {
}
acc.WithWLock(func() {
err := acc.DB.Write(func(tx *bstore.Tx) error {
err := acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
_, _, err := acc.MailboxEnsure(tx, "Testbox", true)
return err
})
tcheck(t, err, "ensure mailbox exists")
err = acc.DB.Read(func(tx *bstore.Tx) error {
err = acc.DB.Read(ctxbg, func(tx *bstore.Tx) error {
_, _, err := acc.MailboxEnsure(tx, "Testbox", true)
return err
})
tcheck(t, err, "ensure mailbox exists")
err = acc.DB.Write(func(tx *bstore.Tx) error {
err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
_, _, err := acc.MailboxEnsure(tx, "Testbox2", false)
tcheck(t, err, "create mailbox")

5
store/export.go
View File

@ -5,6 +5,7 @@ import (
"archive/zip"
"bufio"
"bytes"
"context"
"fmt"
"io"
"os"
@ -104,13 +105,13 @@ func (a DirArchiver) Close() error {
// Some errors are not fatal and result in skipped messages. In that happens, a
// file "errors.txt" is added to the archive describing the errors. The goal is to
// let users export (hopefully) most messages even in the face of errors.
func ExportMessages(log *mlog.Log, db *bstore.DB, accountDir string, archiver Archiver, maildir bool, mailboxOpt string) error {
func ExportMessages(ctx context.Context, log *mlog.Log, db *bstore.DB, accountDir string, archiver Archiver, maildir bool, mailboxOpt string) error {
// todo optimize: should prepare next file to add to archive (can be an mbox with many messages) while writing a file to the archive (which typically compresses, which takes time).
// Start transaction without closure, we are going to close it early, but don't
// want to deal with declaring many variables now to be able to assign them in a
// closure and use them afterwards.
tx, err := db.Begin(false)
tx, err := db.Begin(ctx, false)
if err != nil {
return fmt.Errorf("transaction: %v", err)
}

2
store/export_test.go
View File

@ -49,7 +49,7 @@ func TestExport(t *testing.T) {
archive := func(archiver Archiver, maildir bool) {
t.Helper()
err = ExportMessages(log, acc.DB, acc.Dir, archiver, maildir, "")
err = ExportMessages(ctxbg, log, acc.DB, acc.Dir, archiver, maildir, "")
tcheck(t, err, "export messages")
err = archiver.Close()
tcheck(t, err, "archiver close")

23
store/train.go
View File

@ -1,6 +1,7 @@
package store
import (
"context"
"errors"
"fmt"
"os"
@ -21,7 +22,7 @@ var ErrNoJunkFilter = errors.New("junkfilter: not configured")
// If the account does not have a junk filter enabled, ErrNotConfigured is returned.
// Do not forget to save the filter after modifying, and to always close the filter when done.
// An empty filter is initialized on first access of the filter.
func (a *Account) OpenJunkFilter(log *mlog.Log) (*junk.Filter, *config.JunkFilter, error) {
func (a *Account) OpenJunkFilter(ctx context.Context, log *mlog.Log) (*junk.Filter, *config.JunkFilter, error) {
conf, ok := mox.Conf.Account(a.Name)
if !ok {
return nil, nil, ErrAccountUnknown
@ -36,16 +37,16 @@ func (a *Account) OpenJunkFilter(log *mlog.Log) (*junk.Filter, *config.JunkFilte
bloomPath := filepath.Join(basePath, a.Name, "junkfilter.bloom")
if _, xerr := os.Stat(dbPath); xerr != nil && os.IsNotExist(xerr) {
f, err := junk.NewFilter(log, jf.Params, dbPath, bloomPath)
f, err := junk.NewFilter(ctx, log, jf.Params, dbPath, bloomPath)
return f, jf, err
}
f, err := junk.OpenFilter(log, jf.Params, dbPath, bloomPath, false)
f, err := junk.OpenFilter(ctx, log, jf.Params, dbPath, bloomPath, false)
return f, jf, err
}
// RetrainMessages (un)trains messages, if relevant given their flags. Updates
// m.TrainedJunk after retraining.
func (a *Account) RetrainMessages(log *mlog.Log, tx *bstore.Tx, msgs []Message, absentOK bool) (rerr error) {
func (a *Account) RetrainMessages(ctx context.Context, log *mlog.Log, tx *bstore.Tx, msgs []Message, absentOK bool) (rerr error) {
if len(msgs) == 0 {
return nil
}
@ -60,7 +61,7 @@ func (a *Account) RetrainMessages(log *mlog.Log, tx *bstore.Tx, msgs []Message,
// Lazy open the junk filter.
if jf == nil {
var err error
jf, _, err = a.OpenJunkFilter(log)
jf, _, err = a.OpenJunkFilter(ctx, log)
if err != nil && errors.Is(err, ErrNoJunkFilter) {
// No junk filter configured. Nothing more to do.
return nil
@ -76,7 +77,7 @@ func (a *Account) RetrainMessages(log *mlog.Log, tx *bstore.Tx, msgs []Message,
}
}()
}
if err := a.RetrainMessage(log, tx, jf, &msgs[i], absentOK); err != nil {
if err := a.RetrainMessage(ctx, log, tx, jf, &msgs[i], absentOK); err != nil {
return err
}
}
@ -85,7 +86,7 @@ func (a *Account) RetrainMessages(log *mlog.Log, tx *bstore.Tx, msgs []Message,
// RetrainMessage untrains and/or trains a message, if relevant given m.TrainedJunk
// and m.Junk/m.Notjunk. Updates m.TrainedJunk after retraining.
func (a *Account) RetrainMessage(log *mlog.Log, tx *bstore.Tx, jf *junk.Filter, m *Message, absentOK bool) error {
func (a *Account) RetrainMessage(ctx context.Context, log *mlog.Log, tx *bstore.Tx, jf *junk.Filter, m *Message, absentOK bool) error {
untrain := m.TrainedJunk != nil
untrainJunk := untrain && *m.TrainedJunk
train := m.Junk || m.Notjunk && !(m.Junk && m.Notjunk)
@ -116,14 +117,14 @@ func (a *Account) RetrainMessage(log *mlog.Log, tx *bstore.Tx, jf *junk.Filter,
}
if untrain {
err := jf.Untrain(!untrainJunk, words)
err := jf.Untrain(ctx, !untrainJunk, words)
if err != nil {
return err
}
m.TrainedJunk = nil
}
if train {
err := jf.Train(!trainJunk, words)
err := jf.Train(ctx, !trainJunk, words)
if err != nil {
return err
}
@ -137,7 +138,7 @@ func (a *Account) RetrainMessage(log *mlog.Log, tx *bstore.Tx, jf *junk.Filter,
// TrainMessage trains the junk filter based on the current m.Junk/m.Notjunk flags,
// disregarding m.TrainedJunk and not updating that field.
func (a *Account) TrainMessage(log *mlog.Log, jf *junk.Filter, m Message) (bool, error) {
func (a *Account) TrainMessage(ctx context.Context, log *mlog.Log, jf *junk.Filter, m Message) (bool, error) {
if !m.Junk && !m.Notjunk || (m.Junk && m.Notjunk) {
return false, nil
}
@ -160,5 +161,5 @@ func (a *Account) TrainMessage(log *mlog.Log, jf *junk.Filter, m Message) (bool,
return false, nil
}
return true, jf.Train(m.Notjunk, words)
return true, jf.Train(ctx, m.Notjunk, words)
}