infrastructure/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2025-07-12 17:44:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

add debug logging about bstore db schema upgrades

Browse Source 
bstore was updated to v0.0.6 to add this logging.
this simplifies some of the db-handling code in mtastsdb,tlsrptdb,dmarcdb. we
now call the package-level Init() and Close() in all tests properly.
This commit is contained in:
Mechiel Lukkien
2024-05-10 14:44:37 +02:00
parent 3e4cce826e
commit bf8cfd9724
31 changed files with 298 additions and 428 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
dmarcdb/dmarcdb.go
View File

@ -11,6 +11,15 @@
package dmarcdb
import (
"context"
"fmt"
"os"
"path/filepath"
"time"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
)
@ -19,11 +28,49 @@ import (
// The incoming reports and evaluations for outgoing reports are in separate
// databases for simpler file-based handling of the databases.
func Init() error {
if _, err := reportsDB(mox.Shutdown); err != nil {
return err
if ReportsDB != nil || EvalDB != nil {
return fmt.Errorf("already initialized")
}
if _, err := evalDB(mox.Shutdown); err != nil {
return err
log := mlog.New("dmarcdb", nil)
var err error
ReportsDB, err = openReportsDB(mox.Shutdown, log)
if err != nil {
return fmt.Errorf("open reports db: %v", err)
}
EvalDB, err = openEvalDB(mox.Shutdown, log)
if err != nil {
return fmt.Errorf("open eval db: %v", err)
}
return nil
}
func Close() error {
if err := ReportsDB.Close(); err != nil {
return fmt.Errorf("closing reports db: %w", err)
}
ReportsDB = nil
if err := EvalDB.Close(); err != nil {
return fmt.Errorf("closing eval db: %w", err)
}
EvalDB = nil
return nil
}
func openReportsDB(ctx context.Context, log mlog.Log) (*bstore.DB, error) {
p := mox.DataDirPath("dmarcrpt.db")
os.MkdirAll(filepath.Dir(p), 0770)
opts := bstore.Options{Timeout: 5 * time.Second, Perm: 0660, RegisterLogger: log.Logger}
return bstore.Open(ctx, p, &opts, ReportsDBTypes...)
}
func openEvalDB(ctx context.Context, log mlog.Log) (*bstore.DB, error) {
p := mox.DataDirPath("dmarceval.db")
os.MkdirAll(filepath.Dir(p), 0770)
opts := bstore.Options{Timeout: 5 * time.Second, Perm: 0660, RegisterLogger: log.Logger}
return bstore.Open(ctx, p, &opts, EvalDBTypes...)
}

96
dmarcdb/eval.go
View File

@ -15,7 +15,6 @@ import (
"net/textproto"
"net/url"
"os"
"path/filepath"
"runtime/debug"
"slices"
"sort"
@ -66,8 +65,7 @@ var (
// Exported for backups. For incoming deliveries the SMTP server adds evaluations
// to the database. Every hour, a goroutine wakes up that gathers evaluations from
// the last hour(s), sends a report, and removes the evaluations from the database.
EvalDB *bstore.DB
evalMutex sync.Mutex
EvalDB *bstore.DB
)
// Evaluation is the result of an evaluation of a DMARC policy, to be included
@ -162,21 +160,6 @@ func (e Evaluation) ReportRecord(count int) dmarcrpt.ReportRecord {
}
}
func evalDB(ctx context.Context) (rdb *bstore.DB, rerr error) {
evalMutex.Lock()
defer evalMutex.Unlock()
if EvalDB == nil {
p := mox.DataDirPath("dmarceval.db")
os.MkdirAll(filepath.Dir(p), 0770)
db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, EvalDBTypes...)
if err != nil {
return nil, err
}
EvalDB = db
}
return EvalDB, nil
}
var intervalOpts = []int{24, 12, 8, 6, 4, 3, 2}
func intervalHours(seconds int) int {
@ -197,23 +180,13 @@ func intervalHours(seconds int) int {
func AddEvaluation(ctx context.Context, aggregateReportingIntervalSeconds int, e *Evaluation) error {
e.IntervalHours = intervalHours(aggregateReportingIntervalSeconds)
db, err := evalDB(ctx)
if err != nil {
return err
}
e.ID = 0
return db.Insert(ctx, e)
return EvalDB.Insert(ctx, e)
}
// Evaluations returns all evaluations in the database.
func Evaluations(ctx context.Context) ([]Evaluation, error) {
db, err := evalDB(ctx)
if err != nil {
return nil, err
}
q := bstore.QueryDB[Evaluation](ctx, db)
q := bstore.QueryDB[Evaluation](ctx, EvalDB)
q.SortAsc("Evaluated")
return q.List()
}
@ -229,14 +202,9 @@ type EvaluationStat struct {
// EvaluationStats returns evaluation counts and report-sending status per domain.
func EvaluationStats(ctx context.Context) (map[string]EvaluationStat, error) {
db, err := evalDB(ctx)
if err != nil {
return nil, err
}
r := map[string]EvaluationStat{}
err = bstore.QueryDB[Evaluation](ctx, db).ForEach(func(e Evaluation) error {
err := bstore.QueryDB[Evaluation](ctx, EvalDB).ForEach(func(e Evaluation) error {
if stat, ok := r[e.PolicyDomain]; ok {
if !slices.Contains(stat.Dispositions, string(e.Disposition)) {
stat.Dispositions = append(stat.Dispositions, string(e.Disposition))
@ -263,12 +231,7 @@ func EvaluationStats(ctx context.Context) (map[string]EvaluationStat, error) {
// EvaluationsDomain returns all evaluations for a domain.
func EvaluationsDomain(ctx context.Context, domain dns.Domain) ([]Evaluation, error) {
db, err := evalDB(ctx)
if err != nil {
return nil, err
}
q := bstore.QueryDB[Evaluation](ctx, db)
q := bstore.QueryDB[Evaluation](ctx, EvalDB)
q.FilterNonzero(Evaluation{PolicyDomain: domain.Name()})
q.SortAsc("Evaluated")
return q.List()
@ -277,14 +240,9 @@ func EvaluationsDomain(ctx context.Context, domain dns.Domain) ([]Evaluation, er
// RemoveEvaluationsDomain removes evaluations for domain so they won't be sent in
// an aggregate report.
func RemoveEvaluationsDomain(ctx context.Context, domain dns.Domain) error {
db, err := evalDB(ctx)
if err != nil {
return err
}
q := bstore.QueryDB[Evaluation](ctx, db)
q := bstore.QueryDB[Evaluation](ctx, EvalDB)
q.FilterNonzero(Evaluation{PolicyDomain: domain.Name()})
_, err = q.Delete()
_, err := q.Delete()
return err
}
@ -318,12 +276,6 @@ func Start(resolver dns.Resolver) {
ctx := mox.Shutdown
db, err := evalDB(ctx)
if err != nil {
log.Errorx("opening dmarc evaluations database for sending dmarc aggregate reports, not sending reports", err)
return
}
for {
now := time.Now()
nextEnd := nextWholeHour(now)
@ -355,12 +307,12 @@ func Start(resolver dns.Resolver) {
// 24 hour interval). They should have been processed by now. We may have kept them
// during temporary errors, but persistent temporary errors shouldn't fill up our
// database. This also cleans up evaluations that were all optional for a domain.
_, err := bstore.QueryDB[Evaluation](ctx, db).FilterLess("Evaluated", nextEnd.Add(-48*time.Hour)).Delete()
_, err := bstore.QueryDB[Evaluation](ctx, EvalDB).FilterLess("Evaluated", nextEnd.Add(-48*time.Hour)).Delete()
log.Check(err, "removing stale dmarc evaluations from database")
clog := log.WithCid(mox.Cid())
clog.Info("sending dmarc aggregate reports", slog.Time("end", nextEnd.UTC()), slog.Any("intervals", intervals))
if err := sendReports(ctx, clog, resolver, db, nextEnd, intervals); err != nil {
if err := sendReports(ctx, clog, resolver, EvalDB, nextEnd, intervals); err != nil {
clog.Errorx("sending dmarc aggregate reports", err)
metricReportError.Inc()
} else {
@ -1091,46 +1043,26 @@ func dkimSign(ctx context.Context, log mlog.Log, fromAddr smtp.Address, smtputf8
// SuppressAdd adds an address to the suppress list.
func SuppressAdd(ctx context.Context, ba *SuppressAddress) error {
db, err := evalDB(ctx)
if err != nil {
return err
}
return db.Insert(ctx, ba)
return EvalDB.Insert(ctx, ba)
}
// SuppressList returns all reporting addresses on the suppress list.
func SuppressList(ctx context.Context) ([]SuppressAddress, error) {
db, err := evalDB(ctx)
if err != nil {
return nil, err
}
return bstore.QueryDB[SuppressAddress](ctx, db).SortDesc("ID").List()
return bstore.QueryDB[SuppressAddress](ctx, EvalDB).SortDesc("ID").List()
}
// SuppressRemove removes a reporting address record from the suppress list.
func SuppressRemove(ctx context.Context, id int64) error {
db, err := evalDB(ctx)
if err != nil {
return err
}
return db.Delete(ctx, &SuppressAddress{ID: id})
return EvalDB.Delete(ctx, &SuppressAddress{ID: id})
}
// SuppressUpdate updates the until field of a reporting address record.
func SuppressUpdate(ctx context.Context, id int64, until time.Time) error {
db, err := evalDB(ctx)
if err != nil {
return err
}
ba := SuppressAddress{ID: id}
err = db.Get(ctx, &ba)
err := EvalDB.Get(ctx, &ba)
if err != nil {
return err
}
ba.Until = until
return db.Update(ctx, &ba)
return EvalDB.Update(ctx, &ba)
}

26
dmarcdb/eval_test.go
View File

@ -41,13 +41,13 @@ func TestEvaluations(t *testing.T) {
mox.Context = ctxbg
mox.ConfigStaticPath = filepath.FromSlash("../testdata/dmarcdb/mox.conf")
mox.MustLoadConfig(true, false)
EvalDB = nil
_, err := evalDB(ctxbg)
tcheckf(t, err, "database")
os.Remove(mox.DataDirPath("dmarceval.db"))
err := Init()
tcheckf(t, err, "init")
defer func() {
EvalDB.Close()
EvalDB = nil
err := Close()
tcheckf(t, err, "close")
}()
parseJSON := func(s string) (e Evaluation) {
@ -163,13 +163,13 @@ func TestSendReports(t *testing.T) {
mox.Context = ctxbg
mox.ConfigStaticPath = filepath.FromSlash("../testdata/dmarcdb/mox.conf")
mox.MustLoadConfig(true, false)
EvalDB = nil
db, err := evalDB(ctxbg)
tcheckf(t, err, "database")
os.Remove(mox.DataDirPath("dmarceval.db"))
err := Init()
tcheckf(t, err, "init")
defer func() {
EvalDB.Close()
EvalDB = nil
err := Close()
tcheckf(t, err, "close")
}()
resolver := dns.MockResolver{
@ -288,7 +288,7 @@ func TestSendReports(t *testing.T) {
mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg)
for _, e := range evals {
err := db.Insert(ctxbg, &e)
err := EvalDB.Insert(ctxbg, &e)
tcheckf(t, err, "inserting evaluation")
}
@ -359,13 +359,13 @@ func TestSendReports(t *testing.T) {
// Address is suppressed.
sa := SuppressAddress{ReportingAddress: "dmarcrpt@sender.example", Until: time.Now().Add(time.Minute)}
err = db.Insert(ctxbg, &sa)
err = EvalDB.Insert(ctxbg, &sa)
tcheckf(t, err, "insert suppress address")
test([]Evaluation{eval}, map[string]struct{}{}, map[string]struct{}{}, nil)
// Suppression has expired.
sa.Until = time.Now().Add(-time.Minute)
err = db.Update(ctxbg, &sa)
err = EvalDB.Update(ctxbg, &sa)
tcheckf(t, err, "update suppress address")
test([]Evaluation{eval}, map[string]struct{}{"dmarcrpt@sender.example": {}}, map[string]struct{}{}, expFeedback)

48
dmarcdb/reports.go
View File

@ -3,9 +3,6 @@ package dmarcdb
import (
"context"
"fmt"
"os"
"path/filepath"
"sync"
"time"
"github.com/prometheus/client_golang/prometheus"
@ -15,13 +12,11 @@ import (
"github.com/mjl-/mox/dmarcrpt"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mox-"
)
var (
ReportsDBTypes = []any{DomainFeedback{}} // Types stored in DB.
ReportsDB *bstore.DB // Exported for backups.
reportsMutex sync.Mutex
)
var (
@ -59,38 +54,18 @@ type DomainFeedback struct {
dmarcrpt.Feedback
}
func reportsDB(ctx context.Context) (rdb *bstore.DB, rerr error) {
reportsMutex.Lock()
defer reportsMutex.Unlock()
if ReportsDB == nil {
p := mox.DataDirPath("dmarcrpt.db")
os.MkdirAll(filepath.Dir(p), 0770)
db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, ReportsDBTypes...)
if err != nil {
return nil, err
}
ReportsDB = db
}
return ReportsDB, nil
}
// AddReport adds a DMARC aggregate feedback report from an email to the database,
// and updates prometheus metrics.
//
// fromDomain is the domain in the report message From header.
func AddReport(ctx context.Context, f *dmarcrpt.Feedback, fromDomain dns.Domain) error {
db, err := reportsDB(ctx)
if err != nil {
return err
}
d, err := dns.ParseDomain(f.PolicyPublished.Domain)
if err != nil {
return fmt.Errorf("parsing domain in report: %v", err)
}
df := DomainFeedback{0, d.Name(), fromDomain.Name(), *f}
if err := db.Insert(ctx, &df); err != nil {
if err := ReportsDB.Insert(ctx, &df); err != nil {
return err
}
@ -129,38 +104,23 @@ func AddReport(ctx context.Context, f *dmarcrpt.Feedback, fromDomain dns.Domain)
// Records returns all reports in the database.
func Records(ctx context.Context) ([]DomainFeedback, error) {
db, err := reportsDB(ctx)
if err != nil {
return nil, err
}
return bstore.QueryDB[DomainFeedback](ctx, db).List()
return bstore.QueryDB[DomainFeedback](ctx, ReportsDB).List()
}
// RecordID returns the report for the ID.
func RecordID(ctx context.Context, id int64) (DomainFeedback, error) {
db, err := reportsDB(ctx)
if err != nil {
return DomainFeedback{}, err
}
e := DomainFeedback{ID: id}
err = db.Get(ctx, &e)
err := ReportsDB.Get(ctx, &e)
return e, err
}
// RecordsPeriodDomain returns the reports overlapping start and end, for the given
// domain. If domain is empty, all records match for domain.
func RecordsPeriodDomain(ctx context.Context, start, end time.Time, domain string) ([]DomainFeedback, error) {
db, err := reportsDB(ctx)
if err != nil {
return nil, err
}
s := start.Unix()
e := end.Unix()
q := bstore.QueryDB[DomainFeedback](ctx, db)
q := bstore.QueryDB[DomainFeedback](ctx, ReportsDB)
if domain != "" {
q.FilterNonzero(DomainFeedback{Domain: domain})
}

14
dmarcdb/reports_test.go
View File

@ -20,16 +20,12 @@ func TestDMARCDB(t *testing.T) {
mox.ConfigStaticPath = filepath.FromSlash("../testdata/dmarcdb/mox.conf")
mox.MustLoadConfig(true, false)
dbpath := mox.DataDirPath("dmarcrpt.db")
os.MkdirAll(filepath.Dir(dbpath), 0770)
if err := Init(); err != nil {
t.Fatalf("init database: %s", err)
}
defer os.Remove(dbpath)
os.Remove(mox.DataDirPath("dmarcrpt.db"))
err := Init()
tcheckf(t, err, "init")
defer func() {
ReportsDB.Close()
ReportsDB = nil
err := Close()
tcheckf(t, err, "close")
}()
feedback := &dmarcrpt.Feedback{