infrastructure/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2025-07-12 19:44:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

add debug logging about bstore db schema upgrades

Browse Source 
bstore was updated to v0.0.6 to add this logging.
this simplifies some of the db-handling code in mtastsdb,tlsrptdb,dmarcdb. we
now call the package-level Init() and Close() in all tests properly.
This commit is contained in:
Mechiel Lukkien
2024-05-10 14:44:37 +02:00
parent 3e4cce826e
commit bf8cfd9724
31 changed files with 298 additions and 428 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
tlsrptdb/db.go
View File

@ -1,7 +1,11 @@
package tlsrptdb
import (
"sync"
"context"
"fmt"
"os"
"path/filepath"
"time"
"github.com/mjl-/bstore"
@ -12,7 +16,6 @@ import (
var (
ReportDBTypes = []any{Record{}}
ReportDB *bstore.DB
mutex sync.Mutex
// Accessed directly by tlsrptsend.
ResultDBTypes = []any{TLSResult{}, SuppressAddress{}}
@ -21,29 +24,48 @@ var (
// Init opens and possibly initializes the databases.
func Init() error {
if _, err := reportDB(mox.Shutdown); err != nil {
return err
if ReportDB != nil || ResultDB != nil {
return fmt.Errorf("already initialized")
}
if _, err := resultDB(mox.Shutdown); err != nil {
return err
log := mlog.New("tlsrptdb", nil)
var err error
ReportDB, err = openReportDB(mox.Shutdown, log)
if err != nil {
return fmt.Errorf("opening report db: %v", err)
}
ResultDB, err = openResultDB(mox.Shutdown, log)
if err != nil {
return fmt.Errorf("opening result db: %v", err)
}
return nil
}
// Close closes the database connections.
func Close() {
log := mlog.New("tlsrptdb", nil)
if ResultDB != nil {
err := ResultDB.Close()
log.Check(err, "closing result database")
ResultDB = nil
}
mutex.Lock()
defer mutex.Unlock()
if ReportDB != nil {
err := ReportDB.Close()
log.Check(err, "closing report database")
ReportDB = nil
}
func openReportDB(ctx context.Context, log mlog.Log) (*bstore.DB, error) {
p := mox.DataDirPath("tlsrpt.db")
os.MkdirAll(filepath.Dir(p), 0770)
opts := bstore.Options{Timeout: 5 * time.Second, Perm: 0660, RegisterLogger: log.Logger}
return bstore.Open(ctx, p, &opts, ReportDBTypes...)
}
func openResultDB(ctx context.Context, log mlog.Log) (*bstore.DB, error) {
p := mox.DataDirPath("tlsrptresult.db")
os.MkdirAll(filepath.Dir(p), 0770)
opts := bstore.Options{Timeout: 5 * time.Second, Perm: 0660, RegisterLogger: log.Logger}
return bstore.Open(ctx, p, &opts, ResultDBTypes...)
}
// Close closes the database connections.
func Close() error {
if err := ResultDB.Close(); err != nil {
return fmt.Errorf("closing result db: %w", err)
}
ResultDB = nil
if err := ReportDB.Close(); err != nil {
return fmt.Errorf("closing report db: %w", err)
}
ReportDB = nil
return nil
}

44
tlsrptdb/report.go
View File

@ -5,8 +5,6 @@ import (
"context"
"fmt"
"log/slog"
"os"
"path/filepath"
"time"
"github.com/prometheus/client_golang/prometheus"
@ -55,21 +53,6 @@ type Record struct {
Report tlsrpt.Report
}
func reportDB(ctx context.Context) (rdb *bstore.DB, rerr error) {
mutex.Lock()
defer mutex.Unlock()
if ReportDB == nil {
p := mox.DataDirPath("tlsrpt.db")
os.MkdirAll(filepath.Dir(p), 0770)
db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, ReportDBTypes...)
if err != nil {
return nil, err
}
ReportDB = db
}
return ReportDB, nil
}
// AddReport adds a TLS report to the database.
//
// The report should have come in over SMTP, with a DKIM-validated
@ -82,17 +65,12 @@ func reportDB(ctx context.Context) (rdb *bstore.DB, rerr error) {
//
// Prometheus metrics are updated only for configured domains.
func AddReport(ctx context.Context, log mlog.Log, verifiedFromDomain dns.Domain, mailFrom string, hostReport bool, r *tlsrpt.Report) error {
db, err := reportDB(ctx)
if err != nil {
return err
}
if len(r.Policies) == 0 {
return fmt.Errorf("no policies in report")
}
var inserted int
return db.Write(ctx, func(tx *bstore.Tx) error {
return ReportDB.Write(ctx, func(tx *bstore.Tx) error {
for _, p := range r.Policies {
pp := p.Policy
@ -132,22 +110,13 @@ func AddReport(ctx context.Context, log mlog.Log, verifiedFromDomain dns.Domain,
// Records returns all TLS reports in the database.
func Records(ctx context.Context) ([]Record, error) {
db, err := reportDB(ctx)
if err != nil {
return nil, err
}
return bstore.QueryDB[Record](ctx, db).List()
return bstore.QueryDB[Record](ctx, ReportDB).List()
}
// RecordID returns the report for the ID.
func RecordID(ctx context.Context, id int64) (Record, error) {
db, err := reportDB(ctx)
if err != nil {
return Record{}, err
}
e := Record{ID: id}
err = db.Get(ctx, &e)
err := ReportDB.Get(ctx, &e)
return e, err
}
@ -155,12 +124,7 @@ func RecordID(ctx context.Context, id int64) (Record, error) {
// given policy domain. If policy domain is empty, records for all domains are
// returned.
func RecordsPeriodDomain(ctx context.Context, start, end time.Time, policyDomain dns.Domain) ([]Record, error) {
db, err := reportDB(ctx)
if err != nil {
return nil, err
}
q := bstore.QueryDB[Record](ctx, db)
q := bstore.QueryDB[Record](ctx, ReportDB)
var zerodom dns.Domain
if policyDomain != zerodom {
q.FilterNonzero(Record{Domain: policyDomain.Name()})

90
tlsrptdb/result.go
View File

@ -3,14 +3,11 @@ package tlsrptdb
import (
"context"
"fmt"
"os"
"path/filepath"
"time"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/tlsrpt"
)
@ -70,33 +67,13 @@ type SuppressAddress struct {
Comment string
}
func resultDB(ctx context.Context) (rdb *bstore.DB, rerr error) {
mutex.Lock()
defer mutex.Unlock()
if ResultDB == nil {
p := mox.DataDirPath("tlsrptresult.db")
os.MkdirAll(filepath.Dir(p), 0770)
db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, ResultDBTypes...)
if err != nil {
return nil, err
}
ResultDB = db
}
return ResultDB, nil
}
// AddTLSResults adds or merges all tls results for delivering to a policy domain,
// on its UTC day to a recipient domain to the database. Results may cause multiple
// separate reports to be sent.
func AddTLSResults(ctx context.Context, results []TLSResult) error {
db, err := resultDB(ctx)
if err != nil {
return err
}
now := time.Now()
err = db.Write(ctx, func(tx *bstore.Tx) error {
err := ResultDB.Write(ctx, func(tx *bstore.Tx) error {
for _, result := range results {
// Ensure all slices are non-nil. We do this now so all readers will marshal to
// compliant with the JSON schema. And also for consistent equality checks when
@ -148,102 +125,57 @@ func AddTLSResults(ctx context.Context, results []TLSResult) error {
// Results returns all TLS results in the database, for all policy domains each
// with potentially multiple days. Sorted by RecipientDomain and day.
func Results(ctx context.Context) ([]TLSResult, error) {
db, err := resultDB(ctx)
if err != nil {
return nil, err
}
return bstore.QueryDB[TLSResult](ctx, db).SortAsc("PolicyDomain", "DayUTC", "RecipientDomain").List()
return bstore.QueryDB[TLSResult](ctx, ResultDB).SortAsc("PolicyDomain", "DayUTC", "RecipientDomain").List()
}
// ResultsDomain returns all TLSResults for a policy domain, potentially for
// multiple days.
func ResultsPolicyDomain(ctx context.Context, policyDomain dns.Domain) ([]TLSResult, error) {
db, err := resultDB(ctx)
if err != nil {
return nil, err
}
return bstore.QueryDB[TLSResult](ctx, db).FilterNonzero(TLSResult{PolicyDomain: policyDomain.Name()}).SortAsc("DayUTC", "RecipientDomain").List()
return bstore.QueryDB[TLSResult](ctx, ResultDB).FilterNonzero(TLSResult{PolicyDomain: policyDomain.Name()}).SortAsc("DayUTC", "RecipientDomain").List()
}
// ResultsRecipientDomain returns all TLSResults for a recipient domain,
// potentially for multiple days.
func ResultsRecipientDomain(ctx context.Context, recipientDomain dns.Domain) ([]TLSResult, error) {
db, err := resultDB(ctx)
if err != nil {
return nil, err
}
return bstore.QueryDB[TLSResult](ctx, db).FilterNonzero(TLSResult{RecipientDomain: recipientDomain.Name()}).SortAsc("DayUTC", "PolicyDomain").List()
return bstore.QueryDB[TLSResult](ctx, ResultDB).FilterNonzero(TLSResult{RecipientDomain: recipientDomain.Name()}).SortAsc("DayUTC", "PolicyDomain").List()
}
// RemoveResultsPolicyDomain removes all TLSResults for the policy domain on the
// day from the database.
func RemoveResultsPolicyDomain(ctx context.Context, policyDomain dns.Domain, dayUTC string) error {
db, err := resultDB(ctx)
if err != nil {
return err
}
_, err = bstore.QueryDB[TLSResult](ctx, db).FilterNonzero(TLSResult{PolicyDomain: policyDomain.Name(), DayUTC: dayUTC}).Delete()
_, err := bstore.QueryDB[TLSResult](ctx, ResultDB).FilterNonzero(TLSResult{PolicyDomain: policyDomain.Name(), DayUTC: dayUTC}).Delete()
return err
}
// RemoveResultsRecipientDomain removes all TLSResults for the recipient domain on
// the day from the database.
func RemoveResultsRecipientDomain(ctx context.Context, recipientDomain dns.Domain, dayUTC string) error {
db, err := resultDB(ctx)
if err != nil {
return err
}
_, err = bstore.QueryDB[TLSResult](ctx, db).FilterNonzero(TLSResult{RecipientDomain: recipientDomain.Name(), DayUTC: dayUTC}).Delete()
_, err := bstore.QueryDB[TLSResult](ctx, ResultDB).FilterNonzero(TLSResult{RecipientDomain: recipientDomain.Name(), DayUTC: dayUTC}).Delete()
return err
}
// SuppressAdd adds an address to the suppress list.
func SuppressAdd(ctx context.Context, ba *SuppressAddress) error {
db, err := resultDB(ctx)
if err != nil {
return err
}
return db.Insert(ctx, ba)
return ResultDB.Insert(ctx, ba)
}
// SuppressList returns all reporting addresses on the suppress list.
func SuppressList(ctx context.Context) ([]SuppressAddress, error) {
db, err := resultDB(ctx)
if err != nil {
return nil, err
}
return bstore.QueryDB[SuppressAddress](ctx, db).SortDesc("ID").List()
return bstore.QueryDB[SuppressAddress](ctx, ResultDB).SortDesc("ID").List()
}
// SuppressRemove removes a reporting address record from the suppress list.
func SuppressRemove(ctx context.Context, id int64) error {
db, err := resultDB(ctx)
if err != nil {
return err
}
return db.Delete(ctx, &SuppressAddress{ID: id})
return ResultDB.Delete(ctx, &SuppressAddress{ID: id})
}
// SuppressUpdate updates the until field of a reporting address record.
func SuppressUpdate(ctx context.Context, id int64, until time.Time) error {
db, err := resultDB(ctx)
if err != nil {
return err
}
ba := SuppressAddress{ID: id}
err = db.Get(ctx, &ba)
err := ResultDB.Get(ctx, &ba)
if err != nil {
return err
}
ba.Until = until
return db.Update(ctx, &ba)
return ResultDB.Update(ctx, &ba)
}