mirror of
https://github.com/mjl-/mox.git
synced 2025-07-12 17:44:35 +03:00
mox!
This commit is contained in:
Mechiel Lukkien
186
dmarcdb/db.go
Normal file
186
dmarcdb/db.go
Normal file
@ -0,0 +1,186 @@
|
||||
// Package dmarcdb stores incoming DMARC reports.
|
||||
//
|
||||
// With DMARC, a domain can request emails with DMARC verification results by
|
||||
// remote mail servers to be sent to a specified address. Mox parses such
|
||||
// reports, stores them in its database and makes them available through its
|
||||
// admin web interface.
|
||||
package dmarcdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
|
||||
"github.com/mjl-/bstore"
|
||||
|
||||
"github.com/mjl-/mox/dmarcrpt"
|
||||
"github.com/mjl-/mox/dns"
|
||||
"github.com/mjl-/mox/mlog"
|
||||
"github.com/mjl-/mox/mox-"
|
||||
)
|
||||
|
||||
var xlog = mlog.New("dmarcdb")
|
||||
|
||||
var (
|
||||
dmarcDB *bstore.DB
|
||||
mutex sync.Mutex
|
||||
)
|
||||
|
||||
var (
|
||||
metricEvaluated = promauto.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "mox_dmarcdb_policy_evaluated_total",
|
||||
Help: "Number of policy evaluations.",
|
||||
},
|
||||
// We only register validated domains for which we have a config.
|
||||
[]string{"domain", "disposition", "dkim", "spf"},
|
||||
)
|
||||
metricDKIM = promauto.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "mox_dmarcdb_dkim_result_total",
|
||||
Help: "Number of DKIM results.",
|
||||
},
|
||||
[]string{"result"},
|
||||
)
|
||||
metricSPF = promauto.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "mox_dmarcdb_spf_result_total",
|
||||
Help: "Number of SPF results.",
|
||||
},
|
||||
[]string{"result"},
|
||||
)
|
||||
)
|
||||
|
||||
// DomainFeedback is a single report stored in the database.
|
||||
type DomainFeedback struct {
|
||||
ID int64
|
||||
// Domain where DMARC DNS record was found, could be organizational domain.
|
||||
Domain string `bstore:"index"`
|
||||
// Domain in From-header.
|
||||
FromDomain string `bstore:"index"`
|
||||
dmarcrpt.Feedback
|
||||
}
|
||||
|
||||
func database() (rdb *bstore.DB, rerr error) {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
if dmarcDB == nil {
|
||||
p := mox.DataDirPath("dmarcrpt.db")
|
||||
os.MkdirAll(filepath.Dir(p), 0770)
|
||||
db, err := bstore.Open(p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DomainFeedback{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dmarcDB = db
|
||||
}
|
||||
return dmarcDB, nil
|
||||
}
|
||||
|
||||
// Init opens the database.
|
||||
func Init() error {
|
||||
_, err := database()
|
||||
return err
|
||||
}
|
||||
|
||||
// AddReport adds a DMARC aggregate feedback report from an email to the database,
|
||||
// and updates prometheus metrics.
|
||||
//
|
||||
// fromDomain is the domain in the report message From header.
|
||||
func AddReport(ctx context.Context, f *dmarcrpt.Feedback, fromDomain dns.Domain) error {
|
||||
db, err := database()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
d, err := dns.ParseDomain(f.PolicyPublished.Domain)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing domain in report: %v", err)
|
||||
}
|
||||
|
||||
df := DomainFeedback{0, d.Name(), fromDomain.Name(), *f}
|
||||
if err := db.Insert(&df); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, r := range f.Records {
|
||||
for _, dkim := range r.AuthResults.DKIM {
|
||||
count := r.Row.Count
|
||||
if count > 0 {
|
||||
metricDKIM.With(prometheus.Labels{
|
||||
"result": string(dkim.Result),
|
||||
}).Add(float64(count))
|
||||
}
|
||||
}
|
||||
|
||||
for _, spf := range r.AuthResults.SPF {
|
||||
count := r.Row.Count
|
||||
if count > 0 {
|
||||
metricSPF.With(prometheus.Labels{
|
||||
"result": string(spf.Result),
|
||||
}).Add(float64(count))
|
||||
}
|
||||
}
|
||||
|
||||
count := r.Row.Count
|
||||
if count > 0 {
|
||||
pe := r.Row.PolicyEvaluated
|
||||
metricEvaluated.With(prometheus.Labels{
|
||||
"domain": f.PolicyPublished.Domain,
|
||||
"disposition": string(pe.Disposition),
|
||||
"dkim": string(pe.DKIM),
|
||||
"spf": string(pe.SPF),
|
||||
}).Add(float64(count))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Records returns all reports in the database.
|
||||
func Records(ctx context.Context) ([]DomainFeedback, error) {
|
||||
db, err := database()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return bstore.QueryDB[DomainFeedback](db).List()
|
||||
}
|
||||
|
||||
// RecordID returns the report for the ID.
|
||||
func RecordID(ctx context.Context, id int64) (DomainFeedback, error) {
|
||||
db, err := database()
|
||||
if err != nil {
|
||||
return DomainFeedback{}, err
|
||||
}
|
||||
|
||||
e := DomainFeedback{ID: id}
|
||||
err = db.Get(&e)
|
||||
return e, err
|
||||
}
|
||||
|
||||
// RecordsPeriodDomain returns the reports overlapping start and end, for the given
|
||||
// domain. If domain is empty, all records match for domain.
|
||||
func RecordsPeriodDomain(ctx context.Context, start, end time.Time, domain string) ([]DomainFeedback, error) {
|
||||
db, err := database()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s := start.Unix()
|
||||
e := end.Unix()
|
||||
|
||||
q := bstore.QueryDB[DomainFeedback](db)
|
||||
if domain != "" {
|
||||
q.FilterNonzero(DomainFeedback{Domain: domain})
|
||||
}
|
||||
q.FilterFn(func(d DomainFeedback) bool {
|
||||
m := d.Feedback.ReportMetadata.DateRange
|
||||
return m.Begin >= s && m.Begin < e || m.End > s && m.End <= e
|
||||
})
|
||||
return q.List()
|
||||
}
|
||||
108
dmarcdb/db_test.go
Normal file
108
dmarcdb/db_test.go
Normal file
@ -0,0 +1,108 @@
|
||||
package dmarcdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mjl-/mox/dmarcrpt"
|
||||
"github.com/mjl-/mox/dns"
|
||||
"github.com/mjl-/mox/mox-"
|
||||
)
|
||||
|
||||
func TestDMARCDB(t *testing.T) {
|
||||
mox.ConfigStaticPath = "../testdata/dmarcdb/fake.conf"
|
||||
mox.Conf.Static.DataDir = "."
|
||||
|
||||
dbpath := mox.DataDirPath("dmarcrpt.db")
|
||||
os.MkdirAll(filepath.Dir(dbpath), 0770)
|
||||
defer os.Remove(dbpath)
|
||||
|
||||
if err := Init(); err != nil {
|
||||
t.Fatalf("init database: %s", err)
|
||||
}
|
||||
|
||||
feedback := &dmarcrpt.Feedback{
|
||||
ReportMetadata: dmarcrpt.ReportMetadata{
|
||||
OrgName: "google.com",
|
||||
Email: "noreply-dmarc-support@google.com",
|
||||
ExtraContactInfo: "https://support.google.com/a/answer/2466580",
|
||||
ReportID: "10051505501689795560",
|
||||
DateRange: dmarcrpt.DateRange{
|
||||
Begin: 1596412800,
|
||||
End: 1596499199,
|
||||
},
|
||||
},
|
||||
PolicyPublished: dmarcrpt.PolicyPublished{
|
||||
Domain: "example.org",
|
||||
ADKIM: "r",
|
||||
ASPF: "r",
|
||||
Policy: "reject",
|
||||
SubdomainPolicy: "reject",
|
||||
Percentage: 100,
|
||||
},
|
||||
Records: []dmarcrpt.ReportRecord{
|
||||
{
|
||||
Row: dmarcrpt.Row{
|
||||
SourceIP: "127.0.0.1",
|
||||
Count: 1,
|
||||
PolicyEvaluated: dmarcrpt.PolicyEvaluated{
|
||||
Disposition: dmarcrpt.DispositionNone,
|
||||
DKIM: dmarcrpt.DMARCPass,
|
||||
SPF: dmarcrpt.DMARCPass,
|
||||
},
|
||||
},
|
||||
Identifiers: dmarcrpt.Identifiers{
|
||||
HeaderFrom: "example.org",
|
||||
},
|
||||
AuthResults: dmarcrpt.AuthResults{
|
||||
DKIM: []dmarcrpt.DKIMAuthResult{
|
||||
{
|
||||
Domain: "example.org",
|
||||
Result: dmarcrpt.DKIMPass,
|
||||
Selector: "example",
|
||||
},
|
||||
},
|
||||
SPF: []dmarcrpt.SPFAuthResult{
|
||||
{
|
||||
Domain: "example.org",
|
||||
Result: dmarcrpt.SPFPass,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := AddReport(context.Background(), feedback, dns.Domain{ASCII: "google.com"}); err != nil {
|
||||
t.Fatalf("adding report: %s", err)
|
||||
}
|
||||
|
||||
records, err := Records(context.Background())
|
||||
if err != nil || len(records) != 1 || !reflect.DeepEqual(&records[0].Feedback, feedback) {
|
||||
t.Fatalf("records: got err %v, records %#v, expected no error, single record with feedback %#v", err, records, feedback)
|
||||
}
|
||||
|
||||
record, err := RecordID(context.Background(), records[0].ID)
|
||||
if err != nil || !reflect.DeepEqual(&record.Feedback, feedback) {
|
||||
t.Fatalf("record id: got err %v, record %#v, expected feedback %#v", err, record, feedback)
|
||||
}
|
||||
|
||||
start := time.Unix(1596412800, 0)
|
||||
end := time.Unix(1596499199, 0)
|
||||
records, err = RecordsPeriodDomain(context.Background(), start, end, "example.org")
|
||||
if err != nil || len(records) != 1 || !reflect.DeepEqual(&records[0].Feedback, feedback) {
|
||||
t.Fatalf("records: got err %v, records %#v, expected no error, single record with feedback %#v", err, records, feedback)
|
||||
}
|
||||
|
||||
records, err = RecordsPeriodDomain(context.Background(), end, end, "example.org")
|
||||
if err != nil || len(records) != 0 {
|
||||
t.Fatalf("records: got err %v, records %#v, expected no error and no records", err, records)
|
||||
}
|
||||
records, err = RecordsPeriodDomain(context.Background(), start, end, "other.example")
|
||||
if err != nil || len(records) != 0 {
|
||||
t.Fatalf("records: got err %v, records %#v, expected no error and no records", err, records)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user