mirror of
https://github.com/mjl-/mox.git
synced 2025-07-12 17:04:39 +03:00
mox!
This commit is contained in:
Mechiel Lukkien
192
tlsrptdb/db.go
Normal file
192
tlsrptdb/db.go
Normal file
@ -0,0 +1,192 @@
|
||||
// Package tlsrptdb stores reports from "SMTP TLS Reporting" in its database.
|
||||
package tlsrptdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
|
||||
"github.com/mjl-/bstore"
|
||||
|
||||
"github.com/mjl-/mox/dns"
|
||||
"github.com/mjl-/mox/mlog"
|
||||
"github.com/mjl-/mox/mox-"
|
||||
"github.com/mjl-/mox/tlsrpt"
|
||||
)
|
||||
|
||||
var (
|
||||
xlog = mlog.New("tlsrptdb")
|
||||
|
||||
tlsrptDB *bstore.DB
|
||||
mutex sync.Mutex
|
||||
|
||||
metricSession = promauto.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "mox_tlsrpt_session_total",
|
||||
Help: "Number of sessions, both success and known result types.",
|
||||
},
|
||||
[]string{"type"}, // Known result types, and "success"
|
||||
)
|
||||
|
||||
knownResultTypes = map[tlsrpt.ResultType]struct{}{
|
||||
tlsrpt.ResultSTARTTLSNotSupported: {},
|
||||
tlsrpt.ResultCertificateHostMismatch: {},
|
||||
tlsrpt.ResultCertificateExpired: {},
|
||||
tlsrpt.ResultTLSAInvalid: {},
|
||||
tlsrpt.ResultDNSSECInvalid: {},
|
||||
tlsrpt.ResultDANERequired: {},
|
||||
tlsrpt.ResultCertificateNotTrusted: {},
|
||||
tlsrpt.ResultSTSPolicyInvalid: {},
|
||||
tlsrpt.ResultSTSWebPKIInvalid: {},
|
||||
tlsrpt.ResultValidationFailure: {},
|
||||
tlsrpt.ResultSTSPolicyFetch: {},
|
||||
}
|
||||
)
|
||||
|
||||
// TLSReportRecord is a TLS report as a database record, including information
|
||||
// about the sender.
|
||||
//
|
||||
// todo: should be named just Record, but it would cause a sherpa type name conflict.
|
||||
type TLSReportRecord struct {
|
||||
ID int64 `bstore:"typename Record"`
|
||||
Domain string `bstore:"index"` // Domain to which the TLS report applies.
|
||||
FromDomain string
|
||||
MailFrom string
|
||||
Report tlsrpt.Report
|
||||
}
|
||||
|
||||
func database() (rdb *bstore.DB, rerr error) {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
if tlsrptDB == nil {
|
||||
p := mox.DataDirPath("tlsrpt.db")
|
||||
os.MkdirAll(filepath.Dir(p), 0770)
|
||||
db, err := bstore.Open(p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, TLSReportRecord{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tlsrptDB = db
|
||||
}
|
||||
return tlsrptDB, nil
|
||||
}
|
||||
|
||||
// Init opens and possibly initializes the database.
|
||||
func Init() error {
|
||||
_, err := database()
|
||||
return err
|
||||
}
|
||||
|
||||
// Close closes the database connection.
|
||||
func Close() {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
if tlsrptDB != nil {
|
||||
tlsrptDB.Close()
|
||||
tlsrptDB = nil
|
||||
}
|
||||
}
|
||||
|
||||
// AddReport adds a TLS report to the database.
|
||||
//
|
||||
// The report should have come in over SMTP, with a DKIM-validated
|
||||
// verifiedFromDomain. Using HTTPS for reports is not recommended as there is no
|
||||
// authentication on the reports origin.
|
||||
//
|
||||
// The report is currently required to only cover a single domain in its policy
|
||||
// domain. Only reports for known domains are added to the database.
|
||||
//
|
||||
// Prometheus metrics are updated only for configured domains.
|
||||
func AddReport(ctx context.Context, verifiedFromDomain dns.Domain, mailFrom string, r *tlsrpt.Report) error {
|
||||
log := xlog.WithContext(ctx)
|
||||
|
||||
db, err := database()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(r.Policies) == 0 {
|
||||
return fmt.Errorf("no policies in report")
|
||||
}
|
||||
|
||||
var reportdom, zerodom dns.Domain
|
||||
record := TLSReportRecord{0, "", verifiedFromDomain.Name(), mailFrom, *r}
|
||||
|
||||
for _, p := range r.Policies {
|
||||
pp := p.Policy
|
||||
|
||||
// Check domain, they must all be the same for now (in future, with DANE, this may
|
||||
// no longer apply).
|
||||
d, err := dns.ParseDomain(pp.Domain)
|
||||
if err != nil {
|
||||
log.Errorx("invalid domain in tls report", err, mlog.Field("domain", pp.Domain), mlog.Field("mailfrom", mailFrom))
|
||||
continue
|
||||
}
|
||||
if _, ok := mox.Conf.Domain(d); !ok {
|
||||
log.Info("unknown domain in tls report, not storing", mlog.Field("domain", d), mlog.Field("mailfrom", mailFrom))
|
||||
return fmt.Errorf("unknown domain")
|
||||
}
|
||||
if reportdom != zerodom && d != reportdom {
|
||||
return fmt.Errorf("multiple domains in report %v and %v", reportdom, d)
|
||||
}
|
||||
reportdom = d
|
||||
|
||||
metricSession.WithLabelValues("success").Add(float64(p.Summary.TotalSuccessfulSessionCount))
|
||||
for _, f := range p.FailureDetails {
|
||||
var result string
|
||||
if _, ok := knownResultTypes[f.ResultType]; ok {
|
||||
result = string(f.ResultType)
|
||||
} else {
|
||||
result = "other"
|
||||
}
|
||||
metricSession.WithLabelValues(result).Add(float64(f.FailedSessionCount))
|
||||
}
|
||||
}
|
||||
record.Domain = reportdom.Name()
|
||||
return db.Insert(&record)
|
||||
}
|
||||
|
||||
// Records returns all TLS reports in the database.
|
||||
func Records(ctx context.Context) ([]TLSReportRecord, error) {
|
||||
db, err := database()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bstore.QueryDB[TLSReportRecord](db).List()
|
||||
}
|
||||
|
||||
// RecordID returns the report for the ID.
|
||||
func RecordID(ctx context.Context, id int64) (TLSReportRecord, error) {
|
||||
db, err := database()
|
||||
if err != nil {
|
||||
return TLSReportRecord{}, err
|
||||
}
|
||||
|
||||
e := TLSReportRecord{ID: id}
|
||||
err = db.Get(&e)
|
||||
return e, err
|
||||
}
|
||||
|
||||
// RecordsPeriodDomain returns the reports overlapping start and end, for the given
|
||||
// domain. If domain is empty, all records match for domain.
|
||||
func RecordsPeriodDomain(ctx context.Context, start, end time.Time, domain string) ([]TLSReportRecord, error) {
|
||||
db, err := database()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
q := bstore.QueryDB[TLSReportRecord](db)
|
||||
if domain != "" {
|
||||
q.FilterNonzero(TLSReportRecord{Domain: domain})
|
||||
}
|
||||
q.FilterFn(func(r TLSReportRecord) bool {
|
||||
dr := r.Report.DateRange
|
||||
return !dr.Start.Before(start) && dr.Start.Before(end) || dr.End.After(start) && !dr.End.After(end)
|
||||
})
|
||||
return q.List()
|
||||
}
|
||||
126
tlsrptdb/db_test.go
Normal file
126
tlsrptdb/db_test.go
Normal file
@ -0,0 +1,126 @@
|
||||
package tlsrptdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mjl-/mox/config"
|
||||
"github.com/mjl-/mox/dns"
|
||||
"github.com/mjl-/mox/mox-"
|
||||
"github.com/mjl-/mox/tlsrpt"
|
||||
)
|
||||
|
||||
const reportJSON = `{
|
||||
"organization-name": "Company-X",
|
||||
"date-range": {
|
||||
"start-datetime": "2016-04-01T00:00:00Z",
|
||||
"end-datetime": "2016-04-01T23:59:59Z"
|
||||
},
|
||||
"contact-info": "sts-reporting@company-x.example",
|
||||
"report-id": "5065427c-23d3-47ca-b6e0-946ea0e8c4be",
|
||||
"policies": [{
|
||||
"policy": {
|
||||
"policy-type": "sts",
|
||||
"policy-string": ["version: STSv1","mode: testing",
|
||||
"mx: *.mail.company-y.example","max_age: 86400"],
|
||||
"policy-domain": "test.xmox.nl",
|
||||
"mx-host": ["*.mail.company-y.example"]
|
||||
},
|
||||
"summary": {
|
||||
"total-successful-session-count": 5326,
|
||||
"total-failure-session-count": 303
|
||||
},
|
||||
"failure-details": [{
|
||||
"result-type": "certificate-expired",
|
||||
"sending-mta-ip": "2001:db8:abcd:0012::1",
|
||||
"receiving-mx-hostname": "mx1.mail.company-y.example",
|
||||
"failed-session-count": 100
|
||||
}, {
|
||||
"result-type": "starttls-not-supported",
|
||||
"sending-mta-ip": "2001:db8:abcd:0013::1",
|
||||
"receiving-mx-hostname": "mx2.mail.company-y.example",
|
||||
"receiving-ip": "203.0.113.56",
|
||||
"failed-session-count": 200,
|
||||
"additional-information": "https://reports.company-x.example/report_info ? id = 5065427 c - 23 d3# StarttlsNotSupported "
|
||||
}, {
|
||||
"result-type": "validation-failure",
|
||||
"sending-mta-ip": "198.51.100.62",
|
||||
"receiving-ip": "203.0.113.58",
|
||||
"receiving-mx-hostname": "mx-backup.mail.company-y.example",
|
||||
"failed-session-count": 3,
|
||||
"failure-reason-code": "X509_V_ERR_PROXY_PATH_LENGTH_EXCEEDED"
|
||||
}]
|
||||
}]
|
||||
}`
|
||||
|
||||
func TestReport(t *testing.T) {
|
||||
mox.ConfigStaticPath = "../testdata/tlsrpt/fake.conf"
|
||||
mox.Conf.Static.DataDir = "."
|
||||
// Recognize as configured domain.
|
||||
mox.Conf.Dynamic.Domains = map[string]config.Domain{
|
||||
"test.xmox.nl": {},
|
||||
}
|
||||
|
||||
dbpath := mox.DataDirPath("tlsrpt.db")
|
||||
os.MkdirAll(filepath.Dir(dbpath), 0770)
|
||||
defer os.Remove(dbpath)
|
||||
|
||||
if err := Init(); err != nil {
|
||||
t.Fatalf("init database: %s", err)
|
||||
}
|
||||
defer Close()
|
||||
|
||||
files, err := os.ReadDir("../testdata/tlsreports")
|
||||
if err != nil {
|
||||
t.Fatalf("listing reports: %s", err)
|
||||
}
|
||||
for _, file := range files {
|
||||
f, err := os.Open("../testdata/tlsreports/" + file.Name())
|
||||
if err != nil {
|
||||
t.Fatalf("open %q: %s", file, err)
|
||||
}
|
||||
report, err := tlsrpt.ParseMessage(f)
|
||||
f.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("parsing TLSRPT from message %q: %s", file.Name(), err)
|
||||
}
|
||||
if err := AddReport(context.Background(), dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", report); err != nil {
|
||||
t.Fatalf("adding report to database: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
report, err := tlsrpt.Parse(strings.NewReader(reportJSON))
|
||||
if err != nil {
|
||||
t.Fatalf("parsing report: %v", err)
|
||||
} else if err := AddReport(context.Background(), dns.Domain{ASCII: "company-y.example"}, "tlsrpt@company-y.example", report); err != nil {
|
||||
t.Fatalf("adding report to database: %s", err)
|
||||
}
|
||||
|
||||
records, err := Records(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("fetching records: %s", err)
|
||||
}
|
||||
for _, r := range records {
|
||||
if r.FromDomain != "company-y.example" {
|
||||
continue
|
||||
}
|
||||
if !reflect.DeepEqual(&r.Report, report) {
|
||||
t.Fatalf("report, got %#v, expected %#v", r.Report, report)
|
||||
}
|
||||
if _, err := RecordID(context.Background(), r.ID); err != nil {
|
||||
t.Fatalf("get record by id: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
start, _ := time.Parse(time.RFC3339, "2016-04-01T00:00:00Z")
|
||||
end, _ := time.Parse(time.RFC3339, "2016-04-01T23:59:59Z")
|
||||
records, err = RecordsPeriodDomain(context.Background(), start, end, "test.xmox.nl")
|
||||
if err != nil || len(records) != 1 {
|
||||
t.Fatalf("got err %v, records %#v, expected no error with 1 record", err, records)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user