mox!

2025-07-12 17:44:35 +03:00 · 2023-01-30 14:27:06 +01:00
commit cb229cb6cf
1256 changed files with 491723 additions and 0 deletions
--- a/dns/dns.go
+++ b/dns/dns.go
@ -0,0 +1,109 @@
+// Package dns helps parse internationalized domain names (IDNA), canonicalize
+// names and provides a strict and metrics-keeping logging DNS resolver.
+package dns
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+
+	"golang.org/x/net/idna"
+)
+
+var errTrailingDot = errors.New("dns name has trailing dot")
+
+// Domain is a domain name, with one or more labels, with at least an ASCII
+// representation, and for IDNA non-ASCII domains a unicode representation.
+// The ASCII string must be used for DNS lookups.
+type Domain struct {
+	// A non-unicode domain, e.g. with A-labels (xn--...) or NR-LDH (non-reserved
+	// letters/digits/hyphens) labels. Always in lower case.
+	ASCII string
+
+	// Name as U-labels. Empty if this is an ASCII-only domain.
+	Unicode string
+}
+
+// Name returns the unicode name if set, otherwise the ASCII name.
+func (d Domain) Name() string {
+	if d.Unicode != "" {
+		return d.Unicode
+	}
+	return d.ASCII
+}
+
+// XName is like Name, but only returns a unicode name when utf8 is true.
+func (d Domain) XName(utf8 bool) string {
+	if utf8 && d.Unicode != "" {
+		return d.Unicode
+	}
+	return d.ASCII
+}
+
+// ASCIIExtra returns the ASCII version of the domain name if smtputf8 is true and
+// this is a unicode domain name. Otherwise it returns an empty string.
+//
+// This function is used to add the punycode name in a comment to SMTP message
+// headers, e.g. Received and Authentication-Results.
+func (d Domain) ASCIIExtra(smtputf8 bool) string {
+	if smtputf8 && d.Unicode != "" {
+		return d.ASCII
+	}
+	return ""
+}
+
+// Strings returns a human-readable string.
+// For IDNA names, the string contains both the unicode and ASCII name.
+func (d Domain) String() string {
+	if d.Unicode == "" {
+		return d.ASCII
+	}
+	return d.Unicode + "/" + d.ASCII
+}
+
+// IsZero returns if this is an empty Domain.
+func (d Domain) IsZero() bool {
+	return d == Domain{}
+}
+
+// ParseDomain parses a domain name that can consist of ASCII-only labels or U
+// labels (unicode).
+// Names are IDN-canonicalized and lower-cased.
+// Characters in unicode can be replaced by equivalents. E.g. "Ⓡ" to "r". This
+// means you should only compare parsed domain names, never strings directly.
+func ParseDomain(s string) (Domain, error) {
+	if strings.HasSuffix(s, ".") {
+		return Domain{}, errTrailingDot
+	}
+	ascii, err := idna.Lookup.ToASCII(s)
+	if err != nil {
+		return Domain{}, fmt.Errorf("to ascii: %w", err)
+	}
+	unicode, err := idna.Lookup.ToUnicode(s)
+	if err != nil {
+		return Domain{}, fmt.Errorf("to unicode: %w", err)
+	}
+	// todo: should we cause errors for unicode domains that were not in
+	// canonical form? we are now accepting all kinds of obscure spellings
+	// for even a basic ASCII domain name.
+	// Also see https://daniel.haxx.se/blog/2022/12/14/idn-is-crazy/
+	if ascii == unicode {
+		return Domain{ascii, ""}, nil
+	}
+	return Domain{ascii, unicode}, nil
+}
+
+// IsNotFound returns whether an error is a net.DNSError with IsNotFound set.
+// IsNotFound means the requested type does not exist for the given domain (a
+// nodata or nxdomain response). It doesn't not necessarily mean no other types
+// for that name exist.
+//
+// A DNS server can respond to a lookup with an error "nxdomain" to indicate a
+// name does not exist (at all), or with a success status with an empty list.
+// The Go resolver returns an IsNotFound error for both cases, there is no need
+// to explicitly check for zero entries.
+func IsNotFound(err error) bool {
+	var dnsErr *net.DNSError
+	return err != nil && errors.As(err, &dnsErr) && dnsErr.IsNotFound
+}
--- a/dns/dns_test.go
+++ b/dns/dns_test.go
@ -0,0 +1,27 @@
+package dns
+
+import (
+	"errors"
+	"testing"
+)
+
+func TestParseDomain(t *testing.T) {
+	test := func(s string, exp Domain, expErr error) {
+		t.Helper()
+		dom, err := ParseDomain(s)
+		if (err == nil) != (expErr == nil) || expErr != nil && !errors.Is(err, expErr) {
+			t.Fatalf("parse domain %q: err %v, expected %v", s, err, expErr)
+		}
+		if expErr == nil && dom != exp {
+			t.Fatalf("parse domain %q: got %#v, epxected %#v", s, dom, exp)
+		}
+	}
+
+	// We rely on normalization of names throughout the code base.
+	test("xmox.nl", Domain{"xmox.nl", ""}, nil)
+	test("XMOX.NL", Domain{"xmox.nl", ""}, nil)
+	test("TEST☺.XMOX.NL", Domain{"xn--test-3o3b.xmox.nl", "test☺.xmox.nl"}, nil)
+	test("TEST☺.XMOX.NL", Domain{"xn--test-3o3b.xmox.nl", "test☺.xmox.nl"}, nil)
+	test("ℂᵤⓇℒ。𝐒🄴", Domain{"curl.se", ""}, nil) // https://daniel.haxx.se/blog/2022/12/14/idn-is-crazy/
+	test("xmox.nl.", Domain{}, errTrailingDot)
+}
--- a/dns/ipdomain.go
+++ b/dns/ipdomain.go
@ -0,0 +1,42 @@
+package dns
+
+import (
+	"net"
+)
+
+// IPDomain is an ip address, a domain, or empty.
+type IPDomain struct {
+	IP     net.IP
+	Domain Domain
+}
+
+// IsZero returns if both IP and Domain are zero.
+func (d IPDomain) IsZero() bool {
+	return d.IP == nil && d.Domain == Domain{}
+}
+
+// String returns a string representation of either the IP or domain (with
+// UTF-8).
+func (d IPDomain) String() string {
+	if len(d.IP) > 0 {
+		return d.IP.String()
+	}
+	return d.Domain.Name()
+}
+
+// XString is like String, but only returns UTF-8 domains if utf8 is true.
+func (d IPDomain) XString(utf8 bool) string {
+	if d.IsIP() {
+		// todo: check callers if this is valid syntax for them. should we add [] for ipv6? perhaps also ipv4? probably depends on context. in smtp, the syntax is [<ipv4>] and [IPv6:<ipv6>].
+		return d.IP.String()
+	}
+	return d.Domain.XName(utf8)
+}
+
+func (d IPDomain) IsIP() bool {
+	return len(d.IP) > 0
+}
+
+func (d IPDomain) IsDomain() bool {
+	return !d.Domain.IsZero()
+}
--- a/dns/mock.go
+++ b/dns/mock.go
@ -0,0 +1,156 @@
+package dns
+
+import (
+	"context"
+	"fmt"
+	"net"
+)
+
+// MockResolver is a Resolver used for testing.
+// Set DNS records in the fields, which map FQDNs (with trailing dot) to values.
+type MockResolver struct {
+	PTR   map[string][]string
+	A     map[string][]string
+	AAAA  map[string][]string
+	TXT   map[string][]string
+	MX    map[string][]*net.MX
+	CNAME map[string]string
+	Fail  map[Mockreq]struct{}
+}
+
+type Mockreq struct {
+	Type string // E.g. "cname", "txt", "mx", "ptr", etc.
+	Name string
+}
+
+var _ Resolver = MockResolver{}
+
+func (r MockResolver) nxdomain(s string) *net.DNSError {
+	return &net.DNSError{
+		Err:        "no record",
+		Name:       s,
+		Server:     "localhost",
+		IsNotFound: true,
+	}
+}
+
+func (r MockResolver) servfail(s string) *net.DNSError {
+	return &net.DNSError{
+		Err:         "temp error",
+		Name:        s,
+		Server:      "localhost",
+		IsTemporary: true,
+	}
+}
+
+func (r MockResolver) LookupCNAME(ctx context.Context, name string) (string, error) {
+	if _, ok := r.Fail[Mockreq{"cname", name}]; ok {
+		return "", r.servfail(name)
+	}
+	if cname, ok := r.CNAME[name]; ok {
+		return cname, nil
+	}
+	return "", r.nxdomain(name)
+}
+
+func (r MockResolver) LookupAddr(ctx context.Context, ip string) ([]string, error) {
+	if _, ok := r.Fail[Mockreq{"ptr", ip}]; ok {
+		return nil, r.servfail(ip)
+	}
+	l, ok := r.PTR[ip]
+	if !ok {
+		return nil, r.nxdomain(ip)
+	}
+	return l, nil
+}
+
+func (r MockResolver) LookupNS(ctx context.Context, name string) ([]*net.NS, error) {
+	return nil, r.servfail("ns not implemented")
+}
+
+func (r MockResolver) LookupPort(ctx context.Context, network, service string) (port int, err error) {
+	return 0, r.servfail("port not implemented")
+}
+
+func (r MockResolver) LookupSRV(ctx context.Context, service, proto, name string) (string, []*net.SRV, error) {
+	return "", nil, r.servfail("srv not implemented")
+}
+
+func (r MockResolver) LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error) {
+	if _, ok := r.Fail[Mockreq{"ipaddr", host}]; ok {
+		return nil, r.servfail(host)
+	}
+	addrs, err := r.LookupHost(ctx, host)
+	if err != nil {
+		return nil, err
+	}
+	ips := make([]net.IPAddr, len(addrs))
+	for i, a := range addrs {
+		ip := net.ParseIP(a)
+		if ip == nil {
+			return nil, fmt.Errorf("malformed ip %q", a)
+		}
+		ips[i] = net.IPAddr{IP: ip}
+	}
+	return ips, nil
+}
+
+func (r MockResolver) LookupHost(ctx context.Context, host string) (addrs []string, err error) {
+	if _, ok := r.Fail[Mockreq{"host", host}]; ok {
+		return nil, r.servfail(host)
+	}
+	addrs = append(addrs, r.A[host]...)
+	addrs = append(addrs, r.AAAA[host]...)
+	if len(addrs) > 0 {
+		return addrs, nil
+	}
+	if cname, ok := r.CNAME[host]; ok {
+		return []string{cname}, nil
+	}
+	return nil, r.nxdomain(host)
+}
+
+func (r MockResolver) LookupIP(ctx context.Context, network, host string) ([]net.IP, error) {
+	if _, ok := r.Fail[Mockreq{"ip", host}]; ok {
+		return nil, r.servfail(host)
+	}
+	var ips []net.IP
+	switch network {
+	case "ip", "ip4":
+		for _, ip := range r.A[host] {
+			ips = append(ips, net.ParseIP(ip))
+		}
+	}
+	switch network {
+	case "ip", "ip6":
+		for _, ip := range r.AAAA[host] {
+			ips = append(ips, net.ParseIP(ip))
+		}
+	}
+	if len(ips) == 0 {
+		return nil, r.nxdomain(host)
+	}
+	return ips, nil
+}
+
+func (r MockResolver) LookupMX(ctx context.Context, name string) ([]*net.MX, error) {
+	if _, ok := r.Fail[Mockreq{"mx", name}]; ok {
+		return nil, r.servfail(name)
+	}
+	l, ok := r.MX[name]
+	if !ok {
+		return nil, r.nxdomain(name)
+	}
+	return l, nil
+}
+
+func (r MockResolver) LookupTXT(ctx context.Context, name string) ([]string, error) {
+	if _, ok := r.Fail[Mockreq{"txt", name}]; ok {
+		return nil, r.servfail(name)
+	}
+	l, ok := r.TXT[name]
+	if !ok {
+		return nil, r.nxdomain(name)
+	}
+	return l, nil
+}
--- a/dns/resolver.go
+++ b/dns/resolver.go
@ -0,0 +1,248 @@
+package dns
+
+import (
+	"context"
+	"errors"
+	"net"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	"github.com/mjl-/mox/mlog"
+)
+
+// todo future: replace with a dnssec capable resolver
+// todo future: change to interface that is closer to DNS. 1. expose nxdomain vs success with zero entries: nxdomain means the name does not exist for any dns resource record type, success with zero records means the name exists for other types than the requested type; 2. add ability to not follow cname records when resolving. the net resolver automatically follows cnames for LookupHost, LookupIP, LookupIPAddr. when resolving names found in mx records, we explicitly must not follow cnames. that seems impossible at the moment. 3. when looking up a cname, actually lookup the record? "net" LookupCNAME will return the requested name with no error if there is no CNAME record. because it returns the canonical name.
+// todo future: add option to not use anything in the cache, for the admin pages where you check the latest DNS settings, ignoring old cached info.
+
+var xlog = mlog.New("dns")
+
+var (
+	metricLookup = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "mox_dns_lookup_duration_seconds",
+			Help:    "DNS lookups.",
+			Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20, 30},
+		},
+		[]string{
+			"pkg",
+			"type",   // Lower-case Resolver method name without leading Lookup.
+			"result", // ok, nxdomain, temporary, timeout, canceled, error
+		},
+	)
+)
+
+// Resolver is the interface strict resolver implements.
+type Resolver interface {
+	LookupAddr(ctx context.Context, addr string) ([]string, error)
+	LookupCNAME(ctx context.Context, host string) (string, error) // NOTE: returns an error if no CNAME record is present.
+	LookupHost(ctx context.Context, host string) (addrs []string, err error)
+	LookupIP(ctx context.Context, network, host string) ([]net.IP, error)
+	LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error)
+	LookupMX(ctx context.Context, name string) ([]*net.MX, error)
+	LookupNS(ctx context.Context, name string) ([]*net.NS, error)
+	LookupPort(ctx context.Context, network, service string) (port int, err error)
+	LookupSRV(ctx context.Context, service, proto, name string) (string, []*net.SRV, error)
+	LookupTXT(ctx context.Context, name string) ([]string, error)
+}
+
+// WithPackage sets Pkg on resolver if it is a StrictResolve and does not have a package set yet.
+func WithPackage(resolver Resolver, name string) Resolver {
+	r, ok := resolver.(StrictResolver)
+	if ok && r.Pkg == "" {
+		nr := r
+		r.Pkg = name
+		return nr
+	}
+	return resolver
+}
+
+// StrictResolver is a net.Resolver that enforces that DNS names end with a dot,
+// preventing "search"-relative lookups.
+type StrictResolver struct {
+	Pkg      string        // Name of subsystem that is making DNS requests, for metrics.
+	Resolver *net.Resolver // Where the actual lookups are done. If nil, net.DefaultResolver is used for lookups.
+}
+
+var _ Resolver = StrictResolver{}
+
+var ErrRelativeDNSName = errors.New("dns: host to lookup must be absolute, ending with a dot")
+
+func metricLookupObserve(pkg, typ string, err error, start time.Time) {
+	var result string
+	var dnsErr *net.DNSError
+	switch {
+	case err == nil:
+		result = "ok"
+	case errors.As(err, &dnsErr) && dnsErr.IsNotFound:
+		result = "nxdomain"
+	case errors.As(err, &dnsErr) && dnsErr.IsTemporary:
+		result = "temporary"
+	case errors.Is(err, os.ErrDeadlineExceeded) || errors.Is(err, context.DeadlineExceeded) || errors.As(err, &dnsErr) && dnsErr.IsTimeout:
+		result = "timeout"
+	case errors.Is(err, context.Canceled):
+		result = "canceled"
+	default:
+		result = "error"
+	}
+	metricLookup.WithLabelValues(pkg, typ, result).Observe(float64(time.Since(start)) / float64(time.Second))
+}
+
+func (r StrictResolver) WithPackage(name string) Resolver {
+	nr := r
+	nr.Pkg = name
+	return nr
+}
+
+func (r StrictResolver) resolver() Resolver {
+	if r.Resolver == nil {
+		return net.DefaultResolver
+	}
+	return r.Resolver
+}
+
+func (r StrictResolver) LookupAddr(ctx context.Context, addr string) (resp []string, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "addr", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "addr"), mlog.Field("addr", addr), mlog.Field("resp", resp), mlog.Field("duration", time.Since(start)))
+	}()
+
+	resp, err = r.resolver().LookupAddr(ctx, addr)
+	return
+}
+
+// LookupCNAME looks up a CNAME. Unlike "net" LookupCNAME, it returns a "not found"
+// error if there is no CNAME record.
+func (r StrictResolver) LookupCNAME(ctx context.Context, host string) (resp string, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "cname", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "cname"), mlog.Field("host", host), mlog.Field("resp", resp), mlog.Field("duration", time.Since(start)))
+	}()
+
+	if !strings.HasSuffix(host, ".") {
+		return "", ErrRelativeDNSName
+	}
+	resp, err = r.resolver().LookupCNAME(ctx, host)
+	if err == nil && resp == host {
+		return "", &net.DNSError{
+			Err:        "no cname record",
+			Name:       host,
+			Server:     "",
+			IsNotFound: true,
+		}
+	}
+	return
+}
+func (r StrictResolver) LookupHost(ctx context.Context, host string) (resp []string, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "host", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "host"), mlog.Field("host", host), mlog.Field("resp", resp), mlog.Field("duration", time.Since(start)))
+	}()
+
+	if !strings.HasSuffix(host, ".") {
+		return nil, ErrRelativeDNSName
+	}
+	resp, err = r.resolver().LookupHost(ctx, host)
+	return
+}
+
+func (r StrictResolver) LookupIP(ctx context.Context, network, host string) (resp []net.IP, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "ip", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "ip"), mlog.Field("network", network), mlog.Field("host", host), mlog.Field("resp", resp), mlog.Field("duration", time.Since(start)))
+	}()
+
+	if !strings.HasSuffix(host, ".") {
+		return nil, ErrRelativeDNSName
+	}
+	resp, err = r.resolver().LookupIP(ctx, network, host)
+	return
+}
+
+func (r StrictResolver) LookupIPAddr(ctx context.Context, host string) (resp []net.IPAddr, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "ipaddr", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "ipaddr"), mlog.Field("host", host), mlog.Field("resp", resp), mlog.Field("duration", time.Since(start)))
+	}()
+
+	if !strings.HasSuffix(host, ".") {
+		return nil, ErrRelativeDNSName
+	}
+	resp, err = r.resolver().LookupIPAddr(ctx, host)
+	return
+}
+
+func (r StrictResolver) LookupMX(ctx context.Context, name string) (resp []*net.MX, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "mx", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "mx"), mlog.Field("name", name), mlog.Field("resp", resp), mlog.Field("duration", time.Since(start)))
+	}()
+
+	if !strings.HasSuffix(name, ".") {
+		return nil, ErrRelativeDNSName
+	}
+	resp, err = r.resolver().LookupMX(ctx, name)
+	return
+}
+
+func (r StrictResolver) LookupNS(ctx context.Context, name string) (resp []*net.NS, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "ns", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "ns"), mlog.Field("name", name), mlog.Field("resp", resp), mlog.Field("duration", time.Since(start)))
+	}()
+
+	if !strings.HasSuffix(name, ".") {
+		return nil, ErrRelativeDNSName
+	}
+	resp, err = r.resolver().LookupNS(ctx, name)
+	return
+}
+
+func (r StrictResolver) LookupPort(ctx context.Context, network, service string) (resp int, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "port", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "port"), mlog.Field("network", network), mlog.Field("service", service), mlog.Field("resp", resp), mlog.Field("duration", time.Since(start)))
+	}()
+
+	resp, err = r.resolver().LookupPort(ctx, network, service)
+	return
+}
+
+func (r StrictResolver) LookupSRV(ctx context.Context, service, proto, name string) (resp0 string, resp1 []*net.SRV, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "srv", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "srv"), mlog.Field("service", service), mlog.Field("proto", proto), mlog.Field("name", name), mlog.Field("resp0", resp0), mlog.Field("resp1", resp1), mlog.Field("duration", time.Since(start)))
+	}()
+
+	if !strings.HasSuffix(name, ".") {
+		return "", nil, ErrRelativeDNSName
+	}
+	resp0, resp1, err = r.resolver().LookupSRV(ctx, service, proto, name)
+	return
+}
+
+func (r StrictResolver) LookupTXT(ctx context.Context, name string) (resp []string, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "txt", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "txt"), mlog.Field("name", name), mlog.Field("resp", resp), mlog.Field("duration", time.Since(start)))
+	}()
+
+	if !strings.HasSuffix(name, ".") {
+		return nil, ErrRelativeDNSName
+	}
+	resp, err = r.resolver().LookupTXT(ctx, name)
+	return
+}