quickstart: check if domain was registered recently, and warn about potential deliverability issues

we use 6 weeks as the cutoff, but this is fuzzy, and will vary by mail server/service provider. we check the domain age using RDAP, the replacement for whois. it is a relatively simple protocol, with HTTP/JSON requests. we fetch the "registration"-related events to look for a date of registration. RDAP is not available for all country-level TLDs, but is for most (all?) ICANN global top level domains. some random cctlds i noticed without rdap: .sh, .au, .io. the rdap implementation is very basic, only parsing the fields we need. we don't yet cache the dns registry bootstrap file from iana. we should once we use this functionality from the web interface, with more calls.
2025-06-28 01:08:15 +03:00 · 2025-02-07 11:16:30 +01:00 · 2025-02-07 11:16:30 +01:00 · 2f0997682b
commit 2f0997682b
parent c7354cc22b
5 changed files with 322 additions and 3 deletions
--- a/doc.go
+++ b/doc.go
@ -110,6 +110,7 @@ any parameters. Followed by the help and usage information for each command.
 	mox dnsbl check zone ip
 	mox dnsbl checkhealth zone
 	mox mtasts lookup domain
+	mox rdap domainage domain
 	mox retrain [accountname]
 	mox sendmail [-Fname] [ignoredflags] [-t] [<message]
 	mox spf check domain ip
@ -186,7 +187,7 @@ output of "mox config describe-domains" and see the output of
 	  -hostname string
 	    	hostname mox will run on, by default the hostname of the machine quickstart runs on; if specified, the IPs for the hostname are configured for the public listener
 	  -skipdial
-	    	skip check for outgoing smtp (port 25) connectivity
+	    	skip check for outgoing smtp (port 25) connectivity or for domain age with rdap

 # mox stop

@ -1456,6 +1457,24 @@ should be used, and how long the policy can be cached.

 	usage: mox mtasts lookup domain

+# mox rdap domainage
+
+Lookup the age of domain in RDAP based on latest registration.
+
+RDAP is the registration data access protocol. Registries run RDAP services for
+their top level domains, providing information such as the registration date of
+domains. This command looks up the "age" of a domain by looking at the most
+recent "registration", "reregistration" or "reinstantiation" event.
+
+Email messages from recently registered domains are often treated with
+suspicion, and some mail systems are more likely to classify them as junk.
+
+On each invocation, a bootstrap file with a list of registries (of top-level
+domains) is retrieved, without caching. Do not run this command too often with
+automation.
+
+	usage: mox rdap domainage domain
+
 # mox retrain

 Recreate and retrain the junk filter for the account or all accounts.
--- a/main.go
+++ b/main.go
@ -61,6 +61,7 @@ import (
 	"github.com/mjl-/mox/mtasts"
 	"github.com/mjl-/mox/publicsuffix"
 	"github.com/mjl-/mox/queue"
+	"github.com/mjl-/mox/rdap"
 	"github.com/mjl-/mox/smtp"
 	"github.com/mjl-/mox/smtpclient"
 	"github.com/mjl-/mox/spf"
@ -195,6 +196,7 @@ var commands = []struct {
 	{"dnsbl check", cmdDNSBLCheck},
 	{"dnsbl checkhealth", cmdDNSBLCheckhealth},
 	{"mtasts lookup", cmdMTASTSLookup},
+	{"rdap domainage", cmdRDAPDomainage},
 	{"retrain", cmdRetrain},
 	{"sendmail", cmdSendmail},
 	{"spf check", cmdSPFCheck},
@ -2988,6 +2990,51 @@ should be used, and how long the policy can be cached.
 	}
 }

+func cmdRDAPDomainage(c *cmd) {
+	c.params = "domain"
+	c.help = `Lookup the age of domain in RDAP based on latest registration.
+
+RDAP is the registration data access protocol. Registries run RDAP services for
+their top level domains, providing information such as the registration date of
+domains. This command looks up the "age" of a domain by looking at the most
+recent "registration", "reregistration" or "reinstantiation" event.
+
+Email messages from recently registered domains are often treated with
+suspicion, and some mail systems are more likely to classify them as junk.
+
+On each invocation, a bootstrap file with a list of registries (of top-level
+domains) is retrieved, without caching. Do not run this command too often with
+automation.
+`
+	args := c.Parse()
+	if len(args) != 1 {
+		c.Usage()
+	}
+
+	domain := xparseDomain(args[0], "domain")
+
+	registration, err := rdap.LookupLastDomainRegistration(context.Background(), domain)
+	xcheckf(err, "looking up domain in rdap")
+
+	age := time.Since(registration)
+	const day = 24 * time.Hour
+	const year = 365 * day
+	years := age / year
+	days := (age - years*year) / day
+	var s string
+	if years == 1 {
+		s = "1 year, "
+	} else if years > 0 {
+		s = fmt.Sprintf("%d years, ", years)
+	}
+	if days == 1 {
+		s += "1 day"
+	} else {
+		s += fmt.Sprintf("%d days", days)
+	}
+	fmt.Println(s)
+}
+
 func cmdRetrain(c *cmd) {
 	c.params = "[accountname]"
 	c.help = `Recreate and retrain the junk filter for the account or all accounts.
--- a/quickstart.go
+++ b/quickstart.go
@ -35,6 +35,8 @@ import (
 	"github.com/mjl-/mox/dnsbl"
 	"github.com/mjl-/mox/mlog"
 	"github.com/mjl-/mox/mox-"
+	"github.com/mjl-/mox/publicsuffix"
+	"github.com/mjl-/mox/rdap"
 	"github.com/mjl-/mox/smtp"
 	"github.com/mjl-/mox/store"
 )
@ -102,7 +104,7 @@ output of "mox config describe-domains" and see the output of
 	var skipDial bool
 	c.flag.BoolVar(&existingWebserver, "existing-webserver", false, "use if a webserver is already running, so mox won't listen on port 80 and 443; you'll have to provide tls certificates/keys, and configure the existing webserver as reverse proxy, forwarding requests to mox.")
 	c.flag.StringVar(&hostname, "hostname", "", "hostname mox will run on, by default the hostname of the machine quickstart runs on; if specified, the IPs for the hostname are configured for the public listener")
-	c.flag.BoolVar(&skipDial, "skipdial", false, "skip check for outgoing smtp (port 25) connectivity")
+	c.flag.BoolVar(&skipDial, "skipdial", false, "skip check for outgoing smtp (port 25) connectivity or for domain age with rdap")
 	args := c.Parse()
 	if len(args) != 1 && len(args) != 2 {
 		c.Usage()
@ -582,8 +584,8 @@ messages over SMTP.
 		}
 	}

-	// Check outgoing SMTP connectivity.
 	if !skipDial {
+		// Check outgoing SMTP connectivity.
 		fmt.Printf("Checking if outgoing smtp connections can be made by connecting to gmail.com mx on port 25...")
 		mxctx, mxcancel := context.WithTimeout(context.Background(), 5*time.Second)
 		mx, _, err := resolver.LookupMX(mxctx, "gmail.com.")
@ -619,6 +621,41 @@ in mox.conf and use it in "Routes" in domains.conf. See

 `)
 		}
+
+		// Check if domain is recently registered.
+		rdapctx, rdapcancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer rdapcancel()
+		orgdom := publicsuffix.Lookup(rdapctx, c.log.Logger, domain)
+		fmt.Printf("\nChecking if domain %s was registered recently...", orgdom)
+		registration, err := rdap.LookupLastDomainRegistration(rdapctx, orgdom)
+		rdapcancel()
+		if err != nil {
+			fmt.Printf(" error: %s (continuing)\n\n", err)
+		} else {
+			age := time.Since(registration)
+			const day = 24 * time.Hour
+			const year = 365 * day
+			years := age / year
+			days := (age - years*year) / day
+			var s string
+			if years == 1 {
+				s = "1 year, "
+			} else if years > 0 {
+				s = fmt.Sprintf("%d years, ", years)
+			}
+			if days == 1 {
+				s += "1 day"
+			} else {
+				s += fmt.Sprintf("%d days", days)
+			}
+			fmt.Printf(" %s", s)
+			// 6 weeks is a guess, mail servers/service providers will have different policies.
+			if age < 6*7*day {
+				fmt.Printf(" (recent!)\nWARNING: Mail servers may treat messages coming from recently registered domains\n(in the order of weeks to months) with suspicion, with higher probability of\nmessages being classified as junk.\n\n")
+			} else {
+				fmt.Printf(" OK\n\n")
+			}
+		}
 	}

 	zones := []dns.Domain{
--- a/rdap/rdap.go
+++ b/rdap/rdap.go
@ -0,0 +1,208 @@
+// Package rdap is a basic client for checking the age of domains through RDAP.
+package rdap
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/mjl-/mox/dns"
+)
+
+var ErrNoRegistration = errors.New("registration date not found")
+var ErrNoRDAP = errors.New("rdap not available for top-level domain")
+var ErrNoDomain = errors.New("domain not found in registry")
+var ErrSyntax = errors.New("bad rdap response syntax")
+
+// https://www.iana.org/assignments/rdap-dns/rdap-dns.xhtml
+// ../rfc/9224:115
+const rdapBoostrapDNSURL = "https://data.iana.org/rdap/dns.json"
+
+// Example data: ../rfc/9224:192
+
+// Bootstrap data, parsed from JSON at the IANA DNS bootstrap URL.
+type Bootstrap struct {
+	Version     string    `json:"version"` // Should be "1.0".
+	Description string    `json:"description"`
+	Publication time.Time `json:"publication"` // RFC3339
+
+	// Each entry has two elements: First a list of TLDs, then a list of RDAP service
+	// base URLs ending with a slash.
+	Services [][2][]string `json:"services"`
+}
+
+// todo: when using this more regularly in the admin web interface, store the iana bootstrap response in a database file, including cache-controle results (max-age it seems) and the etag, and do conditional requests when asking for a new version. same for lookups of domains at registries.
+
+// LookupLastDomainRegistration looks up the most recent (re)registration of a
+// domain through RDAP.
+//
+// Not all TLDs have RDAP services yet at the time of writing.
+func LookupLastDomainRegistration(ctx context.Context, dom dns.Domain) (time.Time, error) {
+	// ../rfc/9224:434 Against advice, we do not cache the bootstrap data. This is
+	// currently used by the quickstart, which is run once, or run from the cli without
+	// a place to keep state.
+	req, err := http.NewRequestWithContext(ctx, "GET", rdapBoostrapDNSURL, nil)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("new request for iana dns bootstrap data: %v", err)
+	}
+	// ../rfc/9224:588
+	req.Header.Add("Accept", "application/json")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("http get of iana dns bootstrap data: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode/100 != 2 {
+		return time.Time{}, fmt.Errorf("http get resulted in status %q, expected 200 ok", resp.Status)
+	}
+	var bootstrap Bootstrap
+	if err := json.NewDecoder(resp.Body).Decode(&bootstrap); err != nil {
+		return time.Time{}, fmt.Errorf("%w: parsing iana dns bootstrap data: %v", ErrSyntax, err)
+	}
+
+	// Note: We don't verify version numbers. If the format change incompatibly,
+	// decoding above would have failed. We'll try to work with what we got.
+
+	// ../rfc/9224:184 The bootstrap JSON has A-labels we must match against.
+	// ../rfc/9224:188 Names are lower-case, like our dns.Domain.
+	var urls []string
+	var tldmatch string
+	for _, svc := range bootstrap.Services {
+		for _, s := range svc[0] {
+			// ../rfc/9224:225 We match the longest domain suffix. In practice, there are
+			// currently only single labels, top level domains, in the bootstrap database.
+			if len(s) > len(tldmatch) && (s == dom.ASCII || strings.HasSuffix(dom.ASCII, "."+s)) {
+				urls = svc[1]
+				tldmatch = s
+			}
+		}
+	}
+	// ../rfc/9224:428
+	if len(urls) == 0 {
+		return time.Time{}, ErrNoRDAP
+	}
+	// ../rfc/9224:172 We must try secure transports before insecure (https before http). In practice, there is just a single https URL.
+	sort.Slice(urls, func(i, j int) bool {
+		return strings.HasPrefix(urls[i], "https://")
+	})
+	var lastErr error
+	for _, u := range urls {
+		var reg time.Time
+		reg, lastErr = rdapDomainRequest(ctx, u, dom)
+		if lastErr == nil {
+			return reg, nil
+		}
+	}
+	return time.Time{}, lastErr
+}
+
+// ../rfc/9083:284 We must match json fields case-sensitively, so explicitly.
+// Example domain object: ../rfc/9083:945
+
+// Domain is the RDAP response for a domain request.
+//
+// More fields are available in RDAP responses, we only parse the one(s) a few.
+type Domain struct {
+	// ../rfc/9083:1172
+
+	RDAPConformance []string `json:"rdapConformance"` // E.g. "rdap_level_0"
+	LDHName         string   `json:"ldhName"`         // Domain.
+	Events          []Event  `json:"events"`
+}
+
+// Event is a historic or future change to the domain.
+type Event struct {
+	// ../rfc/9083:573
+
+	EventAction string    `json:"eventAction"` // Required. See https://www.iana.org/assignments/rdap-json-values/rdap-json-values.xhtml.
+	EventDate   time.Time `json:"eventDate"`   // Required. RFC3339. May be in the future, e.g. date of expiry.
+}
+
+// rdapDomainRequest looks up a the most recent registration time of a at an RDAP
+// service base URL.
+func rdapDomainRequest(ctx context.Context, rdapURL string, dom dns.Domain) (time.Time, error) {
+	// ../rfc/9082:316
+	// ../rfc/9224:177 base URLs have a trailing slash.
+	rdapURL += "domain/" + dom.ASCII
+	req, err := http.NewRequestWithContext(ctx, "GET", rdapURL, nil)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("making http request for rdap service: %v", err)
+	}
+	// ../rfc/9083:2372 ../rfc/7480:273
+	req.Header.Add("Accept", "application/rdap+json")
+	// ../rfc/7480:319 Redirects are handled by net/http.
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("http domain rdap get request: %v", err)
+	}
+	defer resp.Body.Close()
+
+	switch {
+	case resp.StatusCode == http.StatusNotFound:
+		// ../rfc/7480:189 ../rfc/7480:359
+		return time.Time{}, ErrNoDomain
+
+	case resp.StatusCode/100 != 2:
+		// We try to read an error message, perhaps a bit too hard, but we may still
+		// truncate utf-8 in the middle of a rune...
+		var msg string
+		var response struct {
+			// For errors, optional fields.
+			Title       string   `json:"title"`
+			Description []string `json:"description"`
+			// ../rfc/9083:2123
+		}
+		buf, err := io.ReadAll(io.LimitReader(resp.Body, 100*1024))
+		if err != nil {
+			msg = fmt.Sprintf("(error reading response: %v)", err)
+		} else if err := json.Unmarshal(buf, &response); err == nil && (response.Title != "" || len(response.Description) > 0) {
+			s := response.Title
+			if s != "" && len(response.Description) > 0 {
+				s += "; "
+			}
+			s += strings.Join(response.Description, " ")
+			if len(s) > 200 {
+				s = s[:150] + "..."
+			}
+			msg = fmt.Sprintf("message from remote: %q", s)
+		} else {
+			var s string
+			if len(buf) > 200 {
+				s = string(buf[:150]) + "..."
+			} else {
+				s = string(buf)
+			}
+			msg = fmt.Sprintf("raw response: %q", s)
+		}
+		return time.Time{}, fmt.Errorf("status %q, expected 200 ok: %s", resp.Status, msg)
+	}
+
+	var domain Domain
+	if err := json.NewDecoder(resp.Body).Decode(&domain); err != nil {
+		return time.Time{}, fmt.Errorf("parse domain rdap response: %v", err)
+	}
+
+	sort.Slice(domain.Events, func(i, j int) bool {
+		return domain.Events[i].EventDate.Before(domain.Events[j].EventDate)
+	})
+
+	now := time.Now()
+	for i := len(domain.Events) - 1; i >= 0; i-- {
+		ev := domain.Events[i]
+		if ev.EventDate.After(now) {
+			continue
+		}
+		switch ev.EventAction {
+		// ../rfc/9083:2690
+		case "registration", "reregistration", "reinstantiation":
+			return ev.EventDate, nil
+		}
+	}
+	return time.Time{}, ErrNoRegistration
+}
--- a/rfc/index.txt
+++ b/rfc/index.txt
@ -454,3 +454,11 @@ See implementation guide, https://jmap.io/server.html
 9077	-?	-	NSEC and NSEC3: TTLs and Aggressive Use
 9157	-?	-	Revised IANA Considerations for DNSSEC
 9276	-?	-	Guidance for NSEC3 Parameter Settings
+
+# RDAP
+7480	-	-	HTTP Usage in the Registration Data Access Protocol (RDAP)
+7481	-	-	Security Services for the Registration Data Access Protocol (RDAP)
+8056	-	-	Extensible Provisioning Protocol (EPP) and Registration Data Access Protocol (RDAP) Status Mapping
+9082	-	-	Registration Data Access Protocol (RDAP) Query Format
+9083	-	-	JSON Responses for the Registration Data Access Protocol (RDAP)
+9224	-	-	Finding the Authoritative Registration Data Access Protocol (RDAP) Service