From 2f0997682beb5c655cdc22a384cf929dbf080ab5 Mon Sep 17 00:00:00 2001 From: Mechiel Lukkien Date: Fri, 7 Feb 2025 11:16:30 +0100 Subject: [PATCH] quickstart: check if domain was registered recently, and warn about potential deliverability issues we use 6 weeks as the cutoff, but this is fuzzy, and will vary by mail server/service provider. we check the domain age using RDAP, the replacement for whois. it is a relatively simple protocol, with HTTP/JSON requests. we fetch the "registration"-related events to look for a date of registration. RDAP is not available for all country-level TLDs, but is for most (all?) ICANN global top level domains. some random cctlds i noticed without rdap: .sh, .au, .io. the rdap implementation is very basic, only parsing the fields we need. we don't yet cache the dns registry bootstrap file from iana. we should once we use this functionality from the web interface, with more calls. --- doc.go | 21 ++++- main.go | 47 ++++++++++++ quickstart.go | 41 +++++++++- rdap/rdap.go | 208 ++++++++++++++++++++++++++++++++++++++++++++++++++ rfc/index.txt | 8 ++ 5 files changed, 322 insertions(+), 3 deletions(-) create mode 100644 rdap/rdap.go diff --git a/doc.go b/doc.go index 214f402..22772c1 100644 --- a/doc.go +++ b/doc.go @@ -110,6 +110,7 @@ any parameters. Followed by the help and usage information for each command. mox dnsbl check zone ip mox dnsbl checkhealth zone mox mtasts lookup domain + mox rdap domainage domain mox retrain [accountname] mox sendmail [-Fname] [ignoredflags] [-t] [ 0 { + s = fmt.Sprintf("%d years, ", years) + } + if days == 1 { + s += "1 day" + } else { + s += fmt.Sprintf("%d days", days) + } + fmt.Println(s) +} + func cmdRetrain(c *cmd) { c.params = "[accountname]" c.help = `Recreate and retrain the junk filter for the account or all accounts. diff --git a/quickstart.go b/quickstart.go index e835c20..770d5c2 100644 --- a/quickstart.go +++ b/quickstart.go @@ -35,6 +35,8 @@ import ( "github.com/mjl-/mox/dnsbl" "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" + "github.com/mjl-/mox/publicsuffix" + "github.com/mjl-/mox/rdap" "github.com/mjl-/mox/smtp" "github.com/mjl-/mox/store" ) @@ -102,7 +104,7 @@ output of "mox config describe-domains" and see the output of var skipDial bool c.flag.BoolVar(&existingWebserver, "existing-webserver", false, "use if a webserver is already running, so mox won't listen on port 80 and 443; you'll have to provide tls certificates/keys, and configure the existing webserver as reverse proxy, forwarding requests to mox.") c.flag.StringVar(&hostname, "hostname", "", "hostname mox will run on, by default the hostname of the machine quickstart runs on; if specified, the IPs for the hostname are configured for the public listener") - c.flag.BoolVar(&skipDial, "skipdial", false, "skip check for outgoing smtp (port 25) connectivity") + c.flag.BoolVar(&skipDial, "skipdial", false, "skip check for outgoing smtp (port 25) connectivity or for domain age with rdap") args := c.Parse() if len(args) != 1 && len(args) != 2 { c.Usage() @@ -582,8 +584,8 @@ messages over SMTP. } } - // Check outgoing SMTP connectivity. if !skipDial { + // Check outgoing SMTP connectivity. fmt.Printf("Checking if outgoing smtp connections can be made by connecting to gmail.com mx on port 25...") mxctx, mxcancel := context.WithTimeout(context.Background(), 5*time.Second) mx, _, err := resolver.LookupMX(mxctx, "gmail.com.") @@ -619,6 +621,41 @@ in mox.conf and use it in "Routes" in domains.conf. See `) } + + // Check if domain is recently registered. + rdapctx, rdapcancel := context.WithTimeout(context.Background(), 10*time.Second) + defer rdapcancel() + orgdom := publicsuffix.Lookup(rdapctx, c.log.Logger, domain) + fmt.Printf("\nChecking if domain %s was registered recently...", orgdom) + registration, err := rdap.LookupLastDomainRegistration(rdapctx, orgdom) + rdapcancel() + if err != nil { + fmt.Printf(" error: %s (continuing)\n\n", err) + } else { + age := time.Since(registration) + const day = 24 * time.Hour + const year = 365 * day + years := age / year + days := (age - years*year) / day + var s string + if years == 1 { + s = "1 year, " + } else if years > 0 { + s = fmt.Sprintf("%d years, ", years) + } + if days == 1 { + s += "1 day" + } else { + s += fmt.Sprintf("%d days", days) + } + fmt.Printf(" %s", s) + // 6 weeks is a guess, mail servers/service providers will have different policies. + if age < 6*7*day { + fmt.Printf(" (recent!)\nWARNING: Mail servers may treat messages coming from recently registered domains\n(in the order of weeks to months) with suspicion, with higher probability of\nmessages being classified as junk.\n\n") + } else { + fmt.Printf(" OK\n\n") + } + } } zones := []dns.Domain{ diff --git a/rdap/rdap.go b/rdap/rdap.go new file mode 100644 index 0000000..182ae44 --- /dev/null +++ b/rdap/rdap.go @@ -0,0 +1,208 @@ +// Package rdap is a basic client for checking the age of domains through RDAP. +package rdap + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "sort" + "strings" + "time" + + "github.com/mjl-/mox/dns" +) + +var ErrNoRegistration = errors.New("registration date not found") +var ErrNoRDAP = errors.New("rdap not available for top-level domain") +var ErrNoDomain = errors.New("domain not found in registry") +var ErrSyntax = errors.New("bad rdap response syntax") + +// https://www.iana.org/assignments/rdap-dns/rdap-dns.xhtml +// ../rfc/9224:115 +const rdapBoostrapDNSURL = "https://data.iana.org/rdap/dns.json" + +// Example data: ../rfc/9224:192 + +// Bootstrap data, parsed from JSON at the IANA DNS bootstrap URL. +type Bootstrap struct { + Version string `json:"version"` // Should be "1.0". + Description string `json:"description"` + Publication time.Time `json:"publication"` // RFC3339 + + // Each entry has two elements: First a list of TLDs, then a list of RDAP service + // base URLs ending with a slash. + Services [][2][]string `json:"services"` +} + +// todo: when using this more regularly in the admin web interface, store the iana bootstrap response in a database file, including cache-controle results (max-age it seems) and the etag, and do conditional requests when asking for a new version. same for lookups of domains at registries. + +// LookupLastDomainRegistration looks up the most recent (re)registration of a +// domain through RDAP. +// +// Not all TLDs have RDAP services yet at the time of writing. +func LookupLastDomainRegistration(ctx context.Context, dom dns.Domain) (time.Time, error) { + // ../rfc/9224:434 Against advice, we do not cache the bootstrap data. This is + // currently used by the quickstart, which is run once, or run from the cli without + // a place to keep state. + req, err := http.NewRequestWithContext(ctx, "GET", rdapBoostrapDNSURL, nil) + if err != nil { + return time.Time{}, fmt.Errorf("new request for iana dns bootstrap data: %v", err) + } + // ../rfc/9224:588 + req.Header.Add("Accept", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return time.Time{}, fmt.Errorf("http get of iana dns bootstrap data: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + return time.Time{}, fmt.Errorf("http get resulted in status %q, expected 200 ok", resp.Status) + } + var bootstrap Bootstrap + if err := json.NewDecoder(resp.Body).Decode(&bootstrap); err != nil { + return time.Time{}, fmt.Errorf("%w: parsing iana dns bootstrap data: %v", ErrSyntax, err) + } + + // Note: We don't verify version numbers. If the format change incompatibly, + // decoding above would have failed. We'll try to work with what we got. + + // ../rfc/9224:184 The bootstrap JSON has A-labels we must match against. + // ../rfc/9224:188 Names are lower-case, like our dns.Domain. + var urls []string + var tldmatch string + for _, svc := range bootstrap.Services { + for _, s := range svc[0] { + // ../rfc/9224:225 We match the longest domain suffix. In practice, there are + // currently only single labels, top level domains, in the bootstrap database. + if len(s) > len(tldmatch) && (s == dom.ASCII || strings.HasSuffix(dom.ASCII, "."+s)) { + urls = svc[1] + tldmatch = s + } + } + } + // ../rfc/9224:428 + if len(urls) == 0 { + return time.Time{}, ErrNoRDAP + } + // ../rfc/9224:172 We must try secure transports before insecure (https before http). In practice, there is just a single https URL. + sort.Slice(urls, func(i, j int) bool { + return strings.HasPrefix(urls[i], "https://") + }) + var lastErr error + for _, u := range urls { + var reg time.Time + reg, lastErr = rdapDomainRequest(ctx, u, dom) + if lastErr == nil { + return reg, nil + } + } + return time.Time{}, lastErr +} + +// ../rfc/9083:284 We must match json fields case-sensitively, so explicitly. +// Example domain object: ../rfc/9083:945 + +// Domain is the RDAP response for a domain request. +// +// More fields are available in RDAP responses, we only parse the one(s) a few. +type Domain struct { + // ../rfc/9083:1172 + + RDAPConformance []string `json:"rdapConformance"` // E.g. "rdap_level_0" + LDHName string `json:"ldhName"` // Domain. + Events []Event `json:"events"` +} + +// Event is a historic or future change to the domain. +type Event struct { + // ../rfc/9083:573 + + EventAction string `json:"eventAction"` // Required. See https://www.iana.org/assignments/rdap-json-values/rdap-json-values.xhtml. + EventDate time.Time `json:"eventDate"` // Required. RFC3339. May be in the future, e.g. date of expiry. +} + +// rdapDomainRequest looks up a the most recent registration time of a at an RDAP +// service base URL. +func rdapDomainRequest(ctx context.Context, rdapURL string, dom dns.Domain) (time.Time, error) { + // ../rfc/9082:316 + // ../rfc/9224:177 base URLs have a trailing slash. + rdapURL += "domain/" + dom.ASCII + req, err := http.NewRequestWithContext(ctx, "GET", rdapURL, nil) + if err != nil { + return time.Time{}, fmt.Errorf("making http request for rdap service: %v", err) + } + // ../rfc/9083:2372 ../rfc/7480:273 + req.Header.Add("Accept", "application/rdap+json") + // ../rfc/7480:319 Redirects are handled by net/http. + resp, err := http.DefaultClient.Do(req) + if err != nil { + return time.Time{}, fmt.Errorf("http domain rdap get request: %v", err) + } + defer resp.Body.Close() + + switch { + case resp.StatusCode == http.StatusNotFound: + // ../rfc/7480:189 ../rfc/7480:359 + return time.Time{}, ErrNoDomain + + case resp.StatusCode/100 != 2: + // We try to read an error message, perhaps a bit too hard, but we may still + // truncate utf-8 in the middle of a rune... + var msg string + var response struct { + // For errors, optional fields. + Title string `json:"title"` + Description []string `json:"description"` + // ../rfc/9083:2123 + } + buf, err := io.ReadAll(io.LimitReader(resp.Body, 100*1024)) + if err != nil { + msg = fmt.Sprintf("(error reading response: %v)", err) + } else if err := json.Unmarshal(buf, &response); err == nil && (response.Title != "" || len(response.Description) > 0) { + s := response.Title + if s != "" && len(response.Description) > 0 { + s += "; " + } + s += strings.Join(response.Description, " ") + if len(s) > 200 { + s = s[:150] + "..." + } + msg = fmt.Sprintf("message from remote: %q", s) + } else { + var s string + if len(buf) > 200 { + s = string(buf[:150]) + "..." + } else { + s = string(buf) + } + msg = fmt.Sprintf("raw response: %q", s) + } + return time.Time{}, fmt.Errorf("status %q, expected 200 ok: %s", resp.Status, msg) + } + + var domain Domain + if err := json.NewDecoder(resp.Body).Decode(&domain); err != nil { + return time.Time{}, fmt.Errorf("parse domain rdap response: %v", err) + } + + sort.Slice(domain.Events, func(i, j int) bool { + return domain.Events[i].EventDate.Before(domain.Events[j].EventDate) + }) + + now := time.Now() + for i := len(domain.Events) - 1; i >= 0; i-- { + ev := domain.Events[i] + if ev.EventDate.After(now) { + continue + } + switch ev.EventAction { + // ../rfc/9083:2690 + case "registration", "reregistration", "reinstantiation": + return ev.EventDate, nil + } + } + return time.Time{}, ErrNoRegistration +} diff --git a/rfc/index.txt b/rfc/index.txt index 459dcd9..06193c3 100644 --- a/rfc/index.txt +++ b/rfc/index.txt @@ -454,3 +454,11 @@ See implementation guide, https://jmap.io/server.html 9077 -? - NSEC and NSEC3: TTLs and Aggressive Use 9157 -? - Revised IANA Considerations for DNSSEC 9276 -? - Guidance for NSEC3 Parameter Settings + +# RDAP +7480 - - HTTP Usage in the Registration Data Access Protocol (RDAP) +7481 - - Security Services for the Registration Data Access Protocol (RDAP) +8056 - - Extensible Provisioning Protocol (EPP) and Registration Data Access Protocol (RDAP) Status Mapping +9082 - - Registration Data Access Protocol (RDAP) Query Format +9083 - - JSON Responses for the Registration Data Access Protocol (RDAP) +9224 - - Finding the Authoritative Registration Data Access Protocol (RDAP) Service