infrastructure/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2025-07-14 19:34:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

add basic rate limiters

Browse Source 
limiting is done based on remote ip's, with 3 ip mask variants to limit networks
of machines. often with two windows, enabling short bursts of activity, but not
sustained high activity. currently only for imap and smtp, not yet http.

limits are currently based on:
- number of open connections
- connection rate
- limits after authentication failures. too many failures, and new connections will be dropped.
- rate of delivery in total number of messages
- rate of delivery in total size of messages

the limits on connections and authentication failures are in-memory. the limits
on delivery of messages are based on stored messages.

the limits themselves are not yet configurable, let's use this first.

in the future, we may also want to have stricter limits for senders without any
reputation.
This commit is contained in:
Mechiel Lukkien
2023-02-07 22:56:03 +01:00
parent 1617b7c0d6
commit 2154392bd8
7 changed files with 584 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
smtpserver/server.go
View File

@ -15,6 +15,7 @@ import (
"fmt"
"hash"
"io"
"math"
"net"
"os"
"runtime/debug"
@ -42,6 +43,7 @@ import (
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/publicsuffix"
"github.com/mjl-/mox/queue"
"github.com/mjl-/mox/ratelimit"
"github.com/mjl-/mox/scram"
"github.com/mjl-/mox/smtp"
"github.com/mjl-/mox/spf"
@ -57,9 +59,39 @@ var xlog = mlog.New("smtpserver")
// We use panic and recover for error handling while executing commands.
// These errors signal the connection must be closed.
var (
errIO = errors.New("fatal io error")
)
var errIO = errors.New("fatal io error")
var limiterConnectionRate, limiterConnections *ratelimit.Limiter
// For delivery rate limiting. Variable because changed during tests.
var limitIPMasked1MessagesPerMinute int = 500
var limitIPMasked1SizePerMinute int64 = 1000 * 1024 * 1024
func init() {
// Also called by tests, so they don't trigger the rate limiter.
limitersInit()
}
func limitersInit() {
mox.LimitersInit()
// todo future: make these configurable
limiterConnectionRate = &ratelimit.Limiter{
WindowLimits: []ratelimit.WindowLimit{
{
Window: time.Minute,
Limits: [...]int64{300, 900, 2700},
},
},
}
limiterConnections = &ratelimit.Limiter{
WindowLimits: []ratelimit.WindowLimit{
{
Window: time.Duration(math.MaxInt64), // All of time.
Limits: [...]int64{30, 90, 270},
},
},
}
}
type codes struct {
code int
@ -503,6 +535,25 @@ func serve(listenerName string, cid int64, hostname dns.Domain, tlsConfig *tls.C
default:
}
if !limiterConnectionRate.Add(c.remoteIP, time.Now(), 1) {
c.writecodeline(smtp.C421ServiceUnavail, smtp.SePol7Other0, "connection rate from your ip or network too high, slow down please", nil)
return
}
// If remote IP/network resulted in too many authentication failures, refuse to serve.
if submission && !mox.LimiterFailedAuth.CanAdd(c.remoteIP, time.Now(), 1) {
c.log.Debug("refusing connection due to many auth failures", mlog.Field("remoteip", c.remoteIP))
c.writecodeline(smtp.C421ServiceUnavail, smtp.SePol7Other0, "too many auth failures", nil)
return
}
if !limiterConnections.Add(c.remoteIP, time.Now(), 1) {
c.log.Debug("refusing connection due to many open connections", mlog.Field("remoteip", c.remoteIP))
c.writecodeline(smtp.C421ServiceUnavail, smtp.SePol7Other0, "too many open connections from your ip or network", nil)
return
}
defer limiterConnections.Add(c.remoteIP, time.Now(), -1)
// We register and unregister the original connection, in case c.conn is replaced
// with a TLS connection later on.
mox.Connections.Register(nc, "smtp", listenerName)
@ -773,6 +824,12 @@ func (c *conn) cmdAuth(p *parser) {
authResult := "error"
defer func() {
metrics.AuthenticationInc("submission", authVariant, authResult)
switch authResult {
case "ok":
mox.LimiterFailedAuth.Reset(c.remoteIP, time.Now())
default:
mox.LimiterFailedAuth.Add(c.remoteIP, time.Now(), 1)
}
}()
// todo: implement "AUTH LOGIN"? it looks like PLAIN, but without the continuation. it is an obsolete sasl mechanism. an account in desktop outlook appears to go through the cloud, attempting to submit email only with unadvertised and AUTH LOGIN. it appears they don't know "plain".
@ -1913,6 +1970,88 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
}
}()
// We don't want to let a single IP or network deliver too many messages to an
// account. They may fill up the mailbox, either with messages that have to be
// purged, or by filling the disk. We check both cases for IP's and networks.
var rateError bool // Whether returned error represents a rate error.
err = acc.DB.Read(func(tx *bstore.Tx) (retErr error) {
now := time.Now()
defer func() {
log.Debugx("checking message and size delivery rates", retErr, mlog.Field("duration", time.Since(now)))
}()
checkCount := func(msg store.Message, window time.Duration, limit int) {
if retErr != nil {
return
}
q := bstore.QueryTx[store.Message](tx)
q.FilterNonzero(msg)
q.FilterGreater("Received", now.Add(-window))
n, err := q.Count()
if err != nil {
retErr = err
return
}
if n >= limit {
rateError = true
retErr = fmt.Errorf("more than %d messages in past %s from your ip/network", limit, window)
}
}
checkSize := func(msg store.Message, window time.Duration, limit int64) {
if retErr != nil {
return
}
q := bstore.QueryTx[store.Message](tx)
q.FilterNonzero(msg)
q.FilterGreater("Received", now.Add(-window))
size := msgWriter.Size
err := q.ForEach(func(v store.Message) error {
size += v.Size
return nil
})
if err != nil {
retErr = err
return
}
if size > limit {
rateError = true
retErr = fmt.Errorf("more than %d bytes in past %s from your ip/network", limit, window)
}
}
// todo future: make these configurable
const day = 24 * time.Hour
checkCount(store.Message{RemoteIPMasked1: ipmasked1}, time.Minute, limitIPMasked1MessagesPerMinute)
checkCount(store.Message{RemoteIPMasked1: ipmasked1}, day, 20*500)
checkCount(store.Message{RemoteIPMasked2: ipmasked2}, time.Minute, 1500)
checkCount(store.Message{RemoteIPMasked2: ipmasked2}, day, 20*1500)
checkCount(store.Message{RemoteIPMasked3: ipmasked3}, time.Minute, 4500)
checkCount(store.Message{RemoteIPMasked3: ipmasked3}, day, 20*4500)
const MB = 1024 * 1024
checkSize(store.Message{RemoteIPMasked1: ipmasked1}, time.Minute, limitIPMasked1SizePerMinute)
checkSize(store.Message{RemoteIPMasked1: ipmasked1}, day, 3*1000*MB)
checkSize(store.Message{RemoteIPMasked2: ipmasked2}, time.Minute, 3000*MB)
checkSize(store.Message{RemoteIPMasked2: ipmasked2}, day, 3*3000*MB)
checkSize(store.Message{RemoteIPMasked3: ipmasked3}, time.Minute, 9000*MB)
checkSize(store.Message{RemoteIPMasked3: ipmasked3}, day, 3*9000*MB)
return retErr
})
if err != nil && !rateError {
log.Errorx("checking delivery rates", err)
metricDelivery.WithLabelValues("checkrates", "").Inc()
addError(rcptAcc, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing")
continue
} else if err != nil {
log.Debugx("refusing due to high delivery rate", err)
metricDelivery.WithLabelValues("highrate", "").Inc()
addError(rcptAcc, smtp.C452StorageFull, smtp.SeMailbox2Full2, true, err.Error())
continue
}
// ../rfc/5321:3204
// ../rfc/5321:3300
// Received-SPF header goes before Received. ../rfc/7208:2038

131
smtpserver/server_test.go
View File

@ -28,6 +28,7 @@ import (
"github.com/mjl-/mox/dkim"
"github.com/mjl-/mox/dmarcdb"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/queue"
"github.com/mjl-/mox/smtp"
@ -70,10 +71,13 @@ type testserver struct {
user, pass string
submission bool
dnsbls []dns.Domain
tlsmode smtpclient.TLSMode
}
func newTestServer(t *testing.T, configPath string, resolver dns.Resolver) *testserver {
ts := testserver{t: t, cid: 1, resolver: resolver}
limitersInit() // Reset rate limiters.
ts := testserver{t: t, cid: 1, resolver: resolver, tlsmode: smtpclient.TLSOpportunistic}
mox.Context = context.Background()
mox.ConfigStaticPath = configPath
@ -125,7 +129,7 @@ func (ts *testserver) run(fn func(helloErr error, client *smtpclient.Client)) {
authLine = fmt.Sprintf("AUTH PLAIN %s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("\u0000%s\u0000%s", ts.user, ts.pass))))
}
client, err := smtpclient.New(context.Background(), xlog.WithCid(ts.cid-1), clientConn, smtpclient.TLSOpportunistic, "mox.example", authLine)
client, err := smtpclient.New(context.Background(), xlog.WithCid(ts.cid-1), clientConn, ts.tlsmode, "mox.example", authLine)
if err != nil {
clientConn.Close()
} else {
@ -745,5 +749,126 @@ func TestTLSReport(t *testing.T) {
run(tlsrpt, 0)
run(strings.ReplaceAll(tlsrpt, "xmox.nl", "mox.example"), 1)
}
func TestRatelimitConnectionrate(t *testing.T) {
ts := newTestServer(t, "../testdata/smtp/mox.conf", dns.MockResolver{})
defer ts.close()
// We'll be creating 300 connections, no TLS and reduce noise.
ts.tlsmode = smtpclient.TLSSkip
mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelInfo})
// We may be passing a window boundary during this tests. The limit is 300/minute.
// So make twice that many connections and hope the tests don't take too long.
for i := 0; i <= 2*300; i++ {
ts.run(func(err error, client *smtpclient.Client) {
t.Helper()
if err != nil && i < 300 {
t.Fatalf("expected smtp connection, got %v", err)
}
if err == nil && i == 600 {
t.Fatalf("expected no smtp connection due to connection rate limit, got connection")
}
if client != nil {
client.Close()
}
})
}
}
func TestRatelimitAuth(t *testing.T) {
ts := newTestServer(t, "../testdata/smtp/mox.conf", dns.MockResolver{})
defer ts.close()
ts.submission = true
ts.tlsmode = smtpclient.TLSSkip
ts.user = "bad"
ts.pass = "bad"
// We may be passing a window boundary during this tests. The limit is 10 auth
// failures/minute. So make twice that many connections and hope the tests don't
// take too long.
for i := 0; i <= 2*10; i++ {
ts.run(func(err error, client *smtpclient.Client) {
t.Helper()
if err == nil {
t.Fatalf("got auth success with bad credentials")
}
var cerr smtpclient.Error
badauth := errors.As(err, &cerr) && cerr.Code == smtp.C535AuthBadCreds
if !badauth && i < 10 {
t.Fatalf("expected auth failure, got %v", err)
}
if badauth && i == 20 {
t.Fatalf("expected no smtp connection due to failed auth rate limit, got other error %v", err)
}
if client != nil {
client.Close()
}
})
}
}
func TestRatelimitDelivery(t *testing.T) {
resolver := dns.MockResolver{
A: map[string][]string{
"example.org.": {"127.0.0.10"}, // For mx check.
},
PTR: map[string][]string{
"127.0.0.10": {"example.org."},
},
}
ts := newTestServer(t, "../testdata/smtp/mox.conf", resolver)
defer ts.close()
orig := limitIPMasked1MessagesPerMinute
limitIPMasked1MessagesPerMinute = 1
defer func() {
limitIPMasked1MessagesPerMinute = orig
}()
ts.run(func(err error, client *smtpclient.Client) {
mailFrom := "remote@example.org"
rcptTo := "mjl@mox.example"
if err == nil {
err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
}
tcheck(t, err, "deliver to remote")
err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
var cerr smtpclient.Error
if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C452StorageFull {
t.Fatalf("got err %v, expected smtpclient error with code 452 for storage full", err)
}
})
limitIPMasked1MessagesPerMinute = orig
origSize := limitIPMasked1SizePerMinute
// Message was already delivered once. We'll do another one. But the 3rd will fail.
// We need the actual size with prepended headers, since that is used in the
// calculations.
msg, err := bstore.QueryDB[store.Message](ts.acc.DB).Get()
if err != nil {
t.Fatalf("getting delivered message for its size: %v", err)
}
limitIPMasked1SizePerMinute = 2*msg.Size + int64(len(deliverMessage)/2)
defer func() {
limitIPMasked1SizePerMinute = origSize
}()
ts.run(func(err error, client *smtpclient.Client) {
mailFrom := "remote@example.org"
rcptTo := "mjl@mox.example"
if err == nil {
err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
}
tcheck(t, err, "deliver to remote")
err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
var cerr smtpclient.Error
if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C452StorageFull {
t.Fatalf("got err %v, expected smtpclient error with code 452 for storage full", err)
}
})
}