.github/workflows/build-test.yml

@@ -27,9 +27,8 @@ jobs:
       # Need to run tests with a temp dir on same file system for os.Rename to succeed.
       - run: 'mkdir -p tmp && TMPDIR=$PWD/tmp make test'
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
-          name: coverage-${{ matrix.go-version }}
           path: cover.html
 
       # Format code, we check below if nothing changed.

.gitignore

@@ -5,7 +5,7 @@
 /local/
 /testdata/check/
 /testdata/*/data/
-/testdata/ctl/config/dkim/
+/testdata/ctl/dkim/
 /testdata/empty/
 /testdata/exportmaildir/
 /testdata/exportmbox/

Makefile

@@ -7,7 +7,13 @@ build0:
 	CGO_ENABLED=0 go build
 	CGO_ENABLED=0 go vet ./...
 	./gendoc.sh
-	./genapidoc.sh
+	# we rewrite some dmarcprt and tlsrpt enums into untyped strings: real-world
+	# reports have invalid values, and our loose Go typed strings accept all values,
+	# but we don't want the typescript runtime checker to fail on those unrecognized
+	# values.
+	(cd webadmin && CGO_ENABLED=0 go run ../vendor/github.com/mjl-/sherpadoc/cmd/sherpadoc/*.go -adjust-function-names none -rename 'config Domain ConfigDomain,dmarc Policy DMARCPolicy,mtasts MX STSMX,tlsrptdb Record TLSReportRecord,tlsrptdb SuppressAddress TLSRPTSuppressAddress,dmarcrpt DKIMResult string,dmarcrpt SPFResult string,dmarcrpt SPFDomainScope string,dmarcrpt DMARCResult string,dmarcrpt PolicyOverride string,dmarcrpt Alignment string,dmarcrpt Disposition string,tlsrpt PolicyType string,tlsrpt ResultType string' Admin) >webadmin/api.json
+	(cd webaccount && CGO_ENABLED=0 go run ../vendor/github.com/mjl-/sherpadoc/cmd/sherpadoc/*.go -adjust-function-names none Account) >webaccount/api.json
+	(cd webmail && CGO_ENABLED=0 go run ../vendor/github.com/mjl-/sherpadoc/cmd/sherpadoc/*.go -adjust-function-names none Webmail) >webmail/api.json
 	./gents.sh webadmin/api.json webadmin/api.ts
 	./gents.sh webaccount/api.json webaccount/api.ts
 	./gents.sh webmail/api.json webmail/api.ts
@@ -23,16 +29,13 @@ race: build0
 	go build -race
 
 test:
-	CGO_ENABLED=0 go test -fullpath -shuffle=on -coverprofile cover.out ./...
+	CGO_ENABLED=0 go test -shuffle=on -coverprofile cover.out ./...
 	go tool cover -html=cover.out -o cover.html
 
 test-race:
-	CGO_ENABLED=1 go test -fullpath -race -shuffle=on -covermode atomic -coverprofile cover.out ./...
+	CGO_ENABLED=1 go test -race -shuffle=on -covermode atomic -coverprofile cover.out ./...
 	go tool cover -html=cover.out -o cover.html
 
-test-more:
-	TZ= CGO_ENABLED=0 go test -fullpath -shuffle=on -count 2 ./...
-
 # note: if testdata/upgradetest.mbox.gz exists, its messages will be imported
 # during tests. helpful for performance/resource consumption tests.
 test-upgrade: build
@@ -42,9 +45,6 @@ test-upgrade: build
 install-staticcheck:
 	CGO_ENABLED=0 go install honnef.co/go/tools/cmd/staticcheck@latest
 
-install-ineffassign:
-	CGO_ENABLED=0 go install github.com/gordonklaus/ineffassign@v0.1.0
-
 check:
 	CGO_ENABLED=0 go vet -tags integration
 	CGO_ENABLED=0 go vet -tags website website/website.go
@@ -52,7 +52,6 @@ check:
 	CGO_ENABLED=0 go vet -tags errata rfc/errata.go
 	CGO_ENABLED=0 go vet -tags xr rfc/xr.go
 	GOARCH=386 CGO_ENABLED=0 go vet ./...
-	CGO_ENABLED=0 ineffassign ./...
 	CGO_ENABLED=0 staticcheck ./...
 	CGO_ENABLED=0 staticcheck -tags integration
 	CGO_ENABLED=0 staticcheck -tags website website/website.go
@@ -74,20 +73,18 @@ check-shadow:
 	CGO_ENABLED=0 go vet -tags xr -vettool=$$(which shadow) rfc/xr.go 2>&1 | grep -v '"err"'
 
 fuzz:
-	go test -fullpath -fuzz FuzzParseSignature -fuzztime 5m ./dkim
+	go test -fuzz FuzzParseSignature -fuzztime 5m ./dkim
-	go test -fullpath -fuzz FuzzParseRecord -fuzztime 5m ./dkim
+	go test -fuzz FuzzParseRecord -fuzztime 5m ./dkim
-	go test -fullpath -fuzz . -fuzztime 5m ./dmarc
+	go test -fuzz . -fuzztime 5m ./dmarc
-	go test -fullpath -fuzz . -fuzztime 5m ./dmarcrpt
+	go test -fuzz . -fuzztime 5m ./dmarcrpt
-	go test -fullpath -fuzz . -parallel 1 -fuzztime 5m ./imapserver
+	go test -fuzz . -parallel 1 -fuzztime 5m ./imapserver
-	go test -fullpath -fuzz . -fuzztime 5m ./imapclient
+	go test -fuzz . -parallel 1 -fuzztime 5m ./junk
-	go test -fullpath -fuzz . -parallel 1 -fuzztime 5m ./junk
+	go test -fuzz FuzzParseRecord -fuzztime 5m ./mtasts
-	go test -fullpath -fuzz FuzzParseRecord -fuzztime 5m ./mtasts
+	go test -fuzz FuzzParsePolicy -fuzztime 5m ./mtasts
-	go test -fullpath -fuzz FuzzParsePolicy -fuzztime 5m ./mtasts
+	go test -fuzz . -parallel 1 -fuzztime 5m ./smtpserver
-	go test -fullpath -fuzz . -fuzztime 5m ./smtp
+	go test -fuzz . -fuzztime 5m ./spf
-	go test -fullpath -fuzz . -parallel 1 -fuzztime 5m ./smtpserver
+	go test -fuzz FuzzParseRecord -fuzztime 5m ./tlsrpt
-	go test -fullpath -fuzz . -fuzztime 5m ./spf
+	go test -fuzz FuzzParseMessage -fuzztime 5m ./tlsrpt
-	go test -fullpath -fuzz FuzzParseRecord -fuzztime 5m ./tlsrpt
-	go test -fullpath -fuzz FuzzParseMessage -fuzztime 5m ./tlsrpt
 
 govendor:
 	go mod tidy
@@ -95,25 +92,23 @@ govendor:
 	./genlicenses.sh
 
 test-integration:
-	-docker compose -f docker-compose-integration.yml kill
-	-docker compose -f docker-compose-integration.yml down
 	docker image build --pull --no-cache -f Dockerfile -t mox_integration_moxmail .
 	docker image build --pull --no-cache -f testdata/integration/Dockerfile.test -t mox_integration_test testdata/integration
 	-rm -rf testdata/integration/moxacmepebble/data
 	-rm -rf testdata/integration/moxmail2/data
 	-rm -f testdata/integration/tmp-pebble-ca.pem
-	MOX_UID=$$(id -u) docker compose -f docker-compose-integration.yml run test
+	MOX_UID=$$(id -u) docker-compose -f docker-compose-integration.yml run test
-	docker compose -f docker-compose-integration.yml kill
+	docker-compose -f docker-compose-integration.yml down --timeout 1
 
 
 imaptest-build:
-	-docker compose -f docker-compose-imaptest.yml build --no-cache --pull mox
+	-docker-compose -f docker-compose-imaptest.yml build --no-cache --pull mox
 
 imaptest-run:
 	-rm -r testdata/imaptest/data
 	mkdir testdata/imaptest/data
-	docker compose -f docker-compose-imaptest.yml run --entrypoint /usr/local/bin/imaptest imaptest host=mox port=1143 user=mjl@mox.example pass=testtest mbox=imaptest.mbox
+	docker-compose -f docker-compose-imaptest.yml run --entrypoint /usr/local/bin/imaptest imaptest host=mox port=1143 user=mjl@mox.example pass=testtest mbox=imaptest.mbox
-	docker compose -f docker-compose-imaptest.yml down
+	docker-compose -f docker-compose-imaptest.yml down
 
 
 fmt:

README.md

@@ -19,7 +19,7 @@ See Quickstart below to get started.
   (similar to greylisting). Rejected emails are stored in a mailbox called Rejects
   for a short period, helping with misclassified legitimate synchronous
   signup/login/transactional emails.
-- Internationalized email (EIA), with unicode in email address usernames
+- Internationalized email, with unicode in email address usernames
   ("localparts"), and in domain names (IDNA).
 - Automatic TLS with ACME, for use with Let's Encrypt and other CA's.
 - DANE and MTA-STS for inbound and outbound delivery over SMTP with STARTTLS,
@@ -99,7 +99,7 @@ for other platforms.
 # Compiling
 
 You can easily (cross) compile mox yourself. You need a recent Go toolchain
-installed.  Run `go version`, it must be >= 1.23. Download the latest version
+installed.  Run `go version`, it must be >= 1.22. Download the latest version
 from https://go.dev/dl/ or see https://go.dev/doc/manage-install.
 
 To download the source code of the latest release, and compile it to binary "mox":
@@ -125,53 +125,44 @@ It is important to run with docker host networking, so mox can use the public
 IPs and has correct remote IP information for incoming connections (important
 for junk filtering and rate-limiting).
 
-# Development
+# Future/development
 
 See develop.txt for instructions/tips for developing on mox.
 
-# Sponsors
+Mox will receive funding for essentially full-time continued work from August
+2023 to August 2024 through NLnet/EU's NGI0 Entrust, see
+https://nlnet.nl/project/Mox/.
 
-Thanks to NLnet foundation, the European Commission's NGI programme, and the
+## Roadmap
-Netherlands Ministry of the Interior and Kingdom Relations for financial
-support:
 
-- 2024/2025, NLnet NGI0 Zero Core, https://nlnet.nl/project/Mox-Automation/
-- 2024, NLnet e-Commons Fund, https://nlnet.nl/project/Mox-API/
-- 2023/2024, NLnet NGI0 Entrust, https://nlnet.nl/project/Mox/
-
-# Roadmap
-
-- "mox setup" command, using admin web interface for interactive setup
-- Automate DNS management, for setup and maintenance, such as DANE/DKIM key rotation
-- Config options for "transactional email domains", for which mox will only
-  send messages
-- Encrypted storage of files (email messages, TLS keys), also with per account keys
-- Recognize common deliverability issues and help postmasters solve them
-- JMAP, IMAP OBJECTID extension, IMAP JMAPACCESS extension
 - Calendaring with CalDAV/iCal
+- More IMAP extensions (PREVIEW, WITHIN, IMPORTANT, COMPRESS=DEFLATE,
+  CREATE-SPECIAL-USE, SAVEDATE, UNAUTHENTICATE, REPLACE, QUOTA, NOTIFY,
+  MULTIAPPEND, OBJECTID, MULTISEARCH, THREAD, SORT)
+- SMTP DSN extension
+- "mox setup" command, with webapp for interactive setup
 - Introbox, to which first-time senders are delivered
+- ARC, with forwarded email from trusted source
+- Forwarding (to an external address)
 - Add special IMAP mailbox ("Queue?") that contains queued but
   undelivered messages, updated with IMAP flags/keywords/tags and message headers.
 - External addresses in aliases/lists.
 - Autoresponder (out of office/vacation)
-- Mailing list manager
+- OAUTH2 support, for single sign on
 - IMAP extensions for "online"/non-syncing/webmail clients (SORT (including
   DISPLAYFROM, DISPLAYTO), THREAD, PARTIAL, CONTEXT=SEARCH CONTEXT=SORT ESORT,
   FILTERS)
-- IMAP ACL support, for account sharing (interacts with many extensions and code)
 - Improve support for mobile clients with extensions: IMAP URLAUTH, SMTP
   CHUNKING and BINARYMIME, IMAP CATENATE
+- Mailing list manager
 - Privilege separation, isolating parts of the application to more restricted
   sandbox (e.g. new unauthenticated connections)
 - Using mox as backup MX
+- JMAP
 - Sieve for filtering (for now see Rulesets in the account config)
-- ARC, with forwarded email from trusted source
 - Milter support, for integration with external tools
-- SMTP DSN extension
 - IMAP Sieve extension, to run Sieve scripts after message changes (not only
   new deliveries)
-- OAUTH2 support, for single sign on
-- Forwarding (to an external address)
 
 There are many smaller improvements to make as well, search for "todo" in the code.
 
@@ -180,10 +171,12 @@ There are many smaller improvements to make as well, search for "todo" in the co
 There is currently no plan to implement the following. Though this may
 change in the future.
 
-- Functioning as an SMTP relay without authentication
+- Functioning as SMTP relay
 - POP3
-- Delivery to (unix) OS system users (mbox/Maildir)
+- Delivery to (unix) OS system users
 - Support for pluggable delivery mechanisms
+- iOS Mail push notifications (with XAPPLEPUSHSERVICE undocumented imap
+  extension and hard to get APNS certificate)
 
 
 # FAQ - Frequently Asked Questions
@@ -295,8 +288,7 @@ MIT license (like mox), and have the rights to do so.
 
 ## Where can I discuss mox?
 
-Join #mox on irc.oftc.net, or #mox:matrix.org (https://matrix.to/#/#mox:matrix.org),
+Join #mox on irc.oftc.net, or #mox:matrix.org, or #mox on the "Gopher slack".
-or #mox on the "Gopher slack".
 
 For bug reports, please file an issue at https://github.com/mjl-/mox/issues/new.
 
@@ -354,18 +346,15 @@ in place and restart. If manual actions are required, the release notes mention
 them. Check the release notes of all version between your current installation
 and the release you're upgrading to.
 
-Before upgrading, make a backup of the config & data directory with `mox backup
+Before upgrading, make a backup of the data directory with `mox backup
-<destdir>`. This copies all files from the config directory to
+<destdir>`. This writes consistent snapshots of the database files, and
-`<destdir>/config`, and creates `<destdir>/data` with a consistent snapshots of
+duplicates message files from the outgoing queue and accounts.  Using the new
-the database files, and message files from the outgoing queue and accounts.
+mox binary, run `mox verifydata <backupdir>` (do NOT use the "live" data
-Using the new mox binary, run `mox verifydata <destdir>/data` (do NOT use the
+directory!) for a dry run. If this fails, an upgrade will probably fail too.
-"live" data directory!) for a dry run. If this fails, an upgrade will probably
+Important: verifydata with the new mox binary can modify the database files (due
-fail too.
+to automatic schema upgrades). So make a fresh backup again before the actual
-
+upgrade. See the help output of the "backup" and "verifydata" commands for more
-Important: verifydata with the new mox binary can modify the database files
+details.
-(due to automatic schema upgrades). So make a fresh backup again before the
-actual upgrade. See the help output of the "backup" and "verifydata" commands
-for more details.
 
 During backup, message files are hardlinked if possible, and copied otherwise.
 Using a destination directory like `data/tmp/backup` increases the odds
@@ -540,13 +529,3 @@ ensuring they don't become too large.  The message index database file for an
 account is at `data/accounts/<account>/index.db`, accessed with the bstore
 database library, which uses bbolt (formerly BoltDB) for storage, a
 transactional key/value library/file format inspired by LMDB.
-
-## How do I block IPs with authentication failures with fail2ban?
-
-Mox includes a rate limiter for IPs/networks that cause too many authentication
-failures. It automatically unblocks such IPs/networks after a while. So you may
-not need fail2ban. If you want to use fail2ban, you could use this snippet:
-
-	[Definition]
-	failregex = .*failed authentication attempt.*remote=<HOST>
-	ignoreregex =

admin/clientconfig.go

@@ -1,175 +0,0 @@
-package admin
-
-import (
-	"fmt"
-	"maps"
-	"slices"
-
-	"github.com/mjl-/mox/config"
-	"github.com/mjl-/mox/dns"
-	"github.com/mjl-/mox/mox-"
-)
-
-type TLSMode uint8
-
-const (
-	TLSModeImmediate TLSMode = 0
-	TLSModeSTARTTLS  TLSMode = 1
-	TLSModeNone      TLSMode = 2
-)
-
-type ProtocolConfig struct {
-	Host           dns.Domain
-	Port           int
-	TLSMode        TLSMode
-	EnabledOnHTTPS bool
-}
-
-type ClientConfig struct {
-	IMAP       ProtocolConfig
-	Submission ProtocolConfig
-}
-
-// ClientConfigDomain returns a single IMAP and Submission client configuration for
-// a domain.
-func ClientConfigDomain(d dns.Domain) (rconfig ClientConfig, rerr error) {
-	var haveIMAP, haveSubmission bool
-
-	domConf, ok := mox.Conf.Domain(d)
-	if !ok {
-		return ClientConfig{}, fmt.Errorf("%w: unknown domain", ErrRequest)
-	}
-
-	gather := func(l config.Listener) (done bool) {
-		host := mox.Conf.Static.HostnameDomain
-		if l.Hostname != "" {
-			host = l.HostnameDomain
-		}
-		if domConf.ClientSettingsDomain != "" {
-			host = domConf.ClientSettingsDNSDomain
-		}
-		if !haveIMAP && l.IMAPS.Enabled {
-			rconfig.IMAP.Host = host
-			rconfig.IMAP.Port = config.Port(l.IMAPS.Port, 993)
-			rconfig.IMAP.TLSMode = TLSModeImmediate
-			rconfig.IMAP.EnabledOnHTTPS = l.IMAPS.EnabledOnHTTPS
-			haveIMAP = true
-		}
-		if !haveIMAP && l.IMAP.Enabled {
-			rconfig.IMAP.Host = host
-			rconfig.IMAP.Port = config.Port(l.IMAP.Port, 143)
-			rconfig.IMAP.TLSMode = TLSModeSTARTTLS
-			if l.TLS == nil {
-				rconfig.IMAP.TLSMode = TLSModeNone
-			}
-			haveIMAP = true
-		}
-		if !haveSubmission && l.Submissions.Enabled {
-			rconfig.Submission.Host = host
-			rconfig.Submission.Port = config.Port(l.Submissions.Port, 465)
-			rconfig.Submission.TLSMode = TLSModeImmediate
-			rconfig.Submission.EnabledOnHTTPS = l.Submissions.EnabledOnHTTPS
-			haveSubmission = true
-		}
-		if !haveSubmission && l.Submission.Enabled {
-			rconfig.Submission.Host = host
-			rconfig.Submission.Port = config.Port(l.Submission.Port, 587)
-			rconfig.Submission.TLSMode = TLSModeSTARTTLS
-			if l.TLS == nil {
-				rconfig.Submission.TLSMode = TLSModeNone
-			}
-			haveSubmission = true
-		}
-		return haveIMAP && haveSubmission
-	}
-
-	// Look at the public listener first. Most likely the intended configuration.
-	if public, ok := mox.Conf.Static.Listeners["public"]; ok {
-		if gather(public) {
-			return
-		}
-	}
-	// Go through the other listeners in consistent order.
-	names := slices.Sorted(maps.Keys(mox.Conf.Static.Listeners))
-	for _, name := range names {
-		if gather(mox.Conf.Static.Listeners[name]) {
-			return
-		}
-	}
-	return ClientConfig{}, fmt.Errorf("%w: no listeners found for imap and/or submission", ErrRequest)
-}
-
-// ClientConfigs holds the client configuration for IMAP/Submission for a
-// domain.
-type ClientConfigs struct {
-	Entries []ClientConfigsEntry
-}
-
-type ClientConfigsEntry struct {
-	Protocol string
-	Host     dns.Domain
-	Port     int
-	Listener string
-	Note     string
-}
-
-// ClientConfigsDomain returns the client configs for IMAP/Submission for a
-// domain.
-func ClientConfigsDomain(d dns.Domain) (ClientConfigs, error) {
-	domConf, ok := mox.Conf.Domain(d)
-	if !ok {
-		return ClientConfigs{}, fmt.Errorf("%w: unknown domain", ErrRequest)
-	}
-
-	c := ClientConfigs{}
-	c.Entries = []ClientConfigsEntry{}
-	var listeners []string
-
-	for name := range mox.Conf.Static.Listeners {
-		listeners = append(listeners, name)
-	}
-	slices.Sort(listeners)
-
-	note := func(tls bool, requiretls bool) string {
-		if !tls {
-			return "plain text, no STARTTLS configured"
-		}
-		if requiretls {
-			return "STARTTLS required"
-		}
-		return "STARTTLS optional"
-	}
-
-	for _, name := range listeners {
-		l := mox.Conf.Static.Listeners[name]
-		host := mox.Conf.Static.HostnameDomain
-		if l.Hostname != "" {
-			host = l.HostnameDomain
-		}
-		if domConf.ClientSettingsDomain != "" {
-			host = domConf.ClientSettingsDNSDomain
-		}
-		if l.Submissions.Enabled {
-			note := "with TLS"
-			if l.Submissions.EnabledOnHTTPS {
-				note += "; also served on port 443 with TLS ALPN \"smtp\""
-			}
-			c.Entries = append(c.Entries, ClientConfigsEntry{"Submission (SMTP)", host, config.Port(l.Submissions.Port, 465), name, note})
-		}
-		if l.IMAPS.Enabled {
-			note := "with TLS"
-			if l.IMAPS.EnabledOnHTTPS {
-				note += "; also served on port 443 with TLS ALPN \"imap\""
-			}
-			c.Entries = append(c.Entries, ClientConfigsEntry{"IMAP", host, config.Port(l.IMAPS.Port, 993), name, note})
-		}
-		if l.Submission.Enabled {
-			c.Entries = append(c.Entries, ClientConfigsEntry{"Submission (SMTP)", host, config.Port(l.Submission.Port, 587), name, note(l.TLS != nil, !l.Submission.NoRequireSTARTTLS)})
-		}
-		if l.IMAP.Enabled {
-			c.Entries = append(c.Entries, ClientConfigsEntry{"IMAP", host, config.Port(l.IMAPS.Port, 143), name, note(l.TLS != nil, !l.IMAP.NoRequireSTARTTLS)})
-		}
-	}
-
-	return c, nil
-}

admin/dnsrecords.go

@@ -1,318 +0,0 @@
-package admin
-
-import (
-	"crypto"
-	"crypto/ed25519"
-	"crypto/rsa"
-	"crypto/sha256"
-	"crypto/x509"
-	"fmt"
-	"net/url"
-	"strings"
-
-	"github.com/mjl-/adns"
-
-	"github.com/mjl-/mox/config"
-	"github.com/mjl-/mox/dkim"
-	"github.com/mjl-/mox/dmarc"
-	"github.com/mjl-/mox/dns"
-	"github.com/mjl-/mox/mox-"
-	"github.com/mjl-/mox/smtp"
-	"github.com/mjl-/mox/spf"
-	"github.com/mjl-/mox/tlsrpt"
-	"slices"
-)
-
-// todo: find a way to automatically create the dns records as it would greatly simplify setting up email for a domain. we could also dynamically make changes, e.g. providing grace periods after disabling a dkim key, only automatically removing the dkim dns key after a few days. but this requires some kind of api and authentication to the dns server. there doesn't appear to be a single commonly used api for dns management. each of the numerous cloud providers have their own APIs and rather large SKDs to use them. we don't want to link all of them in.
-
-// DomainRecords returns text lines describing DNS records required for configuring
-// a domain.
-//
-// If certIssuerDomainName is set, CAA records to limit TLS certificate issuance to
-// that caID will be suggested. If acmeAccountURI is also set, CAA records also
-// restricting issuance to that account ID will be suggested.
-func DomainRecords(domConf config.Domain, domain dns.Domain, hasDNSSEC bool, certIssuerDomainName, acmeAccountURI string) ([]string, error) {
-	d := domain.ASCII
-	h := mox.Conf.Static.HostnameDomain.ASCII
-
-	// The first line with ";" is used by ../testdata/integration/moxacmepebble.sh and
-	// ../testdata/integration/moxmail2.sh for selecting DNS records
-	records := []string{
-		"; Time To Live of 5 minutes, may be recognized if importing as a zone file.",
-		"; Once your setup is working, you may want to increase the TTL.",
-		"$TTL 300",
-		"",
-	}
-
-	if public, ok := mox.Conf.Static.Listeners["public"]; ok && public.TLS != nil && (len(public.TLS.HostPrivateRSA2048Keys) > 0 || len(public.TLS.HostPrivateECDSAP256Keys) > 0) {
-		records = append(records,
-			`; DANE: These records indicate that a remote mail server trying to deliver email`,
-			`; with SMTP (TCP port 25) must verify the TLS certificate with DANE-EE (3), based`,
-			`; on the certificate public key ("SPKI", 1) that is SHA2-256-hashed (1) to the`,
-			`; hexadecimal hash. DANE-EE verification means only the certificate or public`,
-			`; key is verified, not whether the certificate is signed by a (centralized)`,
-			`; certificate authority (CA), is expired, or matches the host name.`,
-			`;`,
-			`; NOTE: Create the records below only once: They are for the machine, and apply`,
-			`; to all hosted domains.`,
-		)
-		if !hasDNSSEC {
-			records = append(records,
-				";",
-				"; WARNING: Domain does not appear to be DNSSEC-signed. To enable DANE, first",
-				"; enable DNSSEC on your domain, then add the TLSA records. Records below have been",
-				"; commented out.",
-			)
-		}
-		addTLSA := func(privKey crypto.Signer) error {
-			spkiBuf, err := x509.MarshalPKIXPublicKey(privKey.Public())
-			if err != nil {
-				return fmt.Errorf("marshal SubjectPublicKeyInfo for DANE record: %v", err)
-			}
-			sum := sha256.Sum256(spkiBuf)
-			tlsaRecord := adns.TLSA{
-				Usage:     adns.TLSAUsageDANEEE,
-				Selector:  adns.TLSASelectorSPKI,
-				MatchType: adns.TLSAMatchTypeSHA256,
-				CertAssoc: sum[:],
-			}
-			var s string
-			if hasDNSSEC {
-				s = fmt.Sprintf("_25._tcp.%-*s TLSA %s", 20+len(d)-len("_25._tcp."), h+".", tlsaRecord.Record())
-			} else {
-				s = fmt.Sprintf(";; _25._tcp.%-*s TLSA %s", 20+len(d)-len(";; _25._tcp."), h+".", tlsaRecord.Record())
-			}
-			records = append(records, s)
-			return nil
-		}
-		for _, privKey := range public.TLS.HostPrivateECDSAP256Keys {
-			if err := addTLSA(privKey); err != nil {
-				return nil, err
-			}
-		}
-		for _, privKey := range public.TLS.HostPrivateRSA2048Keys {
-			if err := addTLSA(privKey); err != nil {
-				return nil, err
-			}
-		}
-		records = append(records, "")
-	}
-
-	if d != h {
-		records = append(records,
-			"; For the machine, only needs to be created once, for the first domain added:",
-			"; ",
-			"; SPF-allow host for itself, resulting in relaxed DMARC pass for (postmaster)",
-			"; messages (DSNs) sent from host:",
-			fmt.Sprintf(`%-*s TXT "v=spf1 a -all"`, 20+len(d), h+"."), // ../rfc/7208:2263 ../rfc/7208:2287
-			"",
-		)
-	}
-	if d != h && mox.Conf.Static.HostTLSRPT.ParsedLocalpart != "" {
-		uri := url.URL{
-			Scheme: "mailto",
-			Opaque: smtp.NewAddress(mox.Conf.Static.HostTLSRPT.ParsedLocalpart, mox.Conf.Static.HostnameDomain).Pack(false),
-		}
-		tlsrptr := tlsrpt.Record{Version: "TLSRPTv1", RUAs: [][]tlsrpt.RUA{{tlsrpt.RUA(uri.String())}}}
-		records = append(records,
-			"; For the machine, only needs to be created once, for the first domain added:",
-			"; ",
-			"; Request reporting about success/failures of TLS connections to (MX) host, for DANE.",
-			fmt.Sprintf(`_smtp._tls.%-*s         TXT "%s"`, 20+len(d)-len("_smtp._tls."), h+".", tlsrptr.String()),
-			"",
-		)
-	}
-
-	records = append(records,
-		"; Deliver email for the domain to this host.",
-		fmt.Sprintf("%s.                    MX 10 %s.", d, h),
-		"",
-
-		"; Outgoing messages will be signed with the first two DKIM keys. The other two",
-		"; configured for backup, switching to them is just a config change.",
-	)
-	var selectors []string
-	for name := range domConf.DKIM.Selectors {
-		selectors = append(selectors, name)
-	}
-	slices.Sort(selectors)
-	for _, name := range selectors {
-		sel := domConf.DKIM.Selectors[name]
-		dkimr := dkim.Record{
-			Version:   "DKIM1",
-			Hashes:    []string{"sha256"},
-			PublicKey: sel.Key.Public(),
-		}
-		if _, ok := sel.Key.(ed25519.PrivateKey); ok {
-			dkimr.Key = "ed25519"
-		} else if _, ok := sel.Key.(*rsa.PrivateKey); !ok {
-			return nil, fmt.Errorf("unrecognized private key for DKIM selector %q: %T", name, sel.Key)
-		}
-		txt, err := dkimr.Record()
-		if err != nil {
-			return nil, fmt.Errorf("making DKIM DNS TXT record: %v", err)
-		}
-
-		if len(txt) > 100 {
-			records = append(records,
-				"; NOTE: The following is a single long record split over several lines for use",
-				"; in zone files. When adding through a DNS operator web interface, combine the",
-				"; strings into a single string, without ().",
-			)
-		}
-		s := fmt.Sprintf("%s._domainkey.%s.   TXT %s", name, d, mox.TXTStrings(txt))
-		records = append(records, s)
-
-	}
-	dmarcr := dmarc.DefaultRecord
-	dmarcr.Policy = "reject"
-	if domConf.DMARC != nil {
-		uri := url.URL{
-			Scheme: "mailto",
-			Opaque: smtp.NewAddress(domConf.DMARC.ParsedLocalpart, domConf.DMARC.DNSDomain).Pack(false),
-		}
-		dmarcr.AggregateReportAddresses = []dmarc.URI{
-			{Address: uri.String(), MaxSize: 10, Unit: "m"},
-		}
-	}
-	dspfr := spf.Record{Version: "spf1"}
-	for _, ip := range mox.DomainSPFIPs() {
-		mech := "ip4"
-		if ip.To4() == nil {
-			mech = "ip6"
-		}
-		dspfr.Directives = append(dspfr.Directives, spf.Directive{Mechanism: mech, IP: ip})
-	}
-	dspfr.Directives = append(dspfr.Directives,
-		spf.Directive{Mechanism: "mx"},
-		spf.Directive{Qualifier: "~", Mechanism: "all"},
-	)
-	dspftxt, err := dspfr.Record()
-	if err != nil {
-		return nil, fmt.Errorf("making domain spf record: %v", err)
-	}
-	records = append(records,
-		"",
-
-		"; Specify the MX host is allowed to send for our domain and for itself (for DSNs).",
-		"; ~all means softfail for anything else, which is done instead of -all to prevent older",
-		"; mail servers from rejecting the message because they never get to looking for a dkim/dmarc pass.",
-		fmt.Sprintf(`%s.                    TXT "%s"`, d, dspftxt),
-		"",
-
-		"; Emails that fail the DMARC check (without aligned DKIM and without aligned SPF)",
-		"; should be rejected, and request reports. If you email through mailing lists that",
-		"; strip DKIM-Signature headers and don't rewrite the From header, you may want to",
-		"; set the policy to p=none.",
-		fmt.Sprintf(`_dmarc.%s.             TXT "%s"`, d, dmarcr.String()),
-		"",
-	)
-
-	if sts := domConf.MTASTS; sts != nil {
-		records = append(records,
-			"; Remote servers can use MTA-STS to verify our TLS certificate with the",
-			"; WebPKI pool of CA's (certificate authorities) when delivering over SMTP with",
-			"; STARTTLS.",
-			fmt.Sprintf(`mta-sts.%s.            CNAME %s.`, d, h),
-			fmt.Sprintf(`_mta-sts.%s.           TXT "v=STSv1; id=%s"`, d, sts.PolicyID),
-			"",
-		)
-	} else {
-		records = append(records,
-			"; Note: No MTA-STS to indicate TLS should be used. Either because disabled for the",
-			"; domain or because mox.conf does not have a listener with MTA-STS configured.",
-			"",
-		)
-	}
-
-	if domConf.TLSRPT != nil {
-		uri := url.URL{
-			Scheme: "mailto",
-			Opaque: smtp.NewAddress(domConf.TLSRPT.ParsedLocalpart, domConf.TLSRPT.DNSDomain).Pack(false),
-		}
-		tlsrptr := tlsrpt.Record{Version: "TLSRPTv1", RUAs: [][]tlsrpt.RUA{{tlsrpt.RUA(uri.String())}}}
-		records = append(records,
-			"; Request reporting about TLS failures.",
-			fmt.Sprintf(`_smtp._tls.%s.         TXT "%s"`, d, tlsrptr.String()),
-			"",
-		)
-	}
-
-	if domConf.ClientSettingsDomain != "" && domConf.ClientSettingsDNSDomain != mox.Conf.Static.HostnameDomain {
-		records = append(records,
-			"; Client settings will reference a subdomain of the hosted domain, making it",
-			"; easier to migrate to a different server in the future by not requiring settings",
-			"; in all clients to be updated.",
-			fmt.Sprintf(`%-*s CNAME %s.`, 20+len(d), domConf.ClientSettingsDNSDomain.ASCII+".", h),
-			"",
-		)
-	}
-
-	records = append(records,
-		"; Autoconfig is used by Thunderbird. Autodiscover is (in theory) used by Microsoft.",
-		fmt.Sprintf(`autoconfig.%s.         CNAME %s.`, d, h),
-		fmt.Sprintf(`_autodiscover._tcp.%s. SRV 0 1 443 %s.`, d, h),
-		"",
-
-		// ../rfc/6186:133 ../rfc/8314:692
-		"; For secure IMAP and submission autoconfig, point to mail host.",
-		fmt.Sprintf(`_imaps._tcp.%s.        SRV 0 1 993 %s.`, d, h),
-		fmt.Sprintf(`_submissions._tcp.%s.  SRV 0 1 465 %s.`, d, h),
-		"",
-		// ../rfc/6186:242
-		"; Next records specify POP3 and non-TLS ports are not to be used.",
-		"; These are optional and safe to leave out (e.g. if you have to click a lot in a",
-		"; DNS admin web interface).",
-		fmt.Sprintf(`_imap._tcp.%s.         SRV 0 0 0 .`, d),
-		fmt.Sprintf(`_submission._tcp.%s.   SRV 0 0 0 .`, d),
-		fmt.Sprintf(`_pop3._tcp.%s.         SRV 0 0 0 .`, d),
-		fmt.Sprintf(`_pop3s._tcp.%s.        SRV 0 0 0 .`, d),
-	)
-
-	if certIssuerDomainName != "" {
-		// ../rfc/8659:18 for CAA records.
-		records = append(records,
-			"",
-			"; Optional:",
-			"; You could mark Let's Encrypt as the only Certificate Authority allowed to",
-			"; sign TLS certificates for your domain.",
-			fmt.Sprintf(`%s.                    CAA 0 issue "%s"`, d, certIssuerDomainName),
-		)
-		if acmeAccountURI != "" {
-			// ../rfc/8657:99 for accounturi.
-			// ../rfc/8657:147 for validationmethods.
-			records = append(records,
-				";",
-				"; Optionally limit certificates for this domain to the account ID and methods used by mox.",
-				fmt.Sprintf(`;; %s.                 CAA 0 issue "%s; accounturi=%s; validationmethods=tls-alpn-01,http-01"`, d, certIssuerDomainName, acmeAccountURI),
-				";",
-				"; Or alternatively only limit for email-specific subdomains, so you can use",
-				"; other accounts/methods for other subdomains.",
-				fmt.Sprintf(`;; autoconfig.%s.      CAA 0 issue "%s; accounturi=%s; validationmethods=tls-alpn-01,http-01"`, d, certIssuerDomainName, acmeAccountURI),
-				fmt.Sprintf(`;; mta-sts.%s.         CAA 0 issue "%s; accounturi=%s; validationmethods=tls-alpn-01,http-01"`, d, certIssuerDomainName, acmeAccountURI),
-			)
-			if domConf.ClientSettingsDomain != "" && domConf.ClientSettingsDNSDomain != mox.Conf.Static.HostnameDomain {
-				records = append(records,
-					fmt.Sprintf(`;; %-*s CAA 0 issue "%s; accounturi=%s; validationmethods=tls-alpn-01,http-01"`, 20-3+len(d), domConf.ClientSettingsDNSDomain.ASCII, certIssuerDomainName, acmeAccountURI),
-				)
-			}
-			if strings.HasSuffix(h, "."+d) {
-				records = append(records,
-					";",
-					"; And the mail hostname.",
-					fmt.Sprintf(`;; %-*s CAA 0 issue "%s; accounturi=%s; validationmethods=tls-alpn-01,http-01"`, 20-3+len(d), h+".", certIssuerDomainName, acmeAccountURI),
-				)
-			}
-		} else {
-			// The string "will be suggested" is used by
-			// ../testdata/integration/moxacmepebble.sh and ../testdata/integration/moxmail2.sh
-			// as end of DNS records.
-			records = append(records,
-				";",
-				"; Note: After starting up, once an ACME account has been created, CAA records",
-				"; that restrict issuance to the account will be suggested.",
-			)
-		}
-	}
-	return records, nil
-}

apidiff.sh

@@ -8,31 +8,20 @@ if ! test -d tmp/mox-$prevversion; then
 fi
 (rm -r tmp/apidiff || exit 0)
 mkdir -p tmp/apidiff/$prevversion tmp/apidiff/next
-(rm apidiff/next.txt.new 2>/dev/null || exit 0)
+(rm apidiff/next.txt || exit 0)
-touch apidiff/next.txt.new
+(
+echo "Below are the incompatible changes between $prevversion and next, per package."
+echo
+) >>apidiff/next.txt
 for p in $(cat apidiff/packages.txt); do
 	if ! test -d tmp/mox-$prevversion/$p; then
 		continue
 	fi
 	(cd tmp/mox-$prevversion && apidiff -w ../apidiff/$prevversion/$p.api ./$p)
 	apidiff -w tmp/apidiff/next/$p.api ./$p
-	apidiff -incompatible tmp/apidiff/$prevversion/$p.api tmp/apidiff/next/$p.api >$p.diff
-	if test -s $p.diff; then
-		(
-		echo '#' $p
-		cat $p.diff
-		echo
-		) >>apidiff/next.txt.new
-	fi
-	rm $p.diff
-done
-if test -s apidiff/next.txt.new; then
 	(
-	echo "Below are the incompatible changes between $prevversion and next, per package."
+	echo '#' $p
+	apidiff -incompatible tmp/apidiff/$prevversion/$p.api tmp/apidiff/next/$p.api
 	echo
-	cat apidiff/next.txt.new
+	) >>apidiff/next.txt
-	) >apidiff/next.txt
+done
-	rm apidiff/next.txt.new
-else
-	mv apidiff/next.txt.new apidiff/next.txt
-fi

apidiff/next.txt

@@ -1,5 +0,0 @@
-Below are the incompatible changes between v0.0.15 and next, per package.
-
-# smtpclient
-- GatherDestinations: changed from func(context.Context, *log/slog.Logger, github.com/mjl-/mox/dns.Resolver, github.com/mjl-/mox/dns.IPDomain) (bool, bool, bool, github.com/mjl-/mox/dns.Domain, []github.com/mjl-/mox/dns.IPDomain, bool, error) to func(context.Context, *log/slog.Logger, github.com/mjl-/mox/dns.Resolver, github.com/mjl-/mox/dns.IPDomain) (bool, bool, bool, github.com/mjl-/mox/dns.Domain, []HostPref, bool, error)
-

apidiff/v0.0.13.txt

@@ -1,5 +0,0 @@
-Below are the incompatible changes between v0.0.13 and next, per package.
-
-# webhook
-- PartStructure: removed
-

apidiff/v0.0.15.txt

@@ -1,7 +0,0 @@
-Below are the incompatible changes between v0.0.14 and next, per package.
-
-# message
-- Part.ContentDescription: changed from string to *string
-- Part.ContentID: changed from string to *string
-- Part.ContentTransferEncoding: changed from string to *string
-

autotls/autotls.go

@@ -42,24 +42,6 @@ import (
 )
 
 var (
-	metricMissingServerName = promauto.NewCounter(
-		prometheus.CounterOpts{
-			Name: "mox_autotls_missing_servername_total",
-			Help: "Number of failed TLS connection attempts with missing SNI where no fallback hostname was configured.",
-		},
-	)
-	metricUnknownServerName = promauto.NewCounter(
-		prometheus.CounterOpts{
-			Name: "mox_autotls_unknown_servername_total",
-			Help: "Number of failed TLS connection attempts with an unrecognized SNI name where no fallback hostname was configured.",
-		},
-	)
-	metricCertRequestErrors = promauto.NewCounter(
-		prometheus.CounterOpts{
-			Name: "mox_autotls_cert_request_errors_total",
-			Help: "Number of errors trying to retrieve a certificate for a hostname, possibly ACME verification errors.",
-		},
-	)
 	metricCertput = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Name: "mox_autotls_certput_total",
@@ -94,7 +76,7 @@ type Manager struct {
 // host, or a newly generated key.
 //
 // When shutdown is closed, no new TLS connections can be created.
-func Load(log mlog.Log, name, acmeDir, contactEmail, directoryURL string, eabKeyID string, eabKey []byte, getPrivateKey func(host string, keyType autocert.KeyType) (crypto.Signer, error), shutdown <-chan struct{}) (*Manager, error) {
+func Load(name, acmeDir, contactEmail, directoryURL string, eabKeyID string, eabKey []byte, getPrivateKey func(host string, keyType autocert.KeyType) (crypto.Signer, error), shutdown <-chan struct{}) (*Manager, error) {
 	if directoryURL == "" {
 		return nil, fmt.Errorf("empty ACME directory URL")
 	}
@@ -107,10 +89,7 @@ func Load(log mlog.Log, name, acmeDir, contactEmail, directoryURL string, eabKey
 	var key crypto.Signer
 	f, err := os.Open(p)
 	if f != nil {
-		defer func() {
+		defer f.Close()
-			err := f.Close()
-			log.Check(err, "closing identify key file")
-		}()
 	}
 	if err != nil && os.IsNotExist(err) {
 		key, err = ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader)
@@ -192,75 +171,45 @@ func Load(log mlog.Log, name, acmeDir, contactEmail, directoryURL string, eabKey
 	return a, nil
 }
 
-// loggingGetCertificate is a helper to implement crypto/tls.Config.GetCertificate,
+// logigngGetCertificate is a helper to implement crypto/tls.Config.GetCertificate,
 // optionally falling back to a certificate for fallbackHostname in case SNI is
 // absent or for an unknown hostname.
 func (m *Manager) loggingGetCertificate(hello *tls.ClientHelloInfo, fallbackHostname dns.Domain, fallbackNoSNI, fallbackUnknownSNI bool) (*tls.Certificate, error) {
-	log := mlog.New("autotls", nil).WithContext(hello.Context()).With(
+	log := mlog.New("autotls", nil).WithContext(hello.Context())
-		slog.Any("localaddr", hello.Conn.LocalAddr()),
-		slog.Any("supportedprotos", hello.SupportedProtos),
-		slog.String("servername", hello.ServerName),
-	)
 
 	// If we can't find a certificate (depending on fallback parameters), we return a
 	// nil certificate and nil error, which crypto/tls turns into a TLS alert
 	// "unrecognized name", which can be interpreted by clients as a hint that they are
-	// using the wrong hostname, or a certificate is missing. ../rfc/9325:578
+	// using the wrong hostname, or a certificate is missing.
-
-	// IP addresses for ServerName are not allowed, but happen in practice. If we
-	// should be lenient (fallbackUnknownSNI), we switch to the fallback hostname,
-	// otherwise we return an error. We don't want to pass IP addresses to
-	// GetCertificate because it will return an error for IPv6 addresses.
-	// ../rfc/6066:367 ../rfc/4366:535
-	if net.ParseIP(hello.ServerName) != nil {
-		if fallbackUnknownSNI {
-			hello.ServerName = fallbackHostname.ASCII
-			log = log.With(slog.String("servername", hello.ServerName))
-		} else {
-			log.Debug("tls request with ip for server name, rejecting")
-			return nil, fmt.Errorf("invalid ip address for sni server name")
-		}
-	}
 
 	if hello.ServerName == "" && fallbackNoSNI {
 		hello.ServerName = fallbackHostname.ASCII
-		log = log.With(slog.String("servername", hello.ServerName))
 	}
 
 	// Handle missing SNI to prevent logging an error below.
 	if hello.ServerName == "" {
-		metricMissingServerName.Inc()
+		log.Debug("tls request without sni servername, rejecting", slog.Any("localaddr", hello.Conn.LocalAddr()), slog.Any("supportedprotos", hello.SupportedProtos))
-		log.Debug("tls request without sni server name, rejecting")
 		return nil, nil
 	}
 
 	cert, err := m.Manager.GetCertificate(hello)
 	if err != nil && errors.Is(err, errHostNotAllowed) {
 		if !fallbackUnknownSNI {
-			metricUnknownServerName.Inc()
+			log.Debugx("requesting certificate", err, slog.String("host", hello.ServerName))
-			log.Debugx("requesting certificate", err)
 			return nil, nil
 		}
 
-		// Some legitimate email deliveries over SMTP use an unknown SNI, e.g. a bare
+		log.Debug("certificate for unknown hostname, using fallback hostname", slog.String("host", hello.ServerName))
-		// domain instead of the MX hostname. We "should" return an error, but that would
-		// break email delivery, so we use the fallback name if it is configured.
-		// ../rfc/9325:589
-
-		log = log.With(slog.String("servername", hello.ServerName))
-		log.Debug("certificate for unknown hostname, using fallback hostname")
 		hello.ServerName = fallbackHostname.ASCII
 		cert, err = m.Manager.GetCertificate(hello)
 		if err != nil {
-			metricCertRequestErrors.Inc()
+			log.Errorx("requesting certificate for fallback hostname", err, slog.String("host", hello.ServerName))
-			log.Errorx("requesting certificate for fallback hostname", err)
 		} else {
-			log.Debug("using certificate for fallback hostname")
+			log.Debugx("requesting certificate for fallback hostname", err, slog.String("host", hello.ServerName))
 		}
 		return cert, err
 	} else if err != nil {
-		metricCertRequestErrors.Inc()
+		log.Errorx("requesting certificate", err, slog.String("host", hello.ServerName))
-		log.Errorx("requesting certificate", err)
 	}
 	return cert, err
 }
@@ -363,12 +312,12 @@ func (m *Manager) SetAllowedHostnames(log mlog.Log, resolver dns.Resolver, hostn
 			for _, h := range added {
 				ips, _, err := resolver.LookupIP(ctx, "ip", h.ASCII+".")
 				if err != nil {
-					log.Warnx("acme tls cert validation for host may fail due to dns lookup error", err, slog.Any("host", h))
+					log.Errorx("warning: acme tls cert validation for host may fail due to dns lookup error", err, slog.Any("host", h))
 					continue
 				}
 				for _, ip := range ips {
 					if _, ok := publicIPstrs[ip.String()]; !ok {
-						log.Warn("acme tls cert validation for host is likely to fail because not all its ips are being listened on",
+						log.Error("warning: acme tls cert validation for host is likely to fail because not all its ips are being listened on",
 							slog.Any("hostname", h),
 							slog.Any("listenedips", publicIPs),
 							slog.Any("hostips", ips),

autotls/autotls_test.go

@@ -25,7 +25,7 @@ func TestAutotls(t *testing.T) {
 	getPrivateKey := func(host string, keyType autocert.KeyType) (crypto.Signer, error) {
 		return nil, fmt.Errorf("not used")
 	}
-	m, err := Load(log, "test", "../testdata/autotls", "mox@localhost", "https://localhost/", "", nil, getPrivateKey, shutdown)
+	m, err := Load("test", "../testdata/autotls", "mox@localhost", "https://localhost/", "", nil, getPrivateKey, shutdown)
 	if err != nil {
 		t.Fatalf("load manager: %v", err)
 	}
@@ -82,7 +82,7 @@ func TestAutotls(t *testing.T) {
 
 	key0 := m.Manager.Client.Key
 
-	m, err = Load(log, "test", "../testdata/autotls", "mox@localhost", "https://localhost/", "", nil, getPrivateKey, shutdown)
+	m, err = Load("test", "../testdata/autotls", "mox@localhost", "https://localhost/", "", nil, getPrivateKey, shutdown)
 	if err != nil {
 		t.Fatalf("load manager again: %v", err)
 	}
@@ -95,7 +95,7 @@ func TestAutotls(t *testing.T) {
 		t.Fatalf("hostpolicy, got err %v, expected no error", err)
 	}
 
-	m2, err := Load(log, "test2", "../testdata/autotls", "mox@localhost", "https://localhost/", "", nil, nil, shutdown)
+	m2, err := Load("test2", "../testdata/autotls", "mox@localhost", "https://localhost/", "", nil, nil, shutdown)
 	if err != nil {
 		t.Fatalf("load another manager: %v", err)
 	}

backup.go

@@ -11,7 +11,6 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
-	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -27,7 +26,7 @@ import (
 	"github.com/mjl-/mox/tlsrptdb"
 )
 
-func xbackupctl(ctx context.Context, xctl *ctl) {
+func backupctl(ctx context.Context, ctl *ctl) {
 	/* protocol:
 	> "backup"
 	> destdir
@@ -41,14 +40,14 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 	// "src" or "dst" are incomplete paths relative to the source or destination data
 	// directories.
 
-	dstDir := xctl.xread()
+	dstDataDir := ctl.xread()
-	verbose := xctl.xread() == "verbose"
+	verbose := ctl.xread() == "verbose"
 
 	// Set when an error is encountered. At the end, we warn if set.
 	var incomplete bool
 
 	// We'll be writing output, and logging both to mox and the ctl stream.
-	xwriter := xctl.writer()
+	writer := ctl.writer()
 
 	// Format easily readable output for the user.
 	formatLog := func(prefix, text string, err error, attrs ...slog.Attr) []byte {
@@ -67,8 +66,10 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 
 	// Log an error to both the mox service as the user running "mox backup".
 	pkglogx := func(prefix, text string, err error, attrs ...slog.Attr) {
-		xctl.log.Errorx(text, err, attrs...)
+		ctl.log.Errorx(text, err, attrs...)
-		xwriter.Write(formatLog(prefix, text, err, attrs...))
+
+		_, werr := writer.Write(formatLog(prefix, text, err, attrs...))
+		ctl.xcheck(werr, "write to ctl")
 	}
 
 	// Log an error but don't mark backup as failed.
@@ -85,100 +86,15 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 
 	// If verbose is enabled, log to the cli command. Always log as info level.
 	xvlog := func(text string, attrs ...slog.Attr) {
-		xctl.log.Info(text, attrs...)
+		ctl.log.Info(text, attrs...)
 		if verbose {
-			xwriter.Write(formatLog("", text, nil, attrs...))
+			_, werr := writer.Write(formatLog("", text, nil, attrs...))
+			ctl.xcheck(werr, "write to ctl")
 		}
 	}
 
-	dstConfigDir := filepath.Join(dstDir, "config")
-	dstDataDir := filepath.Join(dstDir, "data")
-
-	// Warn if directories already exist, will likely cause failures when trying to
-	// write files that already exist.
-	if _, err := os.Stat(dstConfigDir); err == nil {
-		xwarnx("destination config directory already exists", nil, slog.String("configdir", dstConfigDir))
-	}
 	if _, err := os.Stat(dstDataDir); err == nil {
-		xwarnx("destination data directory already exists", nil, slog.String("datadir", dstDataDir))
+		xwarnx("destination data directory already exists", nil, slog.String("dir", dstDataDir))
-	}
-
-	os.MkdirAll(dstDir, 0770)
-	os.MkdirAll(dstConfigDir, 0770)
-	os.MkdirAll(dstDataDir, 0770)
-
-	// Copy all files in the config dir.
-	srcConfigDir := filepath.Clean(mox.ConfigDirPath("."))
-	err := filepath.WalkDir(srcConfigDir, func(srcPath string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-
-		if srcConfigDir == srcPath {
-			return nil
-		}
-
-		// Trim directory and separator.
-		relPath := srcPath[len(srcConfigDir)+1:]
-
-		destPath := filepath.Join(dstConfigDir, relPath)
-
-		if d.IsDir() {
-			if info, err := os.Stat(srcPath); err != nil {
-				return fmt.Errorf("stat config dir %s: %v", srcPath, err)
-			} else if err := os.Mkdir(destPath, info.Mode()&0777); err != nil {
-				return fmt.Errorf("mkdir %s: %v", destPath, err)
-			}
-			return nil
-		}
-		if d.Type()&fs.ModeSymlink != 0 {
-			linkDest, err := os.Readlink(srcPath)
-			if err != nil {
-				return fmt.Errorf("reading symlink %s: %v", srcPath, err)
-			}
-			if err := os.Symlink(linkDest, destPath); err != nil {
-				return fmt.Errorf("creating symlink %s: %v", destPath, err)
-			}
-			return nil
-		}
-		if !d.Type().IsRegular() {
-			xwarnx("skipping non-regular/dir/symlink file in config dir", nil, slog.String("path", srcPath))
-			return nil
-		}
-
-		sf, err := os.Open(srcPath)
-		if err != nil {
-			return fmt.Errorf("open config file %s: %v", srcPath, err)
-		}
-		info, err := sf.Stat()
-		if err != nil {
-			return fmt.Errorf("stat config file %s: %v", srcPath, err)
-		}
-		df, err := os.OpenFile(destPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0777&info.Mode())
-		if err != nil {
-			return fmt.Errorf("create destination config file %s: %v", destPath, err)
-		}
-		defer func() {
-			if df != nil {
-				err := df.Close()
-				xctl.log.Check(err, "closing file")
-			}
-		}()
-		defer func() {
-			err := sf.Close()
-			xctl.log.Check(err, "closing file")
-		}()
-		if _, err := io.Copy(df, sf); err != nil {
-			return fmt.Errorf("copying config file %s to %s: %v", srcPath, destPath, err)
-		}
-		if err := df.Close(); err != nil {
-			return fmt.Errorf("closing destination config file %s: %v", srcPath, err)
-		}
-		df = nil
-		return nil
-	})
-	if err != nil {
-		xerrx("storing config directory", err)
 	}
 
 	srcDataDir := filepath.Clean(mox.DataDirPath("."))
@@ -208,10 +124,7 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 			xerrx("open source file (not backed up)", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
 			return
 		}
-		defer func() {
+		defer sf.Close()
-			err := sf.Close()
-			xctl.log.Check(err, "closing source file")
-		}()
 
 		ensureDestDir(dstpath)
 		df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
@@ -221,8 +134,7 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 		}
 		defer func() {
 			if df != nil {
-				err := df.Close()
+				df.Close()
-				xctl.log.Check(err, "closing destination file")
 			}
 		}()
 		if _, err := io.Copy(df, sf); err != nil {
@@ -264,9 +176,18 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 		xvlog("backed up directory", slog.String("dir", dir), slog.Duration("duration", time.Since(tmDir)))
 	}
 
-	// Backup a database by copying it in a readonly transaction. Wrapped by backupDB
+	// Backup a database by copying it in a readonly transaction.
-	// which logs and returns just a bool.
+	// Always logs on error, so caller doesn't have to, but also returns the error so
-	backupDB0 := func(db *bstore.DB, path string) error {
+	// callers can see result.
+	backupDB := func(db *bstore.DB, path string) (rerr error) {
+		defer func() {
+			if rerr != nil {
+				xerrx("backing up database", rerr, slog.String("path", path))
+			}
+		}()
+
+		tmDB := time.Now()
+
 		dstpath := filepath.Join(dstDataDir, path)
 		ensureDestDir(dstpath)
 		df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
@@ -275,8 +196,7 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 		}
 		defer func() {
 			if df != nil {
-				err := df.Close()
+				df.Close()
-				xctl.log.Check(err, "closing destination database file")
 			}
 		}()
 		err = db.Read(ctx, func(tx *bstore.Tx) error {
@@ -301,20 +221,10 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 		if err != nil {
 			return fmt.Errorf("closing destination database after copy: %v", err)
 		}
+		xvlog("backed up database file", slog.String("path", path), slog.Duration("duration", time.Since(tmDB)))
 		return nil
 	}
 
-	backupDB := func(db *bstore.DB, path string) bool {
-		start := time.Now()
-		err := backupDB0(db, path)
-		if err != nil {
-			xerrx("backing up database", err, slog.String("path", path), slog.Duration("duration", time.Since(start)))
-			return false
-		}
-		xvlog("backed up database file", slog.String("path", path), slog.Duration("duration", time.Since(start)))
-		return true
-	}
-
 	// Try to create a hardlink. Fall back to copying the file (e.g. when on different file system).
 	warnedHardlink := false // We warn once about failing to hardlink.
 	linkOrCopy := func(srcpath, dstpath string) (bool, error) {
@@ -341,7 +251,7 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 		}
 		defer func() {
 			err := sf.Close()
-			xctl.log.Check(err, "closing copied source file")
+			ctl.log.Check(err, "closing copied source file")
 		}()
 
 		df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
@@ -351,7 +261,7 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 		defer func() {
 			if df != nil {
 				err := df.Close()
-				xctl.log.Check(err, "closing partial destination file")
+				ctl.log.Check(err, "closing partial destination file")
 			}
 		}()
 		if _, err := io.Copy(df, sf); err != nil {
@@ -368,16 +278,16 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 	// Start making the backup.
 	tmStart := time.Now()
 
-	xctl.log.Print("making backup", slog.String("destdir", dstDataDir))
+	ctl.log.Print("making backup", slog.String("destdir", dstDataDir))
 
-	if err := os.MkdirAll(dstDataDir, 0770); err != nil {
+	err := os.MkdirAll(dstDataDir, 0770)
+	if err != nil {
 		xerrx("creating destination data directory", err)
 	}
 
 	if err := os.WriteFile(filepath.Join(dstDataDir, "moxversion"), []byte(moxvar.Version), 0660); err != nil {
 		xerrx("writing moxversion", err)
 	}
-	backupDB(store.AuthDB, "auth.db")
 	backupDB(dmarcdb.ReportsDB, "dmarcrpt.db")
 	backupDB(dmarcdb.EvalDB, "dmarceval.db")
 	backupDB(mtastsdb.DB, "mtasts.db")
@@ -389,7 +299,7 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 	srcAcmeDir := filepath.Join(srcDataDir, "acme")
 	if _, err := os.Stat(srcAcmeDir); err == nil {
 		backupDir("acme")
-	} else if !os.IsNotExist(err) {
+	} else if err != nil && !os.IsNotExist(err) {
 		xerrx("copying acme/", err)
 	}
 
@@ -397,12 +307,13 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 	backupQueue := func(path string) {
 		tmQueue := time.Now()
 
-		if !backupDB(queue.DB, path) {
+		if err := backupDB(queue.DB, path); err != nil {
+			xerrx("queue not backed up", err, slog.String("path", path), slog.Duration("duration", time.Since(tmQueue)))
 			return
 		}
 
 		dstdbpath := filepath.Join(dstDataDir, path)
-		opts := bstore.Options{MustExist: true, RegisterLogger: xctl.log.Logger}
+		opts := bstore.Options{MustExist: true, RegisterLogger: ctl.log.Logger}
 		db, err := bstore.Open(ctx, dstdbpath, &opts, queue.DBTypes...)
 		if err != nil {
 			xerrx("open copied queue database", err, slog.String("dstpath", dstdbpath), slog.Duration("duration", time.Since(tmQueue)))
@@ -412,20 +323,17 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 		defer func() {
 			if db != nil {
 				err := db.Close()
-				xctl.log.Check(err, "closing new queue db")
+				ctl.log.Check(err, "closing new queue db")
 			}
 		}()
 
-		// Link/copy known message files. If a message has been removed while we read the
+		// Link/copy known message files. Warn if files are missing or unexpected
-		// database, our backup is not consistent and the backup will be marked failed.
+		// (though a message file could have been removed just now due to delivery, or a
+		// new message may have been queued).
 		tmMsgs := time.Now()
 		seen := map[string]struct{}{}
 		var nlinked, ncopied int
-		var maxID int64
 		err = bstore.QueryDB[queue.Msg](ctx, db).ForEach(func(m queue.Msg) error {
-			if m.ID > maxID {
-				maxID = m.ID
-			}
 			mp := store.MessagePath(m.ID)
 			seen[mp] = struct{}{}
 			srcpath := filepath.Join(srcDataDir, "queue", mp)
@@ -448,9 +356,7 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 				slog.Duration("duration", time.Since(tmMsgs)))
 		}
 
-		// Read through all files in queue directory and warn about anything we haven't
+		// Read through all files in queue directory and warn about anything we haven't handled yet.
-		// handled yet. Message files that are newer than we expect from our consistent
-		// database snapshot are ignored.
 		tmWalk := time.Now()
 		srcqdir := filepath.Join(srcDataDir, "queue")
 		err = filepath.WalkDir(srcqdir, func(srcqpath string, d fs.DirEntry, err error) error {
@@ -468,12 +374,6 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 			if p == "index.db" {
 				return nil
 			}
-			// Skip any messages that were added since we started on our consistent snapshot.
-			// We don't want to cause spurious backup warnings.
-			if id, err := strconv.ParseInt(filepath.Base(p), 10, 64); err == nil && maxID > 0 && id > maxID && p == store.MessagePath(id) {
-				return nil
-			}
-
 			qp := filepath.Join("queue", p)
 			xwarnx("backing up unrecognized file in queue directory", nil, slog.String("path", qp))
 			backupFile(qp)
@@ -490,21 +390,21 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 	backupQueue(filepath.FromSlash("queue/index.db"))
 
 	backupAccount := func(acc *store.Account) {
-		defer func() {
+		defer acc.Close()
-			err := acc.Close()
-			xctl.log.Check(err, "closing account")
-		}()
 
 		tmAccount := time.Now()
 
 		// Copy database file.
 		dbpath := filepath.Join("accounts", acc.Name, "index.db")
-		backupDB(acc.DB, dbpath)
+		err := backupDB(acc.DB, dbpath)
+		if err != nil {
+			xerrx("copying account database", err, slog.String("path", dbpath), slog.Duration("duration", time.Since(tmAccount)))
+		}
 
 		// todo: should document/check not taking a rlock on account.
 
 		// Copy junkfilter files, if configured.
-		if jf, _, err := acc.OpenJunkFilter(ctx, xctl.log); err != nil {
+		if jf, _, err := acc.OpenJunkFilter(ctx, ctl.log); err != nil {
 			if !errors.Is(err, store.ErrNoJunkFilter) {
 				xerrx("opening junk filter for account (not backed up)", err)
 			}
@@ -514,12 +414,13 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 			backupDB(db, jfpath)
 			bloompath := filepath.Join("accounts", acc.Name, "junkfilter.bloom")
 			backupFile(bloompath)
+			db = nil
 			err := jf.Close()
-			xctl.log.Check(err, "closing junkfilter")
+			ctl.log.Check(err, "closing junkfilter")
 		}
 
 		dstdbpath := filepath.Join(dstDataDir, dbpath)
-		opts := bstore.Options{MustExist: true, RegisterLogger: xctl.log.Logger}
+		opts := bstore.Options{MustExist: true, RegisterLogger: ctl.log.Logger}
 		db, err := bstore.Open(ctx, dstdbpath, &opts, store.DBTypes...)
 		if err != nil {
 			xerrx("open copied account database", err, slog.String("dstpath", dstdbpath), slog.Duration("duration", time.Since(tmAccount)))
@@ -529,19 +430,17 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 		defer func() {
 			if db != nil {
 				err := db.Close()
-				xctl.log.Check(err, "close account database")
+				ctl.log.Check(err, "close account database")
 			}
 		}()
 
-		// Link/copy known message files.
+		// Link/copy known message files. Warn if files are missing or unexpected (though a
+		// message file could have been added just now due to delivery, or a message have
+		// been removed).
 		tmMsgs := time.Now()
 		seen := map[string]struct{}{}
-		var maxID int64
 		var nlinked, ncopied int
 		err = bstore.QueryDB[store.Message](ctx, db).FilterEqual("Expunged", false).ForEach(func(m store.Message) error {
-			if m.ID > maxID {
-				maxID = m.ID
-			}
 			mp := store.MessagePath(m.ID)
 			seen[mp] = struct{}{}
 			amp := filepath.Join("accounts", acc.Name, "msg", mp)
@@ -565,18 +464,7 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 				slog.Duration("duration", time.Since(tmMsgs)))
 		}
 
-		eraseIDs := map[int64]struct{}{}
+		// Read through all files in account directory and warn about anything we haven't handled yet.
-		err = bstore.QueryDB[store.MessageErase](ctx, db).ForEach(func(me store.MessageErase) error {
-			eraseIDs[me.ID] = struct{}{}
-			return nil
-		})
-		if err != nil {
-			xerrx("listing erased messages", err)
-		}
-
-		// Read through all files in queue directory and warn about anything we haven't
-		// handled yet. Message files that are newer than we expect from our consistent
-		// database snapshot are ignored.
 		tmWalk := time.Now()
 		srcadir := filepath.Join(srcDataDir, "accounts", acc.Name)
 		err = filepath.WalkDir(srcadir, func(srcapath string, d fs.DirEntry, err error) error {
@@ -594,16 +482,6 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 				if _, ok := seen[mp]; ok {
 					return nil
 				}
-
-				// Skip any messages that were added since we started on our consistent snapshot,
-				// or messages that will be erased. We don't want to cause spurious backup
-				// warnings.
-				id, err := strconv.ParseInt(l[len(l)-1], 10, 64)
-				if err == nil && id > maxID && mp == store.MessagePath(id) {
-					return nil
-				} else if _, ok := eraseIDs[id]; err == nil && ok {
-					return nil
-				}
 			}
 			switch p {
 			case "index.db", "junkfilter.db", "junkfilter.bloom":
@@ -632,7 +510,7 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 	// account directories when handling "all other files" below.
 	accounts := map[string]struct{}{}
 	for _, accName := range mox.Conf.Accounts() {
-		acc, err := store.OpenAccount(xctl.log, accName, false)
+		acc, err := store.OpenAccount(ctl.log, accName)
 		if err != nil {
 			xerrx("opening account for copying (will try to copy as regular files later)", err, slog.String("account", accName))
 			continue
@@ -670,7 +548,7 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 		}
 
 		switch p {
-		case "auth.db", "dmarcrpt.db", "dmarceval.db", "mtasts.db", "tlsrpt.db", "tlsrptresult.db", "receivedid.key", "ctl":
+		case "dmarcrpt.db", "dmarceval.db", "mtasts.db", "tlsrpt.db", "tlsrptresult.db", "receivedid.key", "ctl":
 			// Already handled.
 			return nil
 		case "lastknownversion": // Optional file, not yet handled.
@@ -688,11 +566,11 @@ func xbackupctl(ctx context.Context, xctl *ctl) {
 
 	xvlog("backup finished", slog.Duration("duration", time.Since(tmStart)))
 
-	xwriter.xclose()
+	writer.xclose()
 
 	if incomplete {
-		xctl.xwrite("errors were encountered during backup")
+		ctl.xwrite("errors were encountered during backup")
 	} else {
-		xctl.xwriteok()
+		ctl.xwriteok()
 	}
 }

config/config.go

@@ -61,11 +61,11 @@ type Static struct {
 	HostTLSRPT struct {
 		Account   string `sconf-doc:"Account to deliver TLS reports to. Typically same account as for postmaster."`
 		Mailbox   string `sconf-doc:"Mailbox to deliver TLS reports to. Recommended value: TLSRPT."`
-		Localpart string `sconf-doc:"Localpart at hostname to accept TLS reports at. Recommended value: tlsreports."`
+		Localpart string `sconf-doc:"Localpart at hostname to accept TLS reports at. Recommended value: tls-reports."`
 
 		ParsedLocalpart smtp.Localpart `sconf:"-"`
 	} `sconf:"optional" sconf-doc:"Destination for per-host TLS reports (TLSRPT). TLS reports can be per recipient domain (for MTA-STS), or per MX host (for DANE). The per-domain TLS reporting configuration is in domains.conf. This is the TLS reporting configuration for this host. If absent, no host-based TLSRPT address is configured, and no host TLSRPT DNS record is suggested."`
-	InitialMailboxes InitialMailboxes     `sconf:"optional" sconf-doc:"Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be given a 'special-use' role, which are understood by most mail clients. If absent/empty, the following additional mailboxes are created: Sent, Archive, Trash, Drafts and Junk."`
+	InitialMailboxes InitialMailboxes     `sconf:"optional" sconf-doc:"Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be given a 'special-use' role, which are understood by most mail clients. If absent/empty, the following mailboxes are created: Sent, Archive, Trash, Drafts and Junk."`
 	DefaultMailboxes []string             `sconf:"optional" sconf-doc:"Deprecated in favor of InitialMailboxes. Mailboxes to create when adding an account. Inbox is always created. If no mailboxes are specified, the following are automatically created: Sent, Archive, Trash, Drafts and Junk."`
 	Transports       map[string]Transport `sconf:"optional" sconf-doc:"Transport are mechanisms for delivering messages. Transports can be referenced from Routes in accounts, domains and the global configuration. There is always an implicit/fallback delivery transport doing direct delivery with SMTP from the outgoing message queue. Transports are typically only configured when using smarthosts, i.e. when delivering through another SMTP server. Zero or one transport methods must be set in a transport, never multiple. When using an external party to send email for a domain, keep in mind you may have to add their IP address to your domain's SPF record, and possibly additional DKIM records."`
 	// Awkward naming of fields to get intended default behaviour for zero values.
@@ -158,8 +158,6 @@ type Listener struct {
 
 		FirstTimeSenderDelay *time.Duration `sconf:"optional" sconf-doc:"Delay before accepting a message from a first-time sender for the destination account. Default: 15s."`
 
-		TLSSessionTicketsDisabled *bool `sconf:"optional" sconf-doc:"Override default setting for enabling TLS session tickets. Disabling session tickets may work around TLS interoperability issues."`
-
 		DNSBLZones []dns.Domain `sconf:"-"`
 	} `sconf:"optional"`
 	Submission struct {
@@ -168,9 +166,8 @@ type Listener struct {
 		NoRequireSTARTTLS bool `sconf:"optional" sconf-doc:"Do not require STARTTLS. Since users must login, this means password may be sent without encryption. Not recommended."`
 	} `sconf:"optional" sconf-doc:"SMTP for submitting email, e.g. by email applications. Starts out in plain text, can be upgraded to TLS with the STARTTLS command. Prefer using Submissions which is always a TLS connection."`
 	Submissions struct {
-		Enabled        bool
+		Enabled bool
-		Port           int  `sconf:"optional" sconf-doc:"Default 465."`
+		Port    int `sconf:"optional" sconf-doc:"Default 465."`
-		EnabledOnHTTPS bool `sconf:"optional" sconf-doc:"Additionally enable submission on HTTPS port 443 via TLS ALPN. TLS Application Layer Protocol Negotiation allows clients to request a specific protocol from the server as part of the TLS connection setup. When this setting is enabled and a client requests the 'smtp' protocol after TLS, it will be able to talk SMTP to Mox on port 443. This is meant to be useful as a censorship circumvention technique for Delta Chat."`
 	} `sconf:"optional" sconf-doc:"SMTP over TLS for submitting email, by email applications. Requires a TLS config."`
 	IMAP struct {
 		Enabled           bool
@@ -178,9 +175,8 @@ type Listener struct {
 		NoRequireSTARTTLS bool `sconf:"optional" sconf-doc:"Enable this only when the connection is otherwise encrypted (e.g. through a VPN)."`
 	} `sconf:"optional" sconf-doc:"IMAP for reading email, by email applications. Starts out in plain text, can be upgraded to TLS with the STARTTLS command. Prefer using IMAPS instead which is always a TLS connection."`
 	IMAPS struct {
-		Enabled        bool
+		Enabled bool
-		Port           int  `sconf:"optional" sconf-doc:"Default 993."`
+		Port    int `sconf:"optional" sconf-doc:"Default 993."`
-		EnabledOnHTTPS bool `sconf:"optional" sconf-doc:"Additionally enable IMAP on HTTPS port 443 via TLS ALPN. TLS Application Layer Protocol Negotiation allows clients to request a specific protocol from the server as part of the TLS connection setup. When this setting is enabled and a client requests the 'imap' protocol after TLS, it will be able to talk IMAP to Mox on port 443. This is meant to be useful as a censorship circumvention technique for Delta Chat."`
 	} `sconf:"optional" sconf-doc:"IMAP over TLS for reading email, by email applications. Requires a TLS config."`
 	AccountHTTP  WebService `sconf:"optional" sconf-doc:"Account web interface, for email users wanting to change their accounts, e.g. set new password, set new delivery rulesets. Default path is /."`
 	AccountHTTPS WebService `sconf:"optional" sconf-doc:"Account web interface listener like AccountHTTP, but for HTTPS. Requires a TLS config."`
@@ -209,14 +205,12 @@ type Listener struct {
 		NonTLS  bool `sconf:"optional" sconf-doc:"If set, plain HTTP instead of HTTPS is spoken on the configured port. Can be useful when the mta-sts domain is reverse proxied."`
 	} `sconf:"optional" sconf-doc:"Serve MTA-STS policies describing SMTP TLS requirements. Requires a TLS config."`
 	WebserverHTTP struct {
-		Enabled           bool
+		Enabled bool
-		Port              int  `sconf:"optional" sconf-doc:"Port for plain HTTP (non-TLS) webserver."`
+		Port    int `sconf:"optional" sconf-doc:"Port for plain HTTP (non-TLS) webserver."`
-		RateLimitDisabled bool `sconf:"optional" sconf-doc:"Disable rate limiting for all requests to this port."`
 	} `sconf:"optional" sconf-doc:"All configured WebHandlers will serve on an enabled listener."`
 	WebserverHTTPS struct {
-		Enabled           bool
+		Enabled bool
-		Port              int  `sconf:"optional" sconf-doc:"Port for HTTPS webserver."`
+		Port    int `sconf:"optional" sconf-doc:"Port for HTTPS webserver."`
-		RateLimitDisabled bool `sconf:"optional" sconf-doc:"Disable rate limiting for all requests to this port."`
 	} `sconf:"optional" sconf-doc:"All configured WebHandlers will serve on an enabled listener. Either ACME must be configured, or for each WebHandler domain a TLS certificate must be configured."`
 }
 
@@ -224,7 +218,7 @@ type Listener struct {
 type WebService struct {
 	Enabled   bool
 	Port      int    `sconf:"optional" sconf-doc:"Default 80 for HTTP and 443 for HTTPS. See Hostname at Listener for hostname matching behaviour."`
-	Path      string `sconf:"optional" sconf-doc:"Path to serve requests on. Should end with a slash, related to cookie paths."`
+	Path      string `sconf:"optional" sconf-doc:"Path to serve requests on."`
 	Forwarded bool   `sconf:"optional" sconf-doc:"If set, X-Forwarded-* headers are used for the remote IP address for rate limiting and for the \"secure\" status of cookies."`
 }
 
@@ -237,7 +231,6 @@ type Transport struct {
 	SMTP        *TransportSMTP   `sconf:"optional" sconf-doc:"SMTP over a plain connection (possibly with STARTTLS), typically for old-fashioned unauthenticated relaying to a remote queue."`
 	Socks       *TransportSocks  `sconf:"optional" sconf-doc:"Like regular direct delivery, but makes outgoing connections through a SOCKS proxy."`
 	Direct      *TransportDirect `sconf:"optional" sconf-doc:"Like regular direct delivery, but allows to tweak outgoing connections."`
-	Fail        *TransportFail   `sconf:"optional" sconf-doc:"Immediately fails the delivery attempt."`
 }
 
 // TransportSMTP delivers messages by "submission" (SMTP, typically
@@ -281,29 +274,17 @@ type TransportDirect struct {
 	IPFamily string `sconf:"-" json:"-"`
 }
 
-// TransportFail is a transport that fails all delivery attempts.
-type TransportFail struct {
-	SMTPCode    int    `sconf:"optional" sconf-doc:"SMTP error code and optional enhanced error code to use for the failure. If empty, 554 is used (transaction failed)."`
-	SMTPMessage string `sconf:"optional" sconf-doc:"Message to include for the rejection. It will be shown in the DSN."`
-
-	// Effective values to use, set when parsing.
-	Code    int    `sconf:"-"`
-	Message string `sconf:"-"`
-}
-
 type Domain struct {
-	Disabled                    bool             `sconf:"optional" sconf-doc:"Disabled domains can be useful during/before migrations. Domains that are disabled can still be configured like normal, including adding addresses using the domain to accounts. However, disabled domains: 1. Do not try to fetch ACME certificates. TLS connections to host names involving the email domain will fail. A TLS certificate for the hostname (that wil be used as MX) itself will be requested. 2. Incoming deliveries over SMTP are rejected with a temporary error '450 4.2.1 recipient domain temporarily disabled'. 3. Submissions over SMTP using an (envelope) SMTP MAIL FROM address or message 'From' address of a disabled domain will be rejected with a temporary error '451 4.3.0 sender domain temporarily disabled'. Note that accounts with addresses at disabled domains can still log in and read email (unless the account itself is disabled)."`
+	Description                string           `sconf:"optional" sconf-doc:"Free-form description of domain."`
-	Description                 string           `sconf:"optional" sconf-doc:"Free-form description of domain."`
+	ClientSettingsDomain       string           `sconf:"optional" sconf-doc:"Hostname for client settings instead of the mail server hostname. E.g. mail.<domain>. For future migration to another mail operator without requiring all clients to update their settings, it is convenient to have client settings that reference a subdomain of the hosted domain instead of the hostname of the server where the mail is currently hosted. If empty, the hostname of the mail server is used for client configurations. Unicode name."`
-	ClientSettingsDomain        string           `sconf:"optional" sconf-doc:"Hostname for client settings instead of the mail server hostname. E.g. mail.<domain>. For future migration to another mail operator without requiring all clients to update their settings, it is convenient to have client settings that reference a subdomain of the hosted domain instead of the hostname of the server where the mail is currently hosted. If empty, the hostname of the mail server is used for client configurations. Unicode name."`
+	LocalpartCatchallSeparator string           `sconf:"optional" sconf-doc:"If not empty, only the string before the separator is used to for email delivery decisions. For example, if set to \"+\", you+anything@example.com will be delivered to you@example.com."`
-	LocalpartCatchallSeparator  string           `sconf:"optional" sconf-doc:"If not empty, only the string before the separator is used to for email delivery decisions. For example, if set to \"+\", you+anything@example.com will be delivered to you@example.com."`
+	LocalpartCaseSensitive     bool             `sconf:"optional" sconf-doc:"If set, upper/lower case is relevant for email delivery."`
-	LocalpartCatchallSeparators []string         `sconf:"optional" sconf-doc:"Similar to LocalpartCatchallSeparator, but in case multiple are needed. For example both \"+\" and \"-\". Only of one LocalpartCatchallSeparator or LocalpartCatchallSeparators can be set. If set, the first separator is used to make unique addresses for outgoing SMTP connections with FromIDLoginAddresses."`
+	DKIM                       DKIM             `sconf:"optional" sconf-doc:"With DKIM signing, a domain is taking responsibility for (content of) emails it sends, letting receiving mail servers build up a (hopefully positive) reputation of the domain, which can help with mail delivery."`
-	LocalpartCaseSensitive      bool             `sconf:"optional" sconf-doc:"If set, upper/lower case is relevant for email delivery."`
+	DMARC                      *DMARC           `sconf:"optional" sconf-doc:"With DMARC, a domain publishes, in DNS, a policy on how other mail servers should handle incoming messages with the From-header matching this domain and/or subdomain (depending on the configured alignment). Receiving mail servers use this to build up a reputation of this domain, which can help with mail delivery. A domain can also publish an email address to which reports about DMARC verification results can be sent by verifying mail servers, useful for monitoring. Incoming DMARC reports are automatically parsed, validated, added to metrics and stored in the reporting database for later display in the admin web pages."`
-	DKIM                        DKIM             `sconf:"optional" sconf-doc:"With DKIM signing, a domain is taking responsibility for (content of) emails it sends, letting receiving mail servers build up a (hopefully positive) reputation of the domain, which can help with mail delivery."`
+	MTASTS                     *MTASTS          `sconf:"optional" sconf-doc:"MTA-STS is a mechanism that allows publishing a policy with requirements for WebPKI-verified SMTP STARTTLS connections for email delivered to a domain. Existence of a policy is announced in a DNS TXT record (often unprotected/unverified, MTA-STS's weak spot). If a policy exists, it is fetched with a WebPKI-verified HTTPS request. The policy can indicate that WebPKI-verified SMTP STARTTLS is required, and which MX hosts (optionally with a wildcard pattern) are allowd. MX hosts to deliver to are still taken from DNS (again, not necessarily protected/verified), but messages will only be delivered to domains matching the MX hosts from the published policy. Mail servers look up the MTA-STS policy when first delivering to a domain, then keep a cached copy, periodically checking the DNS record if a new policy is available, and fetching and caching it if so. To update a policy, first serve a new policy with an updated policy ID, then update the DNS record (not the other way around). To remove an enforced policy, publish an updated policy with mode \"none\" for a long enough period so all cached policies have been refreshed (taking DNS TTL and policy max age into account), then remove the policy from DNS, wait for TTL to expire, and stop serving the policy."`
-	DMARC                       *DMARC           `sconf:"optional" sconf-doc:"With DMARC, a domain publishes, in DNS, a policy on how other mail servers should handle incoming messages with the From-header matching this domain and/or subdomain (depending on the configured alignment). Receiving mail servers use this to build up a reputation of this domain, which can help with mail delivery. A domain can also publish an email address to which reports about DMARC verification results can be sent by verifying mail servers, useful for monitoring. Incoming DMARC reports are automatically parsed, validated, added to metrics and stored in the reporting database for later display in the admin web pages."`
+	TLSRPT                     *TLSRPT          `sconf:"optional" sconf-doc:"With TLSRPT a domain specifies in DNS where reports about encountered SMTP TLS behaviour should be sent. Useful for monitoring. Incoming TLS reports are automatically parsed, validated, added to metrics and stored in the reporting database for later display in the admin web pages."`
-	MTASTS                      *MTASTS          `sconf:"optional" sconf-doc:"MTA-STS is a mechanism that allows publishing a policy with requirements for WebPKI-verified SMTP STARTTLS connections for email delivered to a domain. Existence of a policy is announced in a DNS TXT record (often unprotected/unverified, MTA-STS's weak spot). If a policy exists, it is fetched with a WebPKI-verified HTTPS request. The policy can indicate that WebPKI-verified SMTP STARTTLS is required, and which MX hosts (optionally with a wildcard pattern) are allowd. MX hosts to deliver to are still taken from DNS (again, not necessarily protected/verified), but messages will only be delivered to domains matching the MX hosts from the published policy. Mail servers look up the MTA-STS policy when first delivering to a domain, then keep a cached copy, periodically checking the DNS record if a new policy is available, and fetching and caching it if so. To update a policy, first serve a new policy with an updated policy ID, then update the DNS record (not the other way around). To remove an enforced policy, publish an updated policy with mode \"none\" for a long enough period so all cached policies have been refreshed (taking DNS TTL and policy max age into account), then remove the policy from DNS, wait for TTL to expire, and stop serving the policy."`
+	Routes                     []Route          `sconf:"optional" sconf-doc:"Routes for delivering outgoing messages through the queue. Each delivery attempt evaluates account routes, these domain routes and finally global routes. The transport of the first matching route is used in the delivery attempt. If no routes match, which is the default with no configured routes, messages are delivered directly from the queue."`
-	TLSRPT                      *TLSRPT          `sconf:"optional" sconf-doc:"With TLSRPT a domain specifies in DNS where reports about encountered SMTP TLS behaviour should be sent. Useful for monitoring. Incoming TLS reports are automatically parsed, validated, added to metrics and stored in the reporting database for later display in the admin web pages."`
+	Aliases                    map[string]Alias `sconf:"optional" sconf-doc:"Aliases that cause messages to be delivered to one or more locally configured addresses. Keys are localparts (encoded, as they appear in email addresses)."`
-	Routes                      []Route          `sconf:"optional" sconf-doc:"Routes for delivering outgoing messages through the queue. Each delivery attempt evaluates account routes, these domain routes and finally global routes. The transport of the first matching route is used in the delivery attempt. If no routes match, which is the default with no configured routes, messages are delivered directly from the queue."`
-	Aliases                     map[string]Alias `sconf:"optional" sconf-doc:"Aliases that cause messages to be delivered to one or more locally configured addresses. Keys are localparts (encoded, as they appear in email addresses)."`
 
 	Domain                  dns.Domain `sconf:"-"`
 	ClientSettingsDNSDomain dns.Domain `sconf:"-" json:"-"`
@@ -311,8 +292,7 @@ type Domain struct {
 	// Set when DMARC and TLSRPT (when set) has an address with different domain (we're
 	// hosting the reporting), and there are no destination addresses configured for
 	// the domain. Disables some functionality related to hosting a domain.
-	ReportsOnly                          bool     `sconf:"-" json:"-"`
+	ReportsOnly bool `sconf:"-" json:"-"`
-	LocalpartCatchallSeparatorsEffective []string `sconf:"-"` // Either LocalpartCatchallSeparators, the value of LocalpartCatchallSeparator, or empty.
 }
 
 // todo: allow external addresses as members of aliases. we would add messages for them to the queue for outgoing delivery. we should require an admin addresses to which delivery failures will be delivered (locally, and to use in smtp mail from, so dsns go there). also take care to evaluate smtputf8 (if external address requires utf8 and incoming transaction didn't).
@@ -337,12 +317,12 @@ type AliasAddress struct {
 }
 
 type DMARC struct {
-	Localpart string `sconf-doc:"Address-part before the @ that accepts DMARC reports. Must be non-internationalized. Recommended value: dmarcreports."`
+	Localpart string `sconf-doc:"Address-part before the @ that accepts DMARC reports. Must be non-internationalized. Recommended value: dmarc-reports."`
 	Domain    string `sconf:"optional" sconf-doc:"Alternative domain for reporting address, for incoming reports. Typically empty, causing the domain wherein this config exists to be used. Can be used to receive reports for domains that aren't fully hosted on this server. Configure such a domain as a hosted domain without making all the DNS changes, and configure this field with a domain that is fully hosted on this server, so the localpart and the domain of this field form a reporting address. Then only update the DMARC DNS record for the not fully hosted domain, ensuring the reporting address is specified in its \"rua\" field as shown in the suggested DNS settings. Unicode name."`
 	Account   string `sconf-doc:"Account to deliver to."`
 	Mailbox   string `sconf-doc:"Mailbox to deliver to, e.g. DMARC."`
 
-	ParsedLocalpart smtp.Localpart `sconf:"-"` // Lower-case if case-sensitivity is not configured for domain. Not "canonical" for catchall separators for backwards compatibility.
+	ParsedLocalpart smtp.Localpart `sconf:"-"`
 	DNSDomain       dns.Domain     `sconf:"-"` // Effective domain, always set based on Domain field or Domain where this is configured.
 }
 
@@ -355,12 +335,12 @@ type MTASTS struct {
 }
 
 type TLSRPT struct {
-	Localpart string `sconf-doc:"Address-part before the @ that accepts TLSRPT reports. Recommended value: tlsreports."`
+	Localpart string `sconf-doc:"Address-part before the @ that accepts TLSRPT reports. Recommended value: tls-reports."`
 	Domain    string `sconf:"optional" sconf-doc:"Alternative domain for reporting address, for incoming reports. Typically empty, causing the domain wherein this config exists to be used. Can be used to receive reports for domains that aren't fully hosted on this server. Configure such a domain as a hosted domain without making all the DNS changes, and configure this field with a domain that is fully hosted on this server, so the localpart and the domain of this field form a reporting address. Then only update the TLSRPT DNS record for the not fully hosted domain, ensuring the reporting address is specified in its \"rua\" field as shown in the suggested DNS settings. Unicode name."`
 	Account   string `sconf-doc:"Account to deliver to."`
 	Mailbox   string `sconf-doc:"Mailbox to deliver to, e.g. TLSRPT."`
 
-	ParsedLocalpart smtp.Localpart `sconf:"-"` // Lower-case if case-sensitivity is not configured for domain. Not "canonical" for catchall separators for backwards compatibility.
+	ParsedLocalpart smtp.Localpart `sconf:"-"`
 	DNSDomain       dns.Domain     `sconf:"-"` // Effective domain, always set based on Domain field or Domain where this is configured.
 }
 
@@ -436,7 +416,6 @@ type Account struct {
 	KeepRetiredMessagePeriod time.Duration    `sconf:"optional" sconf-doc:"Period to keep messages retired from the queue (delivered or failed) around. Keeping retired messages is useful for maintaining the suppression list for transactional email, for matching incoming DSNs to sent messages, and for debugging. The time at which to clean up (remove) is calculated at retire time. E.g. 168h (1 week)."`
 	KeepRetiredWebhookPeriod time.Duration    `sconf:"optional" sconf-doc:"Period to keep webhooks retired from the queue (delivered or failed) around. Useful for debugging. The time at which to clean up (remove) is calculated at retire time. E.g. 168h (1 week)."`
 
-	LoginDisabled                string                 `sconf:"optional" sconf-doc:"If non-empty, login attempts on all protocols (e.g. SMTP/IMAP, web interfaces) is rejected with this error message. Useful during migrations. Incoming deliveries for addresses of this account are still accepted as normal."`
 	Domain                       string                 `sconf-doc:"Default domain for account. Deprecated behaviour: If a destination is not a full address but only a localpart, this domain is added to form a full address."`
 	Description                  string                 `sconf:"optional" sconf-doc:"Free form description, e.g. full name or alternative contact info."`
 	FullName                     string                 `sconf:"optional" sconf-doc:"Full name, to use in message From header when composing messages in webmail. Can be overridden per destination."`
@@ -450,7 +429,6 @@ type Account struct {
 	MaxOutgoingMessagesPerDay    int                    `sconf:"optional" sconf-doc:"Maximum number of outgoing messages for this account in a 24 hour window. This limits the damage to recipients and the reputation of this mail server in case of account compromise. Default 1000."`
 	MaxFirstTimeRecipientsPerDay int                    `sconf:"optional" sconf-doc:"Maximum number of first-time recipients in outgoing messages for this account in a 24 hour window. This limits the damage to recipients and the reputation of this mail server in case of account compromise. Default 200."`
 	NoFirstTimeSenderDelay       bool                   `sconf:"optional" sconf-doc:"Do not apply a delay to SMTP connections before accepting an incoming message from a first-time sender. Can be useful for accounts that sends automated responses and want instant replies."`
-	NoCustomPassword             bool                   `sconf:"optional" sconf-doc:"If set, this account cannot set a password of their own choice, but can only set a new randomly generated password, preventing password reuse across services and use of weak passwords. Custom account passwords can be set by the admin."`
 	Routes                       []Route                `sconf:"optional" sconf-doc:"Routes for delivering outgoing messages through the queue. Each delivery attempt evaluates these account routes, domain routes and finally global routes. The transport of the first matching route is used in the delivery attempt. If no routes match, which is the default with no configured routes, messages are delivered directly from the queue."`
 
 	DNSDomain                  dns.Domain     `sconf:"-"` // Parsed form of Domain.
@@ -473,19 +451,13 @@ type JunkFilter struct {
 }
 
 type Destination struct {
-	Mailbox                      string    `sconf:"optional" sconf-doc:"Mailbox to deliver to if none of Rulesets match. Default: Inbox."`
+	Mailbox  string    `sconf:"optional" sconf-doc:"Mailbox to deliver to if none of Rulesets match. Default: Inbox."`
-	Rulesets                     []Ruleset `sconf:"optional" sconf-doc:"Delivery rules based on message and SMTP transaction. You may want to match each mailing list by SMTP MailFrom address, VerifiedDomain and/or List-ID header (typically <listname.example.org> if the list address is listname@example.org), delivering them to their own mailbox."`
+	Rulesets []Ruleset `sconf:"optional" sconf-doc:"Delivery rules based on message and SMTP transaction. You may want to match each mailing list by SMTP MailFrom address, VerifiedDomain and/or List-ID header (typically <listname.example.org> if the list address is listname@example.org), delivering them to their own mailbox."`
-	SMTPError                    string    `sconf:"optional" sconf-doc:"If non-empty, incoming delivery attempts to this destination will be rejected during SMTP RCPT TO with this error response line. Useful when a catchall address is configured for the domain and messages to some addresses should be rejected. The response line must start with an error code. Currently the following error resonse codes are allowed: 421 (temporary local error), 550 (user not found). If the line consists of only an error code, an appropriate error message is added. Rejecting messages with a 4xx code invites later retries by the remote, while 5xx codes should prevent further delivery attempts."`
+	FullName string    `sconf:"optional" sconf-doc:"Full name to use in message From header when composing messages coming from this address with webmail."`
-	MessageAuthRequiredSMTPError string    `sconf:"optional" sconf-doc:"If non-empty, an additional DMARC-like message authentication check is done for incoming messages, validating the domain in the From-header of the message. Messages without either an aligned SPF or aligned DKIM pass are rejected during the SMTP DATA command with a permanent error code followed by the message in this field. The domain in the message 'From' header is matched in relaxed or strict mode according to the domain's DMARC policy if present, or relaxed mode (organizational instead of exact domain match) otherwise. Useful for autoresponders that don't want to accept messages they don't want to send an automated reply to."`
-	FullName                     string    `sconf:"optional" sconf-doc:"Full name to use in message From header when composing messages coming from this address with webmail."`
 
 	DMARCReports     bool `sconf:"-" json:"-"`
 	HostTLSReports   bool `sconf:"-" json:"-"`
 	DomainTLSReports bool `sconf:"-" json:"-"`
-	// Ready to use in SMTP responses.
-	SMTPErrorCode   int    `sconf:"-" json:"-"`
-	SMTPErrorSecode string `sconf:"-" json:"-"`
-	SMTPErrorMsg    string `sconf:"-" json:"-"`
 }
 
 // Equal returns whether d and o are equal, only looking at their user-changeable fields.
@@ -544,7 +516,6 @@ type TLS struct {
 	KeyCerts            []KeyCert `sconf:"optional" sconf-doc:"Keys and certificates to use for this listener. The files are opened by the privileged root process and passed to the unprivileged mox process, so no special permissions are required on the files. If the private key will not be replaced when refreshing certificates, also consider adding the private key to HostPrivateKeyFiles and configuring DANE TLSA DNS records."`
 	MinVersion          string    `sconf:"optional" sconf-doc:"Minimum TLS version. Default: TLSv1.2."`
 	HostPrivateKeyFiles []string  `sconf:"optional" sconf-doc:"Private keys used for ACME certificates. Specified explicitly so DANE TLSA DNS records can be generated, even before the certificates are requested. DANE is a mechanism to authenticate remote TLS certificates based on a public key or certificate specified in DNS, protected with DNSSEC. DANE is opportunistic and attempted when delivering SMTP with STARTTLS. The private key files must be in PEM format. PKCS8 is recommended, but PKCS1 and EC private keys are recognized as well. Only RSA 2048 bit and ECDSA P-256 keys are currently used. The first of each is used when requesting new certificates through ACME."`
-	ClientAuthDisabled  bool      `sconf:"optional" sconf-doc:"Disable TLS client authentication with certificates/keys, preventing the TLS server from requesting a TLS certificate from clients. Useful for working around clients that don't handle TLS client authentication well."`
 
 	Config                   *tls.Config     `sconf:"-" json:"-"` // TLS config for non-ACME-verification connections, i.e. SMTP and IMAP, and not port 443. Connections without SNI will use a certificate for the hostname of the listener, connections with an SNI hostname that isn't allowed will be rejected.
 	ConfigFallback           *tls.Config     `sconf:"-" json:"-"` // Like Config, but uses the certificate for the listener hostname when the requested SNI hostname is not allowed, instead of causing the connection to fail.

config/doc.go

@@ -217,11 +217,6 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 				HostPrivateKeyFiles:
 					-
 
-				# Disable TLS client authentication with certificates/keys, preventing the TLS
-				# server from requesting a TLS certificate from clients. Useful for working around
-				# clients that don't handle TLS client authentication well. (optional)
-				ClientAuthDisabled: false
-
 			# Maximum size in bytes for incoming and outgoing messages. Default is 100MB.
 			# (optional)
 			SMTPMaxMessageSize: 0
@@ -267,10 +262,6 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 				# account. Default: 15s. (optional)
 				FirstTimeSenderDelay: 0s
 
-				# Override default setting for enabling TLS session tickets. Disabling session
-				# tickets may work around TLS interoperability issues. (optional)
-				TLSSessionTicketsDisabled: false
-
 			# SMTP for submitting email, e.g. by email applications. Starts out in plain text,
 			# can be upgraded to TLS with the STARTTLS command. Prefer using Submissions which
 			# is always a TLS connection. (optional)
@@ -292,14 +283,6 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 				# Default 465. (optional)
 				Port: 0
 
-				# Additionally enable submission on HTTPS port 443 via TLS ALPN. TLS Application
-				# Layer Protocol Negotiation allows clients to request a specific protocol from
-				# the server as part of the TLS connection setup. When this setting is enabled and
-				# a client requests the 'smtp' protocol after TLS, it will be able to talk SMTP to
-				# Mox on port 443. This is meant to be useful as a censorship circumvention
-				# technique for Delta Chat. (optional)
-				EnabledOnHTTPS: false
-
 			# IMAP for reading email, by email applications. Starts out in plain text, can be
 			# upgraded to TLS with the STARTTLS command. Prefer using IMAPS instead which is
 			# always a TLS connection. (optional)
@@ -321,14 +304,6 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 				# Default 993. (optional)
 				Port: 0
 
-				# Additionally enable IMAP on HTTPS port 443 via TLS ALPN. TLS Application Layer
-				# Protocol Negotiation allows clients to request a specific protocol from the
-				# server as part of the TLS connection setup. When this setting is enabled and a
-				# client requests the 'imap' protocol after TLS, it will be able to talk IMAP to
-				# Mox on port 443. This is meant to be useful as a censorship circumvention
-				# technique for Delta Chat. (optional)
-				EnabledOnHTTPS: false
-
 			# Account web interface, for email users wanting to change their accounts, e.g.
 			# set new password, set new delivery rulesets. Default path is /. (optional)
 			AccountHTTP:
@@ -338,8 +313,7 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 				# matching behaviour. (optional)
 				Port: 0
 
-				# Path to serve requests on. Should end with a slash, related to cookie paths.
+				# Path to serve requests on. (optional)
-				# (optional)
 				Path:
 
 				# If set, X-Forwarded-* headers are used for the remote IP address for rate
@@ -355,8 +329,7 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 				# matching behaviour. (optional)
 				Port: 0
 
-				# Path to serve requests on. Should end with a slash, related to cookie paths.
+				# Path to serve requests on. (optional)
-				# (optional)
 				Path:
 
 				# If set, X-Forwarded-* headers are used for the remote IP address for rate
@@ -375,8 +348,7 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 				# matching behaviour. (optional)
 				Port: 0
 
-				# Path to serve requests on. Should end with a slash, related to cookie paths.
+				# Path to serve requests on. (optional)
-				# (optional)
 				Path:
 
 				# If set, X-Forwarded-* headers are used for the remote IP address for rate
@@ -392,8 +364,7 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 				# matching behaviour. (optional)
 				Port: 0
 
-				# Path to serve requests on. Should end with a slash, related to cookie paths.
+				# Path to serve requests on. (optional)
-				# (optional)
 				Path:
 
 				# If set, X-Forwarded-* headers are used for the remote IP address for rate
@@ -408,8 +379,7 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 				# matching behaviour. (optional)
 				Port: 0
 
-				# Path to serve requests on. Should end with a slash, related to cookie paths.
+				# Path to serve requests on. (optional)
-				# (optional)
 				Path:
 
 				# If set, X-Forwarded-* headers are used for the remote IP address for rate
@@ -425,8 +395,7 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 				# matching behaviour. (optional)
 				Port: 0
 
-				# Path to serve requests on. Should end with a slash, related to cookie paths.
+				# Path to serve requests on. (optional)
-				# (optional)
 				Path:
 
 				# If set, X-Forwarded-* headers are used for the remote IP address for rate
@@ -441,8 +410,7 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 				# matching behaviour. (optional)
 				Port: 0
 
-				# Path to serve requests on. Should end with a slash, related to cookie paths.
+				# Path to serve requests on. (optional)
-				# (optional)
 				Path:
 
 				# If set, X-Forwarded-* headers are used for the remote IP address for rate
@@ -458,8 +426,7 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 				# matching behaviour. (optional)
 				Port: 0
 
-				# Path to serve requests on. Should end with a slash, related to cookie paths.
+				# Path to serve requests on. (optional)
-				# (optional)
 				Path:
 
 				# If set, X-Forwarded-* headers are used for the remote IP address for rate
@@ -519,9 +486,6 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 				# Port for plain HTTP (non-TLS) webserver. (optional)
 				Port: 0
 
-				# Disable rate limiting for all requests to this port. (optional)
-				RateLimitDisabled: false
-
 			# All configured WebHandlers will serve on an enabled listener. Either ACME must
 			# be configured, or for each WebHandler domain a TLS certificate must be
 			# configured. (optional)
@@ -531,9 +495,6 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 				# Port for HTTPS webserver. (optional)
 				Port: 0
 
-				# Disable rate limiting for all requests to this port. (optional)
-				RateLimitDisabled: false
-
 	# Destination for emails delivered to postmaster addresses: a plain 'postmaster'
 	# without domain, 'postmaster@<hostname>' (also for each listener with SMTP
 	# enabled), and as fallback for each domain without explicitly configured
@@ -557,13 +518,13 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 		# Mailbox to deliver TLS reports to. Recommended value: TLSRPT.
 		Mailbox:
 
-		# Localpart at hostname to accept TLS reports at. Recommended value: tlsreports.
+		# Localpart at hostname to accept TLS reports at. Recommended value: tls-reports.
 		Localpart:
 
 	# Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be
 	# given a 'special-use' role, which are understood by most mail clients. If
-	# absent/empty, the following additional mailboxes are created: Sent, Archive,
+	# absent/empty, the following mailboxes are created: Sent, Archive, Trash, Drafts
-	# Trash, Drafts and Junk. (optional)
+	# and Junk. (optional)
 	InitialMailboxes:
 
 		# Special-use roles to mailbox to create. (optional)
@@ -736,16 +697,6 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 				# remote SMTP servers. (optional)
 				DisableIPv6: false
 
-			# Immediately fails the delivery attempt. (optional)
-			Fail:
-
-				# SMTP error code and optional enhanced error code to use for the failure. If
-				# empty, 554 is used (transaction failed). (optional)
-				SMTPCode: 0
-
-				# Message to include for the rejection. It will be shown in the DSN. (optional)
-				SMTPMessage:
-
 	# Do not send DMARC reports (aggregate only). By default, aggregate reports on
 	# DMARC evaluations are sent to domains if their DMARC policy requests them.
 	# Reports are sent at whole hours, with a minimum of 1 hour and maximum of 24
@@ -789,19 +740,6 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 	Domains:
 		x:
 
-			# Disabled domains can be useful during/before migrations. Domains that are
-			# disabled can still be configured like normal, including adding addresses using
-			# the domain to accounts. However, disabled domains: 1. Do not try to fetch ACME
-			# certificates. TLS connections to host names involving the email domain will
-			# fail. A TLS certificate for the hostname (that wil be used as MX) itself will be
-			# requested. 2. Incoming deliveries over SMTP are rejected with a temporary error
-			# '450 4.2.1 recipient domain temporarily disabled'. 3. Submissions over SMTP
-			# using an (envelope) SMTP MAIL FROM address or message 'From' address of a
-			# disabled domain will be rejected with a temporary error '451 4.3.0 sender domain
-			# temporarily disabled'. Note that accounts with addresses at disabled domains can
-			# still log in and read email (unless the account itself is disabled). (optional)
-			Disabled: false
-
 			# Free-form description of domain. (optional)
 			Description:
 
@@ -818,14 +756,6 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 			# delivered to you@example.com. (optional)
 			LocalpartCatchallSeparator:
 
-			# Similar to LocalpartCatchallSeparator, but in case multiple are needed. For
-			# example both "+" and "-". Only of one LocalpartCatchallSeparator or
-			# LocalpartCatchallSeparators can be set. If set, the first separator is used to
-			# make unique addresses for outgoing SMTP connections with FromIDLoginAddresses.
-			# (optional)
-			LocalpartCatchallSeparators:
-				-
-
 			# If set, upper/lower case is relevant for email delivery. (optional)
 			LocalpartCaseSensitive: false
 
@@ -885,7 +815,7 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 			DMARC:
 
 				# Address-part before the @ that accepts DMARC reports. Must be
-				# non-internationalized. Recommended value: dmarcreports.
+				# non-internationalized. Recommended value: dmarc-reports.
 				Localpart:
 
 				# Alternative domain for reporting address, for incoming reports. Typically empty,
@@ -953,7 +883,7 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 			TLSRPT:
 
 				# Address-part before the @ that accepts TLSRPT reports. Recommended value:
-				# tlsreports.
+				# tls-reports.
 				Localpart:
 
 				# Alternative domain for reporting address, for incoming reports. Typically empty,
@@ -1078,12 +1008,6 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 			# retire time. E.g. 168h (1 week). (optional)
 			KeepRetiredWebhookPeriod: 0s
 
-			# If non-empty, login attempts on all protocols (e.g. SMTP/IMAP, web interfaces)
-			# is rejected with this error message. Useful during migrations. Incoming
-			# deliveries for addresses of this account are still accepted as normal.
-			# (optional)
-			LoginDisabled:
-
 			# Default domain for account. Deprecated behaviour: If a destination is not a full
 			# address but only a localpart, this domain is added to form a full address.
 			Domain:
@@ -1176,28 +1100,6 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 							# Free-form comments. (optional)
 							Comment:
 
-					# If non-empty, incoming delivery attempts to this destination will be rejected
-					# during SMTP RCPT TO with this error response line. Useful when a catchall
-					# address is configured for the domain and messages to some addresses should be
-					# rejected. The response line must start with an error code. Currently the
-					# following error resonse codes are allowed: 421 (temporary local error), 550
-					# (user not found). If the line consists of only an error code, an appropriate
-					# error message is added. Rejecting messages with a 4xx code invites later retries
-					# by the remote, while 5xx codes should prevent further delivery attempts.
-					# (optional)
-					SMTPError:
-
-					# If non-empty, an additional DMARC-like message authentication check is done for
-					# incoming messages, validating the domain in the From-header of the message.
-					# Messages without either an aligned SPF or aligned DKIM pass are rejected during
-					# the SMTP DATA command with a permanent error code followed by the message in
-					# this field. The domain in the message 'From' header is matched in relaxed or
-					# strict mode according to the domain's DMARC policy if present, or relaxed mode
-					# (organizational instead of exact domain match) otherwise. Useful for
-					# autoresponders that don't want to accept messages they don't want to send an
-					# automated reply to. (optional)
-					MessageAuthRequiredSMTPError:
-
 					# Full name to use in message From header when composing messages coming from this
 					# address with webmail. (optional)
 					FullName:
@@ -1308,12 +1210,6 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 			# responses and want instant replies. (optional)
 			NoFirstTimeSenderDelay: false
 
-			# If set, this account cannot set a password of their own choice, but can only set
-			# a new randomly generated password, preventing password reuse across services and
-			# use of weak passwords. Custom account passwords can be set by the admin.
-			# (optional)
-			NoCustomPassword: false
-
 			# Routes for delivering outgoing messages through the queue. Each delivery attempt
 			# evaluates these account routes, domain routes and finally global routes. The
 			# transport of the first matching route is used in the delivery attempt. If no

ctl_test.go

@@ -4,13 +4,8 @@ package main
 
 import (
 	"context"
-	"crypto/ed25519"
-	cryptorand "crypto/rand"
-	"crypto/x509"
 	"flag"
 	"fmt"
-	"log/slog"
-	"math/big"
 	"net"
 	"os"
 	"path/filepath"
@@ -20,7 +15,6 @@ import (
 	"github.com/mjl-/mox/config"
 	"github.com/mjl-/mox/dmarcdb"
 	"github.com/mjl-/mox/dns"
-	"github.com/mjl-/mox/imapclient"
 	"github.com/mjl-/mox/mlog"
 	"github.com/mjl-/mox/mox-"
 	"github.com/mjl-/mox/mtastsdb"
@@ -45,70 +39,64 @@ func tcheck(t *testing.T, err error, errmsg string) {
 // unhandled errors would cause a panic.
 func TestCtl(t *testing.T) {
 	os.RemoveAll("testdata/ctl/data")
-	mox.ConfigStaticPath = filepath.FromSlash("testdata/ctl/config/mox.conf")
+	mox.ConfigStaticPath = filepath.FromSlash("testdata/ctl/mox.conf")
-	mox.ConfigDynamicPath = filepath.FromSlash("testdata/ctl/config/domains.conf")
+	mox.ConfigDynamicPath = filepath.FromSlash("testdata/ctl/domains.conf")
 	if errs := mox.LoadConfig(ctxbg, pkglog, true, false); len(errs) > 0 {
 		t.Fatalf("loading mox config: %v", errs)
 	}
-	err := store.Init(ctxbg)
-	tcheck(t, err, "store init")
-	defer store.Close()
 	defer store.Switchboard()()
 
-	err = queue.Init()
+	err := queue.Init()
 	tcheck(t, err, "queue init")
 	defer queue.Shutdown()
 
-	var cid int64
+	testctl := func(fn func(clientctl *ctl)) {
-
-	testctl := func(fn func(clientxctl *ctl)) {
 		t.Helper()
 
 		cconn, sconn := net.Pipe()
-		clientxctl := ctl{conn: cconn, log: pkglog}
+		clientctl := ctl{conn: cconn, log: pkglog}
-		serverxctl := ctl{conn: sconn, log: pkglog}
+		serverctl := ctl{conn: sconn, log: pkglog}
 		done := make(chan struct{})
 		go func() {
-			cid++
+			servectlcmd(ctxbg, &serverctl, func() {})
-			servectlcmd(ctxbg, &serverxctl, cid, func() {})
 			close(done)
 		}()
-		fn(&clientxctl)
+		fn(&clientctl)
 		cconn.Close()
 		<-done
 		sconn.Close()
 	}
 
 	// "deliver"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdDeliver(xctl, "mjl@mox.example")
+		ctlcmdDeliver(ctl, "mjl@mox.example")
 	})
 
 	// "setaccountpassword"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdSetaccountpassword(xctl, "mjl", "test4321")
+		ctlcmdSetaccountpassword(ctl, "mjl", "test4321")
 	})
 
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHoldrulesList(xctl)
+		ctlcmdQueueHoldrulesList(ctl)
 	})
 
 	// All messages.
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHoldrulesAdd(xctl, "", "", "")
+		ctlcmdQueueHoldrulesAdd(ctl, "", "", "")
 	})
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHoldrulesAdd(xctl, "mjl", "", "")
+		ctlcmdQueueHoldrulesAdd(ctl, "mjl", "", "")
 	})
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHoldrulesAdd(xctl, "", "☺.mox.example", "")
+		ctlcmdQueueHoldrulesAdd(ctl, "", "☺.mox.example", "")
 	})
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHoldrulesAdd(xctl, "mox", "☺.mox.example", "example.com")
+		ctlcmdQueueHoldrulesAdd(ctl, "mox", "☺.mox.example", "example.com")
 	})
 
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHoldrulesRemove(xctl, 1)
+		ctlcmdQueueHoldrulesRemove(ctl, 1)
 	})
 
 	// Queue a message to list/change/dump.
@@ -128,321 +116,262 @@ func TestCtl(t *testing.T) {
 	qmid := qml[0].ID
 
 	// Has entries now.
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHoldrulesList(xctl)
+		ctlcmdQueueHoldrulesList(ctl)
 	})
 
 	// "queuelist"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueList(xctl, queue.Filter{}, queue.Sort{})
+		ctlcmdQueueList(ctl, queue.Filter{}, queue.Sort{})
 	})
 
 	// "queueholdset"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHoldSet(xctl, queue.Filter{}, true)
+		ctlcmdQueueHoldSet(ctl, queue.Filter{}, true)
 	})
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHoldSet(xctl, queue.Filter{}, false)
+		ctlcmdQueueHoldSet(ctl, queue.Filter{}, false)
 	})
 
 	// "queueschedule"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueSchedule(xctl, queue.Filter{}, true, time.Minute)
+		ctlcmdQueueSchedule(ctl, queue.Filter{}, true, time.Minute)
 	})
 
 	// "queuetransport"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueTransport(xctl, queue.Filter{}, "socks")
+		ctlcmdQueueTransport(ctl, queue.Filter{}, "socks")
 	})
 
 	// "queuerequiretls"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueRequireTLS(xctl, queue.Filter{}, nil)
+		ctlcmdQueueRequireTLS(ctl, queue.Filter{}, nil)
 	})
 
 	// "queuedump"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueDump(xctl, fmt.Sprintf("%d", qmid))
+		ctlcmdQueueDump(ctl, fmt.Sprintf("%d", qmid))
 	})
 
 	// "queuefail"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueFail(xctl, queue.Filter{})
+		ctlcmdQueueFail(ctl, queue.Filter{})
 	})
 
 	// "queuedrop"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueDrop(xctl, queue.Filter{})
+		ctlcmdQueueDrop(ctl, queue.Filter{})
 	})
 
 	// "queueholdruleslist"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHoldrulesList(xctl)
+		ctlcmdQueueHoldrulesList(ctl)
 	})
 
 	// "queueholdrulesadd"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHoldrulesAdd(xctl, "mjl", "", "")
+		ctlcmdQueueHoldrulesAdd(ctl, "mjl", "", "")
 	})
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHoldrulesAdd(xctl, "mjl", "localhost", "")
+		ctlcmdQueueHoldrulesAdd(ctl, "mjl", "localhost", "")
 	})
 
 	// "queueholdrulesremove"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHoldrulesRemove(xctl, 2)
+		ctlcmdQueueHoldrulesRemove(ctl, 2)
 	})
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHoldrulesList(xctl)
+		ctlcmdQueueHoldrulesList(ctl)
 	})
 
 	// "queuesuppresslist"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueSuppressList(xctl, "mjl")
+		ctlcmdQueueSuppressList(ctl, "mjl")
 	})
 
 	// "queuesuppressadd"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueSuppressAdd(xctl, "mjl", "base@localhost")
+		ctlcmdQueueSuppressAdd(ctl, "mjl", "base@localhost")
 	})
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueSuppressAdd(xctl, "mjl", "other@localhost")
+		ctlcmdQueueSuppressAdd(ctl, "mjl", "other@localhost")
 	})
 
 	// "queuesuppresslookup"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueSuppressLookup(xctl, "mjl", "base@localhost")
+		ctlcmdQueueSuppressLookup(ctl, "mjl", "base@localhost")
 	})
 
 	// "queuesuppressremove"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueSuppressRemove(xctl, "mjl", "base@localhost")
+		ctlcmdQueueSuppressRemove(ctl, "mjl", "base@localhost")
 	})
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueSuppressList(xctl, "mjl")
+		ctlcmdQueueSuppressList(ctl, "mjl")
 	})
 
 	// "queueretiredlist"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueRetiredList(xctl, queue.RetiredFilter{}, queue.RetiredSort{})
+		ctlcmdQueueRetiredList(ctl, queue.RetiredFilter{}, queue.RetiredSort{})
 	})
 
 	// "queueretiredprint"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueRetiredPrint(xctl, "1")
+		ctlcmdQueueRetiredPrint(ctl, "1")
 	})
 
 	// "queuehooklist"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHookList(xctl, queue.HookFilter{}, queue.HookSort{})
+		ctlcmdQueueHookList(ctl, queue.HookFilter{}, queue.HookSort{})
 	})
 
 	// "queuehookschedule"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHookSchedule(xctl, queue.HookFilter{}, true, time.Minute)
+		ctlcmdQueueHookSchedule(ctl, queue.HookFilter{}, true, time.Minute)
 	})
 
 	// "queuehookprint"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHookPrint(xctl, "1")
+		ctlcmdQueueHookPrint(ctl, "1")
 	})
 
 	// "queuehookcancel"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHookCancel(xctl, queue.HookFilter{})
+		ctlcmdQueueHookCancel(ctl, queue.HookFilter{})
 	})
 
 	// "queuehookretiredlist"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHookRetiredList(xctl, queue.HookRetiredFilter{}, queue.HookRetiredSort{})
+		ctlcmdQueueHookRetiredList(ctl, queue.HookRetiredFilter{}, queue.HookRetiredSort{})
 	})
 
 	// "queuehookretiredprint"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdQueueHookRetiredPrint(xctl, "1")
+		ctlcmdQueueHookRetiredPrint(ctl, "1")
 	})
 
 	// "importmbox"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdImport(xctl, true, "mjl", "inbox", "testdata/importtest.mbox")
+		ctlcmdImport(ctl, true, "mjl", "inbox", "testdata/importtest.mbox")
 	})
 
 	// "importmaildir"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdImport(xctl, false, "mjl", "inbox", "testdata/importtest.maildir")
+		ctlcmdImport(ctl, false, "mjl", "inbox", "testdata/importtest.maildir")
 	})
 
 	// "domainadd"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdConfigDomainAdd(xctl, false, dns.Domain{ASCII: "mox2.example"}, "mjl", "")
+		ctlcmdConfigDomainAdd(ctl, dns.Domain{ASCII: "mox2.example"}, "mjl", "")
 	})
 
 	// "accountadd"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdConfigAccountAdd(xctl, "mjl2", "mjl2@mox2.example")
+		ctlcmdConfigAccountAdd(ctl, "mjl2", "mjl2@mox2.example")
 	})
 
 	// "addressadd"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdConfigAddressAdd(xctl, "mjl3@mox2.example", "mjl2")
+		ctlcmdConfigAddressAdd(ctl, "mjl3@mox2.example", "mjl2")
 	})
 
 	// Add a message.
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdDeliver(xctl, "mjl3@mox2.example")
+		ctlcmdDeliver(ctl, "mjl3@mox2.example")
 	})
 	// "retrain", retrain junk filter.
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdRetrain(xctl, "mjl2")
+		ctlcmdRetrain(ctl, "mjl2")
 	})
 
 	// "addressrm"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdConfigAddressRemove(xctl, "mjl3@mox2.example")
+		ctlcmdConfigAddressRemove(ctl, "mjl3@mox2.example")
-	})
-
-	// "accountdisabled"
-	testctl(func(xctl *ctl) {
-		ctlcmdConfigAccountDisabled(xctl, "mjl2", "testing")
-	})
-
-	// "accountlist"
-	testctl(func(xctl *ctl) {
-		ctlcmdConfigAccountList(xctl)
-	})
-
-	testctl(func(xctl *ctl) {
-		ctlcmdConfigAccountDisabled(xctl, "mjl2", "")
 	})
 
 	// "accountrm"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdConfigAccountRemove(xctl, "mjl2")
+		ctlcmdConfigAccountRemove(ctl, "mjl2")
-	})
-
-	// "domaindisabled"
-	testctl(func(xctl *ctl) {
-		ctlcmdConfigDomainDisabled(xctl, dns.Domain{ASCII: "mox2.example"}, true)
-	})
-	testctl(func(xctl *ctl) {
-		ctlcmdConfigDomainDisabled(xctl, dns.Domain{ASCII: "mox2.example"}, false)
 	})
 
 	// "domainrm"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdConfigDomainRemove(xctl, dns.Domain{ASCII: "mox2.example"})
+		ctlcmdConfigDomainRemove(ctl, dns.Domain{ASCII: "mox2.example"})
 	})
 
 	// "aliasadd"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdConfigAliasAdd(xctl, "support@mox.example", config.Alias{Addresses: []string{"mjl@mox.example"}})
+		ctlcmdConfigAliasAdd(ctl, "support@mox.example", config.Alias{Addresses: []string{"mjl@mox.example"}})
 	})
 
 	// "aliaslist"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdConfigAliasList(xctl, "mox.example")
+		ctlcmdConfigAliasList(ctl, "mox.example")
 	})
 
 	// "aliasprint"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdConfigAliasPrint(xctl, "support@mox.example")
+		ctlcmdConfigAliasPrint(ctl, "support@mox.example")
 	})
 
 	// "aliasupdate"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdConfigAliasUpdate(xctl, "support@mox.example", "true", "true", "true")
+		ctlcmdConfigAliasUpdate(ctl, "support@mox.example", "true", "true", "true")
 	})
 
 	// "aliasaddaddr"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdConfigAliasAddaddr(xctl, "support@mox.example", []string{"mjl2@mox.example"})
+		ctlcmdConfigAliasAddaddr(ctl, "support@mox.example", []string{"mjl2@mox.example"})
 	})
 
 	// "aliasrmaddr"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdConfigAliasRmaddr(xctl, "support@mox.example", []string{"mjl2@mox.example"})
+		ctlcmdConfigAliasRmaddr(ctl, "support@mox.example", []string{"mjl2@mox.example"})
 	})
 
 	// "aliasrm"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdConfigAliasRemove(xctl, "support@mox.example")
+		ctlcmdConfigAliasRemove(ctl, "support@mox.example")
 	})
 
-	// accounttlspubkeyadd
-	certDER := fakeCert(t)
-	testctl(func(xctl *ctl) {
-		ctlcmdConfigTlspubkeyAdd(xctl, "mjl@mox.example", "testkey", false, certDER)
-	})
-
-	// "accounttlspubkeylist"
-	testctl(func(xctl *ctl) {
-		ctlcmdConfigTlspubkeyList(xctl, "")
-	})
-	testctl(func(xctl *ctl) {
-		ctlcmdConfigTlspubkeyList(xctl, "mjl")
-	})
-
-	tpkl, err := store.TLSPublicKeyList(ctxbg, "")
-	tcheck(t, err, "list tls public keys")
-	if len(tpkl) != 1 {
-		t.Fatalf("got %d tls public keys, expected 1", len(tpkl))
-	}
-	fingerprint := tpkl[0].Fingerprint
-
-	// "accounttlspubkeyget"
-	testctl(func(xctl *ctl) {
-		ctlcmdConfigTlspubkeyGet(xctl, fingerprint)
-	})
-
-	// "accounttlspubkeyrm"
-	testctl(func(xctl *ctl) {
-		ctlcmdConfigTlspubkeyRemove(xctl, fingerprint)
-	})
-
-	tpkl, err = store.TLSPublicKeyList(ctxbg, "")
-	tcheck(t, err, "list tls public keys")
-	if len(tpkl) != 0 {
-		t.Fatalf("got %d tls public keys, expected 0", len(tpkl))
-	}
-
 	// "loglevels"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdLoglevels(xctl)
+		ctlcmdLoglevels(ctl)
 	})
 
 	// "setloglevels"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdSetLoglevels(xctl, "", "debug")
+		ctlcmdSetLoglevels(ctl, "", "debug")
 	})
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdSetLoglevels(xctl, "smtpserver", "debug")
+		ctlcmdSetLoglevels(ctl, "smtpserver", "debug")
 	})
 
 	// Export data, import it again
 	xcmdExport(true, false, []string{filepath.FromSlash("testdata/ctl/data/tmp/export/mbox/"), filepath.FromSlash("testdata/ctl/data/accounts/mjl")}, &cmd{log: pkglog})
 	xcmdExport(false, false, []string{filepath.FromSlash("testdata/ctl/data/tmp/export/maildir/"), filepath.FromSlash("testdata/ctl/data/accounts/mjl")}, &cmd{log: pkglog})
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdImport(xctl, true, "mjl", "inbox", filepath.FromSlash("testdata/ctl/data/tmp/export/mbox/Inbox.mbox"))
+		ctlcmdImport(ctl, true, "mjl", "inbox", filepath.FromSlash("testdata/ctl/data/tmp/export/mbox/Inbox.mbox"))
 	})
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdImport(xctl, false, "mjl", "inbox", filepath.FromSlash("testdata/ctl/data/tmp/export/maildir/Inbox"))
+		ctlcmdImport(ctl, false, "mjl", "inbox", filepath.FromSlash("testdata/ctl/data/tmp/export/maildir/Inbox"))
 	})
 
 	// "recalculatemailboxcounts"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdRecalculateMailboxCounts(xctl, "mjl")
+		ctlcmdRecalculateMailboxCounts(ctl, "mjl")
 	})
 
 	// "fixmsgsize"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdFixmsgsize(xctl, "mjl")
+		ctlcmdFixmsgsize(ctl, "mjl")
 	})
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		acc, err := store.OpenAccount(xctl.log, "mjl", false)
+		acc, err := store.OpenAccount(ctl.log, "mjl")
 		tcheck(t, err, "open account")
 		defer func() {
 			acc.Close()
-			acc.WaitClosed()
+			acc.CheckClosed()
 		}()
 
 		content := []byte("Subject: hi\r\n\r\nbody\r\n")
@@ -450,17 +379,14 @@ func TestCtl(t *testing.T) {
 		deliver := func(m *store.Message) {
 			t.Helper()
 			m.Size = int64(len(content))
-			msgf, err := store.CreateMessageTemp(xctl.log, "ctltest")
+			msgf, err := store.CreateMessageTemp(ctl.log, "ctltest")
 			tcheck(t, err, "create temp file")
 			defer os.Remove(msgf.Name())
 			defer msgf.Close()
 			_, err = msgf.Write(content)
 			tcheck(t, err, "write message file")
-
+			err = acc.DeliverMailbox(ctl.log, "Inbox", m, msgf)
-			acc.WithWLock(func() {
+			tcheck(t, err, "deliver message")
-				err = acc.DeliverMailbox(xctl.log, "Inbox", m, msgf)
-				tcheck(t, err, "deliver message")
-			})
 		}
 
 		var msgBadSize store.Message
@@ -478,7 +404,7 @@ func TestCtl(t *testing.T) {
 		tcheck(t, err, "update mailbox size")
 
 		// Fix up the size.
-		ctlcmdFixmsgsize(xctl, "")
+		ctlcmdFixmsgsize(ctl, "")
 
 		err = acc.DB.Get(ctxbg, &msgBadSize)
 		tcheck(t, err, "get message")
@@ -488,19 +414,19 @@ func TestCtl(t *testing.T) {
 	})
 
 	// "reparse"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdReparse(xctl, "mjl")
+		ctlcmdReparse(ctl, "mjl")
 	})
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdReparse(xctl, "")
+		ctlcmdReparse(ctl, "")
 	})
 
 	// "reassignthreads"
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdReassignthreads(xctl, "mjl")
+		ctlcmdReassignthreads(ctl, "mjl")
 	})
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		ctlcmdReassignthreads(xctl, "")
+		ctlcmdReassignthreads(ctl, "")
 	})
 
 	// "backup", backup account.
@@ -513,46 +439,17 @@ func TestCtl(t *testing.T) {
 	err = tlsrptdb.Init()
 	tcheck(t, err, "tlsrptdb init")
 	defer tlsrptdb.Close()
-	testctl(func(xctl *ctl) {
+	testctl(func(ctl *ctl) {
-		os.RemoveAll("testdata/ctl/data/tmp/backup")
+		os.RemoveAll("testdata/ctl/data/tmp/backup-data")
 		err := os.WriteFile("testdata/ctl/data/receivedid.key", make([]byte, 16), 0600)
 		tcheck(t, err, "writing receivedid.key")
-		ctlcmdBackup(xctl, filepath.FromSlash("testdata/ctl/data/tmp/backup"), false)
+		ctlcmdBackup(ctl, filepath.FromSlash("testdata/ctl/data/tmp/backup-data"), false)
 	})
 
 	// Verify the backup.
 	xcmd := cmd{
 		flag:     flag.NewFlagSet("", flag.ExitOnError),
-		flagArgs: []string{filepath.FromSlash("testdata/ctl/data/tmp/backup/data")},
+		flagArgs: []string{filepath.FromSlash("testdata/ctl/data/tmp/backup-data")},
 	}
 	cmdVerifydata(&xcmd)
-
-	// IMAP connection.
-	testctl(func(xctl *ctl) {
-		a, b := net.Pipe()
-		go func() {
-			opts := imapclient.Opts{
-				Logger: slog.Default().With("cid", mox.Cid()),
-				Error:  func(err error) { panic(err) },
-			}
-			client, err := imapclient.New(a, &opts)
-			tcheck(t, err, "new imapclient")
-			client.Select("inbox")
-			client.Logout()
-			defer a.Close()
-		}()
-		ctlcmdIMAPServe(xctl, "mjl@mox.example", b, b)
-	})
-}
-
-func fakeCert(t *testing.T) []byte {
-	t.Helper()
-	seed := make([]byte, ed25519.SeedSize)
-	privKey := ed25519.NewKeyFromSeed(seed) // Fake key, don't use this for real!
-	template := &x509.Certificate{
-		SerialNumber: big.NewInt(1), // Required field...
-	}
-	localCertBuf, err := x509.CreateCertificate(cryptorand.Reader, template, template, privKey.Public(), privKey)
-	tcheck(t, err, "making certificate")
-	return localCertBuf
 }

curves.go

@@ -1,14 +0,0 @@
-//go:build !go1.24
-
-package main
-
-import (
-	"crypto/tls"
-)
-
-var curvesList = []tls.CurveID{
-	tls.CurveP256,
-	tls.CurveP384,
-	tls.CurveP521,
-	tls.X25519,
-}

curves_go124.go

@@ -1,15 +0,0 @@
-//go:build go1.24
-
-package main
-
-import (
-	"crypto/tls"
-)
-
-var curvesList = []tls.CurveID{
-	tls.CurveP256,
-	tls.CurveP384,
-	tls.CurveP521,
-	tls.X25519,
-	tls.X25519MLKEM768,
-}

dane/dane.go

@@ -65,7 +65,6 @@ import (
 	"github.com/mjl-/mox/dns"
 	"github.com/mjl-/mox/mlog"
 	"github.com/mjl-/mox/stub"
-	"slices"
 )
 
 var (
@@ -215,9 +214,12 @@ func Dial(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, network
 	if allowedUsages != nil {
 		o := 0
 		for _, r := range records {
-			if slices.Contains(allowedUsages, r.Usage) {
+			for _, usage := range allowedUsages {
-				records[o] = r
+				if r.Usage == usage {
-				o++
+					records[o] = r
+					o++
+					break
+				}
 			}
 		}
 		records = records[:o]
@@ -261,8 +263,7 @@ func Dial(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, network
 	config := TLSClientConfig(log.Logger, records, baseDom, moreAllowedHosts, &verifiedRecord, pkixRoots)
 	tlsConn := tls.Client(conn, &config)
 	if err := tlsConn.HandshakeContext(ctx); err != nil {
-		xerr := conn.Close()
+		conn.Close()
-		log.Check(xerr, "closing connection")
 		return nil, adns.TLSA{}, err
 	}
 	return tlsConn, verifiedRecord, nil
@@ -447,8 +448,7 @@ func verifySingle(log mlog.Log, tlsa adns.TLSA, cs tls.ConnectionState, allowedH
 		// We set roots, so the system defaults don't get used. Verify checks the host name
 		// (set below) and checks for expiration.
 		opts := x509.VerifyOptions{
-			Intermediates: x509.NewCertPool(),
+			Roots: x509.NewCertPool(),
-			Roots:         x509.NewCertPool(),
 		}
 
 		// If the full certificate was included, we must add it to the valid roots, the TLS
@@ -465,13 +465,11 @@ func verifySingle(log mlog.Log, tlsa adns.TLSA, cs tls.ConnectionState, allowedH
 			}
 		}
 
-		for i, cert := range cs.PeerCertificates {
+		for _, cert := range cs.PeerCertificates {
 			if match(cert) {
 				opts.Roots.AddCert(cert)
 				found = true
 				break
-			} else if i > 0 {
-				opts.Intermediates.AddCert(cert)
 			}
 		}
 		if !found {

develop.txt

@@ -13,12 +13,11 @@ code paths are reachable/testable with mox localserve, but some use cases will
 require a full setup.
 
 Before committing, run at least "make fmt" and "make check" (which requires
-staticcheck and ineffassign, run "make install-staticcheck install-ineffassign"
+staticcheck, run "make install-staticcheck" once). Also run "make check-shadow"
-once). Also run "make check-shadow" and fix any shadowed variables other than
+and fix any shadowed variables other than "err" (which are filtered out, but
-"err" (which are filtered out, but causes the command to always exit with an
+causes the command to always exit with an error code; run "make install-shadow"
-error code; run "make install-shadow" once to install the shadow command). If
+once to install the shadow command). If you've updated RFC references, run
-you've updated RFC references, run "make" in rfc/, it verifies the referenced
+"make" in rfc/, it verifies the referenced files exist.
-files exist.
 
 When making changes to the public API of a package listed in
 apidiff/packages.txt, run "make genapidiff" to update the list of changes in
@@ -47,18 +46,6 @@ instructions below.
   standard slog package for logging, not our mlog package. Packages not intended
   for reuse do use mlog as it is more convenient. Internally, we always use
   mlog.Log to do the logging, wrapping an slog.Logger.
-- The code uses panic for error handling in quite a few places, including
-  smtpserver, imapserver and web API calls. Functions/methods, variables, struct
-  fields and types that begin with an "x" indicate they can panic on errors. Both
-  for i/o errors that are fatal for a connection, and also often for user-induced
-  errors, for example bad IMAP commands or invalid web API requests. These panics
-  are caught again at the top of a command or top of the connection. Write code
-  that is panic-safe, using defer to clean up and release resources.
-- Try to check all errors, at the minimum using mlog.Log.Check() to log an error
-  at the appropriate level. Also when just closing a file. Log messages sometimes
-  unexpectedly point out latent issues. Only when there is no point in logging,
-  for example when previous writes to stderr failed, can error logging be skipped.
-  Test code is less strict about checking errors.
 
 
 # Reusable packages
@@ -114,10 +101,10 @@ Large files (images/videos) are in https://github.com/mjl-/mox-website-files to
 keep the repository reasonably sized.
 
 The public website may serve the content from the "website" branch. After a
-release, the main branch (with latest development code and corresponding
+release release, the main branch (with latest development code and
-changes to the website about new features) is merged into the website branch.
+corresponding changes to the website about new features) is merged into the
-Commits to the website branch (e.g. for a news item, or any other change
+website branch.  Commits to the website branch (e.g. for a news item, or any
-unrelated to a new release) is merged back into the main branch.
+other change unrelated to a new release) is merged back into the main branch.
 
 
 # TLS certificates
@@ -316,7 +303,6 @@ done
 
 - Gather feedback on recent changes.
 - Check if dependencies need updates.
-- Update to latest publicsuffix/ list.
 - Check code if there are deprecated features that can be removed.
 - Generate apidiff and check if breaking changes can be prevented. Update moxtools.
 - Update features & roadmap in README.md and website.
@@ -335,6 +321,7 @@ done
 - Add release to the Latest release & News sections of website/index.md.
 - Create git tag (note: "#" is comment, not title/header), push code.
 - Build and publish new docker image.
+- Publish signed release notes for updates.xmox.nl and update DNS record.
 - Deploy update to website.
 - Create new release on the github page, so watchers get a notification.
   Copy/paste it manually from the tag text, and add link to download/compile
@@ -342,4 +329,3 @@ done
 - Publish new cross-referenced code/rfc to www.xmox.nl/xr/.
 - Update moxtools with latest version.
 - Update implementations support matrix.
-- Publish signed release notes for updates.xmox.nl and update DNS record.

dkim/dkim.go

@@ -31,7 +31,6 @@ import (
 	"github.com/mjl-/mox/publicsuffix"
 	"github.com/mjl-/mox/smtp"
 	"github.com/mjl-/mox/stub"
-	"slices"
 )
 
 // If set, signatures for top-level domain "localhost" are accepted.
@@ -174,7 +173,7 @@ func Sign(ctx context.Context, elog *slog.Logger, localpart smtp.Localpart, doma
 		sig.Domain = domain
 		sig.Selector = sel.Domain
 		sig.Identity = &Identity{&localpart, domain}
-		sig.SignedHeaders = slices.Clone(sel.Headers)
+		sig.SignedHeaders = append([]string{}, sel.Headers...)
 		if sel.SealHeaders {
 			// ../rfc/6376:2156
 			// Each time a header name is added to the signature, the next unused value is
@@ -549,7 +548,7 @@ func verifySignatureRecord(r *Record, sig *Sig, hash crypto.Hash, canonHeaderSim
 	if r.PublicKey == nil {
 		return StatusPermerror, ErrKeyRevoked
 	} else if rsaKey, ok := r.PublicKey.(*rsa.PublicKey); ok && rsaKey.N.BitLen() < 1024 {
-		// ../rfc/8301:157
+		// todo: find a reference that supports this.
 		return StatusPermerror, ErrWeakKey
 	}
 
@@ -840,8 +839,8 @@ func parseHeaders(br *bufio.Reader) ([]header, int, error) {
 			return nil, 0, fmt.Errorf("empty header key")
 		}
 		lkey = strings.ToLower(key)
-		value = slices.Clone(t[1])
+		value = append([]byte{}, t[1]...)
-		raw = slices.Clone(line)
+		raw = append([]byte{}, line...)
 	}
 	if key != "" {
 		l = append(l, header{key, lkey, value, raw})

dkim/sig.go

@@ -117,7 +117,7 @@ func (s *Sig) Header() (string, error) {
 			} else if i == len(s.SignedHeaders)-1 {
 				v += ";"
 			}
-			w.Addf(sep, "%s", v)
+			w.Addf(sep, v)
 		}
 	}
 	if len(s.CopiedHeaders) > 0 {
@@ -139,7 +139,7 @@ func (s *Sig) Header() (string, error) {
 			} else if i == len(s.CopiedHeaders)-1 {
 				v += ";"
 			}
-			w.Addf(sep, "%s", v)
+			w.Addf(sep, v)
 		}
 	}
 

dkim/txt_test.go

@@ -32,7 +32,7 @@ func TestParseRecord(t *testing.T) {
 		}
 		if r != nil {
 			pk := r.Pubkey
-			for range 2 {
+			for i := 0; i < 2; i++ {
 				ntxt, err := r.Record()
 				if err != nil {
 					t.Fatalf("making record: %v", err)

dmarc/dmarc.go

@@ -15,7 +15,7 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
-	mathrand2 "math/rand/v2"
+	mathrand "math/rand"
 	"time"
 
 	"github.com/mjl-/mox/dkim"
@@ -257,7 +257,7 @@ func Verify(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, msgFr
 
 	// Record can request sampling of messages to apply policy.
 	// See ../rfc/7489:1432
-	useResult = !applyRandomPercentage || record.Percentage == 100 || mathrand2.IntN(100) < record.Percentage
+	useResult = !applyRandomPercentage || record.Percentage == 100 || mathrand.Intn(100) < record.Percentage
 
 	// We treat "quarantine" and "reject" the same. Thus, we also don't "downgrade"
 	// from reject to quarantine if this message was sampled out.

dmarcdb/dmarcdb.go

@@ -21,7 +21,6 @@ import (
 
 	"github.com/mjl-/mox/mlog"
 	"github.com/mjl-/mox/mox-"
-	"github.com/mjl-/mox/moxvar"
 )
 
 // Init opens the databases.
@@ -65,13 +64,13 @@ func Close() error {
 func openReportsDB(ctx context.Context, log mlog.Log) (*bstore.DB, error) {
 	p := mox.DataDirPath("dmarcrpt.db")
 	os.MkdirAll(filepath.Dir(p), 0770)
-	opts := bstore.Options{Timeout: 5 * time.Second, Perm: 0660, RegisterLogger: moxvar.RegisterLogger(p, log.Logger)}
+	opts := bstore.Options{Timeout: 5 * time.Second, Perm: 0660, RegisterLogger: log.Logger}
 	return bstore.Open(ctx, p, &opts, ReportsDBTypes...)
 }
 
 func openEvalDB(ctx context.Context, log mlog.Log) (*bstore.DB, error) {
 	p := mox.DataDirPath("dmarceval.db")
 	os.MkdirAll(filepath.Dir(p), 0770)
-	opts := bstore.Options{Timeout: 5 * time.Second, Perm: 0660, RegisterLogger: moxvar.RegisterLogger(p, log.Logger)}
+	opts := bstore.Options{Timeout: 5 * time.Second, Perm: 0660, RegisterLogger: log.Logger}
 	return bstore.Open(ctx, p, &opts, EvalDBTypes...)
 }

dmarcdb/eval.go

@@ -10,7 +10,6 @@ import (
 	"fmt"
 	"io"
 	"log/slog"
-	"maps"
 	"mime"
 	"mime/multipart"
 	"net/textproto"
@@ -18,10 +17,13 @@ import (
 	"os"
 	"runtime/debug"
 	"slices"
+	"sort"
 	"strings"
 	"sync"
 	"time"
 
+	"golang.org/x/exp/maps"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 
@@ -250,7 +252,7 @@ var jitterRand = mox.NewPseudoRand()
 // Jitter so we don't cause load at exactly whole hours, other processes may
 // already be doing that.
 var jitteredTimeUntil = func(t time.Time) time.Duration {
-	return time.Until(t.Add(time.Duration(30+jitterRand.IntN(60)) * time.Second))
+	return time.Until(t.Add(time.Duration(30+jitterRand.Intn(60)) * time.Second))
 }
 
 // Start launches a goroutine that wakes up at each whole hour (plus jitter) and
@@ -687,7 +689,9 @@ func sendReportDomain(ctx context.Context, log mlog.Log, resolver dns.Resolver,
 	report.PolicyPublished = last.PolicyPublished
 
 	// Process records in-order for testable results.
-	for _, recstr := range slices.Sorted(maps.Keys(counts)) {
+	recstrs := maps.Keys(counts)
+	sort.Strings(recstrs)
+	for _, recstr := range recstrs {
 		rc := counts[recstr]
 		rc.ReportRecord.Row.Count = rc.count
 		report.Records = append(report.Records, rc.ReportRecord)
@@ -1017,7 +1021,7 @@ func dkimSign(ctx context.Context, log mlog.Log, fromAddr smtp.Address, smtputf8
 	for fd != zerodom {
 		confDom, ok := mox.Conf.Domain(fd)
 		selectors := mox.DKIMSelectors(confDom.DKIM)
-		if len(selectors) > 0 && !confDom.Disabled {
+		if len(selectors) > 0 {
 			dkimHeaders, err := dkim.Sign(ctx, log.Logger, fromAddr.Localpart, fd, selectors, smtputf8, mf)
 			if err != nil {
 				log.Errorx("dkim-signing dmarc report, continuing without signature", err)

dmarcdb/eval_test.go

@@ -19,7 +19,6 @@ import (
 	"github.com/mjl-/mox/mox-"
 	"github.com/mjl-/mox/moxio"
 	"github.com/mjl-/mox/queue"
-	"slices"
 )
 
 func tcheckf(t *testing.T, err error, format string, args ...any) {
@@ -302,7 +301,7 @@ func TestSendReports(t *testing.T) {
 			// Read message file. Also write copy to disk for inspection.
 			buf, err := io.ReadAll(&moxio.AtReader{R: msgFile})
 			tcheckf(t, err, "read report message")
-			err = os.WriteFile("../testdata/dmarcdb/data/report.eml", slices.Concat(qm.MsgPrefix, buf), 0600)
+			err = os.WriteFile("../testdata/dmarcdb/data/report.eml", append(append([]byte{}, qm.MsgPrefix...), buf...), 0600)
 			tcheckf(t, err, "write report message")
 
 			var feedback *dmarcrpt.Feedback

dmarcdb/main_test.go

@@ -1,17 +0,0 @@
-package dmarcdb
-
-import (
-	"fmt"
-	"os"
-	"testing"
-
-	"github.com/mjl-/mox/metrics"
-)
-
-func TestMain(m *testing.M) {
-	m.Run()
-	if metrics.Panics.Load() > 0 {
-		fmt.Println("unhandled panics encountered")
-		os.Exit(2)
-	}
-}

dmarcrpt/parse.go

@@ -52,7 +52,7 @@ func parseMessageReport(log mlog.Log, p message.Part) (*Feedback, error) {
 	// content of the message.
 
 	if p.MediaType != "MULTIPART" {
-		return parseReport(log, p)
+		return parseReport(p)
 	}
 
 	for {
@@ -72,7 +72,7 @@ func parseMessageReport(log mlog.Log, p message.Part) (*Feedback, error) {
 	}
 }
 
-func parseReport(log mlog.Log, p message.Part) (*Feedback, error) {
+func parseReport(p message.Part) (*Feedback, error) {
 	ct := strings.ToLower(p.MediaType + "/" + p.MediaSubType)
 	r := p.Reader()
 
@@ -93,7 +93,7 @@ func parseReport(log mlog.Log, p message.Part) (*Feedback, error) {
 	switch ct {
 	case "application/zip":
 		// Google sends messages with direct application/zip content-type.
-		return parseZip(log, r)
+		return parseZip(r)
 	case "application/gzip", "application/x-gzip":
 		gzr, err := gzip.NewReader(r)
 		if err != nil {
@@ -106,7 +106,7 @@ func parseReport(log mlog.Log, p message.Part) (*Feedback, error) {
 	return nil, ErrNoReport
 }
 
-func parseZip(log mlog.Log, r io.Reader) (*Feedback, error) {
+func parseZip(r io.Reader) (*Feedback, error) {
 	buf, err := io.ReadAll(r)
 	if err != nil {
 		return nil, fmt.Errorf("reading feedback: %s", err)
@@ -122,9 +122,6 @@ func parseZip(log mlog.Log, r io.Reader) (*Feedback, error) {
 	if err != nil {
 		return nil, fmt.Errorf("opening file in zip: %s", err)
 	}
-	defer func() {
+	defer f.Close()
-		err := f.Close()
-		log.Check(err, "closing report file in zip file")
-	}()
 	return ParseReport(f)
 }

dns/dns.go

@@ -5,7 +5,6 @@ package dns
 import (
 	"errors"
 	"fmt"
-	"net"
 	"strings"
 
 	"golang.org/x/net/idna"
@@ -20,7 +19,6 @@ var (
 	errTrailingDot = errors.New("dns name has trailing dot")
 	errUnderscore  = errors.New("domain name with underscore")
 	errIDNA        = errors.New("idna")
-	errIPNotName   = errors.New("ip address while name required")
 )
 
 // Domain is a domain name, with one or more labels, with at least an ASCII
@@ -97,12 +95,6 @@ func ParseDomain(s string) (Domain, error) {
 		return Domain{}, errTrailingDot
 	}
 
-	// IPv4 addresses would be accepted by idna lookups. TLDs cannot be all numerical,
-	// so IP addresses are not valid DNS names.
-	if net.ParseIP(s) != nil {
-		return Domain{}, errIPNotName
-	}
-
 	ascii, err := idna.Lookup.ToASCII(s)
 	if err != nil {
 		return Domain{}, fmt.Errorf("%w: to ascii: %v", errIDNA, err)
@@ -156,9 +148,7 @@ func ParseDomainLax(s string) (Domain, error) {
 	return Domain{ASCII: s}, nil
 }
 
-// IsNotFound returns whether an error is an adns.DNSError or net.DNSError with
+// IsNotFound returns whether an error is an adns.DNSError with IsNotFound set.
-// IsNotFound set.
-//
 // IsNotFound means the requested type does not exist for the given domain (a
 // nodata or nxdomain response). It doesn't not necessarily mean no other types for
 // that name exist.
@@ -168,7 +158,6 @@ func ParseDomainLax(s string) (Domain, error) {
 // The adns resolver (just like the Go resolver) returns an IsNotFound error for
 // both cases, there is no need to explicitly check for zero entries.
 func IsNotFound(err error) bool {
-	var adnsErr *adns.DNSError
+	var dnsErr *adns.DNSError
-	var dnsErr *net.DNSError
+	return err != nil && errors.As(err, &dnsErr) && dnsErr.IsNotFound
-	return err != nil && (errors.As(err, &adnsErr) && adnsErr.IsNotFound || errors.As(err, &dnsErr) && dnsErr.IsNotFound)
 }

dnsbl/dnsbl.go

@@ -10,7 +10,7 @@
 // looked up with an DNS "A" lookup of a name similar to an IPv4 address, but with
 // 4-bit hexadecimal dot-separated characters, in reverse.
 //
-// The health of a DNSBL "zone" can be checked through a lookup of 127.0.0.1
+// The health of a DNSBL "zone" can be check through a lookup of 127.0.0.1
 // (must not be present) and 127.0.0.2 (must be present).
 package dnsbl
 

doc.go

@@ -55,7 +55,7 @@ any parameters. Followed by the help and usage information for each command.
 	mox export mbox [-single] dst-dir account-path [mailbox]
 	mox localserve
 	mox help [command ...]
-	mox backup destdir
+	mox backup dest-dir
 	mox verifydata data-dir
 	mox licenses
 	mox config test
@@ -63,22 +63,12 @@ any parameters. Followed by the help and usage information for each command.
 	mox config dnsrecords domain
 	mox config describe-domains >domains.conf
 	mox config describe-static >mox.conf
-	mox config account list
 	mox config account add account address
 	mox config account rm account
-	mox config account disable account message
-	mox config account enable account
 	mox config address add address account
 	mox config address rm address
-	mox config domain add [-disabled] domain account [localpart]
+	mox config domain add domain account [localpart]
 	mox config domain rm domain
-	mox config domain disable domain
-	mox config domain enable domain
-	mox config tlspubkey list [account]
-	mox config tlspubkey get fingerprint
-	mox config tlspubkey add address [name] < cert.pem
-	mox config tlspubkey rm fingerprint
-	mox config tlspubkey gen stem
 	mox config alias list domain
 	mox config alias print alias
 	mox config alias add alias@domain rcpt1@domain ...
@@ -90,7 +80,6 @@ any parameters. Followed by the help and usage information for each command.
 	mox config printservice >mox.service
 	mox config ensureacmehostprivatekeys
 	mox config example [name]
-	mox admin imapserve preauth-address
 	mox checkupdate
 	mox cid cid
 	mox clientconfig domain
@@ -111,10 +100,8 @@ any parameters. Followed by the help and usage information for each command.
 	mox dnsbl check zone ip
 	mox dnsbl checkhealth zone
 	mox mtasts lookup domain
-	mox rdap domainage domain
+	mox retrain accountname
-	mox retrain [accountname]
 	mox sendmail [-Fname] [ignoredflags] [-t] [<message]
-	mox smtp dial host[:port]
 	mox spf check domain ip
 	mox spf lookup domain
 	mox spf parse txtrecord
@@ -154,8 +141,6 @@ Quickstart writes configuration files, prints initial admin and account
 passwords, DNS records you should create. If you run it on Linux it writes a
 systemd service file and prints commands to enable and start mox as service.
 
-All output is written to quickstart.log for later reference.
-
 The user or uid is optional, defaults to "mox", and is the user or uid/gid mox
 will run as after initialization.
 
@@ -189,7 +174,7 @@ output of "mox config describe-domains" and see the output of
 	  -hostname string
 	    	hostname mox will run on, by default the hostname of the machine quickstart runs on; if specified, the IPs for the hostname are configured for the public listener
 	  -skipdial
-	    	skip check for outgoing smtp (port 25) connectivity or for domain age with rdap
+	    	skip check for outgoing smtp (port 25) connectivity
 
 # mox stop
 
@@ -827,14 +812,13 @@ If a single command matches, its usage and full help text is printed.
 
 # mox backup
 
-Creates a backup of the config and data directory.
+Creates a backup of the data directory.
 
-Backup copies the config directory to <destdir>/config, and creates
+Backup creates consistent snapshots of the databases and message files and
-<destdir>/data with a consistent snapshot of the databases and message files
+copies other files in the data directory. Empty directories are not copied.
-and copies other files from the data directory. Empty directories are not
+These files can then be stored elsewhere for long-term storage, or used to fall
-copied. The backup can then be stored elsewhere for long-term storage, or used
+back to should an upgrade fail. Simply copying files in the data directory
-to fall back to should an upgrade fail. Simply copying files in the data
+while mox is running can result in unusable database files.
-directory while mox is running can result in unusable database files.
 
 Message files never change (they are read-only, though can be removed) and are
 hard-linked so they don't consume additional space. If hardlinking fails, for
@@ -856,19 +840,18 @@ not print any output, but may print warnings. Use the -verbose flag for
 details, including timing.
 
 To restore a backup, first shut down mox, move away the old data directory and
-move an earlier backed up directory in its place, run "mox verifydata
+move an earlier backed up directory in its place, run "mox verifydata",
-<datadir>", possibly with the "-fix" option, and restart mox. After the
+possibly with the "-fix" option, and restart mox. After the restore, you may
-restore, you may also want to run "mox bumpuidvalidity" for each account for
+also want to run "mox bumpuidvalidity" for each account for which messages in a
-which messages in a mailbox changed, to force IMAP clients to synchronize
+mailbox changed, to force IMAP clients to synchronize mailbox state.
-mailbox state.
 
 Before upgrading, to check if the upgrade will likely succeed, first make a
-backup, then use the new mox binary to run "mox verifydata <backupdir>/data".
+backup, then use the new mox binary to run "mox verifydata" on the backup. This
-This can change the backup files (e.g. upgrade database files, move away
+can change the backup files (e.g. upgrade database files, move away
 unrecognized message files), so you should make a new backup before actually
 upgrading.
 
-	usage: mox backup destdir
+	usage: mox backup dest-dir
 	  -verbose
 	    	print progress
 
@@ -955,15 +938,6 @@ may contain unfinished list items.
 
 	usage: mox config describe-static >mox.conf
 
-# mox config account list
-
-List all accounts.
-
-Each account is printed on a line, with optional additional tab-separated
-information, such as "(disabled)".
-
-	usage: mox config account list
-
 # mox config account add
 
 Add an account with an email address and reload the configuration.
@@ -984,26 +958,6 @@ All data for the account will be removed.
 
 	usage: mox config account rm account
 
-# mox config account disable
-
-Disable login for an account, showing message to users when they try to login.
-
-Incoming email will still be accepted for the account, and queued email from the
-account will still be delivered. No new login sessions are possible.
-
-Message must be non-empty, ascii-only without control characters including
-newline, and maximum 256 characters because it is used in SMTP/IMAP.
-
-	usage: mox config account disable account message
-
-# mox config account enable
-
-Enable login again for an account.
-
-Login attempts by the user no long result in an error message.
-
-	usage: mox config account enable account
-
 # mox config address add
 
 Adds an address to an account and reloads the configuration.
@@ -1029,13 +983,7 @@ The account is used for the postmaster mailboxes the domain, including as DMARC
 TLS reporting. Localpart is the "username" at the domain for this account. If
 must be set if and only if account does not yet exist.
 
-The domain can be created in disabled mode, preventing automatically requesting
+	usage: mox config domain add domain account [localpart]
-TLS certificates with ACME, and rejecting incoming/outgoing messages involving
-the domain, but allowing further configuration of the domain.
-
-	usage: mox config domain add [-disabled] domain account [localpart]
-	  -disabled
-	    	disable the new domain
 
 # mox config domain rm
 
@@ -1046,103 +994,27 @@ rejected.
 
 	usage: mox config domain rm domain
 
-# mox config domain disable
-
-Disable a domain and reload the configuration.
-
-This is a dangerous operation. Incoming/outgoing messages involving this domain
-will be rejected.
-
-	usage: mox config domain disable domain
-
-# mox config domain enable
-
-Enable a domain and reload the configuration.
-
-Incoming/outgoing messages involving this domain will be accepted again.
-
-	usage: mox config domain enable domain
-
-# mox config tlspubkey list
-
-List TLS public keys for TLS client certificate authentication.
-
-If account is absent, the TLS public keys for all accounts are listed.
-
-	usage: mox config tlspubkey list [account]
-
-# mox config tlspubkey get
-
-Get a TLS public key for a fingerprint.
-
-Prints the type, name, account and address for the key, and the certificate in
-PEM format.
-
-	usage: mox config tlspubkey get fingerprint
-
-# mox config tlspubkey add
-
-Add a TLS public key to the account of the given address.
-
-The public key is read from the certificate.
-
-The optional name is a human-readable descriptive name of the key. If absent,
-the CommonName from the certificate is used.
-
-	usage: mox config tlspubkey add address [name] < cert.pem
-	  -no-imap-preauth
-	    	Don't automatically switch new IMAP connections authenticated with this key to "authenticated" state after the TLS handshake. For working around clients that ignore the untagged IMAP PREAUTH response and try to authenticate while already authenticated.
-
-# mox config tlspubkey rm
-
-Remove TLS public key for fingerprint.
-
-	usage: mox config tlspubkey rm fingerprint
-
-# mox config tlspubkey gen
-
-Generate an ed25519 private key and minimal certificate for use a TLS public key and write to files starting with stem.
-
-The private key is written to $stem.$timestamp.ed25519privatekey.pkcs8.pem.
-The certificate is written to $stem.$timestamp.certificate.pem.
-The private key and certificate are also written to
-$stem.$timestamp.ed25519privatekey-certificate.pem.
-
-The certificate can be added to an account with "mox config account tlspubkey add".
-
-The combined file can be used with "mox sendmail".
-
-The private key is also written to standard error in raw-url-base64-encoded
-form, also for use with "mox sendmail". The fingerprint is written to standard
-error too, for reference.
-
-	usage: mox config tlspubkey gen stem
-
 # mox config alias list
 
-Show aliases (lists) for domain.
+List aliases for domain.
 
 	usage: mox config alias list domain
 
 # mox config alias print
 
-Print settings and members of alias (list).
+Print settings and members of alias.
 
 	usage: mox config alias print alias
 
 # mox config alias add
 
-Add new alias (list) with one or more addresses and public posting enabled.
+Add new alias with one or more addresses.
-
-An alias is used for delivering incoming email to multiple recipients. If you
-want to add an address to an account, don't use an alias, just add the address
-to the account.
 
 	usage: mox config alias add alias@domain rcpt1@domain ...
 
 # mox config alias update
 
-Update alias (list) configuration.
+Update alias configuration.
 
 	usage: mox config alias update alias@domain [-postpublic false|true -listmembers false|true -allowmsgfrom false|true]
 	  -allowmsgfrom string
@@ -1154,19 +1026,19 @@ Update alias (list) configuration.
 
 # mox config alias rm
 
-Remove alias (list).
+Remove alias.
 
 	usage: mox config alias rm alias@domain
 
 # mox config alias addaddr
 
-Add addresses to alias (list).
+Add addresses to alias.
 
 	usage: mox config alias addaddr alias@domain rcpt1@domain ...
 
 # mox config alias rmaddr
 
-Remove addresses from alias (list).
+Remove addresses from alias.
 
 	usage: mox config alias rmaddr alias@domain rcpt1@domain ...
 
@@ -1217,18 +1089,6 @@ List available config examples, or print a specific example.
 
 	usage: mox config example [name]
 
-# mox admin imapserve
-
-Initiate a preauthenticated IMAP connection on file descriptor 0.
-
-For use with tools that can do IMAP over tunneled connections, e.g. with SSH
-during migrations. TLS is not possible on the connection, and authentication
-does not require TLS.
-
-	usage: mox admin imapserve preauth-address
-	  -fd0
-	    	write IMAP to file descriptor 0 instead of stdout
-
 # mox checkupdate
 
 Check if a newer version of mox is available.
@@ -1468,32 +1328,14 @@ should be used, and how long the policy can be cached.
 
 	usage: mox mtasts lookup domain
 
-# mox rdap domainage
-
-Lookup the age of domain in RDAP based on latest registration.
-
-RDAP is the registration data access protocol. Registries run RDAP services for
-their top level domains, providing information such as the registration date of
-domains. This command looks up the "age" of a domain by looking at the most
-recent "registration", "reregistration" or "reinstantiation" event.
-
-Email messages from recently registered domains are often treated with
-suspicion, and some mail systems are more likely to classify them as junk.
-
-On each invocation, a bootstrap file with a list of registries (of top-level
-domains) is retrieved, without caching. Do not run this command too often with
-automation.
-
-	usage: mox rdap domainage domain
-
 # mox retrain
 
-Recreate and retrain the junk filter for the account or all accounts.
+Recreate and retrain the junk filter for the account.
 
 Useful after having made changes to the junk filter configuration, or if the
 implementation has changed.
 
-	usage: mox retrain [accountname]
+	usage: mox retrain accountname
 
 # mox sendmail
 
@@ -1528,51 +1370,6 @@ binary should be setgid that group:
 
 	usage: mox sendmail [-Fname] [ignoredflags] [-t] [<message]
 
-# mox smtp dial
-
-Dial the address, initialize the SMTP session, including using STARTTLS to enable TLS if the server supports it.
-
-If no port is specified, SMTP port 25 is used.
-
-Data is copied between connection and stdin/stdout until either side closes the
-connection.
-
-The flags influence the TLS configuration, useful for debugging interoperability
-issues.
-
-No MTA-STS or DANE verification is done.
-
-Hint: Use "mox -loglevel trace smtp dial ..." to see the protocol messages
-exchanged during connection set up.
-
-	usage: mox smtp dial host[:port]
-	  -ehlohostname string
-	    	our hostname to use during the SMTP EHLO command
-	  -forcetls
-	    	use TLS, even if remote SMTP server does not announce STARTTLS extension
-	  -notls
-	    	do not use TLS
-	  -remotehostname string
-	    	remote hostname to use for TLS verification, if enabled; the hostname from the parameter is used by default
-	  -tlscerts string
-	    	path to root ca certificates in pem form, for verification
-	  -tlsciphersuites string
-	    	ciphersuites to allow, comma-separated, order is ignored, only for TLS 1.2 and earlier, empty value uses TLS stack defaults; values: tls_ecdhe_ecdsa_with_aes_128_cbc_sha, tls_ecdhe_ecdsa_with_aes_128_gcm_sha256, tls_ecdhe_ecdsa_with_aes_256_cbc_sha, tls_ecdhe_ecdsa_with_aes_256_gcm_sha384, tls_ecdhe_ecdsa_with_chacha20_poly1305_sha256, tls_ecdhe_rsa_with_aes_128_cbc_sha, tls_ecdhe_rsa_with_aes_128_gcm_sha256, tls_ecdhe_rsa_with_aes_256_cbc_sha, tls_ecdhe_rsa_with_aes_256_gcm_sha384, tls_ecdhe_rsa_with_chacha20_poly1305_sha256, and insecure: tls_ecdhe_ecdsa_with_aes_128_cbc_sha256, tls_ecdhe_ecdsa_with_rc4_128_sha, tls_ecdhe_rsa_with_3des_ede_cbc_sha, tls_ecdhe_rsa_with_aes_128_cbc_sha256, tls_ecdhe_rsa_with_rc4_128_sha, tls_rsa_with_3des_ede_cbc_sha, tls_rsa_with_aes_128_cbc_sha, tls_rsa_with_aes_128_cbc_sha256, tls_rsa_with_aes_128_gcm_sha256, tls_rsa_with_aes_256_cbc_sha, tls_rsa_with_aes_256_gcm_sha384, tls_rsa_with_rc4_128_sha
-	  -tlscurves string
-	    	tls ecc key exchange mechanisms to allow, comma-separated, order is ignored, empty value uses TLS stack defaults; values: curvep256, curvep384, curvep521, x25519, x25519mlkem768
-	  -tlsnodynamicrecordsizing
-	    	disable TLS dynamic record sizing
-	  -tlsnosessiontickets
-	    	disable TLS session tickets
-	  -tlsrenegotiation string
-	    	when to allow renegotiation; only applies to tls1.2 and earlier, not tls1.3; values: never, once, always (default "never")
-	  -tlsverify
-	    	verify remote hostname during TLS
-	  -tlsversionmax string
-	    	maximum TLS version, empty value uses TLS stack default; values: tls1.2, etc.
-	  -tlsversionmin string
-	    	minimum TLS version, empty value uses TLS stack default; values: tls1.2, etc.
-
 # mox spf check
 
 Check the status of IP for the policy published in DNS for the domain.

docker-compose-imaptest.yml

@@ -1,12 +1,13 @@
+version: '3.7'
 services:
   mox:
     build:
       context: .
       dockerfile: Dockerfile.moximaptest
     volumes:
-      - ./testdata/imaptest/config:/mox/config:z
+      - ./testdata/imaptest/config:/mox/config
-      - ./testdata/imaptest/data:/mox/data:z
+      - ./testdata/imaptest/data:/mox/data
-      - ./testdata/imaptest/imaptest.mbox:/mox/imaptest.mbox:z
+      - ./testdata/imaptest/imaptest.mbox:/mox/imaptest.mbox
     working_dir: /mox
     tty: true # For job control with set -m.
     command: sh -c 'set -m; mox serve & sleep 1; echo testtest | mox setaccountpassword mjl; fg'
@@ -23,7 +24,7 @@ services:
     command: host=mox port=1143 'user=mjl@mox.example' pass=testtest mbox=/imaptest/imaptest.mbox
     working_dir: /imaptest
     volumes:
-      - ./testdata/imaptest:/imaptest:z
+      - ./testdata/imaptest:/imaptest
     depends_on:
       mox:
         condition: service_healthy

docker-compose-integration.yml

@@ -1,3 +1,4 @@
+version: '3.7'
 services:
   # We run integration_test.go from this container, it connects to the other mox instances.
   test:
@@ -8,11 +9,11 @@ services:
     # dials in integration_test.go succeed.
     command: ["sh", "-c", "set -ex; cat /integration/tmp-pebble-ca.pem /integration/tls/ca.pem >>/etc/ssl/certs/ca-certificates.crt; go test -tags integration"]
     volumes:
-      - ./.go:/.go:z
+      - ./.go:/.go
-      - ./testdata/integration/resolv.conf:/etc/resolv.conf:z
+      - ./testdata/integration/resolv.conf:/etc/resolv.conf
-      - ./testdata/integration:/integration:z
+      - ./testdata/integration:/integration
-      - ./testdata/integration/moxsubmit.conf:/etc/moxsubmit.conf:z
+      - ./testdata/integration/moxsubmit.conf:/etc/moxsubmit.conf
-      - .:/mox:z
+      - .:/mox
     environment:
       GOCACHE: /.go/.cache/go-build
     depends_on:
@@ -25,8 +26,6 @@ services:
         condition: service_healthy
       localserve:
         condition: service_healthy
-      moxacmepebblealpn:
-        condition: service_healthy
     networks:
       mailnet1:
         ipv4_address: 172.28.1.50
@@ -40,8 +39,8 @@ services:
       MOX_UID: "${MOX_UID}"
     command: ["sh", "-c", "/integration/moxacmepebble.sh"]
     volumes:
-      - ./testdata/integration/resolv.conf:/etc/resolv.conf:z
+      - ./testdata/integration/resolv.conf:/etc/resolv.conf
-      - ./testdata/integration:/integration:z
+      - ./testdata/integration:/integration
     healthcheck:
       test: netstat -nlt | grep ':25 '
       interval: 1s
@@ -65,8 +64,8 @@ services:
       MOX_UID: "${MOX_UID}"
     command: ["sh", "-c", "/integration/moxmail2.sh"]
     volumes:
-      - ./testdata/integration/resolv.conf:/etc/resolv.conf:z
+      - ./testdata/integration/resolv.conf:/etc/resolv.conf
-      - ./testdata/integration:/integration:z
+      - ./testdata/integration:/integration
     healthcheck:
       test: netstat -nlt | grep ':25 '
       interval: 1s
@@ -84,40 +83,15 @@ services:
       mailnet1:
         ipv4_address: 172.28.1.20
 
-  # Third mox instance that uses ACME with pebble and has ALPN enabled.
-  moxacmepebblealpn:
-    hostname: moxacmepebblealpn.mox1.example
-    domainname: mox1.example
-    image: mox_integration_moxmail
-    environment:
-      MOX_UID: "${MOX_UID}"
-    command: ["sh", "-c", "/integration/moxacmepebblealpn.sh"]
-    volumes:
-      - ./testdata/integration/resolv.conf:/etc/resolv.conf:z
-      - ./testdata/integration:/integration:z
-    healthcheck:
-      test: netstat -nlt | grep ':25 '
-      interval: 1s
-      timeout: 1s
-      retries: 10
-    depends_on:
-      dns:
-        condition: service_healthy
-      acmepebble:
-        condition: service_healthy
-    networks:
-      mailnet1:
-        ipv4_address: 172.28.1.80
-
   localserve:
     hostname: localserve.mox1.example
     domainname: mox1.example
     image: mox_integration_moxmail
     command: ["sh", "-c", "set -e; chmod o+r /etc/resolv.conf; mox -checkconsistency localserve -ip 172.28.1.60"]
     volumes:
-      - ./.go:/.go:z
+      - ./.go:/.go
-      - ./testdata/integration/resolv.conf:/etc/resolv.conf:z
+      - ./testdata/integration/resolv.conf:/etc/resolv.conf
-      - .:/mox:z
+      - .:/mox
     environment:
       GOCACHE: /.go/.cache/go-build
     healthcheck:
@@ -140,7 +114,7 @@ services:
       context: testdata/integration
     volumes:
       # todo: figure out how to mount files with a uid that the process in the container can read...
-      - ./testdata/integration/resolv.conf:/etc/resolv.conf:z
+      - ./testdata/integration/resolv.conf:/etc/resolv.conf
     command: ["sh", "-c", "set -e; chmod o+r /etc/resolv.conf; (echo 'maillog_file = /dev/stdout'; echo 'mydestination = $$myhostname, localhost.$$mydomain, localhost, $$mydomain'; echo 'smtp_tls_security_level = may') >>/etc/postfix/main.cf; echo 'root: postfix@mox1.example' >>/etc/postfix/aliases; newaliases; postfix start-fg"]
     healthcheck:
       test: netstat -nlt | grep ':25 '
@@ -161,8 +135,8 @@ services:
       # todo: figure out how to build from dockerfile with empty context without creating empty dirs in file system.
       context: testdata/integration
     volumes:
-      - ./testdata/integration/resolv.conf:/etc/resolv.conf:z
+      - ./testdata/integration/resolv.conf:/etc/resolv.conf
-      - ./testdata/integration:/integration:z
+      - ./testdata/integration:/integration
     # We start with a base example.zone, but moxacmepebble appends its records,
     # followed by moxmail2. They restart unbound after appending records.
     command: ["sh", "-c", "set -ex; ls -l /etc/resolv.conf; chmod o+r /etc/resolv.conf; install -m 640 -o unbound /integration/unbound.conf /etc/unbound/; chmod 755 /integration; chmod 644 /integration/*.zone; cp /integration/example.zone /integration/example-integration.zone; ls -ld /integration /integration/reverse.zone; unbound -d -p -v"]
@@ -182,8 +156,8 @@ services:
     hostname: acmepebble.example
     image: docker.io/letsencrypt/pebble:v2.3.1@sha256:fc5a537bf8fbc7cc63aa24ec3142283aa9b6ba54529f86eb8ff31fbde7c5b258
     volumes:
-      - ./testdata/integration/resolv.conf:/etc/resolv.conf:z
+      - ./testdata/integration/resolv.conf:/etc/resolv.conf
-      - ./testdata/integration:/integration:z
+      - ./testdata/integration:/integration
     command: ["sh", "-c", "set -ex; mount; ls -l /etc/resolv.conf; chmod o+r /etc/resolv.conf; pebble -config /integration/pebble-config.json"]
     ports:
       - 14000:14000  # ACME port

docker-compose.yml

@@ -27,6 +27,7 @@
 # The -ip flag ensures connections to the published ports make it to mox, and it
 # prevents listening on ::1 (IPv6 is not enabled in docker by default).
 
+version: '3.7'
 services:
   mox:
     # Replace "latest" with the version you want to run, see https://r.xmox.nl/r/mox/.
@@ -38,11 +39,11 @@ services:
     # machine, and the IPs of incoming connections for spam filtering.
     network_mode: 'host'
     volumes:
-      - ./config:/mox/config:z
+      - ./config:/mox/config
-      - ./data:/mox/data:z
+      - ./data:/mox/data
       # web is optional but recommended to bind in, useful for serving static files with
       # the webserver.
-      - ./web:/mox/web:z
+      - ./web:/mox/web
     working_dir: /mox
     restart: on-failure
     healthcheck:

dsn/dsn.go

@@ -340,7 +340,10 @@ func (m *Message) Compose(log mlog.Log, smtputf8 bool) ([]byte, error) {
 			data := base64.StdEncoding.EncodeToString(headers)
 			for len(data) > 0 {
 				line := data
-				n := min(len(line), 76) // ../rfc/2045:1372
+				n := len(line)
+				if n > 78 {
+					n = 78
+				}
 				line, data = data[:n], data[n:]
 				if _, err := origp.Write([]byte(line + "\r\n")); err != nil {
 					return nil, err

dsn/dsn_test.go

@@ -50,8 +50,8 @@ func tcheckType(t *testing.T, p *message.Part, mt, mst, cte string) {
 	if !strings.EqualFold(p.MediaSubType, mst) {
 		t.Fatalf("got mediasubtype %q, expected %q", p.MediaSubType, mst)
 	}
-	if !(cte == "" && p.ContentTransferEncoding == nil || cte != "" && p.ContentTransferEncoding != nil && strings.EqualFold(cte, *p.ContentTransferEncoding)) {
+	if !strings.EqualFold(p.ContentTransferEncoding, cte) {
-		t.Fatalf("got content-transfer-encoding %v, expected %v", p.ContentTransferEncoding, cte)
+		t.Fatalf("got content-transfer-encoding %q, expected %q", p.ContentTransferEncoding, cte)
 	}
 }
 

dsn/parse.go

@@ -14,7 +14,6 @@ import (
 	"github.com/mjl-/mox/message"
 	"github.com/mjl-/mox/mlog"
 	"github.com/mjl-/mox/smtp"
-	"slices"
 )
 
 // Parse reads a DSN message.
@@ -218,9 +217,15 @@ func parseRecipientHeader(mr *textproto.Reader, utf8 bool) (Recipient, error) {
 		case "Action":
 			a := Action(strings.ToLower(v))
 			actions := []Action{Failed, Delayed, Delivered, Relayed, Expanded}
-			if slices.Contains(actions, a) {
+			var ok bool
-				r.Action = a
+			for _, x := range actions {
-			} else {
+				if a == x {
+					ok = true
+					r.Action = a
+					break
+				}
+			}
+			if !ok {
 				err = fmt.Errorf("unrecognized action %q", v)
 			}
 		case "Status":

export.go

@@ -72,7 +72,7 @@ func xcmdExport(mbox, single bool, args []string, c *cmd) {
 	}()
 
 	a := store.DirArchiver{Dir: dst}
-	err = store.ExportMessages(context.Background(), c.log, db, accountDir, a, !mbox, mailbox, nil, !single)
+	err = store.ExportMessages(context.Background(), c.log, db, accountDir, a, !mbox, mailbox, !single)
 	xcheckf(err, "exporting messages")
 	err = a.Close()
 	xcheckf(err, "closing archiver")

genapidoc.sh

@@ -1,10 +0,0 @@
-#!/bin/sh
-set -eu
-
-# we rewrite some dmarcprt and tlsrpt enums into untyped strings: real-world
-# reports have invalid values, and our loose Go typed strings accept all values,
-# but we don't want the typescript runtime checker to fail on those unrecognized
-# values.
-(cd webadmin && CGO_ENABLED=0 go run ../vendor/github.com/mjl-/sherpadoc/cmd/sherpadoc/*.go -adjust-function-names none -rename 'config Domain ConfigDomain,dmarc Policy DMARCPolicy,mtasts MX STSMX,tlsrptdb Record TLSReportRecord,tlsrptdb SuppressAddress TLSRPTSuppressAddress,dmarcrpt DKIMResult string,dmarcrpt SPFResult string,dmarcrpt SPFDomainScope string,dmarcrpt DMARCResult string,dmarcrpt PolicyOverride string,dmarcrpt Alignment string,dmarcrpt Disposition string,tlsrpt PolicyType string,tlsrpt ResultType string' Admin) >webadmin/api.json
-(cd webaccount && CGO_ENABLED=0 go run ../vendor/github.com/mjl-/sherpadoc/cmd/sherpadoc/*.go -adjust-function-names none Account) >webaccount/api.json
-(cd webmail && CGO_ENABLED=0 go run ../vendor/github.com/mjl-/sherpadoc/cmd/sherpadoc/*.go -adjust-function-names none Webmail) >webmail/api.json

gentestdata.go

@@ -30,7 +30,7 @@ import (
 
 func cmdGentestdata(c *cmd) {
 	c.unlisted = true
-	c.params = "destdir"
+	c.params = "dest-dir"
 	c.help = `Generate a data directory populated, for testing upgrades.`
 	args := c.Parse()
 	if len(args) != 1 {
@@ -187,12 +187,6 @@ Accounts:
 	err = os.WriteFile(filepath.Join(destDataDir, "moxversion"), []byte(moxvar.Version), 0660)
 	xcheckf(err, "writing moxversion")
 
-	// Populate auth.db
-	err = store.Init(ctxbg)
-	xcheckf(err, "store init")
-	err = store.TLSPublicKeyAdd(ctxbg, &store.TLSPublicKey{Name: "testkey", Fingerprint: "...", Type: "ecdsa-p256", CertDER: []byte("..."), Account: "test0", LoginAddress: "test0@mox.example"})
-	xcheckf(err, "adding tlspubkey")
-
 	// Populate dmarc.db.
 	err = dmarcdb.Init()
 	xcheckf(err, "dmarcdb init")
@@ -234,7 +228,8 @@ Accounts:
 	prefix := []byte{}
 	mf := tempfile()
 	xcheckf(err, "temp file for queue message")
-	defer store.CloseRemoveTempFile(c.log, mf, "test message")
+	defer os.Remove(mf.Name())
+	defer mf.Close()
 	const qmsg = "From: <test0@mox.example>\r\nTo: <other@remote.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
 	_, err = fmt.Fprint(mf, qmsg)
 	xcheckf(err, "writing message")
@@ -244,7 +239,7 @@ Accounts:
 
 	// Create three accounts.
 	// First account without messages.
-	accTest0, err := store.OpenAccount(c.log, "test0", false)
+	accTest0, err := store.OpenAccount(c.log, "test0")
 	xcheckf(err, "open account test0")
 	err = accTest0.ThreadingWait(c.log)
 	xcheckf(err, "wait for threading to finish")
@@ -252,7 +247,7 @@ Accounts:
 	xcheckf(err, "close account")
 
 	// Second account with one message.
-	accTest1, err := store.OpenAccount(c.log, "test1", false)
+	accTest1, err := store.OpenAccount(c.log, "test1")
 	xcheckf(err, "open account test1")
 	err = accTest1.ThreadingWait(c.log)
 	xcheckf(err, "wait for threading to finish")
@@ -263,6 +258,7 @@ Accounts:
 		m := store.Message{
 			MailboxID:          inbox.ID,
 			MailboxOrigID:      inbox.ID,
+			MailboxDestinedID:  inbox.ID,
 			RemoteIP:           "1.2.3.4",
 			RemoteIPMasked1:    "1.2.3.4",
 			RemoteIPMasked2:    "1.2.3.0",
@@ -287,13 +283,20 @@ Accounts:
 		}
 		mf := tempfile()
 		xcheckf(err, "creating temp file for delivery")
-		defer store.CloseRemoveTempFile(c.log, mf, "test message")
 		_, err = fmt.Fprint(mf, msg)
 		xcheckf(err, "writing deliver message to file")
+		err = accTest1.DeliverMessage(c.log, tx, &m, mf, false, true, false, true)
 
-		err = accTest1.MessageAdd(c.log, tx, &inbox, &m, mf, store.AddOpts{})
+		mfname := mf.Name()
-		xcheckf(err, "deliver message")
+		xcheckf(err, "add message to account test1")
+		err = mf.Close()
+		xcheckf(err, "closing file")
+		err = os.Remove(mfname)
+		xcheckf(err, "removing temp message file")
 
+		err = tx.Get(&inbox)
+		xcheckf(err, "get inbox")
+		inbox.Add(m.MailboxCounts())
 		err = tx.Update(&inbox)
 		xcheckf(err, "update inbox")
 
@@ -304,7 +307,7 @@ Accounts:
 	xcheckf(err, "close account")
 
 	// Third account with two messages and junkfilter.
-	accTest2, err := store.OpenAccount(c.log, "test2", false)
+	accTest2, err := store.OpenAccount(c.log, "test2")
 	xcheckf(err, "open account test2")
 	err = accTest2.ThreadingWait(c.log)
 	xcheckf(err, "wait for threading to finish")
@@ -315,6 +318,7 @@ Accounts:
 		m0 := store.Message{
 			MailboxID:          inbox.ID,
 			MailboxOrigID:      inbox.ID,
+			MailboxDestinedID:  inbox.ID,
 			RemoteIP:           "::1",
 			RemoteIPMasked1:    "::",
 			RemoteIPMasked2:    "::",
@@ -339,11 +343,20 @@ Accounts:
 		}
 		mf0 := tempfile()
 		xcheckf(err, "creating temp file for delivery")
-		defer store.CloseRemoveTempFile(c.log, mf0, "test message")
 		_, err = fmt.Fprint(mf0, msg0)
 		xcheckf(err, "writing deliver message to file")
-		err = accTest2.MessageAdd(c.log, tx, &inbox, &m0, mf0, store.AddOpts{})
+		err = accTest2.DeliverMessage(c.log, tx, &m0, mf0, false, false, false, true)
 		xcheckf(err, "add message to account test2")
+
+		mf0name := mf0.Name()
+		err = mf0.Close()
+		xcheckf(err, "closing file")
+		err = os.Remove(mf0name)
+		xcheckf(err, "removing temp message file")
+
+		err = tx.Get(&inbox)
+		xcheckf(err, "get inbox")
+		inbox.Add(m0.MailboxCounts())
 		err = tx.Update(&inbox)
 		xcheckf(err, "update inbox")
 
@@ -352,19 +365,29 @@ Accounts:
 		const prefix1 = "Extra: test\r\n"
 		const msg1 = "From: <other@remote.example>\r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
 		m1 := store.Message{
-			MailboxID:     sent.ID,
+			MailboxID:         sent.ID,
-			MailboxOrigID: sent.ID,
+			MailboxOrigID:     sent.ID,
-			Flags:         store.Flags{Seen: true, Junk: true},
+			MailboxDestinedID: sent.ID,
-			Size:          int64(len(prefix1) + len(msg1)),
+			Flags:             store.Flags{Seen: true, Junk: true},
-			MsgPrefix:     []byte(prefix1),
+			Size:              int64(len(prefix1) + len(msg1)),
+			MsgPrefix:         []byte(prefix1),
 		}
 		mf1 := tempfile()
 		xcheckf(err, "creating temp file for delivery")
-		defer store.CloseRemoveTempFile(c.log, mf1, "test message")
 		_, err = fmt.Fprint(mf1, msg1)
 		xcheckf(err, "writing deliver message to file")
-		err = accTest2.MessageAdd(c.log, tx, &sent, &m1, mf1, store.AddOpts{})
+		err = accTest2.DeliverMessage(c.log, tx, &m1, mf1, false, false, false, true)
 		xcheckf(err, "add message to account test2")
+
+		mf1name := mf1.Name()
+		err = mf1.Close()
+		xcheckf(err, "closing file")
+		err = os.Remove(mf1name)
+		xcheckf(err, "removing temp message file")
+
+		err = tx.Get(&sent)
+		xcheckf(err, "get sent")
+		sent.Add(m1.MailboxCounts())
 		err = tx.Update(&sent)
 		xcheckf(err, "update sent")
 

genwebsite.sh

@@ -24,7 +24,7 @@ mkdir html/features
 (
 	cat features/index.md
 	echo
-	sed -n -e 's/^# Roadmap/## Roadmap/' -e '/# FAQ/q' -e '/# Roadmap/,/# FAQ/p' < ../README.md
+	sed -n -e '/# FAQ/q' -e '/## Roadmap/,/# FAQ/p' < ../README.md
 	echo
 	echo 'Also see the [Protocols](../protocols/) page for implementation status, and (non)-plans.'
 ) | go run website.go 'Features' >html/features/index.html

go.mod

@@ -1,12 +1,11 @@
 module github.com/mjl-/mox
 
-go 1.23.0
+go 1.22.0
 
 require (
-	github.com/mjl-/adns v0.0.0-20250321173553-ab04b05bdfea
+	github.com/mjl-/adns v0.0.0-20240509092456-2dc8715bf4af
-	github.com/mjl-/autocert v0.0.0-20250321204043-abab2b936e31
+	github.com/mjl-/autocert v0.0.0-20231214125928-31b7400acb05
-	github.com/mjl-/bstore v0.0.9
+	github.com/mjl-/bstore v0.0.6
-	github.com/mjl-/flate v0.0.0-20250221133712-6372d09eb978
 	github.com/mjl-/sconf v0.0.7
 	github.com/mjl-/sherpa v0.6.7
 	github.com/mjl-/sherpadoc v0.0.16
@@ -15,10 +14,10 @@ require (
 	github.com/prometheus/client_golang v1.18.0
 	github.com/russross/blackfriday/v2 v2.1.0
 	go.etcd.io/bbolt v1.3.11
-	golang.org/x/crypto v0.37.0
+	golang.org/x/crypto v0.27.0
-	golang.org/x/net v0.39.0
+	golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f
-	golang.org/x/sys v0.32.0
+	golang.org/x/net v0.29.0
-	golang.org/x/text v0.24.0
+	golang.org/x/text v0.18.0
 	rsc.io/qr v0.2.0
 )
 
@@ -30,8 +29,9 @@ require (
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.45.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
-	golang.org/x/mod v0.24.0 // indirect
+	golang.org/x/mod v0.21.0 // indirect
-	golang.org/x/sync v0.13.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/tools v0.32.0 // indirect
+	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/tools v0.25.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 )

go.sum

@@ -24,14 +24,12 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
 github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
-github.com/mjl-/adns v0.0.0-20250321173553-ab04b05bdfea h1:8dftsVL1tHhRksXzFZRhSJ7gSlcy/t87Nvucs3JnTGE=
+github.com/mjl-/adns v0.0.0-20240509092456-2dc8715bf4af h1:sEDWZPIi5K1qKk7JQoAZyDwXkRQseIf7y5ony8JeYEQ=
-github.com/mjl-/adns v0.0.0-20250321173553-ab04b05bdfea/go.mod h1:rWZMqGA2HoBm5b5q/A5J8u1sSVuEYh6zBz9tMoVs+RU=
+github.com/mjl-/adns v0.0.0-20240509092456-2dc8715bf4af/go.mod h1:v47qUMJnipnmDTRGaHwpCwzE6oypa5K33mUvBfzZBn8=
-github.com/mjl-/autocert v0.0.0-20250321204043-abab2b936e31 h1:6MFGOLPGf6VzHWkKv8waSzJMMS98EFY2LVKPRHffCyo=
+github.com/mjl-/autocert v0.0.0-20231214125928-31b7400acb05 h1:s6ay4bh4tmpPLdxjyeWG45mcwHfEluBMuGPkqxHWUJ4=
-github.com/mjl-/autocert v0.0.0-20250321204043-abab2b936e31/go.mod h1:taMFU86abMxKLPV4Bynhv8enbYmS67b8LG80qZv2Qus=
+github.com/mjl-/autocert v0.0.0-20231214125928-31b7400acb05/go.mod h1:taMFU86abMxKLPV4Bynhv8enbYmS67b8LG80qZv2Qus=
-github.com/mjl-/bstore v0.0.9 h1:j8HVXL10Arbk4ujeRGwns8gipH1N1TZn853inQ42FgY=
+github.com/mjl-/bstore v0.0.6 h1:ntlu9MkfCkpm2XfBY4+Ws4KK9YzXzewr3+lCueFB+9c=
-github.com/mjl-/bstore v0.0.9/go.mod h1:xzIpSfcFosgPJ6h+vsdIt0pzCq4i8hhMuHPQJ0aHQhM=
+github.com/mjl-/bstore v0.0.6/go.mod h1:/cD25FNBaDfvL/plFRxI3Ba3E+wcB0XVOS8nJDqndg0=
-github.com/mjl-/flate v0.0.0-20250221133712-6372d09eb978 h1:Eg5DfI3/00URzGErujKus6a3O0kyXzF8vjoDZzH/gig=
-github.com/mjl-/flate v0.0.0-20250221133712-6372d09eb978/go.mod h1:QBkFtjai3AiQQuUu7pVh6PA06Vd3oa68E+vddf/UBOs=
 github.com/mjl-/sconf v0.0.7 h1:bdBcSFZCDFMm/UdBsgNCsjkYmKrSgYwp7rAOoufwHe4=
 github.com/mjl-/sconf v0.0.7/go.mod h1:uF8OdWtLT8La3i4ln176i1pB0ps9pXGCaABEU55ZkE0=
 github.com/mjl-/sherpa v0.6.7 h1:C5F8XQdV5nCuS4fvB+ye/ziUQrajEhOoj/t2w5T14BY=
@@ -75,33 +73,35 @@ go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
+golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
-golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
+golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
+golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY=
+golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI=
 golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
-golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
+golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0=
-golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
+golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
+golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
-golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
+golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
-golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
+golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
-golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
+golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
-golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
+golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU=
+golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE=
-golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s=
+golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

http/autoconf.go

@@ -11,8 +11,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"rsc.io/qr"
 
-	"github.com/mjl-/mox/admin"
+	"github.com/mjl-/mox/mox-"
-	"github.com/mjl-/mox/dns"
 	"github.com/mjl-/mox/smtp"
 )
 
@@ -65,35 +64,19 @@ func autoconfHandle(w http.ResponseWriter, r *http.Request) {
 
 	email := r.FormValue("emailaddress")
 	log.Debug("autoconfig request", slog.String("email", email))
-	var domain dns.Domain
+	addr, err := smtp.ParseAddress(email)
-	if email == "" {
+	if err != nil {
-		email = "%EMAILADDRESS%"
+		http.Error(w, "400 - bad request - invalid parameter emailaddress", http.StatusBadRequest)
-		// Declare this here rather than using := to avoid shadowing domain from
+		return
-		// the outer scope.
-		var err error
-		domain, err = dns.ParseDomain(r.Host)
-		if err != nil {
-			http.Error(w, fmt.Sprintf("400 - bad request - invalid domain: %s", r.Host), http.StatusBadRequest)
-			return
-		}
-		domain.ASCII = strings.TrimPrefix(domain.ASCII, "autoconfig.")
-		domain.Unicode = strings.TrimPrefix(domain.Unicode, "autoconfig.")
-	} else {
-		addr, err := smtp.ParseAddress(email)
-		if err != nil {
-			http.Error(w, "400 - bad request - invalid parameter emailaddress", http.StatusBadRequest)
-			return
-		}
-		domain = addr.Domain
 	}
 
-	socketType := func(tlsMode admin.TLSMode) (string, error) {
+	socketType := func(tlsMode mox.TLSMode) (string, error) {
 		switch tlsMode {
-		case admin.TLSModeImmediate:
+		case mox.TLSModeImmediate:
 			return "SSL", nil
-		case admin.TLSModeSTARTTLS:
+		case mox.TLSModeSTARTTLS:
 			return "STARTTLS", nil
-		case admin.TLSModeNone:
+		case mox.TLSModeNone:
 			return "plain", nil
 		default:
 			return "", fmt.Errorf("unknown tls mode %v", tlsMode)
@@ -101,7 +84,7 @@ func autoconfHandle(w http.ResponseWriter, r *http.Request) {
 	}
 
 	var imapTLS, submissionTLS string
-	config, err := admin.ClientConfigDomain(domain)
+	config, err := mox.ClientConfigDomain(addr.Domain)
 	if err == nil {
 		imapTLS, err = socketType(config.IMAP.TLSMode)
 	}
@@ -116,67 +99,37 @@ func autoconfHandle(w http.ResponseWriter, r *http.Request) {
 	// Thunderbird doesn't seem to allow U-labels, always return ASCII names.
 	var resp autoconfigResponse
 	resp.Version = "1.1"
-	resp.EmailProvider.ID = domain.ASCII
+	resp.EmailProvider.ID = addr.Domain.ASCII
-	resp.EmailProvider.Domain = domain.ASCII
+	resp.EmailProvider.Domain = addr.Domain.ASCII
 	resp.EmailProvider.DisplayName = email
-	resp.EmailProvider.DisplayShortName = domain.ASCII
+	resp.EmailProvider.DisplayShortName = addr.Domain.ASCII
 
 	// todo: specify SCRAM-SHA-256 once thunderbird and autoconfig supports it. or perhaps that will fall under "password-encrypted" by then.
-	// todo: let user configure they prefer or require tls client auth and specify "TLS-client-cert"
 
-	incoming := incomingServer{
+	resp.EmailProvider.IncomingServer.Type = "imap"
-		"imap",
+	resp.EmailProvider.IncomingServer.Hostname = config.IMAP.Host.ASCII
-		config.IMAP.Host.ASCII,
+	resp.EmailProvider.IncomingServer.Port = config.IMAP.Port
-		config.IMAP.Port,
+	resp.EmailProvider.IncomingServer.SocketType = imapTLS
-		imapTLS,
+	resp.EmailProvider.IncomingServer.Username = email
-		email,
+	resp.EmailProvider.IncomingServer.Authentication = "password-encrypted"
-		"password-encrypted",
-	}
-	resp.EmailProvider.IncomingServers = append(resp.EmailProvider.IncomingServers, incoming)
-	if config.IMAP.EnabledOnHTTPS {
-		tlsMode, _ := socketType(admin.TLSModeImmediate)
-		incomingALPN := incomingServer{
-			"imap",
-			config.IMAP.Host.ASCII,
-			443,
-			tlsMode,
-			email,
-			"password-encrypted",
-		}
-		resp.EmailProvider.IncomingServers = append(resp.EmailProvider.IncomingServers, incomingALPN)
-	}
 
-	outgoing := outgoingServer{
+	resp.EmailProvider.OutgoingServer.Type = "smtp"
-		"smtp",
+	resp.EmailProvider.OutgoingServer.Hostname = config.Submission.Host.ASCII
-		config.Submission.Host.ASCII,
+	resp.EmailProvider.OutgoingServer.Port = config.Submission.Port
-		config.Submission.Port,
+	resp.EmailProvider.OutgoingServer.SocketType = submissionTLS
-		submissionTLS,
+	resp.EmailProvider.OutgoingServer.Username = email
-		email,
+	resp.EmailProvider.OutgoingServer.Authentication = "password-encrypted"
-		"password-encrypted",
-	}
-	resp.EmailProvider.OutgoingServers = append(resp.EmailProvider.OutgoingServers, outgoing)
-	if config.Submission.EnabledOnHTTPS {
-		tlsMode, _ := socketType(admin.TLSModeImmediate)
-		outgoingALPN := outgoingServer{
-			"smtp",
-			config.Submission.Host.ASCII,
-			443,
-			tlsMode,
-			email,
-			"password-encrypted",
-		}
-		resp.EmailProvider.OutgoingServers = append(resp.EmailProvider.OutgoingServers, outgoingALPN)
-	}
 
 	// todo: should we put the email address in the URL?
-	resp.ClientConfigUpdate.URL = fmt.Sprintf("https://autoconfig.%s/mail/config-v1.1.xml", domain.ASCII)
+	resp.ClientConfigUpdate.URL = fmt.Sprintf("https://autoconfig.%s/mail/config-v1.1.xml", addr.Domain.ASCII)
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	enc := xml.NewEncoder(w)
 	enc.Indent("", "\t")
 	fmt.Fprint(w, xml.Header)
-	err = enc.Encode(resp)
+	if err := enc.Encode(resp); err != nil {
-	log.Check(err, "write autoconfig xml response")
+		log.Errorx("marshal autoconfig response", err)
+	}
 }
 
 // Autodiscover from Microsoft, also used by Thunderbird.
@@ -217,13 +170,13 @@ func autodiscoverHandle(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// tlsmode returns the "ssl" and "encryption" fields.
-	tlsmode := func(tlsMode admin.TLSMode) (string, string, error) {
+	tlsmode := func(tlsMode mox.TLSMode) (string, string, error) {
 		switch tlsMode {
-		case admin.TLSModeImmediate:
+		case mox.TLSModeImmediate:
 			return "on", "TLS", nil
-		case admin.TLSModeSTARTTLS:
+		case mox.TLSModeSTARTTLS:
 			return "on", "", nil
-		case admin.TLSModeNone:
+		case mox.TLSModeNone:
 			return "off", "", nil
 		default:
 			return "", "", fmt.Errorf("unknown tls mode %v", tlsMode)
@@ -232,7 +185,7 @@ func autodiscoverHandle(w http.ResponseWriter, r *http.Request) {
 
 	var imapSSL, imapEncryption string
 	var submissionSSL, submissionEncryption string
-	config, err := admin.ClientConfigDomain(addr.Domain)
+	config, err := mox.ClientConfigDomain(addr.Domain)
 	if err == nil {
 		imapSSL, imapEncryption, err = tlsmode(config.IMAP.TLSMode)
 	}
@@ -255,8 +208,6 @@ func autodiscoverHandle(w http.ResponseWriter, r *http.Request) {
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 
-	// todo: let user configure they prefer or require tls client auth and add "AuthPackage" with value "certificate" to Protocol? see https://learn.microsoft.com/en-us/openspecs/exchange_server_protocols/ms-oxdscli/21fd2dd5-c4ee-485b-94fb-e7db5da93726
-
 	resp := autodiscoverResponse{}
 	resp.XMLName.Local = "Autodiscover"
 	resp.XMLName.Space = "http://schemas.microsoft.com/exchange/autodiscover/responseschema/2006"
@@ -291,8 +242,9 @@ func autodiscoverHandle(w http.ResponseWriter, r *http.Request) {
 	enc := xml.NewEncoder(w)
 	enc.Indent("", "\t")
 	fmt.Fprint(w, xml.Header)
-	err = enc.Encode(resp)
+	if err := enc.Encode(resp); err != nil {
-	log.Check(err, "marshal autodiscover xml response")
+		log.Errorx("marshal autodiscover response", err)
+	}
 }
 
 // Thunderbird requests these URLs for autoconfig/autodiscover:
@@ -300,22 +252,6 @@ func autodiscoverHandle(w http.ResponseWriter, r *http.Request) {
 // https://autodiscover.example.org/autodiscover/autodiscover.xml
 // https://example.org/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress=user%40example.org
 // https://example.org/autodiscover/autodiscover.xml
-type incomingServer struct {
-	Type           string `xml:"type,attr"`
-	Hostname       string `xml:"hostname"`
-	Port           int    `xml:"port"`
-	SocketType     string `xml:"socketType"`
-	Username       string `xml:"username"`
-	Authentication string `xml:"authentication"`
-}
-type outgoingServer struct {
-	Type           string `xml:"type,attr"`
-	Hostname       string `xml:"hostname"`
-	Port           int    `xml:"port"`
-	SocketType     string `xml:"socketType"`
-	Username       string `xml:"username"`
-	Authentication string `xml:"authentication"`
-}
 type autoconfigResponse struct {
 	XMLName xml.Name `xml:"clientConfig"`
 	Version string   `xml:"version,attr"`
@@ -326,8 +262,23 @@ type autoconfigResponse struct {
 		DisplayName      string `xml:"displayName"`
 		DisplayShortName string `xml:"displayShortName"`
 
-		IncomingServers []incomingServer `xml:"incomingServer"`
+		IncomingServer struct {
-		OutgoingServers []outgoingServer `xml:"outgoingServer"`
+			Type           string `xml:"type,attr"`
+			Hostname       string `xml:"hostname"`
+			Port           int    `xml:"port"`
+			SocketType     string `xml:"socketType"`
+			Username       string `xml:"username"`
+			Authentication string `xml:"authentication"`
+		} `xml:"incomingServer"`
+
+		OutgoingServer struct {
+			Type           string `xml:"type,attr"`
+			Hostname       string `xml:"hostname"`
+			Port           int    `xml:"port"`
+			SocketType     string `xml:"socketType"`
+			Username       string `xml:"username"`
+			Authentication string `xml:"authentication"`
+		} `xml:"outgoingServer"`
 	} `xml:"emailProvider"`
 
 	ClientConfigUpdate struct {
@@ -373,8 +324,6 @@ type autodiscoverProtocol struct {
 // Serve a .mobileconfig file. This endpoint is not a standard place where Apple
 // devices look. We point to it from the account page.
 func mobileconfigHandle(w http.ResponseWriter, r *http.Request) {
-	log := pkglog.WithContext(r.Context())
-
 	if r.Method != "GET" {
 		http.Error(w, "405 - method not allowed - get required", http.StatusMethodNotAllowed)
 		return
@@ -400,15 +349,12 @@ func mobileconfigHandle(w http.ResponseWriter, r *http.Request) {
 	filename = strings.ReplaceAll(filename, "@", "-at-")
 	filename = "email-account-" + filename + ".mobileconfig"
 	h.Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filename))
-	_, err = w.Write(buf)
+	w.Write(buf)
-	log.Check(err, "writing mobileconfig response")
 }
 
 // Serve a png file with qrcode with the link to the .mobileconfig file, should be
 // helpful for mobile devices.
 func mobileconfigQRCodeHandle(w http.ResponseWriter, r *http.Request) {
-	log := pkglog.WithContext(r.Context())
-
 	if r.Method != "GET" {
 		http.Error(w, "405 - method not allowed - get required", http.StatusMethodNotAllowed)
 		return
@@ -435,6 +381,5 @@ func mobileconfigQRCodeHandle(w http.ResponseWriter, r *http.Request) {
 	}
 	h := w.Header()
 	h.Set("Content-Type", "image/png")
-	_, err = w.Write(code.PNG())
+	w.Write(code.PNG())
-	log.Check(err, "writing mobileconfig qr code")
 }

http/main_test.go

@@ -1,17 +0,0 @@
-package http
-
-import (
-	"fmt"
-	"os"
-	"testing"
-
-	"github.com/mjl-/mox/metrics"
-)
-
-func TestMain(m *testing.M) {
-	m.Run()
-	if metrics.Panics.Load() > 0 {
-		fmt.Println("unhandled panics encountered")
-		os.Exit(2)
-	}
-}

http/mobileconfig.go

@@ -6,11 +6,13 @@ import (
 	"crypto/sha256"
 	"encoding/xml"
 	"fmt"
-	"maps"
 	"slices"
+	"sort"
 	"strings"
 
-	"github.com/mjl-/mox/admin"
+	"golang.org/x/exp/maps"
+
+	"github.com/mjl-/mox/mox-"
 	"github.com/mjl-/mox/smtp"
 )
 
@@ -37,7 +39,8 @@ func (m dict) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	if err := e.EncodeToken(xml.StartElement{Name: xml.Name{Local: "dict"}}); err != nil {
 		return err
 	}
-	l := slices.Sorted(maps.Keys(m))
+	l := maps.Keys(m)
+	sort.Strings(l)
 	for _, k := range l {
 		tokens := []xml.Token{
 			xml.StartElement{Name: xml.Name{Local: "key"}},
@@ -61,7 +64,7 @@ func (m dict) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 		case int:
 			tokens = []xml.Token{
 				xml.StartElement{Name: xml.Name{Local: "integer"}},
-				xml.CharData(fmt.Appendf(nil, "%d", v)),
+				xml.CharData([]byte(fmt.Sprintf("%d", v))),
 				xml.EndElement{Name: xml.Name{Local: "integer"}},
 			}
 		case bool:
@@ -119,7 +122,7 @@ func MobileConfig(addresses []string, fullName string) ([]byte, error) {
 		return nil, fmt.Errorf("parsing address: %v", err)
 	}
 
-	config, err := admin.ClientConfigDomain(addr.Domain)
+	config, err := mox.ClientConfigDomain(addr.Domain)
 	if err != nil {
 		return nil, fmt.Errorf("getting config for domain: %v", err)
 	}
@@ -172,12 +175,12 @@ func MobileConfig(addresses []string, fullName string) ([]byte, error) {
 					"IncomingMailServerUsername":             addresses[0],
 					"IncomingMailServerHostName":             config.IMAP.Host.ASCII,
 					"IncomingMailServerPortNumber":           config.IMAP.Port,
-					"IncomingMailServerUseSSL":               config.IMAP.TLSMode == admin.TLSModeImmediate,
+					"IncomingMailServerUseSSL":               config.IMAP.TLSMode == mox.TLSModeImmediate,
 					"OutgoingMailServerAuthentication":       "EmailAuthCRAMMD5", // SCRAM not an option at time of writing...
 					"OutgoingMailServerHostName":             config.Submission.Host.ASCII,
 					"OutgoingMailServerPortNumber":           config.Submission.Port,
 					"OutgoingMailServerUsername":             addresses[0],
-					"OutgoingMailServerUseSSL":               config.Submission.TLSMode == admin.TLSModeImmediate,
+					"OutgoingMailServerUseSSL":               config.Submission.TLSMode == mox.TLSModeImmediate,
 					"OutgoingPasswordSameAsIncomingPassword": true,
 					"PayloadIdentifier":                      reverseAddr + ".email.account",
 					"PayloadType":                            "com.apple.mail.managed",

http/web.go

@@ -11,12 +11,10 @@ import (
 	"io"
 	golog "log"
 	"log/slog"
-	"maps"
 	"net"
 	"net/http"
 	"os"
 	"path"
-	"slices"
 	"sort"
 	"strings"
 	"time"
@@ -24,7 +22,7 @@ import (
 	_ "embed"
 	_ "net/http/pprof"
 
-	"golang.org/x/net/http2"
+	"golang.org/x/exp/maps"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
@@ -33,11 +31,9 @@ import (
 	"github.com/mjl-/mox/autotls"
 	"github.com/mjl-/mox/config"
 	"github.com/mjl-/mox/dns"
-	"github.com/mjl-/mox/imapserver"
 	"github.com/mjl-/mox/mlog"
 	"github.com/mjl-/mox/mox-"
 	"github.com/mjl-/mox/ratelimit"
-	"github.com/mjl-/mox/smtpserver"
 	"github.com/mjl-/mox/webaccount"
 	"github.com/mjl-/mox/webadmin"
 	"github.com/mjl-/mox/webapisrv"
@@ -352,7 +348,7 @@ func (w *loggingWriter) Done() {
 		slog.Any("remoteaddr", w.R.RemoteAddr),
 		slog.String("tlsinfo", tlsinfo),
 		slog.String("useragent", w.R.Header.Get("User-Agent")),
-		slog.String("referer", w.R.Header.Get("Referer")),
+		slog.String("referrr", w.R.Header.Get("Referrer")),
 	}
 	if w.WebsocketRequest {
 		attrs = append(attrs,
@@ -386,14 +382,10 @@ type pathHandler struct {
 	Path      string                       // Path to register, like on http.ServeMux.
 	Handler   http.Handler
 }
-
 type serve struct {
-	Kinds             []string // Type of handler and protocol (e.g. acme-tls-alpn-01, account-http, admin-https, imap-https, smtp-https).
+	Kinds     []string // Type of handler and protocol (e.g. acme-tls-alpn-01, account-http, admin-https).
-	TLSConfig         *tls.Config
+	TLSConfig *tls.Config
-	NextProto         tlsNextProtoMap // For HTTP server, when we do submission/imap with ALPN over the HTTPS port.
+	Favicon   bool
-	Favicon           bool
-	Forwarded         bool // Requests are coming from a reverse proxy, we'll use X-Forwarded-For for the IP address to ratelimit.
-	RateLimitDisabled bool // Don't apply ratelimiting.
 
 	// SystemHandlers are for MTA-STS, autoconfig, ACME validation. They can't be
 	// overridden by WebHandlers. WebHandlers are evaluated next, and the internal
@@ -440,41 +432,23 @@ var (
 // metrics.
 func (s *serve) ServeHTTP(xw http.ResponseWriter, r *http.Request) {
 	now := time.Now()
+	// Rate limiting as early as possible.
+	ipstr, _, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		pkglog.Debugx("split host:port client remoteaddr", err, slog.Any("remoteaddr", r.RemoteAddr))
+	} else if ip := net.ParseIP(ipstr); ip == nil {
+		pkglog.Debug("parsing ip for client remoteaddr", slog.Any("remoteaddr", r.RemoteAddr))
+	} else if !limiterConnectionrate.Add(ip, now, 1) {
+		method := metricHTTPMethod(r.Method)
+		proto := "http"
+		if r.TLS != nil {
+			proto = "https"
+		}
+		metricRequest.WithLabelValues("(ratelimited)", proto, method, "429").Observe(0)
+		// No logging, that's just noise.
 
-	// Rate limiting as early as possible, if enabled.
+		http.Error(xw, "429 - too many auth attempts", http.StatusTooManyRequests)
-	if !s.RateLimitDisabled {
+		return
-		// If requests are coming from a reverse proxy, use the IP from X-Forwarded-For.
-		// Otherwise the remote IP for this connection.
-		var ipstr string
-		if s.Forwarded {
-			s := r.Header.Get("X-Forwarded-For")
-			ipstr = strings.TrimSpace(strings.Split(s, ",")[0])
-			if ipstr == "" {
-				pkglog.Debug("ratelimit: no ip address in X-Forwarded-For header")
-			}
-		} else {
-			var err error
-			ipstr, _, err = net.SplitHostPort(r.RemoteAddr)
-			if err != nil {
-				pkglog.Debugx("ratelimit: parsing remote address", err, slog.String("remoteaddr", r.RemoteAddr))
-			}
-		}
-		ip := net.ParseIP(ipstr)
-		if ip == nil && ipstr != "" {
-			pkglog.Debug("ratelimit: invalid ip", slog.String("ip", ipstr))
-		}
-		if ip != nil && !limiterConnectionrate.Add(ip, now, 1) {
-			method := metricHTTPMethod(r.Method)
-			proto := "http"
-			if r.TLS != nil {
-				proto = "https"
-			}
-			metricRequest.WithLabelValues("(ratelimited)", proto, method, "429").Observe(0)
-			// No logging, that's just noise.
-
-			http.Error(xw, "429 - too many auth attempts", http.StatusTooManyRequests)
-			return
-		}
 	}
 
 	ctx := context.WithValue(r.Context(), mlog.CidKey, mox.Cid())
@@ -557,7 +531,7 @@ func redirectToTrailingSlash(srv *serve, hostMatch func(dns.IPDomain) bool, name
 		handler := mox.SafeHeaders(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			http.Redirect(w, r, path, http.StatusSeeOther)
 		}))
-		srv.ServiceHandle(name, hostMatch, strings.TrimRight(path, "/"), handler)
+		srv.ServiceHandle(name, hostMatch, path[:len(path)-1], handler)
 	}
 }
 
@@ -566,22 +540,24 @@ func redirectToTrailingSlash(srv *serve, hostMatch func(dns.IPDomain) bool, name
 func Listen() {
 	// Initialize listeners in deterministic order for the same potential error
 	// messages.
-	names := slices.Sorted(maps.Keys(mox.Conf.Static.Listeners))
+	names := maps.Keys(mox.Conf.Static.Listeners)
+	sort.Strings(names)
 	for _, name := range names {
 		l := mox.Conf.Static.Listeners[name]
-		portServe := portServes(name, l)
+		portServe := portServes(l)
 
-		ports := slices.Sorted(maps.Keys(portServe))
+		ports := maps.Keys(portServe)
+		sort.Ints(ports)
 		for _, port := range ports {
 			srv := portServe[port]
 			for _, ip := range l.IPs {
-				listen1(ip, port, srv.TLSConfig, name, srv.Kinds, srv, srv.NextProto)
+				listen1(ip, port, srv.TLSConfig, name, srv.Kinds, srv)
 			}
 		}
 	}
 }
 
-func portServes(name string, l config.Listener) map[int]*serve {
+func portServes(l config.Listener) map[int]*serve {
 	portServe := map[int]*serve{}
 
 	// For system/services, we serve on host localhost too, for ssh tunnel scenario's.
@@ -604,11 +580,11 @@ func portServes(name string, l config.Listener) map[int]*serve {
 		return mox.Conf.IsClientSettingsDomain(host.Domain)
 	}
 
-	var ensureServe func(https, forwarded, noRateLimiting bool, port int, kind string, favicon bool) *serve
+	var ensureServe func(https bool, port int, kind string, favicon bool) *serve
-	ensureServe = func(https, forwarded, rateLimitDisabled bool, port int, kind string, favicon bool) *serve {
+	ensureServe = func(https bool, port int, kind string, favicon bool) *serve {
 		s := portServe[port]
 		if s == nil {
-			s = &serve{nil, nil, tlsNextProtoMap{}, false, false, false, nil, false, nil}
+			s = &serve{nil, nil, false, nil, false, nil}
 			portServe[port] = s
 		}
 		s.Kinds = append(s.Kinds, kind)
@@ -616,73 +592,34 @@ func portServes(name string, l config.Listener) map[int]*serve {
 			s.ServiceHandle("favicon", accountHostMatch, "/favicon.ico", mox.SafeHeaders(http.HandlerFunc(faviconHandle)))
 			s.Favicon = true
 		}
-		s.Forwarded = s.Forwarded || forwarded
-		s.RateLimitDisabled = s.RateLimitDisabled || rateLimitDisabled
 
-		// We clone TLS configs because we may modify it later on for this server, for
-		// ALPN. And we need copies because multiple listeners on http.Server where the
-		// config is used will try to modify it concurrently.
 		if https && l.TLS.ACME != "" {
-			s.TLSConfig = l.TLS.ACMEConfig.Clone()
+			s.TLSConfig = l.TLS.ACMEConfig
-
-			tlsport := config.Port(mox.Conf.Static.ACME[l.TLS.ACME].Port, 443)
-			if portServe[tlsport] == nil || !slices.Contains(portServe[tlsport].Kinds, "acme-tls-alpn-01") {
-				ensureServe(true, false, false, tlsport, "acme-tls-alpn-01", false)
-			}
 		} else if https {
-			s.TLSConfig = l.TLS.Config.Clone()
+			s.TLSConfig = l.TLS.Config
+			if l.TLS.ACME != "" {
+				tlsport := config.Port(mox.Conf.Static.ACME[l.TLS.ACME].Port, 443)
+				ensureServe(true, tlsport, "acme-tls-alpn-01", false)
+			}
 		}
 		return s
 	}
 
-	// If TLS with ACME is enabled on this plain HTTP port, and it hasn't been enabled
-	// yet, add http-01 validation mechanism handler to server.
-	ensureACMEHTTP01 := func(srv *serve) {
-		if l.TLS != nil && l.TLS.ACME != "" && !slices.Contains(srv.Kinds, "acme-http-01") {
-			m := mox.Conf.Static.ACME[l.TLS.ACME].Manager
-			srv.Kinds = append(srv.Kinds, "acme-http-01")
-			srv.SystemHandle("acme-http-01", nil, "/.well-known/acme-challenge/", m.Manager.HTTPHandler(nil))
-		}
-	}
-
 	if l.TLS != nil && l.TLS.ACME != "" && (l.SMTP.Enabled && !l.SMTP.NoSTARTTLS || l.Submissions.Enabled || l.IMAPS.Enabled) {
 		port := config.Port(mox.Conf.Static.ACME[l.TLS.ACME].Port, 443)
-		ensureServe(true, false, false, port, "acme-tls-alpn-01", false)
+		ensureServe(true, port, "acme-tls-alpn-01", false)
 	}
-	if l.Submissions.Enabled && l.Submissions.EnabledOnHTTPS {
-		s := ensureServe(true, false, false, 443, "smtp-https", false)
-		hostname := mox.Conf.Static.HostnameDomain
-		if l.Hostname != "" {
-			hostname = l.HostnameDomain
-		}
 
-		maxMsgSize := l.SMTPMaxMessageSize
-		if maxMsgSize == 0 {
-			maxMsgSize = config.DefaultMaxMsgSize
-		}
-		requireTLS := !l.SMTP.NoRequireTLS
-
-		s.NextProto["smtp"] = func(_ *http.Server, conn *tls.Conn, _ http.Handler) {
-			smtpserver.ServeTLSConn(name, hostname, conn, s.TLSConfig, true, true, maxMsgSize, requireTLS)
-		}
-	}
-	if l.IMAPS.Enabled && l.IMAPS.EnabledOnHTTPS {
-		s := ensureServe(true, false, false, 443, "imap-https", false)
-		s.NextProto["imap"] = func(_ *http.Server, conn *tls.Conn, _ http.Handler) {
-			imapserver.ServeTLSConn(name, conn, s.TLSConfig)
-		}
-	}
 	if l.AccountHTTP.Enabled {
 		port := config.Port(l.AccountHTTP.Port, 80)
 		path := "/"
 		if l.AccountHTTP.Path != "" {
 			path = l.AccountHTTP.Path
 		}
-		srv := ensureServe(false, l.AccountHTTP.Forwarded, false, port, "account-http at "+path, true)
+		srv := ensureServe(false, port, "account-http at "+path, true)
-		handler := mox.SafeHeaders(http.StripPrefix(strings.TrimRight(path, "/"), http.HandlerFunc(webaccount.Handler(path, l.AccountHTTP.Forwarded))))
+		handler := mox.SafeHeaders(http.StripPrefix(path[:len(path)-1], http.HandlerFunc(webaccount.Handler(path, l.AccountHTTP.Forwarded))))
 		srv.ServiceHandle("account", accountHostMatch, path, handler)
 		redirectToTrailingSlash(srv, accountHostMatch, "account", path)
-		ensureACMEHTTP01(srv)
 	}
 	if l.AccountHTTPS.Enabled {
 		port := config.Port(l.AccountHTTPS.Port, 443)
@@ -690,8 +627,8 @@ func portServes(name string, l config.Listener) map[int]*serve {
 		if l.AccountHTTPS.Path != "" {
 			path = l.AccountHTTPS.Path
 		}
-		srv := ensureServe(true, l.AccountHTTPS.Forwarded, false, port, "account-https at "+path, true)
+		srv := ensureServe(true, port, "account-https at "+path, true)
-		handler := mox.SafeHeaders(http.StripPrefix(strings.TrimRight(path, "/"), http.HandlerFunc(webaccount.Handler(path, l.AccountHTTPS.Forwarded))))
+		handler := mox.SafeHeaders(http.StripPrefix(path[:len(path)-1], http.HandlerFunc(webaccount.Handler(path, l.AccountHTTPS.Forwarded))))
 		srv.ServiceHandle("account", accountHostMatch, path, handler)
 		redirectToTrailingSlash(srv, accountHostMatch, "account", path)
 	}
@@ -702,11 +639,10 @@ func portServes(name string, l config.Listener) map[int]*serve {
 		if l.AdminHTTP.Path != "" {
 			path = l.AdminHTTP.Path
 		}
-		srv := ensureServe(false, l.AdminHTTP.Forwarded, false, port, "admin-http at "+path, true)
+		srv := ensureServe(false, port, "admin-http at "+path, true)
-		handler := mox.SafeHeaders(http.StripPrefix(strings.TrimRight(path, "/"), http.HandlerFunc(webadmin.Handler(path, l.AdminHTTP.Forwarded))))
+		handler := mox.SafeHeaders(http.StripPrefix(path[:len(path)-1], http.HandlerFunc(webadmin.Handler(path, l.AdminHTTP.Forwarded))))
 		srv.ServiceHandle("admin", listenerHostMatch, path, handler)
 		redirectToTrailingSlash(srv, listenerHostMatch, "admin", path)
-		ensureACMEHTTP01(srv)
 	}
 	if l.AdminHTTPS.Enabled {
 		port := config.Port(l.AdminHTTPS.Port, 443)
@@ -714,8 +650,8 @@ func portServes(name string, l config.Listener) map[int]*serve {
 		if l.AdminHTTPS.Path != "" {
 			path = l.AdminHTTPS.Path
 		}
-		srv := ensureServe(true, l.AdminHTTPS.Forwarded, false, port, "admin-https at "+path, true)
+		srv := ensureServe(true, port, "admin-https at "+path, true)
-		handler := mox.SafeHeaders(http.StripPrefix(strings.TrimRight(path, "/"), http.HandlerFunc(webadmin.Handler(path, l.AdminHTTPS.Forwarded))))
+		handler := mox.SafeHeaders(http.StripPrefix(path[:len(path)-1], http.HandlerFunc(webadmin.Handler(path, l.AdminHTTPS.Forwarded))))
 		srv.ServiceHandle("admin", listenerHostMatch, path, handler)
 		redirectToTrailingSlash(srv, listenerHostMatch, "admin", path)
 	}
@@ -731,11 +667,10 @@ func portServes(name string, l config.Listener) map[int]*serve {
 		if l.WebAPIHTTP.Path != "" {
 			path = l.WebAPIHTTP.Path
 		}
-		srv := ensureServe(false, l.WebAPIHTTP.Forwarded, false, port, "webapi-http at "+path, true)
+		srv := ensureServe(false, port, "webapi-http at "+path, true)
-		handler := mox.SafeHeaders(http.StripPrefix(strings.TrimRight(path, "/"), webapisrv.NewServer(maxMsgSize, path, l.WebAPIHTTP.Forwarded)))
+		handler := mox.SafeHeaders(http.StripPrefix(path[:len(path)-1], webapisrv.NewServer(maxMsgSize, path, l.WebAPIHTTP.Forwarded)))
 		srv.ServiceHandle("webapi", accountHostMatch, path, handler)
 		redirectToTrailingSlash(srv, accountHostMatch, "webapi", path)
-		ensureACMEHTTP01(srv)
 	}
 	if l.WebAPIHTTPS.Enabled {
 		port := config.Port(l.WebAPIHTTPS.Port, 443)
@@ -743,8 +678,8 @@ func portServes(name string, l config.Listener) map[int]*serve {
 		if l.WebAPIHTTPS.Path != "" {
 			path = l.WebAPIHTTPS.Path
 		}
-		srv := ensureServe(true, l.WebAPIHTTPS.Forwarded, false, port, "webapi-https at "+path, true)
+		srv := ensureServe(true, port, "webapi-https at "+path, true)
-		handler := mox.SafeHeaders(http.StripPrefix(strings.TrimRight(path, "/"), webapisrv.NewServer(maxMsgSize, path, l.WebAPIHTTPS.Forwarded)))
+		handler := mox.SafeHeaders(http.StripPrefix(path[:len(path)-1], webapisrv.NewServer(maxMsgSize, path, l.WebAPIHTTPS.Forwarded)))
 		srv.ServiceHandle("webapi", accountHostMatch, path, handler)
 		redirectToTrailingSlash(srv, accountHostMatch, "webapi", path)
 	}
@@ -755,7 +690,7 @@ func portServes(name string, l config.Listener) map[int]*serve {
 		if l.WebmailHTTP.Path != "" {
 			path = l.WebmailHTTP.Path
 		}
-		srv := ensureServe(false, l.WebmailHTTP.Forwarded, false, port, "webmail-http at "+path, true)
+		srv := ensureServe(false, port, "webmail-http at "+path, true)
 		var accountPath string
 		if l.AccountHTTP.Enabled {
 			accountPath = "/"
@@ -763,10 +698,9 @@ func portServes(name string, l config.Listener) map[int]*serve {
 				accountPath = l.AccountHTTP.Path
 			}
 		}
-		handler := http.StripPrefix(strings.TrimRight(path, "/"), http.HandlerFunc(webmail.Handler(maxMsgSize, path, l.WebmailHTTP.Forwarded, accountPath)))
+		handler := http.StripPrefix(path[:len(path)-1], http.HandlerFunc(webmail.Handler(maxMsgSize, path, l.WebmailHTTP.Forwarded, accountPath)))
 		srv.ServiceHandle("webmail", accountHostMatch, path, handler)
 		redirectToTrailingSlash(srv, accountHostMatch, "webmail", path)
-		ensureACMEHTTP01(srv)
 	}
 	if l.WebmailHTTPS.Enabled {
 		port := config.Port(l.WebmailHTTPS.Port, 443)
@@ -774,7 +708,7 @@ func portServes(name string, l config.Listener) map[int]*serve {
 		if l.WebmailHTTPS.Path != "" {
 			path = l.WebmailHTTPS.Path
 		}
-		srv := ensureServe(true, l.WebmailHTTPS.Forwarded, false, port, "webmail-https at "+path, true)
+		srv := ensureServe(true, port, "webmail-https at "+path, true)
 		var accountPath string
 		if l.AccountHTTPS.Enabled {
 			accountPath = "/"
@@ -782,14 +716,14 @@ func portServes(name string, l config.Listener) map[int]*serve {
 				accountPath = l.AccountHTTPS.Path
 			}
 		}
-		handler := http.StripPrefix(strings.TrimRight(path, "/"), http.HandlerFunc(webmail.Handler(maxMsgSize, path, l.WebmailHTTPS.Forwarded, accountPath)))
+		handler := http.StripPrefix(path[:len(path)-1], http.HandlerFunc(webmail.Handler(maxMsgSize, path, l.WebmailHTTPS.Forwarded, accountPath)))
 		srv.ServiceHandle("webmail", accountHostMatch, path, handler)
 		redirectToTrailingSlash(srv, accountHostMatch, "webmail", path)
 	}
 
 	if l.MetricsHTTP.Enabled {
 		port := config.Port(l.MetricsHTTP.Port, 8010)
-		srv := ensureServe(false, false, false, port, "metrics-http", false)
+		srv := ensureServe(false, port, "metrics-http", false)
 		srv.SystemHandle("metrics", nil, "/metrics", mox.SafeHeaders(promhttp.Handler()))
 		srv.SystemHandle("metrics", nil, "/", mox.SafeHeaders(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			if r.URL.Path != "/" {
@@ -805,10 +739,7 @@ func portServes(name string, l config.Listener) map[int]*serve {
 	}
 	if l.AutoconfigHTTPS.Enabled {
 		port := config.Port(l.AutoconfigHTTPS.Port, 443)
-		srv := ensureServe(!l.AutoconfigHTTPS.NonTLS, false, false, port, "autoconfig-https", false)
+		srv := ensureServe(!l.AutoconfigHTTPS.NonTLS, port, "autoconfig-https", false)
-		if l.AutoconfigHTTPS.NonTLS {
-			ensureACMEHTTP01(srv)
-		}
 		autoconfigMatch := func(ipdom dns.IPDomain) bool {
 			dom := ipdom.Domain
 			if dom.IsZero() {
@@ -835,10 +766,7 @@ func portServes(name string, l config.Listener) map[int]*serve {
 	}
 	if l.MTASTSHTTPS.Enabled {
 		port := config.Port(l.MTASTSHTTPS.Port, 443)
-		srv := ensureServe(!l.MTASTSHTTPS.NonTLS, false, false, port, "mtasts-https", false)
+		srv := ensureServe(!l.MTASTSHTTPS.NonTLS, port, "mtasts-https", false)
-		if l.MTASTSHTTPS.NonTLS {
-			ensureACMEHTTP01(srv)
-		}
 		mtastsMatch := func(ipdom dns.IPDomain) bool {
 			// todo: may want to check this against the configured domains, could in theory be just a webserver.
 			dom := ipdom.Domain
@@ -855,59 +783,56 @@ func portServes(name string, l config.Listener) map[int]*serve {
 		if _, ok := portServe[port]; ok {
 			pkglog.Fatal("cannot serve pprof on same endpoint as other http services")
 		}
-		srv := &serve{[]string{"pprof-http"}, nil, nil, false, false, false, nil, false, nil}
+		srv := &serve{[]string{"pprof-http"}, nil, false, nil, false, nil}
 		portServe[port] = srv
 		srv.SystemHandle("pprof", nil, "/", http.DefaultServeMux)
 	}
 	if l.WebserverHTTP.Enabled {
 		port := config.Port(l.WebserverHTTP.Port, 80)
-		srv := ensureServe(false, false, l.WebserverHTTP.RateLimitDisabled, port, "webserver-http", false)
+		srv := ensureServe(false, port, "webserver-http", false)
 		srv.Webserver = true
-		ensureACMEHTTP01(srv)
 	}
 	if l.WebserverHTTPS.Enabled {
 		port := config.Port(l.WebserverHTTPS.Port, 443)
-		srv := ensureServe(true, false, l.WebserverHTTPS.RateLimitDisabled, port, "webserver-https", false)
+		srv := ensureServe(true, port, "webserver-https", false)
 		srv.Webserver = true
 	}
 
 	if l.TLS != nil && l.TLS.ACME != "" {
 		m := mox.Conf.Static.ACME[l.TLS.ACME].Manager
-		if ensureManagerHosts[m] == nil {
-			ensureManagerHosts[m] = map[dns.Domain]struct{}{}
-		}
-		hosts := ensureManagerHosts[m]
-		hosts[mox.Conf.Static.HostnameDomain] = struct{}{}
 
+		// If we are listening on port 80 for plain http, also register acme http-01
+		// validation handler.
+		if srv, ok := portServe[80]; ok && srv.TLSConfig == nil {
+			srv.Kinds = append(srv.Kinds, "acme-http-01")
+			srv.SystemHandle("acme-http-01", nil, "/.well-known/acme-challenge/", m.Manager.HTTPHandler(nil))
+		}
+
+		hosts := map[dns.Domain]struct{}{
+			mox.Conf.Static.HostnameDomain: {},
+		}
 		if l.HostnameDomain.ASCII != "" {
 			hosts[l.HostnameDomain] = struct{}{}
 		}
-
 		// All domains are served on all listeners. Gather autoconfig hostnames to ensure
-		// presence of TLS certificates. Fetching a certificate on-demand may be too slow
+		// presence of TLS certificates for.
-		// for the timeouts of clients doing autoconfig.
+		for _, name := range mox.Conf.Domains() {
+			if dom, err := dns.ParseDomain(name); err != nil {
+				pkglog.Errorx("parsing domain from config", err)
+			} else if d, _ := mox.Conf.Domain(dom); d.ReportsOnly {
+				// Do not gather autoconfig name if we aren't accepting email for this domain.
+				continue
+			}
 
-		if l.AutoconfigHTTPS.Enabled && !l.AutoconfigHTTPS.NonTLS {
+			autoconfdom, err := dns.ParseDomain("autoconfig." + name)
-			for _, name := range mox.Conf.Domains() {
+			if err != nil {
-				if dom, err := dns.ParseDomain(name); err != nil {
+				pkglog.Errorx("parsing domain from config for autoconfig", err)
-					pkglog.Errorx("parsing domain from config", err)
+			} else {
-				} else if d, _ := mox.Conf.Domain(dom); d.ReportsOnly || d.Disabled {
+				hosts[autoconfdom] = struct{}{}
-					// Do not gather autoconfig name if we aren't accepting email for this domain or when it is disabled.
-					continue
-				}
-
-				autoconfdom, err := dns.ParseDomain("autoconfig." + name)
-				if err != nil {
-					pkglog.Errorx("parsing domain from config for autoconfig", err)
-				} else {
-					hosts[autoconfdom] = struct{}{}
-				}
 			}
 		}
-	}
 
-	if s := portServe[443]; s != nil && s.TLSConfig != nil && len(s.NextProto) > 0 {
+		ensureManagerHosts[m] = hosts
-		s.TLSConfig.NextProtos = append(s.TLSConfig.NextProtos, slices.Collect(maps.Keys(s.NextProto))...)
 	}
 
 	for _, srv := range portServe {
@@ -941,10 +866,8 @@ var servers []func()
 // the certificate to be given during the first https connection.
 var ensureManagerHosts = map[*autotls.Manager]map[dns.Domain]struct{}{}
 
-type tlsNextProtoMap = map[string]func(*http.Server, *tls.Conn, http.Handler)
-
 // listen prepares a listener, and adds it to "servers", to be launched (if not running as root) through Serve.
-func listen1(ip string, port int, tlsConfig *tls.Config, name string, kinds []string, handler http.Handler, nextProto tlsNextProtoMap) {
+func listen1(ip string, port int, tlsConfig *tls.Config, name string, kinds []string, handler http.Handler) {
 	addr := net.JoinHostPort(ip, fmt.Sprintf("%d", port))
 
 	var protocol string
@@ -978,20 +901,12 @@ func listen1(ip string, port int, tlsConfig *tls.Config, name string, kinds []st
 	}
 
 	server := &http.Server{
-		Handler:           handler,
+		Handler: handler,
-		TLSConfig:         tlsConfig,
+		// Clone because our multiple Server.Serve calls modify config concurrently leading to data race.
+		TLSConfig:         tlsConfig.Clone(),
 		ReadHeaderTimeout: 30 * time.Second,
 		IdleTimeout:       65 * time.Second, // Chrome closes connections after 60 seconds, firefox after 115 seconds.
 		ErrorLog:          golog.New(mlog.LogWriter(pkglog.With(slog.String("pkg", "net/http")), slog.LevelInfo, protocol+" error"), "", 0),
-		TLSNextProto:      nextProto,
-	}
-	// By default, the Go 1.6 and above http.Server includes support for HTTP2.
-	// However, HTTP2 is negotiated via ALPN. Because we are configuring
-	// TLSNextProto above, we have to explicitly enable HTTP2 by importing http2
-	// and calling ConfigureServer.
-	err = http2.ConfigureServer(server, nil)
-	if err != nil {
-		pkglog.Fatalx("https: unable to configure http2", err)
 	}
 	serve := func() {
 		err := server.Serve(ln)

http/web_test.go

@@ -17,7 +17,7 @@ func TestServeHTTP(t *testing.T) {
 	mox.ConfigDynamicPath = filepath.Join(filepath.Dir(mox.ConfigStaticPath), "domains.conf")
 	mox.MustLoadConfig(true, false)
 
-	portSrvs := portServes("local", mox.Conf.Static.Listeners["local"])
+	portSrvs := portServes(mox.Conf.Static.Listeners["local"])
 	srv := portSrvs[80]
 
 	test := func(method, target string, expCode int, expContent string, expHeaders map[string]string) {

http/webserver.go

@@ -30,6 +30,7 @@ import (
 	"github.com/mjl-/mox/dns"
 	"github.com/mjl-/mox/mlog"
 	"github.com/mjl-/mox/mox-"
+	"github.com/mjl-/mox/moxio"
 )
 
 func recvid(r *http.Request) string {
@@ -214,41 +215,31 @@ func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r *
 			}
 			http.NotFound(w, r)
 			return true
-		} else if errors.Is(err, syscall.ENAMETOOLONG) {
-			http.NotFound(w, r)
-			return true
 		} else if os.IsPermission(err) {
 			// If we tried opening a directory, we may not have permission to read it, but
 			// still access files inside it (execute bit), such as index.html. So try to serve it.
 			index, err := os.Open(filepath.Join(fspath, "index.html"))
-			if err != nil {
+			if err == nil {
-				http.Error(w, "403 - permission denied", http.StatusForbidden)
+				defer index.Close()
+				var ifi os.FileInfo
+				ifi, err = index.Stat()
+				if err != nil {
+					log().Errorx("stat index.html in directory we cannot list", err, slog.Any("url", r.URL), slog.String("fspath", fspath))
+					http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError)
+					return true
+				}
+				w.Header().Set("Content-Type", "text/html; charset=utf-8")
+				serveFile("index.html", ifi, index)
 				return true
 			}
-			defer func() {
+			http.Error(w, "403 - permission denied", http.StatusForbidden)
-				err := index.Close()
-				log().Check(err, "closing index file for serving")
-			}()
-			var ifi os.FileInfo
-			ifi, err = index.Stat()
-			if err != nil {
-				log().Errorx("stat index.html in directory we cannot list", err, slog.Any("url", r.URL), slog.String("fspath", fspath))
-				http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError)
-				return true
-			}
-			w.Header().Set("Content-Type", "text/html; charset=utf-8")
-			serveFile("index.html", ifi, index)
 			return true
 		}
 		log().Errorx("open file for static file serving", err, slog.Any("url", r.URL), slog.String("fspath", fspath))
 		http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError)
 		return true
 	}
-	defer func() {
+	defer f.Close()
-		if err := f.Close(); err != nil {
-			log().Check(err, "closing file for static file serving")
-		}
-	}()
 
 	fi, err := f.Stat()
 	if err != nil {
@@ -280,12 +271,7 @@ func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r *
 			http.Error(w, "403 - permission denied", http.StatusForbidden)
 			return true
 		} else if err == nil {
-			defer func() {
+			defer index.Close()
-				if err := index.Close(); err != nil {
-					log().Check(err, "closing index file for serving")
-				}
-			}()
-
 			var ifi os.FileInfo
 			ifi, err = index.Stat()
 			if err == nil {
@@ -352,8 +338,8 @@ func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r *
 			}
 		}
 		err = lsTemplate.Execute(w, map[string]any{"Files": files})
-		if err != nil {
+		if err != nil && !moxio.IsClosed(err) {
-			log().Check(err, "executing directory listing template")
+			log().Errorx("executing directory listing template", err)
 		}
 		return true
 	}
@@ -606,9 +592,7 @@ func forwardWebsocket(h *config.WebForward, w http.ResponseWriter, r *http.Reque
 	}
 	defer func() {
 		if beconn != nil {
-			if err := beconn.Close(); err != nil {
+			beconn.Close()
-				log().Check(err, "closing backend websocket connection")
-			}
 		}
 	}()
 
@@ -624,9 +608,7 @@ func forwardWebsocket(h *config.WebForward, w http.ResponseWriter, r *http.Reque
 	}
 	defer func() {
 		if cconn != nil {
-			if err := cconn.Close(); err != nil {
+			cconn.Close()
-				log().Check(err, "closing client websocket connection")
-			}
 		}
 	}()
 
@@ -679,12 +661,8 @@ func forwardWebsocket(h *config.WebForward, w http.ResponseWriter, r *http.Reque
 	// connection whose closing was already announced with a websocket frame.
 	lw.error(<-errc)
 	// Close connections so other goroutine stops as well.
-	if err := cconn.Close(); err != nil {
+	cconn.Close()
-		log().Check(err, "closing client websocket connection")
+	beconn.Close()
-	}
-	if err := beconn.Close(); err != nil {
-		log().Check(err, "closing backend websocket connection")
-	}
 	// Wait for goroutine so it has updated the logWriter.Size*Client fields before we
 	// continue with logging.
 	<-errc
@@ -737,9 +715,7 @@ func websocketTransact(ctx context.Context, targetURL *url.URL, r *http.Request)
 	}
 	defer func() {
 		if rerr != nil {
-			if xerr := conn.Close(); xerr != nil {
+			conn.Close()
-				log().Check(xerr, "cleaning up websocket connection")
-			}
 		}
 	}()
 
@@ -766,9 +742,7 @@ func websocketTransact(ctx context.Context, targetURL *url.URL, r *http.Request)
 	}
 	defer func() {
 		if rerr != nil {
-			if xerr := resp.Body.Close(); xerr != nil {
+			resp.Body.Close()
-				log().Check(xerr, "closing response body after error")
-			}
 		}
 	}()
 	if err := conn.SetDeadline(time.Time{}); err != nil {

imapclient/client.go

@@ -1,102 +1,40 @@
 /*
-Package imapclient provides an IMAP4 client implementing IMAP4rev1 (RFC 3501),
+Package imapclient provides an IMAP4 client, primarily for testing the IMAP4 server.
-IMAP4rev2 (RFC 9051) and various extensions.
 
-Warning: Currently primarily for testing the mox IMAP4 server. Behaviour that
+Commands can be sent to the server free-form, but responses are parsed strictly.
-may not be required by the IMAP4 specification may be expected by this client.
+Behaviour that may not be required by the IMAP4 specification may be expected by
-
+this client.
-See [Conn] for a high-level client for executing IMAP commands. Use its embedded
-[Proto] for lower-level writing of commands and reading of responses.
 */
 package imapclient
 
+/*
+- Try to keep the parsing method names and the types similar to the ABNF names in the RFCs.
+
+- todo: have mode for imap4rev1 vs imap4rev2, refusing what is not allowed. we are accepting too much now.
+- todo: stricter parsing. xnonspace() and xword() should be replaced by proper parsers.
+*/
+
 import (
 	"bufio"
 	"crypto/tls"
 	"fmt"
-	"io"
-	"log/slog"
 	"net"
+	"reflect"
 	"strings"
-
-	"github.com/mjl-/mox/mlog"
-	"github.com/mjl-/mox/moxio"
 )
 
-// Conn is an connection to an IMAP server.
+// Conn is an IMAP connection to a server.
-//
-// Method names on Conn are the names of IMAP commands. CloseMailbox, which
-// executes the IMAP CLOSE command, is an exception. The Close method closes the
-// connection.
-//
-// The methods starting with MSN are the original (old) IMAP commands. The variants
-// starting with UID should almost always be used instead, if available.
-//
-// The methods on Conn typically return errors of type Error or Response. Error
-// represents protocol and i/o level errors, including io.ErrDeadlineExceeded and
-// various errors for closed connections. Response is returned as error if the IMAP
-// result is NO or BAD instead of OK. The responses returned by the IMAP command
-// methods can also be non-zero on errors. Callers may wish to process any untagged
-// responses.
-//
-// The IMAP command methods defined on Conn don't interpret the untagged responses
-// except for  untagged CAPABILITY and untagged ENABLED responses, and the
-// CAPABILITY response code. Fields CapAvailable and CapEnabled are updated when
-// those untagged responses are received.
-//
-// Capabilities indicate which optional IMAP functionality is supported by a
-// server. Capabilities are typically implicitly enabled when the client sends a
-// command using syntax of an optional extension. Extensions without new syntax
-// from client to server, but with new behaviour or syntax from server to client,
-// the client needs to explicitly enable the capability with the ENABLE command,
-// see the Enable method.
 type Conn struct {
-	// If true, server sent a PREAUTH tag and the connection is already authenticated,
+	conn      net.Conn
-	// e.g. based on TLS certificate authentication.
+	r         *bufio.Reader
-	Preauth bool
+	panic     bool
-
-	// Capabilities available at server, from CAPABILITY command or response code.
-	CapAvailable []Capability
-	// Capabilities marked as enabled by the server, typically after an ENABLE command.
-	CapEnabled []Capability
-
-	// Proto provides lower-level functions for interacting with the IMAP connection,
-	// such as reading and writing individual lines/commands/responses.
-	Proto
-}
-
-// Proto provides low-level operations for writing requests and reading responses
-// on an IMAP connection.
-//
-// To implement the IDLE command, write "IDLE" using [Proto.WriteCommandf], then
-// read a line with [Proto.Readline]. If it starts with "+ ", the connection is in
-// idle mode and untagged responses can be read using [Proto.ReadUntagged]. If the
-// line doesn't start with "+ ", use [ParseResult] to interpret it as a response to
-// IDLE, which should be a NO or BAD. To abort idle mode, write "DONE" using
-// [Proto.Writelinef] and wait until a result line has been read.
-type Proto struct {
-	// Connection, may be original TCP or TLS connection. Reads go through c.br, and
-	// writes through c.xbw. The "x" for the writes indicate that failed writes cause
-	// an i/o panic, which is either turned into a returned error, or passed on (see
-	// boolean panic). The reader and writer wrap a tracing reading/writer and may wrap
-	// flate compression.
-	conn         net.Conn
-	connBroken   bool // If connection is broken, we won't flush (and write) again.
-	br           *bufio.Reader
-	tr           *moxio.TraceReader
-	xbw          *bufio.Writer
-	compress     bool // If compression is enabled, we must flush flateWriter and its target original bufio writer.
-	xflateWriter *moxio.FlateWriter
-	xflateBW     *bufio.Writer
-	xtw          *moxio.TraceWriter
-
-	log       mlog.Log
-	errHandle func(err error) // If set, called for all errors. Can panic. Used for imapserver tests.
 	tagGen    int
 	record    bool // If true, bytes read are added to recordBuf. recorded() resets.
 	recordBuf []byte
 
-	lastTag string
+	LastTag      string
+	CapAvailable map[Capability]struct{} // Capabilities available at server, from CAPABILITY command or response code.
+	CapEnabled   map[Capability]struct{} // Capabilities enabled through ENABLE command.
 }
 
 // Error is a parse or other protocol error.
@@ -110,52 +48,22 @@ func (e Error) Unwrap() error {
 	return e.err
 }
 
-// Opts has optional fields that influence behaviour of a Conn.
+// New creates a new client on conn.
-type Opts struct {
-	Logger *slog.Logger
-
-	// Error is called for IMAP-level and connection-level errors during the IMAP
-	// command methods on Conn, not for errors in calls on Proto. Error is allowed to
-	// call panic.
-	Error func(err error)
-}
-
-// New initializes a new IMAP client on conn.
 //
-// Conn should normally be a TLS connection, typically connected to port 993 of an
+// If xpanic is true, functions that would return an error instead panic. For parse
-// IMAP server. Alternatively, conn can be a plain TCP connection to port 143. TLS
+// errors, the resulting stack traces show typically show what was being parsed.
-// should be enabled on plain TCP connections with the [Conn.StartTLS] method.
 //
-// The initial untagged greeting response is read and must be "OK" or
+// The initial untagged greeting response is read and must be "OK".
-// "PREAUTH". If preauth, the connection is already in authenticated state,
+func New(conn net.Conn, xpanic bool) (client *Conn, rerr error) {
-// typically through TLS client certificate. This is indicated in Conn.Preauth.
-//
-// Logging is written to opts.Logger. In particular, IMAP protocol traces are
-// written with prefixes "CR: " and "CW: " (client read/write) as quoted strings at
-// levels Debug-4, with authentication messages at Debug-6 and (user) data at level
-// Debug-8.
-func New(conn net.Conn, opts *Opts) (client *Conn, rerr error) {
 	c := Conn{
-		Proto: Proto{conn: conn},
+		conn:         conn,
+		r:            bufio.NewReader(conn),
+		panic:        xpanic,
+		CapAvailable: map[Capability]struct{}{},
+		CapEnabled:   map[Capability]struct{}{},
 	}
 
-	var clog *slog.Logger
+	defer c.recover(&rerr)
-	if opts != nil {
-		c.errHandle = opts.Error
-		clog = opts.Logger
-	} else {
-		clog = slog.Default()
-	}
-	c.log = mlog.New("imapclient", clog)
-
-	c.tr = moxio.NewTraceReader(c.log, "CR: ", &c)
-	c.br = bufio.NewReader(c.tr)
-
-	// Writes are buffered and write to Conn, which may panic.
-	c.xtw = moxio.NewTraceWriter(c.log, "CW: ", &c)
-	c.xbw = bufio.NewWriter(c.xtw)
-
-	defer c.recoverErr(&rerr)
 	tag := c.xnonspace()
 	if tag != "*" {
 		c.xerrorf("expected untagged *, got %q", tag)
@@ -167,15 +75,9 @@ func New(conn net.Conn, opts *Opts) (client *Conn, rerr error) {
 		if x.Status != OK {
 			c.xerrorf("greeting, got status %q, expected OK", x.Status)
 		}
-		if x.Code != nil {
-			if caps, ok := x.Code.(CodeCapability); ok {
-				c.CapAvailable = caps
-			}
-		}
 		return &c, nil
 	case UntaggedPreauth:
-		c.Preauth = true
+		c.xerrorf("greeting: unexpected preauth")
-		return &c, nil
 	case UntaggedBye:
 		c.xerrorf("greeting: server sent bye")
 	default:
@@ -184,16 +86,8 @@ func New(conn net.Conn, opts *Opts) (client *Conn, rerr error) {
 	panic("not reached")
 }
 
-func (c *Conn) recoverErr(rerr *error) {
+func (c *Conn) recover(rerr *error) {
-	c.recover(rerr, nil)
+	if c.panic {
-}
-
-func (c *Conn) recover(rerr *error, resp *Response) {
-	if *rerr != nil {
-		if r, ok := (*rerr).(Response); ok && resp != nil {
-			*resp = r
-		}
-		c.errHandle(*rerr)
 		return
 	}
 
@@ -201,163 +95,30 @@ func (c *Conn) recover(rerr *error, resp *Response) {
 	if x == nil {
 		return
 	}
-	var err error
+	err, ok := x.(Error)
-	switch e := x.(type) {
+	if !ok {
-	case Error:
-		err = e
-	case Response:
-		err = e
-		if resp != nil {
-			*resp = e
-		}
-	default:
 		panic(x)
 	}
-	if c.errHandle != nil {
-		c.errHandle(err)
-	}
 	*rerr = err
 }
 
-func (p *Proto) recover(rerr *error) {
+func (c *Conn) xerrorf(format string, args ...any) {
-	if *rerr != nil {
-		return
-	}
-
-	x := recover()
-	if x == nil {
-		return
-	}
-	switch e := x.(type) {
-	case Error:
-		*rerr = e
-	default:
-		panic(x)
-	}
-}
-
-func (p *Proto) xerrorf(format string, args ...any) {
 	panic(Error{fmt.Errorf(format, args...)})
 }
 
-func (p *Proto) xcheckf(err error, format string, args ...any) {
+func (c *Conn) xcheckf(err error, format string, args ...any) {
 	if err != nil {
-		p.xerrorf("%s: %w", fmt.Sprintf(format, args...), err)
+		c.xerrorf("%s: %w", fmt.Sprintf(format, args...), err)
 	}
 }
 
-func (p *Proto) xcheck(err error) {
+func (c *Conn) xcheck(err error) {
 	if err != nil {
 		panic(err)
 	}
 }
 
-// xresponse sets resp if err is a Response and resp is not nil.
+// TLSConnectionState returns the TLS connection state if the connection uses TLS.
-func (p *Proto) xresponse(err error, resp *Response) {
-	if err == nil {
-		return
-	}
-	if r, ok := err.(Response); ok && resp != nil {
-		*resp = r
-	}
-	panic(err)
-}
-
-// Write writes directly to underlying connection (TCP, TLS). For internal use
-// only, to implement io.Writer. Write errors do take the connection's panic mode
-// into account, i.e. Write can panic.
-func (p *Proto) Write(buf []byte) (n int, rerr error) {
-	defer p.recover(&rerr)
-
-	n, rerr = p.conn.Write(buf)
-	if rerr != nil {
-		p.connBroken = true
-	}
-	p.xcheckf(rerr, "write")
-	return n, nil
-}
-
-// Read reads directly from the underlying connection (TCP, TLS). For internal use
-// only, to implement io.Reader.
-func (p *Proto) Read(buf []byte) (n int, err error) {
-	return p.conn.Read(buf)
-}
-
-func (p *Proto) xflush() {
-	// Not writing any more when connection is broken.
-	if p.connBroken {
-		return
-	}
-
-	err := p.xbw.Flush()
-	p.xcheckf(err, "flush")
-
-	// If compression is active, we need to flush the deflate stream.
-	if p.compress {
-		err := p.xflateWriter.Flush()
-		p.xcheckf(err, "flush deflate")
-		err = p.xflateBW.Flush()
-		p.xcheckf(err, "flush deflate buffer")
-	}
-}
-
-func (p *Proto) xtraceread(level slog.Level) func() {
-	if p.tr == nil {
-		// For ParseUntagged and other parse functions.
-		return func() {}
-	}
-	p.tr.SetTrace(level)
-	return func() {
-		p.tr.SetTrace(mlog.LevelTrace)
-	}
-}
-
-func (p *Proto) xtracewrite(level slog.Level) func() {
-	if p.xtw == nil {
-		// For ParseUntagged and other parse functions.
-		return func() {}
-	}
-
-	p.xflush()
-	p.xtw.SetTrace(level)
-	return func() {
-		p.xflush()
-		p.xtw.SetTrace(mlog.LevelTrace)
-	}
-}
-
-// Close closes the connection, flushing and closing any compression and TLS layer.
-//
-// You may want to call Logout first. Closing a connection with a mailbox with
-// deleted messages not yet expunged will not expunge those messages.
-//
-// Closing a TLS connection that is logged out, or closing a TLS connection with
-// compression enabled (i.e. two layered streams), may cause spurious errors
-// because the server may immediate close the underlying connection when it sees
-// the connection is being closed.
-func (c *Conn) Close() (rerr error) {
-	defer c.recoverErr(&rerr)
-
-	if c.conn == nil {
-		return nil
-	}
-	if !c.connBroken && c.xflateWriter != nil {
-		err := c.xflateWriter.Close()
-		c.xcheckf(err, "close deflate writer")
-		err = c.xflateBW.Flush()
-		c.xcheckf(err, "flush deflate buffer")
-		c.xflateWriter = nil
-		c.xflateBW = nil
-	}
-	err := c.conn.Close()
-	c.xcheckf(err, "close connection")
-	c.conn = nil
-	return
-}
-
-// TLSConnectionState returns the TLS connection state if the connection uses TLS,
-// either because the conn passed to [New] was a TLS connection, or because
-// [Conn.StartTLS] was called.
 func (c *Conn) TLSConnectionState() *tls.ConnectionState {
 	if conn, ok := c.conn.(*tls.Conn); ok {
 		cs := conn.ConnectionState()
@@ -366,266 +127,185 @@ func (c *Conn) TLSConnectionState() *tls.ConnectionState {
 	return nil
 }
 
-// WriteCommandf writes a free-form IMAP command to the server. An ending \r\n is
+// Commandf writes a free-form IMAP command to the server.
-// written too.
-//
 // If tag is empty, a next unique tag is assigned.
-func (p *Proto) WriteCommandf(tag string, format string, args ...any) (rerr error) {
+func (c *Conn) Commandf(tag string, format string, args ...any) (rerr error) {
-	defer p.recover(&rerr)
+	defer c.recover(&rerr)
 
 	if tag == "" {
-		p.nextTag()
+		tag = c.nextTag()
-	} else {
-		p.lastTag = tag
 	}
+	c.LastTag = tag
 
-	fmt.Fprintf(p.xbw, "%s %s\r\n", p.lastTag, fmt.Sprintf(format, args...))
+	_, err := fmt.Fprintf(c.conn, "%s %s\r\n", tag, fmt.Sprintf(format, args...))
-	p.xflush()
+	c.xcheckf(err, "write command")
 	return
 }
 
-func (p *Proto) nextTag() string {
+func (c *Conn) nextTag() string {
-	p.tagGen++
+	c.tagGen++
-	p.lastTag = fmt.Sprintf("x%03d", p.tagGen)
+	return fmt.Sprintf("x%03d", c.tagGen)
-	return p.lastTag
 }
 
-// LastTag returns the tag last used for a command. For checking against a command
+// Response reads from the IMAP server until a tagged response line is found.
-// completion result.
-func (p *Proto) LastTag() string {
-	return p.lastTag
-}
-
-// LastTagSet sets a new last tag, as used for checking against a command completion result.
-func (p *Proto) LastTagSet(tag string) {
-	p.lastTag = tag
-}
-
-// ReadResponse reads from the IMAP server until a tagged response line is found.
 // The tag must be the same as the tag for the last written command.
-//
+// Result holds the status of the command. The caller must check if this the status is OK.
-// If an error is returned, resp can still be non-empty, and a caller may wish to
+func (c *Conn) Response() (untagged []Untagged, result Result, rerr error) {
-// process resp.Untagged.
+	defer c.recover(&rerr)
-//
-// Caller should check resp.Status for the result of the command too.
-//
-// Common types for the return error:
-// - Error, for protocol errors
-// - Various I/O errors from the underlying connection, including os.ErrDeadlineExceeded
-func (p *Proto) ReadResponse() (resp Response, rerr error) {
-	defer p.recover(&rerr)
 
 	for {
-		tag := p.xnonspace()
+		tag := c.xnonspace()
-		p.xspace()
+		c.xspace()
 		if tag == "*" {
-			resp.Untagged = append(resp.Untagged, p.xuntagged())
+			untagged = append(untagged, c.xuntagged())
 			continue
 		}
 
-		if tag != p.lastTag {
+		if tag != c.LastTag {
-			p.xerrorf("got tag %q, expected %q", tag, p.lastTag)
+			c.xerrorf("got tag %q, expected %q", tag, c.LastTag)
 		}
 
-		status := p.xstatus()
+		status := c.xstatus()
-		p.xspace()
+		c.xspace()
-		resp.Result = p.xresult(status)
+		result = c.xresult(status)
-		p.xcrlf()
+		c.xcrlf()
 		return
 	}
 }
 
-// ParseCode parses a response code. The string must not have enclosing brackets.
-//
-// Example:
-//
-//	"APPENDUID 123 10"
-func ParseCode(s string) (code Code, rerr error) {
-	p := Proto{br: bufio.NewReader(strings.NewReader(s + "]"))}
-	defer p.recover(&rerr)
-	code = p.xrespCode()
-	p.xtake("]")
-	buf, err := io.ReadAll(p.br)
-	p.xcheckf(err, "read")
-	if len(buf) != 0 {
-		p.xerrorf("leftover data %q", buf)
-	}
-	return code, nil
-}
-
-// ParseResult parses a line, including required crlf, as a command result line.
-//
-// Example:
-//
-//	"tag1 OK [APPENDUID 123 10] message added\r\n"
-func ParseResult(s string) (tag string, result Result, rerr error) {
-	p := Proto{br: bufio.NewReader(strings.NewReader(s))}
-	defer p.recover(&rerr)
-	tag = p.xnonspace()
-	p.xspace()
-	status := p.xstatus()
-	p.xspace()
-	result = p.xresult(status)
-	p.xcrlf()
-	return
-}
-
 // ReadUntagged reads a single untagged response line.
-func (p *Proto) ReadUntagged() (untagged Untagged, rerr error) {
+// Useful for reading lines from IDLE.
-	defer p.recover(&rerr)
+func (c *Conn) ReadUntagged() (untagged Untagged, rerr error) {
-	return p.readUntagged()
+	defer c.recover(&rerr)
-}
 
-// ParseUntagged parses a line, including required crlf, as untagged response.
+	tag := c.xnonspace()
-//
-// Example:
-//
-//	"* BYE shutting down connection\r\n"
-func ParseUntagged(s string) (untagged Untagged, rerr error) {
-	p := Proto{br: bufio.NewReader(strings.NewReader(s))}
-	defer p.recover(&rerr)
-	untagged, rerr = p.readUntagged()
-	return
-}
-
-func (p *Proto) readUntagged() (untagged Untagged, rerr error) {
-	defer p.recover(&rerr)
-	tag := p.xnonspace()
 	if tag != "*" {
-		p.xerrorf("got tag %q, expected untagged", tag)
+		c.xerrorf("got tag %q, expected untagged", tag)
 	}
-	p.xspace()
+	c.xspace()
-	ut := p.xuntagged()
+	ut := c.xuntagged()
 	return ut, nil
 }
 
 // Readline reads a line, including CRLF.
 // Used with IDLE and synchronous literals.
-func (p *Proto) Readline() (line string, rerr error) {
+func (c *Conn) Readline() (line string, rerr error) {
-	defer p.recover(&rerr)
+	defer c.recover(&rerr)
 
-	line, err := p.br.ReadString('\n')
+	line, err := c.r.ReadString('\n')
-	p.xcheckf(err, "read line")
+	c.xcheckf(err, "read line")
 	return line, nil
 }
 
-func (c *Conn) readContinuation() (line string, rerr error) {
+// ReadContinuation reads a line. If it is a continuation, i.e. starts with a +, it
-	defer c.recover(&rerr, nil)
+// is returned without leading "+ " and without trailing crlf. Otherwise, a command
-	line, rerr = c.ReadContinuation()
+// response is returned. A successfully read continuation can return an empty line.
-	if rerr != nil {
+// Callers should check rerr and result.Status being empty to check if a
-		if resp, ok := rerr.(Response); ok {
+// continuation was read.
-			c.processUntagged(resp.Untagged)
+func (c *Conn) ReadContinuation() (line string, untagged []Untagged, result Result, rerr error) {
-			c.processResult(resp.Result)
+	if !c.peek('+') {
-		}
+		untagged, result, rerr = c.Response()
+		c.xcheckf(rerr, "reading non-continuation response")
+		c.xerrorf("response status %q, expected OK", result.Status)
 	}
-	return
+	c.xtake("+ ")
-}
+	line, err := c.Readline()
-
+	c.xcheckf(err, "read line")
-// ReadContinuation reads a line. If it is a continuation, i.e. starts with "+", it
-// is returned without leading "+ " and without trailing crlf. Otherwise, an error
-// is returned, which can be a Response with Untagged that a caller may wish to
-// process. A successfully read continuation can return an empty line.
-func (p *Proto) ReadContinuation() (line string, rerr error) {
-	defer p.recover(&rerr)
-
-	if !p.peek('+') {
-		var resp Response
-		resp, rerr = p.ReadResponse()
-		if rerr == nil {
-			rerr = resp
-		}
-		return "", rerr
-	}
-	p.xtake("+ ")
-	line, err := p.Readline()
-	p.xcheckf(err, "read line")
 	line = strings.TrimSuffix(line, "\r\n")
 	return
 }
 
 // Writelinef writes the formatted format and args as a single line, adding CRLF.
 // Used with IDLE and synchronous literals.
-func (p *Proto) Writelinef(format string, args ...any) (rerr error) {
+func (c *Conn) Writelinef(format string, args ...any) (rerr error) {
-	defer p.recover(&rerr)
+	defer c.recover(&rerr)
 
 	s := fmt.Sprintf(format, args...)
-	fmt.Fprintf(p.xbw, "%s\r\n", s)
+	_, err := fmt.Fprintf(c.conn, "%s\r\n", s)
-	p.xflush()
+	c.xcheckf(err, "writeline")
 	return nil
 }
 
-// WriteSyncLiteral first writes the synchronous literal size, then reads the
+// Write writes directly to the connection. Write errors do take the connections
-// continuation "+" and finally writes the data. If the literal is not accepted, an
+// panic mode into account, i.e. Write can panic.
-// error is returned, which may be a Response.
+func (c *Conn) Write(buf []byte) (n int, rerr error) {
-func (p *Proto) WriteSyncLiteral(s string) (rerr error) {
+	defer c.recover(&rerr)
-	defer p.recover(&rerr)
 
-	fmt.Fprintf(p.xbw, "{%d}\r\n", len(s))
+	n, rerr = c.conn.Write(buf)
-	p.xflush()
+	c.xcheckf(rerr, "write")
+	return n, nil
+}
 
-	plus, err := p.br.Peek(1)
+// WriteSyncLiteral first writes the synchronous literal size, then read the
-	p.xcheckf(err, "read continuation")
+// continuation "+" and finally writes the data.
+func (c *Conn) WriteSyncLiteral(s string) (untagged []Untagged, rerr error) {
+	defer c.recover(&rerr)
+
+	_, err := fmt.Fprintf(c.conn, "{%d}\r\n", len(s))
+	c.xcheckf(err, "write sync literal size")
+
+	plus, err := c.r.Peek(1)
+	c.xcheckf(err, "read continuation")
 	if plus[0] == '+' {
-		_, err = p.Readline()
+		_, err = c.Readline()
-		p.xcheckf(err, "read continuation line")
+		c.xcheckf(err, "read continuation line")
 
-		defer p.xtracewrite(mlog.LevelTracedata)()
+		_, err = c.conn.Write([]byte(s))
-		_, err = p.xbw.Write([]byte(s))
+		c.xcheckf(err, "write literal data")
-		p.xcheckf(err, "write literal data")
+		return nil, nil
-		p.xtracewrite(mlog.LevelTrace)
-		return nil
 	}
-	var resp Response
+	untagged, result, err := c.Response()
-	resp, rerr = p.ReadResponse()
+	if err == nil && result.Status == OK {
-	if rerr == nil {
+		c.xerrorf("no continuation, but invalid ok response (%q)", result.More)
-		rerr = resp
 	}
-	return
+	return untagged, fmt.Errorf("no continuation (%s)", result.Status)
 }
 
-func (c *Conn) processUntagged(l []Untagged) {
+// Transactf writes format and args as an IMAP command, using Commandf with an
-	for _, ut := range l {
-		switch e := ut.(type) {
-		case UntaggedCapability:
-			c.CapAvailable = []Capability(e)
-		case UntaggedEnabled:
-			c.CapEnabled = append(c.CapEnabled, e...)
-		}
-	}
-}
-
-func (c *Conn) processResult(r Result) {
-	if r.Code == nil {
-		return
-	}
-	switch e := r.Code.(type) {
-	case CodeCapability:
-		c.CapAvailable = []Capability(e)
-	}
-}
-
-// transactf writes format and args as an IMAP command, using Commandf with an
 // empty tag. I.e. format must not contain a tag. Transactf then reads a response
 // using ReadResponse and checks the result status is OK.
-func (c *Conn) transactf(format string, args ...any) (resp Response, rerr error) {
+func (c *Conn) Transactf(format string, args ...any) (untagged []Untagged, result Result, rerr error) {
-	defer c.recover(&rerr, &resp)
+	defer c.recover(&rerr)
 
-	err := c.WriteCommandf("", format, args...)
+	err := c.Commandf("", format, args...)
 	if err != nil {
-		return Response{}, err
+		return nil, Result{}, err
 	}
-
+	return c.ResponseOK()
-	return c.responseOK()
 }
 
-func (c *Conn) responseOK() (resp Response, rerr error) {
+func (c *Conn) ResponseOK() (untagged []Untagged, result Result, rerr error) {
-	defer c.recover(&rerr, &resp)
+	untagged, result, rerr = c.Response()
-
+	if rerr != nil {
-	resp, rerr = c.ReadResponse()
+		return nil, Result{}, rerr
-	c.processUntagged(resp.Untagged)
-	c.processResult(resp.Result)
-	if rerr == nil && resp.Status != OK {
-		rerr = resp
 	}
-	return
+	if result.Status != OK {
+		c.xerrorf("response status %q, expected OK", result.Status)
+	}
+	return untagged, result, rerr
+}
+
+func (c *Conn) xgetUntagged(l []Untagged, dst any) {
+	if len(l) != 1 {
+		c.xerrorf("got %d untagged, expected 1: %v", len(l), l)
+	}
+	got := l[0]
+	gotv := reflect.ValueOf(got)
+	dstv := reflect.ValueOf(dst)
+	if gotv.Type() != dstv.Type().Elem() {
+		c.xerrorf("got %v, expected %v", gotv.Type(), dstv.Type().Elem())
+	}
+	dstv.Elem().Set(gotv)
+}
+
+// Close closes the connection without writing anything to the server.
+// You may want to call Logout. Closing a connection with a mailbox with deleted
+// message not yet expunged will not expunge those messages.
+func (c *Conn) Close() error {
+	var err error
+	if c.conn != nil {
+		err = c.conn.Close()
+		c.conn = nil
+	}
+	return err
 }

imapclient/cmds.go

@@ -6,121 +6,73 @@ import (
 	"encoding/base64"
 	"fmt"
 	"hash"
-	"io"
 	"strings"
 	"time"
 
-	"github.com/mjl-/flate"
-
-	"github.com/mjl-/mox/mlog"
-	"github.com/mjl-/mox/moxio"
 	"github.com/mjl-/mox/scram"
 )
 
-// Capability writes the IMAP4 "CAPABILITY" command, requesting a list of
+// Capability requests a list of capabilities from the server. They are returned in
-// capabilities from the server. They are returned in an UntaggedCapability
+// an UntaggedCapability response. The server also sends capabilities in initial
-// response. The server also sends capabilities in initial server greeting, in the
+// server greeting, in the response code.
-// response code.
+func (c *Conn) Capability() (untagged []Untagged, result Result, rerr error) {
-func (c *Conn) Capability() (resp Response, rerr error) {
+	defer c.recover(&rerr)
-	defer c.recover(&rerr, &resp)
+	return c.Transactf("capability")
-	return c.transactf("capability")
 }
 
-// Noop writes the IMAP4 "NOOP" command, which does nothing on its own, but a
+// Noop does nothing on its own, but a server will return any pending untagged
-// server will return any pending untagged responses for new message delivery and
+// responses for new message delivery and changes to mailboxes.
-// changes to mailboxes.
+func (c *Conn) Noop() (untagged []Untagged, result Result, rerr error) {
-func (c *Conn) Noop() (resp Response, rerr error) {
+	defer c.recover(&rerr)
-	defer c.recover(&rerr, &resp)
+	return c.Transactf("noop")
-	return c.transactf("noop")
 }
 
-// Logout ends the IMAP4 session by writing an IMAP "LOGOUT" command. [Conn.Close]
+// Logout ends the IMAP session by writing a LOGOUT command. Close must still be
-// must still be called on this client to close the socket.
+// called on this client to close the socket.
-func (c *Conn) Logout() (resp Response, rerr error) {
+func (c *Conn) Logout() (untagged []Untagged, result Result, rerr error) {
-	defer c.recover(&rerr, &resp)
+	defer c.recover(&rerr)
-	return c.transactf("logout")
+	return c.Transactf("logout")
 }
 
-// StartTLS enables TLS on the connection with the IMAP4 "STARTTLS" command.
+// Starttls enables TLS on the connection with the STARTTLS command.
-func (c *Conn) StartTLS(config *tls.Config) (resp Response, rerr error) {
+func (c *Conn) Starttls(config *tls.Config) (untagged []Untagged, result Result, rerr error) {
-	defer c.recover(&rerr, &resp)
+	defer c.recover(&rerr)
-	resp, rerr = c.transactf("starttls")
+	untagged, result, rerr = c.Transactf("starttls")
 	c.xcheckf(rerr, "starttls command")
-
+	conn := tls.Client(c.conn, config)
-	conn := c.xprefixConn()
+	err := conn.Handshake()
-	tlsConn := tls.Client(conn, config)
-	err := tlsConn.Handshake()
 	c.xcheckf(err, "tls handshake")
-	c.conn = tlsConn
+	c.conn = conn
+	c.r = bufio.NewReader(conn)
+	return untagged, result, nil
+}
+
+// Login authenticates with username and password
+func (c *Conn) Login(username, password string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("login %s %s", astring(username), astring(password))
+}
+
+// Authenticate with plaintext password using AUTHENTICATE PLAIN.
+func (c *Conn) AuthenticatePlain(username, password string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+
+	untagged, result, rerr = c.Transactf("authenticate plain %s", base64.StdEncoding.EncodeToString(fmt.Appendf(nil, "\u0000%s\u0000%s", username, password)))
 	return
 }
 
-// Login authenticates using the IMAP4 "LOGIN" command, sending the plain text
+// Authenticate with SCRAM-SHA-256(-PLUS) or SCRAM-SHA-1(-PLUS). With SCRAM, the
-// password to the server.
+// password is not exchanged in plaintext form, but only derived hashes are
-//
+// exchanged by both parties as proof of knowledge of password.
-// Authentication is not allowed while the "LOGINDISABLED" capability is announced.
-// Call [Conn.StartTLS] first.
-//
-// See [Conn.AuthenticateSCRAM] for a better authentication mechanism.
-func (c *Conn) Login(username, password string) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-
-	fmt.Fprintf(c.xbw, "%s login %s ", c.nextTag(), astring(username))
-	defer c.xtracewrite(mlog.LevelTraceauth)()
-	fmt.Fprintf(c.xbw, "%s\r\n", astring(password))
-	c.xtracewrite(mlog.LevelTrace) // Restore.
-	return c.responseOK()
-}
-
-// AuthenticatePlain executes the AUTHENTICATE command with SASL mechanism "PLAIN",
-// sending the password in plain text password to the server.
-//
-// Required capability: "AUTH=PLAIN"
-//
-// Authentication is not allowed while the "LOGINDISABLED" capability is announced.
-// Call [Conn.StartTLS] first.
-//
-// See [Conn.AuthenticateSCRAM] for a better authentication mechanism.
-func (c *Conn) AuthenticatePlain(username, password string) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-
-	err := c.WriteCommandf("", "authenticate plain")
-	c.xcheckf(err, "writing authenticate command")
-	_, rerr = c.readContinuation()
-	c.xresponse(rerr, &resp)
-
-	defer c.xtracewrite(mlog.LevelTraceauth)()
-	xw := base64.NewEncoder(base64.StdEncoding, c.xbw)
-	fmt.Fprintf(xw, "\u0000%s\u0000%s", username, password)
-	xw.Close()
-	c.xtracewrite(mlog.LevelTrace) // Restore.
-	fmt.Fprintf(c.xbw, "\r\n")
-	c.xflush()
-	return c.responseOK()
-}
-
-// todo: implement cram-md5, write its credentials as traceauth.
-
-// AuthenticateSCRAM executes the IMAP4 "AUTHENTICATE" command with one of the
-// following SASL mechanisms: SCRAM-SHA-256(-PLUS) or SCRAM-SHA-1(-PLUS).//
-//
-// With SCRAM, the password is not sent to the server in plain text, but only
-// derived hashes are exchanged by both parties as proof of knowledge of password.
-//
-// Authentication is not allowed while the "LOGINDISABLED" capability is announced.
-// Call [Conn.StartTLS] first.
-//
-// Required capability: SCRAM-SHA-256-PLUS, SCRAM-SHA-256, SCRAM-SHA-1-PLUS,
-// SCRAM-SHA-1.
 //
 // The PLUS variants bind the authentication exchange to the TLS connection,
 // detecting MitM attacks.
-func (c *Conn) AuthenticateSCRAM(mechanism string, h func() hash.Hash, username, password string) (resp Response, rerr error) {
+func (c *Conn) AuthenticateSCRAM(method string, h func() hash.Hash, username, password string) (untagged []Untagged, result Result, rerr error) {
-	defer c.recover(&rerr, &resp)
+	defer c.recover(&rerr)
 
 	var cs *tls.ConnectionState
-	lmech := strings.ToLower(mechanism)
+	lmethod := strings.ToLower(method)
-	if strings.HasSuffix(lmech, "-plus") {
+	if strings.HasSuffix(lmethod, "-plus") {
 		tlsConn, ok := c.conn.(*tls.Conn)
 		if !ok {
 			c.xerrorf("cannot use scram plus without tls")
@@ -131,14 +83,17 @@ func (c *Conn) AuthenticateSCRAM(mechanism string, h func() hash.Hash, username,
 	sc := scram.NewClient(h, username, "", false, cs)
 	clientFirst, err := sc.ClientFirst()
 	c.xcheckf(err, "scram clientFirst")
-	// todo: only send clientFirst if server has announced SASL-IR
+	c.LastTag = c.nextTag()
-	err = c.Writelinef("%s authenticate %s %s", c.nextTag(), mechanism, base64.StdEncoding.EncodeToString([]byte(clientFirst)))
+	err = c.Writelinef("%s authenticate %s %s", c.LastTag, method, base64.StdEncoding.EncodeToString([]byte(clientFirst)))
 	c.xcheckf(err, "writing command line")
 
 	xreadContinuation := func() []byte {
 		var line string
-		line, rerr = c.readContinuation()
+		line, untagged, result, rerr = c.ReadContinuation()
-		c.xresponse(rerr, &resp)
+		c.xcheckf(err, "read continuation")
+		if result.Status != "" {
+			c.xerrorf("unexpected status %q", result.Status)
+		}
 		buf, err := base64.StdEncoding.DecodeString(line)
 		c.xcheckf(err, "parsing base64 from remote")
 		return buf
@@ -158,131 +113,83 @@ func (c *Conn) AuthenticateSCRAM(mechanism string, h func() hash.Hash, username,
 	err = c.Writelinef("%s", base64.StdEncoding.EncodeToString(nil))
 	c.xcheckf(err, "scram client end")
 
-	return c.responseOK()
+	return c.ResponseOK()
 }
 
-// CompressDeflate enables compression with deflate on the connection by executing
+// Enable enables capabilities for use with the connection, verifying the server has indeed enabled them.
-// the IMAP4 "COMPRESS=DEFAULT" command.
+func (c *Conn) Enable(capabilities ...string) (untagged []Untagged, result Result, rerr error) {
-//
+	defer c.recover(&rerr)
-// Required capability: "COMPRESS=DEFLATE".
-//
-// State: Authenticated or selected.
-func (c *Conn) CompressDeflate() (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
 
-	resp, rerr = c.transactf("compress deflate")
+	untagged, result, rerr = c.Transactf("enable %s", strings.Join(capabilities, " "))
 	c.xcheck(rerr)
-
+	var enabled UntaggedEnabled
-	c.xflateBW = bufio.NewWriter(c)
+	c.xgetUntagged(untagged, &enabled)
-	fw0, err := flate.NewWriter(c.xflateBW, flate.DefaultCompression)
+	got := map[string]struct{}{}
-	c.xcheckf(err, "deflate") // Cannot happen.
+	for _, cap := range enabled {
-	fw := moxio.NewFlateWriter(fw0)
+		got[cap] = struct{}{}
-
+	}
-	c.compress = true
+	for _, cap := range capabilities {
-	c.xflateWriter = fw
+		if _, ok := got[cap]; !ok {
-	c.xtw = moxio.NewTraceWriter(mlog.New("imapclient", nil), "CW: ", fw)
+			c.xerrorf("capability %q not enabled by server", cap)
-	c.xbw = bufio.NewWriter(c.xtw)
+		}
-
+	}
-	rc := c.xprefixConn()
-	fr := flate.NewReaderPartial(rc)
-	c.tr = moxio.NewTraceReader(mlog.New("imapclient", nil), "CR: ", fr)
-	c.br = bufio.NewReader(c.tr)
-
 	return
 }
 
-// Enable enables capabilities for use with the connection by executing the IMAP4 "ENABLE" command.
+// Select opens mailbox as active mailbox.
-//
+func (c *Conn) Select(mailbox string) (untagged []Untagged, result Result, rerr error) {
-// Required capability: "ENABLE" or "IMAP4rev2"
+	defer c.recover(&rerr)
-func (c *Conn) Enable(capabilities ...Capability) (resp Response, rerr error) {
+	return c.Transactf("select %s", astring(mailbox))
-	defer c.recover(&rerr, &resp)
-
-	var caps strings.Builder
-	for _, c := range capabilities {
-		caps.WriteString(" " + string(c))
-	}
-	return c.transactf("enable%s", caps.String())
 }
 
-// Select opens the mailbox with the IMAP4 "SELECT" command.
+// Examine opens mailbox as active mailbox read-only.
-//
+func (c *Conn) Examine(mailbox string) (untagged []Untagged, result Result, rerr error) {
-// If a mailbox is selected/active, it is automatically deselected before
+	defer c.recover(&rerr)
-// selecting the mailbox, without permanently removing ("expunging") messages
+	return c.Transactf("examine %s", astring(mailbox))
-// marked \Deleted.
-//
-// If the mailbox cannot be opened, the connection is left in Authenticated state,
-// not Selected.
-func (c *Conn) Select(mailbox string) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-	return c.transactf("select %s", astring(mailbox))
 }
 
-// Examine opens the mailbox like [Conn.Select], but read-only, with the IMAP4
+// Create makes a new mailbox on the server.
-// "EXAMINE" command.
+func (c *Conn) Create(mailbox string) (untagged []Untagged, result Result, rerr error) {
-func (c *Conn) Examine(mailbox string) (resp Response, rerr error) {
+	defer c.recover(&rerr)
-	defer c.recover(&rerr, &resp)
+	return c.Transactf("create %s", astring(mailbox))
-	return c.transactf("examine %s", astring(mailbox))
 }
 
-// Create makes a new mailbox on the server using the IMAP4 "CREATE" command.
+// Delete removes an entire mailbox and its messages.
-//
+func (c *Conn) Delete(mailbox string) (untagged []Untagged, result Result, rerr error) {
-// SpecialUse can only be used on servers that announced the "CREATE-SPECIAL-USE"
+	defer c.recover(&rerr)
-// capability. Specify flags like \Archive, \Drafts, \Junk, \Sent, \Trash, \All.
+	return c.Transactf("delete %s", astring(mailbox))
-func (c *Conn) Create(mailbox string, specialUse []string) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-	var useStr string
-	if len(specialUse) > 0 {
-		useStr = fmt.Sprintf(" USE (%s)", strings.Join(specialUse, " "))
-	}
-	return c.transactf("create %s%s", astring(mailbox), useStr)
 }
 
-// Delete removes an entire mailbox and its messages using the IMAP4 "DELETE"
+// Rename changes the name of a mailbox and all its child mailboxes.
-// command.
+func (c *Conn) Rename(omailbox, nmailbox string) (untagged []Untagged, result Result, rerr error) {
-func (c *Conn) Delete(mailbox string) (resp Response, rerr error) {
+	defer c.recover(&rerr)
-	defer c.recover(&rerr, &resp)
+	return c.Transactf("rename %s %s", astring(omailbox), astring(nmailbox))
-	return c.transactf("delete %s", astring(mailbox))
 }
 
-// Rename changes the name of a mailbox and all its child mailboxes
+// Subscribe marks a mailbox as subscribed. The mailbox does not have to exist. It
-// using the IMAP4 "RENAME" command.
+// is not an error if the mailbox is already subscribed.
-func (c *Conn) Rename(omailbox, nmailbox string) (resp Response, rerr error) {
+func (c *Conn) Subscribe(mailbox string) (untagged []Untagged, result Result, rerr error) {
-	defer c.recover(&rerr, &resp)
+	defer c.recover(&rerr)
-	return c.transactf("rename %s %s", astring(omailbox), astring(nmailbox))
+	return c.Transactf("subscribe %s", astring(mailbox))
 }
 
-// Subscribe marks a mailbox as subscribed using the IMAP4 "SUBSCRIBE" command.
+// Unsubscribe marks a mailbox as unsubscribed.
-//
+func (c *Conn) Unsubscribe(mailbox string) (untagged []Untagged, result Result, rerr error) {
-// The mailbox does not have to exist. It is not an error if the mailbox is already
+	defer c.recover(&rerr)
-// subscribed.
+	return c.Transactf("unsubscribe %s", astring(mailbox))
-func (c *Conn) Subscribe(mailbox string) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-	return c.transactf("subscribe %s", astring(mailbox))
 }
 
-// Unsubscribe marks a mailbox as unsubscribed using the IMAP4 "UNSUBSCRIBE"
+// List lists mailboxes with the basic LIST syntax.
-// command.
-func (c *Conn) Unsubscribe(mailbox string) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-	return c.transactf("unsubscribe %s", astring(mailbox))
-}
-
-// List lists mailboxes using the IMAP4 "LIST" command with the basic LIST syntax.
 // Pattern can contain * (match any) or % (match any except hierarchy delimiter).
-func (c *Conn) List(pattern string) (resp Response, rerr error) {
+func (c *Conn) List(pattern string) (untagged []Untagged, result Result, rerr error) {
-	defer c.recover(&rerr, &resp)
+	defer c.recover(&rerr)
-	return c.transactf(`list "" %s`, astring(pattern))
+	return c.Transactf(`list "" %s`, astring(pattern))
 }
 
-// ListFull lists mailboxes using the LIST command with the extended LIST
+// ListFull lists mailboxes with the extended LIST syntax requesting all supported data.
-// syntax requesting all supported data.
-//
-// Required capability: "LIST-EXTENDED". If "IMAP4rev2" is announced, the command
-// is also available but only with a single pattern.
-//
 // Pattern can contain * (match any) or % (match any except hierarchy delimiter).
-func (c *Conn) ListFull(subscribedOnly bool, patterns ...string) (resp Response, rerr error) {
+func (c *Conn) ListFull(subscribedOnly bool, patterns ...string) (untagged []Untagged, result Result, rerr error) {
-	defer c.recover(&rerr, &resp)
+	defer c.recover(&rerr)
 	var subscribedStr string
 	if subscribedOnly {
 		subscribedStr = "subscribed recursivematch"
@@ -290,313 +197,115 @@ func (c *Conn) ListFull(subscribedOnly bool, patterns ...string) (resp Response,
 	for i, s := range patterns {
 		patterns[i] = astring(s)
 	}
-	return c.transactf(`list (%s) "" (%s) return (subscribed children special-use status (messages uidnext uidvalidity unseen deleted size recent appendlimit))`, subscribedStr, strings.Join(patterns, " "))
+	return c.Transactf(`list (%s) "" (%s) return (subscribed children special-use status (messages uidnext uidvalidity unseen deleted size recent appendlimit))`, subscribedStr, strings.Join(patterns, " "))
 }
 
-// Namespace requests the hiearchy separator using the IMAP4 "NAMESPACE" command.
+// Namespace returns the hiearchy separator in an UntaggedNamespace response with personal/shared/other namespaces if present.
-//
+func (c *Conn) Namespace() (untagged []Untagged, result Result, rerr error) {
-// Required capability: "NAMESPACE" or "IMAP4rev2".
+	defer c.recover(&rerr)
-//
+	return c.Transactf("namespace")
-// Server will return an UntaggedNamespace response with personal/shared/other
-// namespaces if present.
-func (c *Conn) Namespace() (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-	return c.transactf("namespace")
 }
 
-// Status requests information about a mailbox using the IMAP4 "STATUS" command. For
+// Status requests information about a mailbox, such as number of messages, size,
-// example, number of messages, size, etc. At least one attribute required.
+// etc. At least one attribute required.
-func (c *Conn) Status(mailbox string, attrs ...StatusAttr) (resp Response, rerr error) {
+func (c *Conn) Status(mailbox string, attrs ...StatusAttr) (untagged []Untagged, result Result, rerr error) {
-	defer c.recover(&rerr, &resp)
+	defer c.recover(&rerr)
 	l := make([]string, len(attrs))
 	for i, a := range attrs {
 		l[i] = string(a)
 	}
-	return c.transactf("status %s (%s)", astring(mailbox), strings.Join(l, " "))
+	return c.Transactf("status %s (%s)", astring(mailbox), strings.Join(l, " "))
 }
 
-// Append represents a parameter to the IMAP4 "APPEND" or "REPLACE" commands, for
+// Append adds message to mailbox with flags and optional receive time.
-// adding a message to mailbox, or replacing a message with a new version in a
+func (c *Conn) Append(mailbox string, flags []string, received *time.Time, message []byte) (untagged []Untagged, result Result, rerr error) {
-// mailbox.
+	defer c.recover(&rerr)
-type Append struct {
+	var date string
-	Flags    []string   // Optional, flags for the new message.
+	if received != nil {
-	Received *time.Time // Optional, the INTERNALDATE field, typically time at which a message was received.
+		date = ` "` + received.Format("_2-Jan-2006 15:04:05 -0700") + `"`
-	Size     int64
-	Data     io.Reader // Required, must return Size bytes.
-}
-
-// Append adds message to mailbox with flags and optional receive time using the
-// IMAP4 "APPEND" command.
-func (c *Conn) Append(mailbox string, message Append) (resp Response, rerr error) {
-	return c.MultiAppend(mailbox, message)
-}
-
-// MultiAppend atomatically adds multiple messages to the mailbox.
-//
-// Required capability: "MULTIAPPEND"
-func (c *Conn) MultiAppend(mailbox string, message Append, more ...Append) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-
-	fmt.Fprintf(c.xbw, "%s append %s", c.nextTag(), astring(mailbox))
-
-	msgs := append([]Append{message}, more...)
-	for _, m := range msgs {
-		var date string
-		if m.Received != nil {
-			date = ` "` + m.Received.Format("_2-Jan-2006 15:04:05 -0700") + `"`
-		}
-
-		// todo: use literal8 if needed, with "UTF8()" if required.
-		// todo: for larger messages, use a synchronizing literal.
-
-		fmt.Fprintf(c.xbw, " (%s)%s {%d+}\r\n", strings.Join(m.Flags, " "), date, m.Size)
-		defer c.xtracewrite(mlog.LevelTracedata)()
-		_, err := io.Copy(c.xbw, m.Data)
-		c.xcheckf(err, "write message data")
-		c.xtracewrite(mlog.LevelTrace) // Restore
 	}
-
+	return c.Transactf("append %s (%s)%s {%d+}\r\n%s", astring(mailbox), strings.Join(flags, " "), date, len(message), message)
-	fmt.Fprintf(c.xbw, "\r\n")
-	c.xflush()
-	return c.responseOK()
 }
 
-// note: No Idle or Notify command. Idle/Notify is better implemented by
+// note: No idle command. Idle is better implemented by writing the request and reading and handling the responses as they come in.
-// writing the request and reading and handling the responses as they come in.
 
-// CloseMailbox closes the selected/active mailbox using the IMAP4 "CLOSE" command,
+// CloseMailbox closes the currently selected/active mailbox, permanently removing
-// permanently removing ("expunging") any messages marked with \Deleted.
+// any messages marked with \Deleted.
-//
+func (c *Conn) CloseMailbox() (untagged []Untagged, result Result, rerr error) {
-// See [Conn.Unselect] for closing a mailbox without permanently removing messages.
+	return c.Transactf("close")
-func (c *Conn) CloseMailbox() (resp Response, rerr error) {
-	return c.transactf("close")
 }
 
-// Unselect closes the selected/active mailbox using the IMAP4 "UNSELECT" command,
+// Unselect closes the currently selected/active mailbox, but unlike CloseMailbox
-// but unlike MailboxClose does not permanently remove ("expunge") any messages
+// does not permanently remove any messages marked with \Deleted.
-// marked with \Deleted.
+func (c *Conn) Unselect() (untagged []Untagged, result Result, rerr error) {
-//
+	return c.Transactf("unselect")
-// Required capability: "UNSELECT" or "IMAP4rev2".
-//
-// If Unselect is not available, call [Conn.Select] with a non-existent mailbox for
-// the same effect: Deselecting a mailbox without permanently removing messages
-// marked \Deleted.
-func (c *Conn) Unselect() (resp Response, rerr error) {
-	return c.transactf("unselect")
 }
 
-// Expunge removes all messages marked as deleted for the selected mailbox using
+// Expunge removes messages marked as deleted for the selected mailbox.
-// the IMAP4 "EXPUNGE" command. If other sessions marked messages as deleted, even
+func (c *Conn) Expunge() (untagged []Untagged, result Result, rerr error) {
-// if they aren't visible in the session, they are removed as well.
+	defer c.recover(&rerr)
-//
+	return c.Transactf("expunge")
-// UIDExpunge gives more control over which the messages that are removed.
-func (c *Conn) Expunge() (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-	return c.transactf("expunge")
 }
 
-// UIDExpunge is like expunge, but only removes messages matching UID set, using
+// UIDExpunge is like expunge, but only removes messages matching uidSet.
-// the IMAP4 "UID EXPUNGE" command.
+func (c *Conn) UIDExpunge(uidSet NumSet) (untagged []Untagged, result Result, rerr error) {
-//
+	defer c.recover(&rerr)
-// Required capability: "UIDPLUS" or "IMAP4rev2".
+	return c.Transactf("uid expunge %s", uidSet.String())
-func (c *Conn) UIDExpunge(uidSet NumSet) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-	return c.transactf("uid expunge %s", uidSet.String())
 }
 
 // Note: No search, fetch command yet due to its large syntax.
 
-// MSNStoreFlagsSet stores a new set of flags for messages matching message
+// StoreFlagsSet stores a new set of flags for messages from seqset with the STORE command.
-// sequence numbers (MSNs) from sequence set with the IMAP4 "STORE" command.
+// If silent, no untagged responses with the updated flags will be sent by the server.
-//
+func (c *Conn) StoreFlagsSet(seqset string, silent bool, flags ...string) (untagged []Untagged, result Result, rerr error) {
-// If silent, no untagged responses with the updated flags will be sent by the
+	defer c.recover(&rerr)
-// server.
-//
-// Method [Conn.UIDStoreFlagsSet], which operates on a uid set, should be
-// preferred.
-func (c *Conn) MSNStoreFlagsSet(seqset string, silent bool, flags ...string) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
 	item := "flags"
 	if silent {
 		item += ".silent"
 	}
-	return c.transactf("store %s %s (%s)", seqset, item, strings.Join(flags, " "))
+	return c.Transactf("store %s %s (%s)", seqset, item, strings.Join(flags, " "))
 }
 
-// MSNStoreFlagsAdd is like [Conn.MSNStoreFlagsSet], but only adds flags, leaving
+// StoreFlagsAdd is like StoreFlagsSet, but only adds flags, leaving current flags on the message intact.
-// current flags on the message intact.
+func (c *Conn) StoreFlagsAdd(seqset string, silent bool, flags ...string) (untagged []Untagged, result Result, rerr error) {
-//
+	defer c.recover(&rerr)
-// Method [Conn.UIDStoreFlagsAdd], which operates on a uid set, should be
-// preferred.
-func (c *Conn) MSNStoreFlagsAdd(seqset string, silent bool, flags ...string) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
 	item := "+flags"
 	if silent {
 		item += ".silent"
 	}
-	return c.transactf("store %s %s (%s)", seqset, item, strings.Join(flags, " "))
+	return c.Transactf("store %s %s (%s)", seqset, item, strings.Join(flags, " "))
 }
 
-// MSNStoreFlagsClear is like [Conn.MSNStoreFlagsSet], but only removes flags,
+// StoreFlagsClear is like StoreFlagsSet, but only removes flags, leaving other flags on the message intact.
-// leaving other flags on the message intact.
+func (c *Conn) StoreFlagsClear(seqset string, silent bool, flags ...string) (untagged []Untagged, result Result, rerr error) {
-//
+	defer c.recover(&rerr)
-// Method [Conn.UIDStoreFlagsClear], which operates on a uid set, should be
-// preferred.
-func (c *Conn) MSNStoreFlagsClear(seqset string, silent bool, flags ...string) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
 	item := "-flags"
 	if silent {
 		item += ".silent"
 	}
-	return c.transactf("store %s %s (%s)", seqset, item, strings.Join(flags, " "))
+	return c.Transactf("store %s %s (%s)", seqset, item, strings.Join(flags, " "))
 }
 
-// UIDStoreFlagsSet stores a new set of flags for messages matching UIDs from
+// Copy adds messages from the sequences in seqSet in the currently selected/active mailbox to dstMailbox.
-// uidSet with the IMAP4 "UID STORE" command.
+func (c *Conn) Copy(seqSet NumSet, dstMailbox string) (untagged []Untagged, result Result, rerr error) {
-//
+	defer c.recover(&rerr)
-// If silent, no untagged responses with the updated flags will be sent by the
+	return c.Transactf("copy %s %s", seqSet.String(), astring(dstMailbox))
-// server.
-//
-// Required capability: "UIDPLUS" or "IMAP4rev2".
-func (c *Conn) UIDStoreFlagsSet(uidSet string, silent bool, flags ...string) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-	item := "flags"
-	if silent {
-		item += ".silent"
-	}
-	return c.transactf("uid store %s %s (%s)", uidSet, item, strings.Join(flags, " "))
 }
 
-// UIDStoreFlagsAdd is like UIDStoreFlagsSet, but only adds flags, leaving
+// UIDCopy is like copy, but operates on UIDs.
-// current flags on the message intact.
+func (c *Conn) UIDCopy(uidSet NumSet, dstMailbox string) (untagged []Untagged, result Result, rerr error) {
-//
+	defer c.recover(&rerr)
-// Required capability: "UIDPLUS" or "IMAP4rev2".
+	return c.Transactf("uid copy %s %s", uidSet.String(), astring(dstMailbox))
-func (c *Conn) UIDStoreFlagsAdd(uidSet string, silent bool, flags ...string) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-	item := "+flags"
-	if silent {
-		item += ".silent"
-	}
-	return c.transactf("uid store %s %s (%s)", uidSet, item, strings.Join(flags, " "))
 }
 
-// UIDStoreFlagsClear is like UIDStoreFlagsSet, but only removes flags, leaving
+// Move moves messages from the sequences in seqSet in the currently selected/active mailbox to dstMailbox.
-// other flags on the message intact.
+func (c *Conn) Move(seqSet NumSet, dstMailbox string) (untagged []Untagged, result Result, rerr error) {
-//
+	defer c.recover(&rerr)
-// Required capability: "UIDPLUS" or "IMAP4rev2".
+	return c.Transactf("move %s %s", seqSet.String(), astring(dstMailbox))
-func (c *Conn) UIDStoreFlagsClear(uidSet string, silent bool, flags ...string) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-	item := "-flags"
-	if silent {
-		item += ".silent"
-	}
-	return c.transactf("uid store %s %s (%s)", uidSet, item, strings.Join(flags, " "))
 }
 
-// MSNCopy adds messages from the sequences in the sequence set in the
+// UIDMove is like move, but operates on UIDs.
-// selected/active mailbox to destMailbox using the IMAP4 "COPY" command.
+func (c *Conn) UIDMove(uidSet NumSet, dstMailbox string) (untagged []Untagged, result Result, rerr error) {
-//
+	defer c.recover(&rerr)
-// Method [Conn.UIDCopy], operating on UIDs instead of sequence numbers, should be
+	return c.Transactf("uid move %s %s", uidSet.String(), astring(dstMailbox))
-// preferred.
-func (c *Conn) MSNCopy(seqSet string, destMailbox string) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-	return c.transactf("copy %s %s", seqSet, astring(destMailbox))
-}
-
-// UIDCopy is like copy, but operates on UIDs, using the IMAP4 "UID COPY" command.
-//
-// Required capability: "UIDPLUS" or "IMAP4rev2".
-func (c *Conn) UIDCopy(uidSet string, destMailbox string) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-	return c.transactf("uid copy %s %s", uidSet, astring(destMailbox))
-}
-
-// MSNSearch returns messages from the sequence set in the selected/active mailbox
-// that match the search critera using the IMAP4 "SEARCH" command.
-//
-// Method [Conn.UIDSearch], operating on UIDs instead of sequence numbers, should be
-// preferred.
-func (c *Conn) MSNSearch(seqSet string, criteria string) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-	return c.transactf("seach %s %s", seqSet, criteria)
-}
-
-// UIDSearch returns messages from the uid set in the selected/active mailbox that
-// match the search critera using the IMAP4 "SEARCH" command.
-//
-// Criteria is a search program, see RFC 9051 and RFC 3501 for details.
-//
-// Required capability: "UIDPLUS" or "IMAP4rev2".
-func (c *Conn) UIDSearch(seqSet string, criteria string) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-	return c.transactf("seach %s %s", seqSet, criteria)
-}
-
-// MSNMove moves messages from the sequence set in the selected/active mailbox to
-// destMailbox using the IMAP4 "MOVE" command.
-//
-// Required capability: "MOVE" or "IMAP4rev2".
-//
-// Method [Conn.UIDMove], operating on UIDs instead of sequence numbers, should be
-// preferred.
-func (c *Conn) MSNMove(seqSet string, destMailbox string) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-	return c.transactf("move %s %s", seqSet, astring(destMailbox))
-}
-
-// UIDMove is like move, but operates on UIDs, using the IMAP4 "UID MOVE" command.
-//
-// Required capability: "MOVE" or "IMAP4rev2".
-func (c *Conn) UIDMove(uidSet string, destMailbox string) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-	return c.transactf("uid move %s %s", uidSet, astring(destMailbox))
-}
-
-// MSNReplace is like the preferred [Conn.UIDReplace], but operates on a message
-// sequence number (MSN) instead of a UID.
-//
-// Required capability: "REPLACE".
-//
-// Method [Conn.UIDReplace], operating on UIDs instead of sequence numbers, should be
-// preferred.
-func (c *Conn) MSNReplace(msgseq string, mailbox string, msg Append) (resp Response, rerr error) {
-	// todo: parse msgseq, must be nznumber, with a known msgseq. or "*" with at least one message.
-	return c.replace("replace", msgseq, mailbox, msg)
-}
-
-// UIDReplace uses the IMAP4 "UID REPLACE" command to replace a message from the
-// selected/active mailbox with a new/different version of the message in the named
-// mailbox, which may be the same or different than the selected mailbox.
-//
-// The replaced message is indicated by uid.
-//
-// Required capability: "REPLACE".
-func (c *Conn) UIDReplace(uid string, mailbox string, msg Append) (resp Response, rerr error) {
-	// todo: parse uid, must be nznumber, with a known uid. or "*" with at least one message.
-	return c.replace("uid replace", uid, mailbox, msg)
-}
-
-func (c *Conn) replace(cmd string, num string, mailbox string, msg Append) (resp Response, rerr error) {
-	defer c.recover(&rerr, &resp)
-
-	// todo: use synchronizing literal for larger messages.
-
-	var date string
-	if msg.Received != nil {
-		date = ` "` + msg.Received.Format("_2-Jan-2006 15:04:05 -0700") + `"`
-	}
-	// todo: only use literal8 if needed, possibly with "UTF8()"
-	// todo: encode mailbox
-	err := c.WriteCommandf("", "%s %s %s (%s)%s ~{%d+}", cmd, num, astring(mailbox), strings.Join(msg.Flags, " "), date, msg.Size)
-	c.xcheckf(err, "writing replace command")
-
-	defer c.xtracewrite(mlog.LevelTracedata)()
-	_, err = io.Copy(c.xbw, msg.Data)
-	c.xcheckf(err, "write message data")
-	c.xtracewrite(mlog.LevelTrace)
-
-	fmt.Fprintf(c.xbw, "\r\n")
-	c.xflush()
-
-	return c.responseOK()
 }

imapclient/fuzz_test.go

@@ -1,38 +0,0 @@
-package imapclient
-
-import (
-	"os"
-	"strings"
-	"testing"
-)
-
-func FuzzParser(f *testing.F) {
-	/*
-		Gathering all untagged responses and command completion results from the RFCs:
-
-		cd ../rfc
-		(
-		  grep ' S: \* [A-Z]' * | sed 's/^.*S: //g'
-		  grep -E ' S: [^ *]+ (OK|NO|BAD) ' * | sed 's/^.*S: //g'
-		) | grep -v '\.\.\/' | sort | uniq >../testdata/imapclient/fuzzseed.txt
-	*/
-	buf, err := os.ReadFile("../testdata/imapclient/fuzzseed.txt")
-	if err != nil {
-		f.Fatalf("reading seed: %v", err)
-	}
-	for _, s := range strings.Split(string(buf), "\n") {
-		f.Add(s + "\r\n")
-	}
-	f.Add("1:3")
-	f.Add("3:1")
-	f.Add("3,1")
-	f.Add("*")
-
-	f.Fuzz(func(t *testing.T, data string) {
-		ParseUntagged(data)
-		ParseCode(data)
-		ParseResult(data)
-		ParseNumSet(data)
-		ParseUIDRange(data)
-	})
-}

imapclient/parse_test.go

@@ -1,42 +0,0 @@
-package imapclient
-
-import (
-	"fmt"
-	"reflect"
-	"testing"
-)
-
-func tcheckf(t *testing.T, err error, format string, args ...any) {
-	if err != nil {
-		t.Fatalf("%s: %s", fmt.Sprintf(format, args...), err)
-	}
-}
-
-func tcompare(t *testing.T, a, b any) {
-	if !reflect.DeepEqual(a, b) {
-		t.Fatalf("got:\n%#v\nexpected:\n%#v", a, b)
-	}
-}
-
-func uint32ptr(v uint32) *uint32 { return &v }
-
-func TestParse(t *testing.T) {
-	code, err := ParseCode("COPYUID 1 1:3 2:4")
-	tcheckf(t, err, "parsing code")
-	tcompare(t, code,
-		CodeCopyUID{
-			DestUIDValidity: 1,
-			From:            []NumRange{{First: 1, Last: uint32ptr(3)}},
-			To:              []NumRange{{First: 2, Last: uint32ptr(4)}},
-		},
-	)
-
-	ut, err := ParseUntagged("* BYE done\r\n")
-	tcheckf(t, err, "parsing untagged")
-	tcompare(t, ut, UntaggedBye{Text: "done"})
-
-	tag, result, err := ParseResult("tag1 OK [ALERT] Hello\r\n")
-	tcheckf(t, err, "parsing result")
-	tcompare(t, tag, "tag1")
-	tcompare(t, result, Result{Status: OK, Code: CodeWord("ALERT"), Text: "Hello"})
-}

imapclient/prefixconn.go

@@ -1,41 +0,0 @@
-package imapclient
-
-import (
-	"io"
-	"net"
-)
-
-// prefixConn is a net.Conn with a buffer from which the first reads are satisfied.
-// used for STARTTLS where already did a buffered read of initial TLS data.
-type prefixConn struct {
-	prefix []byte
-	net.Conn
-}
-
-func (c *prefixConn) Read(buf []byte) (int, error) {
-	if len(c.prefix) > 0 {
-		n := min(len(buf), len(c.prefix))
-		copy(buf[:n], c.prefix[:n])
-		c.prefix = c.prefix[n:]
-		if len(c.prefix) == 0 {
-			c.prefix = nil
-		}
-		return n, nil
-	}
-	return c.Conn.Read(buf)
-}
-
-// xprefixConn checks if there are any buffered unconsumed reads. If not, it
-// returns c.conn. Otherwise, it returns a *prefixConn from which the buffered data
-// can be read followed by data from c.conn.
-func (c *Conn) xprefixConn() net.Conn {
-	n := c.br.Buffered()
-	if n == 0 {
-		return c.conn
-	}
-
-	buf := make([]byte, n)
-	_, err := io.ReadFull(c.br, buf)
-	c.xcheckf(err, "get buffered data")
-	return &prefixConn{buf, c.conn}
-}

imapclient/protocol.go

@@ -2,57 +2,35 @@ package imapclient
 
 import (
 	"bufio"
-	"errors"
 	"fmt"
 	"strings"
-	"time"
 )
 
-// Capability is a known string for with the ENABLED command and response and
+// Capability is a known string for with the ENABLED and CAPABILITY command.
-// CAPABILITY responses. Servers could send unknown values. Always in upper case.
 type Capability string
 
 const (
-	CapIMAP4rev1           Capability = "IMAP4REV1"               // ../rfc/3501:1310
+	CapIMAP4rev1     Capability = "IMAP4rev1"
-	CapIMAP4rev2           Capability = "IMAP4REV2"               // ../rfc/9051:1219
+	CapIMAP4rev2     Capability = "IMAP4rev2"
-	CapLoginDisabled       Capability = "LOGINDISABLED"           // ../rfc/3501:3792 ../rfc/9051:5436
+	CapLoginDisabled Capability = "LOGINDISABLED"
-	CapStartTLS            Capability = "STARTTLS"                // ../rfc/3501:1327 ../rfc/9051:1238
+	CapStarttls      Capability = "STARTTLS"
-	CapAuthPlain           Capability = "AUTH=PLAIN"              // ../rfc/3501:1327 ../rfc/9051:1238
+	CapAuthPlain     Capability = "AUTH=PLAIN"
-	CapAuthExternal        Capability = "AUTH=EXTERNAL"           // ../rfc/4422:1575
+	CapLiteralPlus   Capability = "LITERAL+"
-	CapAuthSCRAMSHA256Plus Capability = "AUTH=SCRAM-SHA-256-PLUS" // ../rfc/7677:80
+	CapLiteralMinus  Capability = "LITERAL-"
-	CapAuthSCRAMSHA256     Capability = "AUTH=SCRAM-SHA-256"
+	CapIdle          Capability = "IDLE"
-	CapAuthSCRAMSHA1Plus   Capability = "AUTH=SCRAM-SHA-1-PLUS" // ../rfc/5802:465
+	CapNamespace     Capability = "NAMESPACE"
-	CapAuthSCRAMSHA1       Capability = "AUTH=SCRAM-SHA-1"
+	CapBinary        Capability = "BINARY"
-	CapAuthCRAMMD5         Capability = "AUTH=CRAM-MD5" // ../rfc/2195:80
+	CapUnselect      Capability = "UNSELECT"
-	CapLiteralPlus         Capability = "LITERAL+"      // ../rfc/2088:45
+	CapUidplus       Capability = "UIDPLUS"
-	CapLiteralMinus        Capability = "LITERAL-"      // ../rfc/7888:26 ../rfc/9051:847 Default since IMAP4rev2
+	CapEsearch       Capability = "ESEARCH"
-	CapIdle                Capability = "IDLE"          // ../rfc/2177:69 ../rfc/9051:3542 Default since IMAP4rev2
+	CapEnable        Capability = "ENABLE"
-	CapNamespace           Capability = "NAMESPACE"     // ../rfc/2342:130 ../rfc/9051:135 Default since IMAP4rev2
+	CapSave          Capability = "SAVE"
-	CapBinary              Capability = "BINARY"        // ../rfc/3516:100
+	CapListExtended  Capability = "LIST-EXTENDED"
-	CapUnselect            Capability = "UNSELECT"      // ../rfc/3691:78 ../rfc/9051:3667 Default since IMAP4rev2
+	CapSpecialUse    Capability = "SPECIAL-USE"
-	CapUidplus             Capability = "UIDPLUS"       // ../rfc/4315:36 ../rfc/9051:8015 Default since IMAP4rev2
+	CapMove          Capability = "MOVE"
-	CapEsearch             Capability = "ESEARCH"       // ../rfc/4731:69 ../rfc/9051:8016 Default since IMAP4rev2
+	CapUTF8Only      Capability = "UTF8=ONLY"
-	CapEnable              Capability = "ENABLE"        // ../rfc/5161:52 ../rfc/9051:8016 Default since IMAP4rev2
+	CapUTF8Accept    Capability = "UTF8=ACCEPT"
-	CapListExtended        Capability = "LIST-EXTENDED" // ../rfc/5258:150 ../rfc/9051:7987 Syntax except multiple mailboxes default since IMAP4rev2
+	CapID            Capability = "ID" // ../rfc/2971:80
-	CapSpecialUse          Capability = "SPECIAL-USE"   // ../rfc/6154:156 ../rfc/9051:8021 Special-use attributes in LIST responses by default since IMAP4rev2
-	CapMove                Capability = "MOVE"          // ../rfc/6851:87 ../rfc/9051:8018 Default since IMAP4rev2
-	CapUTF8Only            Capability = "UTF8=ONLY"
-	CapUTF8Accept          Capability = "UTF8=ACCEPT"
-	CapCondstore           Capability = "CONDSTORE"          // ../rfc/7162:411
-	CapQresync             Capability = "QRESYNC"            // ../rfc/7162:1376
-	CapID                  Capability = "ID"                 // ../rfc/2971:80
-	CapMetadata            Capability = "METADATA"           // ../rfc/5464:124
-	CapMetadataServer      Capability = "METADATA-SERVER"    // ../rfc/5464:124
-	CapSaveDate            Capability = "SAVEDATE"           // ../rfc/8514
-	CapCreateSpecialUse    Capability = "CREATE-SPECIAL-USE" // ../rfc/6154:296
-	CapCompressDeflate     Capability = "COMPRESS=DEFLATE"   // ../rfc/4978:65
-	CapListMetadata        Capability = "LIST-METADATA"      // ../rfc/9590:73
-	CapMultiAppend         Capability = "MULTIAPPEND"        // ../rfc/3502:33
-	CapReplace             Capability = "REPLACE"            // ../rfc/8508:155
-	CapPreview             Capability = "PREVIEW"            // ../rfc/8970:114
-	CapMultiSearch         Capability = "MULTISEARCH"        // ../rfc/7377:187
-	CapNotify              Capability = "NOTIFY"             // ../rfc/5465:195
-	CapUIDOnly             Capability = "UIDONLY"            // ../rfc/9586:129
 )
 
 // Status is the tagged final result of a command.
@@ -64,144 +42,73 @@ const (
 	OK  Status = "OK"  // Command succeeded.
 )
 
-// Response is a response to an IMAP command including any preceding untagged
-// responses. Response implements the error interface through result.
-//
-// See [UntaggedResponseGet] and [UntaggedResponseList] to retrieve specific types
-// of untagged responses.
-type Response struct {
-	Untagged []Untagged
-	Result
-}
-
-var (
-	ErrMissing  = errors.New("no response of type")        // Returned by UntaggedResponseGet.
-	ErrMultiple = errors.New("multiple responses of type") // Idem.
-)
-
-// UntaggedResponseGet returns the single untagged response of type T. Only
-// [ErrMissing] or [ErrMultiple] can be returned as error.
-func UntaggedResponseGet[T Untagged](resp Response) (T, error) {
-	var t T
-	var have bool
-	for _, e := range resp.Untagged {
-		if tt, ok := e.(T); ok {
-			if have {
-				return t, ErrMultiple
-			}
-			t = tt
-		}
-	}
-	if !have {
-		return t, ErrMissing
-	}
-	return t, nil
-}
-
-// UntaggedResponseList returns all untagged responses of type T.
-func UntaggedResponseList[T Untagged](resp Response) []T {
-	var l []T
-	for _, e := range resp.Untagged {
-		if tt, ok := e.(T); ok {
-			l = append(l, tt)
-		}
-	}
-	return l
-}
-
 // Result is the final response for a command, indicating success or failure.
 type Result struct {
 	Status Status
-	Code   Code   // Set if response code is present.
+	RespText
-	Text   string // Any remaining text.
 }
 
-func (r Result) Error() string {
+// CodeArg represents a response code with arguments, i.e. the data between [] in the response line.
-	s := fmt.Sprintf("IMAP result %s", r.Status)
+type CodeArg interface {
-	if r.Code != nil {
+	CodeString() string
-		s += "[" + r.Code.CodeString() + "]"
+}
-	}
+
-	if r.Text != "" {
+// CodeOther is a valid but unrecognized response code.
-		s += " " + r.Text
+type CodeOther struct {
+	Code string
+	Args []string
+}
+
+func (c CodeOther) CodeString() string {
+	return c.Code + " " + strings.Join(c.Args, " ")
+}
+
+// CodeWords is a code with space-separated string parameters. E.g. CAPABILITY.
+type CodeWords struct {
+	Code string
+	Args []string
+}
+
+func (c CodeWords) CodeString() string {
+	s := c.Code
+	for _, w := range c.Args {
+		s += " " + w
 	}
 	return s
 }
 
-// Code represents a response code with optional arguments, i.e. the data between [] in the response line.
+// CodeList is a code with a list with space-separated strings as parameters. E.g. BADCHARSET, PERMANENTFLAGS.
-type Code interface {
+type CodeList struct {
-	CodeString() string
+	Code string
+	Args []string // If nil, no list was present. List can also be empty.
 }
 
-// CodeWord is a response code without parameters, always in upper case.
+func (c CodeList) CodeString() string {
-type CodeWord string
+	s := c.Code
-
+	if c.Args == nil {
-func (c CodeWord) CodeString() string {
-	return string(c)
-}
-
-// CodeOther is an unrecognized response code with parameters.
-type CodeParams struct {
-	Code string // Always in upper case.
-	Args []string
-}
-
-func (c CodeParams) CodeString() string {
-	return c.Code + " " + strings.Join(c.Args, " ")
-}
-
-// CodeCapability is a CAPABILITY response code with the capabilities supported by the server.
-type CodeCapability []Capability
-
-func (c CodeCapability) CodeString() string {
-	var s string
-	for _, c := range c {
-		s += " " + string(c)
-	}
-	return "CAPABILITY" + s
-}
-
-type CodeBadCharset []string
-
-func (c CodeBadCharset) CodeString() string {
-	s := "BADCHARSET"
-	if len(c) == 0 {
 		return s
 	}
-	return s + " (" + strings.Join([]string(c), " ") + ")"
+	return s + "(" + strings.Join(c.Args, " ") + ")"
 }
 
-type CodePermanentFlags []string
+// CodeUint is a code with a uint32 parameter, e.g. UIDNEXT and UIDVALIDITY.
-
+type CodeUint struct {
-func (c CodePermanentFlags) CodeString() string {
+	Code string
-	return "PERMANENTFLAGS (" + strings.Join([]string(c), " ") + ")"
+	Num  uint32
 }
 
-type CodeUIDNext uint32
+func (c CodeUint) CodeString() string {
-
+	return fmt.Sprintf("%s %d", c.Code, c.Num)
-func (c CodeUIDNext) CodeString() string {
-	return fmt.Sprintf("UIDNEXT %d", c)
-}
-
-type CodeUIDValidity uint32
-
-func (c CodeUIDValidity) CodeString() string {
-	return fmt.Sprintf("UIDVALIDITY %d", c)
-}
-
-type CodeUnseen uint32
-
-func (c CodeUnseen) CodeString() string {
-	return fmt.Sprintf("UNSEEN %d", c)
 }
 
 // "APPENDUID" response code.
 type CodeAppendUID struct {
 	UIDValidity uint32
-	UIDs        NumRange
+	UID         uint32
 }
 
 func (c CodeAppendUID) CodeString() string {
-	return fmt.Sprintf("APPENDUID %d %s", c.UIDValidity, c.UIDs.String())
+	return fmt.Sprintf("APPENDUID %d %d", c.UIDValidity, c.UID)
 }
 
 // "COPYUID" response code.
@@ -242,66 +149,11 @@ func (c CodeHighestModSeq) CodeString() string {
 	return fmt.Sprintf("HIGHESTMODSEQ %d", c)
 }
 
-// "INPROGRESS" response code.
+// RespText represents a response line minus the leading tag.
-type CodeInProgress struct {
+type RespText struct {
-	Tag     string // Nil is empty string.
+	Code    string  // The first word between [] after the status.
-	Current *uint32
+	CodeArg CodeArg // Set if code has a parameter.
-	Goal    *uint32
+	More    string  // Any remaining text.
-}
-
-func (c CodeInProgress) CodeString() string {
-	// ABNF allows inprogress-tag/state with all nil values. Doesn't seem useful enough
-	// to keep track of.
-	if c.Tag == "" && c.Current == nil && c.Goal == nil {
-		return "INPROGRESS"
-	}
-
-	// todo: quote tag properly
-	current := "nil"
-	goal := "nil"
-	if c.Current != nil {
-		current = fmt.Sprintf("%d", *c.Current)
-	}
-	if c.Goal != nil {
-		goal = fmt.Sprintf("%d", *c.Goal)
-	}
-	return fmt.Sprintf("INPROGRESS (%q %s %s)", c.Tag, current, goal)
-}
-
-// "BADEVENT" response code, with the events that are supported, for the NOTIFY
-// extension.
-type CodeBadEvent []string
-
-func (c CodeBadEvent) CodeString() string {
-	return fmt.Sprintf("BADEVENT (%s)", strings.Join([]string(c), " "))
-}
-
-// "METADATA LONGENTRIES number" response for GETMETADATA command.
-type CodeMetadataLongEntries uint32
-
-func (c CodeMetadataLongEntries) CodeString() string {
-	return fmt.Sprintf("METADATA LONGENTRIES %d", c)
-}
-
-// "METADATA (MAXSIZE number)" response for SETMETADATA command.
-type CodeMetadataMaxSize uint32
-
-func (c CodeMetadataMaxSize) CodeString() string {
-	return fmt.Sprintf("METADATA (MAXSIZE %d)", c)
-}
-
-// "METADATA (TOOMANY)" response for SETMETADATA command.
-type CodeMetadataTooMany struct{}
-
-func (c CodeMetadataTooMany) CodeString() string {
-	return "METADATA (TOOMANY)"
-}
-
-// "METADATA (NOPRIVATE)" response for SETMETADATA command.
-type CodeMetadataNoPrivate struct{}
-
-func (c CodeMetadataNoPrivate) CodeString() string {
-	return "METADATA (NOPRIVATE)"
 }
 
 // atom or string.
@@ -342,30 +194,17 @@ func syncliteral(s string) string {
 // todo: make an interface that the untagged responses implement?
 type Untagged any
 
-type UntaggedBye struct {
+type UntaggedBye RespText
-	Code Code   // Set if response code is present.
+type UntaggedPreauth RespText
-	Text string // Any remaining text.
-}
-type UntaggedPreauth struct {
-	Code Code   // Set if response code is present.
-	Text string // Any remaining text.
-}
 type UntaggedExpunge uint32
 type UntaggedExists uint32
 type UntaggedRecent uint32
-
+type UntaggedCapability []string
-// UntaggedCapability lists all capabilities the server implements.
+type UntaggedEnabled []string
-type UntaggedCapability []Capability
-
-// UntaggedEnabled indicates the capabilities that were enabled on the connection
-// by the server, typically in response to an ENABLE command.
-type UntaggedEnabled []Capability
-
 type UntaggedResult Result
 type UntaggedFlags []string
 type UntaggedList struct {
 	// ../rfc/9051:6690
-
 	Flags     []string
 	Separator byte // 0 for NIL
 	Mailbox   string
@@ -376,19 +215,10 @@ type UntaggedFetch struct {
 	Seq   uint32
 	Attrs []FetchAttr
 }
-
-// UntaggedUIDFetch is like UntaggedFetch, but with UIDs instead of message
-// sequence numbers, and returned instead of regular fetch responses when UIDONLY
-// is enabled.
-type UntaggedUIDFetch struct {
-	UID   uint32
-	Attrs []FetchAttr
-}
 type UntaggedSearch []uint32
 
+// ../rfc/7162:1101
 type UntaggedSearchModSeq struct {
-	// ../rfc/7162:1101
-
 	Nums   []uint32
 	ModSeq int64
 }
@@ -397,35 +227,8 @@ type UntaggedStatus struct {
 	Attrs   map[StatusAttr]int64 // Upper case status attributes.
 }
 
-// Unsolicited response, indicating an annotation has changed.
-type UntaggedMetadataKeys struct {
-	// ../rfc/5464:716
-
-	Mailbox string // Empty means not specific to mailbox.
-
-	// Keys that have changed. To get values (or determine absence), the server must be
-	// queried.
-	Keys []string
-}
-
-// Annotation is a metadata server of mailbox annotation.
-type Annotation struct {
-	Key string
-	// Nil is represented by IsString false and a nil Value.
-	IsString bool
-	Value    []byte
-}
-
-type UntaggedMetadataAnnotations struct {
-	// ../rfc/5464:683
-
-	Mailbox     string // Empty means not specific to mailbox.
-	Annotations []Annotation
-}
-
-type StatusAttr string
-
 // ../rfc/9051:7059 ../9208:712
+type StatusAttr string
 
 const (
 	StatusMessages       StatusAttr = "MESSAGES"
@@ -445,7 +248,6 @@ type UntaggedNamespace struct {
 }
 type UntaggedLsub struct {
 	// ../rfc/3501:4833
-
 	Flags     []string
 	Separator byte
 	Mailbox   string
@@ -453,17 +255,15 @@ type UntaggedLsub struct {
 
 // Fields are optional and zero if absent.
 type UntaggedEsearch struct {
-	Tag         string // ../rfc/9051:6546
+	// ../rfc/9051:6546
-	Mailbox     string // For MULTISEARCH. ../rfc/7377:437
+	Correlator string
-	UIDValidity uint32 // For MULTISEARCH, ../rfc/7377:438
+	UID        bool
-
+	Min        uint32
-	UID    bool
+	Max        uint32
-	Min    uint32
+	All        NumSet
-	Max    uint32
+	Count      *uint32
-	All    NumSet
+	ModSeq     int64
-	Count  *uint32
+	Exts       []EsearchDataExt
-	ModSeq int64
-	Exts   []EsearchDataExt
 }
 
 // UntaggedVanished is used in QRESYNC to send UIDs that have been removed.
@@ -515,7 +315,6 @@ type EsearchDataExt struct {
 
 type NamespaceDescr struct {
 	// ../rfc/9051:6769
-
 	Prefix    string
 	Separator byte // If 0 then separator was absent.
 	Exts      []NamespaceExtension
@@ -523,14 +322,13 @@ type NamespaceDescr struct {
 
 type NamespaceExtension struct {
 	// ../rfc/9051:6773
-
 	Key    string
 	Values []string
 }
 
 // FetchAttr represents a FETCH response attribute.
 type FetchAttr interface {
-	Attr() string // Name of attribute in upper case, e.g. "UID".
+	Attr() string // Name of attribute.
 }
 
 type NumSet struct {
@@ -557,19 +355,12 @@ func (ns NumSet) String() string {
 }
 
 func ParseNumSet(s string) (ns NumSet, rerr error) {
-	c := Proto{br: bufio.NewReader(strings.NewReader(s))}
+	c := Conn{r: bufio.NewReader(strings.NewReader(s))}
 	defer c.recover(&rerr)
 	ns = c.xsequenceSet()
 	return
 }
 
-func ParseUIDRange(s string) (nr NumRange, rerr error) {
-	c := Proto{br: bufio.NewReader(strings.NewReader(s))}
-	defer c.recover(&rerr)
-	nr = c.xuidrange()
-	return
-}
-
 // NumRange is a single number or range.
 type NumRange struct {
 	First uint32  // 0 for "*".
@@ -603,7 +394,6 @@ type TaggedExtComp struct {
 
 type TaggedExtVal struct {
 	// ../rfc/9051:7111
-
 	Number *int64
 	SeqSet *NumSet
 	Comp   *TaggedExtComp // If SimpleNumber and SimpleSeqSet is nil, this is a Comp. But Comp is optional and can also be nil. Not great.
@@ -611,7 +401,6 @@ type TaggedExtVal struct {
 
 type MboxListExtendedItem struct {
 	// ../rfc/9051:6699
-
 	Tag string
 	Val TaggedExtVal
 }
@@ -640,21 +429,9 @@ type Address struct {
 }
 
 // "INTERNALDATE" fetch response.
-type FetchInternalDate struct {
+type FetchInternalDate string            // todo: parsed time
-	Date time.Time
-}
-
 func (f FetchInternalDate) Attr() string { return "INTERNALDATE" }
 
-// "SAVEDATE" fetch response.
-type FetchSaveDate struct {
-	// ../rfc/8514:265
-
-	SaveDate *time.Time // nil means absent for message.
-}
-
-func (f FetchSaveDate) Attr() string { return "SAVEDATE" }
-
 // "RFC822.SIZE" fetch response.
 type FetchRFC822Size int64
 
@@ -678,7 +455,6 @@ func (f FetchRFC822Text) Attr() string { return "RFC822.TEXT" }
 // "BODYSTRUCTURE" fetch response.
 type FetchBodystructure struct {
 	// ../rfc/9051:6355
-
 	RespAttr string
 	Body     any // BodyType*
 }
@@ -688,7 +464,6 @@ func (f FetchBodystructure) Attr() string { return f.RespAttr }
 // "BODY" fetch response.
 type FetchBody struct {
 	// ../rfc/9051:6756 ../rfc/9051:6985
-
 	RespAttr string
 	Section  string // todo: parse more ../rfc/9051:6985
 	Offset   int32
@@ -708,7 +483,6 @@ type BodyFields struct {
 // subparts and the multipart media subtype. Used in a FETCH response.
 type BodyTypeMpart struct {
 	// ../rfc/9051:6411
-
 	Bodies       []any // BodyTypeBasic, BodyTypeMsg, BodyTypeText
 	MediaSubtype string
 	Ext          *BodyExtensionMpart
@@ -718,7 +492,6 @@ type BodyTypeMpart struct {
 // response.
 type BodyTypeBasic struct {
 	// ../rfc/9051:6407
-
 	MediaType, MediaSubtype string
 	BodyFields              BodyFields
 	Ext                     *BodyExtension1Part
@@ -728,7 +501,6 @@ type BodyTypeBasic struct {
 // response.
 type BodyTypeMsg struct {
 	// ../rfc/9051:6415
-
 	MediaType, MediaSubtype string
 	BodyFields              BodyFields
 	Envelope                Envelope
@@ -741,7 +513,6 @@ type BodyTypeMsg struct {
 // response.
 type BodyTypeText struct {
 	// ../rfc/9051:6418
-
 	MediaType, MediaSubtype string
 	BodyFields              BodyFields
 	Lines                   int64
@@ -750,42 +521,26 @@ type BodyTypeText struct {
 
 // BodyExtension1Part has the extensible form fields of a BODYSTRUCTURE for
 // multiparts.
-//
-// Fields in this struct are optional in IMAP4, and can be NIL or contain a value.
-// The first field is always present, otherwise the "parent" struct would have a
-// nil *BodyExtensionMpart. The second and later fields are nil when absent. For
-// non-reference types (e.g. strings), an IMAP4 NIL is represented as a pointer to
-// (*T)(nil). For reference types (e.g. slices), an IMAP4 NIL is represented by a
-// pointer to nil.
 type BodyExtensionMpart struct {
 	// ../rfc/9051:5986 ../rfc/3501:4161 ../rfc/9051:6371 ../rfc/3501:4599
-
 	Params            [][2]string
-	Disposition       **string
+	Disposition       string
-	DispositionParams *[][2]string
+	DispositionParams [][2]string
-	Language          *[]string
+	Language          []string
-	Location          **string
+	Location          string
-	More              []BodyExtension // Nil if absent.
+	More              []BodyExtension
 }
 
 // BodyExtension1Part has the extensible form fields of a BODYSTRUCTURE for
 // non-multiparts.
-//
-// Fields in this struct are optional in IMAP4, and can be NIL or contain a value.
-// The first field is always present, otherwise the "parent" struct would have a
-// nil *BodyExtensionMpart. The second and later fields are nil when absent. For
-// non-reference types (e.g. strings), an IMAP4 NIL is represented as a pointer to
-// (*T)(nil). For reference types (e.g. slices), an IMAP4 NIL is represented by a
-// pointer to nil.
 type BodyExtension1Part struct {
 	// ../rfc/9051:6023 ../rfc/3501:4191 ../rfc/9051:6366 ../rfc/3501:4584
-
+	MD5               string
-	MD5               *string
+	Disposition       string
-	Disposition       **string
+	DispositionParams [][2]string
-	DispositionParams *[][2]string
+	Language          []string
-	Language          *[]string
+	Location          string
-	Location          **string
+	More              []BodyExtension
-	More              []BodyExtension // Nil means absent.
 }
 
 // BodyExtension has the additional extension fields for future expansion of
@@ -823,12 +578,3 @@ func (f FetchUID) Attr() string { return "UID" }
 type FetchModSeq int64
 
 func (f FetchModSeq) Attr() string { return "MODSEQ" }
-
-// "PREVIEW" fetch response.
-type FetchPreview struct {
-	Preview *string
-}
-
-// ../rfc/8970:146
-
-func (f FetchPreview) Attr() string { return "PREVIEW" }

imapserver/append_test.go

@@ -7,30 +7,22 @@ import (
 )
 
 func TestAppend(t *testing.T) {
-	testAppend(t, false)
-}
-
-func TestAppendUIDOnly(t *testing.T) {
-	testAppend(t, true)
-}
-
-func testAppend(t *testing.T, uidonly bool) {
 	defer mockUIDValidity()()
 
-	tc := start(t, uidonly) // note: with switchboard because this connection stays alive unlike tc2.
+	tc := start(t) // note: with switchboard because this connection stays alive unlike tc2.
 	defer tc.close()
 
-	tc2 := startNoSwitchboard(t, uidonly) // note: without switchboard because this connection will break during tests.
+	tc2 := startNoSwitchboard(t) // note: without switchboard because this connection will break during tests.
-	defer tc2.closeNoWait()
+	defer tc2.close()
 
-	tc3 := startNoSwitchboard(t, uidonly)
+	tc3 := startNoSwitchboard(t)
-	defer tc3.closeNoWait()
+	defer tc3.close()
 
-	tc2.login("mjl@mox.example", password0)
+	tc2.client.Login("mjl@mox.example", password0)
 	tc2.client.Select("inbox")
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
 	tc.client.Select("inbox")
-	tc3.login("mjl@mox.example", password0)
+	tc3.client.Login("mjl@mox.example", password0)
 
 	tc2.transactf("bad", "append")              // Missing params.
 	tc2.transactf("bad", `append inbox`)        // Missing message.
@@ -38,44 +30,43 @@ func testAppend(t *testing.T, uidonly bool) {
 
 	// Syntax error for line ending in literal causes connection abort.
 	tc2.transactf("bad", "append inbox (\\Badflag) {1+}\r\nx") // Unknown flag.
-	tc2 = startNoSwitchboard(t, uidonly)
+	tc2 = startNoSwitchboard(t)
-	defer tc2.closeNoWait()
+	defer tc2.close()
-	tc2.login("mjl@mox.example", password0)
+	tc2.client.Login("mjl@mox.example", password0)
 	tc2.client.Select("inbox")
 
 	tc2.transactf("bad", "append inbox () \"bad time\" {1+}\r\nx") // Bad time.
-	tc2 = startNoSwitchboard(t, uidonly)
+	tc2 = startNoSwitchboard(t)
-	defer tc2.closeNoWait()
+	defer tc2.close()
-	tc2.login("mjl@mox.example", password0)
+	tc2.client.Login("mjl@mox.example", password0)
 	tc2.client.Select("inbox")
 
 	tc2.transactf("no", "append nobox (\\Seen) \" 1-Jan-2022 10:10:00 +0100\" {1}")
-	tc2.xcodeWord("TRYCREATE")
+	tc2.xcode("TRYCREATE")
-
-	tc2.transactf("no", "append expungebox (\\Seen) {1}")
-	tc2.xcodeWord("TRYCREATE")
 
 	tc2.transactf("ok", "append inbox (\\Seen Label1 $label2) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
 	tc2.xuntagged(imapclient.UntaggedExists(1))
-	tc2.xcode(imapclient.CodeAppendUID{UIDValidity: 1, UIDs: xparseUIDRange("1")})
+	tc2.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 1})
 
 	tc.transactf("ok", "noop")
+	uid1 := imapclient.FetchUID(1)
 	flags := imapclient.FetchFlags{`\Seen`, "$label2", "label1"}
-	tc.xuntagged(imapclient.UntaggedExists(1), tc.untaggedFetch(1, 1, flags))
+	tc.xuntagged(imapclient.UntaggedExists(1), imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, flags}})
 	tc3.transactf("ok", "noop")
 	tc3.xuntagged() // Inbox is not selected, nothing to report.
 
-	tc2.transactf("ok", "append inbox (\\Seen) \" 1-Jan-2022 10:10:00 +0100\" UTF8 (~{47+}\r\ncontent-type: just completely invalid;;\r\n\r\ntest)")
+	tc2.transactf("ok", "append inbox (\\Seen) \" 1-Jan-2022 10:10:00 +0100\" UTF8 ({47+}\r\ncontent-type: just completely invalid;;\r\n\r\ntest)")
 	tc2.xuntagged(imapclient.UntaggedExists(2))
-	tc2.xcode(imapclient.CodeAppendUID{UIDValidity: 1, UIDs: xparseUIDRange("2")})
+	tc2.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 2})
 
-	tc2.transactf("ok", "append inbox (\\Seen) \" 1-Jan-2022 10:10:00 +0100\" UTF8 (~{31+}\r\ncontent-type: text/plain;\n\ntest)")
+	tc2.transactf("ok", "append inbox (\\Seen) \" 1-Jan-2022 10:10:00 +0100\" UTF8 ({31+}\r\ncontent-type: text/plain;\n\ntest)")
 	tc2.xuntagged(imapclient.UntaggedExists(3))
-	tc2.xcode(imapclient.CodeAppendUID{UIDValidity: 1, UIDs: xparseUIDRange("3")})
+	tc2.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 3})
 
 	// Messages that we cannot parse are marked as application/octet-stream. Perhaps
 	// the imap client knows how to deal with them.
 	tc2.transactf("ok", "uid fetch 2 body")
+	uid2 := imapclient.FetchUID(2)
 	xbs := imapclient.FetchBodystructure{
 		RespAttr: "BODY",
 		Body: imapclient.BodyTypeBasic{
@@ -86,50 +77,16 @@ func testAppend(t *testing.T, uidonly bool) {
 			},
 		},
 	}
-	tc2.xuntagged(tc.untaggedFetch(2, 2, xbs))
+	tc2.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, xbs}})
 
-	// Multiappend with two messages.
+	tclimit := startArgs(t, false, false, true, true, "limit")
-	tc.transactf("ok", "noop") // Flush pending untagged responses.
-	tc.transactf("ok", "append inbox {6+}\r\ntest\r\n ~{6+}\r\ntost\r\n")
-	tc.xuntagged(imapclient.UntaggedExists(5))
-	tc.xcode(imapclient.CodeAppendUID{UIDValidity: 1, UIDs: xparseUIDRange("4:5")})
-
-	// Cancelled with zero-length message.
-	tc.transactf("no", "append inbox {6+}\r\ntest\r\n {0+}\r\n")
-
-	tclimit := startArgs(t, uidonly, false, false, true, true, "limit")
 	defer tclimit.close()
-	tclimit.login("limit@mox.example", password0)
+	tclimit.client.Login("limit@mox.example", password0)
 	tclimit.client.Select("inbox")
 	// First message of 1 byte is within limits.
 	tclimit.transactf("ok", "append inbox (\\Seen Label1 $label2) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
 	tclimit.xuntagged(imapclient.UntaggedExists(1))
 	// Second message would take account past limit.
 	tclimit.transactf("no", "append inbox (\\Seen Label1 $label2) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
-	tclimit.xcodeWord("OVERQUOTA")
+	tclimit.xcode("OVERQUOTA")
-
-	// Empty mailbox.
-	if uidonly {
-		tclimit.transactf("ok", `uid store 1 flags (\deleted)`)
-	} else {
-		tclimit.transactf("ok", `store 1 flags (\deleted)`)
-	}
-	tclimit.transactf("ok", "expunge")
-
-	// Multiappend with first message within quota, and second message with sync
-	// literal causing quota error. Request should get error response immediately.
-	tclimit.transactf("no", "append inbox {1+}\r\nx {100000}")
-	tclimit.xcodeWord("OVERQUOTA")
-
-	// Again, but second message now with non-sync literal, which is fully consumed by server.
-	tclimit.client.WriteCommandf("", "append inbox {1+}\r\nx {4000+}")
-	buf := make([]byte, 4000, 4002)
-	for i := range buf {
-		buf[i] = 'x'
-	}
-	buf = append(buf, "\r\n"...)
-	_, err := tclimit.client.Write(buf)
-	tclimit.check(err, "write append message")
-	tclimit.response("no")
-	tclimit.xcodeWord("OVERQUOTA")
 }

imapserver/authenticate_test.go

@@ -1,74 +1,65 @@
 package imapserver
 
 import (
-	"context"
 	"crypto/hmac"
 	"crypto/md5"
 	"crypto/sha1"
 	"crypto/sha256"
-	"crypto/tls"
 	"encoding/base64"
 	"errors"
 	"fmt"
 	"hash"
-	"net"
-	"os"
-	"path/filepath"
 	"strings"
 	"testing"
-	"time"
 
 	"golang.org/x/text/secure/precis"
 
-	"github.com/mjl-/mox/imapclient"
-	"github.com/mjl-/mox/mox-"
 	"github.com/mjl-/mox/scram"
-	"github.com/mjl-/mox/store"
 )
 
 func TestAuthenticateLogin(t *testing.T) {
 	// NFD username and PRECIS-cleaned password.
-	tc := start(t, false)
+	tc := start(t)
 	tc.client.Login("mo\u0301x@mox.example", password1)
 	tc.close()
 }
 
 func TestAuthenticatePlain(t *testing.T) {
-	tc := start(t, false)
+	tc := start(t)
 
 	tc.transactf("no", "authenticate bogus ")
 	tc.transactf("bad", "authenticate plain not base64...")
 	tc.transactf("no", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000baduser\u0000badpass")))
-	tc.xcodeWord("AUTHENTICATIONFAILED")
+	tc.xcode("AUTHENTICATIONFAILED")
 	tc.transactf("no", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000mjl@mox.example\u0000badpass")))
-	tc.xcodeWord("AUTHENTICATIONFAILED")
+	tc.xcode("AUTHENTICATIONFAILED")
 	tc.transactf("no", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000mjl\u0000badpass"))) // Need email, not account.
-	tc.xcodeWord("AUTHENTICATIONFAILED")
+	tc.xcode("AUTHENTICATIONFAILED")
 	tc.transactf("no", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000mjl@mox.example\u0000test")))
-	tc.xcodeWord("AUTHENTICATIONFAILED")
+	tc.xcode("AUTHENTICATIONFAILED")
 	tc.transactf("no", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000mjl@mox.example\u0000test"+password0)))
-	tc.xcodeWord("AUTHENTICATIONFAILED")
+	tc.xcode("AUTHENTICATIONFAILED")
 	tc.transactf("bad", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000")))
-	tc.xcode(nil)
+	tc.xcode("")
 	tc.transactf("no", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("other\u0000mjl@mox.example\u0000"+password0)))
-	tc.xcodeWord("AUTHORIZATIONFAILED")
+	tc.xcode("AUTHORIZATIONFAILED")
 	tc.transactf("ok", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000mjl@mox.example\u0000"+password0)))
 	tc.close()
 
-	tc = start(t, false)
+	tc = start(t)
 	tc.transactf("ok", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("mjl@mox.example\u0000mjl@mox.example\u0000"+password0)))
 	tc.close()
 
 	// NFD username and PRECIS-cleaned password.
-	tc = start(t, false)
+	tc = start(t)
 	tc.transactf("ok", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("mo\u0301x@mox.example\u0000mo\u0301x@mox.example\u0000"+password1)))
 	tc.close()
 
-	tc = start(t, false)
+	tc = start(t)
 	tc.client.AuthenticatePlain("mjl@mox.example", password0)
 	tc.close()
 
-	tc = start(t, false)
+	tc = start(t)
 	defer tc.close()
 
 	tc.cmdf("", "authenticate plain")
@@ -82,28 +73,6 @@ func TestAuthenticatePlain(t *testing.T) {
 	tc.readstatus("ok")
 }
 
-func TestLoginDisabled(t *testing.T) {
-	tc := start(t, false)
-	defer tc.close()
-
-	acc, err := store.OpenAccount(pkglog, "disabled", false)
-	tcheck(t, err, "open account")
-	err = acc.SetPassword(pkglog, "test1234")
-	tcheck(t, err, "set password")
-	err = acc.Close()
-	tcheck(t, err, "close account")
-
-	tc.transactf("no", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000disabled@mox.example\u0000test1234")))
-	tc.xcode(nil)
-	tc.transactf("no", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000disabled@mox.example\u0000bogus")))
-	tc.xcodeWord("AUTHENTICATIONFAILED")
-
-	tc.transactf("no", "login disabled@mox.example test1234")
-	tc.xcode(nil)
-	tc.transactf("no", "login disabled@mox.example bogus")
-	tc.xcodeWord("AUTHENTICATIONFAILED")
-}
-
 func TestAuthenticateSCRAMSHA1(t *testing.T) {
 	testAuthenticateSCRAM(t, false, "SCRAM-SHA-1", sha1.New)
 }
@@ -121,7 +90,7 @@ func TestAuthenticateSCRAMSHA256PLUS(t *testing.T) {
 }
 
 func testAuthenticateSCRAM(t *testing.T, tls bool, method string, h func() hash.Hash) {
-	tc := startArgs(t, false, true, tls, true, true, "mjl")
+	tc := startArgs(t, true, tls, true, true, "mjl")
 	tc.client.AuthenticateSCRAM(method, h, "mjl@mox.example", password0)
 	tc.close()
 
@@ -132,11 +101,15 @@ func testAuthenticateSCRAM(t *testing.T, tls bool, method string, h func() hash.
 		sc := scram.NewClient(h, username, "", noServerPlus, tc.client.TLSConnectionState())
 		clientFirst, err := sc.ClientFirst()
 		tc.check(err, "scram clientFirst")
-		tc.client.WriteCommandf("", "authenticate %s %s", method, base64.StdEncoding.EncodeToString([]byte(clientFirst)))
+		tc.client.LastTag = "x001"
+		tc.writelinef("%s authenticate %s %s", tc.client.LastTag, method, base64.StdEncoding.EncodeToString([]byte(clientFirst)))
 
 		xreadContinuation := func() []byte {
-			line, err := tc.client.ReadContinuation()
+			line, _, result, rerr := tc.client.ReadContinuation()
-			tcheck(t, err, "read continuation")
+			tc.check(rerr, "read continuation")
+			if result.Status != "" {
+				tc.t.Fatalf("expected continuation")
+			}
 			buf, err := base64.StdEncoding.DecodeString(line)
 			tc.check(err, "parsing base64 from remote")
 			return buf
@@ -159,14 +132,14 @@ func testAuthenticateSCRAM(t *testing.T, tls bool, method string, h func() hash.
 		} else {
 			tc.writelinef("")
 		}
-		resp, err := tc.client.ReadResponse()
+		_, result, err := tc.client.Response()
 		tc.check(err, "read response")
-		if string(resp.Status) != strings.ToUpper(status) {
+		if string(result.Status) != strings.ToUpper(status) {
-			tc.t.Fatalf("got status %q, expected %q", resp.Status, strings.ToUpper(status))
+			tc.t.Fatalf("got status %q, expected %q", result.Status, strings.ToUpper(status))
 		}
 	}
 
-	tc = startArgs(t, false, true, tls, true, true, "mjl")
+	tc = startArgs(t, true, tls, true, true, "mjl")
 	auth("no", scram.ErrInvalidProof, "mjl@mox.example", "badpass")
 	auth("no", scram.ErrInvalidProof, "mjl@mox.example", "")
 	// todo: server aborts due to invalid username. we should probably make client continue with fake determinisitically generated salt and result in error in the end.
@@ -183,7 +156,7 @@ func testAuthenticateSCRAM(t *testing.T, tls bool, method string, h func() hash.
 }
 
 func TestAuthenticateCRAMMD5(t *testing.T) {
-	tc := start(t, false)
+	tc := start(t)
 
 	tc.transactf("no", "authenticate bogus ")
 	tc.transactf("bad", "authenticate CRAM-MD5 not base64...")
@@ -193,11 +166,15 @@ func TestAuthenticateCRAMMD5(t *testing.T) {
 	auth := func(status string, username, password string) {
 		t.Helper()
 
-		tc.client.WriteCommandf("", "authenticate CRAM-MD5")
+		tc.client.LastTag = "x001"
+		tc.writelinef("%s authenticate CRAM-MD5", tc.client.LastTag)
 
 		xreadContinuation := func() []byte {
-			line, err := tc.client.ReadContinuation()
+			line, _, result, rerr := tc.client.ReadContinuation()
-			tcheck(t, err, "read continuation")
+			tc.check(rerr, "read continuation")
+			if result.Status != "" {
+				tc.t.Fatalf("expected continuation")
+			}
 			buf, err := base64.StdEncoding.DecodeString(line)
 			tc.check(err, "parsing base64 from remote")
 			return buf
@@ -210,13 +187,13 @@ func TestAuthenticateCRAMMD5(t *testing.T) {
 		}
 		h := hmac.New(md5.New, []byte(password))
 		h.Write([]byte(chal))
-		data := fmt.Sprintf("%s %x", username, h.Sum(nil))
+		resp := fmt.Sprintf("%s %x", username, h.Sum(nil))
-		tc.writelinef("%s", base64.StdEncoding.EncodeToString([]byte(data)))
+		tc.writelinef("%s", base64.StdEncoding.EncodeToString([]byte(resp)))
 
-		resp, err := tc.client.ReadResponse()
+		_, result, err := tc.client.Response()
 		tc.check(err, "read response")
-		if string(resp.Status) != strings.ToUpper(status) {
+		if string(result.Status) != strings.ToUpper(status) {
-			tc.t.Fatalf("got status %q, expected %q", resp.Status, strings.ToUpper(status))
+			tc.t.Fatalf("got status %q, expected %q", result.Status, strings.ToUpper(status))
 		}
 	}
 
@@ -229,154 +206,7 @@ func TestAuthenticateCRAMMD5(t *testing.T) {
 	tc.close()
 
 	// NFD username, with PRECIS-cleaned password.
-	tc = start(t, false)
+	tc = start(t)
 	auth("ok", "mo\u0301x@mox.example", password1)
 	tc.close()
 }
-
-func TestAuthenticateTLSClientCert(t *testing.T) {
-	tc := startArgsMore(t, false, true, true, nil, nil, true, true, "mjl", nil)
-	tc.transactf("no", "authenticate external ") // No TLS auth.
-	tc.close()
-
-	// Create a certificate, register its public key with account, and make a tls
-	// client config that sends the certificate.
-	clientCert0 := fakeCert(t, true)
-	clientConfig := tls.Config{
-		InsecureSkipVerify: true,
-		Certificates:       []tls.Certificate{clientCert0},
-	}
-
-	tlspubkey, err := store.ParseTLSPublicKeyCert(clientCert0.Certificate[0])
-	tcheck(t, err, "parse certificate")
-	tlspubkey.Account = "mjl"
-	tlspubkey.LoginAddress = "mjl@mox.example"
-	tlspubkey.NoIMAPPreauth = true
-
-	addClientCert := func() error {
-		return store.TLSPublicKeyAdd(ctxbg, &tlspubkey)
-	}
-
-	// No preauth, explicit authenticate with TLS.
-	tc = startArgsMore(t, false, true, true, nil, &clientConfig, false, true, "mjl", addClientCert)
-	if tc.client.Preauth {
-		t.Fatalf("preauthentication while not configured for tls public key")
-	}
-	tc.transactf("ok", "authenticate external ")
-	tc.close()
-
-	// External with explicit username.
-	tc = startArgsMore(t, false, true, true, nil, &clientConfig, false, true, "mjl", addClientCert)
-	if tc.client.Preauth {
-		t.Fatalf("preauthentication while not configured for tls public key")
-	}
-	tc.transactf("ok", "authenticate external %s", base64.StdEncoding.EncodeToString([]byte("mjl@mox.example")))
-	tc.close()
-
-	// No preauth, also allow other mechanisms.
-	tc = startArgsMore(t, false, true, true, nil, &clientConfig, false, true, "mjl", addClientCert)
-	tc.transactf("ok", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000mjl@mox.example\u0000"+password0)))
-	tc.close()
-
-	// No preauth, also allow other username for same account.
-	tc = startArgsMore(t, false, true, true, nil, &clientConfig, false, true, "mjl", addClientCert)
-	tc.transactf("ok", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000móx@mox.example\u0000"+password0)))
-	tc.close()
-
-	// No preauth, other mechanism must be for same account.
-	acc, err := store.OpenAccount(pkglog, "other", false)
-	tcheck(t, err, "open account")
-	err = acc.SetPassword(pkglog, "test1234")
-	tcheck(t, err, "set password")
-	err = acc.Close()
-	tcheck(t, err, "close account")
-	tc = startArgsMore(t, false, true, true, nil, &clientConfig, false, true, "mjl", addClientCert)
-	tc.transactf("no", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000other@mox.example\u0000test1234")))
-	tc.close()
-
-	// Starttls and external auth.
-	tc = startArgsMore(t, false, true, false, nil, &clientConfig, false, true, "mjl", addClientCert)
-	tc.client.StartTLS(&clientConfig)
-	tc.transactf("ok", "authenticate external =")
-	tc.close()
-
-	tlspubkey.NoIMAPPreauth = false
-	err = store.TLSPublicKeyUpdate(ctxbg, &tlspubkey)
-	tcheck(t, err, "update tls public key")
-
-	// With preauth, no authenticate command needed/allowed.
-	// Already set up tls session ticket cache, for next test.
-	serverConfig := tls.Config{
-		Certificates: []tls.Certificate{fakeCert(t, false)},
-	}
-	ctx, cancel := context.WithCancel(ctxbg)
-	defer cancel()
-	mox.StartTLSSessionTicketKeyRefresher(ctx, pkglog, &serverConfig)
-	clientConfig.ClientSessionCache = tls.NewLRUClientSessionCache(10)
-	tc = startArgsMore(t, false, true, true, &serverConfig, &clientConfig, false, true, "mjl", addClientCert)
-	if !tc.client.Preauth {
-		t.Fatalf("not preauthentication while configured for tls public key")
-	}
-	cs := tc.conn.(*tls.Conn).ConnectionState()
-	if cs.DidResume {
-		t.Fatalf("tls connection was resumed")
-	}
-	tc.transactf("no", "authenticate external ") // Not allowed, already in authenticated state.
-	tc.close()
-
-	// Authentication works with TLS resumption.
-	tc = startArgsMore(t, false, true, true, &serverConfig, &clientConfig, false, true, "mjl", addClientCert)
-	if !tc.client.Preauth {
-		t.Fatalf("not preauthentication while configured for tls public key")
-	}
-	cs = tc.conn.(*tls.Conn).ConnectionState()
-	if !cs.DidResume {
-		t.Fatalf("tls connection was not resumed")
-	}
-	// Check that operations that require an account work.
-	tc.client.Enable(imapclient.CapIMAP4rev2)
-	received, err := time.Parse(time.RFC3339, "2022-11-16T10:01:00+01:00")
-	tc.check(err, "parse time")
-	tc.client.Append("inbox", makeAppendTime(exampleMsg, received))
-	tc.client.Select("inbox")
-	tc.close()
-
-	// Authentication with unknown key should fail.
-	// todo: less duplication, change startArgs so this can be merged into it.
-	err = store.Close()
-	tcheck(t, err, "store close")
-	os.RemoveAll("../testdata/imap/data")
-	err = store.Init(ctxbg)
-	tcheck(t, err, "store init")
-	mox.ConfigStaticPath = filepath.FromSlash("../testdata/imap/mox.conf")
-	mox.MustLoadConfig(true, false)
-	switchStop := store.Switchboard()
-	defer switchStop()
-
-	serverConn, clientConn := net.Pipe()
-	defer clientConn.Close()
-
-	done := make(chan struct{})
-	defer func() { <-done }()
-	connCounter++
-	cid := connCounter
-	go func() {
-		defer serverConn.Close()
-		serve("test", cid, &serverConfig, serverConn, true, false, false, false, "")
-		close(done)
-	}()
-
-	clientConfig.ClientSessionCache = nil
-	clientConn = tls.Client(clientConn, &clientConfig)
-	// note: It's not enough to do a handshake and check if that was successful. If the
-	// client cert is not acceptable, we only learn after the handshake, when the first
-	// data messages are exchanged.
-	buf := make([]byte, 100)
-	_, err = clientConn.Read(buf)
-	if err == nil {
-		t.Fatalf("tls handshake with unknown client certificate succeeded")
-	}
-	if alert, ok := mox.AsTLSAlert(err); !ok || alert != 42 {
-		t.Fatalf("got err %#v, expected tls 'bad certificate' alert", err)
-	}
-}

imapserver/compress_test.go

@@ -1,82 +0,0 @@
-package imapserver
-
-import (
-	"crypto/tls"
-	"encoding/base64"
-	"io"
-	mathrand "math/rand/v2"
-	"testing"
-	"time"
-)
-
-func TestCompress(t *testing.T) {
-	tc := start(t, false)
-	defer tc.close()
-
-	tc.login("mjl@mox.example", password0)
-
-	tc.transactf("bad", "compress")
-	tc.transactf("bad", "compress bogus ")
-	tc.transactf("no", "compress bogus")
-
-	tc.client.CompressDeflate()
-	tc.transactf("no", "compress deflate") // Cannot have multiple.
-	tc.xcodeWord("COMPRESSIONACTIVE")
-
-	tc.client.Select("inbox")
-	tc.transactf("ok", "append inbox (\\seen) {%d+}\r\n%s", len(exampleMsg), exampleMsg)
-	tc.transactf("ok", "noop")
-	tc.transactf("ok", "fetch 1 body.peek[1]")
-}
-
-func TestCompressStartTLS(t *testing.T) {
-	tc := start(t, false)
-	defer tc.close()
-
-	tc.client.StartTLS(&tls.Config{InsecureSkipVerify: true})
-	tc.login("mjl@mox.example", password0)
-	tc.client.CompressDeflate()
-	tc.client.Select("inbox")
-	tc.transactf("ok", "append inbox (\\seen) {%d+}\r\n%s", len(exampleMsg), exampleMsg)
-	tc.transactf("ok", "noop")
-	tc.transactf("ok", "fetch 1 body.peek[1]")
-}
-
-func TestCompressBreak(t *testing.T) {
-	// Close the client connection when the server is writing. That causes writes in
-	// the server to fail (panic), jumping out of the flate writer and leaving its
-	// state inconsistent. We must not call into the flate writer again because due to
-	// its broken internal state it may cause array out of bounds accesses.
-
-	tc := start(t, false)
-	defer tc.close()
-
-	msg := exampleMsg
-	// Add random data (so it is not compressible). Don't know why, but only
-	// reproducible with large writes. As if setting socket buffers had no effect.
-	buf := make([]byte, 64*1024)
-	_, err := io.ReadFull(mathrand.NewChaCha8([32]byte{}), buf)
-	tcheck(t, err, "read random")
-	text := base64.StdEncoding.EncodeToString(buf)
-	for len(text) > 0 {
-		n := min(76, len(text))
-		msg += text[:n] + "\r\n"
-		text = text[n:]
-	}
-
-	tc.login("mjl@mox.example", password0)
-	tc.client.CompressDeflate()
-	tc.client.Select("inbox")
-	tc.transactf("ok", "append inbox (\\seen) {%d+}\r\n%s", len(msg), msg)
-	tc.transactf("ok", "noop")
-
-	// Write request. Close connection instead of reading data. Write will panic,
-	// coming through flate writer leaving its state inconsistent. Server must not try
-	// to Flush/Write again on flate writer or it may panic.
-	tc.client.Writelinef("x fetch 1 body.peek[1]")
-
-	// Close client connection and prevent cleanup from closing the client again.
-	time.Sleep(time.Second / 10)
-	tc.client = nil
-	tc.conn.Close() // Simulate client disappearing.
-}

imapserver/copy_test.go

@@ -7,25 +7,17 @@ import (
 )
 
 func TestCopy(t *testing.T) {
-	testCopy(t, false)
-}
-
-func TestCopyUIDOnly(t *testing.T) {
-	testCopy(t, true)
-}
-
-func testCopy(t *testing.T, uidonly bool) {
 	defer mockUIDValidity()()
-	tc := start(t, uidonly)
+	tc := start(t)
 	defer tc.close()
 
-	tc2 := startNoSwitchboard(t, uidonly)
+	tc2 := startNoSwitchboard(t)
-	defer tc2.closeNoWait()
+	defer tc2.close()
 
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
 	tc.client.Select("inbox")
 
-	tc2.login("mjl@mox.example", password0)
+	tc2.client.Login("mjl@mox.example", password0)
 	tc2.client.Select("Trash")
 
 	tc.transactf("bad", "copy")          // Missing params.
@@ -33,53 +25,48 @@ func testCopy(t *testing.T, uidonly bool) {
 	tc.transactf("bad", "copy 1 inbox ") // Leftover.
 
 	// Seqs 1,2 and UIDs 3,4.
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
-	tc.transactf("ok", `Uid Store 1:2 +Flags.Silent (\Deleted)`)
+	tc.client.StoreFlagsSet("1:2", true, `\Deleted`)
 	tc.client.Expunge()
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
 
-	if uidonly {
+	tc.transactf("no", "copy 1 nonexistent")
-		tc.transactf("ok", "uid copy 3:* Trash")
+	tc.xcode("TRYCREATE")
-	} else {
-		tc.transactf("no", "copy 1 nonexistent")
-		tc.xcodeWord("TRYCREATE")
-		tc.transactf("no", "copy 1 expungebox")
-		tc.xcodeWord("TRYCREATE")
 
-		tc.transactf("no", "copy 1 inbox") // Cannot copy to same mailbox.
+	tc.transactf("no", "copy 1 inbox") // Cannot copy to same mailbox.
 
-		tc2.transactf("ok", "noop") // Drain.
+	tc2.transactf("ok", "noop") // Drain.
 
-		tc.transactf("ok", "copy 1:* Trash")
+	tc.transactf("ok", "copy 1:* Trash")
-		tc.xcode(mustParseCode("COPYUID 1 3:4 1:2"))
+	ptr := func(v uint32) *uint32 { return &v }
-	}
+	tc.xcodeArg(imapclient.CodeCopyUID{DestUIDValidity: 1, From: []imapclient.NumRange{{First: 3, Last: ptr(4)}}, To: []imapclient.NumRange{{First: 1, Last: ptr(2)}}})
 	tc2.transactf("ok", "noop")
 	tc2.xuntagged(
 		imapclient.UntaggedExists(2),
-		tc2.untaggedFetch(1, 1, imapclient.FetchFlags(nil)),
+		imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil)}},
-		tc2.untaggedFetch(2, 2, imapclient.FetchFlags(nil)),
+		imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), imapclient.FetchFlags(nil)}},
 	)
 
 	tc.transactf("no", "uid copy 1,2 Trash") // No match.
 	tc.transactf("ok", "uid copy 4,3 Trash")
-	tc.xcode(mustParseCode("COPYUID 1 3:4 3:4"))
+	tc.xcodeArg(imapclient.CodeCopyUID{DestUIDValidity: 1, From: []imapclient.NumRange{{First: 3, Last: ptr(4)}}, To: []imapclient.NumRange{{First: 3, Last: ptr(4)}}})
 	tc2.transactf("ok", "noop")
 	tc2.xuntagged(
 		imapclient.UntaggedExists(4),
-		tc2.untaggedFetch(3, 3, imapclient.FetchFlags(nil)),
+		imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(3), imapclient.FetchFlags(nil)}},
-		tc2.untaggedFetch(4, 4, imapclient.FetchFlags(nil)),
+		imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), imapclient.FetchFlags(nil)}},
 	)
 
-	tclimit := startArgs(t, uidonly, false, false, true, true, "limit")
+	tclimit := startArgs(t, false, false, true, true, "limit")
 	defer tclimit.close()
-	tclimit.login("limit@mox.example", password0)
+	tclimit.client.Login("limit@mox.example", password0)
 	tclimit.client.Select("inbox")
 	// First message of 1 byte is within limits.
 	tclimit.transactf("ok", "append inbox (\\Seen Label1 $label2) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
 	tclimit.xuntagged(imapclient.UntaggedExists(1))
 	// Second message would take account past limit.
-	tclimit.transactf("no", "uid copy 1:* Trash")
+	tclimit.transactf("no", "copy 1:* Trash")
-	tclimit.xcodeWord("OVERQUOTA")
+	tclimit.xcode("OVERQUOTA")
 }

imapserver/create_test.go

@@ -7,42 +7,24 @@ import (
 )
 
 func TestCreate(t *testing.T) {
-	testCreate(t, false)
+	tc := start(t)
-}
-
-func TestCreateUIDOnly(t *testing.T) {
-	testCreate(t, true)
-}
-
-func testCreate(t *testing.T, uidonly bool) {
-	tc := start(t, uidonly)
 	defer tc.close()
 
-	tc2 := startNoSwitchboard(t, uidonly)
+	tc2 := startNoSwitchboard(t)
-	defer tc2.closeNoWait()
+	defer tc2.close()
 
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
-	tc2.login("mjl@mox.example", password0)
+	tc2.client.Login("mjl@mox.example", password0)
 
 	tc.transactf("no", "create inbox") // Already exists and not allowed. ../rfc/9051:1913
 	tc.transactf("no", "create Inbox") // Idem.
 
-	// Don't allow names that can cause trouble when exporting to directories.
-	tc.transactf("no", "create .")
-	tc.transactf("no", "create ..")
-	tc.transactf("no", "create legit/..")
-	tc.transactf("ok", "create ...") // No special meaning.
-
 	// ../rfc/9051:1937
 	tc.transactf("ok", "create inbox/a/c")
 	tc.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "Inbox/a"}, imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "Inbox/a/c"})
 
 	tc2.transactf("ok", "noop")
-	tc2.xuntagged(
+	tc2.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "Inbox/a"}, imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "Inbox/a/c"})
-		imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "..."},
-		imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "Inbox/a"},
-		imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "Inbox/a/c"},
-	)
 
 	tc.transactf("no", "create inbox/a/c") // Exists.
 
@@ -57,7 +39,7 @@ func testCreate(t *testing.T, uidonly bool) {
 	tc.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "mailbox"})
 
 	// OldName is only set for IMAP4rev2 or NOTIFY.
-	tc.client.Enable(imapclient.CapIMAP4rev2)
+	tc.client.Enable("imap4rev2")
 	tc.transactf("ok", "create mailbox2/")
 	tc.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "mailbox2", OldName: "mailbox2/"})
 
@@ -90,19 +72,8 @@ func testCreate(t *testing.T, uidonly bool) {
 	tc.transactf("no", `create "#"`) // Leading hash not allowed.
 	tc.transactf("ok", `create "test#"`)
 
-	// Create with flags.
-	tc.transactf("no", `create "newwithflags" (use (\unknown))`)
-	tc.transactf("no", `create "newwithflags" (use (\all))`)
-	tc.transactf("ok", `create "newwithflags" (use (\archive))`)
-	tc.transactf("ok", "noop")
-	tc.xuntagged()
-	tc.transactf("ok", `create "newwithflags2" (use (\archive) use (\drafts \sent))`)
-
 	// UTF-7 checks are only for IMAP4 before rev2 and without UTF8=ACCEPT.
 	tc.transactf("ok", `create "&"`)      // Interpreted as UTF-8, no UTF-7.
 	tc2.transactf("bad", `create "&"`)    // Bad UTF-7.
 	tc2.transactf("ok", `create "&Jjo-"`) // ☺, valid UTF-7.
-
-	tc.transactf("ok", "create expungebox") // Existed in past.
-	tc.transactf("ok", "delete expungebox") // Gone again.
 }

imapserver/delete_test.go

@@ -7,45 +7,36 @@ import (
 )
 
 func TestDelete(t *testing.T) {
-	testDelete(t, false)
+	tc := start(t)
-}
-
-func TestDeleteUIDOnly(t *testing.T) {
-	testDelete(t, false)
-}
-
-func testDelete(t *testing.T, uidonly bool) {
-	tc := start(t, uidonly)
 	defer tc.close()
 
-	tc2 := startNoSwitchboard(t, uidonly)
+	tc2 := startNoSwitchboard(t)
-	defer tc2.closeNoWait()
+	defer tc2.close()
 
-	tc3 := startNoSwitchboard(t, uidonly)
+	tc3 := startNoSwitchboard(t)
-	defer tc3.closeNoWait()
+	defer tc3.close()
 
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
-	tc2.login("mjl@mox.example", password0)
+	tc2.client.Login("mjl@mox.example", password0)
-	tc3.login("mjl@mox.example", password0)
+	tc3.client.Login("mjl@mox.example", password0)
 
 	tc.transactf("bad", "delete")              // Missing mailbox.
 	tc.transactf("no", "delete inbox")         // Cannot delete inbox.
 	tc.transactf("no", "delete nonexistent")   // Cannot delete mailbox that does not exist.
 	tc.transactf("no", `delete "nonexistent"`) // Again, with quoted string syntax.
-	tc.transactf("no", `delete "expungebox"`)  // Already removed.
 
 	tc.client.Subscribe("x")
 	tc.transactf("no", "delete x") // Subscription does not mean there is a mailbox that can be deleted.
 
-	tc.client.Create("a/b", nil)
+	tc.client.Create("a/b")
 	tc2.transactf("ok", "noop") // Drain changes.
 	tc3.transactf("ok", "noop")
 
 	// ../rfc/9051:2000
 	tc.transactf("no", "delete a") // Still has child.
-	tc.xcodeWord("HASCHILDREN")
+	tc.xcode("HASCHILDREN")
 
-	tc3.client.Enable(imapclient.CapIMAP4rev2) // For \NonExistent support.
+	tc3.client.Enable("IMAP4rev2") // For \NonExistent support.
 	tc.transactf("ok", "delete a/b")
 	tc2.transactf("ok", "noop")
 	tc2.xuntagged() // No IMAP4rev2, no \NonExistent.
@@ -62,12 +53,12 @@ func testDelete(t *testing.T, uidonly bool) {
 	)
 
 	// Let's try again with a message present.
-	tc.client.Create("msgs", nil)
+	tc.client.Create("msgs")
-	tc.client.Append("msgs", makeAppend(exampleMsg))
+	tc.client.Append("msgs", nil, nil, []byte(exampleMsg))
 	tc.transactf("ok", "delete msgs")
 
 	// Delete for inbox/* is allowed.
-	tc.client.Create("inbox/a", nil)
+	tc.client.Create("inbox/a")
 	tc.transactf("ok", "delete inbox/a")
 
 }

imapserver/error.go

@@ -57,9 +57,3 @@ func xsyntaxErrorf(format string, args ...any) {
 	err := errors.New(errmsg)
 	panic(syntaxError{"", "", errmsg, err})
 }
-
-func xsyntaxCodeErrorf(code, format string, args ...any) {
-	errmsg := fmt.Sprintf(format, args...)
-	err := errors.New(errmsg)
-	panic(syntaxError{"", code, errmsg, err})
-}

imapserver/expunge_test.go

@@ -7,25 +7,17 @@ import (
 )
 
 func TestExpunge(t *testing.T) {
-	testExpunge(t, false)
-}
-
-func TestExpungeUIDOnly(t *testing.T) {
-	testExpunge(t, true)
-}
-
-func testExpunge(t *testing.T, uidonly bool) {
 	defer mockUIDValidity()()
-	tc := start(t, uidonly)
+	tc := start(t)
 	defer tc.close()
 
-	tc2 := startNoSwitchboard(t, uidonly)
+	tc2 := startNoSwitchboard(t)
-	defer tc2.closeNoWait()
+	defer tc2.close()
 
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
 	tc.client.Select("inbox")
 
-	tc2.login("mjl@mox.example", password0)
+	tc2.client.Login("mjl@mox.example", password0)
 	tc2.client.Select("inbox")
 
 	tc.transactf("bad", "expunge leftover") // Leftover data.
@@ -39,43 +31,35 @@ func testExpunge(t *testing.T, uidonly bool) {
 
 	tc.client.Unselect()
 	tc.client.Select("inbox")
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
 	tc.transactf("ok", "expunge") // Still nothing to remove.
 	tc.xuntagged()
 
-	tc.transactf("ok", `uid store 1,3 +flags.silent \Deleted`)
+	tc.client.StoreFlagsAdd("1,3", true, `\Deleted`)
 
 	tc2.transactf("ok", "noop") // Drain.
 
 	tc.transactf("ok", "expunge")
-	if uidonly {
+	tc.xuntagged(imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(2))
-		tc.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("1,3")})
-	} else {
-		tc.xuntagged(imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(2))
-	}
 
 	tc2.transactf("ok", "noop") // Drain.
-	if uidonly {
+	tc2.xuntagged(imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(2))
-		tc2.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("1,3")})
-	} else {
-		tc2.xuntagged(imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(2))
-	}
 
 	tc.transactf("ok", "expunge") // Nothing to remove anymore.
 	tc.xuntagged()
 
 	// Only UID 2 is still left. We'll add 3 more. Getting us to UIDs 2,4,5,6.
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
 
 	tc.transactf("bad", "uid expunge")            // Missing uid set.
 	tc.transactf("bad", "uid expunge 1 leftover") // Leftover data.
 	tc.transactf("bad", "uid expunge 1 leftover") // Leftover data.
 
-	tc.transactf("ok", `uid store 2,4,6 +flags.silent \Deleted`)
+	tc.client.StoreFlagsAdd("1,2,4", true, `\Deleted`) // Marks UID 2,4,6 as deleted.
 
 	tc.transactf("ok", "uid expunge 1")
 	tc.xuntagged() // No match.
@@ -83,16 +67,8 @@ func testExpunge(t *testing.T, uidonly bool) {
 	tc2.transactf("ok", "noop") // Drain.
 
 	tc.transactf("ok", "uid expunge 4:6") // Removes UID 4,6 at seqs 2,4.
-	if uidonly {
+	tc.xuntagged(imapclient.UntaggedExpunge(2), imapclient.UntaggedExpunge(3))
-		tc.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("4,6")})
-	} else {
-		tc.xuntagged(imapclient.UntaggedExpunge(2), imapclient.UntaggedExpunge(3))
-	}
 
 	tc2.transactf("ok", "noop")
-	if uidonly {
+	tc.xuntagged(imapclient.UntaggedExpunge(2), imapclient.UntaggedExpunge(3))
-		tc2.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("4,6")})
-	} else {
-		tc2.xuntagged(imapclient.UntaggedExpunge(2), imapclient.UntaggedExpunge(3))
-	}
 }

imapserver/fetch.go

@@ -4,20 +4,19 @@ package imapserver
 
 import (
 	"bytes"
-	"context"
+	"errors"
 	"fmt"
 	"io"
 	"log/slog"
-	"maps"
-	"mime"
 	"net/textproto"
-	"slices"
+	"sort"
 	"strings"
 
+	"golang.org/x/exp/maps"
+
 	"github.com/mjl-/bstore"
 
 	"github.com/mjl-/mox/message"
-	"github.com/mjl-/mox/mlog"
 	"github.com/mjl-/mox/mox-"
 	"github.com/mjl-/mox/moxio"
 	"github.com/mjl-/mox/store"
@@ -26,20 +25,18 @@ import (
 // functions to handle fetch attribute requests are defined on fetchCmd.
 type fetchCmd struct {
 	conn            *conn
-	isUID           bool        // If this is a UID FETCH command.
+	mailboxID       int64
-	rtx             *bstore.Tx  // Read-only transaction, kept open while processing all messages.
+	uid             store.UID
-	updateSeen      []store.UID // To mark as seen after processing all messages. UID instead of message ID since moved messages keep their ID and insert a new ID in the original mailbox.
+	tx              *bstore.Tx     // Writable tx, for storing message when first parsed as mime parts.
-	hasChangedSince bool        // Whether CHANGEDSINCE was set. Enables MODSEQ in response.
+	changes         []store.Change // For updated Seen flag.
-	expungeIssued   bool        // Set if any message has been expunged. Can happen for expunged messages.
+	markSeen        bool
-
+	needFlags       bool
-	// For message currently processing.
+	needModseq      bool                // Whether untagged responses needs modseq.
-	mailboxID int64
+	expungeIssued   bool                // Set if a message cannot be read. Can happen for expunged messages.
-	uid       store.UID
+	modseq          store.ModSeq        // Initialized on first change, for marking messages as seen.
-
+	isUID           bool                // If this is a UID FETCH command.
-	markSeen    bool
+	hasChangedSince bool                // Whether CHANGEDSINCE was set. Enables MODSEQ in response.
-	needFlags   bool
+	deltaCounts     store.MailboxCounts // By marking \Seen, the number of unread/unseen messages will go down. We update counts at the end.
-	needModseq  bool                 // Whether untagged responses needs modseq.
-	newPreviews map[store.UID]string // Save with messages when done.
 
 	// Loaded when first needed, closed when message was processed.
 	m    *store.Message // Message currently being processed.
@@ -79,7 +76,7 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
 	p.xspace()
 	nums := p.xnumSet()
 	p.xspace()
-	atts := p.xfetchAtts()
+	atts := p.xfetchAtts(isUID)
 	var changedSince int64
 	var haveChangedSince bool
 	var vanished bool
@@ -128,66 +125,42 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
 	}
 	p.xempty()
 
-	// We only keep a wlock, only for initial checks and listing the uids. Then we
+	// We don't use c.account.WithRLock because we write to the client while reading messages.
-	// unlock and work without a lock. So changes to the store can happen, and we need
+	// We get the rlock, then we check the mailbox, release the lock and read the messages.
-	// to deal with that. If we need to mark messages as seen, we do so after
+	// The db transaction still locks out any changes to the database...
-	// processing the fetch for all messages, in a single write transaction. We don't
+	c.account.RLock()
-	// send untagged changes for those \seen flag changes before finishing this
+	runlock := c.account.RUnlock
-	// command, because we have to sequence all changes properly, and since we don't
+	// Note: we call runlock in a closure because we replace it below.
-	// (want to) hold a wlock while processing messages (can be many!), other changes
-	// may have happened to the store. So instead, we'll silently mark messages as seen
-	// (the client should know this is happening anyway!), then broadcast the changes
-	// to everyone, including ourselves. A noop/idle command that may come next will
-	// return the \seen flag changes, in the correct order, with the correct modseq. We
-	// also cannot just apply pending changes while processing. It is not allowed at
-	// all for non-uid-fetch. It would also make life more complicated, e.g. we would
-	// perhaps have to check if newly added messages also match uid fetch set that was
-	// requested.
-
-	var uids []store.UID
-	var vanishedUIDs []store.UID
-
-	cmd := &fetchCmd{conn: c, isUID: isUID, hasChangedSince: haveChangedSince, mailboxID: c.mailboxID, newPreviews: map[store.UID]string{}}
-
 	defer func() {
-		if cmd.rtx == nil {
+		runlock()
-			return
-		}
-		err := cmd.rtx.Rollback()
-		c.log.Check(err, "rollback rtx")
-		cmd.rtx = nil
 	}()
 
-	c.account.WithRLock(func() {
+	var vanishedUIDs []store.UID
-		var err error
+	cmd := &fetchCmd{conn: c, mailboxID: c.mailboxID, isUID: isUID, hasChangedSince: haveChangedSince}
-		cmd.rtx, err = c.account.DB.Begin(context.TODO(), false)
+	c.xdbwrite(func(tx *bstore.Tx) {
-		cmd.xcheckf(err, "begin transaction")
+		cmd.tx = tx
 
 		// Ensure the mailbox still exists.
-		c.xmailboxID(cmd.rtx, c.mailboxID)
+		mb := c.xmailboxID(tx, c.mailboxID)
+
+		var uids []store.UID
 
 		// With changedSince, the client is likely asking for a small set of changes. Use a
-		// database query to trim down the uids we need to look at. We need to go through
+		// database query to trim down the uids we need to look at.
-		// the database for "VANISHED (EARLIER)" anyway, to see UIDs that aren't in the
+		// ../rfc/7162:871
-		// session anymore. Vanished must be used with changedSince. ../rfc/7162:871
 		if changedSince > 0 {
-			q := bstore.QueryTx[store.Message](cmd.rtx)
+			q := bstore.QueryTx[store.Message](tx)
 			q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
 			q.FilterGreater("ModSeq", store.ModSeqFromClient(changedSince))
 			if !vanished {
 				q.FilterEqual("Expunged", false)
 			}
 			err := q.ForEach(func(m store.Message) error {
-				if m.UID >= c.uidnext {
+				if m.Expunged {
-					return nil
+					vanishedUIDs = append(vanishedUIDs, m.UID)
-				}
+				} else if isUID {
-				if isUID {
+					if nums.containsUID(m.UID, c.uids, c.searchResult) {
-					if nums.xcontainsKnownUID(m.UID, c.searchResult, func() store.UID { return c.uidnext - 1 }) {
+						uids = append(uids, m.UID)
-						if m.Expunged {
-							vanishedUIDs = append(vanishedUIDs, m.UID)
-						} else {
-							uids = append(uids, m.UID)
-						}
 					}
 				} else {
 					seq := c.sequence(m.UID)
@@ -198,196 +171,115 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
 				return nil
 			})
 			xcheckf(err, "looking up messages with changedsince")
+		} else {
+			uids = c.xnumSetUIDs(isUID, nums)
+		}
 
-			// In case of vanished where we don't have the full history, we must send VANISHED
+		// Send vanished for all missing requested UIDs. ../rfc/7162:1718
-			// for all uids matching nums. ../rfc/7162:1718
+		if vanished {
-			delModSeq, err := c.account.HighestDeletedModSeq(cmd.rtx)
+			delModSeq, err := c.account.HighestDeletedModSeq(tx)
 			xcheckf(err, "looking up highest deleted modseq")
-			if !vanished || changedSince >= delModSeq.Client() {
+			if changedSince < delModSeq.Client() {
-				return
+				// First sort the uids we already found, for fast lookup.
+				sort.Slice(vanishedUIDs, func(i, j int) bool {
+					return vanishedUIDs[i] < vanishedUIDs[j]
+				})
+
+				// We'll be gathering any more vanished uids in more.
+				more := map[store.UID]struct{}{}
+				checkVanished := func(uid store.UID) {
+					if uidSearch(c.uids, uid) <= 0 && uidSearch(vanishedUIDs, uid) <= 0 {
+						more[uid] = struct{}{}
+					}
+				}
+				// Now look through the requested uids. We may have a searchResult, handle it
+				// separately from a numset with potential stars, over which we can more easily
+				// iterate.
+				if nums.searchResult {
+					for _, uid := range c.searchResult {
+						checkVanished(uid)
+					}
+				} else {
+					iter := nums.interpretStar(c.uids).newIter()
+					for {
+						num, ok := iter.Next()
+						if !ok {
+							break
+						}
+						checkVanished(store.UID(num))
+					}
+				}
+				vanishedUIDs = append(vanishedUIDs, maps.Keys(more)...)
 			}
-
-			// We'll iterate through all UIDs in the numset, and add anything that isn't
-			// already in uids and vanishedUIDs. First sort the uids we already found, for fast
-			// lookup. We'll gather new UIDs in more, so we don't break the binary search.
-			slices.Sort(vanishedUIDs)
-			slices.Sort(uids)
-
-			more := map[store.UID]struct{}{} // We'll add them at the end.
-			checkVanished := func(uid store.UID) {
-				if uid < c.uidnext && uidSearch(uids, uid) <= 0 && uidSearch(vanishedUIDs, uid) <= 0 {
-					more[uid] = struct{}{}
-				}
-			}
-
-			// Now look through the requested uids. We may have a searchResult, handle it
-			// separately from a numset with potential stars, over which we can more easily
-			// iterate.
-			if nums.searchResult {
-				for _, uid := range c.searchResult {
-					checkVanished(uid)
-				}
-			} else {
-				xlastUID := c.newCachedLastUID(cmd.rtx, c.mailboxID, func(xerr error) { xuserErrorf("%s", xerr) })
-				iter := nums.xinterpretStar(xlastUID).newIter()
-				for {
-					num, ok := iter.Next()
-					if !ok {
-						break
-					}
-					checkVanished(store.UID(num))
-				}
-			}
-			vanishedUIDs = slices.AppendSeq(vanishedUIDs, maps.Keys(more))
-			slices.Sort(vanishedUIDs)
-		} else {
-			uids = c.xnumSetEval(cmd.rtx, isUID, nums)
 		}
 
-	})
+		// Release the account lock.
-	// We are continuing without a lock, working off our snapshot of uids to process.
+		runlock()
+		runlock = func() {} // Prevent defer from unlocking again.
 
-	// First report all vanished UIDs. ../rfc/7162:1714
+		// First report all vanished UIDs. ../rfc/7162:1714
-	if len(vanishedUIDs) > 0 {
+		if len(vanishedUIDs) > 0 {
-		// Mention all vanished UIDs in compact numset form.
+			// Mention all vanished UIDs in compact numset form.
-		// ../rfc/7162:1985
+			// ../rfc/7162:1985
-		// No hard limit on response sizes, but clients are recommended to not send more
+			sort.Slice(vanishedUIDs, func(i, j int) bool {
-		// than 8k. We send a more conservative max 4k.
+				return vanishedUIDs[i] < vanishedUIDs[j]
-		for _, s := range compactUIDSet(vanishedUIDs).Strings(4*1024 - 32) {
-			c.xbwritelinef("* VANISHED (EARLIER) %s", s)
-		}
-	}
-
-	defer cmd.msgclose() // In case of panic.
-
-	for _, cmd.uid = range uids {
-		cmd.conn.log.Debug("processing uid", slog.Any("uid", cmd.uid))
-		data, err := cmd.process(atts)
-		if err != nil {
-			cmd.conn.log.Infox("processing fetch attribute", err, slog.Any("uid", cmd.uid))
-			xuserErrorf("processing fetch attribute: %v", err)
-		}
-
-		// UIDFETCH in case of uidonly. ../rfc/9586:181
-		if c.uidonly {
-			fmt.Fprintf(cmd.conn.xbw, "* %d UIDFETCH ", cmd.uid)
-		} else {
-			fmt.Fprintf(cmd.conn.xbw, "* %d FETCH ", cmd.conn.xsequence(cmd.uid))
-		}
-		data.xwriteTo(cmd.conn, cmd.conn.xbw)
-		cmd.conn.xbw.Write([]byte("\r\n"))
-
-		cmd.msgclose()
-	}
-
-	// We've returned all data. Now we mark messages as seen in one go, in a new write
-	// transaction. We don't send untagged messages for the changes, since there may be
-	// unprocessed pending changes. Instead, we broadcast them to ourselve too, so a
-	// next noop/idle will return the flags to the client.
-
-	err := cmd.rtx.Rollback()
-	c.log.Check(err, "fetch read tx rollback")
-	cmd.rtx = nil
-
-	// ../rfc/9051:4432 We mark all messages that need it as seen at the end of the
-	// command, in a single transaction.
-	if len(cmd.updateSeen) > 0 || len(cmd.newPreviews) > 0 {
-		c.account.WithWLock(func() {
-			changes := make([]store.Change, 0, len(cmd.updateSeen)+1)
-
-			c.xdbwrite(func(wtx *bstore.Tx) {
-				mb, err := store.MailboxID(wtx, c.mailboxID)
-				if err == store.ErrMailboxExpunged {
-					xusercodeErrorf("NONEXISTENT", "mailbox has been expunged")
-				}
-				xcheckf(err, "get mailbox for updating counts after marking as seen")
-
-				var modseq store.ModSeq
-
-				for _, uid := range cmd.updateSeen {
-					m, err := bstore.QueryTx[store.Message](wtx).FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: uid}).Get()
-					xcheckf(err, "get message")
-					if m.Expunged {
-						// Message has been deleted in the mean time.
-						cmd.expungeIssued = true
-						continue
-					}
-					if m.Seen {
-						// Message already marked as seen by another process.
-						continue
-					}
-
-					if modseq == 0 {
-						modseq, err = c.account.NextModSeq(wtx)
-						xcheckf(err, "get next mod seq")
-					}
-
-					oldFlags := m.Flags
-					mb.Sub(m.MailboxCounts())
-					m.Seen = true
-					mb.Add(m.MailboxCounts())
-					changes = append(changes, m.ChangeFlags(oldFlags, mb))
-
-					m.ModSeq = modseq
-					err = wtx.Update(&m)
-					xcheckf(err, "mark message as seen")
-				}
-
-				changes = append(changes, mb.ChangeCounts())
-
-				for uid, s := range cmd.newPreviews {
-					m, err := bstore.QueryTx[store.Message](wtx).FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: uid}).Get()
-					xcheckf(err, "get message")
-					if m.Expunged {
-						// Message has been deleted in the mean time.
-						cmd.expungeIssued = true
-						continue
-					}
-
-					// note: we are not updating modseq.
-
-					m.Preview = &s
-					err = wtx.Update(&m)
-					xcheckf(err, "saving preview with message")
-				}
-
-				if modseq > 0 {
-					mb.ModSeq = modseq
-					err = wtx.Update(&mb)
-					xcheckf(err, "update mailbox with counts and modseq")
-				}
 			})
+			// No hard limit on response sizes, but clients are recommended to not send more
+			// than 8k. We send a more conservative max 4k.
+			for _, s := range compactUIDSet(vanishedUIDs).Strings(4*1024 - 32) {
+				c.bwritelinef("* VANISHED (EARLIER) %s", s)
+			}
+		}
 
-			// Broadcast these changes also to ourselves, so we'll send the updated flags, but
+		for _, uid := range uids {
-			// in the correct order, after other changes.
+			cmd.uid = uid
-			store.BroadcastChanges(c.account, changes)
+			cmd.conn.log.Debug("processing uid", slog.Any("uid", uid))
-		})
+			cmd.process(atts)
+		}
+
+		var zeromc store.MailboxCounts
+		if cmd.deltaCounts != zeromc {
+			mb.Add(cmd.deltaCounts) // Unseen/Unread will be <= 0.
+			err := tx.Update(&mb)
+			xcheckf(err, "updating mailbox counts")
+			cmd.changes = append(cmd.changes, mb.ChangeCounts())
+			// No need to update account total message size.
+		}
+	})
+
+	if len(cmd.changes) > 0 {
+		// Broadcast seen updates to other connections.
+		c.broadcast(cmd.changes)
 	}
 
 	if cmd.expungeIssued {
 		// ../rfc/2180:343
-		// ../rfc/9051:5102
+		c.writeresultf("%s NO [EXPUNGEISSUED] at least one message was expunged", tag)
-		c.xwriteresultf("%s OK [EXPUNGEISSUED] at least one message was expunged", tag)
 	} else {
 		c.ok(tag, cmdstr)
 	}
 }
 
+func (cmd *fetchCmd) xmodseq() store.ModSeq {
+	if cmd.modseq == 0 {
+		var err error
+		cmd.modseq, err = cmd.conn.account.NextModSeq(cmd.tx)
+		cmd.xcheckf(err, "assigning next modseq")
+	}
+	return cmd.modseq
+}
+
 func (cmd *fetchCmd) xensureMessage() *store.Message {
 	if cmd.m != nil {
 		return cmd.m
 	}
 
-	// We do not filter by Expunged, the message may have been deleted in other
+	q := bstore.QueryTx[store.Message](cmd.tx)
-	// sessions, but not in ours.
-	q := bstore.QueryTx[store.Message](cmd.rtx)
 	q.FilterNonzero(store.Message{MailboxID: cmd.mailboxID, UID: cmd.uid})
+	q.FilterEqual("Expunged", false)
 	m, err := q.Get()
 	cmd.xcheckf(err, "get message for uid %d", cmd.uid)
 	cmd.m = &m
-	if m.Expunged {
-		cmd.expungeIssued = true
-	}
 	return cmd.m
 }
 
@@ -413,20 +305,16 @@ func (cmd *fetchCmd) xensureParsed() (*store.MsgReader, *message.Part) {
 	return cmd.msgr, cmd.part
 }
 
-// msgclose must be called after processing a message (after having written/used
+func (cmd *fetchCmd) process(atts []fetchAtt) {
-// its data), even in the case of a panic.
-func (cmd *fetchCmd) msgclose() {
-	cmd.m = nil
-	cmd.part = nil
-	if cmd.msgr != nil {
-		err := cmd.msgr.Close()
-		cmd.conn.xsanity(err, "closing messagereader")
-		cmd.msgr = nil
-	}
-}
-
-func (cmd *fetchCmd) process(atts []fetchAtt) (rdata listspace, rerr error) {
 	defer func() {
+		cmd.m = nil
+		cmd.part = nil
+		if cmd.msgr != nil {
+			err := cmd.msgr.Close()
+			cmd.conn.xsanity(err, "closing messagereader")
+			cmd.msgr = nil
+		}
+
 		x := recover()
 		if x == nil {
 			return
@@ -434,15 +322,16 @@ func (cmd *fetchCmd) process(atts []fetchAtt) (rdata listspace, rerr error) {
 		err, ok := x.(attrError)
 		if !ok {
 			panic(x)
-		} else if rerr == nil {
-			rerr = err
 		}
+		if errors.Is(err, bstore.ErrAbsent) {
+			cmd.expungeIssued = true
+			return
+		}
+		cmd.conn.log.Infox("processing fetch attribute", err, slog.Any("uid", cmd.uid))
+		xuserErrorf("processing fetch attribute: %v", err)
 	}()
 
-	var data listspace
+	data := listspace{bare("UID"), number(cmd.uid)}
-	if !cmd.conn.uidonly {
-		data = append(data, bare("UID"), number(cmd.uid))
-	}
 
 	cmd.markSeen = false
 	cmd.needFlags = false
@@ -453,7 +342,17 @@ func (cmd *fetchCmd) process(atts []fetchAtt) (rdata listspace, rerr error) {
 	}
 
 	if cmd.markSeen {
-		cmd.updateSeen = append(cmd.updateSeen, cmd.uid)
+		m := cmd.xensureMessage()
+		cmd.deltaCounts.Sub(m.MailboxCounts())
+		origFlags := m.Flags
+		m.Seen = true
+		cmd.deltaCounts.Add(m.MailboxCounts())
+		m.ModSeq = cmd.xmodseq()
+		err := cmd.tx.Update(m)
+		xcheckf(err, "marking message as seen")
+		// No need to update account total message size.
+
+		cmd.changes = append(cmd.changes, m.ChangeFlags(origFlags))
 	}
 
 	if cmd.needFlags {
@@ -476,12 +375,15 @@ func (cmd *fetchCmd) process(atts []fetchAtt) (rdata listspace, rerr error) {
 	// other mentioning of cases elsewhere in the RFC would be too superfluous.
 	//
 	// ../rfc/7162:877 ../rfc/7162:388 ../rfc/7162:909 ../rfc/7162:1426
-	if cmd.needModseq || cmd.hasChangedSince || cmd.conn.enabled[capQresync] && cmd.isUID {
+	if cmd.needModseq || cmd.hasChangedSince || cmd.conn.enabled[capQresync] && (cmd.isUID || cmd.markSeen) {
 		m := cmd.xensureMessage()
 		data = append(data, bare("MODSEQ"), listspace{bare(fmt.Sprintf("%d", m.ModSeq.Client()))})
 	}
 
-	return data, nil
+	// Write errors are turned into panics because we write through c.
+	fmt.Fprintf(cmd.conn.bw, "* %d FETCH ", cmd.conn.xsequence(cmd.uid))
+	data.writeTo(cmd.conn, cmd.conn.bw)
+	cmd.conn.bw.Write([]byte("\r\n"))
 }
 
 // result for one attribute. if processing fails, e.g. because data was requested
@@ -490,12 +392,8 @@ func (cmd *fetchCmd) process(atts []fetchAtt) (rdata listspace, rerr error) {
 func (cmd *fetchCmd) xprocessAtt(a fetchAtt) []token {
 	switch a.field {
 	case "UID":
-		// Present by default without uidonly. For uidonly, we only add it when explicitly
+		// Always present.
-		// requested. ../rfc/9586:184
+		return nil
-		if cmd.conn.uidonly {
-			return []token{bare("UID"), number(cmd.uid)}
-		}
-
 	case "ENVELOPE":
 		_, part := cmd.xensureParsed()
 		envelope := xenvelope(part)
@@ -506,20 +404,9 @@ func (cmd *fetchCmd) xprocessAtt(a fetchAtt) []token {
 		m := cmd.xensureMessage()
 		return []token{bare("INTERNALDATE"), dquote(m.Received.Format("_2-Jan-2006 15:04:05 -0700"))}
 
-	case "SAVEDATE":
-		m := cmd.xensureMessage()
-		// For messages in storage from before we implemented this extension, we don't have
-		// a savedate, and we return nil. This is normally meant to be per mailbox, but
-		// returning it per message should be fine. ../rfc/8514:191
-		var savedate token = nilt
-		if m.SaveDate != nil {
-			savedate = dquote(m.SaveDate.Format("_2-Jan-2006 15:04:05 -0700"))
-		}
-		return []token{bare("SAVEDATE"), savedate}
-
 	case "BODYSTRUCTURE":
 		_, part := cmd.xensureParsed()
-		bs := xbodystructure(cmd.conn.log, part, true)
+		bs := xbodystructure(part)
 		return []token{bare("BODYSTRUCTURE"), bs}
 
 	case "BODY":
@@ -600,37 +487,6 @@ func (cmd *fetchCmd) xprocessAtt(a fetchAtt) []token {
 	case "MODSEQ":
 		cmd.needModseq = true
 
-	case "PREVIEW":
-		m := cmd.xensureMessage()
-		preview := m.Preview
-		// We ignore "lazy", generating the preview is fast enough.
-		if preview == nil {
-			// Get the preview. We'll save all generated previews in a single transaction at
-			// the end.
-			_, p := cmd.xensureParsed()
-			s, err := p.Preview(cmd.conn.log)
-			cmd.xcheckf(err, "generating preview")
-			preview = &s
-			cmd.newPreviews[m.UID] = s
-		}
-		var t token = nilt
-		if preview != nil {
-			s := *preview
-
-			// Limit to 200 characters (not bytes). ../rfc/8970:206
-			var n, o int
-			for o = range s {
-				n++
-				if n > 200 {
-					s = s[:o]
-					break
-				}
-			}
-			s = strings.TrimSpace(s)
-			t = string0(s)
-		}
-		return []token{bare(a.field), t}
-
 	default:
 		xserverErrorf("field %q not yet implemented", a.field)
 	}
@@ -776,15 +632,11 @@ func (cmd *fetchCmd) xbinary(a fetchAtt) (string, token) {
 		cmd.xerrorf("binary only allowed on leaf parts, not multipart/* or message/rfc822 or message/global")
 	}
 
-	var cte string
+	switch p.ContentTransferEncoding {
-	if p.ContentTransferEncoding != nil {
-		cte = *p.ContentTransferEncoding
-	}
-	switch cte {
 	case "", "7BIT", "8BIT", "BINARY", "BASE64", "QUOTED-PRINTABLE":
 	default:
 		// ../rfc/9051:5913
-		xusercodeErrorf("UNKNOWN-CTE", "unknown Content-Transfer-Encoding %q", cte)
+		xusercodeErrorf("UNKNOWN-CTE", "unknown Content-Transfer-Encoding %q", p.ContentTransferEncoding)
 	}
 
 	r := p.Reader()
@@ -808,7 +660,7 @@ func (cmd *fetchCmd) xbody(a fetchAtt) (string, token) {
 
 	if a.section == nil {
 		// Non-extensible form of BODYSTRUCTURE.
-		return a.field, xbodystructure(cmd.conn.log, part, false)
+		return a.field, xbodystructure(part)
 	}
 
 	cmd.peekOrSeen(a.peek)
@@ -820,13 +672,16 @@ func (cmd *fetchCmd) xbody(a fetchAtt) (string, token) {
 		var offset int64
 		count := m.Size
 		if a.partial != nil {
-			offset = min(int64(a.partial.offset), m.Size)
+			offset = int64(a.partial.offset)
+			if offset > m.Size {
+				offset = m.Size
+			}
 			count = int64(a.partial.count)
 			if offset+count > m.Size {
 				count = m.Size - offset
 			}
 		}
-		return respField, readerSizeSyncliteral{&moxio.AtReader{R: msgr, Offset: offset}, count, false}
+		return respField, readerSizeSyncliteral{&moxio.AtReader{R: msgr, Offset: offset}, count}
 	}
 
 	sr := cmd.xsection(a.section, part)
@@ -865,40 +720,35 @@ func (cmd *fetchCmd) xpartnumsDeref(nums []uint32, p *message.Part) *message.Par
 }
 
 func (cmd *fetchCmd) xsection(section *sectionSpec, p *message.Part) io.Reader {
-	// msgtext is not nil, i.e. HEADER* or TEXT (not MIME), for the top-level part (a message).
 	if section.part == nil {
 		return cmd.xsectionMsgtext(section.msgtext, p)
 	}
 
 	p = cmd.xpartnumsDeref(section.part.part, p)
 
-	// If there is no sectionMsgText, then this isn't for HEADER*, TEXT or MIME, i.e. a
-	// part body, e.g. "BODY[1]".
 	if section.part.text == nil {
 		return p.RawReader()
 	}
 
-	// MIME is defined for all parts. Otherwise it's HEADER* or TEXT, which is only
+	// ../rfc/9051:4535
-	// defined for parts that are messages. ../rfc/9051:4500 ../rfc/9051:4517
+	if p.Message != nil {
-	if !section.part.text.mime {
-		if p.Message == nil {
-			cmd.xerrorf("part is not a message, cannot request header* or text")
-		}
-
 		err := p.SetMessageReaderAt()
 		cmd.xcheckf(err, "preparing submessage")
 		p = p.Message
+	}
 
+	if !section.part.text.mime {
 		return cmd.xsectionMsgtext(section.part.text.msgtext, p)
 	}
 
-	// MIME header, see ../rfc/9051:4514 ../rfc/2045:1652
+	// MIME header, see ../rfc/9051:4534 ../rfc/2045:1645
 	h, err := io.ReadAll(p.HeaderReader())
 	cmd.xcheckf(err, "reading header")
 
 	matchesFields := func(line []byte) bool {
 		k := textproto.CanonicalMIMEHeaderKey(string(bytes.TrimRight(bytes.SplitN(line, []byte(":"), 2)[0], " \t")))
-		return strings.HasPrefix(k, "Content-")
+		// Only add MIME-Version and additional CRLF for messages, not other parts. ../rfc/2045:1645 ../rfc/2045:1652
+		return (p.Envelope != nil && k == "Mime-Version") || strings.HasPrefix(k, "Content-")
 	}
 
 	var match bool
@@ -912,7 +762,7 @@ func (cmd *fetchCmd) xsection(section *sectionSpec, p *message.Part) io.Reader {
 		h = h[len(line):]
 
 		match = matchesFields(line) || match && (bytes.HasPrefix(line, []byte(" ")) || bytes.HasPrefix(line, []byte("\t")))
-		if match {
+		if match || len(line) == 2 {
 			hb.Write(line)
 		}
 	}
@@ -920,10 +770,11 @@ func (cmd *fetchCmd) xsection(section *sectionSpec, p *message.Part) io.Reader {
 }
 
 func (cmd *fetchCmd) xsectionMsgtext(smt *sectionMsgtext, p *message.Part) io.Reader {
-	switch smt.s {
+	if smt.s == "HEADER" {
-	case "HEADER":
 		return p.HeaderReader()
+	}
 
+	switch smt.s {
 	case "HEADER.FIELDS":
 		return cmd.xmodifiedHeader(p, smt.headers, false)
 
@@ -931,8 +782,8 @@ func (cmd *fetchCmd) xsectionMsgtext(smt *sectionMsgtext, p *message.Part) io.Re
 		return cmd.xmodifiedHeader(p, smt.headers, true)
 
 	case "TEXT":
-		// TEXT the body (excluding headers) of a message, either the top-level message, or
+		// It appears imap clients expect to get the body of the message, not a "text body"
-		// a nested as message/rfc822 or message/global. ../rfc/9051:4517
+		// which sounds like it means a text/* part of a message. ../rfc/9051:4517
 		return p.RawReader()
 	}
 	panic(serverError{fmt.Errorf("missing case")})
@@ -983,24 +834,27 @@ func (cmd *fetchCmd) sectionMsgtextName(smt *sectionMsgtext) string {
 	return s
 }
 
-func bodyFldParams(p *message.Part) token {
+func bodyFldParams(params map[string]string) token {
-	if len(p.ContentTypeParams) == 0 {
+	if len(params) == 0 {
 		return nilt
 	}
-	params := make(listspace, 0, 2*len(p.ContentTypeParams))
 	// Ensure same ordering, easier for testing.
-	for _, k := range slices.Sorted(maps.Keys(p.ContentTypeParams)) {
+	var keys []string
-		v := p.ContentTypeParams[k]
+	for k := range params {
-		params = append(params, string0(strings.ToUpper(k)), string0(v))
+		keys = append(keys, k)
 	}
-	return params
+	sort.Strings(keys)
+	l := make(listspace, 2*len(keys))
+	i := 0
+	for _, k := range keys {
+		l[i] = string0(strings.ToUpper(k))
+		l[i+1] = string0(params[k])
+		i += 2
+	}
+	return l
 }
 
-func bodyFldEnc(cte *string) token {
+func bodyFldEnc(s string) token {
-	var s string
-	if cte != nil {
-		s = *cte
-	}
 	up := strings.ToUpper(s)
 	switch up {
 	case "7BIT", "8BIT", "BINARY", "BASE64", "QUOTED-PRINTABLE":
@@ -1009,92 +863,25 @@ func bodyFldEnc(cte *string) token {
 	return string0(s)
 }
 
-func bodyFldMd5(p *message.Part) token {
-	if p.ContentMD5 == nil {
-		return nilt
-	}
-	return string0(*p.ContentMD5)
-}
-
-func bodyFldDisp(log mlog.Log, p *message.Part) token {
-	if p.ContentDisposition == nil {
-		return nilt
-	}
-
-	// ../rfc/9051:5989
-	// mime.ParseMediaType recombines parameter value continuations like "title*0" and
-	// "title*1" into "title". ../rfc/2231:147
-	// And decodes character sets and removes language tags, like
-	// "title*0*=us-ascii'en'hello%20world. ../rfc/2231:210
-
-	disp, params, err := mime.ParseMediaType(*p.ContentDisposition)
-	if err != nil {
-		log.Debugx("parsing content-disposition, ignoring", err, slog.String("header", *p.ContentDisposition))
-		return nilt
-	} else if len(params) == 0 {
-		log.Debug("content-disposition has no parameters, ignoring", slog.String("header", *p.ContentDisposition))
-		return nilt
-	}
-	var fields listspace
-	for _, k := range slices.Sorted(maps.Keys(params)) {
-		fields = append(fields, string0(k), string0(params[k]))
-	}
-	return listspace{string0(disp), fields}
-}
-
-func bodyFldLang(p *message.Part) token {
-	// todo: ../rfc/3282:86 ../rfc/5646:218 we currently just split on comma and trim space, should properly parse header.
-	if p.ContentLanguage == nil {
-		return nilt
-	}
-	var l listspace
-	for _, s := range strings.Split(*p.ContentLanguage, ",") {
-		s = strings.TrimSpace(s)
-		if s == "" {
-			return string0(*p.ContentLanguage)
-		}
-		l = append(l, string0(s))
-	}
-	return l
-}
-
-func bodyFldLoc(p *message.Part) token {
-	if p.ContentLocation == nil {
-		return nilt
-	}
-	return string0(*p.ContentLocation)
-}
-
 // xbodystructure returns a "body".
 // calls itself for multipart messages and message/{rfc822,global}.
-func xbodystructure(log mlog.Log, p *message.Part, extensible bool) token {
+func xbodystructure(p *message.Part) token {
 	if p.MediaType == "MULTIPART" {
 		// Multipart, ../rfc/9051:6355 ../rfc/9051:6411
 		var bodies concat
 		for i := range p.Parts {
-			bodies = append(bodies, xbodystructure(log, &p.Parts[i], extensible))
+			bodies = append(bodies, xbodystructure(&p.Parts[i]))
 		}
-		r := listspace{bodies, string0(p.MediaSubType)}
+		return listspace{bodies, string0(p.MediaSubType)}
-		// ../rfc/9051:6371
-		if extensible {
-			r = append(r,
-				bodyFldParams(p),
-				bodyFldDisp(log, p),
-				bodyFldLang(p),
-				bodyFldLoc(p),
-			)
-		}
-		return r
 	}
 
 	// ../rfc/9051:6355
-	var r listspace
 	if p.MediaType == "TEXT" {
 		// ../rfc/9051:6404 ../rfc/9051:6418
-		r = listspace{
+		return listspace{
 			dquote("TEXT"), string0(p.MediaSubType), // ../rfc/9051:6739
 			// ../rfc/9051:6376
-			bodyFldParams(p), // ../rfc/9051:6401
+			bodyFldParams(p.ContentTypeParams), // ../rfc/9051:6401
 			nilOrString(p.ContentID),
 			nilOrString(p.ContentDescription),
 			bodyFldEnc(p.ContentTransferEncoding),
@@ -1104,45 +891,34 @@ func xbodystructure(log mlog.Log, p *message.Part, extensible bool) token {
 	} else if p.MediaType == "MESSAGE" && (p.MediaSubType == "RFC822" || p.MediaSubType == "GLOBAL") {
 		// ../rfc/9051:6415
 		// note: we don't have to prepare p.Message for reading, because we aren't going to read from it.
-		r = listspace{
+		return listspace{
 			dquote("MESSAGE"), dquote(p.MediaSubType), // ../rfc/9051:6732
 			// ../rfc/9051:6376
-			bodyFldParams(p), // ../rfc/9051:6401
+			bodyFldParams(p.ContentTypeParams), // ../rfc/9051:6401
 			nilOrString(p.ContentID),
 			nilOrString(p.ContentDescription),
 			bodyFldEnc(p.ContentTransferEncoding),
 			number(p.EndOffset - p.BodyOffset),
 			xenvelope(p.Message),
-			xbodystructure(log, p.Message, extensible),
+			xbodystructure(p.Message),
 			number(p.RawLineCount), // todo: or mp.RawLineCount?
 		}
-	} else {
-		var media token
-		switch p.MediaType {
-		case "APPLICATION", "AUDIO", "IMAGE", "FONT", "MESSAGE", "MODEL", "VIDEO":
-			media = dquote(p.MediaType)
-		default:
-			media = string0(p.MediaType)
-		}
-		// ../rfc/9051:6404 ../rfc/9051:6407
-		r = listspace{
-			media, string0(p.MediaSubType), // ../rfc/9051:6723
-			// ../rfc/9051:6376
-			bodyFldParams(p), // ../rfc/9051:6401
-			nilOrString(p.ContentID),
-			nilOrString(p.ContentDescription),
-			bodyFldEnc(p.ContentTransferEncoding),
-			number(p.EndOffset - p.BodyOffset),
-		}
 	}
-	if extensible {
+	var media token
-		// ../rfc/9051:6366
+	switch p.MediaType {
-		r = append(r,
+	case "APPLICATION", "AUDIO", "IMAGE", "FONT", "MESSAGE", "MODEL", "VIDEO":
-			bodyFldMd5(p),
+		media = dquote(p.MediaType)
-			bodyFldDisp(log, p),
+	default:
-			bodyFldLang(p),
+		media = string0(p.MediaType)
-			bodyFldLoc(p),
+	}
-		)
+	// ../rfc/9051:6404 ../rfc/9051:6407
+	return listspace{
+		media, string0(p.MediaSubType), // ../rfc/9051:6723
+		// ../rfc/9051:6376
+		bodyFldParams(p.ContentTypeParams), // ../rfc/9051:6401
+		nilOrString(p.ContentID),
+		nilOrString(p.ContentDescription),
+		bodyFldEnc(p.ContentTransferEncoding),
+		number(p.EndOffset - p.BodyOffset),
 	}
-	return r
 }

imapserver/fetch_test.go

@@ -5,33 +5,22 @@ import (
 	"testing"
 	"time"
 
-	"github.com/mjl-/bstore"
-
 	"github.com/mjl-/mox/imapclient"
-	"github.com/mjl-/mox/store"
 )
 
 func TestFetch(t *testing.T) {
-	testFetch(t, false)
+	tc := start(t)
-}
-
-func TestFetchUIDOnly(t *testing.T) {
-	testFetch(t, true)
-}
-
-func testFetch(t *testing.T, uidonly bool) {
-	tc := start(t, uidonly)
 	defer tc.close()
 
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
-	tc.client.Enable(imapclient.CapIMAP4rev2)
+	tc.client.Enable("imap4rev2")
 	received, err := time.Parse(time.RFC3339, "2022-11-16T10:01:00+01:00")
 	tc.check(err, "parse time")
-	tc.client.Append("inbox", makeAppendTime(exampleMsg, received))
+	tc.client.Append("inbox", nil, &received, []byte(exampleMsg))
 	tc.client.Select("inbox")
 
 	uid1 := imapclient.FetchUID(1)
-	date1 := imapclient.FetchInternalDate{Date: received}
+	date1 := imapclient.FetchInternalDate("16-Nov-2022 10:01:00 +0100")
 	rfcsize1 := imapclient.FetchRFC822Size(len(exampleMsg))
 	env1 := imapclient.FetchEnvelope{
 		Date:      "Mon, 7 Feb 1994 21:52:25 -0800",
@@ -43,29 +32,20 @@ func testFetch(t *testing.T, uidonly bool) {
 		MessageID: "<B27397-0100000@Blurdybloop.example>",
 	}
 	noflags := imapclient.FetchFlags(nil)
-	bodystructbody1 := imapclient.BodyTypeText{
-		MediaType:    "TEXT",
-		MediaSubtype: "PLAIN",
-		BodyFields: imapclient.BodyFields{
-			Params: [][2]string{[...]string{"CHARSET", "US-ASCII"}},
-			Octets: 57,
-		},
-		Lines: 2,
-	}
 	bodyxstructure1 := imapclient.FetchBodystructure{
 		RespAttr: "BODY",
-		Body:     bodystructbody1,
+		Body: imapclient.BodyTypeText{
+			MediaType:    "TEXT",
+			MediaSubtype: "PLAIN",
+			BodyFields: imapclient.BodyFields{
+				Params: [][2]string{[...]string{"CHARSET", "US-ASCII"}},
+				Octets: 57,
+			},
+			Lines: 2,
+		},
 	}
 	bodystructure1 := bodyxstructure1
 	bodystructure1.RespAttr = "BODYSTRUCTURE"
-	bodyext1 := imapclient.BodyExtension1Part{
-		Disposition:       ptr((*string)(nil)),
-		DispositionParams: ptr([][2]string(nil)),
-		Language:          ptr([]string(nil)),
-		Location:          ptr((*string)(nil)),
-	}
-	bodystructbody1.Ext = &bodyext1
-	bodystructure1.Body = bodystructbody1
 
 	split := strings.SplitN(exampleMsg, "\r\n\r\n", 2)
 	exampleMsgHeader := split[0] + "\r\n\r\n"
@@ -92,188 +72,136 @@ func testFetch(t *testing.T, uidonly bool) {
 	headerSplit := strings.SplitN(exampleMsgHeader, "\r\n", 2)
 	dateheader1 := imapclient.FetchBody{RespAttr: "BODY[HEADER.FIELDS (Date)]", Section: "HEADER.FIELDS (Date)", Body: headerSplit[0] + "\r\n\r\n"}
 	nodateheader1 := imapclient.FetchBody{RespAttr: "BODY[HEADER.FIELDS.NOT (Date)]", Section: "HEADER.FIELDS.NOT (Date)", Body: headerSplit[1]}
-	mime1 := imapclient.FetchBody{RespAttr: "BODY[1.MIME]", Section: "1.MIME", Body: "Content-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n"}
+	date1header1 := imapclient.FetchBody{RespAttr: "BODY[1.HEADER.FIELDS (Date)]", Section: "1.HEADER.FIELDS (Date)", Body: headerSplit[0] + "\r\n\r\n"}
+	nodate1header1 := imapclient.FetchBody{RespAttr: "BODY[1.HEADER.FIELDS.NOT (Date)]", Section: "1.HEADER.FIELDS.NOT (Date)", Body: headerSplit[1]}
+	mime1 := imapclient.FetchBody{RespAttr: "BODY[1.MIME]", Section: "1.MIME", Body: "MIME-Version: 1.0\r\nContent-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n\r\n"}
 
 	flagsSeen := imapclient.FetchFlags{`\Seen`}
 
-	if !uidonly {
+	tc.transactf("ok", "fetch 1 all")
-		tc.transactf("ok", "fetch 1 all")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, date1, rfcsize1, env1, noflags}})
-		tc.xuntagged(tc.untaggedFetch(1, 1, date1, rfcsize1, env1, noflags))
 
-		tc.transactf("ok", "fetch 1 fast")
+	tc.transactf("ok", "fetch 1 fast")
-		tc.xuntagged(tc.untaggedFetch(1, 1, date1, rfcsize1, noflags))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, date1, rfcsize1, noflags}})
 
-		tc.transactf("ok", "fetch 1 full")
+	tc.transactf("ok", "fetch 1 full")
-		tc.xuntagged(tc.untaggedFetch(1, 1, date1, rfcsize1, env1, bodyxstructure1, noflags))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, date1, rfcsize1, env1, bodyxstructure1, noflags}})
 
-		tc.transactf("ok", "fetch 1 flags")
+	tc.transactf("ok", "fetch 1 flags")
-		tc.xuntagged(tc.untaggedFetch(1, 1, noflags))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, noflags}})
 
-		tc.transactf("ok", "fetch 1 bodystructure")
+	tc.transactf("ok", "fetch 1 bodystructure")
-		tc.xuntagged(tc.untaggedFetch(1, 1, bodystructure1))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodystructure1}})
 
-		// Should be returned unmodified, because there is no content-transfer-encoding.
+	// Should be returned unmodified, because there is no content-transfer-encoding.
-		tc.transactf("ok", "fetch 1 binary[]")
+	tc.transactf("ok", "fetch 1 binary[]")
-		tc.xuntagged(tc.untaggedFetch(1, 1, binary1, noflags))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binary1, flagsSeen}})
-		tc.transactf("ok", "noop")
-		tc.xuntagged(tc.untaggedFetch(1, 1, flagsSeen))
 
-		tc.transactf("ok", "fetch 1 binary[1]")
+	tc.transactf("ok", "fetch 1 binary[1]")
-		tc.xuntagged(tc.untaggedFetch(1, 1, binarypart1)) // Seen flag not changed.
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binarypart1}}) // Seen flag not changed.
 
-		tc.client.MSNStoreFlagsClear("1", true, `\Seen`)
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
-		tc.transactf("ok", "uid fetch 1 binary[]<1.1>")
+	tc.transactf("ok", "fetch 1 binary[]<1.1>")
-		tc.xuntagged(
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binarypartial1, flagsSeen}})
-			tc.untaggedFetch(1, 1, binarypartial1, noflags),
-			tc.untaggedFetch(1, 1, flagsSeen), // For UID FETCH, we get the flags during the command.
-		)
 
-		tc.client.MSNStoreFlagsClear("1", true, `\Seen`)
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
-		tc.transactf("ok", "fetch 1 binary[1]<1.1>")
+	tc.transactf("ok", "fetch 1 binary[1]<1.1>")
-		tc.xuntagged(tc.untaggedFetch(1, 1, binarypartpartial1, noflags))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binarypartpartial1, flagsSeen}})
-		tc.transactf("ok", "noop")
-		tc.xuntagged(tc.untaggedFetch(1, 1, flagsSeen))
 
-		tc.client.MSNStoreFlagsClear("1", true, `\Seen`)
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
-		tc.transactf("ok", "fetch 1 binary[]<10000.10001>")
+	tc.transactf("ok", "fetch 1 binary[]<10000.10001>")
-		tc.xuntagged(tc.untaggedFetch(1, 1, binaryend1, noflags))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binaryend1, flagsSeen}})
-		tc.transactf("ok", "noop")
-		tc.xuntagged(tc.untaggedFetch(1, 1, flagsSeen))
 
-		tc.client.MSNStoreFlagsClear("1", true, `\Seen`)
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
-		tc.transactf("ok", "fetch 1 binary[1]<10000.10001>")
+	tc.transactf("ok", "fetch 1 binary[1]<10000.10001>")
-		tc.xuntagged(tc.untaggedFetch(1, 1, binarypartend1, noflags))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binarypartend1, flagsSeen}})
-		tc.transactf("ok", "noop")
-		tc.xuntagged(tc.untaggedFetch(1, 1, flagsSeen))
 
-		tc.transactf("ok", "fetch 1 binary.size[]")
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
-		tc.xuntagged(tc.untaggedFetch(1, 1, binarysize1))
+	tc.transactf("ok", "fetch 1 binary.size[]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binarysize1}})
 
-		tc.transactf("ok", "fetch 1 binary.size[1]")
+	tc.transactf("ok", "fetch 1 binary.size[1]")
-		tc.xuntagged(tc.untaggedFetch(1, 1, binarysizepart1))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binarysizepart1}})
 
-		tc.client.MSNStoreFlagsClear("1", true, `\Seen`)
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
-		tc.transactf("ok", "fetch 1 body[]")
+	tc.transactf("ok", "fetch 1 body[]")
-		tc.xuntagged(tc.untaggedFetch(1, 1, body1, noflags))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, body1, flagsSeen}})
-		tc.transactf("ok", "noop")
+	tc.transactf("ok", "fetch 1 body[]<1.2>")
-		tc.xuntagged(tc.untaggedFetch(1, 1, flagsSeen))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodyoff1}}) // Already seen.
-		tc.transactf("ok", "fetch 1 body[]<1.2>")
-		tc.xuntagged(tc.untaggedFetch(1, 1, bodyoff1)) // Already seen.
-		tc.transactf("ok", "noop")
-		tc.xuntagged() // Already seen.
 
-		tc.client.MSNStoreFlagsClear("1", true, `\Seen`)
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
-		tc.transactf("ok", "fetch 1 body[1]")
+	tc.transactf("ok", "fetch 1 body[1]")
-		tc.xuntagged(tc.untaggedFetch(1, 1, bodypart1, noflags))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodypart1, flagsSeen}})
-		tc.transactf("ok", "noop")
-		tc.xuntagged(tc.untaggedFetch(1, 1, flagsSeen))
 
-		tc.client.MSNStoreFlagsClear("1", true, `\Seen`)
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
-		tc.transactf("ok", "fetch 1 body[1]<1.2>")
+	tc.transactf("ok", "fetch 1 body[1]<1.2>")
-		tc.xuntagged(tc.untaggedFetch(1, 1, body1off1, noflags))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, body1off1, flagsSeen}})
-		tc.transactf("ok", "noop")
-		tc.xuntagged(tc.untaggedFetch(1, 1, flagsSeen))
 
-		tc.client.MSNStoreFlagsClear("1", true, `\Seen`)
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
-		tc.transactf("ok", "fetch 1 body[1]<100000.100000>")
+	tc.transactf("ok", "fetch 1 body[1]<100000.100000>")
-		tc.xuntagged(tc.untaggedFetch(1, 1, bodyend1, noflags))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodyend1, flagsSeen}})
-		tc.transactf("ok", "noop")
-		tc.xuntagged(tc.untaggedFetch(1, 1, flagsSeen))
 
-		tc.client.MSNStoreFlagsClear("1", true, `\Seen`)
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
-		tc.transactf("ok", "fetch 1 body[header]")
+	tc.transactf("ok", "fetch 1 body[header]")
-		tc.xuntagged(tc.untaggedFetch(1, 1, bodyheader1, noflags))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodyheader1, flagsSeen}})
-		tc.transactf("ok", "noop")
-		tc.xuntagged(tc.untaggedFetch(1, 1, flagsSeen))
 
-		tc.client.MSNStoreFlagsClear("1", true, `\Seen`)
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
-		tc.transactf("ok", "fetch 1 body[text]")
+	tc.transactf("ok", "fetch 1 body[text]")
-		tc.xuntagged(tc.untaggedFetch(1, 1, bodytext1, noflags))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodytext1, flagsSeen}})
-		tc.transactf("ok", "noop")
-		tc.xuntagged(tc.untaggedFetch(1, 1, flagsSeen))
 
-		// equivalent to body.peek[header], ../rfc/3501:3183
+	// equivalent to body.peek[header], ../rfc/3501:3183
-		tc.transactf("ok", "fetch 1 rfc822.header")
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
-		tc.xuntagged(tc.untaggedFetch(1, 1, rfcheader1))
+	tc.transactf("ok", "fetch 1 rfc822.header")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, rfcheader1}})
 
-		// equivalent to body[text], ../rfc/3501:3199
+	// equivalent to body[text], ../rfc/3501:3199
-		tc.client.MSNStoreFlagsClear("1", true, `\Seen`)
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
-		tc.transactf("ok", "fetch 1 rfc822.text")
+	tc.transactf("ok", "fetch 1 rfc822.text")
-		tc.xuntagged(tc.untaggedFetch(1, 1, rfctext1, noflags))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, rfctext1, flagsSeen}})
-		tc.transactf("ok", "noop")
-		tc.xuntagged(tc.untaggedFetch(1, 1, flagsSeen))
 
-		// equivalent to body[], ../rfc/3501:3179
+	// equivalent to body[], ../rfc/3501:3179
-		tc.client.MSNStoreFlagsClear("1", true, `\Seen`)
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
-		tc.transactf("ok", "fetch 1 rfc822")
+	tc.transactf("ok", "fetch 1 rfc822")
-		tc.xuntagged(tc.untaggedFetch(1, 1, rfc1, noflags))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, rfc1, flagsSeen}})
-		tc.transactf("ok", "noop")
-		tc.xuntagged(tc.untaggedFetch(1, 1, flagsSeen))
 
-		// With PEEK, we should not get the \Seen flag.
+	// With PEEK, we should not get the \Seen flag.
-		tc.client.MSNStoreFlagsClear("1", true, `\Seen`)
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
-		tc.transactf("ok", "fetch 1 body.peek[]")
+	tc.transactf("ok", "fetch 1 body.peek[]")
-		tc.xuntagged(tc.untaggedFetch(1, 1, body1))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, body1}})
 
-		tc.transactf("ok", "fetch 1 binary.peek[]")
+	tc.transactf("ok", "fetch 1 binary.peek[]")
-		tc.xuntagged(tc.untaggedFetch(1, 1, binary1))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binary1}})
 
-		// HEADER.FIELDS and .NOT
+	// HEADER.FIELDS and .NOT
-		tc.transactf("ok", "fetch 1 body.peek[header.fields (date)]")
+	tc.transactf("ok", "fetch 1 body.peek[header.fields (date)]")
-		tc.xuntagged(tc.untaggedFetch(1, 1, dateheader1))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, dateheader1}})
-		tc.transactf("ok", "fetch 1 body.peek[header.fields.not (date)]")
+	tc.transactf("ok", "fetch 1 body.peek[header.fields.not (date)]")
-		tc.xuntagged(tc.untaggedFetch(1, 1, nodateheader1))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, nodateheader1}})
-		// For non-multipart messages, 1 means the whole message, but since it's not of
+	// For non-multipart messages, 1 means the whole message. ../rfc/9051:4481
-		// type message/{rfc822,global} (a message), you can't get the message headers.
+	tc.transactf("ok", "fetch 1 body.peek[1.header.fields (date)]")
-		// ../rfc/9051:4481
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, date1header1}})
-		tc.transactf("no", "fetch 1 body.peek[1.header]")
+	tc.transactf("ok", "fetch 1 body.peek[1.header.fields.not (date)]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, nodate1header1}})
 
-		// MIME, part 1 for non-multipart messages is the message itself. ../rfc/9051:4481
+	// MIME, part 1 for non-multipart messages is the message itself. ../rfc/9051:4481
-		tc.transactf("ok", "fetch 1 body.peek[1.mime]")
+	tc.transactf("ok", "fetch 1 body.peek[1.mime]")
-		tc.xuntagged(tc.untaggedFetch(1, 1, mime1))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, mime1}})
 
-		// Missing sequence number. ../rfc/9051:7018
+	// Missing sequence number. ../rfc/9051:7018
-		tc.transactf("bad", "fetch 2 body[]")
+	tc.transactf("bad", "fetch 2 body[]")
 
-		tc.client.MSNStoreFlagsClear("1", true, `\Seen`)
+	tc.transactf("ok", "fetch 1:1 body[]")
-		tc.transactf("ok", "fetch 1:1 body[]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, body1, flagsSeen}})
-		tc.xuntagged(tc.untaggedFetch(1, 1, body1, noflags))
-		tc.transactf("ok", "noop")
-		tc.xuntagged(tc.untaggedFetch(1, 1, flagsSeen))
-	} else {
-		tc.client.UIDStoreFlagsAdd("1", true, `\Seen`)
-		tc.transactf("ok", "noop")
-	}
 
 	// UID fetch
 	tc.transactf("ok", "uid fetch 1 body[]")
-	tc.xuntagged(tc.untaggedFetch(1, 1, body1))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, body1}})
 
+	// UID fetch
 	tc.transactf("ok", "uid fetch 2 body[]")
 	tc.xuntagged()
 
-	// SAVEDATE
+	// Test some invalid syntax.
-	tc.transactf("ok", "uid fetch 1 savedate")
-	// Fetch exact SaveDate we'll be expecting from server.
-	var saveDate time.Time
-	err = tc.account.DB.Read(ctxbg, func(tx *bstore.Tx) error {
-		inbox, err := tc.account.MailboxFind(tx, "Inbox")
-		tc.check(err, "get inbox")
-		if inbox == nil {
-			t.Fatalf("missing inbox")
-		}
-		m, err := bstore.QueryTx[store.Message](tx).FilterNonzero(store.Message{MailboxID: inbox.ID, UID: store.UID(uid1)}).Get()
-		tc.check(err, "get message")
-		if m.SaveDate == nil {
-			t.Fatalf("zero savedate for message")
-		}
-		saveDate = m.SaveDate.Truncate(time.Second)
-		return nil
-	})
-	tc.check(err, "get savedate")
-	tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchSaveDate{SaveDate: &saveDate}))
-
-	// Test some invalid syntax. Also invalid for uidonly.
 	tc.transactf("bad", "fetch")
 	tc.transactf("bad", "fetch ")
 	tc.transactf("bad", "fetch  ")
@@ -296,38 +224,25 @@ func testFetch(t *testing.T, uidonly bool) {
 	tc.transactf("bad", "fetch 1 body[header.fields.not ()]") // List must be non-empty.
 	tc.transactf("bad", "fetch 1 body[mime]")                 // MIME must be prefixed with a number. ../rfc/9051:4497
 
-	if !uidonly {
+	tc.transactf("no", "fetch 1 body[2]") // No such part.
-		tc.transactf("no", "fetch 1 body[2]") // No such part.
-	}
 
 	// Add more complex message.
 
+	uid2 := imapclient.FetchUID(2)
 	bodystructure2 := imapclient.FetchBodystructure{
 		RespAttr: "BODYSTRUCTURE",
 		Body: imapclient.BodyTypeMpart{
 			Bodies: []any{
-				imapclient.BodyTypeBasic{BodyFields: imapclient.BodyFields{Octets: 275}, Ext: &bodyext1},
+				imapclient.BodyTypeBasic{BodyFields: imapclient.BodyFields{Octets: 275}},
-				imapclient.BodyTypeText{MediaType: "TEXT", MediaSubtype: "PLAIN", BodyFields: imapclient.BodyFields{Params: [][2]string{{"CHARSET", "US-ASCII"}}, Octets: 114}, Lines: 3, Ext: &bodyext1},
+				imapclient.BodyTypeText{MediaType: "TEXT", MediaSubtype: "PLAIN", BodyFields: imapclient.BodyFields{Params: [][2]string{{"CHARSET", "US-ASCII"}}, Octets: 114}, Lines: 3},
 				imapclient.BodyTypeMpart{
 					Bodies: []any{
-						imapclient.BodyTypeBasic{MediaType: "AUDIO", MediaSubtype: "BASIC", BodyFields: imapclient.BodyFields{CTE: "BASE64", Octets: 22}, Ext: &bodyext1},
+						imapclient.BodyTypeBasic{MediaType: "AUDIO", MediaSubtype: "BASIC", BodyFields: imapclient.BodyFields{CTE: "BASE64", Octets: 22}},
-						imapclient.BodyTypeBasic{MediaType: "IMAGE", MediaSubtype: "JPEG", BodyFields: imapclient.BodyFields{CTE: "BASE64"}, Ext: &imapclient.BodyExtension1Part{
+						imapclient.BodyTypeBasic{MediaType: "IMAGE", MediaSubtype: "JPEG", BodyFields: imapclient.BodyFields{CTE: "BASE64"}},
-							Disposition:       ptr(ptr("inline")),
-							DispositionParams: ptr([][2]string{{"filename", "image.jpg"}}),
-							Language:          ptr([]string(nil)),
-							Location:          ptr((*string)(nil)),
-						}},
 					},
 					MediaSubtype: "PARALLEL",
-					Ext: &imapclient.BodyExtensionMpart{
-						Params:            [][2]string{{"BOUNDARY", "unique-boundary-2"}},
-						Disposition:       ptr((*string)(nil)), // Present but nil.
-						DispositionParams: ptr([][2]string(nil)),
-						Language:          ptr([]string(nil)),
-						Location:          ptr((*string)(nil)),
-					},
 				},
-				imapclient.BodyTypeText{MediaType: "TEXT", MediaSubtype: "ENRICHED", BodyFields: imapclient.BodyFields{Octets: 145}, Lines: 5, Ext: &bodyext1},
+				imapclient.BodyTypeText{MediaType: "TEXT", MediaSubtype: "ENRICHED", BodyFields: imapclient.BodyFields{Octets: 145}, Lines: 5},
 				imapclient.BodyTypeMsg{
 					MediaType:    "MESSAGE",
 					MediaSubtype: "RFC822",
@@ -340,64 +255,49 @@ func testFetch(t *testing.T, uidonly bool) {
 						To:      []imapclient.Address{{Name: "mox", Adl: "", Mailbox: "info", Host: "mox.example"}},
 					},
 					Bodystructure: imapclient.BodyTypeText{
-						MediaType: "TEXT", MediaSubtype: "PLAIN", BodyFields: imapclient.BodyFields{Params: [][2]string{{"CHARSET", "ISO-8859-1"}}, CTE: "QUOTED-PRINTABLE", Octets: 51}, Lines: 1, Ext: &bodyext1},
+						MediaType: "TEXT", MediaSubtype: "PLAIN", BodyFields: imapclient.BodyFields{Params: [][2]string{{"CHARSET", "ISO-8859-1"}}, CTE: "QUOTED-PRINTABLE", Octets: 51}, Lines: 1},
 					Lines: 7,
-					Ext: &imapclient.BodyExtension1Part{
-						MD5:               ptr("MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY="),
-						Disposition:       ptr((*string)(nil)),
-						DispositionParams: ptr([][2]string(nil)),
-						Language:          ptr([]string{"en", "de"}),
-						Location:          ptr(ptr("http://localhost")),
-					},
 				},
 			},
 			MediaSubtype: "MIXED",
-			Ext: &imapclient.BodyExtensionMpart{
-				Params:            [][2]string{{"BOUNDARY", "unique-boundary-1"}},
-				Disposition:       ptr((*string)(nil)), // Present but nil.
-				DispositionParams: ptr([][2]string(nil)),
-				Language:          ptr([]string(nil)),
-				Location:          ptr((*string)(nil)),
-			},
 		},
 	}
-	tc.client.Append("inbox", makeAppendTime(nestedMessage, received))
+	tc.client.Append("inbox", nil, &received, []byte(nestedMessage))
-	tc.transactf("ok", "uid fetch 2 bodystructure")
+	tc.transactf("ok", "fetch 2 bodystructure")
-	tc.xuntagged(tc.untaggedFetch(2, 2, bodystructure2))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
 
 	// Multiple responses.
-	if !uidonly {
+	tc.transactf("ok", "fetch 1:2 bodystructure")
-		tc.transactf("ok", "fetch 1:2 bodystructure")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodystructure1}}, imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
-		tc.xuntagged(tc.untaggedFetch(1, 1, bodystructure1), tc.untaggedFetch(2, 2, bodystructure2))
+	tc.transactf("ok", "fetch 1,2 bodystructure")
-		tc.transactf("ok", "fetch 1,2 bodystructure")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodystructure1}}, imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
-		tc.xuntagged(tc.untaggedFetch(1, 1, bodystructure1), tc.untaggedFetch(2, 2, bodystructure2))
+	tc.transactf("ok", "fetch 2:1 bodystructure")
-		tc.transactf("ok", "fetch 2:1 bodystructure")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodystructure1}}, imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
-		tc.xuntagged(tc.untaggedFetch(1, 1, bodystructure1), tc.untaggedFetch(2, 2, bodystructure2))
+	tc.transactf("ok", "fetch 1:* bodystructure")
-		tc.transactf("ok", "fetch 1:* bodystructure")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodystructure1}}, imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
-		tc.xuntagged(tc.untaggedFetch(1, 1, bodystructure1), tc.untaggedFetch(2, 2, bodystructure2))
+	tc.transactf("ok", "fetch *:1 bodystructure")
-		tc.transactf("ok", "fetch *:1 bodystructure")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodystructure1}}, imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
-		tc.xuntagged(tc.untaggedFetch(1, 1, bodystructure1), tc.untaggedFetch(2, 2, bodystructure2))
+	tc.transactf("ok", "fetch *:2 bodystructure")
-		tc.transactf("ok", "fetch *:2 bodystructure")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
-		tc.xuntagged(tc.untaggedFetch(2, 2, bodystructure2))
+
-		tc.transactf("ok", "fetch * bodystructure") // Highest msgseq.
+	tc.transactf("ok", "fetch * bodystructure") // Highest msgseq.
-		tc.xuntagged(tc.untaggedFetch(2, 2, bodystructure2))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
-	}
 
 	tc.transactf("ok", "uid fetch 1:* bodystructure")
-	tc.xuntagged(tc.untaggedFetch(1, 1, bodystructure1), tc.untaggedFetch(2, 2, bodystructure2))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodystructure1}}, imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
 
 	tc.transactf("ok", "uid fetch 1:2 bodystructure")
-	tc.xuntagged(tc.untaggedFetch(1, 1, bodystructure1), tc.untaggedFetch(2, 2, bodystructure2))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodystructure1}}, imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
 
 	tc.transactf("ok", "uid fetch 1,2 bodystructure")
-	tc.xuntagged(tc.untaggedFetch(1, 1, bodystructure1), tc.untaggedFetch(2, 2, bodystructure2))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodystructure1}}, imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
 
 	tc.transactf("ok", "uid fetch 2:2 bodystructure")
-	tc.xuntagged(tc.untaggedFetch(2, 2, bodystructure2))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
 
 	// todo: read the bodies/headers of the parts, and of the nested message.
-	tc.transactf("ok", "uid fetch 2 body.peek[]")
+	tc.transactf("ok", "fetch 2 body.peek[]")
-	tc.xuntagged(tc.untaggedFetch(2, 2, imapclient.FetchBody{RespAttr: "BODY[]", Body: nestedMessage}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[]", Body: nestedMessage}}})
 
 	part1 := tocrlf(`  ... Some text appears here ...
 
@@ -407,22 +307,22 @@ func testFetch(t *testing.T, uidonly bool) {
  It could have been done with explicit typing as in the
  next part.]
 `)
-	tc.transactf("ok", "uid fetch 2 body.peek[1]")
+	tc.transactf("ok", "fetch 2 body.peek[1]")
-	tc.xuntagged(tc.untaggedFetch(2, 2, imapclient.FetchBody{RespAttr: "BODY[1]", Section: "1", Body: part1}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[1]", Section: "1", Body: part1}}})
 
-	tc.transactf("no", "uid fetch 2 binary.peek[3]") // Only allowed on leaf parts, not multiparts.
+	tc.transactf("no", "fetch 2 binary.peek[3]") // Only allowed on leaf parts, not multiparts.
-	tc.transactf("no", "uid fetch 2 binary.peek[5]") // Only allowed on leaf parts, not messages.
+	tc.transactf("no", "fetch 2 binary.peek[5]") // Only allowed on leaf parts, not messages.
 
 	part31 := "aGVsbG8NCndvcmxkDQo=\r\n"
 	part31dec := "hello\r\nworld\r\n"
-	tc.transactf("ok", "uid fetch 2 binary.size[3.1]")
+	tc.transactf("ok", "fetch 2 binary.size[3.1]")
-	tc.xuntagged(tc.untaggedFetch(2, 2, imapclient.FetchBinarySize{RespAttr: "BINARY.SIZE[3.1]", Parts: []uint32{3, 1}, Size: int64(len(part31dec))}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBinarySize{RespAttr: "BINARY.SIZE[3.1]", Parts: []uint32{3, 1}, Size: int64(len(part31dec))}}})
 
-	tc.transactf("ok", "uid fetch 2 body.peek[3.1]")
+	tc.transactf("ok", "fetch 2 body.peek[3.1]")
-	tc.xuntagged(tc.untaggedFetch(2, 2, imapclient.FetchBody{RespAttr: "BODY[3.1]", Section: "3.1", Body: part31}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[3.1]", Section: "3.1", Body: part31}}})
 
-	tc.transactf("ok", "uid fetch 2 binary.peek[3.1]")
+	tc.transactf("ok", "fetch 2 binary.peek[3.1]")
-	tc.xuntagged(tc.untaggedFetch(2, 2, imapclient.FetchBinary{RespAttr: "BINARY[3.1]", Parts: []uint32{3, 1}, Data: part31dec}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBinary{RespAttr: "BINARY[3.1]", Parts: []uint32{3, 1}, Data: part31dec}}})
 
 	part3 := tocrlf(`--unique-boundary-2
 Content-Type: audio/basic
@@ -433,18 +333,19 @@ aGVsbG8NCndvcmxkDQo=
 --unique-boundary-2
 Content-Type: image/jpeg
 Content-Transfer-Encoding: base64
-Content-Disposition: inline; filename=image.jpg
 
 
 --unique-boundary-2--
 
 `)
-	tc.transactf("ok", "uid fetch 2 body.peek[3]")
+	tc.transactf("ok", "fetch 2 body.peek[3]")
-	tc.xuntagged(tc.untaggedFetch(2, 2, imapclient.FetchBody{RespAttr: "BODY[3]", Section: "3", Body: part3}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[3]", Section: "3", Body: part3}}})
 
-	part2mime := "Content-type: text/plain; charset=US-ASCII\r\n"
+	part2mime := tocrlf(`Content-type: text/plain; charset=US-ASCII
-	tc.transactf("ok", "uid fetch 2 body.peek[2.mime]")
+
-	tc.xuntagged(tc.untaggedFetch(2, 2, imapclient.FetchBody{RespAttr: "BODY[2.MIME]", Section: "2.MIME", Body: part2mime}))
+`)
+	tc.transactf("ok", "fetch 2 body.peek[2.mime]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[2.MIME]", Section: "2.MIME", Body: part2mime}}})
 
 	part5 := tocrlf(`From: info@mox.example
 To: mox <info@mox.example>
@@ -454,8 +355,8 @@ Content-Transfer-Encoding: Quoted-printable
 
   ... Additional text in ISO-8859-1 goes here ...
 `)
-	tc.transactf("ok", "uid fetch 2 body.peek[5]")
+	tc.transactf("ok", "fetch 2 body.peek[5]")
-	tc.xuntagged(tc.untaggedFetch(2, 2, imapclient.FetchBody{RespAttr: "BODY[5]", Section: "5", Body: part5}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[5]", Section: "5", Body: part5}}})
 
 	part5header := tocrlf(`From: info@mox.example
 To: mox <info@mox.example>
@@ -464,101 +365,39 @@ Content-Type: Text/plain; charset=ISO-8859-1
 Content-Transfer-Encoding: Quoted-printable
 
 `)
-	tc.transactf("ok", "uid fetch 2 body.peek[5.header]")
+	tc.transactf("ok", "fetch 2 body.peek[5.header]")
-	tc.xuntagged(tc.untaggedFetch(2, 2, imapclient.FetchBody{RespAttr: "BODY[5.HEADER]", Section: "5.HEADER", Body: part5header}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[5.HEADER]", Section: "5.HEADER", Body: part5header}}})
+
+	part5mime := tocrlf(`Content-Type: Text/plain; charset=ISO-8859-1
+Content-Transfer-Encoding: Quoted-printable
 
-	part5mime := tocrlf(`Content-Type: message/rfc822
-Content-MD5: MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=
-Content-Language: en,de
-Content-Location: http://localhost
 `)
-	tc.transactf("ok", "uid fetch 2 body.peek[5.mime]")
+	tc.transactf("ok", "fetch 2 body.peek[5.mime]")
-	tc.xuntagged(tc.untaggedFetch(2, 2, imapclient.FetchBody{RespAttr: "BODY[5.MIME]", Section: "5.MIME", Body: part5mime}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[5.MIME]", Section: "5.MIME", Body: part5mime}}})
 
 	part5text := "  ... Additional text in ISO-8859-1 goes here ...\r\n"
+	tc.transactf("ok", "fetch 2 body.peek[5.text]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[5.TEXT]", Section: "5.TEXT", Body: part5text}}})
 
-	tc.transactf("ok", "uid fetch 2 body.peek[5.text]")
+	tc.transactf("ok", "fetch 2 body.peek[5.1]")
-	tc.xuntagged(tc.untaggedFetch(2, 2, imapclient.FetchBody{RespAttr: "BODY[5.TEXT]", Section: "5.TEXT", Body: part5text}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[5.1]", Section: "5.1", Body: part5text}}})
-
-	part5body := "  ... Additional text in ISO-8859-1 goes here ...\r\n"
-	tc.transactf("ok", "uid fetch 2 body.peek[5.1]")
-	tc.xuntagged(tc.untaggedFetch(2, 2, imapclient.FetchBody{RespAttr: "BODY[5.1]", Section: "5.1", Body: part5body}))
-
-	// 5.1 is the part that is the sub message, but not as message/rfc822, but as part,
-	// so we cannot request a header.
-	tc.transactf("no", "uid fetch 2 body.peek[5.1.header]")
 
 	// In case of EXAMINE instead of SELECT, we should not be seeing any changed \Seen flags for non-peek commands.
-	tc.client.UIDStoreFlagsClear("1", true, `\Seen`)
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
 	tc.client.Unselect()
 	tc.client.Examine("inbox")
 
-	// Preview
+	tc.transactf("ok", "fetch 1 binary[]")
-	preview := "Hello Joe, do you think we can meet at 3:30 tomorrow?"
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binary1}})
-	tc.transactf("ok", "uid fetch 1 preview")
-	tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchPreview{Preview: &preview}))
 
-	tc.transactf("ok", "uid fetch 1 preview (lazy)")
+	tc.transactf("ok", "fetch 1 body[]")
-	tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchPreview{Preview: &preview}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, body1}})
 
-	// On-demand preview and saving on first request.
+	tc.transactf("ok", "fetch 1 rfc822.text")
-	err = tc.account.DB.Write(ctxbg, func(tx *bstore.Tx) error {
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, rfctext1}})
-		m := store.Message{ID: 1}
-		err := tx.Get(&m)
-		tcheck(t, err, "get message")
-		if m.UID != 1 {
-			t.Fatalf("uid %d instead of 1", m.UID)
-		}
-		m.Preview = nil
-		err = tx.Update(&m)
-		tcheck(t, err, "remove preview from message")
-		return nil
-	})
-	tcheck(t, err, "remove preview from database")
 
-	tc.transactf("ok", "uid fetch 1 preview")
+	tc.transactf("ok", "fetch 1 rfc822")
-	tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchPreview{Preview: &preview}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, rfc1}})
-	m := store.Message{ID: 1}
-	err = tc.account.DB.Get(ctxbg, &m)
-	tcheck(t, err, "get message")
-	if m.Preview == nil {
-		t.Fatalf("preview missing")
-	} else if *m.Preview != preview+"\n" {
-		t.Fatalf("got preview %q, expected %q", *m.Preview, preview+"\n")
-	}
-
-	tc.transactf("bad", "uid fetch 1 preview (bogus)")
-
-	// Start a second session. Use it to remove the message. First session should still
-	// be able to access the messages.
-	tc2 := startNoSwitchboard(t, uidonly)
-	defer tc2.closeNoWait()
-	tc2.login("mjl@mox.example", password0)
-	tc2.client.Select("inbox")
-	tc2.client.UIDStoreFlagsSet("1", true, `\Deleted`)
-	tc2.client.Expunge()
-	tc2.client.Logout()
-
-	if uidonly {
-		tc.transactf("ok", "uid fetch 1 binary[]")
-		tc.xuntagged(
-			tc.untaggedFetch(1, 1, imapclient.FetchFlags{`\Deleted`}),
-			imapclient.UntaggedVanished{UIDs: xparseNumSet("1")},
-		)
-		// Message no longer available in session.
-	} else {
-		tc.transactf("ok", "fetch 1 binary[]")
-		tc.xuntagged(tc.untaggedFetch(1, 1, binary1))
-
-		tc.transactf("ok", "fetch 1 body[]")
-		tc.xuntagged(tc.untaggedFetch(1, 1, body1))
-
-		tc.transactf("ok", "fetch 1 rfc822.text")
-		tc.xuntagged(tc.untaggedFetch(1, 1, rfctext1))
-
-		tc.transactf("ok", "fetch 1 rfc822")
-		tc.xuntagged(tc.untaggedFetch(1, 1, rfc1))
-	}
 
 	tc.client.Logout()
 }

imapserver/fuzz_test.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"log/slog"
 	"net"
 	"os"
 	"path/filepath"
@@ -60,11 +59,33 @@ func FuzzServer(f *testing.F) {
 		f.Add(tag + cmd)
 	}
 
+	log := mlog.New("imapserver", nil)
+	mox.Context = ctxbg
+	mox.ConfigStaticPath = filepath.FromSlash("../testdata/imapserverfuzz/mox.conf")
+	mox.MustLoadConfig(true, false)
+	dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir)
+	os.RemoveAll(dataDir)
+	acc, err := store.OpenAccount(log, "mjl")
+	if err != nil {
+		f.Fatalf("open account: %v", err)
+	}
+	defer func() {
+		acc.Close()
+		acc.CheckClosed()
+	}()
+	err = acc.SetPassword(log, password0)
+	if err != nil {
+		f.Fatalf("set password: %v", err)
+	}
+	defer store.Switchboard()()
+
+	comm := store.RegisterComm(acc)
+	defer comm.Unregister()
+
 	var cid int64 = 1
 
 	var fl *os.File
 	if false {
-		var err error
 		fl, err = os.OpenFile("fuzz.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
 		if err != nil {
 			f.Fatalf("fuzz log")
@@ -78,34 +99,6 @@ func FuzzServer(f *testing.F) {
 	}
 
 	f.Fuzz(func(t *testing.T, s string) {
-		log := mlog.New("imapserver", nil)
-		mox.ConfigStaticPath = filepath.FromSlash("../testdata/imapserverfuzz/mox.conf")
-		mox.MustLoadConfig(true, false)
-		store.Close() // May not be open, we ignore error.
-		dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir)
-		os.RemoveAll(dataDir)
-		err := store.Init(ctxbg)
-		if err != nil {
-			t.Fatalf("store init: %v", err)
-		}
-		defer store.Switchboard()()
-
-		acc, err := store.OpenAccount(log, "mjl", false)
-		if err != nil {
-			t.Fatalf("open account: %v", err)
-		}
-		defer func() {
-			acc.Close()
-			acc.WaitClosed()
-		}()
-		err = acc.SetPassword(log, password0)
-		if err != nil {
-			t.Fatalf("set password: %v", err)
-		}
-
-		comm := store.RegisterComm(acc)
-		defer comm.Unregister()
-
 		run := func(cmds []string) {
 			limitersInit() // Reset rate limiters.
 			serverConn, clientConn := net.Pipe()
@@ -128,23 +121,19 @@ func FuzzServer(f *testing.F) {
 
 				err := clientConn.SetDeadline(time.Now().Add(time.Second))
 				flog(err, "set client deadline")
-				opts := imapclient.Opts{
+				client, _ := imapclient.New(clientConn, true)
-					Logger: slog.Default().With("cid", mox.Cid()),
-					Error:  func(err error) { panic(err) },
-				}
-				client, _ := imapclient.New(clientConn, &opts)
 
 				for _, cmd := range cmds {
-					client.WriteCommandf("", "%s", cmd)
+					client.Commandf("", "%s", cmd)
-					client.ReadResponse()
+					client.Response()
 				}
-				client.WriteCommandf("", "%s", s)
+				client.Commandf("", "%s", s)
-				client.ReadResponse()
+				client.Response()
 			}()
 
 			err = serverConn.SetDeadline(time.Now().Add(time.Second))
 			flog(err, "set server deadline")
-			serve("test", cid, nil, serverConn, false, false, true, false, "")
+			serve("test", cid, nil, serverConn, false, true)
 			cid++
 		}
 

imapserver/idle_test.go

@@ -9,14 +9,13 @@ import (
 )
 
 func TestIdle(t *testing.T) {
-	tc1 := start(t, false)
+	tc1 := start(t)
 	defer tc1.close()
+	tc1.client.Login("mjl@mox.example", password0)
 
-	tc2 := startNoSwitchboard(t, false)
+	tc2 := startNoSwitchboard(t)
-	defer tc2.closeNoWait()
+	defer tc2.close()
-
+	tc2.client.Login("mjl@mox.example", password0)
-	tc1.login("mjl@mox.example", password0)
-	tc2.login("mjl@mox.example", password0)
 
 	tc1.transactf("ok", "select inbox")
 	tc2.transactf("ok", "select inbox")

imapserver/list.go

@@ -1,14 +1,12 @@
 package imapserver
 
 import (
-	"bytes"
 	"fmt"
+	"path"
 	"sort"
 	"strings"
 
 	"github.com/mjl-/bstore"
-
-	"github.com/mjl-/mox/mox-"
 	"github.com/mjl-/mox/store"
 )
 
@@ -62,7 +60,6 @@ func (c *conn) cmdList(tag, cmd string, p *parser) {
 	isExtended = isExtended || isList
 	var retSubscribed, retChildren bool
 	var retStatusAttrs []string
-	var retMetadata []string
 	if p.take(" RETURN (") {
 		isExtended = true
 		// ../rfc/9051:6613 ../rfc/9051:6915 ../rfc/9051:7072 ../rfc/9051:6821 ../rfc/5819:95
@@ -93,18 +90,6 @@ func (c *conn) cmdList(tag, cmd string, p *parser) {
 					retStatusAttrs = append(retStatusAttrs, p.xstatusAtt())
 				}
 				p.xtake(")")
-			case "METADATA":
-				// ../rfc/9590:167
-				p.xspace()
-				p.xtake("(")
-				for {
-					s := p.xmetadataKey()
-					retMetadata = append(retMetadata, s)
-					if !p.space() {
-						break
-					}
-				}
-				p.xtake(")")
 			default:
 				// ../rfc/9051:2398
 				xsyntaxErrorf("bad list return option %q", w)
@@ -115,7 +100,7 @@ func (c *conn) cmdList(tag, cmd string, p *parser) {
 
 	if !isExtended && reference == "" && patterns[0] == "" {
 		// ../rfc/9051:2277 ../rfc/3501:2221
-		c.xbwritelinef(`* LIST () "/" ""`)
+		c.bwritelinef(`* LIST () "/" ""`)
 		c.ok(tag, cmd)
 		return
 	}
@@ -132,7 +117,6 @@ func (c *conn) cmdList(tag, cmd string, p *parser) {
 	}
 	re := xmailboxPatternMatcher(reference, patterns)
 	var responseLines []string
-	var respMetadata []concatspace
 
 	c.account.WithRLock(func() {
 		c.xdbread(func(tx *bstore.Tx) {
@@ -146,11 +130,10 @@ func (c *conn) cmdList(tag, cmd string, p *parser) {
 			var nameList []string
 
 			q := bstore.QueryTx[store.Mailbox](tx)
-			q.FilterEqual("Expunged", false)
 			err := q.ForEach(func(mb store.Mailbox) error {
 				names[mb.Name] = info{mailbox: &mb}
 				nameList = append(nameList, mb.Name)
-				for p := mox.ParentMailboxName(mb.Name); p != ""; p = mox.ParentMailboxName(p) {
+				for p := path.Dir(mb.Name); p != "."; p = path.Dir(p) {
 					hasChild[p] = true
 				}
 				return nil
@@ -165,7 +148,7 @@ func (c *conn) cmdList(tag, cmd string, p *parser) {
 				if !ok {
 					nameList = append(nameList, sub.Name)
 				}
-				for p := mox.ParentMailboxName(sub.Name); p != ""; p = mox.ParentMailboxName(p) {
+				for p := path.Dir(sub.Name); p != "."; p = path.Dir(p) {
 					hasSubscribedChild[p] = true
 				}
 				return nil
@@ -208,64 +191,39 @@ func (c *conn) cmdList(tag, cmd string, p *parser) {
 					flags = append(flags, bare(`\Subscribed`))
 				}
 				if info.mailbox != nil {
-					add := func(b bool, v string) {
+					if info.mailbox.Archive {
-						if b {
+						flags = append(flags, bare(`\Archive`))
-							flags = append(flags, bare(v))
+					}
-						}
+					if info.mailbox.Draft {
+						flags = append(flags, bare(`\Drafts`))
+					}
+					if info.mailbox.Junk {
+						flags = append(flags, bare(`\Junk`))
+					}
+					if info.mailbox.Sent {
+						flags = append(flags, bare(`\Sent`))
+					}
+					if info.mailbox.Trash {
+						flags = append(flags, bare(`\Trash`))
 					}
-					mb := info.mailbox
-					add(mb.Archive, `\Archive`)
-					add(mb.Draft, `\Drafts`)
-					add(mb.Junk, `\Junk`)
-					add(mb.Sent, `\Sent`)
-					add(mb.Trash, `\Trash`)
 				}
 
 				var extStr string
 				if extended != nil {
 					extStr = " " + extended.pack(c)
 				}
-				line := fmt.Sprintf(`* LIST %s "/" %s%s`, flags.pack(c), mailboxt(name).pack(c), extStr)
+				line := fmt.Sprintf(`* LIST %s "/" %s%s`, flags.pack(c), astring(c.encodeMailbox(name)).pack(c), extStr)
 				responseLines = append(responseLines, line)
 
 				if retStatusAttrs != nil && info.mailbox != nil {
 					responseLines = append(responseLines, c.xstatusLine(tx, *info.mailbox, retStatusAttrs))
 				}
-
-				// ../rfc/9590:101
-				if info.mailbox != nil && len(retMetadata) > 0 {
-					var meta listspace
-					for _, k := range retMetadata {
-						q := bstore.QueryTx[store.Annotation](tx)
-						q.FilterNonzero(store.Annotation{MailboxID: info.mailbox.ID, Key: k})
-						q.FilterEqual("Expunged", false)
-						a, err := q.Get()
-						var v token
-						if err == bstore.ErrAbsent {
-							v = nilt
-						} else {
-							xcheckf(err, "get annotation")
-							if a.IsString {
-								v = string0(string(a.Value))
-							} else {
-								v = readerSizeSyncliteral{bytes.NewReader(a.Value), int64(len(a.Value)), true}
-							}
-						}
-						meta = append(meta, astring(k), v)
-					}
-					line := concatspace{bare("*"), bare("METADATA"), mailboxt(info.mailbox.Name), meta}
-					respMetadata = append(respMetadata, line)
-				}
 			}
 		})
 	})
 
 	for _, line := range responseLines {
-		c.xbwritelinef("%s", line)
+		c.bwritelinef("%s", line)
-	}
-	for _, meta := range respMetadata {
-		meta.xwriteTo(c, c.xbw)
-		c.xbwritelinef("")
 	}
 	c.ok(tag, cmd)
 }

imapserver/list_test.go

@@ -8,18 +8,10 @@ import (
 )
 
 func TestListBasic(t *testing.T) {
-	testListBasic(t, false)
+	tc := start(t)
-}
-
-func TestListBasicUIDOnly(t *testing.T) {
-	testListBasic(t, true)
-}
-
-func testListBasic(t *testing.T, uidonly bool) {
-	tc := start(t, uidonly)
 	defer tc.close()
 
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
 
 	ulist := func(name string, flags ...string) imapclient.UntaggedList {
 		if len(flags) == 0 {
@@ -34,9 +26,6 @@ func testListBasic(t *testing.T, uidonly bool) {
 	tc.last(tc.client.List("Inbox"))
 	tc.xuntagged(ulist("Inbox"))
 
-	tc.last(tc.client.List("expungebox"))
-	tc.xuntagged()
-
 	tc.last(tc.client.List("%"))
 	tc.xuntagged(ulist("Archive", `\Archive`), ulist("Drafts", `\Drafts`), ulist("Inbox"), ulist("Junk", `\Junk`), ulist("Sent", `\Sent`), ulist("Trash", `\Trash`))
 
@@ -46,7 +35,7 @@ func testListBasic(t *testing.T, uidonly bool) {
 	tc.last(tc.client.List("A*"))
 	tc.xuntagged(ulist("Archive", `\Archive`))
 
-	tc.client.Create("Inbox/todo", nil)
+	tc.client.Create("Inbox/todo")
 
 	tc.last(tc.client.List("Inbox*"))
 	tc.xuntagged(ulist("Inbox"), ulist("Inbox/todo"))
@@ -67,20 +56,12 @@ func testListBasic(t *testing.T, uidonly bool) {
 }
 
 func TestListExtended(t *testing.T) {
-	testListExtended(t, false)
-}
-
-func TestListExtendedUIDOnly(t *testing.T) {
-	testListExtended(t, true)
-}
-
-func testListExtended(t *testing.T, uidonly bool) {
 	defer mockUIDValidity()()
 
-	tc := start(t, uidonly)
+	tc := start(t)
 	defer tc.close()
 
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
 
 	ulist := func(name string, flags ...string) imapclient.UntaggedList {
 		if len(flags) == 0 {
@@ -97,7 +78,7 @@ func testListExtended(t *testing.T, uidonly bool) {
 	for _, name := range store.DefaultInitialMailboxes.Regular {
 		uidvals[name] = 1
 	}
-	var uidvalnext uint32 = 3
+	var uidvalnext uint32 = 2
 	uidval := func(name string) uint32 {
 		v, ok := uidvals[name]
 		if !ok {
@@ -165,7 +146,7 @@ func testListExtended(t *testing.T, uidonly bool) {
 	tc.last(tc.client.ListFull(false, "A*", "Junk"))
 	tc.xuntagged(xlist("Archive", Farchive), ustatus("Archive"), xlist("Junk", Fjunk), ustatus("Junk"))
 
-	tc.client.Create("Inbox/todo", nil)
+	tc.client.Create("Inbox/todo")
 
 	tc.last(tc.client.ListFull(false, "Inbox*"))
 	tc.xuntagged(ulist("Inbox", Fhaschildren, Fsubscribed), ustatus("Inbox"), xlist("Inbox/todo"), ustatus("Inbox/todo"))
@@ -223,7 +204,7 @@ func testListExtended(t *testing.T, uidonly bool) {
 	tc.transactf("ok", `list (remote) "inbox" "a"`)
 	tc.xuntagged()
 
-	tc.client.Create("inbox/a", nil)
+	tc.client.Create("inbox/a")
 	tc.transactf("ok", `list (remote) "inbox" "a"`)
 	tc.xuntagged(ulist("Inbox/a"))
 
@@ -235,21 +216,4 @@ func testListExtended(t *testing.T, uidonly bool) {
 	tc.transactf("bad", `list (recursivematch remote) "" "*"`) // "remote" is not a base selection option.
 	tc.transactf("bad", `list (unknown) "" "*"`)               // Unknown selection options must result in BAD.
 	tc.transactf("bad", `list () "" "*" return (unknown)`)     // Unknown return options must result in BAD.
-
-	// Return metadata.
-	tc.transactf("ok", `setmetadata inbox (/private/comment "y")`)
-	tc.transactf("ok", `list () "" ("inbox") return (metadata (/private/comment /shared/comment))`)
-	tc.xuntagged(
-		ulist("Inbox"),
-		imapclient.UntaggedMetadataAnnotations{
-			Mailbox: "Inbox",
-			Annotations: []imapclient.Annotation{
-				{Key: "/private/comment", IsString: true, Value: []byte("y")},
-				{Key: "/shared/comment"},
-			},
-		},
-	)
-
-	tc.transactf("bad", `list () "" ("inbox") return (metadata ())`)                                    // Metadata list must be non-empty.
-	tc.transactf("bad", `list () "" ("inbox") return (metadata (/shared/comment "/private/comment" ))`) // Extra space.
 }

imapserver/lsub_test.go

@@ -7,18 +7,10 @@ import (
 )
 
 func TestLsub(t *testing.T) {
-	testLsub(t, false)
+	tc := start(t)
-}
-
-func TestLsubUIDOnly(t *testing.T) {
-	testLsub(t, true)
-}
-
-func testLsub(t *testing.T, uidonly bool) {
-	tc := start(t, uidonly)
 	defer tc.close()
 
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
 
 	tc.transactf("bad", "lsub")       // Missing params.
 	tc.transactf("bad", `lsub ""`)    // Missing param.
@@ -27,9 +19,6 @@ func testLsub(t *testing.T, uidonly bool) {
 	tc.transactf("ok", `lsub "" x*`)
 	tc.xuntagged()
 
-	tc.transactf("ok", `lsub "" expungebox`)
-	tc.xuntagged(imapclient.UntaggedLsub{Separator: '/', Mailbox: "expungebox"})
-
 	tc.transactf("ok", "create a/b/c")
 	tc.transactf("ok", `lsub "" a/*`)
 	tc.xuntagged(imapclient.UntaggedLsub{Separator: '/', Mailbox: "a/b"}, imapclient.UntaggedLsub{Separator: '/', Mailbox: "a/b/c"})

imapserver/main_test.go

@@ -1,17 +0,0 @@
-package imapserver
-
-import (
-	"fmt"
-	"os"
-	"testing"
-
-	"github.com/mjl-/mox/metrics"
-)
-
-func TestMain(m *testing.M) {
-	m.Run()
-	if metrics.Panics.Load() > 0 {
-		fmt.Println("unhandled panics encountered")
-		os.Exit(2)
-	}
-}

imapserver/metadata.go

@@ -1,317 +0,0 @@
-package imapserver
-
-import (
-	"bytes"
-	"fmt"
-	"strings"
-
-	"github.com/mjl-/bstore"
-
-	"github.com/mjl-/mox/store"
-)
-
-// Changed during tests.
-var metadataMaxKeys = 1000
-var metadataMaxSize = 1000 * 1000
-
-// Metadata errata:
-// ../rfc/5464:183 ../rfc/5464-eid1691
-// ../rfc/5464:564 ../rfc/5464-eid1692
-// ../rfc/5464:494 ../rfc/5464-eid2785 ../rfc/5464-eid2786
-// ../rfc/5464:698 ../rfc/5464-eid3868
-
-// Note: We do not tie the special-use mailbox flags to a (synthetic) private
-// per-mailbox annotation. ../rfc/6154:303
-
-// For registration of names, see https://www.iana.org/assignments/imap-metadata/imap-metadata.xhtml
-
-// Get metadata annotations, per mailbox or globally.
-//
-// State: Authenticated and selected.
-func (c *conn) cmdGetmetadata(tag, cmd string, p *parser) {
-	// Command: ../rfc/5464:412
-
-	// Request syntax: ../rfc/5464:792
-
-	p.xspace()
-	var optMaxSize int64 = -1
-	var optDepth string
-	if p.take("(") {
-		for {
-			if p.take("MAXSIZE") {
-				// ../rfc/5464:804
-				p.xspace()
-				v := p.xnumber()
-				if optMaxSize >= 0 {
-					p.xerrorf("only a single maxsize option accepted")
-				}
-				optMaxSize = int64(v)
-			} else if p.take("DEPTH") {
-				// ../rfc/5464:823
-				p.xspace()
-				s := p.xtakelist("0", "1", "INFINITY")
-				if optDepth != "" {
-					p.xerrorf("only single depth option accepted")
-				}
-				optDepth = s
-			} else {
-				// ../rfc/5464:800 We are not doing anything further parsing for future extensions.
-				p.xerrorf("unknown option for getmetadata, expected maxsize or depth")
-			}
-
-			if p.take(")") {
-				break
-			}
-			p.xspace()
-		}
-		p.xspace()
-	}
-	mailboxName := p.xmailbox()
-	if mailboxName != "" {
-		mailboxName = xcheckmailboxname(mailboxName, true)
-	}
-	p.xspace()
-	// Entries ../rfc/5464:768
-	entryNames := map[string]struct{}{}
-	if p.take("(") {
-		for {
-			s := p.xmetadataKey()
-			entryNames[s] = struct{}{}
-			if p.take(")") {
-				break
-			}
-			p.xtake(" ")
-		}
-	} else {
-		s := p.xmetadataKey()
-		entryNames[s] = struct{}{}
-	}
-	p.xempty()
-
-	var annotations []store.Annotation
-	longentries := -1 // Size of largest value skipped due to optMaxSize. ../rfc/5464:482
-
-	c.account.WithRLock(func() {
-		c.xdbread(func(tx *bstore.Tx) {
-			q := bstore.QueryTx[store.Annotation](tx)
-			if mailboxName == "" {
-				q.FilterEqual("MailboxID", 0)
-			} else {
-				mb := c.xmailbox(tx, mailboxName, "TRYCREATE")
-				q.FilterNonzero(store.Annotation{MailboxID: mb.ID})
-			}
-			q.FilterEqual("Expunged", false)
-			q.SortAsc("MailboxID", "Key") // For tests.
-			err := q.ForEach(func(a store.Annotation) error {
-				// ../rfc/5464:516
-				switch optDepth {
-				case "", "0":
-					if _, ok := entryNames[a.Key]; !ok {
-						return nil
-					}
-				case "1", "INFINITY":
-					// Go through all keys, matching depth.
-					if _, ok := entryNames[a.Key]; ok {
-						break
-					}
-					var match bool
-					for s := range entryNames {
-						prefix := s
-						if s != "/" {
-							prefix += "/"
-						}
-						if !strings.HasPrefix(a.Key, prefix) {
-							continue
-						}
-						if optDepth == "INFINITY" {
-							match = true
-							break
-						}
-						suffix := a.Key[len(prefix):]
-						t := strings.SplitN(suffix, "/", 2)
-						if len(t) == 1 {
-							match = true
-							break
-						}
-					}
-					if !match {
-						return nil
-					}
-				default:
-					xcheckf(fmt.Errorf("%q", optDepth), "missing case for depth")
-				}
-
-				if optMaxSize >= 0 && int64(len(a.Value)) > optMaxSize {
-					longentries = max(longentries, len(a.Value))
-				} else {
-					annotations = append(annotations, a)
-				}
-				return nil
-			})
-			xcheckf(err, "looking up annotations")
-		})
-	})
-
-	// Response syntax: ../rfc/5464:807 ../rfc/5464:778
-	// We can only send untagged responses when we have any matches.
-	if len(annotations) > 0 {
-		fmt.Fprintf(c.xbw, "* METADATA %s (", mailboxt(mailboxName).pack(c))
-		for i, a := range annotations {
-			if i > 0 {
-				fmt.Fprint(c.xbw, " ")
-			}
-			astring(a.Key).xwriteTo(c, c.xbw)
-			fmt.Fprint(c.xbw, " ")
-			if a.IsString {
-				string0(string(a.Value)).xwriteTo(c, c.xbw)
-			} else {
-				v := readerSizeSyncliteral{bytes.NewReader(a.Value), int64(len(a.Value)), true}
-				v.xwriteTo(c, c.xbw)
-			}
-		}
-		c.xbwritelinef(")")
-	}
-
-	if longentries >= 0 {
-		c.xbwritelinef("%s OK [METADATA LONGENTRIES %d] getmetadata done", tag, longentries)
-	} else {
-		c.ok(tag, cmd)
-	}
-}
-
-// Set metadata annotation, per mailbox or globally.
-//
-// We allow both /private/* and /shared/*, we store them in the same way since we
-// don't have ACL extension support yet or another mechanism for access control.
-//
-// State: Authenticated and selected.
-func (c *conn) cmdSetmetadata(tag, cmd string, p *parser) {
-	// Command: ../rfc/5464:547
-
-	// Request syntax: ../rfc/5464:826
-
-	p.xspace()
-	mailboxName := p.xmailbox()
-	// Empty name means a global (per-account) annotation, not for a mailbox.
-	if mailboxName != "" {
-		mailboxName = xcheckmailboxname(mailboxName, true)
-	}
-	p.xspace()
-	p.xtake("(")
-	var l []store.Annotation
-	for {
-		key, isString, value := p.xmetadataKeyValue()
-		l = append(l, store.Annotation{Key: key, IsString: isString, Value: value})
-		if p.take(")") {
-			break
-		}
-		p.xspace()
-	}
-	p.xempty()
-
-	// Additional checks on entry names.
-	for _, a := range l {
-		// ../rfc/5464:217
-		if !strings.HasPrefix(a.Key, "/private/") && !strings.HasPrefix(a.Key, "/shared/") {
-			// ../rfc/5464:346
-			xuserErrorf("only /private/* and /shared/* entry names allowed")
-		}
-
-		// We also enforce that /private/vendor/ is followed by at least 2 elements.
-		// ../rfc/5464:234
-		switch {
-		case a.Key == "/private/vendor",
-			strings.HasPrefix(a.Key, "/private/vendor/"),
-			a.Key == "/shared/vendor", strings.HasPrefix(a.Key, "/shared/vendor/"):
-
-			t := strings.SplitN(a.Key[1:], "/", 4)
-			if len(t) < 4 {
-				xuserErrorf("entry names starting with /private/vendor or /shared/vendor must have at least 4 components")
-			}
-		}
-	}
-
-	// Store the annotations, possibly removing/inserting/updating them.
-	c.account.WithWLock(func() {
-		var changes []store.Change
-		var modseq store.ModSeq
-
-		c.xdbwrite(func(tx *bstore.Tx) {
-			var mb store.Mailbox // mb.ID as 0 is used in query below.
-			if mailboxName != "" {
-				mb = c.xmailbox(tx, mailboxName, "TRYCREATE")
-			}
-
-			for _, a := range l {
-				q := bstore.QueryTx[store.Annotation](tx)
-				q.FilterNonzero(store.Annotation{Key: a.Key})
-				q.FilterEqual("MailboxID", mb.ID) // Can be zero.
-				q.FilterEqual("Expunged", false)
-				oa, err := q.Get()
-				// Nil means remove. ../rfc/5464:579
-				if err == bstore.ErrAbsent && a.Value == nil {
-					continue
-				}
-				if modseq == 0 {
-					var err error
-					modseq, err = c.account.NextModSeq(tx)
-					xcheckf(err, "get next modseq")
-				}
-				if err == bstore.ErrAbsent {
-					a.MailboxID = mb.ID
-					a.CreateSeq = modseq
-					a.ModSeq = modseq
-					err = tx.Insert(&a)
-					xcheckf(err, "inserting annotation")
-					changes = append(changes, a.Change(mailboxName))
-				} else {
-					xcheckf(err, "get metadata")
-					oa.ModSeq = modseq
-					if a.Value == nil {
-						oa.Expunged = true
-					}
-					oa.IsString = a.IsString
-					oa.Value = a.Value
-					err = tx.Update(&oa)
-					xcheckf(err, "updating metdata")
-					changes = append(changes, oa.Change(mailboxName))
-				}
-			}
-
-			c.xcheckMetadataSize(tx)
-
-			// ../rfc/7162:1335
-			if mb.ID != 0 && modseq != 0 {
-				mb.ModSeq = modseq
-				err := tx.Update(&mb)
-				xcheckf(err, "updating mailbox with modseq")
-			}
-		})
-
-		c.broadcast(changes)
-	})
-
-	c.ok(tag, cmd)
-}
-
-func (c *conn) xcheckMetadataSize(tx *bstore.Tx) {
-	// Check for total size. We allow a total of 1000 entries, with total capacity of 1MB.
-	// ../rfc/5464:383
-	var n int
-	var size int
-	err := bstore.QueryTx[store.Annotation](tx).FilterEqual("Expunged", false).ForEach(func(a store.Annotation) error {
-		n++
-		if n > metadataMaxKeys {
-			// ../rfc/5464:590
-			xusercodeErrorf("METADATA (TOOMANY)", "too many metadata entries, 1000 allowed in total")
-		}
-		size += len(a.Key) + len(a.Value)
-		if size > metadataMaxSize {
-			// ../rfc/5464:585 We only have a max total size limit, not per entry. We'll
-			// mention the max total size.
-			xusercodeErrorf(fmt.Sprintf("METADATA (MAXSIZE %d)", metadataMaxSize), "metadata entry values too large, total maximum size is 1MB")
-		}
-		return nil
-	})
-	xcheckf(err, "checking metadata annotation size")
-}

imapserver/metadata_test.go

@@ -1,296 +0,0 @@
-package imapserver
-
-import (
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/mjl-/mox/imapclient"
-)
-
-func TestMetadata(t *testing.T) {
-	testMetadata(t, false)
-}
-
-func TestMetadataUIDOnly(t *testing.T) {
-	testMetadata(t, true)
-}
-
-func testMetadata(t *testing.T, uidonly bool) {
-	tc := start(t, uidonly)
-	defer tc.close()
-
-	tc.login("mjl@mox.example", password0)
-
-	tc.transactf("ok", `getmetadata "" /private/comment`)
-	tc.xuntagged()
-
-	tc.transactf("ok", `getmetadata inbox (/private/comment)`)
-	tc.xuntagged()
-
-	tc.transactf("ok", `setmetadata "" (/PRIVATE/COMMENT "global value")`)
-	tc.transactf("ok", `setmetadata inbox (/private/comment "mailbox value")`)
-
-	tc.transactf("ok", `create metabox`)
-	tc.transactf("ok", `setmetadata metabox (/private/comment "mailbox value")`)
-	tc.transactf("ok", `setmetadata metabox (/shared/comment "mailbox value")`)
-	tc.transactf("ok", `setmetadata metabox (/shared/comment nil)`) // Remove.
-	tc.transactf("ok", `delete metabox`)                            // Delete mailbox with live and expunged metadata.
-
-	tc.transactf("no", `setmetadata expungebox (/private/comment "mailbox value")`)
-	tc.xcodeWord("TRYCREATE")
-
-	tc.transactf("ok", `getmetadata "" ("/private/comment")`)
-	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
-		Mailbox: "",
-		Annotations: []imapclient.Annotation{
-			{Key: "/private/comment", IsString: true, Value: []byte("global value")},
-		},
-	})
-
-	tc.transactf("ok", `setmetadata Inbox (/shared/comment "share")`)
-
-	tc.transactf("ok", `getmetadata inbox (/private/comment /private/unknown /shared/comment)`)
-	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
-		Mailbox: "Inbox",
-		Annotations: []imapclient.Annotation{
-			{Key: "/private/comment", IsString: true, Value: []byte("mailbox value")},
-			{Key: "/shared/comment", IsString: true, Value: []byte("share")},
-		},
-	})
-
-	tc.transactf("no", `setmetadata doesnotexist (/private/comment "test")`) // Bad mailbox.
-	tc.transactf("no", `setmetadata Inbox (/badprefix/comment "")`)
-	tc.transactf("no", `setmetadata Inbox (/private/vendor "")`)          // /*/vendor must have more components.
-	tc.transactf("no", `setmetadata Inbox (/private/vendor/stillbad "")`) // /*/vendor must have more components.
-	tc.transactf("ok", `setmetadata Inbox (/private/vendor/a/b "")`)
-	tc.transactf("bad", `setmetadata Inbox (/private/no* "")`)
-	tc.transactf("bad", `setmetadata Inbox (/private/no%% "")`)
-	tc.transactf("bad", `setmetadata Inbox (/private/notrailingslash/ "")`)
-	tc.transactf("bad", `setmetadata Inbox (/private//nodupslash "")`)
-	tc.transactf("bad", "setmetadata Inbox (/private/\001 \"\")")
-	tc.transactf("bad", "setmetadata Inbox (/private/\u007f \"\")")
-	tc.transactf("bad", `getmetadata (depth 0 depth 0) inbox (/private/a)`) // Duplicate option.
-	tc.transactf("bad", `getmetadata (depth badvalue) inbox (/private/a)`)
-	tc.transactf("bad", `getmetadata (maxsize invalid) inbox (/private/a)`)
-	tc.transactf("bad", `getmetadata (badoption) inbox (/private/a)`)
-
-	// Update existing annotation by key.
-	tc.transactf("ok", `setmetadata "" (/PRIVATE/COMMENT "global updated")`)
-	tc.transactf("ok", `setmetadata inbox (/private/comment "mailbox updated")`)
-	tc.transactf("ok", `getmetadata "" (/private/comment)`)
-	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
-		Mailbox: "",
-		Annotations: []imapclient.Annotation{
-			{Key: "/private/comment", IsString: true, Value: []byte("global updated")},
-		},
-	})
-	tc.transactf("ok", `getmetadata inbox (/private/comment)`)
-	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
-		Mailbox: "Inbox",
-		Annotations: []imapclient.Annotation{
-			{Key: "/private/comment", IsString: true, Value: []byte("mailbox updated")},
-		},
-	})
-
-	// Delete annotation with nil value.
-	tc.transactf("ok", `setmetadata "" (/private/comment nil)`)
-	tc.transactf("ok", `setmetadata inbox (/private/comment nil)`)
-	tc.transactf("ok", `getmetadata "" (/private/comment)`)
-	tc.xuntagged()
-	tc.transactf("ok", `getmetadata inbox (/private/comment)`)
-	tc.xuntagged()
-
-	// Create a literal8 value, not a string.
-	tc.transactf("ok", "setmetadata inbox (/private/comment ~{4+}\r\ntest)")
-	tc.transactf("ok", `getmetadata inbox (/private/comment)`)
-	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
-		Mailbox: "Inbox",
-		Annotations: []imapclient.Annotation{
-			{Key: "/private/comment", IsString: false, Value: []byte("test")},
-		},
-	})
-
-	// Request with a maximum size, we don't get anything larger.
-	tc.transactf("ok", `setmetadata inbox (/private/another "longer")`)
-	tc.transactf("ok", `getmetadata (maxsize 4) inbox (/private/comment /private/another)`)
-	tc.xcode(imapclient.CodeMetadataLongEntries(6))
-	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
-		Mailbox: "Inbox",
-		Annotations: []imapclient.Annotation{
-			{Key: "/private/comment", IsString: false, Value: []byte("test")},
-		},
-	})
-
-	// Request with various depth values.
-	tc.transactf("ok", `setmetadata inbox (/private/a "x" /private/a/b "x" /private/a/b/c "x" /private/a/b/c/d "x")`)
-	tc.transactf("ok", `getmetadata (depth 0) inbox (/private/a)`)
-	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
-		Mailbox: "Inbox",
-		Annotations: []imapclient.Annotation{
-			{Key: "/private/a", IsString: true, Value: []byte("x")},
-		},
-	})
-	tc.transactf("ok", `getmetadata (depth 1) inbox (/private/a)`)
-	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
-		Mailbox: "Inbox",
-		Annotations: []imapclient.Annotation{
-			{Key: "/private/a", IsString: true, Value: []byte("x")},
-			{Key: "/private/a/b", IsString: true, Value: []byte("x")},
-		},
-	})
-	tc.transactf("ok", `getmetadata (depth infinity) inbox (/private/a)`)
-	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
-		Mailbox: "Inbox",
-		Annotations: []imapclient.Annotation{
-			{Key: "/private/a", IsString: true, Value: []byte("x")},
-			{Key: "/private/a/b", IsString: true, Value: []byte("x")},
-			{Key: "/private/a/b/c", IsString: true, Value: []byte("x")},
-			{Key: "/private/a/b/c/d", IsString: true, Value: []byte("x")},
-		},
-	})
-	// Same as previous, but ask for everything below /.
-	tc.transactf("ok", `getmetadata (depth infinity) inbox ("")`)
-	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
-		Mailbox: "Inbox",
-		Annotations: []imapclient.Annotation{
-			{Key: "/private/a", IsString: true, Value: []byte("x")},
-			{Key: "/private/a/b", IsString: true, Value: []byte("x")},
-			{Key: "/private/a/b/c", IsString: true, Value: []byte("x")},
-			{Key: "/private/a/b/c/d", IsString: true, Value: []byte("x")},
-			{Key: "/private/another", IsString: true, Value: []byte("longer")},
-			{Key: "/private/comment", IsString: false, Value: []byte("test")},
-			{Key: "/private/vendor/a/b", IsString: true, Value: []byte("")},
-			{Key: "/shared/comment", IsString: true, Value: []byte("share")},
-		},
-	})
-
-	// Deleting a mailbox with an annotation should work and annotations should not
-	// come back when recreating mailbox.
-	tc.transactf("ok", "create testbox")
-	tc.transactf("ok", `setmetadata testbox (/private/a "x")`)
-	tc.transactf("ok", "delete testbox")
-	tc.transactf("ok", "create testbox")
-	tc.transactf("ok", `getmetadata testbox (/private/a)`)
-	tc.xuntagged()
-
-	// When renaming mailbox, annotations must be copied to destination mailbox.
-	tc.transactf("ok", "rename inbox newbox")
-	tc.transactf("ok", `getmetadata newbox (/private/a)`)
-	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
-		Mailbox: "newbox",
-		Annotations: []imapclient.Annotation{
-			{Key: "/private/a", IsString: true, Value: []byte("x")},
-		},
-	})
-	tc.transactf("ok", `getmetadata inbox (/private/a)`)
-	tc.xuntagged(imapclient.UntaggedMetadataAnnotations{
-		Mailbox: "Inbox",
-		Annotations: []imapclient.Annotation{
-			{Key: "/private/a", IsString: true, Value: []byte("x")},
-		},
-	})
-
-	// Broadcast should not happen when metadata capability is not enabled.
-	tc2 := startNoSwitchboard(t, uidonly)
-	defer tc2.closeNoWait()
-	tc2.login("mjl@mox.example", password0)
-	tc2.client.Select("inbox")
-
-	tc2.cmdf("", "idle")
-	tc2.readprefixline("+ ")
-	done := make(chan error)
-	go func() {
-		defer func() {
-			x := recover()
-			if x != nil {
-				done <- fmt.Errorf("%v", x)
-			}
-		}()
-		untagged, _ := tc2.client.ReadUntagged()
-		var exists imapclient.UntaggedExists
-		tuntagged(tc2.t, untagged, &exists)
-		tc2.writelinef("done")
-		tc2.response("ok")
-		done <- nil
-	}()
-
-	// Should not cause idle to return.
-	tc.transactf("ok", `setmetadata inbox (/private/a "y")`)
-	// Cause to return.
-	tc.transactf("ok", "append inbox {4+}\r\ntest")
-
-	timer := time.NewTimer(time.Second)
-	defer timer.Stop()
-	select {
-	case err := <-done:
-		tc.check(err, "idle")
-	case <-timer.C:
-		t.Fatalf("idle did not finish")
-	}
-
-	// Broadcast should happen when metadata capability is enabled.
-	tc2.client.Enable(imapclient.CapMetadata)
-	tc2.cmdf("", "idle")
-	tc2.readprefixline("+ ")
-	done = make(chan error)
-	go func() {
-		defer func() {
-			x := recover()
-			if x != nil {
-				done <- fmt.Errorf("%v", x)
-			}
-		}()
-		untagged, _ := tc2.client.ReadUntagged()
-		var metadataKeys imapclient.UntaggedMetadataKeys
-		tuntagged(tc2.t, untagged, &metadataKeys)
-		tc2.writelinef("done")
-		tc2.response("ok")
-		done <- nil
-	}()
-
-	// Should cause idle to return.
-	tc.transactf("ok", `setmetadata inbox (/private/a "z")`)
-
-	timer = time.NewTimer(time.Second)
-	defer timer.Stop()
-	select {
-	case err := <-done:
-		tc.check(err, "idle")
-	case <-timer.C:
-		t.Fatalf("idle did not finish")
-	}
-}
-
-func TestMetadataLimit(t *testing.T) {
-	tc := start(t, false)
-	defer tc.close()
-
-	tc.login("mjl@mox.example", password0)
-
-	maxKeys, maxSize := metadataMaxKeys, metadataMaxSize
-	defer func() {
-		metadataMaxKeys = maxKeys
-		metadataMaxSize = maxSize
-	}()
-	metadataMaxKeys = 10
-	metadataMaxSize = 1000
-
-	// Reach max total size limit.
-	buf := make([]byte, metadataMaxSize+1)
-	for i := range buf {
-		buf[i] = 'x'
-	}
-	tc.cmdf("", "setmetadata inbox (/private/large ~{%d+}", len(buf))
-	tc.client.Write(buf)
-	tc.client.Writelinef(")")
-	tc.response("no")
-	tc.xcode(imapclient.CodeMetadataMaxSize(metadataMaxSize))
-
-	// Reach limit for max number.
-	for i := 1; i <= metadataMaxKeys; i++ {
-		tc.transactf("ok", `setmetadata inbox (/private/key%d "test")`, i)
-	}
-	tc.transactf("no", `setmetadata inbox (/private/toomany "test")`)
-	tc.xcode(imapclient.CodeMetadataTooMany{})
-}

imapserver/move_test.go

@@ -7,31 +7,23 @@ import (
 )
 
 func TestMove(t *testing.T) {
-	testMove(t, false)
-}
-
-func TestMoveUIDOnly(t *testing.T) {
-	testMove(t, true)
-}
-
-func testMove(t *testing.T, uidonly bool) {
 	defer mockUIDValidity()()
-	tc := start(t, uidonly)
+	tc := start(t)
 	defer tc.close()
 
-	tc2 := startNoSwitchboard(t, uidonly)
+	tc2 := startNoSwitchboard(t)
-	defer tc2.closeNoWait()
+	defer tc2.close()
 
-	tc3 := startNoSwitchboard(t, uidonly)
+	tc3 := startNoSwitchboard(t)
-	defer tc3.closeNoWait()
+	defer tc3.close()
 
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
 	tc.client.Select("inbox")
 
-	tc2.login("mjl@mox.example", password0)
+	tc2.client.Login("mjl@mox.example", password0)
 	tc2.client.Select("Trash")
 
-	tc3.login("mjl@mox.example", password0)
+	tc3.client.Login("mjl@mox.example", password0)
 	tc3.client.Select("inbox")
 
 	tc.transactf("bad", "move")          // Missing params.
@@ -39,79 +31,62 @@ func testMove(t *testing.T, uidonly bool) {
 	tc.transactf("bad", "move 1 inbox ") // Leftover.
 
 	// Seqs 1,2 and UIDs 3,4.
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
-	tc.client.UIDStoreFlagsSet("1:2", true, `\Deleted`)
+	tc.client.StoreFlagsSet("1:2", true, `\Deleted`)
 	tc.client.Expunge()
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
 
-	if uidonly {
+	tc.client.Unselect()
-		tc.transactf("ok", "uid move 1:* Trash")
+	tc.client.Examine("inbox")
-	} else {
+	tc.transactf("no", "move 1 Trash") // Opened readonly.
-		tc.client.Unselect()
+	tc.client.Unselect()
-		tc.client.Examine("inbox")
+	tc.client.Select("inbox")
-		tc.transactf("no", "move 1 Trash") // Opened readonly.
-		tc.client.Unselect()
-		tc.client.Select("inbox")
 
-		tc.transactf("no", "move 1 nonexistent")
+	tc.transactf("no", "move 1 nonexistent")
-		tc.xcodeWord("TRYCREATE")
+	tc.xcode("TRYCREATE")
 
-		tc.transactf("no", "move 1 expungebox")
+	tc.transactf("no", "move 1 inbox") // Cannot move to same mailbox.
-		tc.xcodeWord("TRYCREATE")
 
-		tc.transactf("no", "move 1 inbox") // Cannot move to same mailbox.
+	tc2.transactf("ok", "noop") // Drain.
+	tc3.transactf("ok", "noop") // Drain.
 
-		tc2.transactf("ok", "noop") // Drain.
+	tc.transactf("ok", "move 1:* Trash")
-		tc3.transactf("ok", "noop") // Drain.
+	ptr := func(v uint32) *uint32 { return &v }
-
+	tc.xuntagged(
-		tc.transactf("ok", "move 1:* Trash")
+		imapclient.UntaggedResult{Status: "OK", RespText: imapclient.RespText{Code: "COPYUID", CodeArg: imapclient.CodeCopyUID{DestUIDValidity: 1, From: []imapclient.NumRange{{First: 3, Last: ptr(4)}}, To: []imapclient.NumRange{{First: 1, Last: ptr(2)}}}, More: "moved"}},
-		tc.xuntagged(
+		imapclient.UntaggedExpunge(1),
-			imapclient.UntaggedResult{Status: "OK", Code: imapclient.CodeCopyUID{DestUIDValidity: 1, From: []imapclient.NumRange{{First: 3, Last: uint32ptr(4)}}, To: []imapclient.NumRange{{First: 1, Last: uint32ptr(2)}}}, Text: "moved"},
+		imapclient.UntaggedExpunge(1),
-			imapclient.UntaggedExpunge(1),
+	)
-			imapclient.UntaggedExpunge(1),
+	tc2.transactf("ok", "noop")
-		)
+	tc2.xuntagged(
-		tc2.transactf("ok", "noop")
+		imapclient.UntaggedExists(2),
-		tc2.xuntagged(
+		imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil)}},
-			imapclient.UntaggedExists(2),
+		imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), imapclient.FetchFlags(nil)}},
-			tc.untaggedFetch(1, 1, imapclient.FetchFlags(nil)),
+	)
-			tc.untaggedFetch(2, 2, imapclient.FetchFlags(nil)),
+	tc3.transactf("ok", "noop")
-		)
+	tc3.xuntagged(imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1))
-		tc3.transactf("ok", "noop")
-		tc3.xuntagged(imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1))
-	}
 
 	// UIDs 5,6
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
 	tc2.transactf("ok", "noop") // Drain.
 	tc3.transactf("ok", "noop") // Drain.
 
 	tc.transactf("no", "uid move 1:4 Trash") // No match.
 	tc.transactf("ok", "uid move 6:5 Trash")
-	if uidonly {
+	tc.xuntagged(
-		tc.xuntagged(
+		imapclient.UntaggedResult{Status: "OK", RespText: imapclient.RespText{Code: "COPYUID", CodeArg: imapclient.CodeCopyUID{DestUIDValidity: 1, From: []imapclient.NumRange{{First: 5, Last: ptr(6)}}, To: []imapclient.NumRange{{First: 3, Last: ptr(4)}}}, More: "moved"}},
-			imapclient.UntaggedResult{Status: "OK", Code: imapclient.CodeCopyUID{DestUIDValidity: 1, From: []imapclient.NumRange{{First: 5, Last: uint32ptr(6)}}, To: []imapclient.NumRange{{First: 3, Last: uint32ptr(4)}}}, Text: "moved"},
+		imapclient.UntaggedExpunge(1),
-			imapclient.UntaggedVanished{UIDs: xparseNumSet("5:6")},
+		imapclient.UntaggedExpunge(1),
-		)
+	)
-	} else {
-		tc.xuntagged(
-			imapclient.UntaggedResult{Status: "OK", Code: imapclient.CodeCopyUID{DestUIDValidity: 1, From: []imapclient.NumRange{{First: 5, Last: uint32ptr(6)}}, To: []imapclient.NumRange{{First: 3, Last: uint32ptr(4)}}}, Text: "moved"},
-			imapclient.UntaggedExpunge(1),
-			imapclient.UntaggedExpunge(1),
-		)
-	}
 	tc2.transactf("ok", "noop")
 	tc2.xuntagged(
 		imapclient.UntaggedExists(4),
-		tc2.untaggedFetch(3, 3, imapclient.FetchFlags(nil)),
+		imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(3), imapclient.FetchFlags(nil)}},
-		tc2.untaggedFetch(4, 4, imapclient.FetchFlags(nil)),
+		imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), imapclient.FetchFlags(nil)}},
 	)
 	tc3.transactf("ok", "noop")
-	if uidonly {
+	tc3.xuntagged(imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1))
-		tc3.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("5:6")})
-	} else {
-		tc3.xuntagged(imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1))
-	}
 }

imapserver/notify.go

@@ -1,329 +0,0 @@
-package imapserver
-
-import (
-	"fmt"
-	"slices"
-	"strings"
-
-	"github.com/mjl-/bstore"
-
-	"github.com/mjl-/mox/store"
-)
-
-// Max number of pending changes for selected-delayed mailbox before we write a
-// NOTIFICATIONOVERFLOW message, flush changes and stop gathering more changes.
-// Changed during tests.
-var selectedDelayedChangesMax = 1000
-
-// notify represents a configuration as passed to the notify command.
-type notify struct {
-	// "NOTIFY NONE" results in an empty list, matching no events.
-	EventGroups []eventGroup
-
-	// Changes for the selected mailbox in case of SELECTED-DELAYED, when we don't send
-	// events asynchrously. These must still be processed later on for their
-	// ChangeRemoveUIDs, to erase expunged message files. At the end of a command (e.g.
-	// NOOP) or immediately upon IDLE we will send untagged responses for these
-	// changes. If the connection breaks, we still process the ChangeRemoveUIDs.
-	Delayed []store.Change
-}
-
-// match checks if an event for a mailbox id/name (optional depending on type)
-// should be turned into a notification to the client.
-func (n notify) match(c *conn, xtxfn func() *bstore.Tx, mailboxID int64, mailbox string, kind eventKind) (mailboxSpecifier, notifyEvent, bool) {
-	// We look through the event groups, and won't stop looking until we've found a
-	// confirmation the event should be notified. ../rfc/5465:756
-
-	// Non-message-related events are only matched by non-"selected" mailbox
-	// specifiers. ../rfc/5465:268
-	// If you read the mailboxes matching paragraph in isolation, you would think only
-	// "SELECTED" and "SELECTED-DELAYED" can match events for the selected mailbox. But
-	// a few other places hint that that only applies to message events, not to mailbox
-	// events, such as subscriptions and mailbox metadata changes. With a strict
-	// interpretation, clients couldn't request notifications for such events for the
-	// selection mailbox. ../rfc/5465:752
-
-	for _, eg := range n.EventGroups {
-		switch eg.MailboxSpecifier.Kind {
-		case mbspecSelected, mbspecSelectedDelayed: // ../rfc/5465:800
-			if mailboxID != c.mailboxID || !slices.Contains(messageEventKinds, kind) {
-				continue
-			}
-			for _, ev := range eg.Events {
-				if eventKind(ev.Kind) == kind {
-					return eg.MailboxSpecifier, ev, true
-				}
-			}
-			// We can only have a single selected for notify, so no point in continuing the search.
-			return mailboxSpecifier{}, notifyEvent{}, false
-
-		default:
-			// The selected mailbox can only match for non-message events for specifiers other
-			// than "selected"/"selected-delayed".
-			if c.mailboxID == mailboxID && slices.Contains(messageEventKinds, kind) {
-				continue
-			}
-		}
-
-		var match bool
-	Match:
-		switch eg.MailboxSpecifier.Kind {
-		case mbspecPersonal: // ../rfc/5465:817
-			match = true
-
-		case mbspecInboxes: // ../rfc/5465:822
-			if mailbox == "Inbox" || strings.HasPrefix(mailbox, "Inbox/") {
-				match = true
-				break Match
-			}
-
-			if mailbox == "" {
-				break Match
-			}
-
-			// Include mailboxes we may deliver to based on destinations, or based on rulesets,
-			// not including deliveries for mailing lists.
-			conf, _ := c.account.Conf()
-			for _, dest := range conf.Destinations {
-				if dest.Mailbox == mailbox {
-					match = true
-					break Match
-				}
-
-				for _, rs := range dest.Rulesets {
-					if rs.ListAllowDomain == "" && rs.Mailbox == mailbox {
-						match = true
-						break Match
-					}
-				}
-			}
-
-		case mbspecSubscribed: // ../rfc/5465:831
-			sub := store.Subscription{Name: mailbox}
-			err := xtxfn().Get(&sub)
-			if err != bstore.ErrAbsent {
-				xcheckf(err, "lookup subscription")
-			}
-			match = err == nil
-
-		case mbspecSubtree: // ../rfc/5465:847
-			for _, name := range eg.MailboxSpecifier.Mailboxes {
-				if mailbox == name || strings.HasPrefix(mailbox, name+"/") {
-					match = true
-					break
-				}
-			}
-
-		case mbspecSubtreeOne: // ../rfc/7377:274
-			ntoken := len(strings.Split(mailbox, "/"))
-			for _, name := range eg.MailboxSpecifier.Mailboxes {
-				if mailbox == name || (strings.HasPrefix(mailbox, name+"/") && len(strings.Split(name, "/"))+1 == ntoken) {
-					match = true
-					break
-				}
-			}
-
-		case mbspecMailboxes: // ../rfc/5465:853
-			match = slices.Contains(eg.MailboxSpecifier.Mailboxes, mailbox)
-
-		default:
-			panic("missing case for " + string(eg.MailboxSpecifier.Kind))
-		}
-
-		if !match {
-			continue
-		}
-
-		// NONE is the signal we shouldn't return events for this mailbox. ../rfc/5465:455
-		if len(eg.Events) == 0 {
-			break
-		}
-
-		// If event kind matches, we will be notifying about this change. If not, we'll
-		// look again at next mailbox specifiers.
-		for _, ev := range eg.Events {
-			if eventKind(ev.Kind) == kind {
-				return eg.MailboxSpecifier, ev, true
-			}
-		}
-	}
-	return mailboxSpecifier{}, notifyEvent{}, false
-}
-
-// Notify enables continuous notifications from the server to the client, without
-// the client issuing an IDLE command. The mailboxes and events to notify about are
-// specified in the account. When notify is enabled, instead of being blocked
-// waiting for a command from the client, we also wait for events from the account,
-// and send events about it.
-//
-// State: Authenticated and selected.
-func (c *conn) cmdNotify(tag, cmd string, p *parser) {
-	// Command: ../rfc/5465:203
-	// Request syntax: ../rfc/5465:923
-
-	p.xspace()
-
-	// NONE indicates client doesn't want any events, also not the "normal" events
-	// without notify. ../rfc/5465:234
-	// ../rfc/5465:930
-	if p.take("NONE") {
-		p.xempty()
-
-		// If we have delayed changes for the selected mailbox, we are no longer going to
-		// notify about them. The client can't know anymore whether messages still exist,
-		// and trying to read them can cause errors if the messages have been expunged and
-		// erased.
-		var changes []store.Change
-		if c.notify != nil {
-			changes = c.notify.Delayed
-		}
-		c.notify = &notify{}
-		c.flushChanges(changes)
-
-		c.ok(tag, cmd)
-		return
-	}
-
-	var n notify
-	var status bool
-
-	// ../rfc/5465:926
-	p.xtake("SET")
-	p.xspace()
-	if p.take("STATUS") {
-		status = true
-		p.xspace()
-	}
-	for {
-		eg := p.xeventGroup()
-		n.EventGroups = append(n.EventGroups, eg)
-		if !p.space() {
-			break
-		}
-	}
-	p.xempty()
-
-	for _, eg := range n.EventGroups {
-		var hasNew, hasExpunge, hasFlag, hasAnnotation bool
-		for _, ev := range eg.Events {
-			switch eventKind(ev.Kind) {
-			case eventMessageNew:
-				hasNew = true
-			case eventMessageExpunge:
-				hasExpunge = true
-			case eventFlagChange:
-				hasFlag = true
-			case eventMailboxName, eventSubscriptionChange, eventMailboxMetadataChange, eventServerMetadataChange:
-				// Nothing special.
-			default: // Including eventAnnotationChange.
-				hasAnnotation = true // Ineffective, we don't implement message annotations yet.
-				// Result must be NO instead of BAD, and we must include BADEVENT and the events we
-				// support. ../rfc/5465:343
-				// ../rfc/5465:1033
-				xusercodeErrorf("BADEVENT (MessageNew MessageExpunge FlagChange MailboxName SubscriptionChange MailboxMetadataChange ServerMetadataChange)", "unimplemented event %s", ev.Kind)
-			}
-		}
-		if hasNew != hasExpunge {
-			// ../rfc/5465:443 ../rfc/5465:987
-			xsyntaxErrorf("MessageNew and MessageExpunge must be specified together")
-		}
-		if (hasFlag || hasAnnotation) && !hasNew {
-			// ../rfc/5465:439
-			xsyntaxErrorf("FlagChange and/or AnnotationChange requires MessageNew and MessageExpunge")
-		}
-	}
-
-	for _, eg := range n.EventGroups {
-		for i, name := range eg.MailboxSpecifier.Mailboxes {
-			eg.MailboxSpecifier.Mailboxes[i] = xcheckmailboxname(name, true)
-		}
-	}
-
-	// Only one selected/selected-delay mailbox filter is allowed. ../rfc/5465:779
-	// Only message events are allowed for selected/selected-delayed. ../rfc/5465:796
-	var haveSelected bool
-	for _, eg := range n.EventGroups {
-		switch eg.MailboxSpecifier.Kind {
-		case mbspecSelected, mbspecSelectedDelayed:
-			if haveSelected {
-				xsyntaxErrorf("cannot have multiple selected/selected-delayed mailbox filters")
-			}
-			haveSelected = true
-
-			// Only events from message-event are allowed with selected mailbox specifiers.
-			// ../rfc/5465:977
-			for _, ev := range eg.Events {
-				if !slices.Contains(messageEventKinds, eventKind(ev.Kind)) {
-					xsyntaxErrorf("selected/selected-delayed is only allowed with message events, not %s", ev.Kind)
-				}
-			}
-		}
-	}
-
-	// We must apply any changes for delayed select. ../rfc/5465:248
-	if c.notify != nil {
-		delayed := c.notify.Delayed
-		c.notify.Delayed = nil
-		c.xapplyChangesNotify(delayed, true)
-	}
-
-	if status {
-		var statuses []string
-
-		// Flush new pending changes before we read the current state from the database.
-		// Don't allow any concurrent changes for a consistent snapshot.
-		c.account.WithRLock(func() {
-			select {
-			case <-c.comm.Pending:
-				overflow, changes := c.comm.Get()
-				c.xapplyChanges(overflow, changes, true)
-			default:
-			}
-
-			c.xdbread(func(tx *bstore.Tx) {
-				// Send STATUS responses for all matching mailboxes. ../rfc/5465:271
-				q := bstore.QueryTx[store.Mailbox](tx)
-				q.FilterEqual("Expunged", false)
-				q.SortAsc("Name")
-				for mb, err := range q.All() {
-					xcheckf(err, "list mailboxes for status")
-
-					if mb.ID == c.mailboxID {
-						continue
-					}
-					_, _, ok := n.match(c, func() *bstore.Tx { return tx }, mb.ID, mb.Name, eventMessageNew)
-					if !ok {
-						continue
-					}
-
-					list := listspace{
-						bare("MESSAGES"), number(mb.MessageCountIMAP()),
-						bare("UIDNEXT"), number(mb.UIDNext),
-						bare("UIDVALIDITY"), number(mb.UIDValidity),
-						// Unseen is not mentioned for STATUS, but clients are able to parse it due to
-						// FlagChange, and it will be useful to have.
-						bare("UNSEEN"), number(mb.MailboxCounts.Unseen),
-					}
-					if c.enabled[capCondstore] || c.enabled[capQresync] {
-						list = append(list, bare("HIGHESTMODSEQ"), number(mb.ModSeq))
-					}
-
-					status := fmt.Sprintf("* STATUS %s %s", mailboxt(mb.Name).pack(c), list.pack(c))
-					statuses = append(statuses, status)
-				}
-			})
-		})
-
-		// Write outside of db transaction and lock.
-		for _, s := range statuses {
-			c.xbwritelinef("%s", s)
-		}
-	}
-
-	// We replace the previous notify config. ../rfc/5465:245
-	c.notify = &n
-
-	// Writing OK will flush any other pending changes for the account according to the
-	// new filters.
-	c.ok(tag, cmd)
-}

imapserver/notify_test.go

@@ -1,516 +0,0 @@
-package imapserver
-
-import (
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/mjl-/mox/imapclient"
-	"github.com/mjl-/mox/store"
-)
-
-func TestNotify(t *testing.T) {
-	testNotify(t, false)
-}
-
-func TestNotifyUIDOnly(t *testing.T) {
-	testNotify(t, true)
-}
-
-func testNotify(t *testing.T, uidonly bool) {
-	defer mockUIDValidity()()
-	tc := start(t, uidonly)
-	defer tc.close()
-	tc.login("mjl@mox.example", password0)
-	tc.client.Select("inbox")
-
-	// Check for some invalid syntax.
-	tc.transactf("bad", "Notify")
-	tc.transactf("bad", "Notify bogus")
-	tc.transactf("bad", "Notify None ") // Trailing space.
-	tc.transactf("bad", "Notify Set")
-	tc.transactf("bad", "Notify Set ")
-	tc.transactf("bad", "Notify Set Status")
-	tc.transactf("bad", "Notify Set Status ()") // Empty list.
-	tc.transactf("bad", "Notify Set Status (UnknownSpecifier (messageNew))")
-	tc.transactf("bad", "Notify Set Status (Personal messageNew)")    // Missing list around events.
-	tc.transactf("bad", "Notify Set Status (Personal (messageNew) )") // Trailing space.
-	tc.transactf("bad", "Notify Set Status (Personal (messageNew)) ") // Trailing space.
-
-	tc.transactf("bad", "Notify Set Status (Selected (mailboxName))")                                  // MailboxName not allowed on Selected.
-	tc.transactf("bad", "Notify Set Status (Selected (messageNew))")                                   // MessageNew must come with MessageExpunge.
-	tc.transactf("bad", "Notify Set Status (Selected (flagChange))")                                   // flagChange must come with MessageNew and MessageExpunge.
-	tc.transactf("bad", "Notify Set Status (Selected (mailboxName)) (Selected-Delayed (mailboxName))") // Duplicate selected.
-	tc.transactf("no", "Notify Set Status (Selected (annotationChange))")                              // We don't implement annotation change.
-	tc.xcode(imapclient.CodeBadEvent{"MessageNew", "MessageExpunge", "FlagChange", "MailboxName", "SubscriptionChange", "MailboxMetadataChange", "ServerMetadataChange"})
-	tc.transactf("no", "Notify Set Status (Personal (unknownEvent))")
-	tc.xcode(imapclient.CodeBadEvent{"MessageNew", "MessageExpunge", "FlagChange", "MailboxName", "SubscriptionChange", "MailboxMetadataChange", "ServerMetadataChange"})
-
-	tc2 := startNoSwitchboard(t, uidonly)
-	defer tc2.closeNoWait()
-	tc2.login("mjl@mox.example", password0)
-	tc2.client.Select("inbox")
-
-	var modseq uint32 = 4
-
-	// Check that we don't get pending changes when we set "notify none". We first make
-	// changes that we drain with noop. Then add new pending changes and execute
-	// "notify none". Server should still process changes to the message sequence
-	// numbers of the selected mailbox.
-	tc2.client.Append("inbox", makeAppend(searchMsg)) // Results in exists and fetch.
-	modseq++
-	tc2.client.Append("Junk", makeAppend(searchMsg)) // Not selected, not mentioned.
-	modseq++
-	tc.transactf("ok", "noop")
-	tc.xuntagged(
-		imapclient.UntaggedExists(1),
-		tc.untaggedFetch(1, 1, imapclient.FetchFlags(nil)),
-	)
-	tc2.client.UIDStoreFlagsAdd("1:*", true, `\Deleted`)
-	modseq++
-	tc2.client.Expunge()
-	modseq++
-	tc.transactf("ok", "Notify None")
-	tc.xuntagged() // No untagged responses for delete/expunge.
-
-	// Enable notify, will first result in a the pending changes, then status.
-	tc.transactf("ok", "Notify Set Status (Selected (messageNew (Uid Modseq Bodystructure Preview) messageExpunge flagChange)) (personal (messageNew messageExpunge flagChange mailboxName subscriptionChange mailboxMetadataChange serverMetadataChange))")
-	tc.xuntagged(
-		imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeHighestModSeq(modseq), Text: "after condstore-enabling command"},
-		// note: no status for Inbox since it is selected.
-		imapclient.UntaggedStatus{Mailbox: "Drafts", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 0, imapclient.StatusUIDNext: 1, imapclient.StatusUIDValidity: 1, imapclient.StatusUnseen: 0, imapclient.StatusHighestModSeq: 2}},
-		imapclient.UntaggedStatus{Mailbox: "Sent", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 0, imapclient.StatusUIDNext: 1, imapclient.StatusUIDValidity: 1, imapclient.StatusUnseen: 0, imapclient.StatusHighestModSeq: 2}},
-		imapclient.UntaggedStatus{Mailbox: "Archive", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 0, imapclient.StatusUIDNext: 1, imapclient.StatusUIDValidity: 1, imapclient.StatusUnseen: 0, imapclient.StatusHighestModSeq: 2}},
-		imapclient.UntaggedStatus{Mailbox: "Trash", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 0, imapclient.StatusUIDNext: 1, imapclient.StatusUIDValidity: 1, imapclient.StatusUnseen: 0, imapclient.StatusHighestModSeq: 2}},
-		imapclient.UntaggedStatus{Mailbox: "Junk", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 1, imapclient.StatusUIDNext: 2, imapclient.StatusUIDValidity: 1, imapclient.StatusUnseen: 1, imapclient.StatusHighestModSeq: int64(modseq - 2)}},
-	)
-
-	// Selecting the mailbox again results in a refresh of the message sequence
-	// numbers, with the deleted message gone (it wasn't acknowledged yet due to
-	// "notify none").
-	tc.client.Select("inbox")
-
-	// Add message, should result in EXISTS and FETCH with the configured attributes.
-	tc2.client.Append("inbox", makeAppend(searchMsg))
-	modseq++
-	tc.readuntagged(
-		imapclient.UntaggedExists(1),
-		tc.untaggedFetchUID(1, 2,
-			imapclient.FetchBodystructure{
-				RespAttr: "BODYSTRUCTURE",
-				Body: imapclient.BodyTypeMpart{
-					Bodies: []any{
-						imapclient.BodyTypeText{
-							MediaType:    "TEXT",
-							MediaSubtype: "PLAIN",
-							BodyFields: imapclient.BodyFields{
-								Params: [][2]string{[...]string{"CHARSET", "utf-8"}},
-								Octets: 21,
-							},
-							Lines: 1,
-							Ext: &imapclient.BodyExtension1Part{
-								Disposition:       ptr((*string)(nil)),
-								DispositionParams: ptr([][2]string(nil)),
-								Language:          ptr([]string(nil)),
-								Location:          ptr((*string)(nil)),
-							},
-						},
-						imapclient.BodyTypeText{
-							MediaType:    "TEXT",
-							MediaSubtype: "HTML",
-							BodyFields: imapclient.BodyFields{
-								Params: [][2]string{[...]string{"CHARSET", "utf-8"}},
-								Octets: 15,
-							},
-							Lines: 1,
-							Ext: &imapclient.BodyExtension1Part{
-								Disposition:       ptr((*string)(nil)),
-								DispositionParams: ptr([][2]string(nil)),
-								Language:          ptr([]string(nil)),
-								Location:          ptr((*string)(nil)),
-							},
-						},
-					},
-					MediaSubtype: "ALTERNATIVE",
-					Ext: &imapclient.BodyExtensionMpart{
-						Params:            [][2]string{{"BOUNDARY", "x"}},
-						Disposition:       ptr((*string)(nil)), // Present but nil.
-						DispositionParams: ptr([][2]string(nil)),
-						Language:          ptr([]string(nil)),
-						Location:          ptr((*string)(nil)),
-					},
-				},
-			},
-			imapclient.FetchPreview{Preview: ptr("this is plain text.")},
-			imapclient.FetchModSeq(modseq),
-		),
-	)
-
-	// Change flags.
-	tc2.client.UIDStoreFlagsAdd("1:*", true, `\Deleted`)
-	modseq++
-	tc.readuntagged(tc.untaggedFetch(1, 2, imapclient.FetchFlags{`\Deleted`}, imapclient.FetchModSeq(modseq)))
-
-	// Remove message.
-	tc2.client.Expunge()
-	modseq++
-	if uidonly {
-		tc.readuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("2")})
-	} else {
-		tc.readuntagged(imapclient.UntaggedExpunge(1))
-	}
-
-	// MailboxMetadataChange for mailbox annotation.
-	tc2.transactf("ok", `setmetadata Archive (/private/comment "test")`)
-	modseq++
-	tc.readuntagged(
-		imapclient.UntaggedMetadataKeys{Mailbox: "Archive", Keys: []string{"/private/comment"}},
-	)
-
-	// MailboxMetadataChange also for the selected Inbox.
-	tc2.transactf("ok", `setmetadata Inbox (/private/comment "test")`)
-	modseq++
-	tc.readuntagged(
-		imapclient.UntaggedMetadataKeys{Mailbox: "Inbox", Keys: []string{"/private/comment"}},
-	)
-
-	// ServerMetadataChange for server annotation.
-	tc2.transactf("ok", `setmetadata "" (/private/vendor/other/x "test")`)
-	modseq++
-	tc.readuntagged(
-		imapclient.UntaggedMetadataKeys{Mailbox: "", Keys: []string{"/private/vendor/other/x"}},
-	)
-
-	// SubscriptionChange for new subscription.
-	tc2.client.Subscribe("doesnotexist")
-	tc.readuntagged(
-		imapclient.UntaggedList{Mailbox: "doesnotexist", Separator: '/', Flags: []string{`\Subscribed`, `\NonExistent`}},
-	)
-
-	// SubscriptionChange for removed subscription.
-	tc2.client.Unsubscribe("doesnotexist")
-	tc.readuntagged(
-		imapclient.UntaggedList{Mailbox: "doesnotexist", Separator: '/', Flags: []string{`\NonExistent`}},
-	)
-
-	// SubscriptionChange for selected mailbox.
-	tc2.client.Unsubscribe("Inbox")
-	tc2.client.Subscribe("Inbox")
-	tc.readuntagged(
-		imapclient.UntaggedList{Mailbox: "Inbox", Separator: '/'},
-		imapclient.UntaggedList{Mailbox: "Inbox", Separator: '/', Flags: []string{`\Subscribed`}},
-	)
-
-	// MailboxName for creating mailbox.
-	tc2.client.Create("newbox", nil)
-	modseq++
-	tc.readuntagged(
-		imapclient.UntaggedList{Mailbox: "newbox", Separator: '/', Flags: []string{`\Subscribed`}},
-	)
-
-	// MailboxName for renaming mailbox.
-	tc2.client.Rename("newbox", "oldbox")
-	modseq++
-	tc.readuntagged(
-		imapclient.UntaggedList{Mailbox: "oldbox", Separator: '/', OldName: "newbox"},
-	)
-
-	// MailboxName for deleting mailbox.
-	tc2.client.Delete("oldbox")
-	modseq++
-	tc.readuntagged(
-		imapclient.UntaggedList{Mailbox: "oldbox", Separator: '/', Flags: []string{`\NonExistent`}},
-	)
-
-	// Add message again to check for modseq. First set notify again with fewer fetch
-	// attributes for simpler checking.
-	tc.transactf("ok", "Notify Set (personal (messageNew messageExpunge flagChange mailboxName subscriptionChange mailboxMetadataChange serverMetadataChange)) (Selected (messageNew (Uid Modseq) messageExpunge flagChange))")
-	tc2.client.Append("inbox", makeAppend(searchMsg))
-	modseq++
-	tc.readuntagged(
-		imapclient.UntaggedExists(1),
-		tc.untaggedFetchUID(1, 3, imapclient.FetchModSeq(modseq)),
-	)
-
-	// Next round of events must be ignored. We shouldn't get anything until we add a
-	// message to "testbox".
-	tc.transactf("ok", "Notify Set (Selected None) (mailboxes testbox (messageNew messageExpunge)) (personal None)")
-	tc2.client.Append("inbox", makeAppend(searchMsg)) // MessageNew
-	modseq++
-	tc2.client.UIDStoreFlagsAdd("1:*", true, `\Deleted`) // FlagChange
-	modseq++
-	tc2.client.Expunge() // MessageExpunge
-	modseq++
-	tc2.transactf("ok", `setmetadata Archive (/private/comment "test2")`) // MailboxMetadataChange
-	modseq++
-	tc2.transactf("ok", `setmetadata "" (/private/vendor/other/x "test2")`) // ServerMetadataChange
-	modseq++
-	tc2.client.Subscribe("doesnotexist2")   // SubscriptionChange
-	tc2.client.Unsubscribe("doesnotexist2") // SubscriptionChange
-	tc2.client.Create("newbox2", nil)       // MailboxName
-	modseq++
-	tc2.client.Rename("newbox2", "oldbox2") // MailboxName
-	modseq++
-	tc2.client.Delete("oldbox2") // MailboxName
-	modseq++
-	// Now trigger receiving a notification.
-	tc2.client.Create("testbox", nil) // MailboxName
-	modseq++
-	tc2.client.Append("testbox", makeAppend(searchMsg)) // MessageNew
-	modseq++
-	tc.readuntagged(
-		imapclient.UntaggedStatus{Mailbox: "testbox", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 1, imapclient.StatusUIDNext: 2, imapclient.StatusUnseen: 1, imapclient.StatusHighestModSeq: int64(modseq)}},
-	)
-
-	// Test filtering per mailbox specifier. We create two mailboxes.
-	tc.client.Create("inbox/a/b", nil)
-	modseq++
-	tc.client.Create("other/a/b", nil)
-	modseq++
-	tc.client.Unsubscribe("other/a/b")
-
-	// Inboxes
-	tc3 := startNoSwitchboard(t, uidonly)
-	defer tc3.closeNoWait()
-	tc3.login("mjl@mox.example", password0)
-	tc3.transactf("ok", "Notify Set (Inboxes (messageNew messageExpunge))")
-
-	// Subscribed
-	tc4 := startNoSwitchboard(t, uidonly)
-	defer tc4.closeNoWait()
-	tc4.login("mjl@mox.example", password0)
-	tc4.transactf("ok", "Notify Set (Subscribed (messageNew messageExpunge))")
-
-	// Subtree
-	tc5 := startNoSwitchboard(t, uidonly)
-	defer tc5.closeNoWait()
-	tc5.login("mjl@mox.example", password0)
-	tc5.transactf("ok", "Notify Set (Subtree (Nonexistent inbox) (messageNew messageExpunge))")
-
-	// Subtree-One
-	tc6 := startNoSwitchboard(t, uidonly)
-	defer tc6.closeNoWait()
-	tc6.login("mjl@mox.example", password0)
-	tc6.transactf("ok", "Notify Set (Subtree-One (Nonexistent Inbox/a other) (messageNew messageExpunge))")
-
-	// We append to other/a/b first. It would normally come first in the notifications,
-	// but we check we only get the second event.
-	tc2.client.Append("other/a/b", makeAppend(searchMsg))
-	modseq++
-	tc2.client.Append("inbox/a/b", makeAppend(searchMsg))
-	modseq++
-
-	// No highestmodseq, these connections don't have CONDSTORE enabled.
-	tc3.readuntagged(
-		imapclient.UntaggedStatus{Mailbox: "Inbox/a/b", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 1, imapclient.StatusUIDNext: 2, imapclient.StatusUnseen: 1}},
-	)
-	tc4.readuntagged(
-		imapclient.UntaggedStatus{Mailbox: "Inbox/a/b", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 1, imapclient.StatusUIDNext: 2, imapclient.StatusUnseen: 1}},
-	)
-	tc5.readuntagged(
-		imapclient.UntaggedStatus{Mailbox: "Inbox/a/b", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 1, imapclient.StatusUIDNext: 2, imapclient.StatusUnseen: 1}},
-	)
-	tc6.readuntagged(
-		imapclient.UntaggedStatus{Mailbox: "Inbox/a/b", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 1, imapclient.StatusUIDNext: 2, imapclient.StatusUnseen: 1}},
-	)
-
-	// Test for STATUS events on non-selected mailbox for message events.
-	tc.transactf("ok", "notify set (personal (messageNew messageExpunge flagChange))")
-	tc.client.Unselect()
-	tc2.client.Create("statusbox", nil)
-	modseq++
-	tc2.client.Append("statusbox", makeAppend(searchMsg))
-	modseq++
-	tc.readuntagged(
-		imapclient.UntaggedStatus{Mailbox: "statusbox", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 1, imapclient.StatusUIDNext: 2, imapclient.StatusUnseen: 1, imapclient.StatusHighestModSeq: int64(modseq)}},
-	)
-
-	// With Selected-Delayed, we only get the events for the selected mailbox for
-	// explicit commands. We still get other events.
-	tc.transactf("ok", "notify set (selected-delayed (messageNew messageExpunge flagChange)) (personal (messageNew messageExpunge flagChange))")
-	tc.client.Select("statusbox")
-	tc2.client.Append("inbox", makeAppend(searchMsg))
-	modseq++
-	tc2.client.UIDStoreFlagsSet("*", true, `\Seen`)
-	modseq++
-	tc2.client.Append("statusbox", imapclient.Append{Flags: []string{"newflag"}, Size: int64(len(searchMsg)), Data: strings.NewReader(searchMsg)})
-	modseq++
-	tc2.client.Select("statusbox")
-
-	tc.readuntagged(
-		imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 1, imapclient.StatusUIDNext: 6, imapclient.StatusUnseen: 1, imapclient.StatusHighestModSeq: int64(modseq - 2)}},
-		imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusUIDValidity: 1, imapclient.StatusUnseen: 0, imapclient.StatusHighestModSeq: int64(modseq - 1)}},
-	)
-
-	tc.transactf("ok", "noop")
-	tc.xuntagged(
-		imapclient.UntaggedExists(2),
-		tc.untaggedFetch(2, 2, imapclient.FetchFlags{"newflag"}, imapclient.FetchModSeq(modseq)),
-		imapclient.UntaggedFlags{`\Seen`, `\Answered`, `\Flagged`, `\Deleted`, `\Draft`, `$Forwarded`, `$Junk`, `$NotJunk`, `$Phishing`, `$MDNSent`, `newflag`},
-	)
-
-	tc2.client.UIDStoreFlagsSet("2", true, `\Deleted`)
-	modseq++
-	tc2.client.Expunge()
-	modseq++
-	tc.transactf("ok", "noop")
-	if uidonly {
-		tc.xuntagged(
-			tc.untaggedFetch(2, 2, imapclient.FetchFlags{`\Deleted`}, imapclient.FetchModSeq(modseq-1)),
-			imapclient.UntaggedVanished{UIDs: xparseNumSet("2")},
-		)
-	} else {
-		tc.xuntagged(
-			tc.untaggedFetch(2, 2, imapclient.FetchFlags{`\Deleted`}, imapclient.FetchModSeq(modseq-1)),
-			imapclient.UntaggedExpunge(2),
-		)
-	}
-
-	// With Selected-Delayed, we should get events for selected mailboxes immediately when using IDLE.
-	tc2.client.UIDStoreFlagsSet("*", true, `\Answered`)
-	modseq++
-	tc2.client.Select("inbox")
-	tc2.client.UIDStoreFlagsClear("*", true, `\Seen`)
-	modseq++
-	tc2.client.Select("statusbox")
-
-	tc.readuntagged(
-		imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusUIDValidity: 1, imapclient.StatusUnseen: 1, imapclient.StatusHighestModSeq: int64(modseq)}},
-	)
-
-	tc.conn.SetReadDeadline(time.Now().Add(3 * time.Second))
-	tc.cmdf("", "idle")
-	tc.readprefixline("+ ")
-	tc.readuntagged(tc.untaggedFetch(1, 1, imapclient.FetchFlags{`\Answered`}, imapclient.FetchModSeq(modseq-1)))
-	tc.writelinef("done")
-	tc.response("ok")
-	tc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
-
-	// If any event matches, we normally return it. But NONE prevents looking further.
-	tc.client.Unselect()
-	tc.transactf("ok", "notify set (mailboxes statusbox NONE) (personal (mailboxName))")
-	tc2.client.UIDStoreFlagsSet("*", true, `\Answered`) // Matches NONE, ignored.
-	//modseq++
-	tc2.client.Create("eventbox", nil)
-	//modseq++
-	tc.readuntagged(
-		imapclient.UntaggedList{Mailbox: "eventbox", Separator: '/', Flags: []string{`\Subscribed`}},
-	)
-
-	// Check we can return message contents.
-	tc.transactf("ok", "notify set (selected (messageNew (body[header] body[text]) messageExpunge))")
-	tc.client.Select("statusbox")
-	tc2.client.Append("statusbox", makeAppend(searchMsg))
-	// modseq++
-	offset := strings.Index(searchMsg, "\r\n\r\n")
-	tc.readuntagged(
-		imapclient.UntaggedExists(2),
-		tc.untaggedFetch(2, 3,
-			imapclient.FetchBody{
-				RespAttr: "BODY[HEADER]",
-				Section:  "HEADER",
-				Body:     searchMsg[:offset+4],
-			},
-			imapclient.FetchBody{
-				RespAttr: "BODY[TEXT]",
-				Section:  "TEXT",
-				Body:     searchMsg[offset+4:],
-			},
-			imapclient.FetchFlags(nil),
-		),
-	)
-
-	// If we encounter an error during fetch, an untagged NO is returned.
-	// We ask for the 2nd part of a message, and we add a message with just 1 part.
-	tc.transactf("ok", "notify set (selected (messageNew (body[2]) messageExpunge))")
-	tc2.client.Append("statusbox", makeAppend(exampleMsg))
-	// modseq++
-	tc.readuntagged(
-		imapclient.UntaggedExists(3),
-		imapclient.UntaggedResult{Status: "NO", Text: "generating notify fetch response: requested part does not exist"},
-		tc.untaggedFetchUID(3, 4),
-	)
-
-	// When adding new tests, uncomment modseq++ lines above.
-}
-
-func TestNotifyOverflow(t *testing.T) {
-	testNotifyOverflow(t, false)
-}
-
-func TestNotifyOverflowUIDOnly(t *testing.T) {
-	testNotifyOverflow(t, true)
-}
-
-func testNotifyOverflow(t *testing.T, uidonly bool) {
-	orig := store.CommPendingChangesMax
-	store.CommPendingChangesMax = 3
-	defer func() {
-		store.CommPendingChangesMax = orig
-	}()
-
-	defer mockUIDValidity()()
-	tc := start(t, uidonly)
-	defer tc.close()
-	tc.login("mjl@mox.example", password0)
-	tc.client.Select("inbox")
-	tc.transactf("ok", "noop")
-
-	tc2 := startNoSwitchboard(t, uidonly)
-	defer tc2.closeNoWait()
-	tc2.login("mjl@mox.example", password0)
-	tc2.client.Select("inbox")
-
-	// Generates 4 changes, crossing max 3.
-	tc2.client.Append("inbox", makeAppend(searchMsg))
-	tc2.client.Append("inbox", makeAppend(searchMsg))
-
-	tc.transactf("ok", "noop")
-	tc.xuntagged(imapclient.UntaggedResult{Status: "OK", Code: imapclient.CodeWord("NOTIFICATIONOVERFLOW"), Text: "out of sync after too many pending changes"})
-
-	// Won't be getting any more notifications until we enable them again with NOTIFY.
-	tc2.client.Append("inbox", makeAppend(searchMsg))
-	tc.transactf("ok", "noop")
-	tc.xuntagged()
-
-	// Enable notify again. Without uidonly, we won't get a notification because the
-	// message isn't known in the session.
-	tc.transactf("ok", "notify set (selected (messageNew messageExpunge flagChange))")
-	tc2.client.UIDStoreFlagsAdd("1", true, `\Seen`)
-	if uidonly {
-		tc.readuntagged(tc.untaggedFetch(1, 1, imapclient.FetchFlags{`\Seen`}))
-	} else {
-		tc.transactf("ok", "noop")
-		tc.xuntagged()
-	}
-
-	// Reselect to get the message visible in the session.
-	tc.client.Select("inbox")
-	tc2.client.UIDStoreFlagsClear("1", true, `\Seen`)
-	tc.transactf("ok", "noop")
-	tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchFlags(nil)))
-
-	// Trigger overflow for changes for "selected-delayed".
-	store.CommPendingChangesMax = 10
-	delayedMax := selectedDelayedChangesMax
-	selectedDelayedChangesMax = 1
-	defer func() {
-		selectedDelayedChangesMax = delayedMax
-	}()
-	tc.transactf("ok", "notify set (selected-delayed (messageNew messageExpunge flagChange))")
-	tc2.client.UIDStoreFlagsAdd("1", true, `\Seen`)
-	tc2.client.UIDStoreFlagsClear("1", true, `\Seen`)
-	tc.transactf("ok", "noop")
-	tc.xuntagged(imapclient.UntaggedResult{Status: "OK", Code: imapclient.CodeWord("NOTIFICATIONOVERFLOW"), Text: "out of sync after too many pending changes for selected mailbox"})
-
-	// Again, no new notifications until we select and enable again.
-	tc2.client.UIDStoreFlagsAdd("1", true, `\Seen`)
-	tc.transactf("ok", "noop")
-	tc.xuntagged()
-
-	tc.client.Select("inbox")
-	tc.transactf("ok", "notify set (selected-delayed (messageNew messageExpunge flagChange))")
-	tc2.client.UIDStoreFlagsClear("1", true, `\Seen`)
-	tc.transactf("ok", "noop")
-	tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchFlags(nil)))
-}

imapserver/pack.go

@@ -9,7 +9,7 @@ import (
 
 type token interface {
 	pack(c *conn) string
-	xwriteTo(c *conn, xw io.Writer) // Writes to xw panic on error.
+	writeTo(c *conn, w io.Writer)
 }
 
 type bare string
@@ -18,8 +18,8 @@ func (t bare) pack(c *conn) string {
 	return string(t)
 }
 
-func (t bare) xwriteTo(c *conn, xw io.Writer) {
+func (t bare) writeTo(c *conn, w io.Writer) {
-	xw.Write([]byte(t.pack(c)))
+	w.Write([]byte(t.pack(c)))
 }
 
 type niltoken struct{}
@@ -30,15 +30,15 @@ func (t niltoken) pack(c *conn) string {
 	return "NIL"
 }
 
-func (t niltoken) xwriteTo(c *conn, xw io.Writer) {
+func (t niltoken) writeTo(c *conn, w io.Writer) {
-	xw.Write([]byte(t.pack(c)))
+	w.Write([]byte(t.pack(c)))
 }
 
-func nilOrString(s *string) token {
+func nilOrString(s string) token {
-	if s == nil {
+	if s == "" {
 		return nilt
 	}
-	return string0(*s)
+	return string0(s)
 }
 
 type string0 string
@@ -60,8 +60,8 @@ func (t string0) pack(c *conn) string {
 	return r
 }
 
-func (t string0) xwriteTo(c *conn, xw io.Writer) {
+func (t string0) writeTo(c *conn, w io.Writer) {
-	xw.Write([]byte(t.pack(c)))
+	w.Write([]byte(t.pack(c)))
 }
 
 type dquote string
@@ -78,8 +78,8 @@ func (t dquote) pack(c *conn) string {
 	return r
 }
 
-func (t dquote) xwriteTo(c *conn, xw io.Writer) {
+func (t dquote) writeTo(c *conn, w io.Writer) {
-	xw.Write([]byte(t.pack(c)))
+	w.Write([]byte(t.pack(c)))
 }
 
 type syncliteral string
@@ -88,16 +88,15 @@ func (t syncliteral) pack(c *conn) string {
 	return fmt.Sprintf("{%d}\r\n", len(t)) + string(t)
 }
 
-func (t syncliteral) xwriteTo(c *conn, xw io.Writer) {
+func (t syncliteral) writeTo(c *conn, w io.Writer) {
-	fmt.Fprintf(xw, "{%d}\r\n", len(t))
+	fmt.Fprintf(w, "{%d}\r\n", len(t))
-	xw.Write([]byte(t))
+	w.Write([]byte(t))
 }
 
 // data from reader with known size.
 type readerSizeSyncliteral struct {
 	r    io.Reader
 	size int64
-	lit8 bool
 }
 
 func (t readerSizeSyncliteral) pack(c *conn) string {
@@ -105,21 +104,13 @@ func (t readerSizeSyncliteral) pack(c *conn) string {
 	if err != nil {
 		panic(err)
 	}
-	var lit string
+	return fmt.Sprintf("{%d}\r\n", t.size) + string(buf)
-	if t.lit8 {
-		lit = "~"
-	}
-	return fmt.Sprintf("%s{%d}\r\n", lit, t.size) + string(buf)
 }
 
-func (t readerSizeSyncliteral) xwriteTo(c *conn, xw io.Writer) {
+func (t readerSizeSyncliteral) writeTo(c *conn, w io.Writer) {
-	var lit string
+	fmt.Fprintf(w, "{%d}\r\n", t.size)
-	if t.lit8 {
+	defer c.xtrace(mlog.LevelTracedata)()
-		lit = "~"
+	if _, err := io.Copy(w, io.LimitReader(t.r, t.size)); err != nil {
-	}
-	fmt.Fprintf(xw, "%s{%d}\r\n", lit, t.size)
-	defer c.xtracewrite(mlog.LevelTracedata)()
-	if _, err := io.Copy(xw, io.LimitReader(t.r, t.size)); err != nil {
 		panic(err)
 	}
 }
@@ -137,14 +128,17 @@ func (t readerSyncliteral) pack(c *conn) string {
 	return fmt.Sprintf("{%d}\r\n", len(buf)) + string(buf)
 }
 
-func (t readerSyncliteral) xwriteTo(c *conn, xw io.Writer) {
+func (t readerSyncliteral) writeTo(c *conn, w io.Writer) {
 	buf, err := io.ReadAll(t.r)
 	if err != nil {
 		panic(err)
 	}
-	fmt.Fprintf(xw, "{%d}\r\n", len(buf))
+	fmt.Fprintf(w, "{%d}\r\n", len(buf))
-	defer c.xtracewrite(mlog.LevelTracedata)()
+	defer c.xtrace(mlog.LevelTracedata)()
-	xw.Write(buf)
+	_, err = w.Write(buf)
+	if err != nil {
+		panic(err)
+	}
 }
 
 // list with tokens space-separated
@@ -162,38 +156,15 @@ func (t listspace) pack(c *conn) string {
 	return s
 }
 
-func (t listspace) xwriteTo(c *conn, xw io.Writer) {
+func (t listspace) writeTo(c *conn, w io.Writer) {
-	fmt.Fprint(xw, "(")
+	fmt.Fprint(w, "(")
 	for i, e := range t {
 		if i > 0 {
-			fmt.Fprint(xw, " ")
+			fmt.Fprint(w, " ")
 		}
-		e.xwriteTo(c, xw)
+		e.writeTo(c, w)
-	}
-	fmt.Fprint(xw, ")")
-}
-
-// concatenate tokens space-separated
-type concatspace []token
-
-func (t concatspace) pack(c *conn) string {
-	var s string
-	for i, e := range t {
-		if i > 0 {
-			s += " "
-		}
-		s += e.pack(c)
-	}
-	return s
-}
-
-func (t concatspace) xwriteTo(c *conn, xw io.Writer) {
-	for i, e := range t {
-		if i > 0 {
-			fmt.Fprint(xw, " ")
-		}
-		e.xwriteTo(c, xw)
 	}
+	fmt.Fprint(w, ")")
 }
 
 // Concatenated tokens, no spaces or list syntax.
@@ -207,9 +178,9 @@ func (t concat) pack(c *conn) string {
 	return s
 }
 
-func (t concat) xwriteTo(c *conn, xw io.Writer) {
+func (t concat) writeTo(c *conn, w io.Writer) {
 	for _, e := range t {
-		e.xwriteTo(c, xw)
+		e.writeTo(c, w)
 	}
 }
 
@@ -231,23 +202,8 @@ next:
 	return string(t)
 }
 
-func (t astring) xwriteTo(c *conn, xw io.Writer) {
+func (t astring) writeTo(c *conn, w io.Writer) {
-	xw.Write([]byte(t.pack(c)))
+	w.Write([]byte(t.pack(c)))
-}
-
-// mailbox with utf7 encoding if connection requires it, or utf8 otherwise.
-type mailboxt string
-
-func (t mailboxt) pack(c *conn) string {
-	s := string(t)
-	if !c.utf8strings() {
-		s = utf7encode(s)
-	}
-	return astring(s).pack(c)
-}
-
-func (t mailboxt) xwriteTo(c *conn, xw io.Writer) {
-	xw.Write([]byte(t.pack(c)))
 }
 
 type number uint32
@@ -256,6 +212,6 @@ func (t number) pack(c *conn) string {
 	return fmt.Sprintf("%d", t)
 }
 
-func (t number) xwriteTo(c *conn, xw io.Writer) {
+func (t number) writeTo(c *conn, w io.Writer) {
-	xw.Write([]byte(t.pack(c)))
+	w.Write([]byte(t.pack(c)))
 }

imapserver/parse.go

@@ -305,10 +305,10 @@ func (p *parser) xstring() (r string) {
 		p.xerrorf("missing closing dquote in string")
 	}
 	size, sync := p.xliteralSize(false, true)
-	buf := p.conn.xreadliteral(size, sync)
+	s := p.conn.xreadliteral(size, sync)
-	line := p.conn.xreadline(false)
+	line := p.conn.readline(false)
 	p.orig, p.upper, p.o = line, toUpper(line), 0
-	return string(buf)
+	return s
 }
 
 func (p *parser) xnil() {
@@ -575,13 +575,11 @@ func (p *parser) xsectionBinary() (r []uint32) {
 var fetchAttWords = []string{
 	"ENVELOPE", "FLAGS", "INTERNALDATE", "RFC822.SIZE", "BODYSTRUCTURE", "UID", "BODY.PEEK", "BODY", "BINARY.PEEK", "BINARY.SIZE", "BINARY",
 	"RFC822.HEADER", "RFC822.TEXT", "RFC822", // older IMAP
-	"MODSEQ",   // CONDSTORE extension.
+	"MODSEQ", // CONDSTORE extension.
-	"SAVEDATE", // SAVEDATE extension, ../rfc/8514:186
-	"PREVIEW",  // ../rfc/8970:345
 }
 
 // ../rfc/9051:6557 ../rfc/3501:4751 ../rfc/7162:2483
-func (p *parser) xfetchAtt() (r fetchAtt) {
+func (p *parser) xfetchAtt(isUID bool) (r fetchAtt) {
 	defer p.context("fetchAtt")()
 	f := p.xtakelist(fetchAttWords...)
 	r.peek = strings.HasSuffix(f, ".PEEK")
@@ -609,14 +607,12 @@ func (p *parser) xfetchAtt() (r fetchAtt) {
 		// The wording about when to respond with a MODSEQ attribute could be more clear. ../rfc/7162:923 ../rfc/7162:388
 		// MODSEQ attribute is a CONDSTORE-enabling parameter. ../rfc/7162:377
 		p.conn.xensureCondstore(nil)
-	case "PREVIEW":
-		r.previewLazy = p.take(" (LAZY)")
 	}
 	return
 }
 
 // ../rfc/9051:6553 ../rfc/3501:4748
-func (p *parser) xfetchAtts() []fetchAtt {
+func (p *parser) xfetchAtts(isUID bool) []fetchAtt {
 	defer p.context("fetchAtts")()
 
 	fields := func(l ...string) []fetchAtt {
@@ -640,13 +636,13 @@ func (p *parser) xfetchAtts() []fetchAtt {
 	}
 
 	if !p.hasPrefix("(") {
-		return []fetchAtt{p.xfetchAtt()}
+		return []fetchAtt{p.xfetchAtt(isUID)}
 	}
 
 	l := []fetchAtt{}
 	p.xtake("(")
 	for {
-		l = append(l, p.xfetchAtt())
+		l = append(l, p.xfetchAtt(isUID))
 		if !p.take(" ") {
 			break
 		}
@@ -796,7 +792,6 @@ var searchKeyWords = []string{
 	"BEFORE", "BODY",
 	"CC", "DELETED", "FLAGGED",
 	"FROM", "KEYWORD",
-	"OLDER", "YOUNGER", // WITHIN extension, ../rfc/5032:72
 	"NEW", "OLD", "ON", "RECENT", "SEEN",
 	"SINCE", "SUBJECT",
 	"TEXT", "TO",
@@ -808,8 +803,7 @@ var searchKeyWords = []string{
 	"SENTBEFORE", "SENTON",
 	"SENTSINCE", "SMALLER",
 	"UID", "UNDRAFT",
-	"MODSEQ",                                                    // CONDSTORE extension.
+	"MODSEQ", // CONDSTORE extension.
-	"SAVEDBEFORE", "SAVEDON", "SAVEDSINCE", "SAVEDATESUPPORTED", // SAVEDATE extension, ../rfc/8514:203
 }
 
 // ../rfc/9051:6923 ../rfc/3501:4957, MODSEQ ../rfc/7162:2492
@@ -933,19 +927,31 @@ func (p *parser) xsearchKey() *searchKey {
 		sk.clientModseq = &v
 		// MODSEQ is a CONDSTORE-enabling parameter. ../rfc/7162:377
 		p.conn.enabled[capCondstore] = true
-	case "SAVEDBEFORE", "SAVEDON", "SAVEDSINCE":
-		p.xspace()
-		sk.date = p.xdate() // ../rfc/8514:267
-	case "SAVEDATESUPPORTED":
-	case "OLDER", "YOUNGER":
-		p.xspace()
-		sk.number = int64(p.xnznumber())
 	default:
 		p.xerrorf("missing case for op %q", sk.op)
 	}
 	return sk
 }
 
+// hasModseq returns whether there is a modseq filter anywhere in the searchkey.
+func (sk searchKey) hasModseq() bool {
+	if sk.clientModseq != nil {
+		return true
+	}
+	for _, e := range sk.searchKeys {
+		if e.hasModseq() {
+			return true
+		}
+	}
+	if sk.searchKey != nil && sk.searchKey.hasModseq() {
+		return true
+	}
+	if sk.searchKey2 != nil && sk.searchKey2.hasModseq() {
+		return true
+	}
+	return false
+}
+
 // ../rfc/9051:6489 ../rfc/3501:4692
 func (p *parser) xdateDay() int {
 	d := p.xdigit()
@@ -968,195 +974,3 @@ func (p *parser) xdate() time.Time {
 	}
 	return time.Date(year, mon, day, 0, 0, 0, 0, time.UTC)
 }
-
-// Parse and validate a metadata key (entry name), returned as lower-case.
-//
-// ../rfc/5464:190
-func (p *parser) xmetadataKey() string {
-	// ../rfc/5464:772
-	s := p.xastring()
-
-	// ../rfc/5464:192
-	if strings.Contains(s, "//") {
-		p.xerrorf("entry name must not contain two slashes")
-	}
-	// We allow a single slash, so it can be used with option "(depth infinity)" to get
-	// all annotations.
-	if s != "/" && strings.HasSuffix(s, "/") {
-		p.xerrorf("entry name must not end with slash")
-	}
-	// ../rfc/5464:202
-	if strings.Contains(s, "*") || strings.Contains(s, "%") {
-		p.xerrorf("entry name must not contain * or %%")
-	}
-	for _, c := range s {
-		if c < ' ' || c >= 0x7f {
-			p.xerrorf("entry name must only contain non-control ascii characters")
-		}
-	}
-	return strings.ToLower(s)
-}
-
-// ../rfc/5464:776
-func (p *parser) xmetadataKeyValue() (key string, isString bool, value []byte) {
-	key = p.xmetadataKey()
-	p.xspace()
-
-	if p.hasPrefix("~{") {
-		size, sync := p.xliteralSize(true, true)
-		value = p.conn.xreadliteral(size, sync)
-		line := p.conn.xreadline(false)
-		p.orig, p.upper, p.o = line, toUpper(line), 0
-	} else if p.hasPrefix(`"`) {
-		value = []byte(p.xstring())
-		isString = true
-	} else if p.take("NIL") {
-		value = nil
-	} else {
-		p.xerrorf("expected metadata value")
-	}
-
-	return
-}
-
-type eventGroup struct {
-	MailboxSpecifier mailboxSpecifier
-	Events           []notifyEvent // NONE is represented by an empty list.
-}
-
-type mbspecKind string
-
-const (
-	mbspecSelected        mbspecKind = "SELECTED"
-	mbspecSelectedDelayed mbspecKind = "SELECTED-DELAYED" // Only for NOTIFY.
-	mbspecInboxes         mbspecKind = "INBOXES"
-	mbspecPersonal        mbspecKind = "PERSONAL"
-	mbspecSubscribed      mbspecKind = "SUBSCRIBED"
-	mbspecSubtreeOne      mbspecKind = "SUBTREE-ONE" // For ESEARCH, we allow it for NOTIFY too.
-	mbspecSubtree         mbspecKind = "SUBTREE"
-	mbspecMailboxes       mbspecKind = "MAILBOXES"
-)
-
-// Used by both the ESEARCH and NOTIFY commands.
-type mailboxSpecifier struct {
-	Kind      mbspecKind
-	Mailboxes []string
-}
-
-type notifyEvent struct {
-	// Kind is always upper case. Should be one of eventKind, anything else must result
-	// in a BADEVENT response code.
-	Kind string
-
-	FetchAtt []fetchAtt // Only for MessageNew
-}
-
-// ../rfc/5465:943
-func (p *parser) xeventGroup() (eg eventGroup) {
-	p.xtake("(")
-	eg.MailboxSpecifier = p.xfilterMailbox(mbspecsNotify)
-	p.xspace()
-	if p.take("NONE") {
-		p.xtake(")")
-		return eg
-	}
-	p.xtake("(")
-	for {
-		e := p.xnotifyEvent()
-		eg.Events = append(eg.Events, e)
-		if !p.space() {
-			break
-		}
-	}
-	p.xtake(")")
-	p.xtake(")")
-	return eg
-}
-
-var mbspecsEsearch = []mbspecKind{
-	mbspecSelected, // selected-delayed is only for NOTIFY.
-	mbspecInboxes,
-	mbspecPersonal,
-	mbspecSubscribed,
-	mbspecSubtreeOne, // Must come before Subtree due to eager parsing.
-	mbspecSubtree,
-	mbspecMailboxes,
-}
-
-var mbspecsNotify = []mbspecKind{
-	mbspecSelectedDelayed, // Must come before mbspecSelected, for eager parsing and mbspecSelected.
-	mbspecSelected,
-	mbspecInboxes,
-	mbspecPersonal,
-	mbspecSubscribed,
-	mbspecSubtreeOne, // From ESEARCH, we also allow it in NOTIFY.
-	mbspecSubtree,
-	mbspecMailboxes,
-}
-
-// If not esearch with "subtree-one", then for notify with "selected-delayed".
-func (p *parser) xfilterMailbox(allowed []mbspecKind) (ms mailboxSpecifier) {
-	var kind mbspecKind
-	for _, s := range allowed {
-		if p.take(string(s)) {
-			kind = s
-			break
-		}
-	}
-	if kind == mbspecKind("") {
-		xsyntaxErrorf("expected mailbox specifier")
-	}
-
-	ms.Kind = kind
-	switch kind {
-	case "SUBTREE", "SUBTREE-ONE", "MAILBOXES":
-		p.xtake(" ")
-		// One or more mailboxes. Multiple start with a list. ../rfc/5465:937
-		if p.take("(") {
-			for {
-				ms.Mailboxes = append(ms.Mailboxes, p.xmailbox())
-				if !p.take(" ") {
-					break
-				}
-			}
-			p.xtake(")")
-		} else {
-			ms.Mailboxes = []string{p.xmailbox()}
-		}
-	}
-	return ms
-}
-
-type eventKind string
-
-const (
-	eventMessageNew            eventKind = "MESSAGENEW"
-	eventMessageExpunge        eventKind = "MESSAGEEXPUNGE"
-	eventFlagChange            eventKind = "FLAGCHANGE"
-	eventAnnotationChange      eventKind = "ANNOTATIONCHANGE"
-	eventMailboxName           eventKind = "MAILBOXNAME"
-	eventSubscriptionChange    eventKind = "SUBSCRIPTIONCHANGE"
-	eventMailboxMetadataChange eventKind = "MAILBOXMETADATACHANGE"
-	eventServerMetadataChange  eventKind = "SERVERMETADATACHANGE"
-)
-
-var messageEventKinds = []eventKind{eventMessageNew, eventMessageExpunge, eventFlagChange, eventAnnotationChange}
-
-// ../rfc/5465:974
-func (p *parser) xnotifyEvent() notifyEvent {
-	s := strings.ToUpper(p.xatom())
-	e := notifyEvent{Kind: s}
-	if eventKind(e.Kind) == eventMessageNew {
-		if p.take(" (") {
-			for {
-				a := p.xfetchAtt()
-				e.FetchAtt = append(e.FetchAtt, a)
-				if !p.take(" ") {
-					break
-				}
-			}
-			p.xtake(")")
-		}
-	}
-	return e
-}

imapserver/prefixconn.go

@@ -1,8 +1,6 @@
 package imapserver
 
 import (
-	"bufio"
-	"io"
 	"net"
 )
 
@@ -15,7 +13,10 @@ type prefixConn struct {
 
 func (c *prefixConn) Read(buf []byte) (int, error) {
 	if len(c.prefix) > 0 {
-		n := min(len(buf), len(c.prefix))
+		n := len(buf)
+		if n > len(c.prefix) {
+			n = len(c.prefix)
+		}
 		copy(buf[:n], c.prefix[:n])
 		c.prefix = c.prefix[n:]
 		if len(c.prefix) == 0 {
@@ -25,18 +26,3 @@ func (c *prefixConn) Read(buf []byte) (int, error) {
 	}
 	return c.Conn.Read(buf)
 }
-
-// xprefixConn returns either the original net.Conn passed as parameter, or returns
-// a *prefixConn returning the buffered data available in br followed data from the
-// net.Conn passed in.
-func xprefixConn(c net.Conn, br *bufio.Reader) net.Conn {
-	n := br.Buffered()
-	if n == 0 {
-		return c
-	}
-
-	buf := make([]byte, n)
-	_, err := io.ReadFull(c, buf)
-	xcheckf(err, "get buffered data")
-	return &prefixConn{buf, c}
-}

imapserver/protocol.go

@@ -32,26 +32,17 @@ func (ss numSet) containsSeq(seq msgseq, uids []store.UID, searchResult []store.
 		uid := uids[int(seq)-1]
 		return uidSearch(searchResult, uid) > 0 && uidSearch(uids, uid) > 0
 	}
-	return ss.containsSeqCount(seq, uint32(len(uids)))
-}
-
-// containsSeqCount returns whether seq is contained in ss, which must not be a
-// searchResult, assuming the message count.
-func (ss numSet) containsSeqCount(seq msgseq, msgCount uint32) bool {
-	if msgCount == 0 {
-		return false
-	}
 	for _, r := range ss.ranges {
 		first := r.first.number
-		if r.first.star || first > msgCount {
+		if r.first.star || first > uint32(len(uids)) {
-			first = msgCount
+			first = uint32(len(uids))
 		}
 
 		last := first
 		if r.last != nil {
 			last = r.last.number
-			if r.last.star || last > msgCount {
+			if r.last.star || last > uint32(len(uids)) {
-				last = msgCount
+				last = uint32(len(uids))
 			}
 		}
 		if first > last {
@@ -65,77 +56,35 @@ func (ss numSet) containsSeqCount(seq msgseq, msgCount uint32) bool {
 	return false
 }
 
-// containsKnownUID returns whether uid, which is known to exist, matches the numSet.
+func (ss numSet) containsUID(uid store.UID, uids []store.UID, searchResult []store.UID) bool {
-// highestUID must return the highest/last UID in the mailbox, or an error. A last UID must
+	if len(uids) == 0 {
-// exist, otherwise this method wouldn't have been called with a known uid.
+		return false
-// highestUID is needed for interpreting UID sets like "<num>:*" where num is
+	}
-// higher than the uid to check.
-func (ss numSet) xcontainsKnownUID(uid store.UID, searchResult []store.UID, xhighestUID func() store.UID) bool {
 	if ss.searchResult {
-		return uidSearch(searchResult, uid) > 0
+		return uidSearch(searchResult, uid) > 0 && uidSearch(uids, uid) > 0
 	}
-
 	for _, r := range ss.ranges {
-		a := store.UID(r.first.number)
+		first := store.UID(r.first.number)
-		// Num in <num>:* can be larger than last, but it still matches the last...
+		if r.first.star || first > uids[len(uids)-1] {
-		// Similar for *:<num>. ../rfc/9051:4814
+			first = uids[len(uids)-1]
-		if r.first.star {
-			if r.last != nil && uid >= store.UID(r.last.number) {
-				return true
-			}
-			a = xhighestUID()
-		}
-		b := a
-		if r.last != nil {
-			b = store.UID(r.last.number)
-			if r.last.star {
-				if uid >= a {
-					return true
-				}
-				b = xhighestUID()
-			}
-		}
-		if a > b {
-			a, b = b, a
-		}
-		if uid >= a && uid <= b {
-			return true
-		}
-	}
-	return false
-}
-
-// xinterpretStar returns a numset that interprets stars in a uid set using
-// xlastUID, returning a new uid set without stars, with increasing first/last, and
-// without unneeded ranges (first.number != last.number).
-// If there are no messages in the mailbox, xlastUID must return zero and the
-// returned numSet will include 0.
-func (s numSet) xinterpretStar(xlastUID func() store.UID) numSet {
-	var ns numSet
-
-	for _, r := range s.ranges {
-		first := r.first.number
-		if r.first.star {
-			first = uint32(xlastUID())
 		}
 		last := first
+		// Num in <num>:* can be larger than last, but it still matches the last...
+		// Similar for *:<num>. ../rfc/9051:4814
 		if r.last != nil {
-			if r.last.star {
+			last = store.UID(r.last.number)
-				last = uint32(xlastUID())
+			if r.last.star || last > uids[len(uids)-1] {
-			} else {
+				last = uids[len(uids)-1]
-				last = r.last.number
 			}
 		}
 		if first > last {
 			first, last = last, first
 		}
-		nr := numRange{first: setNumber{number: first}}
+		if uid >= first && uid <= last && uidSearch(uids, uid) > 0 {
-		if first != last {
+			return true
-			nr.last = &setNumber{number: last}
 		}
-		ns.ranges = append(ns.ranges, nr)
 	}
-	return ns
+	return false
 }
 
 // contains returns whether the numset contains the number.
@@ -209,6 +158,38 @@ func (ss numSet) String() string {
 	return l[0]
 }
 
+// interpretStar returns a numset that interprets stars in a numset, returning a new
+// numset without stars with increasing first/last.
+func (s numSet) interpretStar(uids []store.UID) numSet {
+	var ns numSet
+	if len(uids) == 0 {
+		return ns
+	}
+
+	for _, r := range s.ranges {
+		first := r.first.number
+		if r.first.star || first > uint32(uids[len(uids)-1]) {
+			first = uint32(uids[len(uids)-1])
+		}
+		last := first
+		if r.last != nil {
+			last = r.last.number
+			if r.last.star || last > uint32(uids[len(uids)-1]) {
+				last = uint32(uids[len(uids)-1])
+			}
+		}
+		if first > last {
+			first, last = last, first
+		}
+		nr := numRange{first: setNumber{number: first}}
+		if first != last {
+			nr.last = &setNumber{number: last}
+		}
+		ns.ranges = append(ns.ranges, nr)
+	}
+	return ns
+}
+
 // whether numSet only has numbers (no star/search), and is strictly increasing.
 func (s *numSet) isBasicIncreasing() bool {
 	if s.searchResult {
@@ -326,15 +307,13 @@ type fetchAtt struct {
 	section       *sectionSpec
 	sectionBinary []uint32
 	partial       *partial
-	previewLazy   bool // Not regular "PREVIEW", but "PREVIEW (LAZY)".
 }
 
 type searchKey struct {
 	// Only one of searchKeys, seqSet and op can be non-nil/non-empty.
-	searchKeys []searchKey // In case of nested/multiple keys. Also for the top-level command.
+	searchKeys   []searchKey // In case of nested/multiple keys. Also for the top-level command.
-	seqSet     *numSet     // In case of bare sequence set. For op UID, field uidSet contains the parameter.
+	seqSet       *numSet     // In case of bare sequence set. For op UID, field uidSet contains the parameter.
-	op         string      // Determines which of the fields below are set.
+	op           string      // Determines which of the fields below are set.
-
 	headerField  string
 	astring      string
 	date         time.Time
@@ -346,40 +325,6 @@ type searchKey struct {
 	clientModseq *int64
 }
 
-// Whether we need message sequence numbers to evaluate. Sequence numbers are not
-// allowed with UIDONLY. And if we need sequence numbers we cannot optimize
-// searching for MAX with a query in reverse order.
-func (sk *searchKey) hasSequenceNumbers() bool {
-	for _, k := range sk.searchKeys {
-		if k.hasSequenceNumbers() {
-			return true
-		}
-	}
-	if sk.searchKey != nil && sk.searchKey.hasSequenceNumbers() || sk.searchKey2 != nil && sk.searchKey2.hasSequenceNumbers() {
-		return true
-	}
-	return sk.seqSet != nil && !sk.seqSet.searchResult
-}
-
-// hasModseq returns whether there is a modseq filter anywhere in the searchkey.
-func (sk *searchKey) hasModseq() bool {
-	if sk.clientModseq != nil {
-		return true
-	}
-	for _, e := range sk.searchKeys {
-		if e.hasModseq() {
-			return true
-		}
-	}
-	if sk.searchKey != nil && sk.searchKey.hasModseq() {
-		return true
-	}
-	if sk.searchKey2 != nil && sk.searchKey2.hasModseq() {
-		return true
-	}
-	return false
-}
-
 func compactUIDSet(l []store.UID) (r numSet) {
 	for len(l) > 0 {
 		e := 1

imapserver/protocol_test.go

@@ -23,10 +23,16 @@ func TestNumSetContains(t *testing.T) {
 	check(ss0.containsSeq(1, []store.UID{2}, []store.UID{2}))
 	check(!ss0.containsSeq(1, []store.UID{2}, []store.UID{}))
 
+	check(ss0.containsUID(1, []store.UID{1}, []store.UID{1}))
+	check(ss0.containsUID(2, []store.UID{1, 2, 3}, []store.UID{2}))
+	check(!ss0.containsUID(2, []store.UID{1, 2, 3}, []store.UID{}))
+	check(!ss0.containsUID(2, []store.UID{}, []store.UID{2}))
+
 	ss1 := numSet{false, []numRange{{*num(1), nil}}} // Single number 1.
 	check(ss1.containsSeq(1, []store.UID{2}, nil))
 	check(!ss1.containsSeq(2, []store.UID{1, 2}, nil))
 
+	check(ss1.containsUID(1, []store.UID{1}, nil))
 	check(ss1.containsSeq(1, []store.UID{2}, nil))
 	check(!ss1.containsSeq(2, []store.UID{1, 2}, nil))
 
@@ -38,6 +44,15 @@ func TestNumSetContains(t *testing.T) {
 	check(ss2.containsSeq(3, []store.UID{4, 5, 6}, nil))
 	check(!ss2.containsSeq(4, []store.UID{4, 5, 6}, nil))
 
+	check(ss2.containsUID(2, []store.UID{2}, nil))
+	check(!ss2.containsUID(1, []store.UID{1, 2, 3}, nil))
+	check(ss2.containsUID(3, []store.UID{1, 2, 3}, nil))
+	check(!ss2.containsUID(2, []store.UID{4, 5}, nil))
+	check(!ss2.containsUID(2, []store.UID{1}, nil))
+
+	check(ss2.containsUID(2, []store.UID{2, 6}, nil))
+	check(ss2.containsUID(6, []store.UID{2, 6}, nil))
+
 	// *:2, same as 2:*
 	ss3 := numSet{false, []numRange{{*star, num(2)}}}
 	check(ss3.containsSeq(1, []store.UID{2}, nil))
@@ -45,6 +60,15 @@ func TestNumSetContains(t *testing.T) {
 	check(ss3.containsSeq(2, []store.UID{4, 5}, nil))
 	check(ss3.containsSeq(3, []store.UID{4, 5, 6}, nil))
 	check(!ss3.containsSeq(4, []store.UID{4, 5, 6}, nil))
+
+	check(ss3.containsUID(2, []store.UID{2}, nil))
+	check(!ss3.containsUID(1, []store.UID{1, 2, 3}, nil))
+	check(ss3.containsUID(3, []store.UID{1, 2, 3}, nil))
+	check(!ss3.containsUID(2, []store.UID{4, 5}, nil))
+	check(!ss3.containsUID(2, []store.UID{1}, nil))
+
+	check(ss3.containsUID(2, []store.UID{2, 6}, nil))
+	check(ss3.containsUID(6, []store.UID{2, 6}, nil))
 }
 
 func TestNumSetInterpret(t *testing.T) {
@@ -53,34 +77,38 @@ func TestNumSetInterpret(t *testing.T) {
 		return p.xnumSet0(true, false)
 	}
 
-	checkEqual := func(lastUID store.UID, a, s string) {
+	checkEqual := func(uids []store.UID, a, s string) {
 		t.Helper()
-		n := parseNumSet(a).xinterpretStar(func() store.UID { return lastUID })
+		n := parseNumSet(a).interpretStar(uids)
 		ns := n.String()
 		if ns != s {
 			t.Fatalf("%s != %s", ns, s)
 		}
 	}
 
-	checkEqual(0, "1:*", "0:1")
+	checkEqual([]store.UID{}, "1:*", "")
-	checkEqual(1, "1:*", "1")
+	checkEqual([]store.UID{1}, "1:*", "1")
-	checkEqual(3, "1:*", "1:3")
+	checkEqual([]store.UID{1, 3}, "1:*", "1:3")
-	checkEqual(3, "4:*", "3:4")
+	checkEqual([]store.UID{1, 3}, "4:*", "3")
-	checkEqual(3, "*:4", "3:4")
+	checkEqual([]store.UID{1, 3}, "*:4", "3")
-	checkEqual(3, "*:4", "3:4")
+	checkEqual([]store.UID{2, 3}, "*:4", "3")
-	checkEqual(3, "*:1", "1:3")
+	checkEqual([]store.UID{2, 3}, "*:1", "1:3")
-	checkEqual(3, "1:*", "1:3")
+	checkEqual([]store.UID{2, 3}, "1:*", "1:3")
-	checkEqual(3, "1,2,3", "1,2,3")
+	checkEqual([]store.UID{1, 2, 3}, "1,2,3", "1,2,3")
-	checkEqual(0, "1,2,3", "1,2,3")
+	checkEqual([]store.UID{}, "1,2,3", "")
-	checkEqual(0, "1:3", "1:3")
+	checkEqual([]store.UID{}, "1:3", "")
-	checkEqual(0, "3:1", "1:3")
+	checkEqual([]store.UID{}, "3:1", "")
 
-	iter := parseNumSet("3:1").xinterpretStar(func() store.UID { return 2 }).newIter()
+	iter := parseNumSet("1:3").interpretStar([]store.UID{}).newIter()
+	if _, ok := iter.Next(); ok {
+		t.Fatalf("expected immediate end for empty iter")
+	}
+
+	iter = parseNumSet("3:1").interpretStar([]store.UID{1, 2}).newIter()
 	v0, _ := iter.Next()
 	v1, _ := iter.Next()
-	v2, _ := iter.Next()
 	_, ok := iter.Next()
-	if v0 != 1 || v1 != 2 || v2 != 3 || ok {
+	if v0 != 1 || v1 != 2 || ok {
-		t.Fatalf("got %v %v %v %v, expected 1, 2, 3 false", v0, v1, v2, ok)
+		t.Fatalf("got %v %v %v, expected 1, 2, false", v0, v1, ok)
 	}
 }

imapserver/quota_test.go

@@ -7,10 +7,10 @@ import (
 )
 
 func TestQuota1(t *testing.T) {
-	tc := start(t, false)
+	tc := start(t)
 	defer tc.close()
 
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
 
 	// We don't implement setquota.
 	tc.transactf("bad", `setquota "" (STORAGE 123)`)
@@ -35,10 +35,10 @@ func TestQuota1(t *testing.T) {
 	tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusDeletedStorage: 0}})
 
 	// tclimit does have a limit.
-	tclimit := startArgs(t, false, false, false, true, true, "limit")
+	tclimit := startArgs(t, false, false, true, true, "limit")
 	defer tclimit.close()
 
-	tclimit.login("limit@mox.example", password0)
+	tclimit.client.Login("limit@mox.example", password0)
 
 	tclimit.transactf("ok", "getquotaroot inbox")
 	tclimit.xuntagged(

imapserver/rename_test.go

@@ -6,39 +6,29 @@ import (
 	"github.com/mjl-/mox/imapclient"
 )
 
-func TestRename(t *testing.T) {
-	testRename(t, false)
-}
-
-func TestRenameUIDOnly(t *testing.T) {
-	testRename(t, true)
-}
-
 // todo: check that UIDValidity is indeed updated properly.
-func testRename(t *testing.T, uidonly bool) {
+func TestRename(t *testing.T) {
-	tc := start(t, uidonly)
+	tc := start(t)
 	defer tc.close()
 
-	tc2 := startNoSwitchboard(t, uidonly)
+	tc2 := startNoSwitchboard(t)
-	defer tc2.closeNoWait()
+	defer tc2.close()
 
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
-	tc2.login("mjl@mox.example", password0)
+	tc2.client.Login("mjl@mox.example", password0)
 
 	tc.transactf("bad", "rename")      // Missing parameters.
 	tc.transactf("bad", "rename x")    // Missing destination.
 	tc.transactf("bad", "rename x y ") // Leftover data.
 
 	tc.transactf("no", "rename doesnotexist newbox") // Does not exist.
-	tc.xcodeWord("NONEXISTENT")                      // ../rfc/9051:5140
+	tc.xcode("NONEXISTENT")                          // ../rfc/9051:5140
-	tc.transactf("no", "rename expungebox newbox")   // No longer exists.
+	tc.transactf("no", `rename "Sent" "Trash"`)      // Already exists.
-	tc.xcodeWord("NONEXISTENT")
+	tc.xcode("ALREADYEXISTS")
-	tc.transactf("no", `rename "Sent" "Trash"`) // Already exists.
-	tc.xcodeWord("ALREADYEXISTS")
 
-	tc.client.Create("x", nil)
+	tc.client.Create("x")
 	tc.client.Subscribe("sub")
-	tc.client.Create("a/b/c", nil)
+	tc.client.Create("a/b/c")
 	tc.client.Subscribe("x/y/c") // For later rename, but not affected by rename of x.
 	tc2.transactf("ok", "noop")  // Drain.
 
@@ -47,7 +37,7 @@ func testRename(t *testing.T, uidonly bool) {
 	tc2.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "z"})
 
 	// OldName is only set for IMAP4rev2 or NOTIFY.
-	tc2.client.Enable(imapclient.CapIMAP4rev2)
+	tc2.client.Enable("IMAP4rev2")
 	tc.transactf("ok", "rename z y")
 	tc2.transactf("ok", "noop")
 	tc2.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "y", OldName: "z"})
@@ -59,16 +49,16 @@ func testRename(t *testing.T, uidonly bool) {
 
 	// Cannot rename a child to a parent. It already exists.
 	tc.transactf("no", "rename a/b/c a/b")
-	tc.xcodeWord("ALREADYEXISTS")
+	tc.xcode("ALREADYEXISTS")
 	tc.transactf("no", "rename a/b a")
-	tc.xcodeWord("ALREADYEXISTS")
+	tc.xcode("ALREADYEXISTS")
 
 	tc2.transactf("ok", "noop")          // Drain.
 	tc.transactf("ok", "rename a/b x/y") // This will cause new parent "x" to be created, and a/b and a/b/c to be renamed.
 	tc2.transactf("ok", "noop")
 	tc2.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "x"}, imapclient.UntaggedList{Separator: '/', Mailbox: "x/y", OldName: "a/b"}, imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "x/y/c", OldName: "a/b/c"})
 
-	tc.client.Create("k/l", nil)
+	tc.client.Create("k/l")
 	tc.transactf("ok", "rename k/l k/l/m") // With "l" renamed, a new "k" will be created.
 	tc.transactf("ok", `list "" "k*" return (subscribed)`)
 	tc.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "k"}, imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "k/l"}, imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/m"})
@@ -80,51 +70,28 @@ func testRename(t *testing.T, uidonly bool) {
 	tc.client.Unsubscribe("k")
 	tc.transactf("ok", "rename k/l k/l/m") // With "l" renamed, a new "k" will be created.
 	tc.transactf("ok", `list "" "k*" return (subscribed)`)
-	tc.xuntagged(
+	tc.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "k"}, imapclient.UntaggedList{Flags: []string{"\\Subscribed"}, Separator: '/', Mailbox: "k/l"}, imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/m"})
-		imapclient.UntaggedList{Separator: '/', Mailbox: "k"},
-		imapclient.UntaggedList{Flags: []string{"\\Subscribed"}, Separator: '/', Mailbox: "k/l"},
-		imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/m"},
-	)
-
-	tc.transactf("ok", "rename k/l/m k/l/x/y/m") // k/l/x and k/l/x/y will be created.
-	tc.transactf("ok", `list "" "k/l/x*" return (subscribed)`)
-	tc.xuntagged(
-		imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/x"},
-		imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/x/y"},
-		imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/x/y/m"},
-	)
 
 	// Renaming inbox keeps inbox in existence, moves messages, and does not rename children.
 	tc.transactf("ok", "create inbox/a")
 	// To check if UIDs are renumbered properly, we add UIDs 1 and 2. Expunge 1,
 	// keeping only 2. Then rename the inbox, which should renumber UID 2 in the old
 	// inbox to UID 1 in the newly created mailbox.
-	tc.transactf("ok", "append inbox (\\deleted) {1+}\r\nx")
+	tc.transactf("ok", "append inbox (\\deleted) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
-	tc.transactf("ok", "append inbox (label1) {1+}\r\nx")
+	tc.transactf("ok", "append inbox (label1) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
 	tc.transactf("ok", `select inbox`)
 	tc.transactf("ok", "expunge")
-	tc.transactf("ok", "rename inbox x/minbox")
+	tc.transactf("ok", "rename inbox minbox")
-	tc.transactf("ok", `list "" (inbox inbox/a x/minbox)`)
+	tc.transactf("ok", `list "" (inbox inbox/a minbox)`)
-	tc.xuntagged(
+	tc.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"}, imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox/a"}, imapclient.UntaggedList{Separator: '/', Mailbox: "minbox"})
-		imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"},
+	tc.transactf("ok", `select minbox`)
-		imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox/a"},
-		imapclient.UntaggedList{Separator: '/', Mailbox: "x/minbox"},
-	)
-	tc.transactf("ok", `select x/minbox`)
 	tc.transactf("ok", `uid fetch 1:* flags`)
-	tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchFlags{"label1"}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}}})
 
 	// Renaming to new hiearchy that does not have any subscribes.
-	tc.transactf("ok", "rename x/minbox w/w")
+	tc.transactf("ok", "rename minbox w/w")
 	tc.transactf("ok", `list "" "w*"`)
 	tc.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "w"}, imapclient.UntaggedList{Separator: '/', Mailbox: "w/w"})
 
-	tc.transactf("ok", "rename inbox misc/old/inbox")
-	tc.transactf("ok", `list "" (misc misc/old/inbox)`)
-	tc.xuntagged(
-		imapclient.UntaggedList{Separator: '/', Mailbox: "misc"},
-		imapclient.UntaggedList{Separator: '/', Mailbox: "misc/old/inbox"},
-	)
-
 	// todo: test create+delete+rename of/to a name results in a higher uidvalidity.
 }

imapserver/replace.go

@@ -1,369 +0,0 @@
-package imapserver
-
-import (
-	"errors"
-	"io"
-	"os"
-	"time"
-
-	"github.com/mjl-/bstore"
-
-	"github.com/mjl-/mox/message"
-	"github.com/mjl-/mox/mlog"
-	"github.com/mjl-/mox/store"
-)
-
-// Replace relaces a message for another, atomically, possibly in another mailbox,
-// without needing a sequence of: append message, store \deleted flag, expunge.
-//
-// State: Selected
-func (c *conn) cmdxReplace(isUID bool, tag, cmd string, p *parser) {
-	// Command: ../rfc/8508:158 ../rfc/8508:198
-
-	// Request syntax: ../rfc/8508:471
-	p.xspace()
-	star := p.take("*")
-	var num uint32
-	if !star {
-		num = p.xnznumber()
-	}
-	p.xspace()
-	name := p.xmailbox()
-
-	// ../rfc/4466:473
-	p.xspace()
-	var storeFlags store.Flags
-	var keywords []string
-	if p.hasPrefix("(") {
-		// Error must be a syntax error, to properly abort the connection due to literal.
-		var err error
-		storeFlags, keywords, err = store.ParseFlagsKeywords(p.xflagList())
-		if err != nil {
-			xsyntaxErrorf("parsing flags: %v", err)
-		}
-		p.xspace()
-	}
-
-	var tm time.Time
-	if p.hasPrefix(`"`) {
-		tm = p.xdateTime()
-		p.xspace()
-	} else {
-		tm = time.Now()
-	}
-
-	// todo: only with utf8 should we we accept message headers with utf-8. we currently always accept them.
-	// todo: this is only relevant if we also support the CATENATE extension?
-	// ../rfc/6855:204
-	utf8 := p.take("UTF8 (")
-	if utf8 {
-		p.xtake("~")
-	}
-	// Always allow literal8, for binary extension. ../rfc/4466:486
-	// For utf8, we already consumed the required ~ above.
-	size, synclit := p.xliteralSize(!utf8, false)
-
-	// Check the request, including old message in database, whether the message fits
-	// in quota. If a non-nil func is returned, an error was found. Calling the
-	// function aborts handling this command.
-	var uidOld store.UID
-	checkMessage := func(tx *bstore.Tx) func() {
-		if c.readonly {
-			return func() { xuserErrorf("mailbox open in read-only mode") }
-		}
-
-		mb, err := c.account.MailboxFind(tx, name)
-		if err != nil {
-			return func() { xserverErrorf("finding mailbox: %v", err) }
-		}
-		if mb == nil {
-			return func() { xusercodeErrorf("TRYCREATE", "%w", store.ErrUnknownMailbox) }
-		}
-
-		// Resolve "*" for UID or message sequence.
-		if star {
-			if c.uidonly {
-				q := bstore.QueryTx[store.Message](tx)
-				q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
-				q.FilterEqual("Expunged", false)
-				q.FilterLess("UID", c.uidnext)
-				q.SortDesc("UID")
-				q.Limit(1)
-				m, err := q.Get()
-				if err == bstore.ErrAbsent {
-					return func() { xsyntaxErrorf("cannot use * on empty mailbox") }
-				}
-				xcheckf(err, "get last message in mailbox")
-				num = uint32(m.UID)
-			} else if c.exists == 0 {
-				return func() { xsyntaxErrorf("cannot use * on empty mailbox") }
-			} else if isUID {
-				num = uint32(c.uids[c.exists-1])
-			} else {
-				num = uint32(c.exists)
-			}
-			star = false
-		}
-
-		// Find or verify UID of message to replace.
-		if isUID {
-			uidOld = store.UID(num)
-		} else if num > c.exists {
-			return func() { xuserErrorf("invalid msgseq") }
-		} else {
-			uidOld = c.uids[int(num)-1]
-		}
-
-		// Check the message still exists in the database. If it doesn't, it may have been
-		// deleted just now and we won't check the quota. We'll raise an error later on,
-		// when we are not possibly reading a sync literal and can respond with unsolicited
-		// expunges.
-		q := bstore.QueryTx[store.Message](tx)
-		q.FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: uidOld})
-		q.FilterEqual("Expunged", false)
-		q.FilterLess("UID", c.uidnext)
-		_, err = q.Get()
-		if err == bstore.ErrAbsent {
-			return nil
-		}
-		if err != nil {
-			return func() { xserverErrorf("get message to replace: %v", err) }
-		}
-
-		// Check if we can add size bytes. We can't necessarily remove the current message yet.
-		ok, maxSize, err := c.account.CanAddMessageSize(tx, size)
-		if err != nil {
-			return func() { xserverErrorf("check quota: %v", err) }
-		}
-		if !ok {
-			// ../rfc/9208:472
-			return func() { xusercodeErrorf("OVERQUOTA", "account over maximum total message size %d", maxSize) }
-		}
-		return nil
-	}
-
-	var errfn func()
-	if synclit {
-		// Check request, if it cannot succeed, fail it now before client is sending the data.
-
-		name = xcheckmailboxname(name, true)
-
-		c.account.WithRLock(func() {
-			c.xdbread(func(tx *bstore.Tx) {
-				errfn = checkMessage(tx)
-				if errfn != nil {
-					errfn()
-				}
-			})
-		})
-
-		c.xwritelinef("+ ")
-	} else {
-		var err error
-		name, _, err = store.CheckMailboxName(name, true)
-		if err != nil {
-			errfn = func() { xusercodeErrorf("CANNOT", "%s", err) }
-		} else {
-			c.account.WithRLock(func() {
-				c.xdbread(func(tx *bstore.Tx) {
-					errfn = checkMessage(tx)
-				})
-			})
-		}
-	}
-
-	var file *os.File
-	var newID int64 // Delivered message ID, file removed on error.
-	var f io.Writer
-	var commit bool
-
-	if errfn != nil {
-		// We got a non-sync literal, we will consume some data, but abort if there's too
-		// much. We draw the line at 1mb. Client should have used synchronizing literal.
-		if size > 1000*1000 {
-			// ../rfc/9051:357 ../rfc/3501:347
-			err := errors.New("error condition and non-synchronizing literal too big")
-			bye := "* BYE [ALERT] " + err.Error()
-			panic(syntaxError{bye, "TOOBIG", err.Error(), err})
-		}
-		// Message will not be accepted.
-		f = io.Discard
-	} else {
-		// Read the message into a temporary file.
-		var err error
-		file, err = store.CreateMessageTemp(c.log, "imap-replace")
-		xcheckf(err, "creating temp file for message")
-		defer store.CloseRemoveTempFile(c.log, file, "temporary message file")
-		f = file
-
-		defer func() {
-			if !commit && newID != 0 {
-				p := c.account.MessagePath(newID)
-				err := os.Remove(p)
-				c.xsanity(err, "remove message file for replace after error")
-			}
-		}()
-	}
-
-	// Read the message data.
-	defer c.xtraceread(mlog.LevelTracedata)()
-	mw := message.NewWriter(f)
-	msize, err := io.Copy(mw, io.LimitReader(c.br, size))
-	c.xtraceread(mlog.LevelTrace) // Restore.
-	if err != nil {
-		// Cannot use xcheckf due to %w handling of errIO.
-		c.xbrokenf("reading literal message: %s (%w)", err, errIO)
-	}
-	if msize != size {
-		c.xbrokenf("read %d bytes for message, expected %d (%w)", msize, size, errIO)
-	}
-
-	// Finish reading the command.
-	line := c.xreadline(false)
-	p = newParser(line, c)
-	if utf8 {
-		p.xtake(")")
-	}
-	p.xempty()
-
-	// If an error was found earlier, abort the command now that we've read the message.
-	if errfn != nil {
-		errfn()
-	}
-
-	var oldMsgExpunged bool
-
-	var om, nm store.Message
-	var mbSrc, mbDst store.Mailbox // Src and dst mailboxes can be different. ../rfc/8508:263
-	var overflow bool
-	var pendingChanges []store.Change
-	defer func() {
-		// In case of panic.
-		c.flushChanges(pendingChanges)
-	}()
-
-	c.account.WithWLock(func() {
-		var changes []store.Change
-
-		c.xdbwrite(func(tx *bstore.Tx) {
-			mbSrc = c.xmailboxID(tx, c.mailboxID)
-
-			// Get old message. If it has been expunged, we should have a pending change for
-			// it. We'll send untagged responses and fail the command.
-			var err error
-			qom := bstore.QueryTx[store.Message](tx)
-			qom.FilterNonzero(store.Message{MailboxID: mbSrc.ID, UID: uidOld})
-			om, err = qom.Get()
-			xcheckf(err, "get old message to replace from database")
-			if om.Expunged {
-				oldMsgExpunged = true
-				return
-			}
-
-			// Check quota for addition of new message. We can't necessarily yet remove the old message.
-			ok, maxSize, err := c.account.CanAddMessageSize(tx, mw.Size)
-			xcheckf(err, "checking quota")
-			if !ok {
-				// ../rfc/9208:472
-				xusercodeErrorf("OVERQUOTA", "account over maximum total message size %d", maxSize)
-			}
-
-			modseq, err := c.account.NextModSeq(tx)
-			xcheckf(err, "get next mod seq")
-
-			chremuids, _, err := c.account.MessageRemove(c.log, tx, modseq, &mbSrc, store.RemoveOpts{}, om)
-			xcheckf(err, "expunge old message")
-			changes = append(changes, chremuids)
-			// Note: we only add a mbSrc counts change later on, if it is not equal to mbDst.
-
-			err = tx.Update(&mbSrc)
-			xcheckf(err, "updating source mailbox counts")
-
-			mbDst = c.xmailbox(tx, name, "TRYCREATE")
-			mbDst.ModSeq = modseq
-
-			nkeywords := len(mbDst.Keywords)
-
-			// Make new message to deliver.
-			nm = store.Message{
-				MailboxID:     mbDst.ID,
-				MailboxOrigID: mbDst.ID,
-				Received:      tm,
-				Flags:         storeFlags,
-				Keywords:      keywords,
-				Size:          mw.Size,
-				ModSeq:        modseq,
-				CreateSeq:     modseq,
-			}
-
-			err = c.account.MessageAdd(c.log, tx, &mbDst, &nm, file, store.AddOpts{})
-			xcheckf(err, "delivering message")
-			newID = nm.ID
-
-			changes = append(changes, nm.ChangeAddUID(mbDst), mbDst.ChangeCounts())
-			if nkeywords != len(mbDst.Keywords) {
-				changes = append(changes, mbDst.ChangeKeywords())
-			}
-
-			err = tx.Update(&mbDst)
-			xcheckf(err, "updating destination mailbox")
-		})
-
-		// Fetch pending changes, possibly with new UIDs, so we can apply them before adding our own new UID.
-		overflow, pendingChanges = c.comm.Get()
-
-		if oldMsgExpunged {
-			return
-		}
-
-		// Success, make sure messages aren't cleaned up anymore.
-		commit = true
-
-		// Broadcast the change to other connections.
-		if mbSrc.ID != mbDst.ID {
-			changes = append(changes, mbSrc.ChangeCounts())
-		}
-		c.broadcast(changes)
-	})
-
-	// Must update our msgseq/uids tracking with latest pending changes.
-	l := pendingChanges
-	pendingChanges = nil
-	c.xapplyChanges(overflow, l, false)
-
-	// If we couldn't find the message, send a NO response. We've just applied pending
-	// changes, which should have expunged the absent message.
-	if oldMsgExpunged {
-		xuserErrorf("message to be replaced has been expunged")
-	}
-
-	// If the destination mailbox is our currently selected mailbox, we register and
-	// announce the new message.
-	if mbDst.ID == c.mailboxID {
-		c.uidAppend(nm.UID)
-		// We send an untagged OK with APPENDUID, for sane bookkeeping in clients. ../rfc/8508:401
-		c.xbwritelinef("* OK [APPENDUID %d %d] ", mbDst.UIDValidity, nm.UID)
-		c.xbwritelinef("* %d EXISTS", c.exists)
-	}
-
-	// We must return vanished instead of expunge, and also highestmodseq, when qresync
-	// was enabled. ../rfc/8508:422 ../rfc/7162:1883
-	qresync := c.enabled[capQresync]
-
-	// Now that we are in sync with msgseq, we can find our old msgseq and say it is
-	// expunged or vanished. ../rfc/7162:1900
-	var oseq msgseq
-	if c.uidonly {
-		c.exists--
-	} else {
-		oseq = c.xsequence(om.UID)
-		c.sequenceRemove(oseq, om.UID)
-	}
-	if qresync || c.uidonly {
-		c.xbwritelinef("* VANISHED %d", om.UID)
-		// ../rfc/7162:1916
-	} else {
-		c.xbwritelinef("* %d EXPUNGE", oseq)
-	}
-	c.xwriteresultf("%s OK [HIGHESTMODSEQ %d] replaced", tag, nm.ModSeq.Client())
-}

imapserver/replace_test.go

@@ -1,233 +0,0 @@
-package imapserver
-
-import (
-	"testing"
-
-	"github.com/mjl-/mox/imapclient"
-)
-
-func TestReplace(t *testing.T) {
-	testReplace(t, false)
-}
-
-func TestReplaceUIDOnly(t *testing.T) {
-	testReplace(t, true)
-}
-
-func testReplace(t *testing.T, uidonly bool) {
-	defer mockUIDValidity()()
-
-	tc := start(t, uidonly)
-	defer tc.close()
-
-	tc2 := startNoSwitchboard(t, uidonly)
-	defer tc2.closeNoWait()
-
-	tc.login("mjl@mox.example", password0)
-	tc.client.Select("inbox")
-
-	// Star not allowed on empty mailbox.
-	tc.transactf("bad", "uid replace * inbox {1}")
-	if !uidonly {
-		tc.transactf("bad", "replace * inbox {1}")
-	}
-
-	// Append 3 messages, remove first. Leaves msgseq 1,2 with uid 2,3.
-	tc.client.MultiAppend("inbox", makeAppend(exampleMsg), makeAppend(exampleMsg), makeAppend(exampleMsg))
-	tc.client.UIDStoreFlagsSet("1", true, `\deleted`)
-	tc.client.Expunge()
-
-	tc.transactf("no", "uid replace 1 expungebox {1}") // Mailbox no longer exists.
-	tc.xcodeWord("TRYCREATE")
-
-	tc2.login("mjl@mox.example", password0)
-	tc2.client.Select("inbox")
-
-	// Replace last message (msgseq 2, uid 3) in same mailbox.
-	if uidonly {
-		tc.lastResponse, tc.lastErr = tc.client.UIDReplace("3", "INBOX", makeAppend(searchMsg))
-	} else {
-		tc.lastResponse, tc.lastErr = tc.client.MSNReplace("2", "INBOX", makeAppend(searchMsg))
-	}
-	tcheck(tc.t, tc.lastErr, "read imap response")
-	if uidonly {
-		tc.xuntagged(
-			imapclient.UntaggedResult{Status: "OK", Code: imapclient.CodeAppendUID{UIDValidity: 1, UIDs: xparseUIDRange("4")}, Text: ""},
-			imapclient.UntaggedExists(3),
-			imapclient.UntaggedVanished{UIDs: xparseNumSet("3")},
-		)
-	} else {
-		tc.xuntagged(
-			imapclient.UntaggedResult{Status: "OK", Code: imapclient.CodeAppendUID{UIDValidity: 1, UIDs: xparseUIDRange("4")}, Text: ""},
-			imapclient.UntaggedExists(3),
-			imapclient.UntaggedExpunge(2),
-		)
-	}
-	tc.xcode(imapclient.CodeHighestModSeq(8))
-
-	// Check that other client sees Exists and Expunge.
-	tc2.transactf("ok", "noop")
-	if uidonly {
-		tc2.xuntagged(
-			imapclient.UntaggedVanished{UIDs: xparseNumSet("3")},
-			imapclient.UntaggedExists(2),
-			tc.untaggedFetch(2, 4, imapclient.FetchFlags(nil)),
-		)
-	} else {
-		tc2.xuntagged(
-			imapclient.UntaggedExpunge(2),
-			imapclient.UntaggedExists(2),
-			tc.untaggedFetch(2, 4, imapclient.FetchFlags(nil)),
-		)
-	}
-
-	// Enable qresync, replace uid 2 (msgseq 1) to different mailbox, see that we get vanished instead of expunged.
-	tc.transactf("ok", "enable qresync")
-	tc.lastResponse, tc.lastErr = tc.client.UIDReplace("2", "INBOX", makeAppend(searchMsg))
-	tcheck(tc.t, tc.lastErr, "read imap response")
-	tc.xuntagged(
-		imapclient.UntaggedResult{Status: "OK", Code: imapclient.CodeAppendUID{UIDValidity: 1, UIDs: xparseUIDRange("5")}, Text: ""},
-		imapclient.UntaggedExists(3),
-		imapclient.UntaggedVanished{UIDs: xparseNumSet("2")},
-	)
-	tc.xcode(imapclient.CodeHighestModSeq(9))
-
-	// Use "*" for replacing.
-	tc.transactf("ok", "uid replace * inbox {1+}\r\nx")
-	tc.xuntagged(
-		imapclient.UntaggedResult{Status: "OK", Code: imapclient.CodeAppendUID{UIDValidity: 1, UIDs: xparseUIDRange("6")}, Text: ""},
-		imapclient.UntaggedExists(3),
-		imapclient.UntaggedVanished{UIDs: xparseNumSet("5")},
-	)
-	if !uidonly {
-		tc.transactf("ok", "replace * inbox {1+}\r\ny")
-		tc.xuntagged(
-			imapclient.UntaggedResult{Status: "OK", Code: imapclient.CodeAppendUID{UIDValidity: 1, UIDs: xparseUIDRange("7")}, Text: ""},
-			imapclient.UntaggedExists(3),
-			imapclient.UntaggedVanished{UIDs: xparseNumSet("6")},
-		)
-	}
-
-	// Non-existent mailbox with non-synchronizing literal should consume the literal.
-	if uidonly {
-		tc.transactf("no", "uid replace 1 bogusbox {1+}\r\nx")
-	} else {
-		tc.transactf("no", "replace 1 bogusbox {1+}\r\nx")
-	}
-
-	// Leftover data.
-	tc.transactf("bad", "replace 1 inbox () {6+}\r\ntest\r\n ")
-}
-
-func TestReplaceBigNonsyncLit(t *testing.T) {
-	tc := start(t, false)
-	defer tc.close()
-
-	tc.login("mjl@mox.example", password0)
-	tc.client.Select("inbox")
-
-	// Adding a message >1mb with non-sync literal to non-existent mailbox should abort entire connection.
-	tc.transactf("bad", "replace 12345 inbox {2000000+}")
-	tc.xuntagged(
-		imapclient.UntaggedBye{Code: imapclient.CodeWord("ALERT"), Text: "error condition and non-synchronizing literal too big"},
-	)
-	tc.xcodeWord("TOOBIG")
-}
-
-func TestReplaceQuota(t *testing.T) {
-	testReplaceQuota(t, false)
-}
-
-func TestReplaceQuotaUIDOnly(t *testing.T) {
-	testReplaceQuota(t, true)
-}
-
-func testReplaceQuota(t *testing.T, uidonly bool) {
-	// with quota limit
-	tc := startArgs(t, uidonly, true, false, true, true, "limit")
-	defer tc.close()
-
-	tc.login("limit@mox.example", password0)
-	tc.client.Select("inbox")
-	tc.client.Append("inbox", makeAppend("x"))
-
-	// Synchronizing literal, we get failure immediately.
-	tc.transactf("no", "uid replace 1 inbox {6}\r\n")
-	tc.xcodeWord("OVERQUOTA")
-
-	// Synchronizing literal to non-existent mailbox, we get failure immediately.
-	tc.transactf("no", "uid replace 1 badbox {6}\r\n")
-	tc.xcodeWord("TRYCREATE")
-
-	buf := make([]byte, 4000, 4002)
-	for i := range buf {
-		buf[i] = 'x'
-	}
-	buf = append(buf, "\r\n"...)
-
-	// Non-synchronizing literal. We get to write our data.
-	tc.client.WriteCommandf("", "uid replace 1 inbox ~{4000+}")
-	_, err := tc.client.Write(buf)
-	tc.check(err, "write replace message")
-	tc.response("no")
-	tc.xcodeWord("OVERQUOTA")
-
-	// Non-synchronizing literal to bad mailbox.
-	tc.client.WriteCommandf("", "uid replace 1 badbox {4000+}")
-	_, err = tc.client.Write(buf)
-	tc.check(err, "write replace message")
-	tc.response("no")
-	tc.xcodeWord("TRYCREATE")
-}
-
-func TestReplaceExpunged(t *testing.T) {
-	testReplaceExpunged(t, false)
-}
-
-func TestReplaceExpungedUIDOnly(t *testing.T) {
-	testReplaceExpunged(t, true)
-}
-
-func testReplaceExpunged(t *testing.T, uidonly bool) {
-	tc := start(t, uidonly)
-	defer tc.close()
-
-	tc.login("mjl@mox.example", password0)
-	tc.client.Select("inbox")
-	tc.client.Append("inbox", makeAppend(exampleMsg))
-
-	// We start the command, but don't write data yet.
-	tc.client.WriteCommandf("", "uid replace 1 inbox {4000}")
-
-	// Get in with second client and remove the message we are replacing.
-	tc2 := startNoSwitchboard(t, uidonly)
-	defer tc2.closeNoWait()
-	tc2.login("mjl@mox.example", password0)
-	tc2.client.Select("inbox")
-	tc2.client.UIDStoreFlagsSet("1", true, `\Deleted`)
-	tc2.client.Expunge()
-	tc2.client.Unselect()
-	tc2.client.Close()
-
-	// Now continue trying to replace the message. We should get an error and an expunge.
-	tc.readprefixline("+ ")
-	buf := make([]byte, 4000, 4002)
-	for i := range buf {
-		buf[i] = 'x'
-	}
-	buf = append(buf, "\r\n"...)
-	_, err := tc.client.Write(buf)
-	tc.check(err, "write replace message")
-	tc.response("no")
-	if uidonly {
-		tc.xuntagged(
-			tc.untaggedFetch(1, 1, imapclient.FetchFlags{`\Deleted`}),
-			imapclient.UntaggedVanished{UIDs: xparseNumSet("1")},
-		)
-	} else {
-		tc.xuntagged(
-			tc.untaggedFetch(1, 1, imapclient.FetchFlags{`\Deleted`}),
-			imapclient.UntaggedExpunge(1),
-		)
-	}
-}

imapserver/search.go

@@ -1,14 +1,10 @@
 package imapserver
 
 import (
-	"cmp"
 	"fmt"
 	"log/slog"
-	"maps"
 	"net/textproto"
-	"slices"
 	"strings"
-	"time"
 
 	"github.com/mjl-/bstore"
 
@@ -16,54 +12,22 @@ import (
 	"github.com/mjl-/mox/store"
 )
 
-// If last search output was this long ago, we write an untagged inprogress
-// response. Changed during tests. ../rfc/9585:109
-var inProgressPeriod = time.Duration(10 * time.Second)
-
-// ESEARCH allows searching multiple mailboxes, referenced through mailbox filters
-// borrowed from the NOTIFY extension. Unlike the regular extended SEARCH/UID
-// SEARCH command that always returns an ESEARCH response, the ESEARCH command only
-// returns ESEARCH responses when there were matches in a mailbox.
-//
-// ../rfc/7377:159
-func (c *conn) cmdEsearch(tag, cmd string, p *parser) {
-	c.cmdxSearch(true, true, tag, cmd, p)
-}
-
 // Search returns messages matching criteria specified in parameters.
 //
-// State: Selected for SEARCH and UID SEARCH, Authenticated or selectd for ESEARCH.
+// State: Selected
-func (c *conn) cmdxSearch(isUID, isE bool, tag, cmd string, p *parser) {
+func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
-	// Command: ../rfc/9051:3716 ../rfc/7377:159 ../rfc/6237:142 ../rfc/4731:31 ../rfc/4466:354 ../rfc/3501:2723
+	// Command: ../rfc/9051:3716 ../rfc/4731:31 ../rfc/4466:354 ../rfc/3501:2723
-	// Examples: ../rfc/9051:3986 ../rfc/7377:385 ../rfc/6237:323 ../rfc/4731:153 ../rfc/3501:2975
+	// Examples: ../rfc/9051:3986 ../rfc/4731:153 ../rfc/3501:2975
-	// Syntax: ../rfc/9051:6918 ../rfc/7377:462 ../rfc/6237:403 ../rfc/4466:611 ../rfc/3501:4954
+	// Syntax: ../rfc/9051:6918 ../rfc/4466:611 ../rfc/3501:4954
 
-	// We will respond with ESEARCH instead of SEARCH if "RETURN" is present or for IMAP4rev2 or for isE (ESEARCH command).
+	// We will respond with ESEARCH instead of SEARCH if "RETURN" is present or for IMAP4rev2.
 	var eargs map[string]bool // Options except SAVE. Nil means old-style SEARCH response.
 	var save bool             // For SAVE option. Kept separately for easier handling of MIN/MAX later.
 
-	if c.enabled[capIMAP4rev2] || isE {
+	// IMAP4rev2 always returns ESEARCH, even with absent RETURN.
+	if c.enabled[capIMAP4rev2] {
 		eargs = map[string]bool{}
 	}
-
-	// The ESEARCH command has various ways to specify which mailboxes are to be
-	// searched. We parse and gather the request first, and evaluate them to mailboxes
-	// after parsing, when we start and have a DB transaction.
-	var mailboxSpecs []mailboxSpecifier
-
-	// ../rfc/7377:468
-	if isE && p.take(" IN (") {
-		for {
-			ms := p.xfilterMailbox(mbspecsEsearch)
-			mailboxSpecs = append(mailboxSpecs, ms)
-
-			if !p.take(" ") {
-				break
-			}
-		}
-		p.xtake(")")
-		// We are not parsing the scope-options since there aren't any defined yet. ../rfc/7377:469
-	}
 	// ../rfc/9051:6967
 	if p.take(" RETURN (") {
 		eargs = map[string]bool{}
@@ -107,11 +71,6 @@ func (c *conn) cmdxSearch(isUID, isE bool, tag, cmd string, p *parser) {
 		sk.searchKeys = append(sk.searchKeys, *p.xsearchKey())
 	}
 
-	// Sequence set search program must be rejected with UIDONLY enabled. ../rfc/9586:220
-	if c.uidonly && sk.hasSequenceNumbers() {
-		xsyntaxCodeErrorf("UIDREQUIRED", "cannot search message sequence numbers in search program with uidonly enabled")
-	}
-
 	// Even in case of error, we ensure search result is changed.
 	if save {
 		c.searchResult = []store.UID{}
@@ -166,344 +125,68 @@ func (c *conn) cmdxSearch(isUID, isE bool, tag, cmd string, p *parser) {
 
 	// If we only have a MIN and/or MAX, we can stop processing as soon as we
 	// have those matches.
-	var min1, max1 int
+	var min, max int
 	if eargs["MIN"] {
-		min1 = 1
+		min = 1
 	}
 	if eargs["MAX"] {
-		max1 = 1
+		max = 1
 	}
 
-	// We'll have one Result per mailbox we are searching. For regular (UID) SEARCH
+	var expungeIssued bool
-	// commands, we'll have just one, for the selected mailbox.
+	var maxModSeq store.ModSeq
-	type Result struct {
-		Mailbox   store.Mailbox
-		MaxModSeq store.ModSeq
-		UIDs      []store.UID
-	}
-	var results []Result
-
-	// We periodically send an untagged OK with INPROGRESS code while searching, to let
-	// clients doing slow searches know we're still working.
-	inProgressLast := time.Now()
-	// Only respond with tag if it can't be confused as end of response code. ../rfc/9585:122
-	inProgressTag := "nil"
-	if !strings.Contains(tag, "]") {
-		inProgressTag = dquote(tag).pack(c)
-	}
 
+	var uids []store.UID
 	c.xdbread(func(tx *bstore.Tx) {
-		// Gather mailboxes to operate on. Usually just the selected mailbox. But with the
+		c.xmailboxID(tx, c.mailboxID) // Validate.
-		// ESEARCH command, we may be searching multiple.
-		var mailboxes []store.Mailbox
-		if len(mailboxSpecs) > 0 {
-			// While gathering, we deduplicate mailboxes. ../rfc/7377:312
-			m := map[int64]store.Mailbox{}
-			for _, ms := range mailboxSpecs {
-				switch ms.Kind {
-				case mbspecSelected:
-					// ../rfc/7377:306
-					if c.state != stateSelected {
-						xsyntaxErrorf("cannot use ESEARCH with selected when state is not selected")
-					}
-
-					mb := c.xmailboxID(tx, c.mailboxID) // Validate.
-					m[mb.ID] = mb
-
-				case mbspecInboxes:
-					// Inbox and everything below. And we look at destinations and rulesets. We all
-					// mailboxes from the destinations, and all from the rulesets except when
-					// ListAllowDomain is non-empty.
-					// ../rfc/5465:822
-					q := bstore.QueryTx[store.Mailbox](tx)
-					q.FilterEqual("Expunged", false)
-					q.FilterGreaterEqual("Name", "Inbox")
-					q.SortAsc("Name")
-					for mb, err := range q.All() {
-						xcheckf(err, "list mailboxes")
-						if mb.Name != "Inbox" && !strings.HasPrefix(mb.Name, "Inbox/") {
-							break
-						}
-						m[mb.ID] = mb
-					}
-
-					conf, _ := c.account.Conf()
-					for _, dest := range conf.Destinations {
-						if dest.Mailbox != "" && dest.Mailbox != "Inbox" {
-							mb, err := c.account.MailboxFind(tx, dest.Mailbox)
-							xcheckf(err, "find mailbox from destination")
-							if mb != nil {
-								m[mb.ID] = *mb
-							}
-						}
-
-						for _, rs := range dest.Rulesets {
-							if rs.ListAllowDomain != "" || rs.Mailbox == "" {
-								continue
-							}
-
-							mb, err := c.account.MailboxFind(tx, rs.Mailbox)
-							xcheckf(err, "find mailbox from ruleset")
-							if mb != nil {
-								m[mb.ID] = *mb
-							}
-						}
-					}
-
-				case mbspecPersonal:
-					// All mailboxes in the personal namespace. Which is all mailboxes for us.
-					// ../rfc/5465:817
-					for mb, err := range bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).All() {
-						xcheckf(err, "list mailboxes")
-						m[mb.ID] = mb
-					}
-
-				case mbspecSubscribed:
-					// Mailboxes that are subscribed. Will typically be same as personal, since we
-					// subscribe to all mailboxes. But user can manage subscriptions differently.
-					// ../rfc/5465:831
-					for mb, err := range bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).All() {
-						xcheckf(err, "list mailboxes")
-						if err := tx.Get(&store.Subscription{Name: mb.Name}); err == nil {
-							m[mb.ID] = mb
-						} else if err != bstore.ErrAbsent {
-							xcheckf(err, "lookup subscription for mailbox")
-						}
-					}
-
-				case mbspecSubtree, mbspecSubtreeOne:
-					// The mailbox name itself, and children. ../rfc/5465:847
-					// SUBTREE is arbitrarily deep, SUBTREE-ONE is one level deeper than requested
-					// mailbox. The mailbox itself is included too ../rfc/7377:274
-
-					// We don't have to worry about loops. Mailboxes are not in the file system.
-					// ../rfc/7377:291
-
-					for _, name := range ms.Mailboxes {
-						name = xcheckmailboxname(name, true)
-
-						one := ms.Kind == mbspecSubtreeOne
-						var ntoken int
-						if one {
-							ntoken = len(strings.Split(name, "/")) + 1
-						}
-
-						q := bstore.QueryTx[store.Mailbox](tx)
-						q.FilterEqual("Expunged", false)
-						q.FilterGreaterEqual("Name", name)
-						q.SortAsc("Name")
-						for mb, err := range q.All() {
-							xcheckf(err, "list mailboxes")
-							if mb.Name != name && !strings.HasPrefix(mb.Name, name+"/") {
-								break
-							}
-							if !one || mb.Name == name || len(strings.Split(mb.Name, "/")) == ntoken {
-								m[mb.ID] = mb
-							}
-						}
-					}
-
-				case mbspecMailboxes:
-					// Just the specified mailboxes. ../rfc/5465:853
-					for _, name := range ms.Mailboxes {
-						name = xcheckmailboxname(name, true)
-
-						// If a mailbox doesn't exist, we don't treat it as an error. Seems reasonable
-						// giving we are searching. Messages may not exist. And likewise for the mailbox.
-						// Just results in no hits.
-						mb, err := c.account.MailboxFind(tx, name)
-						xcheckf(err, "looking up mailbox")
-						if mb != nil {
-							m[mb.ID] = *mb
-						}
-					}
-
-				default:
-					panic("missing case")
-				}
-			}
-			mailboxes = slices.Collect(maps.Values(m))
-			slices.SortFunc(mailboxes, func(a, b store.Mailbox) int {
-				return cmp.Compare(a.Name, b.Name)
-			})
-
-			// If no source mailboxes were specified (no mailboxSpecs), the selected mailbox is
-			// used below. ../rfc/7377:298
-		} else {
-			mb := c.xmailboxID(tx, c.mailboxID) // Validate.
-			mailboxes = []store.Mailbox{mb}
-		}
-
-		if save && !(len(mailboxes) == 1 && mailboxes[0].ID == c.mailboxID) {
-			// ../rfc/7377:319
-			xsyntaxErrorf("can only use SAVE on selected mailbox")
-		}
-
 		runlock()
 		runlock = func() {}
 
-		// Determine if search has a sequence set without search results. If so, we need
+		// Normal forward search when we don't have MAX only.
-		// sequence numbers for matching, and we must always go through the messages in
+		var lastIndex = -1
-		// forward order. No reverse search for MAX only.
+		if eargs == nil || max == 0 || len(eargs) != 1 {
-		needSeq := (len(mailboxes) > 1 || len(mailboxes) == 1 && mailboxes[0].ID != c.mailboxID) && sk.hasSequenceNumbers()
+			for i, uid := range c.uids {
-
+				lastIndex = i
-		forward := eargs == nil || max1 == 0 || len(eargs) != 1 || needSeq
+				if match, modseq := c.searchMatch(tx, msgseq(i+1), uid, *sk, bodySearch, textSearch, &expungeIssued); match {
-		reverse := max1 == 1 && (len(eargs) == 1 || min1+max1 == len(eargs)) && !needSeq
+					uids = append(uids, uid)
-
+					if modseq > maxModSeq {
-		// We set a worst-case "goal" of having gone through all messages in all mailboxes.
+						maxModSeq = modseq
-		// Sometimes, we can be faster, when we only do a MIN and/or MAX query and we can
-		// stop early. We'll account for that as we go. For the selected mailbox, we'll
-		// only look at those the session has already seen.
-		goal := "nil"
-		var total uint32
-		for _, mb := range mailboxes {
-			if mb.ID == c.mailboxID && !c.uidonly {
-				total += c.exists
-			} else {
-				total += uint32(mb.Total + mb.Deleted)
-			}
-		}
-		if total > 0 {
-			// Goal is always non-zero. ../rfc/9585:232
-			goal = fmt.Sprintf("%d", total)
-		}
-
-		var progress uint32
-		for _, mb := range mailboxes {
-			var lastUID store.UID
-
-			result := Result{Mailbox: mb}
-
-			msgCount := uint32(mb.MailboxCounts.Total + mb.MailboxCounts.Deleted)
-			if mb.ID == c.mailboxID && !c.uidonly {
-				msgCount = c.exists
-			}
-
-			// Used for interpreting UID sets with a star, like "1:*" and "10:*". Only called
-			// for UIDs that are higher than the number, since "10:*" evaluates to "10:5" if 5
-			// is the highest UID, and UID 5-10 would all match.
-			var cachedHighestUID store.UID
-			xhighestUID := func() store.UID {
-				if cachedHighestUID > 0 {
-					return cachedHighestUID
-				}
-
-				q := bstore.QueryTx[store.Message](tx)
-				q.FilterNonzero(store.Message{MailboxID: mb.ID})
-				q.FilterEqual("Expunged", false)
-				if mb.ID == c.mailboxID {
-					q.FilterLess("UID", c.uidnext)
-				}
-				q.SortDesc("UID")
-				q.Limit(1)
-				m, err := q.Get()
-				if err == bstore.ErrAbsent {
-					xuserErrorf("cannot use * on empty mailbox")
-				}
-				xcheckf(err, "get last uid")
-				cachedHighestUID = m.UID
-				return cachedHighestUID
-			}
-
-			progressOrig := progress
-
-			if forward {
-				// We track this for non-selected mailboxes. searchMatch will look the message
-				// sequence number for this session up if we are searching the selected mailbox.
-				var seq msgseq = 1
-
-				q := bstore.QueryTx[store.Message](tx)
-				q.FilterNonzero(store.Message{MailboxID: mb.ID})
-				q.FilterEqual("Expunged", false)
-				if mb.ID == c.mailboxID {
-					q.FilterLess("UID", c.uidnext)
-				}
-				q.SortAsc("UID")
-				for m, err := range q.All() {
-					xcheckf(err, "list messages in mailbox")
-
-					// We track this for the "reverse" case, we'll stop before seeing lastUID.
-					lastUID = m.UID
-
-					if time.Since(inProgressLast) > inProgressPeriod {
-						c.xwritelinef("* OK [INPROGRESS (%s %d %s)] still searching", inProgressTag, progress, goal)
-						inProgressLast = time.Now()
 					}
-					progress++
+					if min == 1 && min+max == len(eargs) {
-
-					if c.searchMatch(tx, msgCount, seq, m, *sk, bodySearch, textSearch, xhighestUID) {
-						result.UIDs = append(result.UIDs, m.UID)
-						result.MaxModSeq = max(result.MaxModSeq, m.ModSeq)
-						if min1 == 1 && min1+max1 == len(eargs) {
-							if !needSeq {
-								break
-							}
-							// We only need a MIN and a MAX, but we also need sequence numbers so we are
-							// walking through and collecting all UIDs. Correct for that, keeping only the MIN
-							// (first)
-							// and MAX (second).
-							if len(result.UIDs) == 3 {
-								result.UIDs[1] = result.UIDs[2]
-								result.UIDs = result.UIDs[:2]
-							}
-						}
-					}
-					seq++
-				}
-			}
-			// And reverse search for MAX if we have only MAX or MAX combined with MIN, and
-			// don't need sequence numbers. We just need a single match, then we stop.
-			if reverse {
-				q := bstore.QueryTx[store.Message](tx)
-				q.FilterNonzero(store.Message{MailboxID: mb.ID})
-				q.FilterEqual("Expunged", false)
-				q.FilterGreater("UID", lastUID)
-				if mb.ID == c.mailboxID {
-					q.FilterLess("UID", c.uidnext)
-				}
-				q.SortDesc("UID")
-				for m, err := range q.All() {
-					xcheckf(err, "list messages in mailbox")
-
-					if time.Since(inProgressLast) > inProgressPeriod {
-						c.xwritelinef("* OK [INPROGRESS (%s %d %s)] still searching", inProgressTag, progress, goal)
-						inProgressLast = time.Now()
-					}
-					progress++
-
-					var seq msgseq // Filled in by searchMatch for messages in selected mailbox.
-					if c.searchMatch(tx, msgCount, seq, m, *sk, bodySearch, textSearch, xhighestUID) {
-						result.UIDs = append(result.UIDs, m.UID)
-						result.MaxModSeq = max(result.MaxModSeq, m.ModSeq)
 						break
 					}
 				}
 			}
-
+		}
-			// We could have finished searching the mailbox with fewer
+		// And reverse search for MAX if we have only MAX or MAX combined with MIN.
-			mailboxProcessed := progress - progressOrig
+		if max == 1 && (len(eargs) == 1 || min+max == len(eargs)) {
-			mailboxTotal := uint32(mb.MailboxCounts.Total + mb.MailboxCounts.Deleted)
+			for i := len(c.uids) - 1; i > lastIndex; i-- {
-			progress += max(0, mailboxTotal-mailboxProcessed)
+				if match, modseq := c.searchMatch(tx, msgseq(i+1), c.uids[i], *sk, bodySearch, textSearch, &expungeIssued); match {
-
+					uids = append(uids, c.uids[i])
-			results = append(results, result)
+					if modseq > maxModSeq {
+						maxModSeq = modseq
+					}
+					break
+				}
+			}
 		}
 	})
 
 	if eargs == nil {
-		// We'll only have a result for the one selected mailbox.
-		result := results[0]
-
 		// In IMAP4rev1, an untagged SEARCH response is required. ../rfc/3501:2728
-		if len(result.UIDs) == 0 {
+		if len(uids) == 0 {
-			c.xbwritelinef("* SEARCH")
+			c.bwritelinef("* SEARCH")
 		}
 
 		// Old-style SEARCH response. We must spell out each number. So we may be splitting
 		// into multiple responses. ../rfc/9051:6809 ../rfc/3501:4833
-		for len(result.UIDs) > 0 {
+		for len(uids) > 0 {
-			n := min(100, len(result.UIDs))
+			n := len(uids)
+			if n > 100 {
+				n = 100
+			}
 			s := ""
-			for _, v := range result.UIDs[:n] {
+			for _, v := range uids[:n] {
 				if !isUID {
 					v = store.UID(c.xsequence(v))
 				}
@@ -519,111 +202,91 @@ func (c *conn) cmdxSearch(isUID, isE bool, tag, cmd string, p *parser) {
 			var modseq string
 			if sk.hasModseq() {
 				// ../rfc/7162:2557
-				modseq = fmt.Sprintf(" (MODSEQ %d)", result.MaxModSeq.Client())
+				modseq = fmt.Sprintf(" (MODSEQ %d)", maxModSeq.Client())
 			}
 
-			c.xbwritelinef("* SEARCH%s%s", s, modseq)
+			c.bwritelinef("* SEARCH%s%s", s, modseq)
-			result.UIDs = result.UIDs[n:]
+			uids = uids[n:]
 		}
 	} else {
 		// New-style ESEARCH response syntax: ../rfc/9051:6546 ../rfc/4466:522
 
 		if save {
 			// ../rfc/9051:3784 ../rfc/5182:13
-			c.searchResult = results[0].UIDs
+			c.searchResult = uids
-			c.checkUIDs(c.searchResult, false)
+			if sanityChecks {
+				checkUIDs(c.searchResult)
+			}
 		}
 
 		// No untagged ESEARCH response if nothing was requested. ../rfc/9051:4160
 		if len(eargs) > 0 {
-			for _, result := range results {
+			// The tag was originally a string, became an astring in IMAP4rev2, better stick to
-				// For the ESEARCH command, we must not return a response if there were no matching
+			// string. ../rfc/4466:707 ../rfc/5259:1163 ../rfc/9051:7087
-				// messages. This is unlike the later IMAP4rev2, where an ESEARCH response must be
+			resp := fmt.Sprintf(`* ESEARCH (TAG "%s")`, tag)
-				// sent if there were no matches. ../rfc/7377:243 ../rfc/9051:3775
+			if isUID {
-				if isE && len(result.UIDs) == 0 {
+				resp += " UID"
-					continue
-				}
-
-				// The tag was originally a string, became an astring in IMAP4rev2, better stick to
-				// string. ../rfc/4466:707 ../rfc/5259:1163 ../rfc/9051:7087
-				if isE {
-					fmt.Fprintf(c.xbw, `* ESEARCH (TAG "%s" MAILBOX %s UIDVALIDITY %d)`, tag, result.Mailbox.Name, result.Mailbox.UIDValidity)
-				} else {
-					fmt.Fprintf(c.xbw, `* ESEARCH (TAG "%s")`, tag)
-				}
-				if isUID {
-					fmt.Fprintf(c.xbw, " UID")
-				}
-
-				// NOTE: we are potentially converting UIDs to msgseq, but keep the store.UID type
-				// for convenience.
-				nums := result.UIDs
-				if !isUID {
-					// If searchResult is hanging on to the slice, we need to work on a copy.
-					if save {
-						nums = slices.Clone(nums)
-					}
-					for i, uid := range nums {
-						nums[i] = store.UID(c.xsequence(uid))
-					}
-				}
-
-				// If no matches, then no MIN/MAX response. ../rfc/4731:98 ../rfc/9051:3758
-				if eargs["MIN"] && len(nums) > 0 {
-					fmt.Fprintf(c.xbw, " MIN %d", nums[0])
-				}
-				if eargs["MAX"] && len(result.UIDs) > 0 {
-					fmt.Fprintf(c.xbw, " MAX %d", nums[len(nums)-1])
-				}
-				if eargs["COUNT"] {
-					fmt.Fprintf(c.xbw, " COUNT %d", len(nums))
-				}
-				if eargs["ALL"] && len(nums) > 0 {
-					fmt.Fprintf(c.xbw, " ALL %s", compactUIDSet(nums).String())
-				}
-
-				// Interaction between ESEARCH and CONDSTORE: ../rfc/7162:1211 ../rfc/4731:273
-				// Summary: send the highest modseq of the returned messages.
-				if sk.hasModseq() && len(nums) > 0 {
-					fmt.Fprintf(c.xbw, " MODSEQ %d", result.MaxModSeq.Client())
-				}
-
-				c.xbwritelinef("")
 			}
+
+			// NOTE: we are converting UIDs to msgseq in the uids slice (if needed) while
+			// keeping the "uids" name!
+			if !isUID {
+				// If searchResult is hanging on to the slice, we need to work on a copy.
+				if save {
+					nuids := make([]store.UID, len(uids))
+					copy(nuids, uids)
+					uids = nuids
+				}
+				for i, uid := range uids {
+					uids[i] = store.UID(c.xsequence(uid))
+				}
+			}
+
+			// If no matches, then no MIN/MAX response. ../rfc/4731:98 ../rfc/9051:3758
+			if eargs["MIN"] && len(uids) > 0 {
+				resp += fmt.Sprintf(" MIN %d", uids[0])
+			}
+			if eargs["MAX"] && len(uids) > 0 {
+				resp += fmt.Sprintf(" MAX %d", uids[len(uids)-1])
+			}
+			if eargs["COUNT"] {
+				resp += fmt.Sprintf(" COUNT %d", len(uids))
+			}
+			if eargs["ALL"] && len(uids) > 0 {
+				resp += fmt.Sprintf(" ALL %s", compactUIDSet(uids).String())
+			}
+
+			// Interaction between ESEARCH and CONDSTORE: ../rfc/7162:1211 ../rfc/4731:273
+			// Summary: send the highest modseq of the returned messages.
+			if sk.hasModseq() && len(uids) > 0 {
+				resp += fmt.Sprintf(" MODSEQ %d", maxModSeq.Client())
+			}
+
+			c.bwritelinef("%s", resp)
 		}
 	}
-
+	if expungeIssued {
-	c.ok(tag, cmd)
+		// ../rfc/9051:5102
+		c.writeresultf("%s OK [EXPUNGEISSUED] done", tag)
+	} else {
+		c.ok(tag, cmd)
+	}
 }
 
 type search struct {
-	c           *conn
+	c             *conn
-	tx          *bstore.Tx
+	tx            *bstore.Tx
-	msgCount    uint32 // Number of messages in mailbox (or session when selected).
+	seq           msgseq
-	seq         msgseq // Can be 0, for other mailboxes than selected in case of MAX.
+	uid           store.UID
-	m           store.Message
+	mr            *store.MsgReader
-	mr          *store.MsgReader
+	m             store.Message
-	p           *message.Part
+	p             *message.Part
-	xhighestUID func() store.UID
+	expungeIssued *bool
+	hasModseq     bool
 }
 
-func (c *conn) searchMatch(tx *bstore.Tx, msgCount uint32, seq msgseq, m store.Message, sk searchKey, bodySearch, textSearch *store.WordSearch, xhighestUID func() store.UID) bool {
+func (c *conn) searchMatch(tx *bstore.Tx, seq msgseq, uid store.UID, sk searchKey, bodySearch, textSearch *store.WordSearch, expungeIssued *bool) (bool, store.ModSeq) {
-	if m.MailboxID == c.mailboxID {
+	s := search{c: c, tx: tx, seq: seq, uid: uid, expungeIssued: expungeIssued, hasModseq: sk.hasModseq()}
-		// If session doesn't know about the message yet, don't return it.
-		if c.uidonly {
-			if m.UID >= c.uidnext {
-				return false
-			}
-		} else {
-			// Set seq for use in evaluations.
-			seq = c.sequence(m.UID)
-			if seq == 0 {
-				return false
-			}
-		}
-	}
-
-	s := search{c: c, tx: tx, msgCount: msgCount, seq: seq, m: m, xhighestUID: xhighestUID}
 	defer func() {
 		if s.mr != nil {
 			err := s.mr.Close()
@@ -634,7 +297,18 @@ func (c *conn) searchMatch(tx *bstore.Tx, msgCount uint32, seq msgseq, m store.M
 	return s.match(sk, bodySearch, textSearch)
 }
 
-func (s *search) match(sk searchKey, bodySearch, textSearch *store.WordSearch) (match bool) {
+func (s *search) match(sk searchKey, bodySearch, textSearch *store.WordSearch) (match bool, modseq store.ModSeq) {
+	// Instead of littering all the cases in match0 with calls to get modseq, we do it once
+	// here in case of a match.
+	defer func() {
+		if match && s.hasModseq {
+			if s.m.ID == 0 {
+				match = s.xensureMessage()
+			}
+			modseq = s.m.ModSeq
+		}
+	}()
+
 	match = s.match0(sk)
 	if match && bodySearch != nil {
 		if !s.xensurePart() {
@@ -657,6 +331,24 @@ func (s *search) match(sk searchKey, bodySearch, textSearch *store.WordSearch) (
 	return
 }
 
+func (s *search) xensureMessage() bool {
+	if s.m.ID > 0 {
+		return true
+	}
+
+	q := bstore.QueryTx[store.Message](s.tx)
+	q.FilterNonzero(store.Message{MailboxID: s.c.mailboxID, UID: s.uid})
+	m, err := q.Get()
+	if err == bstore.ErrAbsent || err == nil && m.Expunged {
+		// ../rfc/2180:607
+		*s.expungeIssued = true
+		return false
+	}
+	xcheckf(err, "get message")
+	s.m = m
+	return true
+}
+
 // ensure message, reader and part are loaded. returns whether that was
 // successful.
 func (s *search) xensurePart() bool {
@@ -664,6 +356,10 @@ func (s *search) xensurePart() bool {
 		return s.p != nil
 	}
 
+	if !s.xensureMessage() {
+		return false
+	}
+
 	// Closed by searchMatch after all (recursive) search.match calls are finished.
 	s.mr = s.c.account.MessageReader(s.m)
 
@@ -690,23 +386,14 @@ func (s *search) match0(sk searchKey) bool {
 		}
 		return true
 	} else if sk.seqSet != nil {
-		if sk.seqSet.searchResult {
+		return sk.seqSet.containsSeq(s.seq, c.uids, c.searchResult)
-			// Interpreting search results on a mailbox that isn't selected during multisearch
-			// is likely a mistake. No mention about it in the RFC. ../rfc/7377:257
-			if s.m.MailboxID != c.mailboxID {
-				xuserErrorf("can only use search result with the selected mailbox")
-			}
-			return uidSearch(c.searchResult, s.m.UID) > 0
-		}
-		// For multisearch, we have arranged to have a seq for non-selected mailboxes too.
-		return sk.seqSet.containsSeqCount(s.seq, s.msgCount)
 	}
 
 	filterHeader := func(field, value string) bool {
 		lower := strings.ToLower(value)
 		h, err := s.p.Header()
 		if err != nil {
-			c.log.Debugx("parsing message header", err, slog.Any("uid", s.m.UID), slog.Int64("msgid", s.m.ID))
+			c.log.Debugx("parsing message header", err, slog.Any("uid", s.uid))
 			return false
 		}
 		for _, v := range h.Values(field) {
@@ -736,12 +423,7 @@ func (s *search) match0(sk searchKey) bool {
 	case "OR":
 		return s.match0(*sk.searchKey) || s.match0(*sk.searchKey2)
 	case "UID":
-		if sk.uidSet.searchResult && s.m.MailboxID != c.mailboxID {
+		return sk.uidSet.containsUID(s.uid, c.uids, c.searchResult)
-			// Interpreting search results on a mailbox that isn't selected during multisearch
-			// is likely a mistake. No mention about it in the RFC. ../rfc/7377:257
-			xuserErrorf("cannot use search result from another mailbox")
-		}
-		return sk.uidSet.xcontainsKnownUID(s.m.UID, c.searchResult, s.xhighestUID)
 	}
 
 	// Parsed part.
@@ -771,7 +453,12 @@ func (s *search) match0(sk searchKey) bool {
 		case "$mdnsent":
 			return s.m.MDNSent
 		default:
-			return slices.Contains(s.m.Keywords, kw)
+			for _, k := range s.m.Keywords {
+				if k == kw {
+					return true
+				}
+			}
+			return false
 		}
 	case "SEEN":
 		return s.m.Seen
@@ -795,7 +482,12 @@ func (s *search) match0(sk searchKey) bool {
 		case "$mdnsent":
 			return !s.m.MDNSent
 		default:
-			return !slices.Contains(s.m.Keywords, kw)
+			for _, k := range s.m.Keywords {
+				if k == kw {
+					return false
+				}
+			}
+			return true
 		}
 	case "UNSEEN":
 		return !s.m.Seen
@@ -822,42 +514,10 @@ func (s *search) match0(sk searchKey) bool {
 	case "MODSEQ":
 		// ../rfc/7162:1045
 		return s.m.ModSeq.Client() >= *sk.clientModseq
-	case "SAVEDBEFORE", "SAVEDON", "SAVEDSINCE":
-		// If we don't have a savedate for this message (for messages received before we
-		// implemented this feature), we use the "internal date" (received timestamp) of
-		// the message. ../rfc/8514:237
-		rt := s.m.Received
-		if s.m.SaveDate != nil {
-			rt = *s.m.SaveDate
-		}
-
-		skdt := sk.date.Format("2006-01-02")
-		rdt := rt.Format("2006-01-02")
-		switch sk.op {
-		case "SAVEDBEFORE":
-			return rdt < skdt
-		case "SAVEDON":
-			return rdt == skdt
-		case "SAVEDSINCE":
-			return rdt >= skdt
-		}
-		panic("missing case")
-	case "SAVEDATESUPPORTED":
-		// We return whether we have a savedate for this message. We support it on all
-		// mailboxes, but we only have this metadata from the time we implemented this
-		// feature.
-		return s.m.SaveDate != nil
-	case "OLDER":
-		// ../rfc/5032:76
-		seconds := int64(time.Since(s.m.Received) / time.Second)
-		return seconds >= sk.number
-	case "YOUNGER":
-		seconds := int64(time.Since(s.m.Received) / time.Second)
-		return seconds <= sk.number
 	}
 
 	if s.p == nil {
-		c.log.Info("missing parsed message, not matching", slog.Any("uid", s.m.UID), slog.Int64("msgid", s.m.ID))
+		c.log.Info("missing parsed message, not matching", slog.Any("uid", s.uid))
 		return false
 	}
 
@@ -886,7 +546,7 @@ func (s *search) match0(sk searchKey) bool {
 		lower := strings.ToLower(sk.astring)
 		h, err := s.p.Header()
 		if err != nil {
-			c.log.Errorx("parsing header for search", err, slog.Any("uid", s.m.UID), slog.Int64("msgid", s.m.ID))
+			c.log.Errorx("parsing header for search", err, slog.Any("uid", s.uid))
 			return false
 		}
 		k := textproto.CanonicalMIMEHeaderKey(sk.headerField)

imapserver/search_test.go

@@ -34,10 +34,6 @@ this is html.
 --x--
 `, "\n", "\r\n")
 
-func uint32ptr(v uint32) *uint32 {
-	return &v
-}
-
 func (tc *testconn) xsearch(nums ...uint32) {
 	tc.t.Helper()
 
@@ -57,35 +53,26 @@ func (tc *testconn) xsearchmodseq(modseq int64, nums ...uint32) {
 func (tc *testconn) xesearch(exp imapclient.UntaggedEsearch) {
 	tc.t.Helper()
 
-	exp.Tag = tc.client.LastTag()
+	exp.Correlator = tc.client.LastTag
 	tc.xuntagged(exp)
 }
 
 func TestSearch(t *testing.T) {
-	testSearch(t, false)
+	tc := start(t)
-}
-
-func TestSearchUIDOnly(t *testing.T) {
-	testSearch(t, true)
-}
-
-func testSearch(t *testing.T, uidonly bool) {
-	tc := start(t, uidonly)
 	defer tc.close()
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
 	tc.client.Select("inbox")
 
 	// Add 5 and delete first 4 messages. So UIDs start at 5.
 	received := time.Date(2020, time.January, 1, 10, 0, 0, 0, time.UTC)
-	saveDate := time.Now()
+	for i := 0; i < 5; i++ {
-	for range 5 {
+		tc.client.Append("inbox", nil, &received, []byte(exampleMsg))
-		tc.client.Append("inbox", makeAppendTime(exampleMsg, received))
 	}
-	tc.client.UIDStoreFlagsSet("1:4", true, `\Deleted`)
+	tc.client.StoreFlagsSet("1:4", true, `\Deleted`)
 	tc.client.Expunge()
 
 	received = time.Date(2022, time.January, 1, 9, 0, 0, 0, time.UTC)
-	tc.client.Append("inbox", makeAppendTime(searchMsg, received))
+	tc.client.Append("inbox", nil, &received, []byte(searchMsg))
 
 	received = time.Date(2022, time.January, 1, 9, 0, 0, 0, time.UTC)
 	mostFlags := []string{
@@ -102,315 +89,256 @@ func testSearch(t *testing.T, uidonly bool) {
 		`custom1`,
 		`Custom2`,
 	}
-	tc.client.Append("inbox", imapclient.Append{Flags: mostFlags, Received: &received, Size: int64(len(searchMsg)), Data: strings.NewReader(searchMsg)})
+	tc.client.Append("inbox", mostFlags, &received, []byte(searchMsg))
 
 	// We now have sequence numbers 1,2,3 and UIDs 5,6,7.
 
-	if uidonly {
+	tc.transactf("ok", "search all")
-		// We need to be selected. Not the case for ESEARCH command.
+	tc.xsearch(1, 2, 3)
-		tc.client.Unselect()
-		tc.transactf("no", "uid search all")
-		tc.client.Select("inbox")
-	} else {
-		// We need to be selected. Not the case for ESEARCH command.
-		tc.client.Unselect()
-		tc.transactf("no", "search all")
-		tc.client.Select("inbox")
-
-		tc.transactf("ok", "search all")
-		tc.xsearch(1, 2, 3)
-	}
 
 	tc.transactf("ok", "uid search all")
 	tc.xsearch(5, 6, 7)
 
+	tc.transactf("ok", "search answered")
+	tc.xsearch(3)
+
+	tc.transactf("ok", `search bcc "bcc@mox.example"`)
+	tc.xsearch(2, 3)
+
+	tc.transactf("ok", "search before 1-Jan-2038")
+	tc.xsearch(1, 2, 3)
+	tc.transactf("ok", "search before 1-Jan-2020")
+	tc.xsearch() // Before is about received, not date header of message.
+
+	tc.transactf("ok", `search body "Joe"`)
+	tc.xsearch(1)
+	tc.transactf("ok", `search body "Joe" body "bogus"`)
+	tc.xsearch()
+	tc.transactf("ok", `search body "Joe" text "Blurdybloop"`)
+	tc.xsearch(1)
+	tc.transactf("ok", `search body "Joe" not text "mox"`)
+	tc.xsearch(1)
+	tc.transactf("ok", `search body "Joe" not not body "Joe"`)
+	tc.xsearch(1)
+	tc.transactf("ok", `search body "this is plain text"`)
+	tc.xsearch(2, 3)
+	tc.transactf("ok", `search body "this is html"`)
+	tc.xsearch(2, 3)
+
+	tc.transactf("ok", `search cc "xcc@mox.example"`)
+	tc.xsearch(2, 3)
+
+	tc.transactf("ok", `search deleted`)
+	tc.xsearch(3)
+
+	tc.transactf("ok", `search flagged`)
+	tc.xsearch(3)
+
+	tc.transactf("ok", `search from "foobar@Blurdybloop.example"`)
+	tc.xsearch(1)
+
+	tc.transactf("ok", `search keyword $Forwarded`)
+	tc.xsearch(3)
+
+	tc.transactf("ok", `search keyword Custom1`)
+	tc.xsearch(3)
+
+	tc.transactf("ok", `search keyword custom2`)
+	tc.xsearch(3)
+
+	tc.transactf("ok", `search new`)
+	tc.xsearch() // New requires a message to be recent. We pretend all messages are not recent.
+
+	tc.transactf("ok", `search old`)
+	tc.xsearch(1, 2, 3)
+
+	tc.transactf("ok", `search on 1-Jan-2022`)
+	tc.xsearch(2, 3)
+
+	tc.transactf("ok", `search recent`)
+	tc.xsearch()
+
+	tc.transactf("ok", `search seen`)
+	tc.xsearch(3)
+
+	tc.transactf("ok", `search since 1-Jan-2020`)
+	tc.xsearch(1, 2, 3)
+
+	tc.transactf("ok", `search subject "afternoon"`)
+	tc.xsearch(1)
+
+	tc.transactf("ok", `search text "Joe"`)
+	tc.xsearch(1)
+
+	tc.transactf("ok", `search to "mooch@owatagu.siam.edu.example"`)
+	tc.xsearch(1)
+
+	tc.transactf("ok", `search unanswered`)
+	tc.xsearch(1, 2)
+
+	tc.transactf("ok", `search undeleted`)
+	tc.xsearch(1, 2)
+
+	tc.transactf("ok", `search unflagged`)
+	tc.xsearch(1, 2)
+
+	tc.transactf("ok", `search unkeyword $Junk`)
+	tc.xsearch(1, 2)
+
+	tc.transactf("ok", `search unkeyword custom1`)
+	tc.xsearch(1, 2)
+
+	tc.transactf("ok", `search unseen`)
+	tc.xsearch(1, 2)
+
+	tc.transactf("ok", `search draft`)
+	tc.xsearch(3)
+
+	tc.transactf("ok", `search header "subject" "afternoon"`)
+	tc.xsearch(1)
+
+	tc.transactf("ok", `search larger 1`)
+	tc.xsearch(1, 2, 3)
+
+	tc.transactf("ok", `search not text "mox"`)
+	tc.xsearch(1)
+
+	tc.transactf("ok", `search or seen unseen`)
+	tc.xsearch(1, 2, 3)
+
+	tc.transactf("ok", `search or unseen seen`)
+	tc.xsearch(1, 2, 3)
+
+	tc.transactf("ok", `search sentbefore 8-Feb-1994`)
+	tc.xsearch(1)
+
+	tc.transactf("ok", `search senton 7-Feb-1994`)
+	tc.xsearch(1)
+
+	tc.transactf("ok", `search sentsince 6-Feb-1994`)
+	tc.xsearch(1, 2, 3)
+
+	tc.transactf("ok", `search smaller 9999999`)
+	tc.xsearch(1, 2, 3)
+
+	tc.transactf("ok", `search uid 1`)
+	tc.xsearch()
+
+	tc.transactf("ok", `search uid 5`)
+	tc.xsearch(1)
+
+	tc.transactf("ok", `search or larger 1000000 smaller 1`)
+	tc.xsearch()
+
+	tc.transactf("ok", `search undraft`)
+	tc.xsearch(1, 2)
+
+	tc.transactf("no", `search charset unknown text "mox"`)
+	tc.transactf("ok", `search charset us-ascii text "mox"`)
+	tc.xsearch(2, 3)
+	tc.transactf("ok", `search charset utf-8 text "mox"`)
+	tc.xsearch(2, 3)
+
 	esearchall := func(ss string) imapclient.UntaggedEsearch {
 		return imapclient.UntaggedEsearch{All: esearchall0(ss)}
 	}
 
-	if !uidonly {
+	uint32ptr := func(v uint32) *uint32 {
-		tc.transactf("ok", "search answered")
+		return &v
-		tc.xsearch(3)
-
-		tc.transactf("ok", `search bcc "bcc@mox.example"`)
-		tc.xsearch(2, 3)
-
-		tc.transactf("ok", "search before 1-Jan-2038")
-		tc.xsearch(1, 2, 3)
-		tc.transactf("ok", "search before 1-Jan-2020")
-		tc.xsearch() // Before is about received, not date header of message.
-
-		// WITHIN extension with OLDER & YOUNGER.
-		tc.transactf("ok", "search older 60")
-		tc.xsearch(1, 2, 3)
-		tc.transactf("ok", "search younger 60")
-		tc.xsearch()
-
-		// SAVEDATE extension.
-		tc.transactf("ok", "search savedbefore %s", saveDate.Add(24*time.Hour).Format("2-Jan-2006"))
-		tc.xsearch(1, 2, 3)
-		tc.transactf("ok", "search savedbefore %s", saveDate.Add(-24*time.Hour).Format("2-Jan-2006"))
-		tc.xsearch()
-		tc.transactf("ok", "search savedon %s", saveDate.Format("2-Jan-2006"))
-		tc.xsearch(1, 2, 3)
-		tc.transactf("ok", "search savedon %s", saveDate.Add(-24*time.Hour).Format("2-Jan-2006"))
-		tc.xsearch()
-		tc.transactf("ok", "search savedsince %s", saveDate.Add(-24*time.Hour).Format("2-Jan-2006"))
-		tc.xsearch(1, 2, 3)
-		tc.transactf("ok", "search savedsince %s", saveDate.Add(24*time.Hour).Format("2-Jan-2006"))
-		tc.xsearch()
-
-		tc.transactf("ok", `search body "Joe"`)
-		tc.xsearch(1)
-		tc.transactf("ok", `search body "Joe" body "bogus"`)
-		tc.xsearch()
-		tc.transactf("ok", `search body "Joe" text "Blurdybloop"`)
-		tc.xsearch(1)
-		tc.transactf("ok", `search body "Joe" not text "mox"`)
-		tc.xsearch(1)
-		tc.transactf("ok", `search body "Joe" not not body "Joe"`)
-		tc.xsearch(1)
-		tc.transactf("ok", `search body "this is plain text"`)
-		tc.xsearch(2, 3)
-		tc.transactf("ok", `search body "this is html"`)
-		tc.xsearch(2, 3)
-
-		tc.transactf("ok", `search cc "xcc@mox.example"`)
-		tc.xsearch(2, 3)
-
-		tc.transactf("ok", `search deleted`)
-		tc.xsearch(3)
-
-		tc.transactf("ok", `search flagged`)
-		tc.xsearch(3)
-
-		tc.transactf("ok", `search from "foobar@Blurdybloop.example"`)
-		tc.xsearch(1)
-
-		tc.transactf("ok", `search keyword $Forwarded`)
-		tc.xsearch(3)
-
-		tc.transactf("ok", `search keyword Custom1`)
-		tc.xsearch(3)
-
-		tc.transactf("ok", `search keyword custom2`)
-		tc.xsearch(3)
-
-		tc.transactf("ok", `search new`)
-		tc.xsearch() // New requires a message to be recent. We pretend all messages are not recent.
-
-		tc.transactf("ok", `search old`)
-		tc.xsearch(1, 2, 3)
-
-		tc.transactf("ok", `search on 1-Jan-2022`)
-		tc.xsearch(2, 3)
-
-		tc.transactf("ok", `search recent`)
-		tc.xsearch()
-
-		tc.transactf("ok", `search seen`)
-		tc.xsearch(3)
-
-		tc.transactf("ok", `search since 1-Jan-2020`)
-		tc.xsearch(1, 2, 3)
-
-		tc.transactf("ok", `search subject "afternoon"`)
-		tc.xsearch(1)
-
-		tc.transactf("ok", `search text "Joe"`)
-		tc.xsearch(1)
-
-		tc.transactf("ok", `search to "mooch@owatagu.siam.edu.example"`)
-		tc.xsearch(1)
-
-		tc.transactf("ok", `search unanswered`)
-		tc.xsearch(1, 2)
-
-		tc.transactf("ok", `search undeleted`)
-		tc.xsearch(1, 2)
-
-		tc.transactf("ok", `search unflagged`)
-		tc.xsearch(1, 2)
-
-		tc.transactf("ok", `search unkeyword $Junk`)
-		tc.xsearch(1, 2)
-
-		tc.transactf("ok", `search unkeyword custom1`)
-		tc.xsearch(1, 2)
-
-		tc.transactf("ok", `search unseen`)
-		tc.xsearch(1, 2)
-
-		tc.transactf("ok", `search draft`)
-		tc.xsearch(3)
-
-		tc.transactf("ok", `search header "subject" "afternoon"`)
-		tc.xsearch(1)
-
-		tc.transactf("ok", `search larger 1`)
-		tc.xsearch(1, 2, 3)
-
-		tc.transactf("ok", `search not text "mox"`)
-		tc.xsearch(1)
-
-		tc.transactf("ok", `search or seen unseen`)
-		tc.xsearch(1, 2, 3)
-
-		tc.transactf("ok", `search or unseen seen`)
-		tc.xsearch(1, 2, 3)
-
-		tc.transactf("ok", `search sentbefore 8-Feb-1994`)
-		tc.xsearch(1)
-
-		tc.transactf("ok", `search senton 7-Feb-1994`)
-		tc.xsearch(1)
-
-		tc.transactf("ok", `search sentsince 6-Feb-1994`)
-		tc.xsearch(1, 2, 3)
-
-		tc.transactf("ok", `search smaller 9999999`)
-		tc.xsearch(1, 2, 3)
-
-		tc.transactf("ok", `search uid 1`)
-		tc.xsearch()
-
-		tc.transactf("ok", `search uid 5`)
-		tc.xsearch(1)
-
-		tc.transactf("ok", `search or larger 1000000 smaller 1`)
-		tc.xsearch()
-
-		tc.transactf("ok", `search undraft`)
-		tc.xsearch(1, 2)
-
-		tc.transactf("no", `search charset unknown text "mox"`)
-		tc.transactf("ok", `search charset us-ascii text "mox"`)
-		tc.xsearch(2, 3)
-		tc.transactf("ok", `search charset utf-8 text "mox"`)
-		tc.xsearch(2, 3)
-
-		// Check for properly formed INPROGRESS response code.
-		orig := inProgressPeriod
-		inProgressPeriod = 0
-		tc.cmdf("tag1", "search undraft")
-		tc.response("ok")
-
-		inprogress := func(cur, goal uint32) imapclient.UntaggedResult {
-			return imapclient.UntaggedResult{
-				Status: "OK",
-				Code:   imapclient.CodeInProgress{Tag: "tag1", Current: &cur, Goal: &goal},
-				Text:   "still searching",
-			}
-		}
-		tc.xuntagged(
-			imapclient.UntaggedSearch([]uint32{1, 2}),
-			// Due to inProgressPeriod 0, we get an inprogress response for each message in the mailbox.
-			inprogress(0, 3),
-			inprogress(1, 3),
-			inprogress(2, 3),
-		)
-		inProgressPeriod = orig
-
-		// Do new-style ESEARCH requests with RETURN. We should get an ESEARCH response.
-		tc.transactf("ok", "search return () all")
-		tc.xesearch(esearchall("1:3")) // Without any options, "ALL" is implicit.
-
-		tc.transactf("ok", "search return (min max count all) all")
-		tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uint32ptr(3), All: esearchall0("1:3")})
-
-		tc.transactf("ok", "search return (min) all")
-		tc.xesearch(imapclient.UntaggedEsearch{Min: 1})
-
-		tc.transactf("ok", "search return (min) 3")
-		tc.xesearch(imapclient.UntaggedEsearch{Min: 3})
-
-		tc.transactf("ok", "search return (min) NOT all")
-		tc.xesearch(imapclient.UntaggedEsearch{}) // Min not present if no match.
-
-		tc.transactf("ok", "search return (max) all")
-		tc.xesearch(imapclient.UntaggedEsearch{Max: 3})
-
-		tc.transactf("ok", "search return (max) 1")
-		tc.xesearch(imapclient.UntaggedEsearch{Max: 1})
-
-		tc.transactf("ok", "search return (max) not all")
-		tc.xesearch(imapclient.UntaggedEsearch{}) // Max not present if no match.
-
-		tc.transactf("ok", "search return (min max) all")
-		tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3})
-
-		tc.transactf("ok", "search return (min max) 1")
-		tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1})
-
-		tc.transactf("ok", "search return (min max) not all")
-		tc.xesearch(imapclient.UntaggedEsearch{})
-
-		tc.transactf("ok", "search return (all) not all")
-		tc.xesearch(imapclient.UntaggedEsearch{}) // All not present if no match.
-
-		tc.transactf("ok", "search return (min max all) not all")
-		tc.xesearch(imapclient.UntaggedEsearch{})
-
-		tc.transactf("ok", "search return (min max all count) not all")
-		tc.xesearch(imapclient.UntaggedEsearch{Count: uint32ptr(0)})
-
-		tc.transactf("ok", "search return (min max count all) 1,3")
-		tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uint32ptr(2), All: esearchall0("1,3")})
-
-		tc.transactf("ok", "search return (min max count all) UID 5,7")
-		tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uint32ptr(2), All: esearchall0("1,3")})
 	}
 
+	// Do new-style ESEARCH requests with RETURN. We should get an ESEARCH response.
+	tc.transactf("ok", "search return () all")
+	tc.xesearch(esearchall("1:3")) // Without any options, "ALL" is implicit.
+
+	tc.transactf("ok", "search return (min max count all) all")
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uint32ptr(3), All: esearchall0("1:3")})
+
 	tc.transactf("ok", "UID search return (min max count all) all")
 	tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uint32ptr(3), All: esearchall0("5:7")})
 
-	if !uidonly {
+	tc.transactf("ok", "search return (min) all")
-		tc.transactf("ok", "uid search return (min max count all) 1,3")
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1})
-		tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uint32ptr(2), All: esearchall0("5,7")})
+
-	}
+	tc.transactf("ok", "search return (min) 3")
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 3})
+
+	tc.transactf("ok", "search return (min) NOT all")
+	tc.xesearch(imapclient.UntaggedEsearch{}) // Min not present if no match.
+
+	tc.transactf("ok", "search return (max) all")
+	tc.xesearch(imapclient.UntaggedEsearch{Max: 3})
+
+	tc.transactf("ok", "search return (max) 1")
+	tc.xesearch(imapclient.UntaggedEsearch{Max: 1})
+
+	tc.transactf("ok", "search return (max) not all")
+	tc.xesearch(imapclient.UntaggedEsearch{}) // Max not present if no match.
+
+	tc.transactf("ok", "search return (min max) all")
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3})
+
+	tc.transactf("ok", "search return (min max) 1")
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1})
+
+	tc.transactf("ok", "search return (min max) not all")
+	tc.xesearch(imapclient.UntaggedEsearch{})
+
+	tc.transactf("ok", "search return (all) not all")
+	tc.xesearch(imapclient.UntaggedEsearch{}) // All not present if no match.
+
+	tc.transactf("ok", "search return (min max all) not all")
+	tc.xesearch(imapclient.UntaggedEsearch{})
+
+	tc.transactf("ok", "search return (min max all count) not all")
+	tc.xesearch(imapclient.UntaggedEsearch{Count: uint32ptr(0)})
+
+	tc.transactf("ok", "search return (min max count all) 1,3")
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uint32ptr(2), All: esearchall0("1,3")})
+
+	tc.transactf("ok", "search return (min max count all) UID 5,7")
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uint32ptr(2), All: esearchall0("1,3")})
+
+	tc.transactf("ok", "uid search return (min max count all) 1,3")
+	tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uint32ptr(2), All: esearchall0("5,7")})
 
 	tc.transactf("ok", "uid search return (min max count all) UID 5,7")
 	tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uint32ptr(2), All: esearchall0("5,7")})
 
-	if !uidonly {
+	tc.transactf("no", `search return () charset unknown text "mox"`)
-		tc.transactf("no", `search return () charset unknown text "mox"`)
+	tc.transactf("ok", `search return () charset us-ascii text "mox"`)
-		tc.transactf("ok", `search return () charset us-ascii text "mox"`)
+	tc.xesearch(esearchall("2:3"))
-		tc.xesearch(esearchall("2:3"))
+	tc.transactf("ok", `search return () charset utf-8 text "mox"`)
-		tc.transactf("ok", `search return () charset utf-8 text "mox"`)
+	tc.xesearch(esearchall("2:3"))
-		tc.xesearch(esearchall("2:3"))
 
-		tc.transactf("bad", `search return (unknown) all`)
+	tc.transactf("bad", `search return (unknown) all`)
 
-		tc.transactf("ok", "search return (save) 2")
+	tc.transactf("ok", "search return (save) 2")
-		tc.xnountagged() // ../rfc/9051:3800
+	tc.xnountagged() // ../rfc/9051:3800
-		tc.transactf("ok", "fetch $ (uid)")
+	tc.transactf("ok", "fetch $ (uid)")
-		tc.xuntagged(tc.untaggedFetch(2, 6))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6)}})
 
-		tc.transactf("ok", "search return (all) $")
+	tc.transactf("ok", "search return (all) $")
-		tc.xesearch(esearchall("2"))
+	tc.xesearch(esearchall("2"))
 
-		tc.transactf("ok", "search return (save) $")
+	tc.transactf("ok", "search return (save) $")
-		tc.xnountagged()
+	tc.xnountagged()
 
-		tc.transactf("ok", "search return (save all) all")
+	tc.transactf("ok", "search return (save all) all")
-		tc.xesearch(esearchall("1:3"))
+	tc.xesearch(esearchall("1:3"))
 
-		tc.transactf("ok", "search return (all save) all")
+	tc.transactf("ok", "search return (all save) all")
-		tc.xesearch(esearchall("1:3"))
+	tc.xesearch(esearchall("1:3"))
 
-		tc.transactf("ok", "search return (min save) all")
+	tc.transactf("ok", "search return (min save) all")
-		tc.xesearch(imapclient.UntaggedEsearch{Min: 1})
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1})
-		tc.transactf("ok", "fetch $ (uid)")
+	tc.transactf("ok", "fetch $ (uid)")
-		tc.xuntagged(tc.untaggedFetch(1, 5))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5)}})
-	}
 
 	// Do a seemingly old-style search command with IMAP4rev2 enabled. We'll still get ESEARCH responses.
-	tc.client.Enable(imapclient.CapIMAP4rev2)
+	tc.client.Enable("IMAP4rev2")
-
+	tc.transactf("ok", `search undraft`)
-	if !uidonly {
+	tc.xesearch(esearchall("1:2"))
-		tc.transactf("ok", `search undraft`)
-		tc.xesearch(esearchall("1:2"))
-	}
 
 	// Long commands should be rejected, not allocating too much memory.
 	lit := make([]byte, 100*1024+1)
@@ -445,7 +373,7 @@ func testSearch(t *testing.T, uidonly bool) {
 	// More than 1mb total for literals.
 	_, err = fmt.Fprintf(tc.client, "x0 uid search")
 	tcheck(t, err, "write start of uit search")
-	for range 10 {
+	for i := 0; i < 10; i++ {
 		writeTextLit(100*1024, true)
 	}
 	writeTextLit(1, false)
@@ -453,10 +381,11 @@ func testSearch(t *testing.T, uidonly bool) {
 	// More than 1000 literals.
 	_, err = fmt.Fprintf(tc.client, "x0 uid search")
 	tcheck(t, err, "write start of uit search")
-	for range 1000 {
+	for i := 0; i < 1000; i++ {
 		writeTextLit(1, true)
 	}
 	writeTextLit(1, false)
+
 }
 
 // esearchall makes an UntaggedEsearch response with All set, for comparisons.
@@ -490,367 +419,3 @@ func esearchall0(ss string) imapclient.NumSet {
 	}
 	return seqset
 }
-
-func TestSearchMultiUnselected(t *testing.T) {
-	testSearchMulti(t, false, false)
-}
-
-func TestSearchMultiSelected(t *testing.T) {
-	testSearchMulti(t, true, false)
-}
-
-func TestSearchMultiSelectedUIDOnly(t *testing.T) {
-	testSearchMulti(t, true, true)
-}
-
-// Test the MULTISEARCH extension, with and without selected mailbx. Operating
-// without messag sequence numbers, and return untagged esearch responses that
-// include the mailbox and uidvalidity.
-func testSearchMulti(t *testing.T, selected, uidonly bool) {
-	defer mockUIDValidity()()
-
-	tc := start(t, uidonly)
-	defer tc.close()
-	tc.login("mjl@mox.example", password0)
-	tc.client.Select("inbox")
-
-	// Add 5 messages to Inbox and delete first 4 messages. So UIDs start at 5.
-	received := time.Date(2020, time.January, 1, 10, 0, 0, 0, time.UTC)
-	for range 6 {
-		tc.client.Append("inbox", makeAppendTime(exampleMsg, received))
-	}
-	tc.client.UIDStoreFlagsSet("1:4", true, `\Deleted`)
-	tc.client.Expunge()
-
-	// Unselecting mailbox, esearch works in authenticated state.
-	if !selected {
-		tc.client.Unselect()
-	}
-
-	received = time.Date(2022, time.January, 1, 9, 0, 0, 0, time.UTC)
-	tc.client.Append("inbox", makeAppendTime(searchMsg, received))
-
-	received = time.Date(2022, time.January, 1, 9, 0, 0, 0, time.UTC)
-	mostFlags := []string{
-		`\Deleted`,
-		`\Seen`,
-		`\Answered`,
-		`\Flagged`,
-		`\Draft`,
-		`$Forwarded`,
-		`$Junk`,
-		`$Notjunk`,
-		`$Phishing`,
-		`$MDNSent`,
-		`custom1`,
-		`Custom2`,
-	}
-	tc.client.Append("Archive", imapclient.Append{Flags: mostFlags, Received: &received, Size: int64(len(searchMsg)), Data: strings.NewReader(searchMsg)})
-
-	// We now have sequence numbers 1,2,3 and UIDs 5,6,7 in Inbox, and UID 1 in Archive.
-
-	// Basic esearch with mailboxes.
-	tc.cmdf("Tag1", `Esearch In (Personal) Return () All`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:7")},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, All: esearchall0("1")},
-	)
-
-	// Again, but with progress information.
-	orig := inProgressPeriod
-	inProgressPeriod = 0
-	inprogress := func(cur, goal uint32) imapclient.UntaggedResult {
-		return imapclient.UntaggedResult{
-			Status: "OK",
-			Code:   imapclient.CodeInProgress{Tag: "Tag1", Current: &cur, Goal: &goal},
-			Text:   "still searching",
-		}
-	}
-	tc.cmdf("Tag1", `Esearch In (Personal) Return () All`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:7")},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, All: esearchall0("1")},
-		inprogress(0, 4),
-		inprogress(1, 4),
-		inprogress(2, 4),
-		inprogress(3, 4),
-	)
-	inProgressPeriod = orig
-
-	// Explicit mailboxes listed, including non-existent one that is ignored,
-	// duplicates are ignored as well.
-	tc.cmdf("Tag1", `Esearch In (Mailboxes (INBOX Archive Archive)) Return (Min Max Count All) All`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5, Max: 7, Count: uint32ptr(3), All: esearchall0("5:7")},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Min: 1, Max: 1, Count: uint32ptr(1), All: esearchall0("1")},
-	)
-
-	// No response if none of the mailboxes exist.
-	tc.cmdf("Tag1", `Esearch In (Mailboxes bogus Mailboxes (nonexistent)) Return (Min Max Count All) All`)
-	tc.response("ok")
-	tc.xuntagged()
-
-	// Inboxes evaluates to just inbox on new account. We'll add more mailboxes
-	// matching "inboxes" later on.
-	tc.cmdf("Tag1", `Esearch In (Inboxes) Return () All`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:7")},
-	)
-
-	// Subscribed is set for created mailboxes by default.
-	tc.cmdf("Tag1", `Esearch In (Subscribed) Return (Max) All`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Max: 7},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Max: 1},
-	)
-
-	// Asking for max does a reverse search.
-	tc.cmdf("Tag1", `Esearch In (Personal) Return (Max) All`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Max: 7},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Max: 1},
-	)
-
-	// Min stops early.
-	tc.cmdf("Tag1", `Esearch In (Personal) Return (Min) All`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Min: 1},
-	)
-
-	// Min and max do forward and reverse search, stopping early.
-	tc.cmdf("Tag1", `Esearch In (Personal) Return (Min Max) All`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5, Max: 7},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Min: 1, Max: 1},
-	)
-
-	if selected {
-		// With only 1 inbox, we can use SAVE with Inboxes. Can't anymore when we have multiple.
-		tc.transactf("ok", `Esearch In (Inboxes) Return (Save) All`)
-		tc.xuntagged()
-
-		// Using search result ($) works with selected mailbox.
-		tc.cmdf("Tag1", `Esearch In (Selected) Return () $`)
-		tc.response("ok")
-		tc.xuntagged(
-			imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:7")},
-		)
-	} else {
-		// Cannot use "selected" if we are not in selected state.
-		tc.transactf("bad", `Esearch In (Selected) Return () All`)
-	}
-
-	// Add more "inboxes", and other mailboxes for testing "subtree" and "subtree-one".
-	more := []string{
-		"Inbox/Sub1",
-		"Inbox/Sub2",
-		"Inbox/Sub2/SubA",
-		"Inbox/Sub2/SubB",
-		"Other",
-		"Other/Sub1", // sub1@mox.example in config.
-		"Other/Sub2",
-		"Other/Sub2/SubA", // ruleset for sub2@mox.example in config.
-		"Other/Sub2/SubB",
-		"List", // ruleset for a mailing list
-	}
-	for _, name := range more {
-		tc.client.Create(name, nil)
-		tc.client.Append(name, makeAppendTime(exampleMsg, received))
-	}
-
-	// Cannot use SAVE with multiple mailboxes that match.
-	tc.transactf("bad", `Esearch In (Inboxes) Return (Save) All`)
-
-	// "inboxes" includes everything below Inbox, and also anything that we might
-	// deliver to based on account addresses and rulesets, but not mailing lists.
-	tc.cmdf("Tag1", `Esearch In (Inboxes) Return () All`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:7")},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox/Sub1", UIDValidity: 3, UID: true, All: esearchall0("1")},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox/Sub2", UIDValidity: 4, UID: true, All: esearchall0("1")},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox/Sub2/SubA", UIDValidity: 5, UID: true, All: esearchall0("1")},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox/Sub2/SubB", UIDValidity: 6, UID: true, All: esearchall0("1")},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub1", UIDValidity: 8, UID: true, All: esearchall0("1")},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub2/SubA", UIDValidity: 10, UID: true, All: esearchall0("1")},
-	)
-
-	// subtree
-	tc.cmdf("Tag1", `Esearch In (Subtree Other) Return () All`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other", UIDValidity: 7, UID: true, All: esearchall0("1")},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub1", UIDValidity: 8, UID: true, All: esearchall0("1")},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub2", UIDValidity: 9, UID: true, All: esearchall0("1")},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub2/SubA", UIDValidity: 10, UID: true, All: esearchall0("1")},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub2/SubB", UIDValidity: 11, UID: true, All: esearchall0("1")},
-	)
-
-	// subtree-one
-	tc.cmdf("Tag1", `Esearch In (Subtree-One Other) Return () All`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other", UIDValidity: 7, UID: true, All: esearchall0("1")},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub1", UIDValidity: 8, UID: true, All: esearchall0("1")},
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub2", UIDValidity: 9, UID: true, All: esearchall0("1")},
-	)
-
-	// Search with sequence set also for non-selected mailboxes(!). The min/max would
-	// get the first and last message, but the message sequence set forces a scan. Not
-	// allowed with UIDONLY.
-	if !uidonly {
-		tc.cmdf("Tag1", `Esearch In (Mailboxes Inbox) Return (Min Max) 1:*`)
-		tc.response("ok")
-		tc.xuntagged(
-			imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5, Max: 7},
-		)
-	}
-
-	// Search with uid set with "$highnum:*" forces getting highest uid.
-	tc.cmdf("Tag1", `Esearch In (Mailboxes Inbox) Return (Min Max) Uid *:100`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 7, Max: 7},
-	)
-	tc.cmdf("Tag1", `Esearch In (Mailboxes Inbox) Return (Min Max) Uid 100:*`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 7, Max: 7},
-	)
-	tc.cmdf("Tag1", `Esearch In (Mailboxes Inbox) Return (Min Max) Uid 1:*`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5, Max: 7},
-	)
-
-	// We use another session to add a new message to Inbox and to Archive. Searching
-	// with Inbox selected will not return the new message since it isn't available in
-	// the session yet. The message in Archive is returned, since there is no session
-	// limitation.
-	tc2 := startNoSwitchboard(t, uidonly)
-	defer tc2.closeNoWait()
-	tc2.login("mjl@mox.example", password0)
-	tc2.client.Append("inbox", makeAppendTime(searchMsg, received))
-	tc2.client.Append("Archive", makeAppendTime(searchMsg, received))
-
-	tc.cmdf("Tag1", `Esearch In (Mailboxes (Inbox Archive)) Return (Count) All`)
-	tc.response("ok")
-	if selected {
-		tc.xuntagged(
-			imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Count: uint32ptr(3)},
-			imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Count: uint32ptr(2)},
-			imapclient.UntaggedExists(4),
-			tc.untaggedFetch(4, 8, imapclient.FetchFlags(nil)),
-		)
-	} else {
-		tc.xuntagged(
-			imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Count: uint32ptr(4)},
-			imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Count: uint32ptr(2)},
-		)
-	}
-
-	if selected {
-		// Saving a search result, and then using it with another mailbox results in error.
-		tc.transactf("ok", `Esearch In (Mailboxes Inbox) Return (Save) All`)
-		tc.transactf("no", `Esearch In (Mailboxes Archive) Return () $`)
-	} else {
-		tc.transactf("bad", `Esearch In (Inboxes) Return (Save) All`) // Need a selected mailbox with SAVE.
-		tc.transactf("no", `Esearch In (Inboxes) Return () $`)        // Cannot use saved result with non-selected mailbox.
-	}
-
-	tc.transactf("bad", `Esearch In () Return () All`)                    // Missing values for "IN"-list.
-	tc.transactf("bad", `Esearch In (Bogus) Return () All`)               // Bogus word for "IN".
-	tc.transactf("bad", `Esearch In ("Selected") Return () All`)          // IN-words can't be quoted.
-	tc.transactf("bad", `Esearch In (Selected-Delayed) Return () All`)    // From NOTIFY, not in ESEARCH.
-	tc.transactf("bad", `Esearch In (Subtree-One) Return () All`)         // After subtree-one we need a list.
-	tc.transactf("bad", `Esearch In (Subtree-One ) Return () All`)        // After subtree-one we need a list.
-	tc.transactf("bad", `Esearch In (Subtree-One (Test) ) Return () All`) // Bogus space.
-
-	if !selected {
-		return
-	}
-	// From now on, we are in selected state.
-
-	tc.cmdf("Tag1", `Esearch In (Selected) Return () All`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:8")},
-	)
-
-	// Testing combinations of SAVE with MIN/MAX/others ../rfc/9051:4100
-	tc.transactf("ok", `Esearch In (Selected) Return (Save) All`)
-	tc.xuntagged()
-
-	tc.cmdf("Tag1", `Esearch In (Selected) Return () $`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:8")},
-	)
-
-	// Inbox happens to be the selected mailbox, so OK.
-	tc.cmdf("Tag1", `Esearch In (Mailboxes Inbox) Return () $`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:8")},
-	)
-
-	// Non-selected mailboxes aren't allowed to use the saved result.
-	tc.transactf("no", `Esearch In (Mailboxes Archive) Return () $`)
-	tc.transactf("no", `Esearch In (Mailboxes Archive) Return () uid $`)
-
-	tc.cmdf("Tag1", `Esearch In (Selected) Return (Save Min Max) All`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5, Max: 8},
-	)
-	tc.cmdf("Tag1", `Esearch In (Selected) Return () $`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5,8")},
-	)
-
-	tc.cmdf("Tag1", `Esearch In (Selected) Return (Save Min) All`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5},
-	)
-
-	tc.cmdf("Tag1", `Esearch In (Selected) Return () $`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5")},
-	)
-
-	tc.cmdf("Tag1", `Esearch In (Selected) Return (Save Max) All`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Max: 8},
-	)
-
-	tc.cmdf("Tag1", `Esearch In (Selected) Return () $`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("8")},
-	)
-
-	tc.cmdf("Tag1", `Esearch In (Selected) Return (Save Min Max Count) All`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5, Max: 8, Count: uint32ptr(4)},
-	)
-
-	tc.cmdf("Tag1", `Esearch In (Selected) Return () $`)
-	tc.response("ok")
-	tc.xuntagged(
-		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:8")},
-	)
-}

imapserver/selectexamine_test.go

@@ -8,28 +8,20 @@ import (
 )
 
 func TestSelect(t *testing.T) {
-	testSelectExamine(t, false, false)
+	testSelectExamine(t, false)
 }
 
 func TestExamine(t *testing.T) {
-	testSelectExamine(t, true, false)
+	testSelectExamine(t, true)
-}
-
-func TestSelectUIDOnly(t *testing.T) {
-	testSelectExamine(t, false, true)
-}
-
-func TestExamineUIDOnly(t *testing.T) {
-	testSelectExamine(t, true, true)
 }
 
 // select and examine are pretty much the same. but examine opens readonly instead of readwrite.
-func testSelectExamine(t *testing.T, examine, uidonly bool) {
+func testSelectExamine(t *testing.T, examine bool) {
 	defer mockUIDValidity()()
-	tc := start(t, uidonly)
+	tc := start(t)
 	defer tc.close()
 
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
 
 	cmd := "select"
 	okcode := "READ-WRITE"
@@ -38,48 +30,43 @@ func testSelectExamine(t *testing.T, examine, uidonly bool) {
 		okcode = "READ-ONLY"
 	}
 
-	uclosed := imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeWord("CLOSED"), Text: "x"}
+	uclosed := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "CLOSED", More: "x"}}
 	flags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent`, " ")
 	permflags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent \*`, " ")
 	uflags := imapclient.UntaggedFlags(flags)
-	upermflags := imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodePermanentFlags(permflags), Text: "x"}
+	upermflags := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "PERMANENTFLAGS", CodeArg: imapclient.CodeList{Code: "PERMANENTFLAGS", Args: permflags}, More: "x"}}
 	urecent := imapclient.UntaggedRecent(0)
 	uexists0 := imapclient.UntaggedExists(0)
 	uexists1 := imapclient.UntaggedExists(1)
-	uuidval1 := imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeUIDValidity(1), Text: "x"}
+	uuidval1 := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UIDVALIDITY", CodeArg: imapclient.CodeUint{Code: "UIDVALIDITY", Num: 1}, More: "x"}}
-	uuidnext1 := imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeUIDNext(1), Text: "x"}
+	uuidnext1 := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UIDNEXT", CodeArg: imapclient.CodeUint{Code: "UIDNEXT", Num: 1}, More: "x"}}
 	ulist := imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"}
-	uunseen := imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeUnseen(1), Text: "x"}
+	uunseen := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UNSEEN", CodeArg: imapclient.CodeUint{Code: "UNSEEN", Num: 1}, More: "x"}}
-	uuidnext2 := imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeUIDNext(2), Text: "x"}
+	uuidnext2 := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UIDNEXT", CodeArg: imapclient.CodeUint{Code: "UIDNEXT", Num: 2}, More: "x"}}
 
 	// Parameter required.
-	tc.transactf("bad", "%s", cmd)
+	tc.transactf("bad", cmd)
 
 	// Mailbox does not exist.
-	tc.transactf("no", "%s bogus", cmd)
+	tc.transactf("no", cmd+" bogus")
-	tc.transactf("no", "%s expungebox", cmd)
 
-	tc.transactf("ok", "%s inbox", cmd)
+	tc.transactf("ok", cmd+" inbox")
 	tc.xuntagged(uflags, upermflags, urecent, uexists0, uuidval1, uuidnext1, ulist)
-	tc.xcodeWord(okcode)
+	tc.xcode(okcode)
 
-	tc.transactf("ok", `%s "inbox"`, cmd)
+	tc.transactf("ok", cmd+` "inbox"`)
 	tc.xuntagged(uclosed, uflags, upermflags, urecent, uexists0, uuidval1, uuidnext1, ulist)
-	tc.xcodeWord(okcode)
+	tc.xcode(okcode)
 
 	// Append a message. It will be reported as UNSEEN.
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
-	tc.transactf("ok", "%s inbox", cmd)
+	tc.transactf("ok", cmd+" inbox")
-	if uidonly {
+	tc.xuntagged(uclosed, uflags, upermflags, urecent, uunseen, uexists1, uuidval1, uuidnext2, ulist)
-		tc.xuntagged(uclosed, uflags, upermflags, urecent, uexists1, uuidval1, uuidnext2, ulist)
+	tc.xcode(okcode)
-	} else {
-		tc.xuntagged(uclosed, uflags, upermflags, urecent, uunseen, uexists1, uuidval1, uuidnext2, ulist)
-	}
-	tc.xcodeWord(okcode)
 
 	// With imap4rev2, we no longer get untagged RECENT or untagged UNSEEN.
-	tc.client.Enable(imapclient.CapIMAP4rev2)
+	tc.client.Enable("imap4rev2")
-	tc.transactf("ok", "%s inbox", cmd)
+	tc.transactf("ok", cmd+" inbox")
 	tc.xuntagged(uclosed, uflags, upermflags, uexists1, uuidval1, uuidnext2, ulist)
-	tc.xcodeWord(okcode)
+	tc.xcode(okcode)
 }

imapserver/server_test.go

@@ -7,7 +7,6 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
-	"log/slog"
 	"math/big"
 	"net"
 	"os"
@@ -17,16 +16,11 @@ import (
 	"testing"
 	"time"
 
-	"golang.org/x/sys/unix"
-
-	"github.com/mjl-/bstore"
-
 	"github.com/mjl-/mox/imapclient"
 	"github.com/mjl-/mox/mlog"
 	"github.com/mjl-/mox/mox-"
 	"github.com/mjl-/mox/moxvar"
 	"github.com/mjl-/mox/store"
-	"slices"
 )
 
 var ctxbg = context.Background()
@@ -38,12 +32,6 @@ func init() {
 	// Don't slow down tests.
 	badClientDelay = 0
 	authFailDelay = 0
-
-	mox.Context = ctxbg
-}
-
-func ptr[T any](v T) *T {
-	return &v
 }
 
 func tocrlf(s string) string {
@@ -123,7 +111,6 @@ aGVsbG8NCndvcmxkDQo=
 --unique-boundary-2
 Content-Type: image/jpeg
 Content-Transfer-Encoding: base64
-Content-Disposition: inline; filename=image.jpg
 
 
 --unique-boundary-2--
@@ -139,9 +126,6 @@ Isn't it
 
 --unique-boundary-1
 Content-Type: message/rfc822
-Content-MD5: MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=
-Content-Language: en,de
-Content-Location: http://localhost
 
 From: info@mox.example
 To: mox <info@mox.example>
@@ -161,22 +145,6 @@ func tcheck(t *testing.T, err error, msg string) {
 	}
 }
 
-func mustParseUntagged(s string) imapclient.Untagged {
-	r, err := imapclient.ParseUntagged(s + "\r\n")
-	if err != nil {
-		panic(err)
-	}
-	return r
-}
-
-func mustParseCode(s string) imapclient.Code {
-	r, err := imapclient.ParseCode(s)
-	if err != nil {
-		panic(err)
-	}
-	return r
-}
-
 func mockUIDValidity() func() {
 	orig := store.InitialUIDValidity
 	store.InitialUIDValidity = func() uint32 {
@@ -188,18 +156,16 @@ func mockUIDValidity() func() {
 }
 
 type testconn struct {
-	t           *testing.T
+	t          *testing.T
-	conn        net.Conn
+	conn       net.Conn
-	client      *imapclient.Conn
+	client     *imapclient.Conn
-	uidonly     bool
+	done       chan struct{}
-	done        chan struct{}
+	serverConn net.Conn
-	serverConn  net.Conn
+	account    *store.Account
-	account     *store.Account
-	switchStop  func()
-	clientPanic bool
 
 	// Result of last command.
-	lastResponse imapclient.Response
+	lastUntagged []imapclient.Untagged
+	lastResult   imapclient.Result
 	lastErr      error
 }
 
@@ -210,21 +176,24 @@ func (tc *testconn) check(err error, msg string) {
 	}
 }
 
-func (tc *testconn) last(resp imapclient.Response, err error) {
+func (tc *testconn) last(l []imapclient.Untagged, r imapclient.Result, err error) {
-	tc.lastResponse = resp
+	tc.lastUntagged = l
+	tc.lastResult = r
 	tc.lastErr = err
 }
 
-func (tc *testconn) xcode(c imapclient.Code) {
+func (tc *testconn) xcode(s string) {
 	tc.t.Helper()
-	if !reflect.DeepEqual(tc.lastResponse.Code, c) {
+	if tc.lastResult.Code != s {
-		tc.t.Fatalf("got last code %#v, expected %#v", tc.lastResponse.Code, c)
+		tc.t.Fatalf("got last code %q, expected %q", tc.lastResult.Code, s)
 	}
 }
 
-func (tc *testconn) xcodeWord(s string) {
+func (tc *testconn) xcodeArg(v any) {
 	tc.t.Helper()
-	tc.xcode(imapclient.CodeWord(s))
+	if !reflect.DeepEqual(tc.lastResult.CodeArg, v) {
+		tc.t.Fatalf("got last code argument %v, expected %v", tc.lastResult.CodeArg, v)
+	}
 }
 
 func (tc *testconn) xuntagged(exps ...imapclient.Untagged) {
@@ -234,7 +203,7 @@ func (tc *testconn) xuntagged(exps ...imapclient.Untagged) {
 
 func (tc *testconn) xuntaggedOpt(all bool, exps ...imapclient.Untagged) {
 	tc.t.Helper()
-	last := slices.Clone(tc.lastResponse.Untagged)
+	last := append([]imapclient.Untagged{}, tc.lastUntagged...)
 	var mismatch any
 next:
 	for ei, exp := range exps {
@@ -254,10 +223,10 @@ next:
 			tc.t.Fatalf("untagged data mismatch, got:\n\t%T %#v\nexpected:\n\t%T %#v", mismatch, mismatch, exp, exp)
 		}
 		var next string
-		if len(tc.lastResponse.Untagged) > 0 {
+		if len(tc.lastUntagged) > 0 {
-			next = fmt.Sprintf(", next:\n%#v", tc.lastResponse.Untagged[0])
+			next = fmt.Sprintf(", next %#v", tc.lastUntagged[0])
 		}
-		tc.t.Fatalf("did not find untagged response:\n%#v %T (%d)\nin %v%s", exp, exp, ei, tc.lastResponse.Untagged, next)
+		tc.t.Fatalf("did not find untagged response %#v %T (%d) in %v%s", exp, exp, ei, tc.lastUntagged, next)
 	}
 	if len(last) > 0 && all {
 		tc.t.Fatalf("leftover untagged responses %v", last)
@@ -269,27 +238,15 @@ func tuntagged(t *testing.T, got imapclient.Untagged, dst any) {
 	gotv := reflect.ValueOf(got)
 	dstv := reflect.ValueOf(dst)
 	if gotv.Type() != dstv.Type().Elem() {
-		t.Fatalf("got %#v, expected %#v", got, dstv.Elem().Interface())
+		t.Fatalf("got %v, expected %v", gotv.Type(), dstv.Type().Elem())
 	}
 	dstv.Elem().Set(gotv)
 }
 
 func (tc *testconn) xnountagged() {
 	tc.t.Helper()
-	if len(tc.lastResponse.Untagged) != 0 {
+	if len(tc.lastUntagged) != 0 {
-		tc.t.Fatalf("got %v untagged, expected 0", tc.lastResponse.Untagged)
+		tc.t.Fatalf("got %v untagged, expected 0", tc.lastUntagged)
-	}
-}
-
-func (tc *testconn) readuntagged(exps ...imapclient.Untagged) {
-	tc.t.Helper()
-	for i, exp := range exps {
-		tc.conn.SetReadDeadline(time.Now().Add(3 * time.Second))
-		v, err := tc.client.ReadUntagged()
-		tcheck(tc.t, err, "reading untagged")
-		if !reflect.DeepEqual(v, exp) {
-			tc.t.Fatalf("got %#v, expected %#v, response %d/%d", v, exp, i+1, len(exps))
-		}
 	}
 }
 
@@ -301,24 +258,16 @@ func (tc *testconn) transactf(status, format string, args ...any) {
 
 func (tc *testconn) response(status string) {
 	tc.t.Helper()
-	tc.lastResponse, tc.lastErr = tc.client.ReadResponse()
+	tc.lastUntagged, tc.lastResult, tc.lastErr = tc.client.Response()
-	if tc.lastErr != nil {
+	tcheck(tc.t, tc.lastErr, "read imap response")
-		if resp, ok := tc.lastErr.(imapclient.Response); ok {
+	if strings.ToUpper(status) != string(tc.lastResult.Status) {
-			if !reflect.DeepEqual(resp, tc.lastResponse) {
+		tc.t.Fatalf("got status %q, expected %q", tc.lastResult.Status, status)
-				tc.t.Fatalf("response error %#v != returned response %#v", tc.lastErr, tc.lastResponse)
-			}
-		} else {
-			tcheck(tc.t, tc.lastErr, "read imap response")
-		}
-	}
-	if strings.ToUpper(status) != string(tc.lastResponse.Status) {
-		tc.t.Fatalf("got status %q, expected %q", tc.lastResponse.Status, status)
 	}
 }
 
 func (tc *testconn) cmdf(tag, format string, args ...any) {
 	tc.t.Helper()
-	err := tc.client.WriteCommandf(tag, format, args...)
+	err := tc.client.Commandf(tag, format, args...)
 	tcheck(tc.t, err, "writing imap command")
 }
 
@@ -354,67 +303,18 @@ func (tc *testconn) waitDone() {
 	}
 }
 
-func (tc *testconn) login(username, password string) {
-	tc.client.Login(username, password)
-	if tc.uidonly {
-		tc.transactf("ok", "enable uidonly")
-	}
-}
-
-// untaggedFetch returns an imapclient.UntaggedFetch or
-// imapclient.UntaggedUIDFetch, depending on whether uidonly is enabled for the
-// connection.
-func (tc *testconn) untaggedFetch(seq, uid uint32, attrs ...imapclient.FetchAttr) any {
-	if tc.uidonly {
-		return imapclient.UntaggedUIDFetch{UID: uid, Attrs: attrs}
-	}
-	attrs = append([]imapclient.FetchAttr{imapclient.FetchUID(uid)}, attrs...)
-	return imapclient.UntaggedFetch{Seq: seq, Attrs: attrs}
-}
-
-// like untaggedFetch, but with explicit UID fetch attribute in case of uidonly.
-func (tc *testconn) untaggedFetchUID(seq, uid uint32, attrs ...imapclient.FetchAttr) any {
-	attrs = append([]imapclient.FetchAttr{imapclient.FetchUID(uid)}, attrs...)
-	if tc.uidonly {
-		return imapclient.UntaggedUIDFetch{UID: uid, Attrs: attrs}
-	}
-	return imapclient.UntaggedFetch{Seq: seq, Attrs: attrs}
-}
-
 func (tc *testconn) close() {
-	tc.close0(true)
-}
-
-func (tc *testconn) closeNoWait() {
-	tc.close0(false)
-}
-
-func (tc *testconn) close0(waitclose bool) {
-	defer func() {
-		if unhandledPanics.Swap(0) > 0 {
-			tc.t.Fatalf("unhandled panic in server")
-		}
-	}()
-
 	if tc.account == nil {
 		// Already closed, we are not strict about closing multiple times.
 		return
 	}
-	if tc.client != nil {
-		tc.clientPanic = false // Ignore errors writing to TLS connection the server also closed.
-		tc.client.Close()
-	}
 	err := tc.account.Close()
 	tc.check(err, "close account")
-	if waitclose {
+	// no account.CheckClosed(), the tests open accounts multiple times.
-		tc.account.WaitClosed()
-	}
 	tc.account = nil
+	tc.client.Close()
 	tc.serverConn.Close()
 	tc.waitDone()
-	if tc.switchStop != nil {
-		tc.switchStop()
-	}
 }
 
 func xparseNumSet(s string) imapclient.NumSet {
@@ -425,171 +325,66 @@ func xparseNumSet(s string) imapclient.NumSet {
 	return ns
 }
 
-func xparseUIDRange(s string) imapclient.NumRange {
-	nr, err := imapclient.ParseUIDRange(s)
-	if err != nil {
-		panic(fmt.Sprintf("parsing uid range %s: %s", s, err))
-	}
-	return nr
-}
-
-func makeAppend(msg string) imapclient.Append {
-	return imapclient.Append{Size: int64(len(msg)), Data: strings.NewReader(msg)}
-}
-
-func makeAppendTime(msg string, tm time.Time) imapclient.Append {
-	return imapclient.Append{Received: &tm, Size: int64(len(msg)), Data: strings.NewReader(msg)}
-}
-
 var connCounter int64
 
-func start(t *testing.T, uidonly bool) *testconn {
+func start(t *testing.T) *testconn {
-	return startArgs(t, uidonly, true, false, true, true, "mjl")
+	return startArgs(t, true, false, true, true, "mjl")
 }
 
-func startNoSwitchboard(t *testing.T, uidonly bool) *testconn {
+func startNoSwitchboard(t *testing.T) *testconn {
-	return startArgs(t, uidonly, false, false, true, false, "mjl")
+	return startArgs(t, false, false, true, false, "mjl")
 }
 
 const password0 = "te\u0301st \u00a0\u2002\u200a" // NFD and various unicode spaces.
 const password1 = "tést    "                      // PRECIS normalized, with NFC.
 
-func startArgs(t *testing.T, uidonly, first, immediateTLS bool, allowLoginWithoutTLS, setPassword bool, accname string) *testconn {
+func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS, setPassword bool, accname string) *testconn {
-	return startArgsMore(t, uidonly, first, immediateTLS, nil, nil, allowLoginWithoutTLS, setPassword, accname, nil)
-}
-
-// namedConn wraps a conn so it can return a RemoteAddr with a non-empty name.
-// The TLS resumption test needs a non-empty name, but on BSDs, the unix domain
-// socket pair has an empty peer name.
-type namedConn struct {
-	net.Conn
-}
-
-func (c namedConn) RemoteAddr() net.Addr {
-	return &net.TCPAddr{IP: net.ParseIP("127.0.0.10"), Port: 1234}
-}
-
-// todo: the parameters and usage are too much now. change to scheme similar to smtpserver, with params in a struct, and a separate method for init and making a connection.
-func startArgsMore(t *testing.T, uidonly, first, immediateTLS bool, serverConfig, clientConfig *tls.Config, allowLoginWithoutTLS, setPassword bool, accname string, afterInit func() error) *testconn {
 	limitersInit() // Reset rate limiters.
 
-	switchStop := func() {}
 	if first {
-		mox.ConfigStaticPath = filepath.FromSlash("../testdata/imap/mox.conf")
-		mox.MustLoadConfig(true, false)
-		store.Close() // May not be open, we ignore error.
 		os.RemoveAll("../testdata/imap/data")
-		err := store.Init(ctxbg)
-		tcheck(t, err, "store init")
-		switchStop = store.Switchboard()
 	}
-
+	mox.Context = ctxbg
-	acc, err := store.OpenAccount(pkglog, accname, false)
+	mox.ConfigStaticPath = filepath.FromSlash("../testdata/imap/mox.conf")
+	mox.MustLoadConfig(true, false)
+	acc, err := store.OpenAccount(pkglog, accname)
 	tcheck(t, err, "open account")
 	if setPassword {
 		err = acc.SetPassword(pkglog, password0)
 		tcheck(t, err, "set password")
 	}
+	switchStop := func() {}
 	if first {
-		// Add a deleted mailbox, may excercise some code paths.
+		switchStop = store.Switchboard()
-		err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
-			// todo: add a message to inbox and remove it again. need to change all uids in the tests.
-			// todo: add tests for operating on an expunged mailbox. it should say it doesn't exist.
-
-			mb, _, _, _, err := acc.MailboxCreate(tx, "expungebox", store.SpecialUse{})
-			if err != nil {
-				return fmt.Errorf("create mailbox: %v", err)
-			}
-			if _, _, err := acc.MailboxDelete(ctxbg, pkglog, tx, &mb); err != nil {
-				return fmt.Errorf("delete mailbox: %v", err)
-			}
-			return nil
-		})
-		tcheck(t, err, "add expunged mailbox")
 	}
 
-	if afterInit != nil {
+	serverConn, clientConn := net.Pipe()
-		err := afterInit()
+
-		tcheck(t, err, "after init")
+	tlsConfig := &tls.Config{
+		Certificates: []tls.Certificate{fakeCert(t)},
 	}
-
+	if isTLS {
-	// We get actual sockets for their buffering behaviour. net.Pipe is synchronous,
+		serverConn = tls.Server(serverConn, tlsConfig)
-	// and the implementation of the compress extension can write a sync message to an
+		clientConn = tls.Client(clientConn, &tls.Config{InsecureSkipVerify: true})
-	// imap client when that client isn't reading but is trying to write. In the real
-	// world, network buffer will take up those few bytes, so assume the buffer in the
-	// test too.
-	fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM, 0)
-	tcheck(t, err, "socketpair")
-	xfdconn := func(fd int, name string) net.Conn {
-		f := os.NewFile(uintptr(fd), name)
-		fc, err := net.FileConn(f)
-		tcheck(t, err, "fileconn")
-		err = f.Close()
-		tcheck(t, err, "close file for conn")
-
-		// Small read/write buffers, for detecting closed/broken connections quickly.
-		uc := fc.(*net.UnixConn)
-		err = uc.SetReadBuffer(512)
-		tcheck(t, err, "set read buffer")
-		uc.SetWriteBuffer(512)
-		tcheck(t, err, "set write buffer")
-
-		return namedConn{uc}
-	}
-	serverConn := xfdconn(fds[0], "server")
-	clientConn := xfdconn(fds[1], "client")
-
-	if serverConfig == nil {
-		serverConfig = &tls.Config{
-			Certificates: []tls.Certificate{fakeCert(t, false)},
-		}
-	}
-	if immediateTLS {
-		if clientConfig == nil {
-			clientConfig = &tls.Config{InsecureSkipVerify: true}
-		}
-		clientConn = tls.Client(clientConn, clientConfig)
 	}
 
 	done := make(chan struct{})
-	connCounter += 2
+	connCounter++
-	cid := connCounter - 1
+	cid := connCounter
 	go func() {
-		const viaHTTPS = false
+		serve("test", cid, tlsConfig, serverConn, isTLS, allowLoginWithoutTLS)
-		serve("test", cid, serverConfig, serverConn, immediateTLS, false, allowLoginWithoutTLS, viaHTTPS, "")
+		switchStop()
 		close(done)
 	}()
-	var tc *testconn
+	client, err := imapclient.New(clientConn, true)
-	var opts imapclient.Opts
+	tcheck(t, err, "new client")
-	opts = imapclient.Opts{
+	return &testconn{t: t, conn: clientConn, client: client, done: done, serverConn: serverConn, account: acc}
-		Logger: slog.Default().With("cid", connCounter),
-		Error: func(err error) {
-			if tc.clientPanic {
-				panic(err)
-			} else {
-				opts.Logger.Error("imapclient error", "err", err)
-			}
-		},
-	}
-	client, _ := imapclient.New(clientConn, &opts)
-	tc = &testconn{t: t, conn: clientConn, client: client, uidonly: uidonly, done: done, serverConn: serverConn, account: acc, clientPanic: true}
-	if first {
-		tc.switchStop = switchStop
-	}
-	return tc
 }
 
-func fakeCert(t *testing.T, randomkey bool) tls.Certificate {
+func fakeCert(t *testing.T) tls.Certificate {
-	seed := make([]byte, ed25519.SeedSize)
+	privKey := ed25519.NewKeyFromSeed(make([]byte, ed25519.SeedSize)) // Fake key, don't use this for real!
-	if randomkey {
-		cryptorand.Read(seed)
-	}
-	privKey := ed25519.NewKeyFromSeed(seed) // Fake key, don't use this for real!
 	template := &x509.Certificate{
 		SerialNumber: big.NewInt(1), // Required field...
-		// Valid period is needed to get session resumption enabled.
-		NotBefore: time.Now().Add(-time.Minute),
-		NotAfter:  time.Now().Add(time.Hour),
 	}
 	localCertBuf, err := x509.CreateCertificate(cryptorand.Reader, template, template, privKey.Public(), privKey)
 	if err != nil {
@@ -608,7 +403,7 @@ func fakeCert(t *testing.T, randomkey bool) tls.Certificate {
 }
 
 func TestLogin(t *testing.T) {
-	tc := start(t, false)
+	tc := start(t)
 	defer tc.close()
 
 	tc.transactf("bad", "login too many args")
@@ -622,11 +417,11 @@ func TestLogin(t *testing.T) {
 	tc.transactf("ok", `login mjl@mox.example "%s"`, password0)
 	tc.close()
 
-	tc = start(t, false)
+	tc = start(t)
 	tc.transactf("ok", `login "mjl@mox.example" "%s"`, password0)
 	tc.close()
 
-	tc = start(t, false)
+	tc = start(t)
 	tc.transactf("ok", `login "\"\"@mox.example" "%s"`, password0)
 	defer tc.close()
 
@@ -636,7 +431,7 @@ func TestLogin(t *testing.T) {
 
 // Test that commands don't work in the states they are not supposed to.
 func TestState(t *testing.T) {
-	tc := start(t, false)
+	tc := start(t)
 
 	notAuthenticated := []string{"starttls", "authenticate", "login"}
 	authenticatedOrSelected := []string{"enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub"}
@@ -647,17 +442,17 @@ func TestState(t *testing.T) {
 	tc.transactf("ok", "noop")
 	tc.transactf("ok", "logout")
 	tc.close()
-	tc = start(t, false)
+	tc = start(t)
 	defer tc.close()
 
 	// Not authenticated, lots of commands not allowed.
-	for _, cmd := range slices.Concat(authenticatedOrSelected, selected) {
+	for _, cmd := range append(append([]string{}, authenticatedOrSelected...), selected...) {
 		tc.transactf("no", "%s", cmd)
 	}
 
 	// Some commands not allowed when authenticated.
 	tc.transactf("ok", `login mjl@mox.example "%s"`, password0)
-	for _, cmd := range slices.Concat(notAuthenticated, selected) {
+	for _, cmd := range append(append([]string{}, notAuthenticated...), selected...) {
 		tc.transactf("no", "%s", cmd)
 	}
 
@@ -665,7 +460,7 @@ func TestState(t *testing.T) {
 }
 
 func TestNonIMAP(t *testing.T) {
-	tc := start(t, false)
+	tc := start(t)
 	defer tc.close()
 
 	// imap greeting has already been read, we sidestep the imapclient.
@@ -678,41 +473,33 @@ func TestNonIMAP(t *testing.T) {
 }
 
 func TestLiterals(t *testing.T) {
-	tc := start(t, false)
+	tc := start(t)
 	defer tc.close()
 
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
-	tc.client.Create("tmpbox", nil)
+	tc.client.Create("tmpbox")
 
 	tc.transactf("ok", "rename {6+}\r\ntmpbox {7+}\r\nntmpbox")
 
 	from := "ntmpbox"
 	to := "tmpbox"
-	tc.client.LastTagSet("xtag")
 	fmt.Fprint(tc.client, "xtag rename ")
 	tc.client.WriteSyncLiteral(from)
 	fmt.Fprint(tc.client, " ")
 	tc.client.WriteSyncLiteral(to)
 	fmt.Fprint(tc.client, "\r\n")
-	tc.lastResponse, tc.lastErr = tc.client.ReadResponse()
+	tc.client.LastTag = "xtag"
-	if tc.lastResponse.Status != "OK" {
+	tc.last(tc.client.Response())
-		tc.t.Fatalf(`got %q, expected "OK"`, tc.lastResponse.Status)
+	if tc.lastResult.Status != "OK" {
+		tc.t.Fatalf(`got %q, expected "OK"`, tc.lastResult.Status)
 	}
 }
 
 // Test longer scenario with login, lists, subscribes, status, selects, etc.
 func TestScenario(t *testing.T) {
-	testScenario(t, false)
+	tc := start(t)
-}
-
-func TestScenarioUIDOnly(t *testing.T) {
-	testScenario(t, true)
-}
-
-func testScenario(t *testing.T, uidonly bool) {
-	tc := start(t, uidonly)
 	defer tc.close()
-	tc.login("mjl@mox.example", password0)
+	tc.transactf("ok", `login mjl@mox.example "%s"`, password0)
 
 	tc.transactf("bad", " missingcommand")
 
@@ -756,44 +543,6 @@ func testScenario(t *testing.T, uidonly bool) {
 	tc.check(err, "write message")
 	tc.response("ok")
 
-	tc.transactf("ok", "uid fetch 1 all")
-	tc.transactf("ok", "uid fetch 1 body")
-	tc.transactf("ok", "uid fetch 1 binary[]")
-
-	tc.transactf("ok", `uid store 1 flags (\seen \answered)`)
-	tc.transactf("ok", `uid store 1 +flags ($junk)`) // should train as junk.
-	tc.transactf("ok", `uid store 1 -flags ($junk)`) // should retrain as non-junk.
-	tc.transactf("ok", `uid store 1 -flags (\seen)`) // should untrain completely.
-	tc.transactf("ok", `uid store 1 -flags (\answered)`)
-	tc.transactf("ok", `uid store 1 +flags (\answered)`)
-	tc.transactf("ok", `uid store 1 flags.silent (\seen \answered)`)
-	tc.transactf("ok", `uid store 1 -flags.silent (\answered)`)
-	tc.transactf("ok", `uid store 1 +flags.silent (\answered)`)
-	tc.transactf("bad", `uid store 1 flags (\badflag)`)
-	tc.transactf("ok", "noop")
-
-	tc.transactf("ok", "uid copy 1 Trash")
-	tc.transactf("ok", "uid copy 1 Trash")
-	tc.transactf("ok", "uid move 1 Trash")
-
-	tc.transactf("ok", "close")
-	tc.transactf("ok", "select Trash")
-	tc.transactf("ok", `uid store 1 flags (\deleted)`)
-	tc.transactf("ok", "expunge")
-	tc.transactf("ok", "noop")
-
-	tc.transactf("ok", `uid store 1 flags (\deleted)`)
-	tc.transactf("ok", "close")
-	tc.transactf("ok", "delete Trash")
-
-	if uidonly {
-		return
-	}
-
-	tc.transactf("ok", "create Trash")
-	tc.transactf("ok", "append inbox (\\seen) {%d+}\r\n%s", len(exampleMsg), exampleMsg)
-	tc.transactf("ok", "select inbox")
-
 	tc.transactf("ok", "fetch 1 all")
 	tc.transactf("ok", "fetch 1 body")
 	tc.transactf("ok", "fetch 1 binary[]")
@@ -826,9 +575,9 @@ func testScenario(t *testing.T, uidonly bool) {
 }
 
 func TestMailbox(t *testing.T) {
-	tc := start(t, false)
+	tc := start(t)
 	defer tc.close()
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
 
 	invalid := []string{
 		"e\u0301", // é but as e + acute, not unicode-normalized
@@ -848,16 +597,15 @@ func TestMailbox(t *testing.T) {
 }
 
 func TestMailboxDeleted(t *testing.T) {
-	tc := start(t, false)
+	tc := start(t)
 	defer tc.close()
+	tc.client.Login("mjl@mox.example", password0)
 
-	tc2 := startNoSwitchboard(t, false)
+	tc2 := startNoSwitchboard(t)
-	defer tc2.closeNoWait()
+	defer tc2.close()
+	tc2.client.Login("mjl@mox.example", password0)
 
-	tc.login("mjl@mox.example", password0)
+	tc.client.Create("testbox")
-	tc2.login("mjl@mox.example", password0)
-
-	tc.client.Create("testbox", nil)
 	tc2.client.Select("testbox")
 	tc.client.Delete("testbox")
 
@@ -871,23 +619,23 @@ func TestMailboxDeleted(t *testing.T) {
 	tc2.transactf("no", "uid fetch 1 all")
 	tc2.transactf("no", "store 1 flags ()")
 	tc2.transactf("no", "uid store 1 flags ()")
-	tc2.transactf("no", "copy 1 inbox")
+	tc2.transactf("bad", "copy 1 inbox") // msgseq 1 not available.
 	tc2.transactf("no", "uid copy 1 inbox")
-	tc2.transactf("no", "move 1 inbox")
+	tc2.transactf("bad", "move 1 inbox") // msgseq 1 not available.
 	tc2.transactf("no", "uid move 1 inbox")
 
 	tc2.transactf("ok", "unselect")
 
-	tc.client.Create("testbox", nil)
+	tc.client.Create("testbox")
 	tc2.client.Select("testbox")
 	tc.client.Delete("testbox")
 	tc2.transactf("ok", "close")
 }
 
 func TestID(t *testing.T) {
-	tc := start(t, false)
+	tc := start(t)
 	defer tc.close()
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
 
 	tc.transactf("ok", "id nil")
 	tc.xuntagged(imapclient.UntaggedID{"name": "mox", "version": moxvar.Version})
@@ -899,86 +647,54 @@ func TestID(t *testing.T) {
 }
 
 func TestSequence(t *testing.T) {
-	testSequence(t, false)
+	tc := start(t)
-}
-
-func TestSequenceUIDOnly(t *testing.T) {
-	testSequence(t, true)
-}
-
-func testSequence(t *testing.T, uidonly bool) {
-	tc := start(t, uidonly)
 	defer tc.close()
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
 	tc.client.Select("inbox")
 
 	tc.transactf("bad", "fetch * all") // ../rfc/9051:7018
-	tc.transactf("bad", "fetch 1:* all")
-	tc.transactf("bad", "fetch 1:2 all")
 	tc.transactf("bad", "fetch 1 all") // ../rfc/9051:7018
 
 	tc.transactf("ok", "uid fetch 1 all") // non-existing messages are OK for uids.
 	tc.transactf("ok", "uid fetch * all") // * is like uidnext, a non-existing message.
 
-	tc.transactf("ok", "uid search return (save) all") // Empty result.
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
-	tc.transactf("ok", "uid fetch $ uid")
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
-	tc.xuntagged()
+	tc.transactf("ok", "fetch 2:1,1 uid") // We reorder 2:1 to 1:2, but we don't deduplicate numbers.
-
-	tc.client.Append("inbox", makeAppend(exampleMsg))
-	tc.client.Append("inbox", makeAppend(exampleMsg))
-	if !uidonly {
-		tc.transactf("ok", "fetch 2:1,1 uid") // We reorder 2:1 to 1:2, and we deduplicate numbers.
-		tc.xuntagged(
-			tc.untaggedFetch(1, 1),
-			tc.untaggedFetch(2, 2),
-		)
-
-		tc.transactf("bad", "fetch 1:3 all")
-	}
-
-	tc.transactf("ok", "uid fetch * flags")
-	tc.xuntagged(tc.untaggedFetch(2, 2, imapclient.FetchFlags(nil)))
-
-	tc.transactf("ok", "uid fetch 3:* flags") // Because * is the last message, which is 2, the range becomes 3:2, which matches the last message.
-	tc.xuntagged(tc.untaggedFetch(2, 2, imapclient.FetchFlags(nil)))
-
-	tc.transactf("ok", "uid fetch *:3 flags")
-	tc.xuntagged(tc.untaggedFetch(2, 2, imapclient.FetchFlags(nil)))
-
-	tc.transactf("ok", "uid search return (save) all") // Empty result.
-	tc.transactf("ok", "uid fetch $ flags")
 	tc.xuntagged(
-		tc.untaggedFetch(1, 1, imapclient.FetchFlags(nil)),
+		imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1)}},
-		tc.untaggedFetch(2, 2, imapclient.FetchFlags(nil)),
+		imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2)}},
+		imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1)}},
 	)
+
+	tc.transactf("ok", "uid fetch 3:* uid") // Because * is the last message, which is 2, the range becomes 3:2, which matches the last message.
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2)}})
 }
 
 // Test that a message that is expunged by another session can be read as long as a
 // reference is held by a session. New sessions do not see the expunged message.
-func TestReference(t *testing.T) {
+// todo: possibly implement the additional reference counting. so far it hasn't been worth the trouble.
-	tc := start(t, false)
+func DisabledTestReference(t *testing.T) {
+	tc := start(t)
 	defer tc.close()
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
 	tc.client.Select("inbox")
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
 
-	tc2 := startNoSwitchboard(t, false)
+	tc2 := startNoSwitchboard(t)
-	defer tc2.closeNoWait()
+	defer tc2.close()
-	tc2.login("mjl@mox.example", password0)
+	tc2.client.Login("mjl@mox.example", password0)
 	tc2.client.Select("inbox")
 
-	tc.client.MSNStoreFlagsSet("1", true, `\Deleted`)
+	tc.client.StoreFlagsSet("1", true, `\Deleted`)
 	tc.client.Expunge()
 
-	tc3 := startNoSwitchboard(t, false)
+	tc3 := startNoSwitchboard(t)
-	defer tc3.closeNoWait()
+	defer tc3.close()
-	tc3.login("mjl@mox.example", password0)
+	tc3.client.Login("mjl@mox.example", password0)
 	tc3.transactf("ok", `list "" "inbox" return (status (messages))`)
-	tc3.xuntagged(
+	tc3.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"}, imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 0}})
-		mustParseUntagged(`* LIST () "/" Inbox`),
-		imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 0}},
-	)
 
 	tc2.transactf("ok", "fetch 1 rfc822.size")
-	tc2.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchRFC822Size(len(exampleMsg))))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchRFC822Size(len(exampleMsg))}})
 }

imapserver/starttls_test.go

@@ -7,22 +7,22 @@ import (
 )
 
 func TestStarttls(t *testing.T) {
-	tc := start(t, false)
+	tc := start(t)
-	tc.client.StartTLS(&tls.Config{InsecureSkipVerify: true})
+	tc.client.Starttls(&tls.Config{InsecureSkipVerify: true})
 	tc.transactf("bad", "starttls") // TLS already active.
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
 	tc.close()
 
-	tc = startArgs(t, false, true, true, false, true, "mjl")
+	tc = startArgs(t, true, true, false, true, "mjl")
 	tc.transactf("bad", "starttls") // TLS already active.
 	tc.close()
 
-	tc = startArgs(t, false, true, false, false, true, "mjl")
+	tc = startArgs(t, true, false, false, true, "mjl")
 	tc.transactf("no", `login "mjl@mox.example" "%s"`, password0)
-	tc.xcodeWord("PRIVACYREQUIRED")
+	tc.xcode("PRIVACYREQUIRED")
 	tc.transactf("no", "authenticate PLAIN %s", base64.StdEncoding.EncodeToString([]byte("\u0000mjl@mox.example\u0000"+password0)))
-	tc.xcodeWord("PRIVACYREQUIRED")
+	tc.xcode("PRIVACYREQUIRED")
-	tc.client.StartTLS(&tls.Config{InsecureSkipVerify: true})
+	tc.client.Starttls(&tls.Config{InsecureSkipVerify: true})
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
 	tc.close()
 }

imapserver/status_test.go

@@ -7,19 +7,11 @@ import (
 )
 
 func TestStatus(t *testing.T) {
-	testStatus(t, false)
-}
-
-func TestStatusUIDOnly(t *testing.T) {
-	testStatus(t, true)
-}
-
-func testStatus(t *testing.T, uidonly bool) {
 	defer mockUIDValidity()()
-	tc := start(t, uidonly)
+	tc := start(t)
 	defer tc.close()
 
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
 
 	tc.transactf("bad", "status")                      // Missing param.
 	tc.transactf("bad", "status inbox")                // Missing param.
@@ -27,8 +19,6 @@ func testStatus(t *testing.T, uidonly bool) {
 	tc.transactf("bad", "status inbox (uidvalidity) ") // Leftover data.
 	tc.transactf("bad", "status inbox (unknown)")      // Unknown attribute.
 
-	tc.transactf("no", "status expungebox (messages)") // No longer exists.
-
 	tc.transactf("ok", "status inbox (messages uidnext uidvalidity unseen deleted size recent appendlimit)")
 	tc.xuntagged(imapclient.UntaggedStatus{
 		Mailbox: "Inbox",
@@ -61,7 +51,7 @@ func testStatus(t *testing.T, uidonly bool) {
 	})
 
 	tc.client.Select("inbox")
-	tc.client.UIDStoreFlagsSet("1", true, `\Deleted`)
+	tc.client.StoreFlagsSet("1", true, `\Deleted`)
 	tc.transactf("ok", "status inbox (messages uidnext uidvalidity unseen deleted size recent appendlimit)")
 	tc.xuntagged(imapclient.UntaggedStatus{
 		Mailbox: "Inbox",

imapserver/store_test.go

@@ -8,74 +8,63 @@ import (
 )
 
 func TestStore(t *testing.T) {
-	testStore(t, false)
+	tc := start(t)
-}
-
-func TestStoreUIDOnly(t *testing.T) {
-	testStore(t, true)
-}
-
-func testStore(t *testing.T, uidonly bool) {
-	tc := start(t, uidonly)
 	defer tc.close()
 
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
-	tc.client.Enable(imapclient.CapIMAP4rev2)
+	tc.client.Enable("imap4rev2")
 
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
 	tc.client.Select("inbox")
 
+	uid1 := imapclient.FetchUID(1)
 	noflags := imapclient.FetchFlags(nil)
 
-	if !uidonly {
+	tc.transactf("ok", "store 1 flags.silent ()")
-		tc.transactf("ok", "store 1 flags.silent ()")
-		tc.xuntagged()
-	}
-
-	tc.transactf("ok", `uid store 1 flags ()`)
-	tc.xuntagged(tc.untaggedFetch(1, 1, noflags))
-	tc.transactf("ok", `uid fetch 1 flags`)
-	tc.xuntagged(tc.untaggedFetch(1, 1, noflags))
-
-	tc.transactf("ok", `uid store 1 flags.silent (\Seen)`)
 	tc.xuntagged()
-	tc.transactf("ok", `uid fetch 1 flags`)
-	tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchFlags{`\Seen`}))
 
-	tc.transactf("ok", `uid store 1 flags ($Junk)`)
+	tc.transactf("ok", `store 1 flags ()`)
-	tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchFlags{`$Junk`}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, noflags}})
-	tc.transactf("ok", `uid fetch 1 flags`)
+	tc.transactf("ok", `fetch 1 flags`)
-	tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchFlags{`$Junk`}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, noflags}})
 
-	tc.transactf("ok", `uid store 1 +flags ()`)
+	tc.transactf("ok", `store 1 flags.silent (\Seen)`)
-	tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchFlags{`$Junk`}))
+	tc.xuntagged()
-	tc.transactf("ok", `uid store 1 +flags (\Deleted)`)
+	tc.transactf("ok", `fetch 1 flags`)
-	tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchFlags{`\Deleted`, `$Junk`}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchFlags{`\Seen`}}})
-	tc.transactf("ok", `uid fetch 1 flags`)
-	tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchFlags{`\Deleted`, `$Junk`}))
 
-	tc.transactf("ok", `uid store 1 -flags \Deleted $Junk`)
+	tc.transactf("ok", `store 1 flags ($Junk)`)
-	tc.xuntagged(tc.untaggedFetch(1, 1, noflags))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchFlags{`$Junk`}}})
-	tc.transactf("ok", `uid fetch 1 flags`)
+	tc.transactf("ok", `fetch 1 flags`)
-	tc.xuntagged(tc.untaggedFetch(1, 1, noflags))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchFlags{`$Junk`}}})
 
-	if !uidonly {
+	tc.transactf("ok", `store 1 +flags ()`)
-		tc.transactf("bad", "store 2 flags ()") // ../rfc/9051:7018
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchFlags{`$Junk`}}})
-	}
+	tc.transactf("ok", `store 1 +flags (\Deleted)`)
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchFlags{`\Deleted`, `$Junk`}}})
+	tc.transactf("ok", `fetch 1 flags`)
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchFlags{`\Deleted`, `$Junk`}}})
+
+	tc.transactf("ok", `store 1 -flags \Deleted $Junk`)
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, noflags}})
+	tc.transactf("ok", `fetch 1 flags`)
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, noflags}})
+
+	tc.transactf("bad", "store 2 flags ()") // ../rfc/9051:7018
 
 	tc.transactf("ok", "uid store 1 flags ()")
-	tc.xuntagged(tc.untaggedFetch(1, 1, noflags))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, noflags}})
 
-	tc.transactf("ok", "uid store 1 flags (new)") // New flag.
+	tc.transactf("ok", "store 1 flags (new)") // New flag.
-	tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchFlags{"new"}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchFlags{"new"}}})
-	tc.transactf("ok", "uid store 1 flags (new new a b c)") // Duplicates are ignored.
+	tc.transactf("ok", "store 1 flags (new new a b c)") // Duplicates are ignored.
-	tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchFlags{"a", "b", "c", "new"}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchFlags{"a", "b", "c", "new"}}})
-	tc.transactf("ok", "uid store 1 +flags (new new c d e)")
+	tc.transactf("ok", "store 1 +flags (new new c d e)")
-	tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchFlags{"a", "b", "c", "d", "e", "new"}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchFlags{"a", "b", "c", "d", "e", "new"}}})
-	tc.transactf("ok", "uid store 1 -flags (new new e a c)")
+	tc.transactf("ok", "store 1 -flags (new new e a c)")
-	tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchFlags{"b", "d"}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchFlags{"b", "d"}}})
-	tc.transactf("ok", "uid store 1 flags ($Forwarded Different)")
+	tc.transactf("ok", "store 1 flags ($Forwarded Different)")
-	tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchFlags{"$Forwarded", "different"}))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchFlags{"$Forwarded", "different"}}})
 
 	tc.transactf("bad", "store")          // Need numset, flags and args.
 	tc.transactf("bad", "store 1")        // Need flags.
@@ -91,5 +80,5 @@ func testStore(t *testing.T, uidonly bool) {
 	flags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent a b c d different e new`, " ")
 	tc.xuntaggedOpt(false, imapclient.UntaggedFlags(flags))
 
-	tc.transactf("no", `uid store 1 flags ()`) // No permission to set flags.
+	tc.transactf("no", `store 1 flags ()`) // No permission to set flags.
 }

imapserver/subscribe_test.go

@@ -7,14 +7,14 @@ import (
 )
 
 func TestSubscribe(t *testing.T) {
-	tc := start(t, false)
+	tc := start(t)
 	defer tc.close()
 
-	tc2 := startNoSwitchboard(t, false)
+	tc2 := startNoSwitchboard(t)
-	defer tc2.closeNoWait()
+	defer tc2.close()
 
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
-	tc2.login("mjl@mox.example", password0)
+	tc2.client.Login("mjl@mox.example", password0)
 
 	tc.transactf("bad", "subscribe")       // Missing param.
 	tc.transactf("bad", "subscribe ")      // Missing param.

imapserver/uidonly_test.go

@@ -1,39 +0,0 @@
-package imapserver
-
-import (
-	"testing"
-)
-
-func TestUIDOnly(t *testing.T) {
-	tc := start(t, true)
-	defer tc.close()
-	tc.login("mjl@mox.example", password0)
-	tc.client.Select("inbox")
-
-	tc.transactf("bad", "Fetch 1")
-	tc.xcodeWord("UIDREQUIRED")
-	tc.transactf("bad", "Fetch 1")
-	tc.xcodeWord("UIDREQUIRED")
-	tc.transactf("bad", "Search 1")
-	tc.xcodeWord("UIDREQUIRED")
-	tc.transactf("bad", "Store 1 Flags ()")
-	tc.xcodeWord("UIDREQUIRED")
-	tc.transactf("bad", "Copy 1 Archive")
-	tc.xcodeWord("UIDREQUIRED")
-	tc.transactf("bad", "Move 1 Archive")
-	tc.xcodeWord("UIDREQUIRED")
-
-	// Sequence numbers in search program.
-	tc.transactf("bad", "Uid Search 1")
-	tc.xcodeWord("UIDREQUIRED")
-
-	// Sequence number in last qresync parameter.
-	tc.transactf("ok", "Enable Qresync")
-	tc.transactf("bad", "Select inbox (Qresync (1 5 (1,3,6 1,3,6)))")
-	tc.xcodeWord("UIDREQUIRED")
-	tc.client.Select("inbox") // Select again.
-
-	// Breaks connection.
-	tc.transactf("bad", "replace 1 inbox {1+}\r\nx")
-	tc.xcodeWord("UIDREQUIRED")
-}

imapserver/unselect_test.go

@@ -7,18 +7,10 @@ import (
 )
 
 func TestUnselect(t *testing.T) {
-	testUnselect(t, false)
+	tc := start(t)
-}
-
-func TestUnselectUIDOnly(t *testing.T) {
-	testUnselect(t, true)
-}
-
-func testUnselect(t *testing.T, uidonly bool) {
-	tc := start(t, uidonly)
 	defer tc.close()
 
-	tc.login("mjl@mox.example", password0)
+	tc.client.Login("mjl@mox.example", password0)
 	tc.client.Select("inbox")
 
 	tc.transactf("bad", "unselect bogus") // Leftover data.
@@ -26,8 +18,8 @@ func testUnselect(t *testing.T, uidonly bool) {
 	tc.transactf("no", "fetch 1 all") // Invalid when not selected.
 
 	tc.client.Select("inbox")
-	tc.client.Append("inbox", makeAppend(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
-	tc.client.UIDStoreFlagsAdd("1", true, `\Deleted`)
+	tc.client.StoreFlagsAdd("1", true, `\Deleted`)
 	tc.transactf("ok", "unselect")
 	tc.transactf("ok", "status inbox (messages)")
 	tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 1}}) // Message not removed.