infrastructure/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2025-07-12 17:04:39 +03:00
Code Issues Packages Projects Releases Wiki Activity

make mox compile on windows, without "mox serve" but with working "mox localserve"

Browse Source 
getting mox to compile required changing code in only a few places where
package "syscall" was used: for accessing file access times and for umask
handling. an open problem is how to start a process as an unprivileged user on
windows.  that's why "mox serve" isn't implemented yet. and just finding a way
to implement it now may not be good enough in the near future: we may want to
starting using a more complete privilege separation approach, with a process
handling sensitive tasks (handling private keys, authentication), where we may
want to pass file descriptors between processes. how would that work on
windows?

anyway, getting mox to compile for windows doesn't mean it works properly on
windows. the largest issue: mox would normally open a file, rename or remove
it, and finally close it. this happens during message delivery. that doesn't
work on windows, the rename/remove would fail because the file is still open.
so this commit swaps many "remove" and "close" calls. renames are a longer
story: message delivery had two ways to deliver: with "consuming" the
(temporary) message file (which would rename it to its final destination), and
without consuming (by hardlinking the file, falling back to copying). the last
delivery to a recipient of a message (and the only one in the common case of a
single recipient) would consume the message, and the earlier recipients would
not.  during delivery, the already open message file was used, to parse the
message.  we still want to use that open message file, and the caller now stays
responsible for closing it, but we no longer try to rename (consume) the file.
we always hardlink (or copy) during delivery (this works on windows), and the
caller is responsible for closing and removing (in that order) the original
temporary file. this does cost one syscall more. but it makes the delivery code
(responsibilities) a bit simpler.

there is one more obvious issue: the file system path separator. mox already
used the "filepath" package to join paths in many places, but not everywhere.
and it still used strings with slashes for local file access. with this commit,
the code now uses filepath.FromSlash for path strings with slashes, uses
"filepath" in a few more places where it previously didn't. also switches from
"filepath" to regular "path" package when handling mailbox names in a few
places, because those always use forward slashes, regardless of local file
system conventions.  windows can handle forward slashes when opening files, so
test code that passes path strings with forward slashes straight to go stdlib
file i/o functions are left unchanged to reduce code churn. the regular
non-test code, or test code that uses path strings in places other than
standard i/o functions, does have the paths converted for consistent paths
(otherwise we would end up with paths with mixed forward/backward slashes in
log messages).

windows cannot dup a listening socket. for "mox localserve", it isn't
important, and we can work around the issue. the current approach for "mox
serve" (forking a process and passing file descriptors of listening sockets on
"privileged" ports) won't work on windows. perhaps it isn't needed on windows,
and any user can listen on "privileged" ports? that would be welcome.

on windows, os.Open cannot open a directory, so we cannot call Sync on it after
message delivery. a cursory internet search indicates that directories cannot
be synced on windows. the story is probably much more nuanced than that, with
long deep technical details/discussions/disagreement/confusion, like on unix.
for "mox localserve" we can get away with making syncdir a no-op.
This commit is contained in:
Mechiel Lukkien
2023-10-14 10:54:07 +02:00
parent 96774de8d6
commit 28fae96a9b
78 changed files with 1155 additions and 938 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
queue/dsn.go
View File

@ -154,12 +154,11 @@ func queueDSN(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg stri
return
}
defer func() {
if msgFile != nil {
err := os.Remove(msgFile.Name())
log.Check(err, "removing message file", mlog.Field("path", msgFile.Name()))
err = msgFile.Close()
log.Check(err, "closing message file")
}
name := msgFile.Name()
err := msgFile.Close()
log.Check(err, "closing message file")
err = os.Remove(name)
log.Check(err, "removing message file", mlog.Field("path", name))
}()
msgWriter := message.NewWriter(msgFile)
@ -174,12 +173,9 @@ func queueDSN(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg stri
MsgPrefix: []byte{},
}
acc.WithWLock(func() {
if err := acc.DeliverMailbox(log, mailbox, msg, msgFile, true); err != nil {
if err := acc.DeliverMailbox(log, mailbox, msg, msgFile); err != nil {
qlog("delivering dsn to mailbox", err)
return
}
})
err = msgFile.Close()
log.Check(err, "closing dsn file")
msgFile = nil
}

16
queue/queue.go
View File

@ -118,7 +118,7 @@ func (m Msg) MessagePath() string {
// Init opens the queue database without starting delivery.
func Init() error {
qpath := mox.DataDirPath("queue/index.db")
qpath := mox.DataDirPath(filepath.FromSlash("queue/index.db"))
os.MkdirAll(filepath.Dir(qpath), 0770)
isNew := false
if _, err := os.Stat(qpath); err != nil && os.IsNotExist(err) {
@ -176,14 +176,11 @@ func Count(ctx context.Context) (int, error) {
// Add a new message to the queue. The queue is kicked immediately to start a
// first delivery attempt.
//
// If consumeFile is true, it is removed as part of delivery (by rename or copy
// and remove). msgFile is never closed by Add.
//
// dnsutf8Opt is a utf8-version of the message, to be used only for DNSs. If set,
// this data is used as the message when delivering the DSN and the remote SMTP
// server supports SMTPUTF8. If the remote SMTP server does not support SMTPUTF8,
// the regular non-utf8 message is delivered.
func Add(ctx context.Context, log *mlog.Log, senderAccount string, mailFrom, rcptTo smtp.Path, has8bit, smtputf8 bool, size int64, messageID string, msgPrefix []byte, msgFile *os.File, dsnutf8Opt []byte, consumeFile bool) (int64, error) {
func Add(ctx context.Context, log *mlog.Log, senderAccount string, mailFrom, rcptTo smtp.Path, has8bit, smtputf8 bool, size int64, messageID string, msgPrefix []byte, msgFile *os.File, dsnutf8Opt []byte) (int64, error) {
// todo: Add should accept multiple rcptTo if they are for the same domain. so we can queue them for delivery in one (or just a few) session(s), transferring the data only once. ../rfc/5321:3759
if Localserve {
@ -202,7 +199,7 @@ func Add(ctx context.Context, log *mlog.Log, senderAccount string, mailFrom, rcp
conf, _ := acc.Conf()
dest := conf.Destinations[mailFrom.String()]
acc.WithWLock(func() {
err = acc.DeliverDestination(log, dest, &m, msgFile, consumeFile)
err = acc.DeliverDestination(log, dest, &m, msgFile)
})
if err != nil {
return 0, fmt.Errorf("delivering message: %v", err)
@ -239,12 +236,7 @@ func Add(ctx context.Context, log *mlog.Log, senderAccount string, mailFrom, rcp
}()
dstDir := filepath.Dir(dst)
os.MkdirAll(dstDir, 0770)
if consumeFile {
if err := os.Rename(msgFile.Name(), dst); err != nil {
// Could be due to cross-filesystem rename. Users shouldn't configure their systems that way.
return 0, fmt.Errorf("move message into queue dir: %w", err)
}
} else if err := moxio.LinkOrCopy(log, dst, msgFile.Name(), nil, true); err != nil {
if err := moxio.LinkOrCopy(log, dst, msgFile.Name(), nil, true); err != nil {
return 0, fmt.Errorf("linking/copying message to new file: %s", err)
} else if err := moxio.SyncDir(dstDir); err != nil {
return 0, fmt.Errorf("sync directory: %v", err)

36
queue/queue_test.go
View File

@ -13,6 +13,7 @@ import (
"math/big"
"net"
"os"
"path/filepath"
"strings"
"testing"
"time"
@ -40,7 +41,7 @@ func setup(t *testing.T) (*store.Account, func()) {
// Prepare config so email can be delivered to mjl@mox.example.
os.RemoveAll("../testdata/queue/data")
mox.Context = ctxbg
mox.ConfigStaticPath = "../testdata/queue/mox.conf"
mox.ConfigStaticPath = filepath.FromSlash("../testdata/queue/mox.conf")
mox.MustLoadConfig(true, false)
acc, err := store.OpenAccount("mjl")
tcheck(t, err, "open account")
@ -86,13 +87,15 @@ func TestQueue(t *testing.T) {
}
path := smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "mox.example"}}}
_, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, prepareFile(t), nil, true)
mf := prepareFile(t)
defer os.Remove(mf.Name())
defer mf.Close()
_, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf, nil)
tcheck(t, err, "add message to queue for delivery")
mf2 := prepareFile(t)
_, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf2, nil, false)
_, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf, nil)
tcheck(t, err, "add message to queue for delivery")
os.Remove(mf2.Name())
msgs, err = List(ctxbg)
tcheck(t, err, "listing queue")
@ -385,7 +388,7 @@ func TestQueue(t *testing.T) {
// Add a message to be delivered with submit because of its route.
topath := smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "submit.example"}}}
_, err = Add(ctxbg, xlog, "mjl", path, topath, false, false, int64(len(testmsg)), "<test@localhost>", nil, prepareFile(t), nil, true)
_, err = Add(ctxbg, xlog, "mjl", path, topath, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf, nil)
tcheck(t, err, "add message to queue for delivery")
wasNetDialer = testDeliver(fakeSubmitServer)
if !wasNetDialer {
@ -393,7 +396,7 @@ func TestQueue(t *testing.T) {
}
// Add a message to be delivered with submit because of explicitly configured transport, that uses TLS.
msgID, err := Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, prepareFile(t), nil, true)
msgID, err := Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf, nil)
tcheck(t, err, "add message to queue for delivery")
transportSubmitTLS := "submittls"
n, err = Kick(ctxbg, msgID, "", "", &transportSubmitTLS)
@ -417,7 +420,7 @@ func TestQueue(t *testing.T) {
}
// Add a message to be delivered with socks.
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<socks@localhost>", nil, prepareFile(t), nil, true)
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<socks@localhost>", nil, mf, nil)
tcheck(t, err, "add message to queue for delivery")
transportSocks := "socks"
n, err = Kick(ctxbg, msgID, "", "", &transportSocks)
@ -431,7 +434,7 @@ func TestQueue(t *testing.T) {
}
// Add message to be delivered with opportunistic TLS verification.
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, prepareFile(t), nil, true)
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, mf, nil)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
tcheck(t, err, "kick queue")
@ -441,7 +444,7 @@ func TestQueue(t *testing.T) {
testDeliver(fakeSMTPSTARTTLSServer)
// Test fallback to plain text with TLS handshake fails.
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<badtls@localhost>", nil, prepareFile(t), nil, true)
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<badtls@localhost>", nil, mf, nil)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
tcheck(t, err, "kick queue")
@ -457,7 +460,7 @@ func TestQueue(t *testing.T) {
{Usage: adns.TLSAUsageDANEEE, Selector: adns.TLSASelectorSPKI, MatchType: adns.TLSAMatchTypeFull, CertAssoc: moxCert.Leaf.RawSubjectPublicKeyInfo},
},
}
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<dane@localhost>", nil, prepareFile(t), nil, true)
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<dane@localhost>", nil, mf, nil)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
tcheck(t, err, "kick queue")
@ -472,7 +475,7 @@ func TestQueue(t *testing.T) {
{},
},
}
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<daneunusable@localhost>", nil, prepareFile(t), nil, true)
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<daneunusable@localhost>", nil, mf, nil)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
tcheck(t, err, "kick queue")
@ -489,7 +492,7 @@ func TestQueue(t *testing.T) {
{Usage: adns.TLSAUsageDANEEE, Selector: adns.TLSASelectorSPKI, MatchType: adns.TLSAMatchTypeFull, CertAssoc: make([]byte, sha256.Size)},
},
}
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<daneinsecure@localhost>", nil, prepareFile(t), nil, true)
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<daneinsecure@localhost>", nil, mf, nil)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
tcheck(t, err, "kick queue")
@ -504,7 +507,7 @@ func TestQueue(t *testing.T) {
resolver.TLSA = nil
// Add another message that we'll fail to deliver entirely.
_, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, prepareFile(t), nil, true)
_, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf, nil)
tcheck(t, err, "add message to queue for delivery")
msgs, err = List(ctxbg)
@ -660,7 +663,10 @@ func TestQueueStart(t *testing.T) {
}
path := smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "mox.example"}}}
_, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, prepareFile(t), nil, true)
mf := prepareFile(t)
defer os.Remove(mf.Name())
defer mf.Close()
_, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf, nil)
tcheck(t, err, "add message to queue for delivery")
checkDialed(true)