diff --git a/queue/localserve.go b/queue/localserve.go
index 770f1ed..d8aaf52 100644
--- a/queue/localserve.go
+++ b/queue/localserve.go
@@ -82,8 +82,10 @@ func deliverLocalserve(ctx context.Context, log mlog.Log, msgs []*Msg, backoff t
 		}
 		msgr = store.FileMsgReader(m0.MsgPrefix, f)
 		defer func() {
-			err := msgr.Close()
-			log.Check(err, "closing message after delivery attempt")
+			if msgr != nil {
+				err := msgr.Close()
+				log.Check(err, "closing message after delivery attempt")
+			}
 		}()
 	}
 
@@ -99,5 +101,12 @@ func deliverLocalserve(ctx context.Context, log mlog.Log, msgs []*Msg, backoff t
 	if err != nil {
 		log.Infox("smtp transaction for delivery failed", err)
 	}
+
+	// Must close before processing, because that may try to remove the message file,
+	// and on Windows we can't have it open when we remove it.
+	cerr := msgr.Close()
+	log.Check(cerr, "closing message after delivery attempt")
+	msgr = nil
+
 	processDeliveries(log, m0, msgs, addr, "localhost", backoff, rcptErrs, err)
 }
diff --git a/queue/submit.go b/queue/submit.go
index f258968..c800561 100644
--- a/queue/submit.go
+++ b/queue/submit.go
@@ -215,8 +215,10 @@ func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
 		}
 		msgr = store.FileMsgReader(m0.MsgPrefix, f)
 		defer func() {
-			err := msgr.Close()
-			qlog.Check(err, "closing message after delivery attempt")
+			if msgr != nil {
+				err := msgr.Close()
+				qlog.Check(err, "closing message after delivery attempt")
+			}
 		}()
 	}
 
@@ -230,6 +232,13 @@ func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
 	if submiterr != nil {
 		qlog.Infox("smtp transaction for delivery failed", submiterr)
 	}
+
+	// Must close before processing, because that may try to remove the message file,
+	// and on Windows we can't have it open when we remove it.
+	cerr := msgr.Close()
+	qlog.Check(cerr, "closing message after delivery attempt")
+	msgr = nil
+
 	failed, delivered = processDeliveries(qlog, m0, msgs, addr, transport.Host, backoff, rcptErrs, submiterr)
 }