infrastructure/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2025-07-13 00:54:38 +03:00
Code Issues Packages Projects Releases Wiki Activity

imapserver: return all the extensible fields for bodystructure, notably for content-disposition

Browse Source 
The gmail iOS/Android app were showing mime image parts as (garbled) text
instead of rendering them as image. By returning all the optional fields in the
bodystructure fetch attribute, the gmail app renders the image as expected by
the user. So we now add all fields. We didn't before, because we weren't
keeping track of Content-MD5, Content-Language and Content-Location header
fields, since they aren't that useful.

Messages in mailboxes have to be reparsed:
	./mox reparse

Without reparsing, imap responses will claim the extra fields
(content-disposition) are absent for existing messages, instead of not claiming
anything at all, which is what we did before.

Accounts and all/some mailboxes can get their "uid validity" bumped ("./mox
bumpuidvalidity $account [$mailbox]"), which should trigger clients to load all
messages from scratch, but gmail doesn't appear to notice, so it would be
better to remove & add the account in gmail.

For issue #327, also relevant to issue #217.
This commit is contained in:
Mechiel Lukkien
2025-04-05 15:46:17 +02:00
parent 69d2699961
commit 2defbce0bc
11 changed files with 185 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

124
imapserver/fetch.go
View File

@ -9,17 +9,18 @@ import (
"io"
"log/slog"
"maps"
"mime"
"net/textproto"
"sort"
"slices"
"strings"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/store"
"slices"
)
// functions to handle fetch attribute requests are defined on fetchCmd.
@ -483,7 +484,7 @@ func (cmd *fetchCmd) xprocessAtt(a fetchAtt) []token {
case "BODYSTRUCTURE":
_, part := cmd.xensureParsed()
bs := xbodystructure(part, true)
bs := xbodystructure(cmd.conn.log, part, true)
return []token{bare("BODYSTRUCTURE"), bs}
case "BODY":
@ -768,7 +769,7 @@ func (cmd *fetchCmd) xbody(a fetchAtt) (string, token) {
if a.section == nil {
// Non-extensible form of BODYSTRUCTURE.
return a.field, xbodystructure(part, false)
return a.field, xbodystructure(cmd.conn.log, part, false)
}
cmd.peekOrSeen(a.peek)
@ -939,24 +940,17 @@ func (cmd *fetchCmd) sectionMsgtextName(smt *sectionMsgtext) string {
return s
}
func bodyFldParams(params map[string]string) token {
if len(params) == 0 {
func bodyFldParams(p *message.Part) token {
if len(p.ContentTypeParams) == 0 {
return nilt
}
params := make(listspace, 0, 2*len(p.ContentTypeParams))
// Ensure same ordering, easier for testing.
var keys []string
for k := range params {
keys = append(keys, k)
for _, k := range slices.Sorted(maps.Keys(p.ContentTypeParams)) {
v := p.ContentTypeParams[k]
params = append(params, string0(strings.ToUpper(k)), string0(v))
}
sort.Strings(keys)
l := make(listspace, 2*len(keys))
i := 0
for _, k := range keys {
l[i] = string0(strings.ToUpper(k))
l[i+1] = string0(params[k])
i += 2
}
return l
return params
}
func bodyFldEnc(s string) token {
@ -968,26 +962,80 @@ func bodyFldEnc(s string) token {
return string0(s)
}
func bodyFldMd5(p *message.Part) token {
if p.ContentMD5 == "" {
return nilt
}
return string0(p.ContentMD5)
}
func bodyFldDisp(log mlog.Log, p *message.Part) token {
if p.ContentDisposition == "" {
return nilt
}
// ../rfc/9051:5989
// mime.ParseMediaType recombines parameter value continuations like "title*0" and
// "title*1" into "title". ../rfc/2231:147
// And decodes character sets and removes language tags, like
// "title*0*=us-ascii'en'hello%20world. ../rfc/2231:210
disp, params, err := mime.ParseMediaType(p.ContentDisposition)
if err != nil {
log.Debugx("parsing content-disposition, ignoring", err, slog.String("header", p.ContentDisposition))
return nilt
} else if len(params) == 0 {
log.Debug("content-disposition has no parameters, ignoring", slog.String("header", p.ContentDisposition))
return nilt
}
var fields listspace
for _, k := range slices.Sorted(maps.Keys(params)) {
fields = append(fields, string0(k), string0(params[k]))
}
return listspace{string0(disp), fields}
}
func bodyFldLang(p *message.Part) token {
// todo: ../rfc/3282:86 ../rfc/5646:218 we currently just split on comma and trim space, should properly parse header.
if p.ContentLanguage == "" {
return nilt
}
var l listspace
for _, s := range strings.Split(p.ContentLanguage, ",") {
s = strings.TrimSpace(s)
if s == "" {
return string0(p.ContentLanguage)
}
l = append(l, string0(s))
}
return l
}
func bodyFldLoc(p *message.Part) token {
if p.ContentLocation == "" {
return nilt
}
return string0(p.ContentLocation)
}
// xbodystructure returns a "body".
// calls itself for multipart messages and message/{rfc822,global}.
func xbodystructure(p *message.Part, extensible bool) token {
func xbodystructure(log mlog.Log, p *message.Part, extensible bool) token {
if p.MediaType == "MULTIPART" {
// Multipart, ../rfc/9051:6355 ../rfc/9051:6411
var bodies concat
for i := range p.Parts {
bodies = append(bodies, xbodystructure(&p.Parts[i], extensible))
bodies = append(bodies, xbodystructure(log, &p.Parts[i], extensible))
}
r := listspace{bodies, string0(p.MediaSubType)}
// ../rfc/9051:6371
if extensible {
if len(p.ContentTypeParams) == 0 {
r = append(r, nilt)
} else {
params := make(listspace, 0, 2*len(p.ContentTypeParams))
for k, v := range p.ContentTypeParams {
params = append(params, string0(k), string0(v))
}
r = append(r, params)
}
r = append(r,
bodyFldParams(p),
bodyFldDisp(log, p),
bodyFldLang(p),
bodyFldLoc(p),
)
}
return r
}
@ -999,7 +1047,7 @@ func xbodystructure(p *message.Part, extensible bool) token {
r = listspace{
dquote("TEXT"), string0(p.MediaSubType), // ../rfc/9051:6739
// ../rfc/9051:6376
bodyFldParams(p.ContentTypeParams), // ../rfc/9051:6401
bodyFldParams(p), // ../rfc/9051:6401
nilOrString(p.ContentID),
nilOrString(p.ContentDescription),
bodyFldEnc(p.ContentTransferEncoding),
@ -1012,13 +1060,13 @@ func xbodystructure(p *message.Part, extensible bool) token {
r = listspace{
dquote("MESSAGE"), dquote(p.MediaSubType), // ../rfc/9051:6732
// ../rfc/9051:6376
bodyFldParams(p.ContentTypeParams), // ../rfc/9051:6401
bodyFldParams(p), // ../rfc/9051:6401
nilOrString(p.ContentID),
nilOrString(p.ContentDescription),
bodyFldEnc(p.ContentTransferEncoding),
number(p.EndOffset - p.BodyOffset),
xenvelope(p.Message),
xbodystructure(p.Message, extensible),
xbodystructure(log, p.Message, extensible),
number(p.RawLineCount), // todo: or mp.RawLineCount?
}
} else {
@ -1033,13 +1081,21 @@ func xbodystructure(p *message.Part, extensible bool) token {
r = listspace{
media, string0(p.MediaSubType), // ../rfc/9051:6723
// ../rfc/9051:6376
bodyFldParams(p.ContentTypeParams), // ../rfc/9051:6401
bodyFldParams(p), // ../rfc/9051:6401
nilOrString(p.ContentID),
nilOrString(p.ContentDescription),
bodyFldEnc(p.ContentTransferEncoding),
number(p.EndOffset - p.BodyOffset),
}
}
// todo: if "extensible", we could add the value of the "content-md5" header. we don't have it in our parsed data structure, so we don't add it. likely no one would use it, also not any of the other optional fields. ../rfc/9051:6366
if extensible {
// ../rfc/9051:6366
r = append(r,
bodyFldMd5(p),
bodyFldDisp(log, p),
bodyFldLang(p),
bodyFldLoc(p),
)
}
return r
}

43
imapserver/fetch_test.go
View File

@ -35,20 +35,23 @@ func TestFetch(t *testing.T) {
MessageID: "<B27397-0100000@Blurdybloop.example>",
}
noflags := imapclient.FetchFlags(nil)
bodystructbody1 := imapclient.BodyTypeText{
MediaType: "TEXT",
MediaSubtype: "PLAIN",
BodyFields: imapclient.BodyFields{
Params: [][2]string{[...]string{"CHARSET", "US-ASCII"}},
Octets: 57,
},
Lines: 2,
}
bodyxstructure1 := imapclient.FetchBodystructure{
RespAttr: "BODY",
Body: imapclient.BodyTypeText{
MediaType: "TEXT",
MediaSubtype: "PLAIN",
BodyFields: imapclient.BodyFields{
Params: [][2]string{[...]string{"CHARSET", "US-ASCII"}},
Octets: 57,
},
Lines: 2,
},
Body: bodystructbody1,
}
bodystructure1 := bodyxstructure1
bodystructure1.RespAttr = "BODYSTRUCTURE"
bodystructbody1.Ext = &imapclient.BodyExtension1Part{}
bodystructure1.Body = bodystructbody1
split := strings.SplitN(exampleMsg, "\r\n\r\n", 2)
exampleMsgHeader := split[0] + "\r\n\r\n"
@ -288,17 +291,17 @@ func TestFetch(t *testing.T) {
RespAttr: "BODYSTRUCTURE",
Body: imapclient.BodyTypeMpart{
Bodies: []any{
imapclient.BodyTypeBasic{BodyFields: imapclient.BodyFields{Octets: 275}},
imapclient.BodyTypeText{MediaType: "TEXT", MediaSubtype: "PLAIN", BodyFields: imapclient.BodyFields{Params: [][2]string{{"CHARSET", "US-ASCII"}}, Octets: 114}, Lines: 3},
imapclient.BodyTypeBasic{BodyFields: imapclient.BodyFields{Octets: 275}, Ext: &imapclient.BodyExtension1Part{}},
imapclient.BodyTypeText{MediaType: "TEXT", MediaSubtype: "PLAIN", BodyFields: imapclient.BodyFields{Params: [][2]string{{"CHARSET", "US-ASCII"}}, Octets: 114}, Lines: 3, Ext: &imapclient.BodyExtension1Part{}},
imapclient.BodyTypeMpart{
Bodies: []any{
imapclient.BodyTypeBasic{MediaType: "AUDIO", MediaSubtype: "BASIC", BodyFields: imapclient.BodyFields{CTE: "BASE64", Octets: 22}},
imapclient.BodyTypeBasic{MediaType: "IMAGE", MediaSubtype: "JPEG", BodyFields: imapclient.BodyFields{CTE: "BASE64"}},
imapclient.BodyTypeBasic{MediaType: "AUDIO", MediaSubtype: "BASIC", BodyFields: imapclient.BodyFields{CTE: "BASE64", Octets: 22}, Ext: &imapclient.BodyExtension1Part{}},
imapclient.BodyTypeBasic{MediaType: "IMAGE", MediaSubtype: "JPEG", BodyFields: imapclient.BodyFields{CTE: "BASE64"}, Ext: &imapclient.BodyExtension1Part{Disposition: "inline", DispositionParams: [][2]string{{"filename", "image.jpg"}}}},
},
MediaSubtype: "PARALLEL",
Ext: &imapclient.BodyExtensionMpart{Params: [][2]string{{"boundary", "unique-boundary-2"}}},
Ext: &imapclient.BodyExtensionMpart{Params: [][2]string{{"BOUNDARY", "unique-boundary-2"}}},
},
imapclient.BodyTypeText{MediaType: "TEXT", MediaSubtype: "ENRICHED", BodyFields: imapclient.BodyFields{Octets: 145}, Lines: 5},
imapclient.BodyTypeText{MediaType: "TEXT", MediaSubtype: "ENRICHED", BodyFields: imapclient.BodyFields{Octets: 145}, Lines: 5, Ext: &imapclient.BodyExtension1Part{}},
imapclient.BodyTypeMsg{
MediaType: "MESSAGE",
MediaSubtype: "RFC822",
@ -311,12 +314,17 @@ func TestFetch(t *testing.T) {
To: []imapclient.Address{{Name: "mox", Adl: "", Mailbox: "info", Host: "mox.example"}},
},
Bodystructure: imapclient.BodyTypeText{
MediaType: "TEXT", MediaSubtype: "PLAIN", BodyFields: imapclient.BodyFields{Params: [][2]string{{"CHARSET", "ISO-8859-1"}}, CTE: "QUOTED-PRINTABLE", Octets: 51}, Lines: 1},
MediaType: "TEXT", MediaSubtype: "PLAIN", BodyFields: imapclient.BodyFields{Params: [][2]string{{"CHARSET", "ISO-8859-1"}}, CTE: "QUOTED-PRINTABLE", Octets: 51}, Lines: 1, Ext: &imapclient.BodyExtension1Part{}},
Lines: 7,
Ext: &imapclient.BodyExtension1Part{
MD5: "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=",
Language: []string{"en", "de"},
Location: "http://localhost",
},
},
},
MediaSubtype: "MIXED",
Ext: &imapclient.BodyExtensionMpart{Params: [][2]string{{"boundary", "unique-boundary-1"}}},
Ext: &imapclient.BodyExtensionMpart{Params: [][2]string{{"BOUNDARY", "unique-boundary-1"}}},
},
}
tc.client.Append("inbox", makeAppendTime(nestedMessage, received))
@ -390,6 +398,7 @@ aGVsbG8NCndvcmxkDQo=
--unique-boundary-2
Content-Type: image/jpeg
Content-Transfer-Encoding: base64
Content-Disposition: inline; filename=image.jpg
--unique-boundary-2--

4
imapserver/server_test.go
View File

@ -118,6 +118,7 @@ aGVsbG8NCndvcmxkDQo=
--unique-boundary-2
Content-Type: image/jpeg
Content-Transfer-Encoding: base64
Content-Disposition: inline; filename=image.jpg
--unique-boundary-2--
@ -133,6 +134,9 @@ Isn't it
--unique-boundary-1
Content-Type: message/rfc822
Content-MD5: MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=
Content-Language: en,de
Content-Location: http://localhost
From: info@mox.example
To: mox <info@mox.example>