infrastructure/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2025-07-19 03:26:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

imapserver: implement PREVIEW extension (RFC 8970), and store previews in message database

Browse Source 
We were already generating previews of plain text parts for the webmail
interface, but we didn't store them, so were generating the previews each time
messages were listed.

Now we store previews in the database for faster handling. And we also generate
previews for html parts if needed. We use the first part that has textual
content.

For IMAP, the previews can be requested by an IMAP client. When we get the
"LAZY" variant, which doesn't require us to generate a preview, we generate it
anyway, because it should be fast enough. So don't make clients first ask for
"PREVIEW (LAZY)" and then again a request for "PREVIEW".

We now also generate a preview when a message is added to the account. Except
for imports. It would slow us down, the previews aren't urgent, and they will
be generated on-demand at first-request.
This commit is contained in:
Mechiel Lukkien
2025-03-28 16:57:44 +01:00
parent 8b418a9ca2
commit aa631c604c
23 changed files with 735 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
webmail/api.ts
View File

@ -299,12 +299,11 @@ export interface EventViewMsgs {
// message.Part, made for the needs of the message items in the message list.
// messages.
export interface MessageItem {
Message: Message // Without ParsedBuf and MsgPrefix, for size.
Message: Message // Without ParsedBuf and MsgPrefix, for size. With Preview, even with value not yet stored in the database.
Envelope: MessageEnvelope
Attachments?: Attachment[] | null
IsSigned: boolean
IsEncrypted: boolean
FirstLine: string // Of message body, for showing as preview.
MatchQuery: boolean // If message does not match query, it can still be included because of threading.
MoreHeaders?: (string[] | null)[] | null // All headers from store.Settings.ShowHeaders that are present.
}
@ -378,6 +377,7 @@ export interface Message {
Size: number
TrainedJunk?: boolean | null // If nil, no training done yet. Otherwise, true is trained as junk, false trained as nonjunk.
MsgPrefix?: string | null // Typically holds received headers and/or header separator.
Preview?: string | null // If non-nil, a preview of the message based on text and/or html parts of the message. Used in the webmail and IMAP PREVIEW extension. If non-nil, it is empty if no preview could be created, or the message has not textual content or couldn't be parsed. Previews are typically created when delivering a message, but not when importing messages, for speed. Previews are generated on first request (in the webmail, or through the IMAP fetch attribute "PREVIEW" (without "LAZY")), and stored with the message at that time. The preview is at most 256 characters (can be more bytes), with detected quoted text replaced with "[..."].
ParsedBuf?: string | null // ParsedBuf message structure. Currently saved as JSON of message.Part because bstore cannot yet store recursive types. Created when first needed, and saved in the database. todo: once replaced with non-json storage, remove date fixup in ../message/part.go.
}
@ -625,8 +625,8 @@ export const types: TypenameMap = {
"EventViewErr": {"Name":"EventViewErr","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"RequestID","Docs":"","Typewords":["int64"]},{"Name":"Err","Docs":"","Typewords":["string"]}]},
"EventViewReset": {"Name":"EventViewReset","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"RequestID","Docs":"","Typewords":["int64"]}]},
"EventViewMsgs": {"Name":"EventViewMsgs","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"RequestID","Docs":"","Typewords":["int64"]},{"Name":"MessageItems","Docs":"","Typewords":["[]","[]","MessageItem"]},{"Name":"ParsedMessage","Docs":"","Typewords":["nullable","ParsedMessage"]},{"Name":"ViewEnd","Docs":"","Typewords":["bool"]}]},
"MessageItem": {"Name":"MessageItem","Docs":"","Fields":[{"Name":"Message","Docs":"","Typewords":["Message"]},{"Name":"Envelope","Docs":"","Typewords":["MessageEnvelope"]},{"Name":"Attachments","Docs":"","Typewords":["[]","Attachment"]},{"Name":"IsSigned","Docs":"","Typewords":["bool"]},{"Name":"IsEncrypted","Docs":"","Typewords":["bool"]},{"Name":"FirstLine","Docs":"","Typewords":["string"]},{"Name":"MatchQuery","Docs":"","Typewords":["bool"]},{"Name":"MoreHeaders","Docs":"","Typewords":["[]","[]","string"]}]},
"Message": {"Name":"Message","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"CreateSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Expunged","Docs":"","Typewords":["bool"]},{"Name":"IsReject","Docs":"","Typewords":["bool"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"MailboxOrigID","Docs":"","Typewords":["int64"]},{"Name":"MailboxDestinedID","Docs":"","Typewords":["int64"]},{"Name":"Received","Docs":"","Typewords":["timestamp"]},{"Name":"SaveDate","Docs":"","Typewords":["nullable","timestamp"]},{"Name":"RemoteIP","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked1","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked2","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked3","Docs":"","Typewords":["string"]},{"Name":"EHLODomain","Docs":"","Typewords":["string"]},{"Name":"MailFrom","Docs":"","Typewords":["string"]},{"Name":"MailFromLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"MailFromDomain","Docs":"","Typewords":["string"]},{"Name":"RcptToLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"RcptToDomain","Docs":"","Typewords":["string"]},{"Name":"MsgFromLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"MsgFromDomain","Docs":"","Typewords":["string"]},{"Name":"MsgFromOrgDomain","Docs":"","Typewords":["string"]},{"Name":"EHLOValidated","Docs":"","Typewords":["bool"]},{"Name":"MailFromValidated","Docs":"","Typewords":["bool"]},{"Name":"MsgFromValidated","Docs":"","Typewords":["bool"]},{"Name":"EHLOValidation","Docs":"","Typewords":["Validation"]},{"Name":"MailFromValidation","Docs":"","Typewords":["Validation"]},{"Name":"MsgFromValidation","Docs":"","Typewords":["Validation"]},{"Name":"DKIMDomains","Docs":"","Typewords":["[]","string"]},{"Name":"OrigEHLODomain","Docs":"","Typewords":["string"]},{"Name":"OrigDKIMDomains","Docs":"","Typewords":["[]","string"]},{"Name":"MessageID","Docs":"","Typewords":["string"]},{"Name":"SubjectBase","Docs":"","Typewords":["string"]},{"Name":"MessageHash","Docs":"","Typewords":["nullable","string"]},{"Name":"ThreadID","Docs":"","Typewords":["int64"]},{"Name":"ThreadParentIDs","Docs":"","Typewords":["[]","int64"]},{"Name":"ThreadMissingLink","Docs":"","Typewords":["bool"]},{"Name":"ThreadMuted","Docs":"","Typewords":["bool"]},{"Name":"ThreadCollapsed","Docs":"","Typewords":["bool"]},{"Name":"IsMailingList","Docs":"","Typewords":["bool"]},{"Name":"DSN","Docs":"","Typewords":["bool"]},{"Name":"ReceivedTLSVersion","Docs":"","Typewords":["uint16"]},{"Name":"ReceivedTLSCipherSuite","Docs":"","Typewords":["uint16"]},{"Name":"ReceivedRequireTLS","Docs":"","Typewords":["bool"]},{"Name":"Seen","Docs":"","Typewords":["bool"]},{"Name":"Answered","Docs":"","Typewords":["bool"]},{"Name":"Flagged","Docs":"","Typewords":["bool"]},{"Name":"Forwarded","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Notjunk","Docs":"","Typewords":["bool"]},{"Name":"Deleted","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Phishing","Docs":"","Typewords":["bool"]},{"Name":"MDNSent","Docs":"","Typewords":["bool"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"Size","Docs":"","Typewords":["int64"]},{"Name":"TrainedJunk","Docs":"","Typewords":["nullable","bool"]},{"Name":"MsgPrefix","Docs":"","Typewords":["nullable","string"]},{"Name":"ParsedBuf","Docs":"","Typewords":["nullable","string"]}]},
"MessageItem": {"Name":"MessageItem","Docs":"","Fields":[{"Name":"Message","Docs":"","Typewords":["Message"]},{"Name":"Envelope","Docs":"","Typewords":["MessageEnvelope"]},{"Name":"Attachments","Docs":"","Typewords":["[]","Attachment"]},{"Name":"IsSigned","Docs":"","Typewords":["bool"]},{"Name":"IsEncrypted","Docs":"","Typewords":["bool"]},{"Name":"MatchQuery","Docs":"","Typewords":["bool"]},{"Name":"MoreHeaders","Docs":"","Typewords":["[]","[]","string"]}]},
"Message": {"Name":"Message","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"CreateSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Expunged","Docs":"","Typewords":["bool"]},{"Name":"IsReject","Docs":"","Typewords":["bool"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"MailboxOrigID","Docs":"","Typewords":["int64"]},{"Name":"MailboxDestinedID","Docs":"","Typewords":["int64"]},{"Name":"Received","Docs":"","Typewords":["timestamp"]},{"Name":"SaveDate","Docs":"","Typewords":["nullable","timestamp"]},{"Name":"RemoteIP","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked1","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked2","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked3","Docs":"","Typewords":["string"]},{"Name":"EHLODomain","Docs":"","Typewords":["string"]},{"Name":"MailFrom","Docs":"","Typewords":["string"]},{"Name":"MailFromLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"MailFromDomain","Docs":"","Typewords":["string"]},{"Name":"RcptToLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"RcptToDomain","Docs":"","Typewords":["string"]},{"Name":"MsgFromLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"MsgFromDomain","Docs":"","Typewords":["string"]},{"Name":"MsgFromOrgDomain","Docs":"","Typewords":["string"]},{"Name":"EHLOValidated","Docs":"","Typewords":["bool"]},{"Name":"MailFromValidated","Docs":"","Typewords":["bool"]},{"Name":"MsgFromValidated","Docs":"","Typewords":["bool"]},{"Name":"EHLOValidation","Docs":"","Typewords":["Validation"]},{"Name":"MailFromValidation","Docs":"","Typewords":["Validation"]},{"Name":"MsgFromValidation","Docs":"","Typewords":["Validation"]},{"Name":"DKIMDomains","Docs":"","Typewords":["[]","string"]},{"Name":"OrigEHLODomain","Docs":"","Typewords":["string"]},{"Name":"OrigDKIMDomains","Docs":"","Typewords":["[]","string"]},{"Name":"MessageID","Docs":"","Typewords":["string"]},{"Name":"SubjectBase","Docs":"","Typewords":["string"]},{"Name":"MessageHash","Docs":"","Typewords":["nullable","string"]},{"Name":"ThreadID","Docs":"","Typewords":["int64"]},{"Name":"ThreadParentIDs","Docs":"","Typewords":["[]","int64"]},{"Name":"ThreadMissingLink","Docs":"","Typewords":["bool"]},{"Name":"ThreadMuted","Docs":"","Typewords":["bool"]},{"Name":"ThreadCollapsed","Docs":"","Typewords":["bool"]},{"Name":"IsMailingList","Docs":"","Typewords":["bool"]},{"Name":"DSN","Docs":"","Typewords":["bool"]},{"Name":"ReceivedTLSVersion","Docs":"","Typewords":["uint16"]},{"Name":"ReceivedTLSCipherSuite","Docs":"","Typewords":["uint16"]},{"Name":"ReceivedRequireTLS","Docs":"","Typewords":["bool"]},{"Name":"Seen","Docs":"","Typewords":["bool"]},{"Name":"Answered","Docs":"","Typewords":["bool"]},{"Name":"Flagged","Docs":"","Typewords":["bool"]},{"Name":"Forwarded","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Notjunk","Docs":"","Typewords":["bool"]},{"Name":"Deleted","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Phishing","Docs":"","Typewords":["bool"]},{"Name":"MDNSent","Docs":"","Typewords":["bool"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"Size","Docs":"","Typewords":["int64"]},{"Name":"TrainedJunk","Docs":"","Typewords":["nullable","bool"]},{"Name":"MsgPrefix","Docs":"","Typewords":["nullable","string"]},{"Name":"Preview","Docs":"","Typewords":["nullable","string"]},{"Name":"ParsedBuf","Docs":"","Typewords":["nullable","string"]}]},
"MessageEnvelope": {"Name":"MessageEnvelope","Docs":"","Fields":[{"Name":"Date","Docs":"","Typewords":["timestamp"]},{"Name":"Subject","Docs":"","Typewords":["string"]},{"Name":"From","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"Sender","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"ReplyTo","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"To","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"CC","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"BCC","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"InReplyTo","Docs":"","Typewords":["string"]},{"Name":"MessageID","Docs":"","Typewords":["string"]}]},
"Attachment": {"Name":"Attachment","Docs":"","Fields":[{"Name":"Path","Docs":"","Typewords":["[]","int32"]},{"Name":"Filename","Docs":"","Typewords":["string"]},{"Name":"Part","Docs":"","Typewords":["Part"]}]},
"EventViewChanges": {"Name":"EventViewChanges","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"Changes","Docs":"","Typewords":["[]","[]","any"]}]},