imapserver: implement PREVIEW extension (RFC 8970), and store previews in message database

We were already generating previews of plain text parts for the webmail
interface, but we didn't store them, so were generating the previews each time
messages were listed.

Now we store previews in the database for faster handling. And we also generate
previews for html parts if needed. We use the first part that has textual
content.

For IMAP, the previews can be requested by an IMAP client. When we get the
"LAZY" variant, which doesn't require us to generate a preview, we generate it
anyway, because it should be fast enough. So don't make clients first ask for
"PREVIEW (LAZY)" and then again a request for "PREVIEW".

We now also generate a preview when a message is added to the account. Except
for imports. It would slow us down, the previews aren't urgent, and they will
be generated on-demand at first-request.

imapserver/fetch.go

@@ -31,10 +31,11 @@ type fetchCmd struct {
 	hasChangedSince bool        // Whether CHANGEDSINCE was set. Enables MODSEQ in response.
 	expungeIssued   bool        // Set if any message has been expunged. Can happen for expunged messages.
 
-	uid        store.UID // UID currently processing.
-	markSeen   bool
-	needFlags  bool
-	needModseq bool // Whether untagged responses needs modseq.
+	uid         store.UID // UID currently processing.
+	markSeen    bool
+	needFlags   bool
+	needModseq  bool                 // Whether untagged responses needs modseq.
+	newPreviews map[store.UID]string // Save with messages when done.
 
 	// Loaded when first needed, closed when message was processed.
 	m    *store.Message // Message currently being processed.
@@ -261,7 +262,7 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
 
 	// ../rfc/9051:4432 We mark all messages that need it as seen at the end of the
 	// command, in a single transaction.
-	if len(cmd.updateSeen) > 0 {
+	if len(cmd.updateSeen) > 0 || len(cmd.newPreviews) > 0 {
 		c.account.WithWLock(func() {
 			changes := make([]store.Change, 0, len(cmd.updateSeen)+1)
 
@@ -305,9 +306,27 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
 
 				changes = append(changes, mb.ChangeCounts())
 
-				mb.ModSeq = modseq
-				err = wtx.Update(&mb)
-				xcheckf(err, "update mailbox with counts and modseq")
+				for uid, s := range cmd.newPreviews {
+					m, err := bstore.QueryTx[store.Message](wtx).FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: uid}).Get()
+					xcheckf(err, "get message")
+					if m.Expunged {
+						// Message has been deleted in the mean time.
+						cmd.expungeIssued = true
+						continue
+					}
+
+					// note: we are not updating modseq.
+
+					m.Preview = &s
+					err = wtx.Update(&m)
+					xcheckf(err, "saving preview with message")
+				}
+
+				if modseq > 0 {
+					mb.ModSeq = modseq
+					err = wtx.Update(&mb)
+					xcheckf(err, "update mailbox with counts and modseq")
+				}
 			})
 
 			// Broadcast these changes also to ourselves, so we'll send the updated flags, but
@@ -545,6 +564,37 @@ func (cmd *fetchCmd) xprocessAtt(a fetchAtt) []token {
 	case "MODSEQ":
 		cmd.needModseq = true
 
+	case "PREVIEW":
+		m := cmd.xensureMessage()
+		preview := m.Preview
+		// We ignore "lazy", generating the preview is fast enough.
+		if preview == nil {
+			// Get the preview. We'll save all generated previews in a single transaction at
+			// the end.
+			_, p := cmd.xensureParsed()
+			s, err := p.Preview(cmd.conn.log)
+			cmd.xcheckf(err, "generating preview")
+			preview = &s
+			cmd.newPreviews[m.UID] = s
+		}
+		var t token = nilt
+		if preview != nil {
+			s := *preview
+
+			// Limit to 200 characters (not bytes). ../rfc/8970:206
+			var n, o int
+			for o = range s {
+				n++
+				if n > 200 {
+					s = s[:o]
+					break
+				}
+			}
+			s = strings.TrimSpace(s)
+			t = string0(s)
+		}
+		return []token{bare(a.field), t}
+
 	default:
 		xserverErrorf("field %q not yet implemented", a.field)
 	}

imapserver/fetch_test.go

@@ -466,5 +466,15 @@ Content-Transfer-Encoding: Quoted-printable
 	tc.transactf("ok", "fetch 1 rfc822")
 	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, rfc1}})
 
+	// Preview
+	preview := "Hello Joe, do you think we can meet at 3:30 tomorrow?"
+	tc.transactf("ok", "fetch 1 preview")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchPreview{Preview: &preview}}})
+
+	tc.transactf("ok", "fetch 1 preview (lazy)")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchPreview{Preview: &preview}}})
+
+	tc.transactf("bad", "fetch 1 preview (bogus)")
+
 	tc.client.Logout()
 }

imapserver/parse.go

@@ -577,6 +577,7 @@ var fetchAttWords = []string{
 	"RFC822.HEADER", "RFC822.TEXT", "RFC822", // older IMAP
 	"MODSEQ",   // CONDSTORE extension.
 	"SAVEDATE", // SAVEDATE extension, ../rfc/8514:186
+	"PREVIEW",  // ../rfc/8970:345
 }
 
 // ../rfc/9051:6557 ../rfc/3501:4751 ../rfc/7162:2483
@@ -608,6 +609,8 @@ func (p *parser) xfetchAtt(isUID bool) (r fetchAtt) {
 		// The wording about when to respond with a MODSEQ attribute could be more clear. ../rfc/7162:923 ../rfc/7162:388
 		// MODSEQ attribute is a CONDSTORE-enabling parameter. ../rfc/7162:377
 		p.conn.xensureCondstore(nil)
+	case "PREVIEW":
+		r.previewLazy = p.take(" (LAZY)")
 	}
 	return
 }

imapserver/protocol.go

@@ -307,6 +307,7 @@ type fetchAtt struct {
 	section       *sectionSpec
 	sectionBinary []uint32
 	partial       *partial
+	previewLazy   bool // Not regular "PREVIEW", but "PREVIEW (LAZY)".
 }
 
 type searchKey struct {