imapserver: implement PREVIEW extension (RFC 8970), and store previews in message database

We were already generating previews of plain text parts for the webmail
interface, but we didn't store them, so were generating the previews each time
messages were listed.

Now we store previews in the database for faster handling. And we also generate
previews for html parts if needed. We use the first part that has textual
content.

For IMAP, the previews can be requested by an IMAP client. When we get the
"LAZY" variant, which doesn't require us to generate a preview, we generate it
anyway, because it should be fast enough. So don't make clients first ask for
"PREVIEW (LAZY)" and then again a request for "PREVIEW".

We now also generate a preview when a message is added to the account. Except
for imports. It would slow us down, the previews aren't urgent, and they will
be generated on-demand at first-request.

imapclient/parse.go

@@ -776,6 +776,18 @@ func (c *Conn) xmsgatt1() FetchAttr {
 		modseq := c.xint64()
 		c.xtake(")")
 		return FetchModSeq(modseq)
+
+	case "PREVIEW":
+		// ../rfc/8970:348
+		c.xspace()
+		var preview *string
+		if c.peek('n') || c.peek('N') {
+			c.xtake("nil")
+		} else {
+			s := c.xstring()
+			preview = &s
+		}
+		return FetchPreview{preview}
 	}
 	c.xerrorf("unknown fetch attribute %q", f)
 	panic("not reached")

imapclient/protocol.go

@@ -627,3 +627,12 @@ func (f FetchUID) Attr() string { return "UID" }
 type FetchModSeq int64
 
 func (f FetchModSeq) Attr() string { return "MODSEQ" }
+
+// "PREVIEW" fetch response.
+type FetchPreview struct {
+	Preview *string
+}
+
+// ../rfc/8970:146
+
+func (f FetchPreview) Attr() string { return "PREVIEW" }

imapserver/fetch.go

@@ -35,6 +35,7 @@ type fetchCmd struct {
 	markSeen    bool
 	needFlags   bool
 	needModseq  bool                 // Whether untagged responses needs modseq.
+	newPreviews map[store.UID]string // Save with messages when done.
 
 	// Loaded when first needed, closed when message was processed.
 	m    *store.Message // Message currently being processed.
@@ -261,7 +262,7 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
 
 	// ../rfc/9051:4432 We mark all messages that need it as seen at the end of the
 	// command, in a single transaction.
-	if len(cmd.updateSeen) > 0 {
+	if len(cmd.updateSeen) > 0 || len(cmd.newPreviews) > 0 {
 		c.account.WithWLock(func() {
 			changes := make([]store.Change, 0, len(cmd.updateSeen)+1)
 
@@ -305,9 +306,27 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
 
 				changes = append(changes, mb.ChangeCounts())
 
+				for uid, s := range cmd.newPreviews {
+					m, err := bstore.QueryTx[store.Message](wtx).FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: uid}).Get()
+					xcheckf(err, "get message")
+					if m.Expunged {
+						// Message has been deleted in the mean time.
+						cmd.expungeIssued = true
+						continue
+					}
+
+					// note: we are not updating modseq.
+
+					m.Preview = &s
+					err = wtx.Update(&m)
+					xcheckf(err, "saving preview with message")
+				}
+
+				if modseq > 0 {
 					mb.ModSeq = modseq
 					err = wtx.Update(&mb)
 					xcheckf(err, "update mailbox with counts and modseq")
+				}
 			})
 
 			// Broadcast these changes also to ourselves, so we'll send the updated flags, but
@@ -545,6 +564,37 @@ func (cmd *fetchCmd) xprocessAtt(a fetchAtt) []token {
 	case "MODSEQ":
 		cmd.needModseq = true
 
+	case "PREVIEW":
+		m := cmd.xensureMessage()
+		preview := m.Preview
+		// We ignore "lazy", generating the preview is fast enough.
+		if preview == nil {
+			// Get the preview. We'll save all generated previews in a single transaction at
+			// the end.
+			_, p := cmd.xensureParsed()
+			s, err := p.Preview(cmd.conn.log)
+			cmd.xcheckf(err, "generating preview")
+			preview = &s
+			cmd.newPreviews[m.UID] = s
+		}
+		var t token = nilt
+		if preview != nil {
+			s := *preview
+
+			// Limit to 200 characters (not bytes). ../rfc/8970:206
+			var n, o int
+			for o = range s {
+				n++
+				if n > 200 {
+					s = s[:o]
+					break
+				}
+			}
+			s = strings.TrimSpace(s)
+			t = string0(s)
+		}
+		return []token{bare(a.field), t}
+
 	default:
 		xserverErrorf("field %q not yet implemented", a.field)
 	}

imapserver/fetch_test.go

@@ -466,5 +466,15 @@ Content-Transfer-Encoding: Quoted-printable
 	tc.transactf("ok", "fetch 1 rfc822")
 	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, rfc1}})
 
+	// Preview
+	preview := "Hello Joe, do you think we can meet at 3:30 tomorrow?"
+	tc.transactf("ok", "fetch 1 preview")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchPreview{Preview: &preview}}})
+
+	tc.transactf("ok", "fetch 1 preview (lazy)")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchPreview{Preview: &preview}}})
+
+	tc.transactf("bad", "fetch 1 preview (bogus)")
+
 	tc.client.Logout()
 }

imapserver/parse.go

@@ -577,6 +577,7 @@ var fetchAttWords = []string{
 	"RFC822.HEADER", "RFC822.TEXT", "RFC822", // older IMAP
 	"MODSEQ",   // CONDSTORE extension.
 	"SAVEDATE", // SAVEDATE extension, ../rfc/8514:186
+	"PREVIEW",  // ../rfc/8970:345
 }
 
 // ../rfc/9051:6557 ../rfc/3501:4751 ../rfc/7162:2483
@@ -608,6 +609,8 @@ func (p *parser) xfetchAtt(isUID bool) (r fetchAtt) {
 		// The wording about when to respond with a MODSEQ attribute could be more clear. ../rfc/7162:923 ../rfc/7162:388
 		// MODSEQ attribute is a CONDSTORE-enabling parameter. ../rfc/7162:377
 		p.conn.xensureCondstore(nil)
+	case "PREVIEW":
+		r.previewLazy = p.take(" (LAZY)")
 	}
 	return
 }

imapserver/protocol.go

@@ -307,6 +307,7 @@ type fetchAtt struct {
 	section       *sectionSpec
 	sectionBinary []uint32
 	partial       *partial
+	previewLazy   bool // Not regular "PREVIEW", but "PREVIEW (LAZY)".
 }
 
 type searchKey struct {

import.go

@@ -375,6 +375,7 @@ func ximportctl(ctx context.Context, xctl *ctl, mbox bool) {
 				SkipThreads:         true, // We do this efficiently when we have all messages.
 				SkipUpdateDiskUsage: true, // We do this once at the end.
 				SkipCheckQuota:      true, // We check before.
+				SkipPreview:         true, // We'll do this on-demand when messages are requested. Saves time.
 			}
 			err = a.MessageAdd(xctl.log, tx, &mb, m, msgf, opts)
 			xctl.xcheck(err, "delivering message")

message/preview.go

@@ -0,0 +1,350 @@
+package message
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"regexp"
+	"slices"
+	"strings"
+
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/moxio"
+)
+
+// Preview returns a message preview, based on the first text/plain or text/html
+// part of the message that has textual content. Preview returns at most 256
+// characters (possibly more bytes). Callers may want to truncate and trim trailing
+// whitespace before using the preview.
+//
+// Preview logs at debug level for invalid messages. An error is only returned for
+// serious errors, like i/o errors.
+func (p Part) Preview(log mlog.Log) (string, error) {
+	// ../rfc/8970:190
+
+	// Don't use if Content-Disposition attachment.
+	disp, _, err := p.DispositionFilename()
+	if err != nil {
+		log.Debugx("parsing disposition/filename", err)
+	} else if strings.EqualFold(disp, "attachment") {
+		return "", nil
+	}
+
+	mt := p.MediaType + "/" + p.MediaSubType
+	switch mt {
+	case "TEXT/PLAIN", "/":
+		r := &moxio.LimitReader{R: p.ReaderUTF8OrBinary(), Limit: 100 * 1024}
+		s, err := previewText(r)
+		if err != nil {
+			return "", fmt.Errorf("making preview from text part: %v", err)
+		}
+		return s, nil
+
+	case "TEXT/HTML":
+		r := &moxio.LimitReader{R: p.ReaderUTF8OrBinary(), Limit: 1024 * 1024}
+
+		// First turn the HTML into text.
+		s, err := previewHTML(r)
+		if err != nil {
+			log.Debugx("parsing html part for preview (ignored)", err)
+			return "", nil
+		}
+
+		// Turn text body into a preview text.
+		s, err = previewText(strings.NewReader(s))
+		if err != nil {
+			return "", fmt.Errorf("making preview from text from html: %v", err)
+		}
+		return s, nil
+
+	case "MULTIPART/ENCRYPTED":
+		return "", nil
+	}
+
+	for i, sp := range p.Parts {
+		if mt == "MULTIPART/SIGNED" && i >= 1 {
+			break
+		}
+		s, err := sp.Preview(log)
+		if err != nil || s != "" {
+			return s, err
+		}
+	}
+	return "", nil
+}
+
+// previewText returns a line the client can display next to the subject line
+// in a mailbox. It will replace quoted text, and any prefixing "On ... wrote:"
+// line with "[...]" so only new and useful information will be displayed.
+// Trailing signatures are not included.
+func previewText(r io.Reader) (string, error) {
+	// We look quite a bit of lines ahead for trailing signatures with trailing empty lines.
+	var lines []string
+	scanner := bufio.NewScanner(r)
+	ensureLines := func() {
+		for len(lines) < 10 && scanner.Scan() {
+			lines = append(lines, strings.TrimSpace(scanner.Text()))
+		}
+	}
+	ensureLines()
+
+	isSnipped := func(s string) bool {
+		return s == "[...]" || s == "[…]" || s == "..."
+	}
+
+	nextLineQuoted := func(i int) bool {
+		if i+1 < len(lines) && lines[i+1] == "" {
+			i++
+		}
+		return i+1 < len(lines) && (strings.HasPrefix(lines[i+1], ">") || isSnipped(lines[i+1]))
+	}
+
+	// Remainder is signature if we see a line with only and minimum 2 dashes, and
+	// there are no more empty lines, and there aren't more than 5 lines left.
+	isSignature := func() bool {
+		if len(lines) == 0 || !strings.HasPrefix(lines[0], "--") || strings.Trim(strings.TrimSpace(lines[0]), "-") != "" {
+			return false
+		}
+		l := lines[1:]
+		for len(l) > 0 && l[len(l)-1] == "" {
+			l = l[:len(l)-1]
+		}
+		if len(l) >= 5 {
+			return false
+		}
+		return !slices.Contains(l, "")
+	}
+
+	result := ""
+
+	resultSnipped := func() bool {
+		return strings.HasSuffix(result, "[...]\n") || strings.HasSuffix(result, "[…]")
+	}
+
+	// Quick check for initial wrapped "On ... wrote:" line.
+	if len(lines) > 3 && strings.HasPrefix(lines[0], "On ") && !strings.HasSuffix(lines[0], "wrote:") && strings.HasSuffix(lines[1], ":") && nextLineQuoted(1) {
+		result = "[...]\n"
+		lines = lines[3:]
+		ensureLines()
+	}
+
+	for ; len(lines) > 0 && !isSignature(); ensureLines() {
+		line := lines[0]
+		if strings.HasPrefix(line, ">") {
+			if !resultSnipped() {
+				result += "[...]\n"
+			}
+			lines = lines[1:]
+			continue
+		}
+		if line == "" {
+			lines = lines[1:]
+			continue
+		}
+		// Check for a "On <date>, <person> wrote:", we require digits before a quoted
+		// line, with an optional empty line in between. If we don't have any text yet, we
+		// don't require the digits.
+		if strings.HasSuffix(line, ":") && (strings.ContainsAny(line, "0123456789") || result == "") && nextLineQuoted(0) {
+			if !resultSnipped() {
+				result += "[...]\n"
+			}
+			lines = lines[1:]
+			continue
+		}
+		// Skip possibly duplicate snipping by author.
+		if !isSnipped(line) || !resultSnipped() {
+			result += line + "\n"
+		}
+		lines = lines[1:]
+		if len(result) > 250 {
+			break
+		}
+	}
+
+	// Limit number of characters (not bytes). ../rfc/8970:200
+	// To 256 characters. ../rfc/8970:211
+	var o, n int
+	for o = range result {
+		n++
+		if n > 256 {
+			result = result[:o]
+			break
+		}
+	}
+
+	return result, scanner.Err()
+}
+
+// Any text inside these html elements (recursively) is ignored.
+var ignoreAtoms = atomMap(
+	atom.Dialog,
+	atom.Head,
+	atom.Map,
+	atom.Math,
+	atom.Script,
+	atom.Style,
+	atom.Svg,
+	atom.Template,
+)
+
+// Inline elements don't force newlines at beginning & end of text in this element.
+// https://developer.mozilla.org/en-US/docs/Web/HTML/Element#inline_text_semantics
+var inlineAtoms = atomMap(
+	atom.A,
+	atom.Abbr,
+	atom.B,
+	atom.Bdi,
+	atom.Bdo,
+	atom.Cite,
+	atom.Code,
+	atom.Data,
+	atom.Dfn,
+	atom.Em,
+	atom.I,
+	atom.Kbd,
+	atom.Mark,
+	atom.Q,
+	atom.Rp,
+	atom.Rt,
+	atom.Ruby,
+	atom.S,
+	atom.Samp,
+	atom.Small,
+	atom.Span,
+	atom.Strong,
+	atom.Sub,
+	atom.Sup,
+	atom.Time,
+	atom.U,
+	atom.Var,
+	atom.Wbr,
+
+	atom.Del,
+	atom.Ins,
+
+	// We treat these specially, inserting a space after them instead of a newline.
+	atom.Td,
+	atom.Th,
+)
+
+func atomMap(l ...atom.Atom) map[atom.Atom]bool {
+	m := map[atom.Atom]bool{}
+	for _, a := range l {
+		m[a] = true
+	}
+	return m
+}
+
+var regexpSpace = regexp.MustCompile(`[ \t]+`)                                                    // Replaced with single space.
+var regexpNewline = regexp.MustCompile(`\n\n\n+`)                                                 // Replaced with single newline.
+var regexpZeroWidth = regexp.MustCompile("[\u00a0\u200b\u200c\u200d][\u00a0\u200b\u200c\u200d]+") // Removed, combinations don't make sense, generated.
+
+func previewHTML(r io.Reader) (string, error) {
+	// Stack/state, based on elements.
+	var ignores []bool
+	var inlines []bool
+
+	var text string // Collecting text.
+	var err error   // Set when walking DOM.
+	var quoteLevel int
+
+	// We'll walk the DOM nodes, keeping track of whether we are ignoring text, and
+	// whether we are in an inline or block element, and building up the text. We stop
+	// when we have enough data, returning false in that case.
+	var walk func(n *html.Node) bool
+	walk = func(n *html.Node) bool {
+		switch n.Type {
+		case html.ErrorNode:
+			err = fmt.Errorf("unexpected error node")
+			return false
+
+		case html.ElementNode:
+			ignores = append(ignores, ignoreAtoms[n.DataAtom])
+			inline := inlineAtoms[n.DataAtom]
+			inlines = append(inlines, inline)
+			if n.DataAtom == atom.Blockquote {
+				quoteLevel++
+			}
+			defer func() {
+				if n.DataAtom == atom.Blockquote {
+					quoteLevel--
+				}
+				if !inline && !strings.HasSuffix(text, "\n\n") {
+					text += "\n"
+				} else if (n.DataAtom == atom.Td || n.DataAtom == atom.Th) && !strings.HasSuffix(text, " ") {
+					text += " "
+				}
+
+				ignores = ignores[:len(ignores)-1]
+				inlines = inlines[:len(inlines)-1]
+			}()
+
+		case html.TextNode:
+			if slices.Contains(ignores, true) {
+				return true
+			}
+			// Collapse all kinds of weird whitespace-like characters into a space, except for newline and ignoring carriage return.
+			var s string
+			for _, c := range n.Data {
+				if c == '\r' {
+					continue
+				} else if c == '\t' {
+					s += " "
+				} else {
+					s += string(c)
+				}
+			}
+			s = regexpSpace.ReplaceAllString(s, " ")
+			s = regexpNewline.ReplaceAllString(s, "\n")
+			s = regexpZeroWidth.ReplaceAllString(s, "")
+
+			inline := len(inlines) > 0 && inlines[len(inlines)-1]
+			ts := strings.TrimSpace(s)
+			if !inline && ts == "" {
+				break
+			}
+			if ts != "" || !strings.HasSuffix(s, " ") && !strings.HasSuffix(s, "\n") {
+				if quoteLevel > 0 {
+					q := strings.Repeat("> ", quoteLevel)
+					var sb strings.Builder
+					for line := range strings.Lines(s) {
+						sb.WriteString(q)
+						sb.WriteString(line)
+					}
+					s = sb.String()
+				}
+				text += s
+			}
+			// We need to generate at most 256 characters of preview. The text we're gathering
+			// will be cleaned up, with quoting removed, so we'll end up with less. Hopefully,
+			// 4k bytes is enough to read.
+			if len(text) >= 4*1024 {
+				return false
+			}
+		}
+		// Ignored: DocumentNode, CommentNode, DoctypeNode, RawNode
+
+		for cn := range n.ChildNodes() {
+			if !walk(cn) {
+				break
+			}
+		}
+
+		return true
+	}
+
+	node, err := html.Parse(r)
+	if err != nil {
+		return "", fmt.Errorf("parsing html: %v", err)
+	}
+
+	// Build text.
+	walk(node)
+
+	text = strings.TrimSpace(text)
+	text = regexpSpace.ReplaceAllString(text, " ")
+	return text, err
+}

message/preview_test.go

@@ -0,0 +1,159 @@
+package message
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log/slog"
+	"mime/multipart"
+	"net/textproto"
+	"strings"
+	"testing"
+
+	"github.com/mjl-/mox/mlog"
+)
+
+func TestPreviewText(t *testing.T) {
+	check := func(body, expLine string) {
+		t.Helper()
+
+		line, err := previewText(strings.NewReader(body))
+		tcompare(t, err, nil)
+		if line != expLine {
+			t.Fatalf("got %q, expected %q, for body %q", line, expLine, body)
+		}
+	}
+
+	check("", "")
+	check("single line", "single line\n")
+	check("single line\n", "single line\n")
+	check("> quoted\n", "[...]\n")
+	check("> quoted\nresponse\n", "[...]\nresponse\n")
+	check("> quoted\n[...]\nresponse after author snip\n", "[...]\nresponse after author snip\n")
+	check("[...]\nresponse after author snip\n", "[...]\nresponse after author snip\n")
+	check("[…]\nresponse after author snip\n", "[…]\nresponse after author snip\n")
+	check(">> quoted0\n> quoted1\n>quoted2\n[...]\nresponse after author snip\n", "[...]\nresponse after author snip\n")
+	check(">quoted\n\n>quoted\ncoalesce line-separated quotes\n", "[...]\ncoalesce line-separated quotes\n")
+	check("On <date> <user> wrote:\n> hi\nresponse", "[...]\nresponse\n")
+	check("On <longdate>\n<user> wrote:\n> hi\nresponse", "[...]\nresponse\n")
+	check("> quote\nresponse\n--\nsignature\n", "[...]\nresponse\n")
+	check("> quote\nline1\nline2\nline3\n", "[...]\nline1\nline2\nline3\n")
+}
+
+func tcompose(t *testing.T, typeContents ...string) *bytes.Reader {
+	var b bytes.Buffer
+
+	xc := NewComposer(&b, 100*1024, true)
+	xc.Header("MIME-Version", "1.0")
+
+	var cur, alt *multipart.Writer
+
+	xcreateMultipart := func(subtype string) *multipart.Writer {
+		mp := multipart.NewWriter(xc)
+		if cur == nil {
+			xc.Header("Content-Type", fmt.Sprintf(`multipart/%s; boundary="%s"`, subtype, mp.Boundary()))
+			xc.Line()
+		} else {
+			_, err := cur.CreatePart(textproto.MIMEHeader{"Content-Type": []string{fmt.Sprintf(`multipart/%s; boundary="%s"`, subtype, mp.Boundary())}})
+			tcheck(t, err, "adding multipart")
+		}
+		cur = mp
+		return mp
+	}
+	xcreatePart := func(header textproto.MIMEHeader) io.Writer {
+		if cur == nil {
+			for k, vl := range header {
+				for _, v := range vl {
+					xc.Header(k, v)
+				}
+			}
+			xc.Line()
+			return xc
+		}
+		p, err := cur.CreatePart(header)
+		tcheck(t, err, "adding part")
+		return p
+	}
+
+	if len(typeContents)/2 > 1 {
+		alt = xcreateMultipart("alternative")
+	}
+	for i := 0; i < len(typeContents); i += 2 {
+		body, ct, cte := xc.TextPart(typeContents[i], typeContents[i+1])
+		tp := xcreatePart(textproto.MIMEHeader{"Content-Type": []string{ct}, "Content-Transfer-Encoding": []string{cte}})
+		_, err := tp.Write([]byte(body))
+		tcheck(t, err, "write part")
+	}
+	if alt != nil {
+		err := alt.Close()
+		tcheck(t, err, "close multipart")
+	}
+	xc.Flush()
+
+	buf := b.Bytes()
+	return bytes.NewReader(buf)
+}
+
+func TestPreviewHTML(t *testing.T) {
+	check := func(r *bytes.Reader, exp string) {
+		t.Helper()
+
+		p, err := Parse(slog.Default(), false, r)
+		tcheck(t, err, "parse")
+		err = p.Walk(slog.Default(), nil)
+		tcheck(t, err, "walk")
+		log := mlog.New("message", nil)
+		s, err := p.Preview(log)
+		tcheck(t, err, "preview")
+		tcompare(t, s, exp)
+	}
+
+	// We use the first part for the preview.
+	m := tcompose(t, "plain", "the text", "html", "<html><body>the html</body></html>")
+	check(m, "the text\n")
+
+	// HTML before text.
+	m = tcompose(t, "html", "<body>the html</body>", "plain", "the text")
+	check(m, "the html\n")
+
+	// Only text.
+	m = tcompose(t, "plain", "the text")
+	check(m, "the text\n")
+
+	// Only html.
+	m = tcompose(t, "html", "<body>the html</body>")
+	check(m, "the html\n")
+
+	// No preview
+	m = tcompose(t, "other", "other text")
+	check(m, "")
+
+	// HTML with quoted text.
+	m = tcompose(t, "html", "<html><div>On ... someone wrote:</div><blockquote>something worth replying</blockquote><div>agreed</div></body>")
+	check(m, "[...]\nagreed\n")
+
+	// HTML with ignored elements, inline elements and tables.
+	const moreHTML = `<!doctype html>
+<html>
+	<head>
+		<title>title</title>
+		<style>head style</style>
+		<script>head script</script>
+	</head>
+<body>
+<script>body script</script>
+<style>body style</style>
+<div>line1</div>
+<div>line2</div>
+<div><a href="about:blank">link1   </a> text <span>word</span><span>word2</span>.</div>
+<table><tr><td>col1</td><th>col2</th></tr><tr><td>row2</td></tr></table>
+</body></html>
+`
+	m = tcompose(t, "html", moreHTML)
+	check(m, `line1
+line2
+link1 text wordword2.
+col1 col2
+row2
+`)
+}

rfc/index.txt

@@ -234,7 +234,7 @@ https://www.iana.org/assignments/message-headers/message-headers.xhtml
 8474	Roadmap	-	IMAP Extension for Object Identifiers
 8508	Yes	-	IMAP REPLACE Extension
 8514	Yes	-	Internet Message Access Protocol (IMAP) - SAVEDATE Extension
-8970	Roadmap	-	IMAP4 Extension: Message Preview Generation
+8970	Yes	-	IMAP4 Extension: Message Preview Generation
 9208	Partial	-	IMAP QUOTA Extension
 9394	Roadmap	-	IMAP PARTIAL Extension for Paged SEARCH and FETCH
 9585	?	-	IMAP Response Code for Command Progress Notifications

store/account.go

@@ -560,6 +560,18 @@ type Message struct {
 	TrainedJunk *bool  // If nil, no training done yet. Otherwise, true is trained as junk, false trained as nonjunk.
 	MsgPrefix   []byte // Typically holds received headers and/or header separator.
 
+	// If non-nil, a preview of the message based on text and/or html parts of the
+	// message. Used in the webmail and IMAP PREVIEW extension. If non-nil, it is empty
+	// if no preview could be created, or the message has not textual content or
+	// couldn't be parsed.
+	// Previews are typically created when delivering a message, but not when importing
+	// messages, for speed. Previews are generated on first request (in the webmail, or
+	// through the IMAP fetch attribute "PREVIEW" (without "LAZY")), and stored with
+	// the message at that time.
+	// The preview is at most 256 characters (can be more bytes), with detected quoted
+	// text replaced with "[..."].
+	Preview *string
+
 	// ParsedBuf message structure. Currently saved as JSON of message.Part because bstore
 	// cannot yet store recursive types. Created when first needed, and saved in the
 	// database.
@@ -2126,6 +2138,9 @@ type AddOpts struct {
 	JunkFilter *junk.Filter
 
 	SkipTraining bool
+
+	// If true, a preview will be generated if the Message doesn't already have one.
+	SkipPreview bool
 }
 
 // todo optimization: when moving files, we open the original, call MessageAdd() which hardlinks it and close the file gain. when passing the filename, we could just use os.Link, saves 2 syscalls.
@@ -2255,6 +2270,8 @@ func (a *Account) MessageAdd(log mlog.Log, tx *bstore.Tx, mb *Mailbox, m *Messag
 		if err := json.Unmarshal(m.ParsedBuf, &p); err != nil {
 			log.Errorx("unmarshal parsed message, continuing", err, slog.String("parse", ""))
 		} else {
+			mr := FileMsgReader(m.MsgPrefix, msgFile)
+			p.SetReaderAt(mr)
 			part = &p
 		}
 		return part
@@ -2269,6 +2286,16 @@ func (a *Account) MessageAdd(log mlog.Log, tx *bstore.Tx, mb *Mailbox, m *Messag
 		m.PrepareThreading(log, part)
 	}
 
+	if !opts.SkipPreview && m.Preview == nil {
+		if p := getPart(); p != nil {
+			s, err := p.Preview(log)
+			if err != nil {
+				return fmt.Errorf("generating preview: %v", err)
+			}
+			m.Preview = &s
+		}
+	}
+
 	// Assign to thread (if upgrade has completed).
 	noThreadID := opts.SkipThreads
 	if m.ThreadID == 0 && !opts.SkipThreads && getPart() != nil {

webaccount/import.go

@@ -551,6 +551,7 @@ func importMessages(ctx context.Context, log mlog.Log, token string, acc *store.
 			SkipThreads:         true,
 			SkipUpdateDiskUsage: true,
 			SkipCheckQuota:      true,
+			SkipPreview:         true,
 		}
 		if err := acc.MessageAdd(log, tx, mb, m, f, opts); err != nil {
 			problemf("delivering message %s: %s (continuing)", pos, err)

webmail/api.go

@@ -171,7 +171,7 @@ func (Webmail) ParsedMessage(ctx context.Context, msgID int64) (pm ParsedMessage
 		state := msgState{acc: acc}
 		defer state.clear()
 		var err error
-		pm, err = parsedMessage(log, m, &state, true, false, false)
+		pm, err = parsedMessage(log, &m, &state, true, false, false)
 		xcheckf(ctx, err, "parsing message")
 
 		if len(pm.envelope.From) == 1 {
@@ -1849,7 +1849,7 @@ func (Webmail) RulesetSuggestMove(ctx context.Context, msgID, mbSrcID, mbDstID i
 		// Parse message for List-Id header.
 		state := msgState{acc: acc}
 		defer state.clear()
-		pm, err := parsedMessage(log, m, &state, true, false, false)
+		pm, err := parsedMessage(log, &m, &state, true, false, false)
 		xcheckf(ctx, err, "parsing message")
 
 		// The suggested ruleset. Once all is checked, we'll return it.

webmail/api.json

@@ -2115,7 +2115,7 @@
 			"Fields": [
 				{
 					"Name": "Message",
-					"Docs": "Without ParsedBuf and MsgPrefix, for size.",
+					"Docs": "Without ParsedBuf and MsgPrefix, for size. With Preview, even with value not yet stored in the database.",
 					"Typewords": [
 						"Message"
 					]
@@ -2149,13 +2149,6 @@
 						"bool"
 					]
 				},
-				{
-					"Name": "FirstLine",
-					"Docs": "Of message body, for showing as preview.",
-					"Typewords": [
-						"string"
-					]
-				},
 				{
 					"Name": "MatchQuery",
 					"Docs": "If message does not match query, it can still be included because of threading.",
@@ -2613,6 +2606,14 @@
 						"uint8"
 					]
 				},
+				{
+					"Name": "Preview",
+					"Docs": "If non-nil, a preview of the message based on text and/or html parts of the message. Used in the webmail and IMAP PREVIEW extension. If non-nil, it is empty if no preview could be created, or the message has not textual content or couldn't be parsed. Previews are typically created when delivering a message, but not when importing messages, for speed. Previews are generated on first request (in the webmail, or through the IMAP fetch attribute \"PREVIEW\" (without \"LAZY\")), and stored with the message at that time. The preview is at most 256 characters (can be more bytes), with detected quoted text replaced with \"[...\"].",
+					"Typewords": [
+						"nullable",
+						"string"
+					]
+				},
 				{
 					"Name": "ParsedBuf",
 					"Docs": "ParsedBuf message structure. Currently saved as JSON of message.Part because bstore cannot yet store recursive types. Created when first needed, and saved in the database. todo: once replaced with non-json storage, remove date fixup in ../message/part.go.",

webmail/api.ts

@@ -299,12 +299,11 @@ export interface EventViewMsgs {
 // message.Part, made for the needs of the message items in the message list.
 // messages.
 export interface MessageItem {
-	Message: Message  // Without ParsedBuf and MsgPrefix, for size.
+	Message: Message  // Without ParsedBuf and MsgPrefix, for size. With Preview, even with value not yet stored in the database.
 	Envelope: MessageEnvelope
 	Attachments?: Attachment[] | null
 	IsSigned: boolean
 	IsEncrypted: boolean
-	FirstLine: string  // Of message body, for showing as preview.
 	MatchQuery: boolean  // If message does not match query, it can still be included because of threading.
 	MoreHeaders?: (string[] | null)[] | null  // All headers from store.Settings.ShowHeaders that are present.
 }
@@ -378,6 +377,7 @@ export interface Message {
 	Size: number
 	TrainedJunk?: boolean | null  // If nil, no training done yet. Otherwise, true is trained as junk, false trained as nonjunk.
 	MsgPrefix?: string | null  // Typically holds received headers and/or header separator.
+	Preview?: string | null  // If non-nil, a preview of the message based on text and/or html parts of the message. Used in the webmail and IMAP PREVIEW extension. If non-nil, it is empty if no preview could be created, or the message has not textual content or couldn't be parsed. Previews are typically created when delivering a message, but not when importing messages, for speed. Previews are generated on first request (in the webmail, or through the IMAP fetch attribute "PREVIEW" (without "LAZY")), and stored with the message at that time. The preview is at most 256 characters (can be more bytes), with detected quoted text replaced with "[..."].
 	ParsedBuf?: string | null  // ParsedBuf message structure. Currently saved as JSON of message.Part because bstore cannot yet store recursive types. Created when first needed, and saved in the database. todo: once replaced with non-json storage, remove date fixup in ../message/part.go.
 }
 
@@ -625,8 +625,8 @@ export const types: TypenameMap = {
 	"EventViewErr": {"Name":"EventViewErr","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"RequestID","Docs":"","Typewords":["int64"]},{"Name":"Err","Docs":"","Typewords":["string"]}]},
 	"EventViewReset": {"Name":"EventViewReset","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"RequestID","Docs":"","Typewords":["int64"]}]},
 	"EventViewMsgs": {"Name":"EventViewMsgs","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"RequestID","Docs":"","Typewords":["int64"]},{"Name":"MessageItems","Docs":"","Typewords":["[]","[]","MessageItem"]},{"Name":"ParsedMessage","Docs":"","Typewords":["nullable","ParsedMessage"]},{"Name":"ViewEnd","Docs":"","Typewords":["bool"]}]},
-	"MessageItem": {"Name":"MessageItem","Docs":"","Fields":[{"Name":"Message","Docs":"","Typewords":["Message"]},{"Name":"Envelope","Docs":"","Typewords":["MessageEnvelope"]},{"Name":"Attachments","Docs":"","Typewords":["[]","Attachment"]},{"Name":"IsSigned","Docs":"","Typewords":["bool"]},{"Name":"IsEncrypted","Docs":"","Typewords":["bool"]},{"Name":"FirstLine","Docs":"","Typewords":["string"]},{"Name":"MatchQuery","Docs":"","Typewords":["bool"]},{"Name":"MoreHeaders","Docs":"","Typewords":["[]","[]","string"]}]},
-	"Message": {"Name":"Message","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"CreateSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Expunged","Docs":"","Typewords":["bool"]},{"Name":"IsReject","Docs":"","Typewords":["bool"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"MailboxOrigID","Docs":"","Typewords":["int64"]},{"Name":"MailboxDestinedID","Docs":"","Typewords":["int64"]},{"Name":"Received","Docs":"","Typewords":["timestamp"]},{"Name":"SaveDate","Docs":"","Typewords":["nullable","timestamp"]},{"Name":"RemoteIP","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked1","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked2","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked3","Docs":"","Typewords":["string"]},{"Name":"EHLODomain","Docs":"","Typewords":["string"]},{"Name":"MailFrom","Docs":"","Typewords":["string"]},{"Name":"MailFromLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"MailFromDomain","Docs":"","Typewords":["string"]},{"Name":"RcptToLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"RcptToDomain","Docs":"","Typewords":["string"]},{"Name":"MsgFromLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"MsgFromDomain","Docs":"","Typewords":["string"]},{"Name":"MsgFromOrgDomain","Docs":"","Typewords":["string"]},{"Name":"EHLOValidated","Docs":"","Typewords":["bool"]},{"Name":"MailFromValidated","Docs":"","Typewords":["bool"]},{"Name":"MsgFromValidated","Docs":"","Typewords":["bool"]},{"Name":"EHLOValidation","Docs":"","Typewords":["Validation"]},{"Name":"MailFromValidation","Docs":"","Typewords":["Validation"]},{"Name":"MsgFromValidation","Docs":"","Typewords":["Validation"]},{"Name":"DKIMDomains","Docs":"","Typewords":["[]","string"]},{"Name":"OrigEHLODomain","Docs":"","Typewords":["string"]},{"Name":"OrigDKIMDomains","Docs":"","Typewords":["[]","string"]},{"Name":"MessageID","Docs":"","Typewords":["string"]},{"Name":"SubjectBase","Docs":"","Typewords":["string"]},{"Name":"MessageHash","Docs":"","Typewords":["nullable","string"]},{"Name":"ThreadID","Docs":"","Typewords":["int64"]},{"Name":"ThreadParentIDs","Docs":"","Typewords":["[]","int64"]},{"Name":"ThreadMissingLink","Docs":"","Typewords":["bool"]},{"Name":"ThreadMuted","Docs":"","Typewords":["bool"]},{"Name":"ThreadCollapsed","Docs":"","Typewords":["bool"]},{"Name":"IsMailingList","Docs":"","Typewords":["bool"]},{"Name":"DSN","Docs":"","Typewords":["bool"]},{"Name":"ReceivedTLSVersion","Docs":"","Typewords":["uint16"]},{"Name":"ReceivedTLSCipherSuite","Docs":"","Typewords":["uint16"]},{"Name":"ReceivedRequireTLS","Docs":"","Typewords":["bool"]},{"Name":"Seen","Docs":"","Typewords":["bool"]},{"Name":"Answered","Docs":"","Typewords":["bool"]},{"Name":"Flagged","Docs":"","Typewords":["bool"]},{"Name":"Forwarded","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Notjunk","Docs":"","Typewords":["bool"]},{"Name":"Deleted","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Phishing","Docs":"","Typewords":["bool"]},{"Name":"MDNSent","Docs":"","Typewords":["bool"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"Size","Docs":"","Typewords":["int64"]},{"Name":"TrainedJunk","Docs":"","Typewords":["nullable","bool"]},{"Name":"MsgPrefix","Docs":"","Typewords":["nullable","string"]},{"Name":"ParsedBuf","Docs":"","Typewords":["nullable","string"]}]},
+	"MessageItem": {"Name":"MessageItem","Docs":"","Fields":[{"Name":"Message","Docs":"","Typewords":["Message"]},{"Name":"Envelope","Docs":"","Typewords":["MessageEnvelope"]},{"Name":"Attachments","Docs":"","Typewords":["[]","Attachment"]},{"Name":"IsSigned","Docs":"","Typewords":["bool"]},{"Name":"IsEncrypted","Docs":"","Typewords":["bool"]},{"Name":"MatchQuery","Docs":"","Typewords":["bool"]},{"Name":"MoreHeaders","Docs":"","Typewords":["[]","[]","string"]}]},
+	"Message": {"Name":"Message","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"CreateSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Expunged","Docs":"","Typewords":["bool"]},{"Name":"IsReject","Docs":"","Typewords":["bool"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"MailboxOrigID","Docs":"","Typewords":["int64"]},{"Name":"MailboxDestinedID","Docs":"","Typewords":["int64"]},{"Name":"Received","Docs":"","Typewords":["timestamp"]},{"Name":"SaveDate","Docs":"","Typewords":["nullable","timestamp"]},{"Name":"RemoteIP","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked1","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked2","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked3","Docs":"","Typewords":["string"]},{"Name":"EHLODomain","Docs":"","Typewords":["string"]},{"Name":"MailFrom","Docs":"","Typewords":["string"]},{"Name":"MailFromLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"MailFromDomain","Docs":"","Typewords":["string"]},{"Name":"RcptToLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"RcptToDomain","Docs":"","Typewords":["string"]},{"Name":"MsgFromLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"MsgFromDomain","Docs":"","Typewords":["string"]},{"Name":"MsgFromOrgDomain","Docs":"","Typewords":["string"]},{"Name":"EHLOValidated","Docs":"","Typewords":["bool"]},{"Name":"MailFromValidated","Docs":"","Typewords":["bool"]},{"Name":"MsgFromValidated","Docs":"","Typewords":["bool"]},{"Name":"EHLOValidation","Docs":"","Typewords":["Validation"]},{"Name":"MailFromValidation","Docs":"","Typewords":["Validation"]},{"Name":"MsgFromValidation","Docs":"","Typewords":["Validation"]},{"Name":"DKIMDomains","Docs":"","Typewords":["[]","string"]},{"Name":"OrigEHLODomain","Docs":"","Typewords":["string"]},{"Name":"OrigDKIMDomains","Docs":"","Typewords":["[]","string"]},{"Name":"MessageID","Docs":"","Typewords":["string"]},{"Name":"SubjectBase","Docs":"","Typewords":["string"]},{"Name":"MessageHash","Docs":"","Typewords":["nullable","string"]},{"Name":"ThreadID","Docs":"","Typewords":["int64"]},{"Name":"ThreadParentIDs","Docs":"","Typewords":["[]","int64"]},{"Name":"ThreadMissingLink","Docs":"","Typewords":["bool"]},{"Name":"ThreadMuted","Docs":"","Typewords":["bool"]},{"Name":"ThreadCollapsed","Docs":"","Typewords":["bool"]},{"Name":"IsMailingList","Docs":"","Typewords":["bool"]},{"Name":"DSN","Docs":"","Typewords":["bool"]},{"Name":"ReceivedTLSVersion","Docs":"","Typewords":["uint16"]},{"Name":"ReceivedTLSCipherSuite","Docs":"","Typewords":["uint16"]},{"Name":"ReceivedRequireTLS","Docs":"","Typewords":["bool"]},{"Name":"Seen","Docs":"","Typewords":["bool"]},{"Name":"Answered","Docs":"","Typewords":["bool"]},{"Name":"Flagged","Docs":"","Typewords":["bool"]},{"Name":"Forwarded","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Notjunk","Docs":"","Typewords":["bool"]},{"Name":"Deleted","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Phishing","Docs":"","Typewords":["bool"]},{"Name":"MDNSent","Docs":"","Typewords":["bool"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"Size","Docs":"","Typewords":["int64"]},{"Name":"TrainedJunk","Docs":"","Typewords":["nullable","bool"]},{"Name":"MsgPrefix","Docs":"","Typewords":["nullable","string"]},{"Name":"Preview","Docs":"","Typewords":["nullable","string"]},{"Name":"ParsedBuf","Docs":"","Typewords":["nullable","string"]}]},
 	"MessageEnvelope": {"Name":"MessageEnvelope","Docs":"","Fields":[{"Name":"Date","Docs":"","Typewords":["timestamp"]},{"Name":"Subject","Docs":"","Typewords":["string"]},{"Name":"From","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"Sender","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"ReplyTo","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"To","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"CC","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"BCC","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"InReplyTo","Docs":"","Typewords":["string"]},{"Name":"MessageID","Docs":"","Typewords":["string"]}]},
 	"Attachment": {"Name":"Attachment","Docs":"","Fields":[{"Name":"Path","Docs":"","Typewords":["[]","int32"]},{"Name":"Filename","Docs":"","Typewords":["string"]},{"Name":"Part","Docs":"","Typewords":["Part"]}]},
 	"EventViewChanges": {"Name":"EventViewChanges","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"Changes","Docs":"","Typewords":["[]","[]","any"]}]},

webmail/message.go

@@ -1,7 +1,6 @@
 package webmail
 
 import (
-	"bufio"
 	"errors"
 	"fmt"
 	"io"
@@ -86,10 +85,10 @@ func messageItemMoreHeaders(moreHeaders []string, pm ParsedMessage) (l [][2]stri
 
 func messageItem(log mlog.Log, m store.Message, state *msgState, moreHeaders []string) (MessageItem, error) {
 	headers := len(moreHeaders) > 0
-	pm, err := parsedMessage(log, m, state, false, true, headers)
+	pm, err := parsedMessage(log, &m, state, false, true, headers)
 	if err != nil && errors.Is(err, message.ErrHeader) && headers {
 		log.Debugx("load message item without parsing headers after error", err, slog.Int64("msgid", m.ID))
-		pm, err = parsedMessage(log, m, state, false, true, false)
+		pm, err = parsedMessage(log, &m, state, false, true, false)
 	}
 	if err != nil {
 		return MessageItem{}, fmt.Errorf("parsing message %d for item: %v", m.ID, err)
@@ -98,117 +97,33 @@ func messageItem(log mlog.Log, m store.Message, state *msgState, moreHeaders []s
 	m.MsgPrefix = nil
 	m.ParsedBuf = nil
 	l := messageItemMoreHeaders(moreHeaders, pm)
-	return MessageItem{m, pm.envelope, pm.attachments, pm.isSigned, pm.isEncrypted, pm.firstLine, true, l}, nil
+	return MessageItem{m, pm.envelope, pm.attachments, pm.isSigned, pm.isEncrypted, true, l}, nil
 }
 
-// formatFirstLine returns a line the client can display next to the subject line
-// in a mailbox. It will replace quoted text, and any prefixing "On ... write:"
-// line with "[...]" so only new and useful information will be displayed.
-// Trailing signatures are not included.
-func formatFirstLine(r io.Reader) (string, error) {
-	// We look quite a bit of lines ahead for trailing signatures with trailing empty lines.
-	var lines []string
-	scanner := bufio.NewScanner(r)
-	ensureLines := func() {
-		for len(lines) < 10 && scanner.Scan() {
-			lines = append(lines, strings.TrimSpace(scanner.Text()))
-		}
-	}
-	ensureLines()
-
-	isSnipped := func(s string) bool {
-		return s == "[...]" || s == "[…]" || s == "..."
-	}
-
-	nextLineQuoted := func(i int) bool {
-		if i+1 < len(lines) && lines[i+1] == "" {
-			i++
-		}
-		return i+1 < len(lines) && (strings.HasPrefix(lines[i+1], ">") || isSnipped(lines[i+1]))
-	}
-
-	// Remainder is signature if we see a line with only and minimum 2 dashes, and
-	// there are no more empty lines, and there aren't more than 5 lines left.
-	isSignature := func() bool {
-		if len(lines) == 0 || !strings.HasPrefix(lines[0], "--") || strings.Trim(strings.TrimSpace(lines[0]), "-") != "" {
-			return false
-		}
-		l := lines[1:]
-		for len(l) > 0 && l[len(l)-1] == "" {
-			l = l[:len(l)-1]
-		}
-		if len(l) >= 5 {
-			return false
-		}
-		return !slices.Contains(l, "")
-	}
-
-	result := ""
-
-	resultSnipped := func() bool {
-		return strings.HasSuffix(result, "[...]\n") || strings.HasSuffix(result, "[…]")
-	}
-
-	// Quick check for initial wrapped "On ... wrote:" line.
-	if len(lines) > 3 && strings.HasPrefix(lines[0], "On ") && !strings.HasSuffix(lines[0], "wrote:") && strings.HasSuffix(lines[1], ":") && nextLineQuoted(1) {
-		result = "[...]\n"
-		lines = lines[3:]
-		ensureLines()
-	}
-
-	for ; len(lines) > 0 && !isSignature(); ensureLines() {
-		line := lines[0]
-		if strings.HasPrefix(line, ">") {
-			if !resultSnipped() {
-				result += "[...]\n"
-			}
-			lines = lines[1:]
-			continue
-		}
-		if line == "" {
-			lines = lines[1:]
-			continue
-		}
-		// Check for a "On <date>, <person> wrote:", we require digits before a quoted
-		// line, with an optional empty line in between. If we don't have any text yet, we
-		// don't require the digits.
-		if strings.HasSuffix(line, ":") && (strings.ContainsAny(line, "0123456789") || result == "") && nextLineQuoted(0) {
-			if !resultSnipped() {
-				result += "[...]\n"
-			}
-			lines = lines[1:]
-			continue
-		}
-		// Skip possibly duplicate snipping by author.
-		if !isSnipped(line) || !resultSnipped() {
-			result += line + "\n"
-		}
-		lines = lines[1:]
-		if len(result) > 250 {
-			break
-		}
-	}
-	if len(result) > 250 {
-		result = result[:230] + "..."
-	}
-	return result, scanner.Err()
-}
-
-func parsedMessage(log mlog.Log, m store.Message, state *msgState, full, msgitem, msgitemHeaders bool) (pm ParsedMessage, rerr error) {
+func parsedMessage(log mlog.Log, m *store.Message, state *msgState, full, msgitem, msgitemHeaders bool) (pm ParsedMessage, rerr error) {
 	pm.ViewMode = store.ModeText // Valid default, in case this makes it to frontend.
 
-	if full || msgitem {
-		if !state.ensurePart(m, true) {
+	if full || msgitem || state.newPreviews != nil && m.Preview == nil {
+		if !state.ensurePart(*m, true) {
 			return pm, state.err
 		}
 		if full {
 			pm.Part = *state.part
 		}
 	} else {
-		if !state.ensurePart(m, false) {
+		if !state.ensurePart(*m, false) {
 			return pm, state.err
 		}
 	}
+	if state.newPreviews != nil && m.Preview == nil {
+		s, err := state.part.Preview(log)
+		if err != nil {
+			log.Infox("generating preview", err, slog.Int64("msgid", m.ID))
+		}
+		// Set preview on m now, and let it be saved later on.
+		m.Preview = &s
+		state.newPreviews[m.ID] = s
+	}
 
 	// todo: we should store this form in message.Part, requires a data structure update.
 
@@ -311,13 +226,6 @@ func parsedMessage(log mlog.Log, m store.Message, state *msgState, full, msgitem
 				pm.Texts = append(pm.Texts, string(buf))
 				pm.TextPaths = append(pm.TextPaths, slices.Clone(path))
 			}
-			if msgitem && pm.firstLine == "" {
-				pm.firstLine, rerr = formatFirstLine(p.ReaderUTF8OrBinary())
-				if rerr != nil {
-					rerr = fmt.Errorf("reading text for first line snippet: %v", rerr)
-					return
-				}
-			}
 
 		case "TEXT/HTML":
 			pm.HasHTML = true

webmail/message_test.go

@@ -1,39 +1,11 @@
 package webmail
 
 import (
-	"strings"
 	"testing"
 
 	"github.com/mjl-/mox/dns"
 )
 
-func TestFormatFirstLine(t *testing.T) {
-	check := func(body, expLine string) {
-		t.Helper()
-
-		line, err := formatFirstLine(strings.NewReader(body))
-		tcompare(t, err, nil)
-		if line != expLine {
-			t.Fatalf("got %q, expected %q, for body %q", line, expLine, body)
-		}
-	}
-
-	check("", "")
-	check("single line", "single line\n")
-	check("single line\n", "single line\n")
-	check("> quoted\n", "[...]\n")
-	check("> quoted\nresponse\n", "[...]\nresponse\n")
-	check("> quoted\n[...]\nresponse after author snip\n", "[...]\nresponse after author snip\n")
-	check("[...]\nresponse after author snip\n", "[...]\nresponse after author snip\n")
-	check("[…]\nresponse after author snip\n", "[…]\nresponse after author snip\n")
-	check(">> quoted0\n> quoted1\n>quoted2\n[...]\nresponse after author snip\n", "[...]\nresponse after author snip\n")
-	check(">quoted\n\n>quoted\ncoalesce line-separated quotes\n", "[...]\ncoalesce line-separated quotes\n")
-	check("On <date> <user> wrote:\n> hi\nresponse", "[...]\nresponse\n")
-	check("On <longdate>\n<user> wrote:\n> hi\nresponse", "[...]\nresponse\n")
-	check("> quote\nresponse\n--\nsignature\n", "[...]\nresponse\n")
-	check("> quote\nline1\nline2\nline3\n", "[...]\nline1\nline2\nline3\n")
-}
-
 func TestParseListPostAddress(t *testing.T) {
 	check := func(s string, exp *MessageAddress) {
 		t.Helper()

webmail/msg.js

@@ -318,8 +318,8 @@ var api;
 		"EventViewErr": { "Name": "EventViewErr", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Err", "Docs": "", "Typewords": ["string"] }] },
 		"EventViewReset": { "Name": "EventViewReset", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }] },
 		"EventViewMsgs": { "Name": "EventViewMsgs", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "[]", "MessageItem"] }, { "Name": "ParsedMessage", "Docs": "", "Typewords": ["nullable", "ParsedMessage"] }, { "Name": "ViewEnd", "Docs": "", "Typewords": ["bool"] }] },
-		"MessageItem": { "Name": "MessageItem", "Docs": "", "Fields": [{ "Name": "Message", "Docs": "", "Typewords": ["Message"] }, { "Name": "Envelope", "Docs": "", "Typewords": ["MessageEnvelope"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "Attachment"] }, { "Name": "IsSigned", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsEncrypted", "Docs": "", "Typewords": ["bool"] }, { "Name": "FirstLine", "Docs": "", "Typewords": ["string"] }, { "Name": "MatchQuery", "Docs": "", "Typewords": ["bool"] }, { "Name": "MoreHeaders", "Docs": "", "Typewords": ["[]", "[]", "string"] }] },
-		"Message": { "Name": "Message", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsReject", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxOrigID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxDestinedID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Received", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "SaveDate", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "RemoteIP", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked1", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked2", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked3", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFrom", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MailFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "RcptToLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "RcptToDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MsgFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromOrgDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLOValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MsgFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "EHLOValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MailFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MsgFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "DKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "OrigEHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "OrigDKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }, { "Name": "SubjectBase", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageHash", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ThreadID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ThreadParentIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "ThreadMissingLink", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadMuted", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadCollapsed", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsMailingList", "Docs": "", "Typewords": ["bool"] }, { "Name": "DSN", "Docs": "", "Typewords": ["bool"] }, { "Name": "ReceivedTLSVersion", "Docs": "", "Typewords": ["uint16"] }, { "Name": "ReceivedTLSCipherSuite", "Docs": "", "Typewords": ["uint16"] }, { "Name": "ReceivedRequireTLS", "Docs": "", "Typewords": ["bool"] }, { "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }, { "Name": "TrainedJunk", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "MsgPrefix", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ParsedBuf", "Docs": "", "Typewords": ["nullable", "string"] }] },
+		"MessageItem": { "Name": "MessageItem", "Docs": "", "Fields": [{ "Name": "Message", "Docs": "", "Typewords": ["Message"] }, { "Name": "Envelope", "Docs": "", "Typewords": ["MessageEnvelope"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "Attachment"] }, { "Name": "IsSigned", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsEncrypted", "Docs": "", "Typewords": ["bool"] }, { "Name": "MatchQuery", "Docs": "", "Typewords": ["bool"] }, { "Name": "MoreHeaders", "Docs": "", "Typewords": ["[]", "[]", "string"] }] },
+		"Message": { "Name": "Message", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsReject", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxOrigID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxDestinedID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Received", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "SaveDate", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "RemoteIP", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked1", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked2", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked3", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFrom", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MailFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "RcptToLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "RcptToDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MsgFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromOrgDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLOValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MsgFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "EHLOValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MailFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MsgFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "DKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "OrigEHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "OrigDKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }, { "Name": "SubjectBase", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageHash", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ThreadID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ThreadParentIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "ThreadMissingLink", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadMuted", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadCollapsed", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsMailingList", "Docs": "", "Typewords": ["bool"] }, { "Name": "DSN", "Docs": "", "Typewords": ["bool"] }, { "Name": "ReceivedTLSVersion", "Docs": "", "Typewords": ["uint16"] }, { "Name": "ReceivedTLSCipherSuite", "Docs": "", "Typewords": ["uint16"] }, { "Name": "ReceivedRequireTLS", "Docs": "", "Typewords": ["bool"] }, { "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }, { "Name": "TrainedJunk", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "MsgPrefix", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "Preview", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ParsedBuf", "Docs": "", "Typewords": ["nullable", "string"] }] },
 		"MessageEnvelope": { "Name": "MessageEnvelope", "Docs": "", "Fields": [{ "Name": "Date", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "Sender", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "CC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "BCC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "InReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }] },
 		"Attachment": { "Name": "Attachment", "Docs": "", "Fields": [{ "Name": "Path", "Docs": "", "Typewords": ["[]", "int32"] }, { "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "Part", "Docs": "", "Typewords": ["Part"] }] },
 		"EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] },

webmail/text.js

@@ -318,8 +318,8 @@ var api;
 		"EventViewErr": { "Name": "EventViewErr", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Err", "Docs": "", "Typewords": ["string"] }] },
 		"EventViewReset": { "Name": "EventViewReset", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }] },
 		"EventViewMsgs": { "Name": "EventViewMsgs", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "[]", "MessageItem"] }, { "Name": "ParsedMessage", "Docs": "", "Typewords": ["nullable", "ParsedMessage"] }, { "Name": "ViewEnd", "Docs": "", "Typewords": ["bool"] }] },
-		"MessageItem": { "Name": "MessageItem", "Docs": "", "Fields": [{ "Name": "Message", "Docs": "", "Typewords": ["Message"] }, { "Name": "Envelope", "Docs": "", "Typewords": ["MessageEnvelope"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "Attachment"] }, { "Name": "IsSigned", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsEncrypted", "Docs": "", "Typewords": ["bool"] }, { "Name": "FirstLine", "Docs": "", "Typewords": ["string"] }, { "Name": "MatchQuery", "Docs": "", "Typewords": ["bool"] }, { "Name": "MoreHeaders", "Docs": "", "Typewords": ["[]", "[]", "string"] }] },
-		"Message": { "Name": "Message", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsReject", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxOrigID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxDestinedID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Received", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "SaveDate", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "RemoteIP", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked1", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked2", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked3", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFrom", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MailFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "RcptToLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "RcptToDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MsgFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromOrgDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLOValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MsgFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "EHLOValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MailFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MsgFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "DKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "OrigEHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "OrigDKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }, { "Name": "SubjectBase", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageHash", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ThreadID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ThreadParentIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "ThreadMissingLink", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadMuted", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadCollapsed", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsMailingList", "Docs": "", "Typewords": ["bool"] }, { "Name": "DSN", "Docs": "", "Typewords": ["bool"] }, { "Name": "ReceivedTLSVersion", "Docs": "", "Typewords": ["uint16"] }, { "Name": "ReceivedTLSCipherSuite", "Docs": "", "Typewords": ["uint16"] }, { "Name": "ReceivedRequireTLS", "Docs": "", "Typewords": ["bool"] }, { "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }, { "Name": "TrainedJunk", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "MsgPrefix", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ParsedBuf", "Docs": "", "Typewords": ["nullable", "string"] }] },
+		"MessageItem": { "Name": "MessageItem", "Docs": "", "Fields": [{ "Name": "Message", "Docs": "", "Typewords": ["Message"] }, { "Name": "Envelope", "Docs": "", "Typewords": ["MessageEnvelope"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "Attachment"] }, { "Name": "IsSigned", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsEncrypted", "Docs": "", "Typewords": ["bool"] }, { "Name": "MatchQuery", "Docs": "", "Typewords": ["bool"] }, { "Name": "MoreHeaders", "Docs": "", "Typewords": ["[]", "[]", "string"] }] },
+		"Message": { "Name": "Message", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsReject", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxOrigID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxDestinedID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Received", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "SaveDate", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "RemoteIP", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked1", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked2", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked3", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFrom", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MailFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "RcptToLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "RcptToDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MsgFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromOrgDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLOValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MsgFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "EHLOValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MailFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MsgFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "DKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "OrigEHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "OrigDKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }, { "Name": "SubjectBase", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageHash", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ThreadID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ThreadParentIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "ThreadMissingLink", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadMuted", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadCollapsed", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsMailingList", "Docs": "", "Typewords": ["bool"] }, { "Name": "DSN", "Docs": "", "Typewords": ["bool"] }, { "Name": "ReceivedTLSVersion", "Docs": "", "Typewords": ["uint16"] }, { "Name": "ReceivedTLSCipherSuite", "Docs": "", "Typewords": ["uint16"] }, { "Name": "ReceivedRequireTLS", "Docs": "", "Typewords": ["bool"] }, { "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }, { "Name": "TrainedJunk", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "MsgPrefix", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "Preview", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ParsedBuf", "Docs": "", "Typewords": ["nullable", "string"] }] },
 		"MessageEnvelope": { "Name": "MessageEnvelope", "Docs": "", "Fields": [{ "Name": "Date", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "Sender", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "CC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "BCC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "InReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }] },
 		"Attachment": { "Name": "Attachment", "Docs": "", "Fields": [{ "Name": "Path", "Docs": "", "Typewords": ["[]", "int32"] }, { "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "Part", "Docs": "", "Typewords": ["Part"] }] },
 		"EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] },

webmail/view.go

@@ -166,12 +166,11 @@ type MessageEnvelope struct {
 // message.Part, made for the needs of the message items in the message list.
 // messages.
 type MessageItem struct {
-	Message     store.Message // Without ParsedBuf and MsgPrefix, for size.
+	Message     store.Message // Without ParsedBuf and MsgPrefix, for size. With Preview, even if it isn't stored yet in the database.
 	Envelope    MessageEnvelope
 	Attachments []Attachment
 	IsSigned    bool
 	IsEncrypted bool
-	FirstLine   string      // Of message body, for showing as preview.
 	MatchQuery  bool        // If message does not match query, it can still be included because of threading.
 	MoreHeaders [][2]string // All headers from store.Settings.ShowHeaders that are present.
 }
@@ -204,7 +203,6 @@ type ParsedMessage struct {
 	attachments []Attachment
 	isSigned    bool
 	isEncrypted bool
-	firstLine   string
 }
 
 // EventStart is the first message sent on an SSE connection, giving the client
@@ -816,6 +814,9 @@ func serveEvents(ctx context.Context, log mlog.Log, accountPath string, w http.R
 	xprocessChanges := func(changes []store.Change) {
 		taggedChanges := [][2]any{}
 
+		newPreviews := map[int64]string{}
+		defer storeNewPreviews(ctx, log, acc, newPreviews)
+
 		// We get a transaction first time we need it.
 		var xtx *bstore.Tx
 		defer func() {
@@ -891,7 +892,7 @@ func serveEvents(ctx context.Context, log mlog.Log, accountPath string, w http.R
 					continue
 				}
 
-				state := msgState{acc: acc, log: log}
+				state := msgState{acc: acc, log: log, newPreviews: newPreviews}
 				mi, err := messageItem(log, m, &state, xmoreHeaders())
 				state.clear()
 				xcheckf(ctx, err, "make messageitem")
@@ -901,7 +902,7 @@ func serveEvents(ctx context.Context, log mlog.Log, accountPath string, w http.R
 				if !thread && req.Query.Threading != ThreadOff {
 					err := ensureTx()
 					xcheckf(ctx, err, "transaction")
-					more, _, err := gatherThread(log, xtx, acc, v, m, 0, false, xmoreHeaders())
+					more, _, err := gatherThread(log, xtx, acc, v, m, 0, false, xmoreHeaders(), newPreviews)
 					xcheckf(ctx, err, "gathering thread messages for id %d, thread %d", m.ID, m.ThreadID)
 					mil = append(mil, more...)
 					v.threadIDs[m.ThreadID] = struct{}{}
@@ -1265,18 +1266,55 @@ type msgResp struct {
 	pm      *ParsedMessage // If m was the target page.DestMessageID, or this is the first match, this is the parsed message of mi.
 }
 
+func storeNewPreviews(ctx context.Context, log mlog.Log, acc *store.Account, newPreviews map[int64]string) {
+	if len(newPreviews) == 0 {
+		return
+	}
+
+	defer func() {
+		x := recover()
+		if x != nil {
+			log.Error("unhandled panic in storeNewPreviews", slog.Any("err", x))
+			debug.PrintStack()
+			metrics.PanicInc(metrics.Store)
+		}
+	}()
+
+	err := acc.DB.Write(ctx, func(tx *bstore.Tx) error {
+		for id, preview := range newPreviews {
+			m := store.Message{ID: id}
+			if err := tx.Get(&m); err != nil {
+				return fmt.Errorf("get message with id %d to store preview: %w", id, err)
+			} else if !m.Expunged {
+				m.Preview = &preview
+				if err := tx.Update(&m); err != nil {
+					return fmt.Errorf("updating message with id %d: %v", m.ID, err)
+				}
+			}
+		}
+		return nil
+	})
+	log.Check(err, "saving new previews with messages")
+}
+
 // viewRequestTx executes a request (query with filters, pagination) by
 // launching a new goroutine with queryMessages, receiving results as msgResp,
 // and sending Event* to the SSE connection.
 //
 // It always closes tx.
 func viewRequestTx(ctx context.Context, log mlog.Log, acc *store.Account, tx *bstore.Tx, v view, msgc chan EventViewMsgs, errc chan EventViewErr, resetc chan EventViewReset, donec chan int64) {
+	// Newly generated previews which we'll save when the operation is done.
+	newPreviews := map[int64]string{}
+
 	defer func() {
 		err := tx.Rollback()
 		log.Check(err, "rolling back query transaction")
 
 		donec <- v.Request.ID
 
+		// ctx can be canceled, we still want to store the previews.
+		storeNewPreviews(context.Background(), log, acc, newPreviews)
+
 		x := recover() // Should not happen, but don't take program down if it does.
 		if x != nil {
 			log.WithContext(ctx).Error("viewRequestTx panic", slog.Any("err", x))
@@ -1308,7 +1346,7 @@ func viewRequestTx(ctx context.Context, log mlog.Log, acc *store.Account, tx *bs
 	// todo: should probably rewrite code so we don't start yet another goroutine, but instead handle the query responses directly (through a struct that keeps state?) in the sse connection goroutine.
 
 	mrc := make(chan msgResp, 1)
-	go queryMessages(ctx, log, acc, tx, v, mrc)
+	go queryMessages(ctx, log, acc, tx, v, mrc, newPreviews)
 
 	for {
 		select {
@@ -1358,7 +1396,8 @@ func viewRequestTx(ctx context.Context, log mlog.Log, acc *store.Account, tx *bs
 // It sends on msgc, with several types of messages: errors, whether the view is
 // reset due to missing AnchorMessageID, and when the end of the view was reached
 // and/or for a message.
-func queryMessages(ctx context.Context, log mlog.Log, acc *store.Account, tx *bstore.Tx, v view, mrc chan msgResp) {
+// newPreviews is filled with previews, the caller must save them.
+func queryMessages(ctx context.Context, log mlog.Log, acc *store.Account, tx *bstore.Tx, v view, mrc chan msgResp, newPreviews map[int64]string) {
 	defer func() {
 		x := recover() // Should not happen, but don't take program down if it does.
 		if x != nil {
@@ -1453,7 +1492,7 @@ func queryMessages(ctx context.Context, log mlog.Log, acc *store.Account, tx *bs
 	// implement reporting errors, or anything else, just a bool. So when making the
 	// filter functions, we give them a place to store parsed message state, and an
 	// error. We check the error during and after query execution.
-	state := msgState{acc: acc, log: log}
+	state := msgState{acc: acc, log: log, newPreviews: newPreviews}
 	defer state.clear()
 
 	flagfilter := query.flagFilterFn()
@@ -1530,7 +1569,7 @@ func queryMessages(ctx context.Context, log mlog.Log, acc *store.Account, tx *bs
 			// expected to read first, that would be the first unread, which we'll get below
 			// when gathering the thread.
 			found = true
-			xpm, err := parsedMessage(log, m, &state, true, false, false)
+			xpm, err := parsedMessage(log, &m, &state, true, false, false)
 			if err != nil && errors.Is(err, message.ErrHeader) {
 				log.Debug("not returning parsed message due to invalid headers", slog.Int64("msgid", m.ID), slog.Any("err", err))
 			} else if err != nil {
@@ -1552,7 +1591,7 @@ func queryMessages(ctx context.Context, log mlog.Log, acc *store.Account, tx *bs
 		}
 		mil := []MessageItem{mi}
 		if query.Threading != ThreadOff {
-			more, xpm, err := gatherThread(log, tx, acc, v, m, page.DestMessageID, page.AnchorMessageID == 0 && have == 0, moreHeaders)
+			more, xpm, err := gatherThread(log, tx, acc, v, m, page.DestMessageID, page.AnchorMessageID == 0 && have == 0, moreHeaders, state.newPreviews)
 			if err != nil {
 				return fmt.Errorf("gathering thread messages for id %d, thread %d: %v", m.ID, m.ThreadID, err)
 			}
@@ -1621,7 +1660,7 @@ func queryMessages(ctx context.Context, log mlog.Log, acc *store.Account, tx *bs
 	}
 }
 
-func gatherThread(log mlog.Log, tx *bstore.Tx, acc *store.Account, v view, m store.Message, destMessageID int64, first bool, moreHeaders []string) ([]MessageItem, *ParsedMessage, error) {
+func gatherThread(log mlog.Log, tx *bstore.Tx, acc *store.Account, v view, m store.Message, destMessageID int64, first bool, moreHeaders []string, newPreviews map[int64]string) ([]MessageItem, *ParsedMessage, error) {
 	if m.ThreadID == 0 {
 		// If we would continue, FilterNonzero would fail because there are no non-zero fields.
 		return nil, nil, fmt.Errorf("message has threadid 0, account is probably still being upgraded, try turning threading off until the upgrade is done")
@@ -1643,7 +1682,7 @@ func gatherThread(log mlog.Log, tx *bstore.Tx, acc *store.Account, v view, m sto
 	var firstUnread bool
 	for _, tm := range tml {
 		err := func() error {
-			xstate := msgState{acc: acc, log: log}
+			xstate := msgState{acc: acc, log: log, newPreviews: newPreviews}
 			defer xstate.clear()
 
 			mi, err := messageItem(log, tm, &xstate, moreHeaders)
@@ -1660,7 +1699,7 @@ func gatherThread(log mlog.Log, tx *bstore.Tx, acc *store.Account, v view, m sto
 
 			if tm.ID == destMessageID || destMessageID == 0 && first && (pm == nil || !firstUnread && !tm.Seen) {
 				firstUnread = !tm.Seen
-				xpm, err := parsedMessage(log, tm, &xstate, true, false, false)
+				xpm, err := parsedMessage(log, &tm, &xstate, true, false, false)
 				if err != nil && errors.Is(err, message.ErrHeader) {
 					log.Debug("not returning parsed message due to invalid headers", slog.Int64("msgid", m.ID), slog.Any("err", err))
 				} else if err != nil {
@@ -1681,7 +1720,7 @@ func gatherThread(log mlog.Log, tx *bstore.Tx, acc *store.Account, v view, m sto
 	if destMessageID == 0 && first && !m.Seen && !firstUnread {
 		xstate := msgState{acc: acc, log: log}
 		defer xstate.clear()
-		xpm, err := parsedMessage(log, m, &xstate, true, false, false)
+		xpm, err := parsedMessage(log, &m, &xstate, true, false, false)
 		if err != nil && errors.Is(err, message.ErrHeader) {
 			log.Debug("not returning parsed message due to invalid headers", slog.Int64("msgid", m.ID), slog.Any("err", err))
 		} else if err != nil {
@@ -1706,6 +1745,11 @@ type msgState struct {
 	part *message.Part // Will be without Reader when msgr is nil.
 	msgr *store.MsgReader
 	log  mlog.Log
+
+	// If not nil, messages will get their Preview field filled when nil, and message
+	// id and preview added to newPreviews, and saved in a separate write transaction
+	// when the operation is done.
+	newPreviews map[int64]string
 }
 
 func (ms *msgState) clear() {
@@ -1714,7 +1758,7 @@ func (ms *msgState) clear() {
 		ms.log.Check(err, "closing message reader from state")
 		ms.msgr = nil
 	}
-	*ms = msgState{acc: ms.acc, err: ms.err, log: ms.log}
+	*ms = msgState{acc: ms.acc, err: ms.err, log: ms.log, newPreviews: ms.newPreviews}
 }
 
 func (ms *msgState) ensureMsg(m store.Message) {
@@ -1864,7 +1908,7 @@ var attachmentExtensions = map[string]AttachmentType{
 func attachmentTypes(log mlog.Log, m store.Message, state *msgState) (map[AttachmentType]bool, error) {
 	types := map[AttachmentType]bool{}
 
-	pm, err := parsedMessage(log, m, state, false, false, false)
+	pm, err := parsedMessage(log, &m, state, false, false, false)
 	if err != nil {
 		return nil, fmt.Errorf("parsing message for attachments: %w", err)
 	}

webmail/webmail.go

@@ -655,7 +655,7 @@ func handle(apiHandler http.Handler, isForwarded bool, accountPath string, w htt
 		state := msgState{acc: acc, m: m, msgr: msgr, part: &p}
 		// note: state is cleared by cleanup
 
-		pm, err := parsedMessage(log, m, &state, true, true, true)
+		pm, err := parsedMessage(log, &m, &state, true, true, true)
 		xcheckf(ctx, err, "getting parsed message")
 		if t[1] == "msgtext" && len(pm.Texts) == 0 || t[1] != "msgtext" && !pm.HasHTML {
 			http.Error(w, "400 - bad request - no such part", http.StatusBadRequest)
@@ -687,7 +687,7 @@ func handle(apiHandler http.Handler, isForwarded bool, accountPath string, w htt
 		state := msgState{acc: acc, m: m, msgr: msgr, part: &p}
 		// note: state is cleared by cleanup
 
-		pm, err := parsedMessage(log, m, &state, true, true, true)
+		pm, err := parsedMessage(log, &m, &state, true, true, true)
 		xcheckf(ctx, err, "parsing parsedmessage")
 		pmjson, err := json.Marshal(pm)
 		xcheckf(ctx, err, "marshal parsedmessage")
@@ -695,7 +695,7 @@ func handle(apiHandler http.Handler, isForwarded bool, accountPath string, w htt
 		m.MsgPrefix = nil
 		m.ParsedBuf = nil
 		hl := messageItemMoreHeaders(moreHeaders, pm)
-		mi := MessageItem{m, pm.envelope, pm.attachments, pm.isSigned, pm.isEncrypted, pm.firstLine, false, hl}
+		mi := MessageItem{m, pm.envelope, pm.attachments, pm.isSigned, pm.isEncrypted, false, hl}
 		mijson, err := json.Marshal(mi)
 		xcheckf(ctx, err, "marshal messageitem")
 
@@ -720,7 +720,7 @@ func handle(apiHandler http.Handler, isForwarded bool, accountPath string, w htt
 		state := msgState{acc: acc, m: m, msgr: msgr, part: &p}
 		// note: state is cleared by cleanup
 
-		pm, err := parsedMessage(log, m, &state, true, true, true)
+		pm, err := parsedMessage(log, &m, &state, true, true, true)
 		xcheckf(ctx, err, "parsing parsedmessage")
 
 		if len(pm.Texts) == 0 {

webmail/webmail.js

@@ -318,8 +318,8 @@ var api;
 		"EventViewErr": { "Name": "EventViewErr", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Err", "Docs": "", "Typewords": ["string"] }] },
 		"EventViewReset": { "Name": "EventViewReset", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }] },
 		"EventViewMsgs": { "Name": "EventViewMsgs", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "[]", "MessageItem"] }, { "Name": "ParsedMessage", "Docs": "", "Typewords": ["nullable", "ParsedMessage"] }, { "Name": "ViewEnd", "Docs": "", "Typewords": ["bool"] }] },
-		"MessageItem": { "Name": "MessageItem", "Docs": "", "Fields": [{ "Name": "Message", "Docs": "", "Typewords": ["Message"] }, { "Name": "Envelope", "Docs": "", "Typewords": ["MessageEnvelope"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "Attachment"] }, { "Name": "IsSigned", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsEncrypted", "Docs": "", "Typewords": ["bool"] }, { "Name": "FirstLine", "Docs": "", "Typewords": ["string"] }, { "Name": "MatchQuery", "Docs": "", "Typewords": ["bool"] }, { "Name": "MoreHeaders", "Docs": "", "Typewords": ["[]", "[]", "string"] }] },
-		"Message": { "Name": "Message", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsReject", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxOrigID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxDestinedID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Received", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "SaveDate", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "RemoteIP", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked1", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked2", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked3", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFrom", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MailFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "RcptToLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "RcptToDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MsgFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromOrgDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLOValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MsgFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "EHLOValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MailFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MsgFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "DKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "OrigEHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "OrigDKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }, { "Name": "SubjectBase", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageHash", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ThreadID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ThreadParentIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "ThreadMissingLink", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadMuted", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadCollapsed", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsMailingList", "Docs": "", "Typewords": ["bool"] }, { "Name": "DSN", "Docs": "", "Typewords": ["bool"] }, { "Name": "ReceivedTLSVersion", "Docs": "", "Typewords": ["uint16"] }, { "Name": "ReceivedTLSCipherSuite", "Docs": "", "Typewords": ["uint16"] }, { "Name": "ReceivedRequireTLS", "Docs": "", "Typewords": ["bool"] }, { "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }, { "Name": "TrainedJunk", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "MsgPrefix", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ParsedBuf", "Docs": "", "Typewords": ["nullable", "string"] }] },
+		"MessageItem": { "Name": "MessageItem", "Docs": "", "Fields": [{ "Name": "Message", "Docs": "", "Typewords": ["Message"] }, { "Name": "Envelope", "Docs": "", "Typewords": ["MessageEnvelope"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "Attachment"] }, { "Name": "IsSigned", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsEncrypted", "Docs": "", "Typewords": ["bool"] }, { "Name": "MatchQuery", "Docs": "", "Typewords": ["bool"] }, { "Name": "MoreHeaders", "Docs": "", "Typewords": ["[]", "[]", "string"] }] },
+		"Message": { "Name": "Message", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsReject", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxOrigID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxDestinedID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Received", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "SaveDate", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "RemoteIP", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked1", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked2", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked3", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFrom", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MailFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "RcptToLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "RcptToDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MsgFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromOrgDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLOValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MsgFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "EHLOValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MailFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MsgFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "DKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "OrigEHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "OrigDKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }, { "Name": "SubjectBase", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageHash", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ThreadID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ThreadParentIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "ThreadMissingLink", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadMuted", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadCollapsed", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsMailingList", "Docs": "", "Typewords": ["bool"] }, { "Name": "DSN", "Docs": "", "Typewords": ["bool"] }, { "Name": "ReceivedTLSVersion", "Docs": "", "Typewords": ["uint16"] }, { "Name": "ReceivedTLSCipherSuite", "Docs": "", "Typewords": ["uint16"] }, { "Name": "ReceivedRequireTLS", "Docs": "", "Typewords": ["bool"] }, { "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }, { "Name": "TrainedJunk", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "MsgPrefix", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "Preview", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ParsedBuf", "Docs": "", "Typewords": ["nullable", "string"] }] },
 		"MessageEnvelope": { "Name": "MessageEnvelope", "Docs": "", "Fields": [{ "Name": "Date", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "Sender", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "CC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "BCC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "InReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }] },
 		"Attachment": { "Name": "Attachment", "Docs": "", "Fields": [{ "Name": "Path", "Docs": "", "Typewords": ["[]", "int32"] }, { "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "Part", "Docs": "", "Typewords": ["Part"] }] },
 		"EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] },
@@ -3585,7 +3585,7 @@ const newMsgitemView = (mi, msglistView, otherMailbox, listMailboxes, receivedTi
 		((msgitemView.parent || msgitemView.kids.length > 0) && !msgitemView.threadRoot().collapsed) ?
 			dom.div(css('msgItemThreadBar', { position: 'absolute', right: 0, top: 0, bottom: 0, borderRight: '2px solid', borderRightColor: styles.colorMilder }), !msgitemView.parent ? css('msgItemThreadBarFirst', { top: '50%', bottom: '-1px' }) : (isThreadLast() ?
 				css('msgItemThreadBarLast', { top: '-1px', bottom: '50%' }) :
-				css('msgItemThreadBarMiddle', { top: '-1px', bottom: '-1px' }))) : []), dom.div(msgItemCellStyle, css('msgItemSubject', { position: 'relative' }), dom.div(css('msgItemSubjectSpread', { display: 'flex', justifyContent: 'space-between', position: 'relative' }), dom.div(css('msgItemSubjectText', { whiteSpace: 'nowrap', overflow: 'hidden' }), threadIndent > 0 ? dom.span(threadChar, style({ paddingLeft: (threadIndent / 2) + 'em' }), css('msgItemThreadChar', { opacity: '.75', fontWeight: 'normal' }), threadCharTitle ? attr.title(threadCharTitle) : []) : [], msgitemView.parent ? [] : mi.Envelope.Subject || '(no subject)', dom.span(css('msgItemSubjectSnippet', { fontWeight: 'normal', color: styles.colorMilder }), ' ' + mi.FirstLine)), dom.div(keywords, mailboxtags))), dom.div(msgItemCellStyle, dom._class('msgItemAge'), age(received())), function click(e) {
+				css('msgItemThreadBarMiddle', { top: '-1px', bottom: '-1px' }))) : []), dom.div(msgItemCellStyle, css('msgItemSubject', { position: 'relative' }), dom.div(css('msgItemSubjectSpread', { display: 'flex', justifyContent: 'space-between', position: 'relative' }), dom.div(css('msgItemSubjectText', { whiteSpace: 'nowrap', overflow: 'hidden' }), threadIndent > 0 ? dom.span(threadChar, style({ paddingLeft: (threadIndent / 2) + 'em' }), css('msgItemThreadChar', { opacity: '.75', fontWeight: 'normal' }), threadCharTitle ? attr.title(threadCharTitle) : []) : [], msgitemView.parent ? [] : mi.Envelope.Subject || '(no subject)', dom.span(css('msgItemSubjectSnippet', { fontWeight: 'normal', color: styles.colorMilder }), ' ' + (mi.Message.Preview || ''))), dom.div(keywords, mailboxtags))), dom.div(msgItemCellStyle, dom._class('msgItemAge'), age(received())), function click(e) {
 			e.preventDefault();
 			e.stopPropagation();
 			msglistView.click(msgitemView, e.ctrlKey, e.shiftKey);

webmail/webmail.ts

@@ -2809,7 +2809,7 @@ const newMsgitemView = (mi: api.MessageItem, msglistView: MsglistView, otherMail
 						css('msgItemSubjectText', {whiteSpace: 'nowrap', overflow: 'hidden'}),
 						threadIndent > 0 ? dom.span(threadChar, style({paddingLeft: (threadIndent/2)+'em'}), css('msgItemThreadChar', {opacity: '.75', fontWeight: 'normal'}), threadCharTitle ? attr.title(threadCharTitle) : []) : [],
 						msgitemView.parent ? [] : mi.Envelope.Subject || '(no subject)',
-						dom.span(css('msgItemSubjectSnippet', {fontWeight: 'normal', color: styles.colorMilder}), ' '+mi.FirstLine),
+						dom.span(css('msgItemSubjectSnippet', {fontWeight: 'normal', color: styles.colorMilder}), ' '+(mi.Message.Preview || '')),
 					),
 					dom.div(
 						keywords,