infrastructure/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2025-07-13 08:54:38 +03:00
Code Issues Packages Projects Releases Wiki Activity

implement storing non-system/well-known flags (keywords) for messages and mailboxes, with imap

Browse Source 
the mailbox select/examine responses now return all flags used in a mailbox in
the FLAGS response. and indicate in the PERMANENTFLAGS response that clients
can set new keywords. we store these values on the new Message.Keywords field.
system/well-known flags are still in Message.Flags, so we're recognizing those
and handling them separately.

the imap store command handles the new flags. as does the append command, and
the search command.

we store keywords in a mailbox when a message in that mailbox gets the keyword.
we don't automatically remove the keywords from a mailbox. there is currently
no way at all to remove a keyword from a mailbox.

the import commands now handle non-system/well-known keywords too, when
importing from mbox/maildir.

jmap requires keyword support, so best to get it out of the way now.
This commit is contained in:
Mechiel Lukkien
2023-06-24 00:24:43 +02:00
parent afefadf2c0
commit 40163bd145
30 changed files with 1927 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
store/account.go
View File

@ -39,6 +39,7 @@ import (
"time"
"golang.org/x/crypto/bcrypt"
"golang.org/x/exp/slices"
"golang.org/x/text/unicode/norm"
"github.com/mjl-/bstore"
@ -172,6 +173,11 @@ type Mailbox struct {
Junk bool
Sent bool
Trash bool
// Keywords as used in messages. Storing a non-system keyword for a message
// automatically adds it to this list. Used in the IMAP FLAGS response. Only
// "atoms", stored in lower case.
Keywords []string
}
// Subscriptions are separate from existence of mailboxes.
@ -286,6 +292,7 @@ type Message struct {
MessageHash []byte // Hash of message. For rejects delivery, so optional like MessageID.
Flags
Keywords []string `bstore:"index"` // Non-system or well-known $-flags. Only in "atom" syntax, stored in lower case.
Size int64
TrainedJunk *bool // If nil, no training done yet. Otherwise, true is trained as junk, false trained as nonjunk.
MsgPrefix []byte // Typically holds received headers and/or header separator.
@ -1054,7 +1061,7 @@ func (a *Account) DeliverMailbox(log *mlog.Log, mailbox string, m *Message, msgF
return err
}
changes = append(changes, ChangeAddUID{m.MailboxID, m.UID, m.Flags})
changes = append(changes, ChangeAddUID{m.MailboxID, m.UID, m.Flags, m.Keywords})
comm := RegisterComm(a)
defer comm.Unregister()
comm.Broadcast(changes)
@ -1348,24 +1355,44 @@ func (f Flags) Set(mask, flags Flags) Flags {
return r
}
// FlagsQuerySet returns a map with the flags that are true in mask, with
// values from flags.
func FlagsQuerySet(mask, flags Flags) map[string]any {
r := map[string]any{}
set := func(f string, m, v bool) {
if m {
r[f] = v
// RemoveKeywords removes keywords from l, modifying and returning it. Should only
// be used with lower-case keywords, not with system flags like \Seen.
func RemoveKeywords(l, remove []string) []string {
for _, k := range remove {
if i := slices.Index(l, k); i >= 0 {
copy(l[i:], l[i+1:])
l = l[:len(l)-1]
}
}
set("Seen", mask.Seen, flags.Seen)
set("Answered", mask.Answered, flags.Answered)
set("Flagged", mask.Flagged, flags.Flagged)
set("Forwarded", mask.Forwarded, flags.Forwarded)
set("Junk", mask.Junk, flags.Junk)
set("Notjunk", mask.Notjunk, flags.Notjunk)
set("Deleted", mask.Deleted, flags.Deleted)
set("Draft", mask.Draft, flags.Draft)
set("Phishing", mask.Phishing, flags.Phishing)
set("MDNSent", mask.MDNSent, flags.MDNSent)
return r
return l
}
// MergeKeywords adds keywords from add into l, updating and returning it along
// with whether it added any keyword. Keywords are only added if they aren't
// already present. Should only be used with lower-case keywords, not with system
// flags like \Seen.
func MergeKeywords(l, add []string) ([]string, bool) {
var changed bool
for _, k := range add {
if slices.Index(l, k) < 0 {
l = append(l, k)
changed = true
}
}
return l, changed
}
// ValidLowercaseKeyword returns whether s is a valid, lower-case, keyword.
func ValidLowercaseKeyword(s string) bool {
for _, c := range s {
if c >= 'a' && c <= 'z' {
continue
}
// ../rfc/9051:6334
const atomspecials = `(){%*"\]`
if c <= ' ' || c > 0x7e || strings.ContainsRune(atomspecials, c) {
return false
}
}
return len(s) > 0
}

58
store/import.go
View File

@ -12,6 +12,8 @@ import (
"strings"
"time"
"golang.org/x/exp/maps"
"github.com/mjl-/mox/mlog"
)
@ -92,6 +94,7 @@ func (mr *MboxReader) Next() (*Message, *os.File, string, error) {
fromLine := mr.fromLine
bf := bufio.NewWriter(f)
var flags Flags
keywords := map[string]bool{}
var size int64
for {
line, err := mr.r.ReadBytes('\n')
@ -132,7 +135,23 @@ func (mr *MboxReader) Next() (*Message, *os.File, string, error) {
} else if bytes.HasPrefix(line, []byte("X-Keywords:")) {
s := strings.TrimSpace(strings.SplitN(string(line), ":", 2)[1])
for _, t := range strings.Split(s, ",") {
flagSet(&flags, strings.ToLower(strings.TrimSpace(t)))
word := strings.ToLower(strings.TrimSpace(t))
switch word {
case "forwarded", "$forwarded":
flags.Forwarded = true
case "junk", "$junk":
flags.Junk = true
case "notjunk", "$notjunk", "nonjunk", "$nonjunk":
flags.Notjunk = true
case "phishing", "$phishing":
flags.Phishing = true
case "mdnsent", "$mdnsent":
flags.MDNSent = true
default:
if ValidLowercaseKeyword(word) {
keywords[word] = true
}
}
}
}
}
@ -165,7 +184,7 @@ func (mr *MboxReader) Next() (*Message, *os.File, string, error) {
return nil, nil, mr.Position(), fmt.Errorf("flush: %v", err)
}
m := &Message{Flags: flags, Size: size}
m := &Message{Flags: flags, Keywords: maps.Keys(keywords), Size: size}
if t := strings.SplitN(fromLine, " ", 3); len(t) == 3 {
layouts := []string{time.ANSIC, time.UnixDate, time.RubyDate}
@ -297,6 +316,7 @@ func (mr *MaildirReader) Next() (*Message, *os.File, string, error) {
// Parse flags. See https://cr.yp.to/proto/maildir.html.
flags := Flags{}
keywords := map[string]bool{}
t = strings.SplitN(filepath.Base(sf.Name()), ":2,", 2)
if len(t) == 2 {
for _, c := range t[1] {
@ -319,26 +339,29 @@ func (mr *MaildirReader) Next() (*Message, *os.File, string, error) {
if index >= len(mr.dovecotKeywords) {
continue
}
kw := mr.dovecotKeywords[index]
kw := strings.ToLower(mr.dovecotKeywords[index])
switch kw {
case "$Forwarded", "Forwarded":
case "$forwarded", "forwarded":
flags.Forwarded = true
case "$Junk", "Junk":
case "$junk", "junk":
flags.Junk = true
case "$NotJunk", "NotJunk", "NonJunk":
case "$notjunk", "notjunk", "nonjunk":
flags.Notjunk = true
case "$MDNSent":
case "$mdnsent", "mdnsent":
flags.MDNSent = true
case "$Phishing", "Phishing":
case "$phishing", "phishing":
flags.Phishing = true
default:
if ValidLowercaseKeyword(kw) {
keywords[kw] = true
}
}
// todo: custom labels, e.g. $label1, JunkRecorded?
}
}
}
}
m := &Message{Received: received, Flags: flags, Size: size}
m := &Message{Received: received, Flags: flags, Keywords: maps.Keys(keywords), Size: size}
// Prevent cleanup by defer.
mf := f
@ -397,18 +420,3 @@ func ParseDovecotKeywords(r io.Reader, log *mlog.Log) ([]string, error) {
}
return keywords[:end], err
}
func flagSet(flags *Flags, word string) {
switch word {
case "forwarded", "$forwarded":
flags.Forwarded = true
case "junk", "$junk":
flags.Junk = true
case "notjunk", "$notjunk", "nonjunk", "$nonjunk":
flags.Notjunk = true
case "phishing", "$phishing":
flags.Phishing = true
case "mdnsent", "$mdnsent":
flags.MDNSent = true
}
}

8
store/state.go
View File

@ -26,7 +26,8 @@ type Change any
type ChangeAddUID struct {
MailboxID int64
UID UID
Flags Flags
Flags Flags // System flags.
Keywords []string // Other flags.
}
// ChangeRemoveUIDs is sent for removal of one or more messages from a mailbox.
@ -39,8 +40,9 @@ type ChangeRemoveUIDs struct {
type ChangeFlags struct {
MailboxID int64
UID UID
Mask Flags // Which flags are actually modified.
Flags Flags // New flag values. All are set, not just mask.
Mask Flags // Which flags are actually modified.
Flags Flags // New flag values. All are set, not just mask.
Keywords []string // Other flags.
}
// ChangeRemoveMailbox is sent for a removed mailbox.