infrastructure/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2025-07-12 17:44:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

implement storing non-system/well-known flags (keywords) for messages and mailboxes, with imap

Browse Source 
the mailbox select/examine responses now return all flags used in a mailbox in
the FLAGS response. and indicate in the PERMANENTFLAGS response that clients
can set new keywords. we store these values on the new Message.Keywords field.
system/well-known flags are still in Message.Flags, so we're recognizing those
and handling them separately.

the imap store command handles the new flags. as does the append command, and
the search command.

we store keywords in a mailbox when a message in that mailbox gets the keyword.
we don't automatically remove the keywords from a mailbox. there is currently
no way at all to remove a keyword from a mailbox.

the import commands now handle non-system/well-known keywords too, when
importing from mbox/maildir.

jmap requires keyword support, so best to get it out of the way now.
This commit is contained in:
Mechiel Lukkien
2023-06-24 00:24:43 +02:00
parent afefadf2c0
commit 40163bd145
30 changed files with 1927 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
http/account_test.go
View File

@ -14,14 +14,19 @@ import (
"os"
"path"
"path/filepath"
"sort"
"strings"
"testing"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/store"
)
var ctxbg = context.Background()
func tcheck(t *testing.T, err error, msg string) {
t.Helper()
if err != nil {
@ -50,7 +55,7 @@ func TestAccount(t *testing.T) {
if authHdr != "" {
r.Header.Add("Authorization", authHdr)
}
ok := checkAccountAuth(context.Background(), log, w, r)
ok := checkAccountAuth(ctxbg, log, w, r)
if ok != expect {
t.Fatalf("got %v, expected %v", ok, expect)
}
@ -59,7 +64,7 @@ func TestAccount(t *testing.T) {
const authOK = "Basic bWpsQG1veC5leGFtcGxlOnRlc3QxMjM0" // mjl@mox.example:test1234
const authBad = "Basic bWpsQG1veC5leGFtcGxlOmJhZHBhc3N3b3Jk" // mjl@mox.example:badpassword
authCtx := context.WithValue(context.Background(), authCtxKey, "mjl")
authCtx := context.WithValue(ctxbg, authCtxKey, "mjl")
test(authOK, "") // No password set yet.
Account{}.SetPassword(authCtx, "test1234")
@ -132,6 +137,39 @@ func TestAccount(t *testing.T) {
testImport("../testdata/importtest.mbox.zip", 2)
testImport("../testdata/importtest.maildir.tgz", 2)
// Check there are messages, with the right flags.
acc.DB.Read(ctxbg, func(tx *bstore.Tx) error {
_, err = bstore.QueryTx[store.Message](tx).FilterIn("Keywords", "other").FilterIn("Keywords", "test").Get()
tcheck(t, err, `fetching message with keywords "other" and "test"`)
mb, err := acc.MailboxFind(tx, "importtest")
tcheck(t, err, "looking up mailbox importtest")
if mb == nil {
t.Fatalf("missing mailbox importtest")
}
sort.Strings(mb.Keywords)
if strings.Join(mb.Keywords, " ") != "other test" {
t.Fatalf(`expected mailbox keywords "other" and "test", got %v`, mb.Keywords)
}
n, err := bstore.QueryTx[store.Message](tx).FilterIn("Keywords", "custom").Count()
tcheck(t, err, `fetching message with keyword "custom"`)
if n != 2 {
t.Fatalf(`got %d messages with keyword "custom", expected 2`, n)
}
mb, err = acc.MailboxFind(tx, "maildir")
tcheck(t, err, "looking up mailbox maildir")
if mb == nil {
t.Fatalf("missing mailbox maildir")
}
if strings.Join(mb.Keywords, " ") != "custom" {
t.Fatalf(`expected mailbox keywords "custom", got %v`, mb.Keywords)
}
return nil
})
testExport := func(httppath string, iszip bool, expectFiles int) {
t.Helper()

9
http/admin_test.go
View File

@ -1,7 +1,6 @@
package http
import (
"context"
"crypto/ed25519"
"net"
"net/http/httptest"
@ -29,7 +28,7 @@ func TestAdminAuth(t *testing.T) {
if authHdr != "" {
r.Header.Add("Authorization", authHdr)
}
ok := checkAdminAuth(context.Background(), passwordfile, w, r)
ok := checkAdminAuth(ctxbg, passwordfile, w, r)
if ok != expect {
t.Fatalf("got %v, expected %v", ok, expect)
}
@ -125,9 +124,9 @@ func TestCheckDomain(t *testing.T) {
close(done)
dialer := &net.Dialer{Deadline: time.Now().Add(-time.Second), Cancel: done}
checkDomain(context.Background(), resolver, dialer, "mox.example")
checkDomain(ctxbg, resolver, dialer, "mox.example")
// todo: check returned data
Admin{}.Domains(context.Background()) // todo: check results
dnsblsStatus(context.Background(), resolver) // todo: check results
Admin{}.Domains(ctxbg) // todo: check results
dnsblsStatus(ctxbg, resolver) // todo: check results
}

81
http/import.go
View File

@ -19,6 +19,7 @@ import (
"strings"
"time"
"golang.org/x/exp/maps"
"golang.org/x/text/unicode/norm"
"github.com/mjl-/bstore"
@ -361,10 +362,16 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
mailboxes := map[string]store.Mailbox{}
messages := map[string]int{}
// For maildirs, we are likely to get a possible dovecot-keywords file after having imported the messages. Once we see the keywords, we use them. But before that time we remember which messages miss a keywords. Once the keywords become available, we'll fix up the flags for the unknown messages
// For maildirs, we are likely to get a possible dovecot-keywords file after having
// imported the messages. Once we see the keywords, we use them. But before that
// time we remember which messages miss a keywords. Once the keywords become
// available, we'll fix up the flags for the unknown messages
mailboxKeywords := map[string]map[rune]string{} // Mailbox to 'a'-'z' to flag name.
mailboxMissingKeywordMessages := map[string]map[int64]string{} // Mailbox to message id to string consisting of the unrecognized flags.
// We keep the mailboxes we deliver to up to date with their keywords (non-system flags).
destMailboxKeywords := map[int64]map[string]bool{}
// Previous mailbox an event was sent for. We send an event for new mailboxes, when
// another 100 messages were added, when adding a message to another mailbox, and
// finally at the end as a closing statement.
@ -471,6 +478,15 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
m.MailboxID = mb.ID
m.MailboxOrigID = mb.ID
if len(m.Keywords) > 0 {
if destMailboxKeywords[mb.ID] == nil {
destMailboxKeywords[mb.ID] = map[string]bool{}
}
for _, k := range m.Keywords {
destMailboxKeywords[mb.ID][k] = true
}
}
// Parse message and store parsed information for later fast retrieval.
p, err := message.EnsurePart(f, m.Size)
if err != nil {
@ -503,7 +519,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
return
}
deliveredIDs = append(deliveredIDs, m.ID)
changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, Flags: m.Flags})
changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, Flags: m.Flags, Keywords: m.Keywords})
messages[mb.Name]++
if messages[mb.Name]%100 == 0 || prevMailbox != mb.Name {
prevMailbox = mb.Name
@ -583,7 +599,8 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
// Parse flags. See https://cr.yp.to/proto/maildir.html.
var keepFlags string
flags := store.Flags{}
var flags store.Flags
keywords := map[string]bool{}
t = strings.SplitN(path.Base(filename), ":2,", 2)
if len(t) == 2 {
for _, c := range t[1] {
@ -602,12 +619,12 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
flags.Flagged = true
default:
if c >= 'a' && c <= 'z' {
keywords, ok := mailboxKeywords[mailbox]
dovecotKeywords, ok := mailboxKeywords[mailbox]
if !ok {
// No keywords file seen yet, we'll try later if it comes in.
keepFlags += string(c)
} else if kw, ok := keywords[c]; ok {
flagSet(&flags, strings.ToLower(kw))
} else if kw, ok := dovecotKeywords[c]; ok {
flagSet(&flags, keywords, strings.ToLower(kw))
}
}
}
@ -617,6 +634,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
m := store.Message{
Received: received,
Flags: flags,
Keywords: maps.Keys(keywords),
Size: size,
}
xdeliver(mb, &m, f, filename)
@ -663,38 +681,52 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
default:
if path.Base(name) == "dovecot-keywords" {
mailbox := path.Dir(name)
keywords := map[rune]string{}
dovecotKeywords := map[rune]string{}
words, err := store.ParseDovecotKeywords(r, log)
log.Check(err, "parsing dovecot keywords for mailbox", mlog.Field("mailbox", mailbox))
for i, kw := range words {
keywords['a'+rune(i)] = kw
dovecotKeywords['a'+rune(i)] = kw
}
mailboxKeywords[mailbox] = keywords
mailboxKeywords[mailbox] = dovecotKeywords
for id, chars := range mailboxMissingKeywordMessages[mailbox] {
var flags, zeroflags store.Flags
keywords := map[string]bool{}
for _, c := range chars {
kw, ok := keywords[c]
kw, ok := dovecotKeywords[c]
if !ok {
problemf("unspecified message flag %c for message id %d (continuing)", c, id)
problemf("unspecified dovecot message flag %c for message id %d (continuing)", c, id)
continue
}
flagSet(&flags, strings.ToLower(kw))
flagSet(&flags, keywords, strings.ToLower(kw))
}
if flags == zeroflags {
if flags == zeroflags && len(keywords) == 0 {
continue
}
m := store.Message{ID: id}
err := tx.Get(&m)
ximportcheckf(err, "get imported message for flag update")
m.Flags = m.Flags.Set(flags, flags)
m.Keywords = maps.Keys(keywords)
if len(m.Keywords) > 0 {
if destMailboxKeywords[m.MailboxID] == nil {
destMailboxKeywords[m.MailboxID] = map[string]bool{}
}
for _, k := range m.Keywords {
destMailboxKeywords[m.MailboxID][k] = true
}
}
// We train before updating, training may set m.TrainedJunk.
if jf != nil && m.NeedsTraining() {
openTrainMessage(&m)
}
err = tx.Update(&m)
ximportcheckf(err, "updating message after flag update")
changes = append(changes, store.ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, Mask: flags, Flags: flags})
changes = append(changes, store.ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, Mask: flags, Flags: flags, Keywords: m.Keywords})
}
delete(mailboxMissingKeywordMessages, mailbox)
} else {
@ -744,6 +776,19 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
sendEvent("count", importCount{prevMailbox, messages[prevMailbox]})
}
// Update mailboxes with keywords.
for mbID, keywords := range destMailboxKeywords {
mb := store.Mailbox{ID: mbID}
err := tx.Get(&mb)
ximportcheckf(err, "loading mailbox for updating keywords")
var changed bool
mb.Keywords, changed = store.MergeKeywords(mb.Keywords, maps.Keys(keywords))
if changed {
err = tx.Update(&mb)
ximportcheckf(err, "updating mailbox with keywords")
}
}
err = tx.Commit()
tx = nil
ximportcheckf(err, "commit")
@ -768,9 +813,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
sendEvent("done", importDone{})
}
func flagSet(flags *store.Flags, word string) {
// todo: custom labels, e.g. $label1, JunkRecorded?
func flagSet(flags *store.Flags, keywords map[string]bool, word string) {
switch word {
case "forwarded", "$forwarded":
flags.Forwarded = true
@ -782,5 +825,9 @@ func flagSet(flags *store.Flags, word string) {
flags.Phishing = true
case "mdnsent", "$mdnsent":
flags.MDNSent = true
default:
if store.ValidLowercaseKeyword(word) {
keywords[word] = true
}
}
}