add debug logging about bstore db schema upgrades

bstore was updated to v0.0.6 to add this logging.
this simplifies some of the db-handling code in mtastsdb,tlsrptdb,dmarcdb. we
now call the package-level Init() and Close() in all tests properly.
This commit is contained in:
Mechiel Lukkien
2024-05-10 14:44:37 +02:00
parent 3e4cce826e
commit bf8cfd9724
31 changed files with 298 additions and 428 deletions

38
vendor/github.com/mjl-/bstore/doc.go generated vendored
View File

@ -216,24 +216,24 @@ stays trivial (though not plan9, unfortunately). Although bstore is much more
limited in so many aspects than sqlite, bstore also offers some advantages as
well. Some points of comparison:
- Cross-compilation and reproducibility: Trivial with bstore due to pure Go,
much harder with sqlite because of cgo.
- Code complexity: low with bstore (7k lines including comments/docs), high
with sqlite.
- Query language: mostly-type-checked function calls in bstore, free-form query
strings only checked at runtime with sqlite.
- Functionality: very limited with bstore, much more full-featured with sqlite.
- Schema management: mostly automatic based on Go type definitions in bstore,
manual with ALTER statements in sqlite.
- Types and packing/parsing: automatic/transparent in bstore based on Go types
(including maps, slices, structs and custom MarshalBinary encoding), versus
manual scanning and parameter passing with sqlite with limited set of SQL
types.
- Performance: low to good performance with bstore, high performance with
sqlite.
- Database files: single file with bstore, several files with sqlite (due to
WAL or journal files).
- Test coverage: decent coverage but limited real-world for bstore, versus
extremely thoroughly tested and with enormous real-world use.
- Cross-compilation and reproducibility: Trivial with bstore due to pure Go,
much harder with sqlite because of cgo.
- Code complexity: low with bstore (7k lines including comments/docs), high
with sqlite.
- Query language: mostly-type-checked function calls in bstore, free-form query
strings only checked at runtime with sqlite.
- Functionality: very limited with bstore, much more full-featured with sqlite.
- Schema management: mostly automatic based on Go type definitions in bstore,
manual with ALTER statements in sqlite.
- Types and packing/parsing: automatic/transparent in bstore based on Go types
(including maps, slices, structs and custom MarshalBinary encoding), versus
manual scanning and parameter passing with sqlite with limited set of SQL
types.
- Performance: low to good performance with bstore, high performance with
sqlite.
- Database files: single file with bstore, several files with sqlite (due to
WAL or journal files).
- Test coverage: decent coverage but limited real-world for bstore, versus
extremely thoroughly tested and with enormous real-world use.
*/
package bstore

View File

@ -7,6 +7,7 @@ import (
"encoding/json"
"errors"
"fmt"
"log/slog"
"os"
"reflect"
"sort"
@ -50,6 +51,17 @@ var errSchemaCheck = errors.New("schema check")
// to "changed", an error is returned if there is no schema change. If it is set to
// "unchanged", an error is returned if there was a schema change.
func (db *DB) Register(ctx context.Context, typeValues ...any) error {
return db.register(ctx, slog.New(discardHandler{}), typeValues...)
}
type discardHandler struct{}
func (l discardHandler) Enabled(context.Context, slog.Level) bool { return false }
func (l discardHandler) Handle(context.Context, slog.Record) error { return nil }
func (l discardHandler) WithAttrs(attrs []slog.Attr) slog.Handler { return l }
func (l discardHandler) WithGroup(name string) slog.Handler { return l }
func (db *DB) register(ctx context.Context, log *slog.Logger, typeValues ...any) error {
// We will drop/create new indices as needed. For changed indices, we drop
// and recreate. E.g. if an index becomes a unique index, or if a field in
// an index changes. These values map type and index name to their index.
@ -148,6 +160,9 @@ func (db *DB) Register(ctx context.Context, typeValues ...any) error {
tv.Version = 1
if st.Current != nil {
tv.Version = st.Current.Version + 1
log.Debug("updating schema for type", slog.String("type", tv.name), slog.Uint64("version", uint64(tv.Version)))
} else {
log.Debug("adding schema for new type", slog.String("type", tv.name), slog.Uint64("version", uint64(tv.Version)))
}
k, v, err := packSchema(tv)
if err != nil {
@ -299,6 +314,7 @@ func (db *DB) Register(ctx context.Context, typeValues ...any) error {
}
foundField = true
log.Debug("verifying foreign key constraint for new reference", slog.String("fromtype", ntname), slog.String("fromfield", f.Name), slog.String("totype", name))
// For newly added references, check they are valid.
b, err := tx.recordsBucket(ntname, ntv.fillPercent)
@ -396,6 +412,7 @@ func (db *DB) Register(ctx context.Context, typeValues ...any) error {
db.types[st.Type] = *st
db.typeNames[st.Name] = *st
ntvp = &ntv
log.Debug("updating schema for type due to new incoming reference", slog.String("type", name), slog.Uint64("version", uint64(ntv.Version)))
}
k, v, err := packSchema(ntvp)
@ -453,6 +470,7 @@ func (db *DB) Register(ctx context.Context, typeValues ...any) error {
if !drop {
continue
}
log.Debug("dropping old/modified index", slog.String("type", name), slog.String("indexname", iname))
b, err := tx.typeBucket(name)
if err != nil {
return err
@ -482,6 +500,7 @@ func (db *DB) Register(ctx context.Context, typeValues ...any) error {
if !create {
continue
}
log.Debug("preparing for new/modified index for type", slog.String("type", name), slog.String("indexname", iname))
b, err := tx.typeBucket(name)
if err != nil {
return err
@ -578,6 +597,8 @@ func (db *DB) Register(ctx context.Context, typeValues ...any) error {
return nil
}
log.Debug("creating new/modified indexes for type", slog.String("type", name))
// Now do all sorts + inserts.
for i, ib := range ibs {
idx := idxs[i]

View File

@ -8,6 +8,7 @@ import (
"fmt"
"io"
"io/fs"
"log/slog"
"os"
"reflect"
"sync"
@ -282,9 +283,10 @@ type fieldType struct {
// Options configure how a database should be opened or initialized.
type Options struct {
Timeout time.Duration // Abort if opening DB takes longer than Timeout. If not set, the deadline from the context is used.
Perm fs.FileMode // Permissions for new file if created. If zero, 0600 is used.
MustExist bool // Before opening, check that file exists. If not, io/fs.ErrNotExist is returned.
Timeout time.Duration // Abort if opening DB takes longer than Timeout. If not set, the deadline from the context is used.
Perm fs.FileMode // Permissions for new file if created. If zero, 0600 is used.
MustExist bool // Before opening, check that file exists. If not, io/fs.ErrNotExist is returned.
RegisterLogger *slog.Logger // For debug logging about schema upgrades.
}
// Open opens a bstore database and registers types by calling Register.
@ -326,7 +328,16 @@ func Open(ctx context.Context, path string, opts *Options, typeValues ...any) (*
typeNames := map[string]storeType{}
types := map[reflect.Type]storeType{}
db := &DB{bdb: bdb, typeNames: typeNames, types: types}
if err := db.Register(ctx, typeValues...); err != nil {
var log *slog.Logger
if opts != nil {
log = opts.RegisterLogger
}
if log == nil {
log = slog.New(discardHandler{})
} else {
log = log.With("dbpath", path)
}
if err := db.register(ctx, log, typeValues...); err != nil {
bdb.Close()
return nil, err
}