update to latest bstore, with a bugfix for queries with multiple orders that were partially handled by an index

causing returned order to be incorrect.
was triggered by new code i'm working on.

vendor/github.com/mjl-/bstore/plan.go

@@ -8,7 +8,6 @@ import (
 )
 
 // todo: cache query plans? perhaps explicitly through something like a prepared statement. the current plan includes values in keys,start,stop, which would need to be calculated for each execution. should benchmark time spent in planning first.
-// todo optimize: handle multiple sorts with multikey indices if they match
 // todo optimize: combine multiple filter (not)in/equals calls for same field
 // todo optimize: efficiently pack booleans in an index (eg for Message.Flags), and use it to query.
 // todo optimize: do multiple range scans if necessary when we can use an index for an equal check with multiple values.
@@ -31,10 +30,15 @@ type plan[T any] struct {
 	// index. Required non-nil for unique.
 	keys [][]byte
 
-	desc           bool   // Direction of the range scan.
-	start          []byte // First key to scan. Filters below may still apply. If desc, this value is > than stop (if it is set). If nil, we begin ranging at the first or last (for desc) key.
-	stop           []byte // Last key to scan. Can be nil independently of start.
-	startInclusive bool   // If the start and stop values are inclusive or exclusive.
+	desc bool // Direction of the range scan.
+	// First key to scan. Filters below may still apply. If desc, this value is > than
+	// stop (if it is set). If nil, we begin ranging at the first or last (for desc)
+	// key.
+	start []byte
+	// Last key to scan. Can be nil independently of start.
+	stop []byte
+	// If the start and stop values are inclusive or exclusive.
+	startInclusive bool
 	stopInclusive  bool
 
 	// Filter we need to apply after retrieving the record. If all original filters
@@ -42,9 +46,13 @@ type plan[T any] struct {
 	// empty.
 	filters []filter[T]
 
-	// Orders we need to apply after first retrieving all records. As with
-	// filters, if a range scan takes care of an ordering from the query,
-	// this field is empty.
+	// Number of fields from index used to group results before applying in-memory
+	// ordering with "orders" below.
+	norderidxuse int
+
+	// Orders we need to apply after first retrieving all records with equal values for
+	// first norderidxuse fields. As with filters, if a range scan takes care of all
+	// orderings from the query, this field is empty.
 	orders []order
 }
 
@@ -177,7 +185,7 @@ indices:
 	var p *plan[T]
 	var nexact int
 	var nrange int
-	var ordered bool
+	var norder int
 
 	evaluatePKOrIndex := func(idx *index) error {
 		var isPK bool
@@ -205,25 +213,40 @@ indices:
 		}
 
 		var nex = 0
-		// log.Printf("idx %v", idx)
+		// log.Printf("evaluating idx %#v", idx)
 		var skipFilters []*filter[T]
 		for _, f := range idx.Fields {
 			if equals[f.Name] != nil && f.Type.Kind != kindSlice {
 				skipFilters = append(skipFilters, equals[f.Name])
-				nex++
 			} else if inslices[f.Name] != nil && f.Type.Kind == kindSlice {
 				skipFilters = append(skipFilters, inslices[f.Name])
-				nex++
 			} else {
 				break
 			}
+			nex++
 		}
 
-		// See if the next field can be used for compare.
-		var gx, lx *filterCompare[T]
-		var nrng int
-		var order *order
+		// For ordering, skip leading filters we already match on exactly.
 		orders := q.xorders
+		trim := 0
+	TrimOrders:
+		for _, o := range orders {
+			for _, f := range idx.Fields[:nex] {
+				if o.field.Name == f.Name {
+					trim++
+					continue TrimOrders
+				}
+			}
+			break
+		}
+		orders = orders[trim:]
+
+		// Fields from the index that we use for grouping before in-memory sorting.
+		var norderidxuse int
+
+		// See if the next index field can be used for compare and ordering.
+		var gx, lx *filterCompare[T]
+		var nrng int // for nrange
 		if nex < len(idx.Fields) {
 			nf := idx.Fields[nex]
 			for i := range q.xfilters {
@@ -250,22 +273,45 @@ indices:
 				}
 			}
 
-			// See if it can be used for ordering.
-			// todo optimize: we could use multiple orders
-			if len(orders) > 0 && orders[0].field.Name == nf.Name {
-				order = &orders[0]
-				orders = orders[1:]
+			// We can use multiple orderings as long as the asc/desc direction stays the same.
+			nord := 0
+			for i, o := range orders {
+				if nex+i < len(idx.Fields) && o.field.Name == idx.Fields[nex+i].Name && (nord == 0 || o.asc == orders[0].asc) {
+					nord++
+					continue
+				}
+				break
 			}
+			norderidxuse = nex + nord
+			prevorders := orders
+			orders = orders[nord:]
+
+			// The stored index key ends with the primary key, so if we're there, and the next
+			// ordering key is the primary key, we use the index for it too.
+			if norderidxuse == len(idx.Fields) && len(orders) > 0 && orders[0].field.Name == q.st.Current.Fields[0].Name && (nord == 0 || orders[0].asc == prevorders[nord-1].asc) {
+				orders = orders[1:]
+				norderidxuse++
+			}
+		} else if len(orders) > 0 && orders[0].field.Name == q.st.Current.Fields[0].Name {
+			// We only had equals filters that used all of the index, but we're also sorting by
+			// the primary key, so use the index for that too.
+			orders = orders[1:]
+			norderidxuse++
 		}
 
+		// Orders handled by the index, excluding exact match filters.
+		idxorders := q.xorders[trim : len(q.xorders)-len(orders)]
+
+		// log.Printf("index fields to match for index order: %d, orders for index %d, in-memory ordering %d, total orders %d", norderidxuse, len(idxorders), len(orders), len(q.xorders))
+
 		// See if this is better than what we had.
-		if !(nex > nexact || (nex == nexact && (nrng > nrange || order != nil && !ordered && (q.xlimit > 0 || nrng == nrange)))) {
-			// log.Printf("plan not better, nex %d, nrng %d, limit %d, order %v ordered %v", nex, nrng, q.limit, order, ordered)
+		if !(nex > nexact || (nex == nexact && (nrng > nrange || len(idxorders) > norder && (q.xlimit > 0 || nrng == nrange)))) {
+			// log.Printf("plan not better, nex %d, nrng %d, limit %d, nidxorders %v ordered %v", nex, nrng, q.xlimit, len(idxorders), norder)
 			return nil
 		}
 		nexact = nex
 		nrange = nrng
-		ordered = order != nil
+		norder = len(idxorders)
 
 		// Calculate the prefix key.
 		var kvalues []reflect.Value
@@ -307,7 +353,8 @@ indices:
 
 		startInclusive := gx == nil || gx.op != opGreater
 		stopInclusive := lx == nil || lx.op != opLess
-		if order != nil && !order.asc {
+		desc := len(idxorders) > 0 && !idxorders[0].asc
+		if desc {
 			start, stop = stop, start
 			startInclusive, stopInclusive = stopInclusive, startInclusive
 		}
@@ -318,12 +365,13 @@ indices:
 
 		p = &plan[T]{
 			idx:            idx,
-			desc:           order != nil && !order.asc,
+			desc:           desc,
 			start:          start,
 			stop:           stop,
 			startInclusive: startInclusive,
 			stopInclusive:  stopInclusive,
 			filters:        dropFilters(q.xfilters, skipFilters),
+			norderidxuse:   norderidxuse,
 			orders:         orders,
 		}
 		return nil
@@ -341,6 +389,7 @@ indices:
 
 	}
 	if p != nil {
+		// log.Printf("using index plan %v", p)
 		return p, nil
 	}