certificate: improve gob migration

`storage.KeyValueStore` requires ordered iteration, which redis
doesn't support natively. This would require loading all the keys
into memory and then processing them, rather than iterating over them
one-by-one.

This adds a temporary `IterateUnordered` to handle the migrations
more gracefully.

Change-Id: I55b763500523077c7ab8fdfad175c32cc7788e47

certificate/authorization/db.go

@@ -300,9 +300,7 @@ func (authDB *DB) put(ctx context.Context, userID string, auths Group) (err erro
 // MigrateGob migrates gob encoded Group to protobuf encoded Group.
 func (authDB *DB) MigrateGob(ctx context.Context, progress func(userID string)) (err error) {
 	defer mon.Task()(&ctx)(&err)
-	err = authDB.db.Iterate(ctx, storage.IterateOptions{
+	err = authDB.db.IterateUnordered(ctx, func(ctx context.Context, it storage.Iterator) error {
-		Recurse: true,
-	}, func(ctx context.Context, it storage.Iterator) error {
 		var item storage.ListItem
 
 		for it.Next(ctx, &item) {

cmd/certificates/migrate.go

@@ -35,7 +35,7 @@ func cmdMigrate(cmd *cobra.Command, args []string) error {
 	count := 0
 	return authorizationDB.MigrateGob(ctx, func(userID string) {
 		if count%100 == 0 {
-			log.Info("progress", zap.String("user", userID), zap.Int("count", count))
+			log.Info("progress", zap.String("last", userID), zap.Int("total-processed-count", count))
 		}
 		count++
 	})

storage/boltdb/client.go

@@ -230,6 +230,13 @@ func (client *Client) Iterate(ctx context.Context, opts storage.IterateOptions,
 	return client.IterateWithoutLookupLimit(ctx, opts, fn)
 }
 
+// IterateUnordered iterates over all data, however, does not guarantee ordering.
+// It only guarantees all items are iterated at least once.
+func (client *Client) IterateUnordered(ctx context.Context, fn func(context.Context, storage.Iterator) error) (err error) {
+	defer mon.Task()(&ctx)(&err)
+	return client.Iterate(ctx, storage.IterateOptions{}, fn)
+}
+
 // IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
 func (client *Client) IterateWithoutLookupLimit(ctx context.Context, opts storage.IterateOptions, fn func(context.Context, storage.Iterator) error) (err error) {
 	defer mon.Task()(&ctx)(&err)

storage/common.go

@@ -72,6 +72,9 @@ type KeyValueStore interface {
 	List(ctx context.Context, start Key, limit int) (Keys, error)
 	// Iterate iterates over items based on opts.
 	Iterate(ctx context.Context, opts IterateOptions, fn func(context.Context, Iterator) error) error
+	// IterateUnordered iterates over all data, however, does not guarantee ordering.
+	// It only guarantees all items are iterated at least once.
+	IterateUnordered(ctx context.Context, fn func(context.Context, Iterator) error) error
 	// IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
 	IterateWithoutLookupLimit(ctx context.Context, opts IterateOptions, fn func(context.Context, Iterator) error) error
 	// CompareAndSwap atomically compares and swaps oldValue with newValue.

storage/redis/client.go

@@ -193,6 +193,17 @@ func (client *Client) Iterate(ctx context.Context, opts storage.IterateOptions,
 	return client.IterateWithoutLookupLimit(ctx, opts, fn)
 }
 
+// IterateUnordered iterates over all data, however, does not guarantee ordering.
+// It only guarantees all items are iterated at least once.
+func (client *Client) IterateUnordered(ctx context.Context, fn func(context.Context, storage.Iterator) error) (err error) {
+	defer mon.Task()(&ctx)(&err)
+
+	return fn(ctx, &ScanIterator{
+		db: client.db,
+		it: client.db.Scan(ctx, 0, "*", 0).Iterator(),
+	})
+}
+
 // IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
 func (client *Client) IterateWithoutLookupLimit(ctx context.Context, opts storage.IterateOptions, fn func(context.Context, storage.Iterator) error) (err error) {
 	defer mon.Task()(&ctx)(&err)

storage/redis/util.go

@@ -8,6 +8,8 @@ import (
 	"context"
 	"sort"
 
+	"github.com/go-redis/redis/v8"
+
 	"storj.io/storj/storage"
 )
 
@@ -75,3 +77,29 @@ func (it *StaticIterator) Next(ctx context.Context, item *storage.ListItem) bool
 	it.Index++
 	return true
 }
+
+// ScanIterator iterates over scan command items.
+type ScanIterator struct {
+	db *redis.Client
+	it *redis.ScanIterator
+}
+
+// Next returns the next item from the iterator.
+func (it *ScanIterator) Next(ctx context.Context, item *storage.ListItem) bool {
+	ok := it.it.Next(ctx)
+	if !ok {
+		return false
+	}
+
+	key := it.it.Val()
+	value, err := it.db.Get(ctx, key).Bytes()
+	if err != nil {
+		return false
+	}
+
+	item.Key = storage.Key(key)
+	item.Value = storage.Value(value)
+	item.IsPrefix = false
+
+	return true
+}

storage/storelogger/logger.go

@@ -100,6 +100,26 @@ func (store *Logger) Iterate(ctx context.Context, opts storage.IterateOptions, f
 	})
 }
 
+// IterateUnordered iterates over all data, however, does not guarantee ordering.
+// It only guarantees all items are iterated at least once.
+func (store *Logger) IterateUnordered(ctx context.Context, fn func(context.Context, storage.Iterator) error) (err error) {
+	defer mon.Task()(&ctx)(&err)
+	store.log.Debug("IterateUnordered")
+	return store.store.IterateUnordered(ctx, func(ctx context.Context, it storage.Iterator) error {
+		return fn(ctx, storage.IteratorFunc(func(ctx context.Context, item *storage.ListItem) bool {
+			ok := it.Next(ctx, item)
+			if ok {
+				store.log.Debug("  ",
+					zap.ByteString("key", item.Key),
+					zap.Int("value length", len(item.Value)),
+					zap.Binary("truncated value", truncate(item.Value)),
+				)
+			}
+			return ok
+		}))
+	})
+}
+
 // IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
 func (store *Logger) IterateWithoutLookupLimit(ctx context.Context, opts storage.IterateOptions, fn func(context.Context, storage.Iterator) error) (err error) {
 	defer mon.Task()(&ctx)(&err)

storage/teststore/store.go

@@ -237,6 +237,13 @@ func (store *Client) Iterate(ctx context.Context, opts storage.IterateOptions, f
 	return store.IterateWithoutLookupLimit(ctx, opts, fn)
 }
 
+// IterateUnordered iterates over all data, however, does not guarantee ordering.
+// It only guarantees all items are iterated at least once.
+func (store *Client) IterateUnordered(ctx context.Context, fn func(context.Context, storage.Iterator) error) (err error) {
+	defer mon.Task()(&ctx)(&err)
+	return store.IterateWithoutLookupLimit(ctx, storage.IterateOptions{}, fn)
+}
+
 // IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
 func (store *Client) IterateWithoutLookupLimit(ctx context.Context, opts storage.IterateOptions, fn func(context.Context, storage.Iterator) error) (err error) {
 	defer mon.Task()(&ctx)(&err)