certificate: improve gob migration
`storage.KeyValueStore` requires ordered iteration, which redis doesn't support natively. This would require loading all the keys into memory and then processing them, rather than iterating over them one-by-one. This adds a temporary `IterateUnordered` to handle the migrations more gracefully. Change-Id: I55b763500523077c7ab8fdfad175c32cc7788e47
This commit is contained in:
parent
fa26ae85e9
commit
b8c7dcbf7b
@ -300,9 +300,7 @@ func (authDB *DB) put(ctx context.Context, userID string, auths Group) (err erro
|
||||
// MigrateGob migrates gob encoded Group to protobuf encoded Group.
|
||||
func (authDB *DB) MigrateGob(ctx context.Context, progress func(userID string)) (err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
err = authDB.db.Iterate(ctx, storage.IterateOptions{
|
||||
Recurse: true,
|
||||
}, func(ctx context.Context, it storage.Iterator) error {
|
||||
err = authDB.db.IterateUnordered(ctx, func(ctx context.Context, it storage.Iterator) error {
|
||||
var item storage.ListItem
|
||||
|
||||
for it.Next(ctx, &item) {
|
||||
|
@ -35,7 +35,7 @@ func cmdMigrate(cmd *cobra.Command, args []string) error {
|
||||
count := 0
|
||||
return authorizationDB.MigrateGob(ctx, func(userID string) {
|
||||
if count%100 == 0 {
|
||||
log.Info("progress", zap.String("user", userID), zap.Int("count", count))
|
||||
log.Info("progress", zap.String("last", userID), zap.Int("total-processed-count", count))
|
||||
}
|
||||
count++
|
||||
})
|
||||
|
@ -230,6 +230,13 @@ func (client *Client) Iterate(ctx context.Context, opts storage.IterateOptions,
|
||||
return client.IterateWithoutLookupLimit(ctx, opts, fn)
|
||||
}
|
||||
|
||||
// IterateUnordered iterates over all data, however, does not guarantee ordering.
|
||||
// It only guarantees all items are iterated at least once.
|
||||
func (client *Client) IterateUnordered(ctx context.Context, fn func(context.Context, storage.Iterator) error) (err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
return client.Iterate(ctx, storage.IterateOptions{}, fn)
|
||||
}
|
||||
|
||||
// IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
|
||||
func (client *Client) IterateWithoutLookupLimit(ctx context.Context, opts storage.IterateOptions, fn func(context.Context, storage.Iterator) error) (err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
|
@ -72,6 +72,9 @@ type KeyValueStore interface {
|
||||
List(ctx context.Context, start Key, limit int) (Keys, error)
|
||||
// Iterate iterates over items based on opts.
|
||||
Iterate(ctx context.Context, opts IterateOptions, fn func(context.Context, Iterator) error) error
|
||||
// IterateUnordered iterates over all data, however, does not guarantee ordering.
|
||||
// It only guarantees all items are iterated at least once.
|
||||
IterateUnordered(ctx context.Context, fn func(context.Context, Iterator) error) error
|
||||
// IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
|
||||
IterateWithoutLookupLimit(ctx context.Context, opts IterateOptions, fn func(context.Context, Iterator) error) error
|
||||
// CompareAndSwap atomically compares and swaps oldValue with newValue.
|
||||
|
@ -193,6 +193,17 @@ func (client *Client) Iterate(ctx context.Context, opts storage.IterateOptions,
|
||||
return client.IterateWithoutLookupLimit(ctx, opts, fn)
|
||||
}
|
||||
|
||||
// IterateUnordered iterates over all data, however, does not guarantee ordering.
|
||||
// It only guarantees all items are iterated at least once.
|
||||
func (client *Client) IterateUnordered(ctx context.Context, fn func(context.Context, storage.Iterator) error) (err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
|
||||
return fn(ctx, &ScanIterator{
|
||||
db: client.db,
|
||||
it: client.db.Scan(ctx, 0, "*", 0).Iterator(),
|
||||
})
|
||||
}
|
||||
|
||||
// IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
|
||||
func (client *Client) IterateWithoutLookupLimit(ctx context.Context, opts storage.IterateOptions, fn func(context.Context, storage.Iterator) error) (err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
|
@ -8,6 +8,8 @@ import (
|
||||
"context"
|
||||
"sort"
|
||||
|
||||
"github.com/go-redis/redis/v8"
|
||||
|
||||
"storj.io/storj/storage"
|
||||
)
|
||||
|
||||
@ -75,3 +77,29 @@ func (it *StaticIterator) Next(ctx context.Context, item *storage.ListItem) bool
|
||||
it.Index++
|
||||
return true
|
||||
}
|
||||
|
||||
// ScanIterator iterates over scan command items.
|
||||
type ScanIterator struct {
|
||||
db *redis.Client
|
||||
it *redis.ScanIterator
|
||||
}
|
||||
|
||||
// Next returns the next item from the iterator.
|
||||
func (it *ScanIterator) Next(ctx context.Context, item *storage.ListItem) bool {
|
||||
ok := it.it.Next(ctx)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
key := it.it.Val()
|
||||
value, err := it.db.Get(ctx, key).Bytes()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
item.Key = storage.Key(key)
|
||||
item.Value = storage.Value(value)
|
||||
item.IsPrefix = false
|
||||
|
||||
return true
|
||||
}
|
||||
|
@ -100,6 +100,26 @@ func (store *Logger) Iterate(ctx context.Context, opts storage.IterateOptions, f
|
||||
})
|
||||
}
|
||||
|
||||
// IterateUnordered iterates over all data, however, does not guarantee ordering.
|
||||
// It only guarantees all items are iterated at least once.
|
||||
func (store *Logger) IterateUnordered(ctx context.Context, fn func(context.Context, storage.Iterator) error) (err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
store.log.Debug("IterateUnordered")
|
||||
return store.store.IterateUnordered(ctx, func(ctx context.Context, it storage.Iterator) error {
|
||||
return fn(ctx, storage.IteratorFunc(func(ctx context.Context, item *storage.ListItem) bool {
|
||||
ok := it.Next(ctx, item)
|
||||
if ok {
|
||||
store.log.Debug(" ",
|
||||
zap.ByteString("key", item.Key),
|
||||
zap.Int("value length", len(item.Value)),
|
||||
zap.Binary("truncated value", truncate(item.Value)),
|
||||
)
|
||||
}
|
||||
return ok
|
||||
}))
|
||||
})
|
||||
}
|
||||
|
||||
// IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
|
||||
func (store *Logger) IterateWithoutLookupLimit(ctx context.Context, opts storage.IterateOptions, fn func(context.Context, storage.Iterator) error) (err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
|
@ -237,6 +237,13 @@ func (store *Client) Iterate(ctx context.Context, opts storage.IterateOptions, f
|
||||
return store.IterateWithoutLookupLimit(ctx, opts, fn)
|
||||
}
|
||||
|
||||
// IterateUnordered iterates over all data, however, does not guarantee ordering.
|
||||
// It only guarantees all items are iterated at least once.
|
||||
func (store *Client) IterateUnordered(ctx context.Context, fn func(context.Context, storage.Iterator) error) (err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
return store.IterateWithoutLookupLimit(ctx, storage.IterateOptions{}, fn)
|
||||
}
|
||||
|
||||
// IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
|
||||
func (store *Client) IterateWithoutLookupLimit(ctx context.Context, opts storage.IterateOptions, fn func(context.Context, storage.Iterator) error) (err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
|
Loading…
Reference in New Issue
Block a user