storj/storage/listv2.go
Egon Elbre 76fdb5d863 storage: add configurable lookup limits
Currently storage tests were tied to the default lookup limit.
By increasing the limits, the tests will take longer and sometimes
cause a large number of goroutines to be started.

This change adds configurable lookup limit to all storage backends.

Also remove boltdb.NewShared, since it's not used any more.

Change-Id: I1a052f149da471246fac5745da133c3cfc27582e
2020-01-22 21:35:56 +02:00

93 lines
2.1 KiB
Go

// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package storage
import (
"context"
)
// ListOptions are items that are optional for the LIST method
type ListOptions struct {
Prefix Key
StartAfter Key // StartAfter is relative to Prefix
Recursive bool
IncludeValue bool
Limit int
}
// ListV2 lists all keys corresponding to ListOptions
// limit is capped to LookupLimit
//
// more indicates if the result was truncated. If false
// then the result []ListItem includes all requested keys.
// If true then the caller must call List again to get more
// results by setting `StartAfter` appropriately.
func ListV2(ctx context.Context, store KeyValueStore, opts ListOptions) (result Items, more bool, err error) {
defer mon.Task()(&ctx)(&err)
limit := opts.Limit
if limit <= 0 || limit > store.LookupLimit() {
limit = store.LookupLimit()
}
more = true
first := opts.StartAfter
iterate := func(ctx context.Context, it Iterator) error {
var item ListItem
skipFirst := true
for ; limit > 0; limit-- {
if !it.Next(ctx, &item) {
more = false
return nil
}
relativeKey := item.Key[len(opts.Prefix):]
if skipFirst {
skipFirst = false
if relativeKey.Equal(first) {
// skip the first element in iteration
// if it matches the search key
limit++
continue
}
}
if opts.IncludeValue {
result = append(result, ListItem{
Key: CloneKey(relativeKey),
Value: CloneValue(item.Value),
IsPrefix: item.IsPrefix,
})
} else {
result = append(result, ListItem{
Key: CloneKey(relativeKey),
IsPrefix: item.IsPrefix,
})
}
}
// we still need to consume one item for the more flag
more = it.Next(ctx, &item)
return nil
}
var firstFull Key
if !opts.StartAfter.IsZero() {
firstFull = joinKey(opts.Prefix, opts.StartAfter)
}
err = store.Iterate(ctx, IterateOptions{
Prefix: opts.Prefix,
First: firstFull,
Recurse: opts.Recursive,
Limit: limit,
}, iterate)
return result, more, err
}
func joinKey(a, b Key) Key {
return append(append(Key{}, a...), b...)
}