2c0d61b18e
Currently ListV2 loaded the whole data into memory, even when all the data wasn't being used, using up more memory than needed. Change-Id: I5846d979344729b447c108a6cc9f4227229ec981
119 lines
3.0 KiB
Go
119 lines
3.0 KiB
Go
// Copyright (C) 2019 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package storage
|
|
|
|
import (
|
|
"context"
|
|
)
|
|
|
|
// ListOptions are items that are optional for the LIST method.
|
|
type ListOptions struct {
|
|
Prefix Key
|
|
StartAfter Key // StartAfter is relative to Prefix
|
|
Recursive bool
|
|
IncludeValue bool
|
|
Limit int
|
|
}
|
|
|
|
// ListV2 lists all keys corresponding to ListOptions.
|
|
// limit is capped to LookupLimit.
|
|
//
|
|
// more indicates if the result was truncated. If false
|
|
// then the result []ListItem includes all requested keys.
|
|
// If true then the caller must call List again to get more
|
|
// results by setting `StartAfter` appropriately.
|
|
func ListV2(ctx context.Context, store KeyValueStore, opts ListOptions) (result Items, more bool, err error) {
|
|
more, err = ListV2Iterate(ctx, store, opts, func(ctx context.Context, item *ListItem) error {
|
|
if opts.IncludeValue {
|
|
result = append(result, ListItem{
|
|
Key: CloneKey(item.Key),
|
|
Value: CloneValue(item.Value),
|
|
IsPrefix: item.IsPrefix,
|
|
})
|
|
} else {
|
|
result = append(result, ListItem{
|
|
Key: CloneKey(item.Key),
|
|
IsPrefix: item.IsPrefix,
|
|
})
|
|
}
|
|
return nil
|
|
})
|
|
return result, more, err
|
|
}
|
|
|
|
// ListV2Iterate lists all keys corresponding to ListOptions.
|
|
// limit is capped to LookupLimit.
|
|
//
|
|
// more indicates if the result was truncated. If false
|
|
// then the result []ListItem includes all requested keys.
|
|
// If true then the caller must call List again to get more
|
|
// results by setting `StartAfter` appropriately.
|
|
//
|
|
// The opts.IncludeValue is ignored for this func.
|
|
// The callback item will be reused for next calls.
|
|
// If the user needs the preserve the value, it must call storage.CloneValue or storage.CloneKey.
|
|
func ListV2Iterate(ctx context.Context, store KeyValueStore, opts ListOptions, fn func(context.Context, *ListItem) error) (more bool, err error) {
|
|
defer mon.Task()(&ctx)(&err)
|
|
|
|
limit := opts.Limit
|
|
if limit <= 0 || limit > store.LookupLimit() {
|
|
limit = store.LookupLimit()
|
|
}
|
|
|
|
more = true
|
|
|
|
first := opts.StartAfter
|
|
iterate := func(ctx context.Context, it Iterator) error {
|
|
var item ListItem
|
|
skipFirst := true
|
|
for ; limit > 0; limit-- {
|
|
if !it.Next(ctx, &item) {
|
|
more = false
|
|
return nil
|
|
}
|
|
|
|
relativeKey := item.Key[len(opts.Prefix):]
|
|
if skipFirst {
|
|
skipFirst = false
|
|
if relativeKey.Equal(first) {
|
|
// skip the first element in iteration
|
|
// if it matches the search key
|
|
limit++
|
|
continue
|
|
}
|
|
}
|
|
|
|
task := mon.TaskNamed("handling_item")(nil)
|
|
item.Key = relativeKey
|
|
err := fn(ctx, &item)
|
|
task(nil)
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// we still need to consume one item for the more flag
|
|
more = it.Next(ctx, &item)
|
|
return nil
|
|
}
|
|
|
|
var firstFull Key
|
|
if !opts.StartAfter.IsZero() {
|
|
firstFull = joinKey(opts.Prefix, opts.StartAfter)
|
|
}
|
|
err = store.Iterate(ctx, IterateOptions{
|
|
Prefix: opts.Prefix,
|
|
First: firstFull,
|
|
Recurse: opts.Recursive,
|
|
Limit: limit,
|
|
}, iterate)
|
|
|
|
return more, err
|
|
}
|
|
|
|
func joinKey(a, b Key) Key {
|
|
return append(append(Key{}, a...), b...)
|
|
}
|