storage: delete unused code and lower visibility of static iterator
Change-Id: I8ec6ec9a710650611d272b03b2927759a8b02f91
This commit is contained in:
Egon Elbre
parent
948589d38b
commit
8bef560ab9
@ -12,7 +12,7 @@ import (
|
||||
"github.com/zeebo/errs"
|
||||
)
|
||||
|
||||
// ErrInvalidBlobRef is returned when an blob reference is invalid
|
||||
// ErrInvalidBlobRef is returned when an blob reference is invalid.
|
||||
var ErrInvalidBlobRef = errs.Class("invalid blob ref")
|
||||
|
||||
// FormatVersion represents differing storage format version values. Different Blobs implementors
|
||||
@ -31,7 +31,7 @@ type BlobRef struct {
|
||||
Key []byte
|
||||
}
|
||||
|
||||
// IsValid returns whether both namespace and key are specified
|
||||
// IsValid returns whether both namespace and key are specified.
|
||||
func (ref *BlobRef) IsValid() bool {
|
||||
return len(ref.Namespace) > 0 && len(ref.Key) > 0
|
||||
}
|
||||
@ -42,7 +42,7 @@ type BlobReader interface {
|
||||
io.ReaderAt
|
||||
io.Seeker
|
||||
io.Closer
|
||||
// Size returns the size of the blob
|
||||
// Size returns the size of the blob.
|
||||
Size() (int64, error)
|
||||
// StorageFormatVersion returns the storage format version associated with the blob.
|
||||
StorageFormatVersion() FormatVersion
|
||||
@ -67,37 +67,37 @@ type BlobWriter interface {
|
||||
//
|
||||
// architecture: Database
|
||||
type Blobs interface {
|
||||
// Create creates a new blob that can be written
|
||||
// optionally takes a size argument for performance improvements, -1 is unknown size
|
||||
// Create creates a new blob that can be written.
|
||||
// Optionally takes a size argument for performance improvements, -1 is unknown size.
|
||||
Create(ctx context.Context, ref BlobRef, size int64) (BlobWriter, error)
|
||||
// Open opens a reader with the specified namespace and key
|
||||
// Open opens a reader with the specified namespace and key.
|
||||
Open(ctx context.Context, ref BlobRef) (BlobReader, error)
|
||||
// OpenWithStorageFormat opens a reader for the already-located blob, avoiding the potential
|
||||
// need to check multiple storage formats to find the blob.
|
||||
OpenWithStorageFormat(ctx context.Context, ref BlobRef, formatVer FormatVersion) (BlobReader, error)
|
||||
// Delete deletes the blob with the namespace and key
|
||||
// Delete deletes the blob with the namespace and key.
|
||||
Delete(ctx context.Context, ref BlobRef) error
|
||||
// DeleteWithStorageFormat deletes a blob of a specific storage format
|
||||
// DeleteWithStorageFormat deletes a blob of a specific storage format.
|
||||
DeleteWithStorageFormat(ctx context.Context, ref BlobRef, formatVer FormatVersion) error
|
||||
// Trash marks a file for pending deletion
|
||||
// Trash marks a file for pending deletion.
|
||||
Trash(ctx context.Context, ref BlobRef) error
|
||||
// RestoreTrash restores all files in the trash for a given namespace and returns the keys restored
|
||||
// RestoreTrash restores all files in the trash for a given namespace and returns the keys restored.
|
||||
RestoreTrash(ctx context.Context, namespace []byte) ([][]byte, error)
|
||||
// EmptyTrash removes all files in trash that were moved to trash prior to trashedBefore and returns the total bytes emptied and keys deleted
|
||||
// EmptyTrash removes all files in trash that were moved to trash prior to trashedBefore and returns the total bytes emptied and keys deleted.
|
||||
EmptyTrash(ctx context.Context, namespace []byte, trashedBefore time.Time) (int64, [][]byte, error)
|
||||
// Stat looks up disk metadata on the blob file
|
||||
// Stat looks up disk metadata on the blob file.
|
||||
Stat(ctx context.Context, ref BlobRef) (BlobInfo, error)
|
||||
// StatWithStorageFormat looks up disk metadata for the blob file with the given storage format
|
||||
// version. This avoids the potential need to check multiple storage formats for the blob
|
||||
// when the format is already known.
|
||||
StatWithStorageFormat(ctx context.Context, ref BlobRef, formatVer FormatVersion) (BlobInfo, error)
|
||||
// FreeSpace return how much free space is available to the blobstore
|
||||
// FreeSpace return how much free space is available to the blobstore.
|
||||
FreeSpace() (int64, error)
|
||||
// SpaceUsedForTrash returns the total space used by the trash
|
||||
// SpaceUsedForTrash returns the total space used by the trash.
|
||||
SpaceUsedForTrash(ctx context.Context) (int64, error)
|
||||
// SpaceUsedForBlobs adds up how much is used in all namespaces
|
||||
// SpaceUsedForBlobs adds up how much is used in all namespaces.
|
||||
SpaceUsedForBlobs(ctx context.Context) (int64, error)
|
||||
// SpaceUsedForBlobsInNamespace adds up how much is used in the given namespace
|
||||
// SpaceUsedForBlobsInNamespace adds up how much is used in the given namespace.
|
||||
SpaceUsedForBlobsInNamespace(ctx context.Context, namespace []byte) (int64, error)
|
||||
// ListNamespaces finds all namespaces in which keys might currently be stored.
|
||||
ListNamespaces(ctx context.Context) ([][]byte, error)
|
||||
@ -111,14 +111,14 @@ type Blobs interface {
|
||||
}
|
||||
|
||||
// BlobInfo allows lazy inspection of a blob and its underlying file during iteration with
|
||||
// WalkNamespace-type methods
|
||||
// WalkNamespace-type methods.
|
||||
type BlobInfo interface {
|
||||
// BlobRef returns the relevant BlobRef for the blob
|
||||
// BlobRef returns the relevant BlobRef for the blob.
|
||||
BlobRef() BlobRef
|
||||
// StorageFormatVersion indicates the storage format version used to store the piece
|
||||
// StorageFormatVersion indicates the storage format version used to store the piece.
|
||||
StorageFormatVersion() FormatVersion
|
||||
// FullPath gives the full path to the on-disk blob file
|
||||
// FullPath gives the full path to the on-disk blob file.
|
||||
FullPath(ctx context.Context) (string, error)
|
||||
// Stat does a stat on the on-disk blob file
|
||||
// Stat does a stat on the on-disk blob file.
|
||||
Stat(ctx context.Context) (os.FileInfo, error)
|
||||
}
|
||||
|
||||
@ -14,114 +14,120 @@ import (
|
||||
|
||||
var mon = monkit.Package()
|
||||
|
||||
// Delimiter separates nested paths in storage
|
||||
// Delimiter separates nested paths in storage.
|
||||
const Delimiter = '/'
|
||||
|
||||
//ErrKeyNotFound used when something doesn't exist
|
||||
var ErrKeyNotFound = errs.Class("key not found")
|
||||
|
||||
// ErrEmptyKey is returned when an empty key is used in Put or in CompareAndSwap
|
||||
// ErrEmptyKey is returned when an empty key is used in Put or in CompareAndSwap.
|
||||
var ErrEmptyKey = errs.Class("empty key")
|
||||
|
||||
// ErrValueChanged is returned when the current value of the key does not match the oldValue in CompareAndSwap
|
||||
// ErrValueChanged is returned when the current value of the key does not match the oldValue in CompareAndSwap.
|
||||
var ErrValueChanged = errs.Class("value changed")
|
||||
|
||||
// ErrEmptyQueue is returned when attempting to Dequeue from an empty queue
|
||||
// ErrEmptyQueue is returned when attempting to Dequeue from an empty queue.
|
||||
var ErrEmptyQueue = errs.Class("empty queue")
|
||||
|
||||
// ErrLimitExceeded is returned when request limit is exceeded
|
||||
// ErrLimitExceeded is returned when request limit is exceeded.
|
||||
var ErrLimitExceeded = errors.New("limit exceeded")
|
||||
|
||||
// Key is the type for the keys in a `KeyValueStore`
|
||||
// Key is the type for the keys in a `KeyValueStore`.
|
||||
type Key []byte
|
||||
|
||||
// Value is the type for the values in a `ValueValueStore`
|
||||
// Value is the type for the values in a `ValueValueStore`.
|
||||
type Value []byte
|
||||
|
||||
// Keys is the type for a slice of keys in a `KeyValueStore`
|
||||
// Keys is the type for a slice of keys in a `KeyValueStore`.
|
||||
type Keys []Key
|
||||
|
||||
// Values is the type for a slice of Values in a `KeyValueStore`
|
||||
// Values is the type for a slice of Values in a `KeyValueStore`.
|
||||
type Values []Value
|
||||
|
||||
// Items keeps all ListItem
|
||||
// Items keeps all ListItem.
|
||||
type Items []ListItem
|
||||
|
||||
// DefaultLookupLimit is the default lookup limit for storage implementations
|
||||
// DefaultLookupLimit is the default lookup limit for storage implementations.
|
||||
const DefaultLookupLimit = 10000
|
||||
|
||||
// ListItem returns Key, Value, IsPrefix
|
||||
// ListItem returns Key, Value, IsPrefix.
|
||||
type ListItem struct {
|
||||
Key Key
|
||||
Value Value
|
||||
IsPrefix bool
|
||||
}
|
||||
|
||||
// KeyValueStore describes key/value stores like redis and boltdb
|
||||
// KeyValueStore describes key/value stores like redis and boltdb.
|
||||
type KeyValueStore interface {
|
||||
// Put adds a value to store
|
||||
// Put adds a value to store.
|
||||
Put(context.Context, Key, Value) error
|
||||
// Get gets a value to store
|
||||
// Get gets a value to store.
|
||||
Get(context.Context, Key) (Value, error)
|
||||
// GetAll gets all values from the store
|
||||
// GetAll gets all values from the store.
|
||||
GetAll(context.Context, Keys) (Values, error)
|
||||
// Delete deletes key and the value
|
||||
// Delete deletes key and the value.
|
||||
Delete(context.Context, Key) error
|
||||
// DeleteMultiple deletes keys and returns nil for
|
||||
// DeleteMultiple deletes keys and returns nil for.
|
||||
DeleteMultiple(context.Context, []Key) (Items, error)
|
||||
// List lists all keys starting from start and upto limit items
|
||||
// List lists all keys starting from start and upto limit items.
|
||||
List(ctx context.Context, start Key, limit int) (Keys, error)
|
||||
// Iterate iterates over items based on opts
|
||||
// Iterate iterates over items based on opts.
|
||||
Iterate(ctx context.Context, opts IterateOptions, fn func(context.Context, Iterator) error) error
|
||||
// CompareAndSwap atomically compares and swaps oldValue with newValue
|
||||
// CompareAndSwap atomically compares and swaps oldValue with newValue.
|
||||
CompareAndSwap(ctx context.Context, key Key, oldValue, newValue Value) error
|
||||
// Close closes the store
|
||||
// Close closes the store.
|
||||
Close() error
|
||||
|
||||
// LookupLimit returns the maximum limit that is allowed.
|
||||
LookupLimit() int
|
||||
}
|
||||
|
||||
// IterateOptions contains options for iterator
|
||||
// IterateOptions contains options for iterator.
|
||||
type IterateOptions struct {
|
||||
// Prefix ensure
|
||||
// Prefix ensure.
|
||||
Prefix Key
|
||||
// First will be the first item iterator returns or the next item (previous when reverse)
|
||||
// First will be the first item iterator returns or the next item (previous when reverse).
|
||||
First Key
|
||||
// Recurse, do not collapse items based on Delimiter
|
||||
// Recurse, do not collapse items based on Delimiter.
|
||||
Recurse bool
|
||||
// The maximum number of elements to be returned
|
||||
// The maximum number of elements to be returned.
|
||||
Limit int
|
||||
}
|
||||
|
||||
// Iterator iterates over a sequence of ListItems
|
||||
// Iterator iterates over a sequence of ListItems.
|
||||
type Iterator interface {
|
||||
// Next prepares the next list item.
|
||||
// It returns true on success, or false if there is no next result row or an error happened while preparing it.
|
||||
Next(ctx context.Context, item *ListItem) bool
|
||||
}
|
||||
|
||||
// IsZero returns true if the value struct is it's zero value
|
||||
// IteratorFunc implements basic iterator.
|
||||
type IteratorFunc func(ctx context.Context, item *ListItem) bool
|
||||
|
||||
// Next returns the next item.
|
||||
func (next IteratorFunc) Next(ctx context.Context, item *ListItem) bool { return next(ctx, item) }
|
||||
|
||||
// IsZero returns true if the value struct is it's zero value.
|
||||
func (value Value) IsZero() bool {
|
||||
return len(value) == 0
|
||||
}
|
||||
|
||||
// IsZero returns true if the key struct is it's zero value
|
||||
// IsZero returns true if the key struct is it's zero value.
|
||||
func (key Key) IsZero() bool {
|
||||
return len(key) == 0
|
||||
}
|
||||
|
||||
// MarshalBinary implements the encoding.BinaryMarshaler interface for the Value type
|
||||
// MarshalBinary implements the encoding.BinaryMarshaler interface for the Value type.
|
||||
func (value Value) MarshalBinary() ([]byte, error) {
|
||||
return value, nil
|
||||
}
|
||||
|
||||
// MarshalBinary implements the encoding.BinaryMarshaler interface for the Key type
|
||||
// MarshalBinary implements the encoding.BinaryMarshaler interface for the Key type.
|
||||
func (key Key) MarshalBinary() ([]byte, error) {
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// ByteSlices converts a `Keys` struct to a slice of byte-slices (i.e. `[][]byte`)
|
||||
// ByteSlices converts a `Keys` struct to a slice of byte-slices (i.e. `[][]byte`).
|
||||
func (keys Keys) ByteSlices() [][]byte {
|
||||
result := make([][]byte, len(keys))
|
||||
|
||||
@ -132,10 +138,10 @@ func (keys Keys) ByteSlices() [][]byte {
|
||||
return result
|
||||
}
|
||||
|
||||
// String implements the Stringer interface
|
||||
// String implements the Stringer interface.
|
||||
func (key Key) String() string { return string(key) }
|
||||
|
||||
// Strings returns everything as strings
|
||||
// Strings returns everything as strings.
|
||||
func (keys Keys) Strings() []string {
|
||||
strs := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
@ -144,7 +150,7 @@ func (keys Keys) Strings() []string {
|
||||
return strs
|
||||
}
|
||||
|
||||
// GetKeys gets all the Keys in []ListItem and converts them to Keys
|
||||
// GetKeys gets all the Keys in []ListItem and converts them to Keys.
|
||||
func (items Items) GetKeys() Keys {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
@ -166,11 +172,11 @@ func (items Items) Less(i, k int) bool { return items[i].Less(items[k]) }
|
||||
// Swap swaps the elements with indexes i and j.
|
||||
func (items Items) Swap(i, k int) { items[i], items[k] = items[k], items[i] }
|
||||
|
||||
// Less returns whether item should be sorted before b
|
||||
// Less returns whether item should be sorted before b.
|
||||
func (item ListItem) Less(b ListItem) bool { return item.Key.Less(b.Key) }
|
||||
|
||||
// Less returns whether key should be sorted before b
|
||||
// Less returns whether key should be sorted before b.
|
||||
func (key Key) Less(b Key) bool { return bytes.Compare([]byte(key), []byte(b)) < 0 }
|
||||
|
||||
// Equal returns whether key and b are equal
|
||||
// Equal returns whether key and b are equal.
|
||||
func (key Key) Equal(b Key) bool { return bytes.Equal([]byte(key), []byte(b)) }
|
||||
|
||||
@ -1,77 +0,0 @@
|
||||
// Copyright (C) 2019 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// IteratorFunc implements basic iterator
|
||||
type IteratorFunc func(ctx context.Context, item *ListItem) bool
|
||||
|
||||
// Next returns the next item
|
||||
func (next IteratorFunc) Next(ctx context.Context, item *ListItem) bool { return next(ctx, item) }
|
||||
|
||||
// SelectPrefixed keeps only items that have prefix
|
||||
// items will be reused and modified
|
||||
// TODO: remove this
|
||||
func SelectPrefixed(items Items, prefix []byte) Items {
|
||||
result := items[:0]
|
||||
for _, item := range items {
|
||||
if bytes.HasPrefix(item.Key, prefix) {
|
||||
result = append(result, item)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// SortAndCollapse sorts items and combines elements based on Delimiter
|
||||
// items will be reused and modified
|
||||
// TODO: remove this
|
||||
func SortAndCollapse(items Items, prefix []byte) Items {
|
||||
sort.Sort(items)
|
||||
result := items[:0]
|
||||
|
||||
var currentPrefix []byte
|
||||
var prefixed bool
|
||||
for _, item := range items {
|
||||
if prefixed {
|
||||
if bytes.HasPrefix(item.Key, currentPrefix) {
|
||||
continue
|
||||
}
|
||||
prefixed = false
|
||||
}
|
||||
|
||||
if p := bytes.IndexByte(item.Key[len(prefix):], Delimiter); p >= 0 {
|
||||
currentPrefix = item.Key[:len(prefix)+p+1]
|
||||
prefixed = true
|
||||
result = append(result, ListItem{
|
||||
Key: currentPrefix,
|
||||
IsPrefix: true,
|
||||
})
|
||||
} else {
|
||||
result = append(result, item)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// StaticIterator implements an iterator over list of items
|
||||
type StaticIterator struct {
|
||||
Items Items
|
||||
Index int
|
||||
}
|
||||
|
||||
// Next returns the next item from the iterator
|
||||
func (it *StaticIterator) Next(ctx context.Context, item *ListItem) bool {
|
||||
if it.Index >= len(it.Items) {
|
||||
return false
|
||||
}
|
||||
*item = it.Items[it.Index]
|
||||
it.Index++
|
||||
return true
|
||||
}
|
||||
@ -7,8 +7,8 @@ import (
|
||||
"context"
|
||||
)
|
||||
|
||||
// ListKeys returns keys starting from first and upto limit
|
||||
// limit is capped to LookupLimit
|
||||
// ListKeys returns keys starting from first and upto limit.
|
||||
// limit is capped to LookupLimit.
|
||||
func ListKeys(ctx context.Context, store KeyValueStore, first Key, limit int) (_ Keys, err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
if limit <= 0 || limit > store.LookupLimit() {
|
||||
|
||||
@ -7,7 +7,7 @@ import (
|
||||
"context"
|
||||
)
|
||||
|
||||
// ListOptions are items that are optional for the LIST method
|
||||
// ListOptions are items that are optional for the LIST method.
|
||||
type ListOptions struct {
|
||||
Prefix Key
|
||||
StartAfter Key // StartAfter is relative to Prefix
|
||||
@ -16,8 +16,8 @@ type ListOptions struct {
|
||||
Limit int
|
||||
}
|
||||
|
||||
// ListV2 lists all keys corresponding to ListOptions
|
||||
// limit is capped to LookupLimit
|
||||
// ListV2 lists all keys corresponding to ListOptions.
|
||||
// limit is capped to LookupLimit.
|
||||
//
|
||||
// more indicates if the result was truncated. If false
|
||||
// then the result []ListItem includes all requested keys.
|
||||
|
||||
@ -190,10 +190,10 @@ func (client *Client) Iterate(ctx context.Context, opts storage.IterateOptions,
|
||||
}
|
||||
|
||||
if !opts.Recurse {
|
||||
all = storage.SortAndCollapse(all, opts.Prefix)
|
||||
all = sortAndCollapse(all, opts.Prefix)
|
||||
}
|
||||
|
||||
return fn(ctx, &storage.StaticIterator{
|
||||
return fn(ctx, &StaticIterator{
|
||||
Items: all,
|
||||
})
|
||||
}
|
||||
|
||||
@ -3,6 +3,14 @@
|
||||
|
||||
package redis
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"sort"
|
||||
|
||||
"storj.io/storj/storage"
|
||||
)
|
||||
|
||||
func escapeMatch(match []byte) []byte {
|
||||
start := 0
|
||||
escaped := []byte{}
|
||||
@ -20,3 +28,50 @@ func escapeMatch(match []byte) []byte {
|
||||
|
||||
return append(escaped, match[start:]...)
|
||||
}
|
||||
|
||||
// sortAndCollapse sorts items and combines elements based on Delimiter.
|
||||
// items will be reused and modified.
|
||||
func sortAndCollapse(items storage.Items, prefix []byte) storage.Items {
|
||||
sort.Sort(items)
|
||||
result := items[:0]
|
||||
|
||||
var currentPrefix []byte
|
||||
var prefixed bool
|
||||
for _, item := range items {
|
||||
if prefixed {
|
||||
if bytes.HasPrefix(item.Key, currentPrefix) {
|
||||
continue
|
||||
}
|
||||
prefixed = false
|
||||
}
|
||||
|
||||
if p := bytes.IndexByte(item.Key[len(prefix):], storage.Delimiter); p >= 0 {
|
||||
currentPrefix = item.Key[:len(prefix)+p+1]
|
||||
prefixed = true
|
||||
result = append(result, storage.ListItem{
|
||||
Key: currentPrefix,
|
||||
IsPrefix: true,
|
||||
})
|
||||
} else {
|
||||
result = append(result, item)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// StaticIterator implements an iterator over list of items.
|
||||
type StaticIterator struct {
|
||||
Items storage.Items
|
||||
Index int
|
||||
}
|
||||
|
||||
// Next returns the next item from the iterator.
|
||||
func (it *StaticIterator) Next(ctx context.Context, item *storage.ListItem) bool {
|
||||
if it.Index >= len(it.Items) {
|
||||
return false
|
||||
}
|
||||
*item = it.Items[it.Index]
|
||||
it.Index++
|
||||
return true
|
||||
}
|
||||
|
||||
@ -8,25 +8,25 @@ import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// NextKey returns the successive key
|
||||
// NextKey returns the successive key.
|
||||
func NextKey(key Key) Key {
|
||||
return append(CloneKey(key), 0)
|
||||
}
|
||||
|
||||
// AfterPrefix returns the key after prefix
|
||||
// AfterPrefix returns the key after prefix.
|
||||
func AfterPrefix(key Key) Key {
|
||||
after := CloneKey(key)
|
||||
after[len(after)-1]++
|
||||
return after
|
||||
}
|
||||
|
||||
// CloneKey creates a copy of key
|
||||
// CloneKey creates a copy of key.
|
||||
func CloneKey(key Key) Key { return append(Key{}, key...) }
|
||||
|
||||
// CloneValue creates a copy of value
|
||||
// CloneValue creates a copy of value.
|
||||
func CloneValue(value Value) Value { return append(Value{}, value...) }
|
||||
|
||||
// CloneItem creates a deep copy of item
|
||||
// CloneItem creates a deep copy of item.
|
||||
func CloneItem(item ListItem) ListItem {
|
||||
return ListItem{
|
||||
Key: CloneKey(item.Key),
|
||||
@ -35,7 +35,7 @@ func CloneItem(item ListItem) ListItem {
|
||||
}
|
||||
}
|
||||
|
||||
// CloneItems creates a deep copy of items
|
||||
// CloneItems creates a deep copy of items.
|
||||
func CloneItems(items Items) Items {
|
||||
var result = make(Items, len(items))
|
||||
for i, item := range items {
|
||||
@ -44,7 +44,7 @@ func CloneItems(items Items) Items {
|
||||
return result
|
||||
}
|
||||
|
||||
// PutAll adds multiple values to the store
|
||||
// PutAll adds multiple values to the store.
|
||||
func PutAll(ctx context.Context, store KeyValueStore, items ...ListItem) (err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user