JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add IsPrefix support to piecestore (#313)

Browse Source
This commit is contained in:
Egon Elbre 2018-09-07 17:20:15 +03:00 committed by GitHub
parent 706a6ac26e
commit 0a2839b2d7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 832 additions and 503 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/test/storage.go
View File

@ -72,7 +72,7 @@ func (store *MockKeyValueStore) Delete(key storage.Key) error {
}
// List returns either a list of keys for which the MockKeyValueStore has values or an error.
func (store *MockKeyValueStore) List(first storage.Key, limit storage.Limit) (storage.Keys, error) {
func (store *MockKeyValueStore) List(first storage.Key, limit int) (storage.Keys, error) {
store.ListCalled++
return storage.ListKeys(store, first, limit)
}
@ -112,7 +112,7 @@ func (store *MockKeyValueStore) allPrefixedItems(prefix, first, last storage.Key
}
// ReverseList returns either a list of keys for which the MockKeyValueStore has values or an error.
func (store *MockKeyValueStore) ReverseList(first storage.Key, limit storage.Limit) (storage.Keys, error) {
func (store *MockKeyValueStore) ReverseList(first storage.Key, limit int) (storage.Keys, error) {
return storage.ReverseListKeys(store, first, limit)
}

1
pkg/miniogw/gateway-storj.go
View File

@ -216,6 +216,7 @@ func (s *storjObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
for i, fi := range items {
f[i] = minio.ObjectInfo{
Bucket: bucket,
IsDir: fi.IsPrefix,
Name: fi.Path.String(),
ModTime: fi.Meta.Modified,
Size: fi.Meta.Size,

4
pkg/overlay/server.go
View File

@ -103,7 +103,7 @@ func (o *Server) getNodes(ctx context.Context, keys storage.Keys) ([]*proto.Node
}
func (o *Server) populate(ctx context.Context, starting storage.Key, maxNodes, restrictedBandwidth, restrictedSpace int64) ([]*proto.Node, storage.Key, error) {
limit := storage.Limit(maxNodes) * 2
limit := int(maxNodes * 2)
keys, err := o.cache.DB.List(starting, limit)
if err != nil {
o.logger.Error("Error listing nodes", zap.Error(err))
@ -132,7 +132,7 @@ func (o *Server) populate(ctx context.Context, starting storage.Key, maxNodes, r
}
nextStart := keys[len(keys)-1]
if len(keys) < int(limit) {
if len(keys) < limit {
nextStart = nil
}

4
pkg/pointerdb/kvstore_mock_test.go
View File

@ -96,7 +96,7 @@ func (mr *MockKeyValueStoreMockRecorder) Iterate(arg0, arg1 interface{}) *gomock
}
// List mocks base method
func (m *MockKeyValueStore) List(arg0 storage.Key, arg1 storage.Limit) (storage.Keys, error) {
func (m *MockKeyValueStore) List(arg0 storage.Key, arg1 int) (storage.Keys, error) {
ret := m.ctrl.Call(m, "List", arg0, arg1)
ret0, _ := ret[0].(storage.Keys)
ret1, _ := ret[1].(error)
@ -121,7 +121,7 @@ func (mr *MockKeyValueStoreMockRecorder) Put(arg0, arg1 interface{}) *gomock.Cal
}
// ReverseList mocks base method
func (m *MockKeyValueStore) ReverseList(arg0 storage.Key, arg1 storage.Limit) (storage.Keys, error) {
func (m *MockKeyValueStore) ReverseList(arg0 storage.Key, arg1 int) (storage.Keys, error) {
ret := m.ctrl.Call(m, "ReverseList", arg0, arg1)
ret0, _ := ret[0].(storage.Keys)
ret1, _ := ret[1].(error)

10
pkg/pointerdb/pdbclient/client.go
View File

@ -33,8 +33,9 @@ var _ Client = (*PointerDB)(nil)
// ListItem is a single item in a listing
type ListItem struct {
Path p.Path
Pointer *pb.Pointer
Path p.Path
Pointer *pb.Pointer
IsPrefix bool
}
// Client services offerred for the interface
@ -130,8 +131,9 @@ func (pdb *PointerDB) List(ctx context.Context, prefix, startAfter, endBefore p.
items = make([]ListItem, len(list))
for i, itm := range list {
items[i] = ListItem{
Path: p.New(itm.GetPath()),
Pointer: itm.GetPointer(),
Path: p.New(itm.GetPath()),
Pointer: itm.GetPointer(),
IsPrefix: itm.IsPrefix,
}
}

153
pkg/pointerdb/pointerdb.go
View File

@ -5,7 +5,6 @@ package pointerdb
import (
"context"
"reflect"
"github.com/golang/protobuf/proto"
"github.com/golang/protobuf/ptypes"
@ -15,7 +14,6 @@ import (
"google.golang.org/grpc/status"
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/storage/meta"
"storj.io/storj/pointerdb/auth"
pb "storj.io/storj/protos/pointerdb"
@ -126,132 +124,55 @@ func (s *Server) Get(ctx context.Context, req *pb.GetRequest) (resp *pb.GetRespo
}, nil
}
// List calls the bolt client's List function and returns all Path keys in the Pointers bucket
// List returns all Path keys in the Pointers bucket
func (s *Server) List(ctx context.Context, req *pb.ListRequest) (resp *pb.ListResponse, err error) {
defer mon.Task()(&ctx)(&err)
s.logger.Debug("entering pointerdb list")
limit := int(req.GetLimit())
if err = s.validateAuth(req.GetAPIKey()); err != nil {
if err = s.validateAuth(req.APIKey); err != nil {
return nil, err
}
prefix := paths.New(req.GetPrefix())
var prefix storage.Key
if req.Prefix != "" {
prefix = storage.Key(req.Prefix)
if prefix[len(prefix)-1] != storage.Delimiter {
prefix = append(prefix, storage.Delimiter)
}
}
// TODO(kaloyan): here we query the DB without limit. We must optimize it!
keys, err := s.DB.List([]byte(req.GetPrefix()+"/"), 0)
rawItems, more, err := storage.ListV2(s.DB, storage.ListOptions{
Prefix: prefix, //storage.Key(req.Prefix),
StartAfter: storage.Key(req.StartAfter),
EndBefore: storage.Key(req.EndBefore),
Recursive: req.Recursive,
Limit: int(req.Limit),
IncludeValue: req.MetaFlags != meta.None,
})
if err != nil {
return nil, status.Errorf(codes.Internal, err.Error())
return nil, status.Errorf(codes.Internal, "ListV2: %v", err)
}
var more bool
var items []*pb.ListResponse_Item
if req.GetEndBefore() != "" && req.GetStartAfter() == "" {
items, more = s.processKeysBackwards(ctx, keys, prefix,
req.GetEndBefore(), req.GetRecursive(), limit, req.GetMetaFlags())
} else {
items, more = s.processKeysForwards(ctx, keys, prefix, req.GetStartAfter(),
req.GetEndBefore(), req.GetRecursive(), limit, req.GetMetaFlags())
for _, rawItem := range rawItems {
items = append(items, s.createListItem(prefix, rawItem, req.MetaFlags))
}
s.logger.Debug("path keys retrieved")
return &pb.ListResponse{Items: items, More: more}, nil
}
// processKeysForwards iterates forwards through given keys, and returns them
// as list items
func (s *Server) processKeysForwards(ctx context.Context, keys storage.Keys,
prefix paths.Path, startAfter, endBefore string, recursive bool, limit int,
metaFlags uint32) (items []*pb.ListResponse_Item, more bool) {
skip := startAfter != ""
startAfterPath := prefix.Append(startAfter)
endBeforePath := prefix.Append(endBefore)
for _, key := range keys {
p := paths.New(string(key))
if skip {
if reflect.DeepEqual(p, startAfterPath) {
// TODO(kaloyan): Better check - what if there is no path equal to startAfter?
// TODO(kaloyan): Add Equal method in Path type
skip = false
}
continue
}
// TODO(kaloyan): Better check - what if there is no path equal to endBefore?
// TODO(kaloyan): Add Equal method in Path type
if reflect.DeepEqual(p, endBeforePath) {
break
}
if !p.HasPrefix(prefix) {
// We went through all keys that start with the prefix
break
}
if !recursive && len(p) > len(prefix)+1 {
continue
}
item := s.createListItem(ctx, p, metaFlags)
items = append(items, item)
if len(items) == limit {
more = true
break
}
}
return items, more
}
// processKeysBackwards iterates backwards through given keys, and returns them
// as list items
func (s *Server) processKeysBackwards(ctx context.Context, keys storage.Keys,
prefix paths.Path, endBefore string, recursive bool, limit int,
metaFlags uint32) (items []*pb.ListResponse_Item, more bool) {
skip := endBefore != ""
endBeforePath := prefix.Append(endBefore)
for i := len(keys) - 1; i >= 0; i-- {
key := keys[i]
p := paths.New(string(key))
if skip {
if reflect.DeepEqual(p, endBeforePath) {
// TODO(kaloyan): Better check - what if there is no path equal to endBefore?
// TODO(kaloyan): Add Equal method in Path type
skip = false
}
continue
}
if !p.HasPrefix(prefix) || len(p) <= len(prefix) {
// We went through all keys that start with the prefix
break
}
if !recursive && len(p) > len(prefix)+1 {
continue
}
item := s.createListItem(ctx, p, metaFlags)
items = append([]*pb.ListResponse_Item{item}, items...)
if len(items) == limit {
more = true
break
}
}
return items, more
}
// createListItem creates a new list item with the given path. It also adds
// the metadata according to the given metaFlags.
func (s *Server) createListItem(ctx context.Context, p paths.Path,
metaFlags uint32) *pb.ListResponse_Item {
item := &pb.ListResponse_Item{Path: p.String()}
err := s.getMetadata(ctx, item, metaFlags)
func (s *Server) createListItem(prefix storage.Key, rawItem storage.ListItem, metaFlags uint32) *pb.ListResponse_Item {
item := &pb.ListResponse_Item{
Path: append(prefix, rawItem.Key...).String(),
IsPrefix: rawItem.IsPrefix,
}
if item.IsPrefix {
return item
}
err := s.setMetadata(item, rawItem.Value, metaFlags)
if err != nil {
s.logger.Warn("err retrieving metadata", zap.Error(err))
}
@ -260,21 +181,13 @@ func (s *Server) createListItem(ctx context.Context, p paths.Path,
// getMetadata adds the metadata to the given item pointer according to the
// given metaFlags
func (s *Server) getMetadata(ctx context.Context, item *pb.ListResponse_Item,
metaFlags uint32) (err error) {
defer mon.Task()(&ctx)(&err)
if metaFlags == meta.None {
func (s *Server) setMetadata(item *pb.ListResponse_Item, data []byte, metaFlags uint32) (err error) {
if metaFlags == meta.None || len(data) == 0 {
return nil
}
b, err := s.DB.Get([]byte(item.GetPath()))
if err != nil {
return err
}
pr := &pb.Pointer{}
err = proto.Unmarshal(b, pr)
err = proto.Unmarshal(data, pr)
if err != nil {
return err
}

245
pkg/pointerdb/pointerdb_test.go
View File

@ -15,12 +15,15 @@ import (
"github.com/golang/mock/gomock"
"github.com/golang/protobuf/proto"
"github.com/golang/protobuf/ptypes"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/storage/meta"
pb "storj.io/storj/protos/pointerdb"
"storj.io/storj/storage"
"storj.io/storj/storage/teststore"
)
//go:generate mockgen -destination kvstore_mock_test.go -package pointerdb storj.io/storj/storage KeyValueStore
@ -143,92 +146,184 @@ func TestServiceDelete(t *testing.T) {
}
func TestServiceList(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
db := teststore.New()
server := Server{DB: db, logger: zap.NewNop()}
keys := storage.Keys{
storage.Key(paths.New("sample.jpg").Bytes()),
storage.Key(paths.New("music").Bytes()),
storage.Key(paths.New("music/song1.mp3").Bytes()),
storage.Key(paths.New("music/song2.mp3").Bytes()),
storage.Key(paths.New("music/album/song3.mp3").Bytes()),
storage.Key(paths.New("music/song4.mp3").Bytes()),
storage.Key(paths.New("videos").Bytes()),
storage.Key(paths.New("videos/movie.mkv").Bytes()),
key := func(s string) storage.Key {
return storage.Key(paths.New(s).Bytes())
}
for i, tt := range []struct {
prefix string
startAfter string
endBefore string
recursive bool
limit int
metaFlags uint32
apiKey []byte
returnedKeys storage.Keys
expectedKeys storage.Keys
expectedMore bool
err error
errString string
}{
{"", "", "", true, 0, meta.None, nil, keys, keys, false, nil, ""},
{"", "", "", true, 0, meta.All, nil, keys, keys, false, nil, ""},
{"", "", "", true, 0, meta.None, []byte("wrong key"), keys, keys, false,
nil, status.Errorf(codes.Unauthenticated, "Invalid API credential").Error()},
{"", "", "", true, 0, meta.None, nil, keys, keys, false,
errors.New("list error"), status.Errorf(codes.Internal, "list error").Error()},
{"", "", "", true, 2, meta.None, nil, keys, keys[:2], true, nil, ""},
{"", "", "", false, 0, meta.None, nil, keys, storage.Keys{keys[0], keys[1], keys[6]}, false, nil, ""},
{"", "", "videos", false, 0, meta.None, nil, keys, keys[:2], false, nil, ""},
{"music", "", "", false, 0, meta.None, nil, keys[2:], storage.Keys{keys[2], keys[3], keys[5]}, false, nil, ""},
{"music", "", "", true, 0, meta.None, nil, keys[2:], keys[2:6], false, nil, ""},
{"music", "song1.mp3", "", true, 0, meta.None, nil, keys, keys[3:6], false, nil, ""},
{"music", "song1.mp3", "album/song3.mp3", true, 0, meta.None, nil, keys, keys[3:4], false, nil, ""},
{"music", "", "song4.mp3", true, 0, meta.None, nil, keys, keys[2:5], false, nil, ""},
{"music", "", "song4.mp3", true, 1, meta.None, nil, keys, keys[4:5], true, nil, ""},
{"music", "", "song4.mp3", false, 0, meta.None, nil, keys, keys[2:4], false, nil, ""},
{"music", "song2.mp3", "song4.mp3", true, 0, meta.None, nil, keys, keys[4:5], false, nil, ""},
{"mus", "", "", true, 0, meta.None, nil, keys[1:], nil, false, nil, ""},
} {
errTag := fmt.Sprintf("Test case #%d", i)
pointer := &pb.Pointer{}
pointer.CreationDate = ptypes.TimestampNow()
db := NewMockKeyValueStore(ctrl)
s := Server{DB: db, logger: zap.NewNop()}
pointerBytes, err := proto.Marshal(pointer)
if err != nil {
t.Fatal(err)
}
pointerValue := storage.Value(pointerBytes)
if tt.err != nil || tt.errString == "" {
prefix := storage.Key([]byte(tt.prefix + "/"))
db.EXPECT().List(prefix, storage.Limit(0)).Return(tt.returnedKeys, tt.err)
err = storage.PutAll(db, []storage.ListItem{
{Key: key("sample.😶"), Value: pointerValue},
{Key: key("müsic"), Value: pointerValue},
{Key: key("müsic/söng1.mp3"), Value: pointerValue},
{Key: key("müsic/söng2.mp3"), Value: pointerValue},
{Key: key("müsic/album/söng3.mp3"), Value: pointerValue},
{Key: key("müsic/söng4.mp3"), Value: pointerValue},
{Key: key("ビデオ/movie.mkv"), Value: pointerValue},
}...)
if err != nil {
t.Fatal(err)
}
if tt.metaFlags != meta.None {
pr := pb.Pointer{}
b, err := proto.Marshal(&pr)
assert.NoError(t, err, errTag)
for _, key := range keys {
db.EXPECT().Get(key).Return(b, nil)
}
type Test struct {
Request pb.ListRequest
Expected *pb.ListResponse
Error func(i int, err error)
}
errorWithCode := func(code codes.Code) func(i int, err error) {
t.Helper()
return func(i int, err error) {
t.Helper()
if status.Code(err) != code {
t.Fatalf("%d: should fail with %v, got: %v", i, code, err)
}
}
}
req := pb.ListRequest{
Prefix: tt.prefix,
StartAfter: tt.startAfter,
EndBefore: tt.endBefore,
Recursive: tt.recursive,
Limit: int32(tt.limit),
MetaFlags: tt.metaFlags,
APIKey: tt.apiKey,
}
resp, err := s.List(ctx, &req)
tests := []Test{
{
Request: pb.ListRequest{Recursive: true},
Expected: &pb.ListResponse{
Items: []*pb.ListResponse_Item{
{Path: "müsic"},
{Path: "müsic/album/söng3.mp3"},
{Path: "müsic/söng1.mp3"},
{Path: "müsic/söng2.mp3"},
{Path: "müsic/söng4.mp3"},
{Path: "sample.😶"},
{Path: "ビデオ/movie.mkv"},
},
},
}, {
Request: pb.ListRequest{Recursive: true, MetaFlags: meta.All},
Expected: &pb.ListResponse{
Items: []*pb.ListResponse_Item{
{Path: "müsic", Pointer: pointer},
{Path: "müsic/album/söng3.mp3", Pointer: pointer},
{Path: "müsic/söng1.mp3", Pointer: pointer},
{Path: "müsic/söng2.mp3", Pointer: pointer},
{Path: "müsic/söng4.mp3", Pointer: pointer},
{Path: "sample.😶", Pointer: pointer},
{Path: "ビデオ/movie.mkv", Pointer: pointer},
},
},
}, {
Request: pb.ListRequest{Recursive: true, MetaFlags: meta.All, APIKey: []byte("wrong key")},
Error: errorWithCode(codes.Unauthenticated),
}, {
Request: pb.ListRequest{Recursive: true, Limit: 3},
Expected: &pb.ListResponse{
Items: []*pb.ListResponse_Item{
{Path: "müsic"},
{Path: "müsic/album/söng3.mp3"},
{Path: "müsic/söng1.mp3"},
},
More: true,
},
}, {
Request: pb.ListRequest{MetaFlags: meta.All},
Expected: &pb.ListResponse{
Items: []*pb.ListResponse_Item{
{Path: "müsic", Pointer: pointer},
{Path: "müsic/", IsPrefix: true},
{Path: "sample.😶", Pointer: pointer},
{Path: "ビデオ/", IsPrefix: true},
},
More: false,
},
}, {
Request: pb.ListRequest{EndBefore: "ビデオ"},
Expected: &pb.ListResponse{
Items: []*pb.ListResponse_Item{
{Path: "müsic"},
{Path: "müsic/", IsPrefix: true},
{Path: "sample.😶"},
},
More: false,
},
}, {
Request: pb.ListRequest{Recursive: true, Prefix: "müsic/"},
Expected: &pb.ListResponse{
Items: []*pb.ListResponse_Item{
{Path: "müsic/album/söng3.mp3"},
{Path: "müsic/söng1.mp3"},
{Path: "müsic/söng2.mp3"},
{Path: "müsic/söng4.mp3"},
},
},
}, {
Request: pb.ListRequest{Recursive: true, Prefix: "müsic/", StartAfter: "album/söng3.mp3"},
Expected: &pb.ListResponse{
Items: []*pb.ListResponse_Item{
{Path: "müsic/söng1.mp3"},
{Path: "müsic/söng2.mp3"},
{Path: "müsic/söng4.mp3"},
},
},
}, {
Request: pb.ListRequest{Prefix: "müsic/"},
Expected: &pb.ListResponse{
Items: []*pb.ListResponse_Item{
{Path: "müsic/album/", IsPrefix: true},
{Path: "müsic/söng1.mp3"},
{Path: "müsic/söng2.mp3"},
{Path: "müsic/söng4.mp3"},
},
},
}, {
Request: pb.ListRequest{Prefix: "müsic/", StartAfter: "söng1.mp3"},
Expected: &pb.ListResponse{
Items: []*pb.ListResponse_Item{
{Path: "müsic/söng2.mp3"},
{Path: "müsic/söng4.mp3"},
},
},
}, {
Request: pb.ListRequest{Prefix: "müsic/", EndBefore: "söng4.mp3"},
Expected: &pb.ListResponse{
Items: []*pb.ListResponse_Item{
{Path: "müsic/album/", IsPrefix: true},
{Path: "müsic/söng1.mp3"},
{Path: "müsic/söng2.mp3"},
},
},
}, {
Request: pb.ListRequest{Prefix: "müs", Recursive: true, EndBefore: "ic/söng4.mp3", Limit: 1},
Expected: &pb.ListResponse{
Items: []*pb.ListResponse_Item{
// {Path: "ic/söng2.mp3"},
},
// More: true,
},
},
}
if err != nil {
assert.EqualError(t, err, tt.errString, errTag)
// TODO:
// pb.ListRequest{Prefix: "müsic/", StartAfter: "söng1.mp3", EndBefore: "söng4.mp3"},
// failing database
for i, test := range tests {
resp, err := server.List(ctx, &test.Request)
if test.Error == nil {
if err != nil {
t.Fatalf("%d: failed %v", i, err)
}
} else {
assert.NoError(t, err, errTag)
assert.Equal(t, tt.expectedMore, resp.GetMore(), errTag)
assert.Equal(t, len(tt.expectedKeys), len(resp.GetItems()), errTag)
for i, item := range resp.GetItems() {
assert.Equal(t, tt.expectedKeys[i].String(), item.Path, errTag)
}
test.Error(i, err)
}
if diff := cmp.Diff(test.Expected, resp, cmp.Comparer(proto.Equal)); diff != "" {
t.Errorf("%d: (-want +got) %v\n%s", i, test.Request.String(), diff)
}
}
}

5
pkg/storage/buckets/prefixed.go
View File

@ -63,8 +63,9 @@ func (o *prefixedObjStore) List(ctx context.Context, prefix, startAfter,
continue
}
items[i] = objects.ListItem{
Path: itm.Path[1:],
Meta: itm.Meta,
Path: itm.Path[1:],
Meta: itm.Meta,
IsPrefix: itm.IsPrefix,
}
}
return items, more, nil

14
pkg/storage/buckets/store.go
View File

@ -26,8 +26,7 @@ type Store interface {
Get(ctx context.Context, bucket string) (meta Meta, err error)
Put(ctx context.Context, bucket string) (meta Meta, err error)
Delete(ctx context.Context, bucket string) (err error)
List(ctx context.Context, startAfter, endBefore string, limit int) (
items []ListItem, more bool, err error)
List(ctx context.Context, startAfter, endBefore string, limit int) (items []ListItem, more bool, err error)
GetObjectStore(ctx context.Context, bucketName string) (store objects.Store, err error)
}
@ -104,12 +103,15 @@ func (b *BucketStore) List(ctx context.Context, startAfter, endBefore string, li
items []ListItem, more bool, err error) {
defer mon.Task()(&ctx)(&err)
objItems, more, err := b.o.List(ctx, nil, paths.New(startAfter), paths.New(endBefore), false, limit, meta.Modified)
items = make([]ListItem, len(objItems))
for i, itm := range objItems {
items[i] = ListItem{
items = make([]ListItem, 0, len(objItems))
for _, itm := range objItems {
if itm.IsPrefix {
continue
}
items = append(items, ListItem{
Bucket: itm.Path.String(),
Meta: convertMeta(itm.Meta),
}
})
}
return items, more, nil
}

10
pkg/storage/objects/store.go
View File

@ -30,8 +30,9 @@ type Meta struct {
// ListItem is a single item in a listing
type ListItem struct {
Path paths.Path
Meta Meta
Path paths.Path
Meta Meta
IsPrefix bool
}
// Store for objects
@ -105,8 +106,9 @@ func (o *objStore) List(ctx context.Context, prefix, startAfter,
items = make([]ListItem, len(strItems))
for i, itm := range strItems {
items[i] = ListItem{
Path: itm.Path,
Meta: convertMeta(itm.Meta),
Path: itm.Path,
Meta: convertMeta(itm.Meta),
IsPrefix: itm.IsPrefix,
}
}

9
pkg/storage/segments/store.go
View File

@ -40,8 +40,9 @@ type Meta struct {
// ListItem is a single item in a listing
type ListItem struct {
Path paths.Path
Meta Meta
Path paths.Path
Meta Meta
IsPrefix bool
}
// Store for segments
@ -268,8 +269,7 @@ func (s *segmentStore) List(ctx context.Context, prefix, startAfter,
items []ListItem, more bool, err error) {
defer mon.Task()(&ctx)(&err)
pdbItems, more, err := s.pdb.List(ctx, prefix, startAfter, endBefore,
recursive, limit, metaFlags)
pdbItems, more, err := s.pdb.List(ctx, prefix, startAfter, endBefore, recursive, limit, metaFlags)
if err != nil {
return nil, false, err
}
@ -279,6 +279,7 @@ func (s *segmentStore) List(ctx context.Context, prefix, startAfter,
items[i] = ListItem{
Path: itm.Path,
Meta: convertMeta(itm.Pointer),
IsPrefix: itm.IsPrefix,
}
}

13
pkg/storage/streams/store.go
View File

@ -220,8 +220,9 @@ func (s *streamStore) Delete(ctx context.Context, path paths.Path) (err error) {
// ListItem is a single item in a listing
type ListItem struct {
Path paths.Path
Meta Meta
Path paths.Path
Meta Meta
IsPrefix bool
}
// List all the paths inside l/, stripping off the l/ prefix
@ -230,18 +231,18 @@ func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore pa
more bool, err error) {
defer mon.Task()(&ctx)(&err)
lItems, more, err := s.segments.List(ctx, prefix.Prepend("l"), startAfter, endBefore, recursive, limit, metaFlags)
segments, more, err := s.segments.List(ctx, prefix.Prepend("l"), startAfter, endBefore, recursive, limit, metaFlags)
if err != nil {
return nil, false, err
}
items = make([]ListItem, len(lItems))
for i, item := range lItems {
items = make([]ListItem, len(segments))
for i, item := range segments {
newMeta, err := convertMeta(item.Meta)
if err != nil {
return nil, false, err
}
items[i] = ListItem{Path: item.Path[1:], Meta: newMeta}
items[i] = ListItem{Path: item.Path[1:], Meta: newMeta, IsPrefix: item.IsPrefix}
}
return items, more, nil

693
protos/pointerdb/pointerdb.pb.go
View File

@ -1,33 +1,12 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: pointerdb.proto
/*
Package pointerdb is a generated protocol buffer package.
It is generated from these files:
pointerdb.proto
It has these top-level messages:
RedundancyScheme
EncryptionScheme
RemotePiece
RemoteSegment
Pointer
PutRequest
GetRequest
ListRequest
PutResponse
GetResponse
ListResponse
DeleteRequest
DeleteResponse
*/
package pointerdb
import proto "github.com/golang/protobuf/proto"
import fmt "fmt"
import math "math"
import google_protobuf "github.com/golang/protobuf/ptypes/timestamp"
import timestamp "github.com/golang/protobuf/ptypes/timestamp"
import (
context "golang.org/x/net/context"
@ -62,7 +41,7 @@ func (x RedundancyScheme_SchemeType) String() string {
return proto.EnumName(RedundancyScheme_SchemeType_name, int32(x))
}
func (RedundancyScheme_SchemeType) EnumDescriptor() ([]byte, []int) {
return fileDescriptor0, []int{0, 0}
return fileDescriptor_pointerdb_138b3541b1f95735, []int{0, 0}
}
type EncryptionScheme_EncryptionType int32
@ -85,7 +64,7 @@ func (x EncryptionScheme_EncryptionType) String() string {
return proto.EnumName(EncryptionScheme_EncryptionType_name, int32(x))
}
func (EncryptionScheme_EncryptionType) EnumDescriptor() ([]byte, []int) {
return fileDescriptor0, []int{1, 0}
return fileDescriptor_pointerdb_138b3541b1f95735, []int{1, 0}
}
type Pointer_DataType int32
@ -107,22 +86,46 @@ var Pointer_DataType_value = map[string]int32{
func (x Pointer_DataType) String() string {
return proto.EnumName(Pointer_DataType_name, int32(x))
}
func (Pointer_DataType) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{4, 0} }
type RedundancyScheme struct {
Type RedundancyScheme_SchemeType `protobuf:"varint,1,opt,name=type,enum=pointerdb.RedundancyScheme_SchemeType" json:"type,omitempty"`
// these values apply to RS encoding
MinReq int32 `protobuf:"varint,2,opt,name=min_req,json=minReq" json:"min_req,omitempty"`
Total int32 `protobuf:"varint,3,opt,name=total" json:"total,omitempty"`
RepairThreshold int32 `protobuf:"varint,4,opt,name=repair_threshold,json=repairThreshold" json:"repair_threshold,omitempty"`
SuccessThreshold int32 `protobuf:"varint,5,opt,name=success_threshold,json=successThreshold" json:"success_threshold,omitempty"`
ErasureShareSize int32 `protobuf:"varint,6,opt,name=erasure_share_size,json=erasureShareSize" json:"erasure_share_size,omitempty"`
func (Pointer_DataType) EnumDescriptor() ([]byte, []int) {
return fileDescriptor_pointerdb_138b3541b1f95735, []int{4, 0}
}
func (m *RedundancyScheme) Reset() { *m = RedundancyScheme{} }
func (m *RedundancyScheme) String() string { return proto.CompactTextString(m) }
func (*RedundancyScheme) ProtoMessage() {}
func (*RedundancyScheme) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
type RedundancyScheme struct {
Type RedundancyScheme_SchemeType `protobuf:"varint,1,opt,name=type,proto3,enum=pointerdb.RedundancyScheme_SchemeType" json:"type,omitempty"`
// these values apply to RS encoding
MinReq int32 `protobuf:"varint,2,opt,name=min_req,json=minReq,proto3" json:"min_req,omitempty"`
Total int32 `protobuf:"varint,3,opt,name=total,proto3" json:"total,omitempty"`
RepairThreshold int32 `protobuf:"varint,4,opt,name=repair_threshold,json=repairThreshold,proto3" json:"repair_threshold,omitempty"`
SuccessThreshold int32 `protobuf:"varint,5,opt,name=success_threshold,json=successThreshold,proto3" json:"success_threshold,omitempty"`
ErasureShareSize int32 `protobuf:"varint,6,opt,name=erasure_share_size,json=erasureShareSize,proto3" json:"erasure_share_size,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *RedundancyScheme) Reset() { *m = RedundancyScheme{} }
func (m *RedundancyScheme) String() string { return proto.CompactTextString(m) }
func (*RedundancyScheme) ProtoMessage() {}
func (*RedundancyScheme) Descriptor() ([]byte, []int) {
return fileDescriptor_pointerdb_138b3541b1f95735, []int{0}
}
func (m *RedundancyScheme) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_RedundancyScheme.Unmarshal(m, b)
}
func (m *RedundancyScheme) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_RedundancyScheme.Marshal(b, m, deterministic)
}
func (dst *RedundancyScheme) XXX_Merge(src proto.Message) {
xxx_messageInfo_RedundancyScheme.Merge(dst, src)
}
func (m *RedundancyScheme) XXX_Size() int {
return xxx_messageInfo_RedundancyScheme.Size(m)
}
func (m *RedundancyScheme) XXX_DiscardUnknown() {
xxx_messageInfo_RedundancyScheme.DiscardUnknown(m)
}
var xxx_messageInfo_RedundancyScheme proto.InternalMessageInfo
func (m *RedundancyScheme) GetType() RedundancyScheme_SchemeType {
if m != nil {
@ -167,15 +170,37 @@ func (m *RedundancyScheme) GetErasureShareSize() int32 {
}
type EncryptionScheme struct {
Type EncryptionScheme_EncryptionType `protobuf:"varint,1,opt,name=type,enum=pointerdb.EncryptionScheme_EncryptionType" json:"type,omitempty"`
Type EncryptionScheme_EncryptionType `protobuf:"varint,1,opt,name=type,proto3,enum=pointerdb.EncryptionScheme_EncryptionType" json:"type,omitempty"`
EncryptedEncryptionKey []byte `protobuf:"bytes,2,opt,name=encrypted_encryption_key,json=encryptedEncryptionKey,proto3" json:"encrypted_encryption_key,omitempty"`
EncryptedStartingNonce []byte `protobuf:"bytes,3,opt,name=encrypted_starting_nonce,json=encryptedStartingNonce,proto3" json:"encrypted_starting_nonce,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *EncryptionScheme) Reset() { *m = EncryptionScheme{} }
func (m *EncryptionScheme) String() string { return proto.CompactTextString(m) }
func (*EncryptionScheme) ProtoMessage() {}
func (*EncryptionScheme) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
func (m *EncryptionScheme) Reset() { *m = EncryptionScheme{} }
func (m *EncryptionScheme) String() string { return proto.CompactTextString(m) }
func (*EncryptionScheme) ProtoMessage() {}
func (*EncryptionScheme) Descriptor() ([]byte, []int) {
return fileDescriptor_pointerdb_138b3541b1f95735, []int{1}
}
func (m *EncryptionScheme) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_EncryptionScheme.Unmarshal(m, b)
}
func (m *EncryptionScheme) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_EncryptionScheme.Marshal(b, m, deterministic)
}
func (dst *EncryptionScheme) XXX_Merge(src proto.Message) {
xxx_messageInfo_EncryptionScheme.Merge(dst, src)
}
func (m *EncryptionScheme) XXX_Size() int {
return xxx_messageInfo_EncryptionScheme.Size(m)
}
func (m *EncryptionScheme) XXX_DiscardUnknown() {
xxx_messageInfo_EncryptionScheme.DiscardUnknown(m)
}
var xxx_messageInfo_EncryptionScheme proto.InternalMessageInfo
func (m *EncryptionScheme) GetType() EncryptionScheme_EncryptionType {
if m != nil {
@ -199,14 +224,36 @@ func (m *EncryptionScheme) GetEncryptedStartingNonce() []byte {
}
type RemotePiece struct {
PieceNum int32 `protobuf:"varint,1,opt,name=piece_num,json=pieceNum" json:"piece_num,omitempty"`
NodeId string `protobuf:"bytes,2,opt,name=node_id,json=nodeId" json:"node_id,omitempty"`
PieceNum int32 `protobuf:"varint,1,opt,name=piece_num,json=pieceNum,proto3" json:"piece_num,omitempty"`
NodeId string `protobuf:"bytes,2,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *RemotePiece) Reset() { *m = RemotePiece{} }
func (m *RemotePiece) String() string { return proto.CompactTextString(m) }
func (*RemotePiece) ProtoMessage() {}
func (*RemotePiece) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
func (m *RemotePiece) Reset() { *m = RemotePiece{} }
func (m *RemotePiece) String() string { return proto.CompactTextString(m) }
func (*RemotePiece) ProtoMessage() {}
func (*RemotePiece) Descriptor() ([]byte, []int) {
return fileDescriptor_pointerdb_138b3541b1f95735, []int{2}
}
func (m *RemotePiece) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_RemotePiece.Unmarshal(m, b)
}
func (m *RemotePiece) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_RemotePiece.Marshal(b, m, deterministic)
}
func (dst *RemotePiece) XXX_Merge(src proto.Message) {
xxx_messageInfo_RemotePiece.Merge(dst, src)
}
func (m *RemotePiece) XXX_Size() int {
return xxx_messageInfo_RemotePiece.Size(m)
}
func (m *RemotePiece) XXX_DiscardUnknown() {
xxx_messageInfo_RemotePiece.DiscardUnknown(m)
}
var xxx_messageInfo_RemotePiece proto.InternalMessageInfo
func (m *RemotePiece) GetPieceNum() int32 {
if m != nil {
@ -223,16 +270,38 @@ func (m *RemotePiece) GetNodeId() string {
}
type RemoteSegment struct {
Redundancy *RedundancyScheme `protobuf:"bytes,1,opt,name=redundancy" json:"redundancy,omitempty"`
PieceId string `protobuf:"bytes,2,opt,name=piece_id,json=pieceId" json:"piece_id,omitempty"`
RemotePieces []*RemotePiece `protobuf:"bytes,3,rep,name=remote_pieces,json=remotePieces" json:"remote_pieces,omitempty"`
MerkleRoot []byte `protobuf:"bytes,4,opt,name=merkle_root,json=merkleRoot,proto3" json:"merkle_root,omitempty"`
Redundancy *RedundancyScheme `protobuf:"bytes,1,opt,name=redundancy,proto3" json:"redundancy,omitempty"`
PieceId string `protobuf:"bytes,2,opt,name=piece_id,json=pieceId,proto3" json:"piece_id,omitempty"`
RemotePieces []*RemotePiece `protobuf:"bytes,3,rep,name=remote_pieces,json=remotePieces,proto3" json:"remote_pieces,omitempty"`
MerkleRoot []byte `protobuf:"bytes,4,opt,name=merkle_root,json=merkleRoot,proto3" json:"merkle_root,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *RemoteSegment) Reset() { *m = RemoteSegment{} }
func (m *RemoteSegment) String() string { return proto.CompactTextString(m) }
func (*RemoteSegment) ProtoMessage() {}
func (*RemoteSegment) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
func (m *RemoteSegment) Reset() { *m = RemoteSegment{} }
func (m *RemoteSegment) String() string { return proto.CompactTextString(m) }
func (*RemoteSegment) ProtoMessage() {}
func (*RemoteSegment) Descriptor() ([]byte, []int) {
return fileDescriptor_pointerdb_138b3541b1f95735, []int{3}
}
func (m *RemoteSegment) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_RemoteSegment.Unmarshal(m, b)
}
func (m *RemoteSegment) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_RemoteSegment.Marshal(b, m, deterministic)
}
func (dst *RemoteSegment) XXX_Merge(src proto.Message) {
xxx_messageInfo_RemoteSegment.Merge(dst, src)
}
func (m *RemoteSegment) XXX_Size() int {
return xxx_messageInfo_RemoteSegment.Size(m)
}
func (m *RemoteSegment) XXX_DiscardUnknown() {
xxx_messageInfo_RemoteSegment.DiscardUnknown(m)
}
var xxx_messageInfo_RemoteSegment proto.InternalMessageInfo
func (m *RemoteSegment) GetRedundancy() *RedundancyScheme {
if m != nil {
@ -263,19 +332,41 @@ func (m *RemoteSegment) GetMerkleRoot() []byte {
}
type Pointer struct {
Type Pointer_DataType `protobuf:"varint,1,opt,name=type,enum=pointerdb.Pointer_DataType" json:"type,omitempty"`
InlineSegment []byte `protobuf:"bytes,3,opt,name=inline_segment,json=inlineSegment,proto3" json:"inline_segment,omitempty"`
Remote *RemoteSegment `protobuf:"bytes,4,opt,name=remote" json:"remote,omitempty"`
Size int64 `protobuf:"varint,5,opt,name=size" json:"size,omitempty"`
CreationDate *google_protobuf.Timestamp `protobuf:"bytes,6,opt,name=creation_date,json=creationDate" json:"creation_date,omitempty"`
ExpirationDate *google_protobuf.Timestamp `protobuf:"bytes,7,opt,name=expiration_date,json=expirationDate" json:"expiration_date,omitempty"`
Metadata []byte `protobuf:"bytes,8,opt,name=metadata,proto3" json:"metadata,omitempty"`
Type Pointer_DataType `protobuf:"varint,1,opt,name=type,proto3,enum=pointerdb.Pointer_DataType" json:"type,omitempty"`
InlineSegment []byte `protobuf:"bytes,3,opt,name=inline_segment,json=inlineSegment,proto3" json:"inline_segment,omitempty"`
Remote *RemoteSegment `protobuf:"bytes,4,opt,name=remote,proto3" json:"remote,omitempty"`
Size int64 `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"`
CreationDate *timestamp.Timestamp `protobuf:"bytes,6,opt,name=creation_date,json=creationDate,proto3" json:"creation_date,omitempty"`
ExpirationDate *timestamp.Timestamp `protobuf:"bytes,7,opt,name=expiration_date,json=expirationDate,proto3" json:"expiration_date,omitempty"`
Metadata []byte `protobuf:"bytes,8,opt,name=metadata,proto3" json:"metadata,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Pointer) Reset() { *m = Pointer{} }
func (m *Pointer) String() string { return proto.CompactTextString(m) }
func (*Pointer) ProtoMessage() {}
func (*Pointer) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
func (m *Pointer) Reset() { *m = Pointer{} }
func (m *Pointer) String() string { return proto.CompactTextString(m) }
func (*Pointer) ProtoMessage() {}
func (*Pointer) Descriptor() ([]byte, []int) {
return fileDescriptor_pointerdb_138b3541b1f95735, []int{4}
}
func (m *Pointer) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_Pointer.Unmarshal(m, b)
}
func (m *Pointer) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_Pointer.Marshal(b, m, deterministic)
}
func (dst *Pointer) XXX_Merge(src proto.Message) {
xxx_messageInfo_Pointer.Merge(dst, src)
}
func (m *Pointer) XXX_Size() int {
return xxx_messageInfo_Pointer.Size(m)
}
func (m *Pointer) XXX_DiscardUnknown() {
xxx_messageInfo_Pointer.DiscardUnknown(m)
}
var xxx_messageInfo_Pointer proto.InternalMessageInfo
func (m *Pointer) GetType() Pointer_DataType {
if m != nil {
@ -305,14 +396,14 @@ func (m *Pointer) GetSize() int64 {
return 0
}
func (m *Pointer) GetCreationDate() *google_protobuf.Timestamp {
func (m *Pointer) GetCreationDate() *timestamp.Timestamp {
if m != nil {
return m.CreationDate
}
return nil
}
func (m *Pointer) GetExpirationDate() *google_protobuf.Timestamp {
func (m *Pointer) GetExpirationDate() *timestamp.Timestamp {
if m != nil {
return m.ExpirationDate
}
@ -328,15 +419,37 @@ func (m *Pointer) GetMetadata() []byte {
// PutRequest is a request message for the Put rpc call
type PutRequest struct {
Path string `protobuf:"bytes,1,opt,name=path" json:"path,omitempty"`
Pointer *Pointer `protobuf:"bytes,2,opt,name=pointer" json:"pointer,omitempty"`
APIKey []byte `protobuf:"bytes,3,opt,name=API_key,json=APIKey,proto3" json:"API_key,omitempty"`
Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
Pointer *Pointer `protobuf:"bytes,2,opt,name=pointer,proto3" json:"pointer,omitempty"`
APIKey []byte `protobuf:"bytes,3,opt,name=API_key,json=APIKey,proto3" json:"API_key,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *PutRequest) Reset() { *m = PutRequest{} }
func (m *PutRequest) String() string { return proto.CompactTextString(m) }
func (*PutRequest) ProtoMessage() {}
func (*PutRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
func (m *PutRequest) Reset() { *m = PutRequest{} }
func (m *PutRequest) String() string { return proto.CompactTextString(m) }
func (*PutRequest) ProtoMessage() {}
func (*PutRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_pointerdb_138b3541b1f95735, []int{5}
}
func (m *PutRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_PutRequest.Unmarshal(m, b)
}
func (m *PutRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_PutRequest.Marshal(b, m, deterministic)
}
func (dst *PutRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_PutRequest.Merge(dst, src)
}
func (m *PutRequest) XXX_Size() int {
return xxx_messageInfo_PutRequest.Size(m)
}
func (m *PutRequest) XXX_DiscardUnknown() {
xxx_messageInfo_PutRequest.DiscardUnknown(m)
}
var xxx_messageInfo_PutRequest proto.InternalMessageInfo
func (m *PutRequest) GetPath() string {
if m != nil {
@ -361,14 +474,36 @@ func (m *PutRequest) GetAPIKey() []byte {
// GetRequest is a request message for the Get rpc call
type GetRequest struct {
Path string `protobuf:"bytes,1,opt,name=path" json:"path,omitempty"`
APIKey []byte `protobuf:"bytes,2,opt,name=API_key,json=APIKey,proto3" json:"API_key,omitempty"`
Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
APIKey []byte `protobuf:"bytes,2,opt,name=API_key,json=APIKey,proto3" json:"API_key,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *GetRequest) Reset() { *m = GetRequest{} }
func (m *GetRequest) String() string { return proto.CompactTextString(m) }
func (*GetRequest) ProtoMessage() {}
func (*GetRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
func (m *GetRequest) Reset() { *m = GetRequest{} }
func (m *GetRequest) String() string { return proto.CompactTextString(m) }
func (*GetRequest) ProtoMessage() {}
func (*GetRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_pointerdb_138b3541b1f95735, []int{6}
}
func (m *GetRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_GetRequest.Unmarshal(m, b)
}
func (m *GetRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_GetRequest.Marshal(b, m, deterministic)
}
func (dst *GetRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_GetRequest.Merge(dst, src)
}
func (m *GetRequest) XXX_Size() int {
return xxx_messageInfo_GetRequest.Size(m)
}
func (m *GetRequest) XXX_DiscardUnknown() {
xxx_messageInfo_GetRequest.DiscardUnknown(m)
}
var xxx_messageInfo_GetRequest proto.InternalMessageInfo
func (m *GetRequest) GetPath() string {
if m != nil {
@ -386,19 +521,41 @@ func (m *GetRequest) GetAPIKey() []byte {
// ListRequest is a request message for the List rpc call
type ListRequest struct {
Prefix string `protobuf:"bytes,1,opt,name=prefix" json:"prefix,omitempty"`
StartAfter string `protobuf:"bytes,2,opt,name=start_after,json=startAfter" json:"start_after,omitempty"`
EndBefore string `protobuf:"bytes,3,opt,name=end_before,json=endBefore" json:"end_before,omitempty"`
Recursive bool `protobuf:"varint,4,opt,name=recursive" json:"recursive,omitempty"`
Limit int32 `protobuf:"varint,5,opt,name=limit" json:"limit,omitempty"`
MetaFlags uint32 `protobuf:"fixed32,6,opt,name=meta_flags,json=metaFlags" json:"meta_flags,omitempty"`
APIKey []byte `protobuf:"bytes,7,opt,name=API_key,json=APIKey,proto3" json:"API_key,omitempty"`
Prefix string `protobuf:"bytes,1,opt,name=prefix,proto3" json:"prefix,omitempty"`
StartAfter string `protobuf:"bytes,2,opt,name=start_after,json=startAfter,proto3" json:"start_after,omitempty"`
EndBefore string `protobuf:"bytes,3,opt,name=end_before,json=endBefore,proto3" json:"end_before,omitempty"`
Recursive bool `protobuf:"varint,4,opt,name=recursive,proto3" json:"recursive,omitempty"`
Limit int32 `protobuf:"varint,5,opt,name=limit,proto3" json:"limit,omitempty"`
MetaFlags uint32 `protobuf:"fixed32,6,opt,name=meta_flags,json=metaFlags,proto3" json:"meta_flags,omitempty"`
APIKey []byte `protobuf:"bytes,7,opt,name=API_key,json=APIKey,proto3" json:"API_key,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *ListRequest) Reset() { *m = ListRequest{} }
func (m *ListRequest) String() string { return proto.CompactTextString(m) }
func (*ListRequest) ProtoMessage() {}
func (*ListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
func (m *ListRequest) Reset() { *m = ListRequest{} }
func (m *ListRequest) String() string { return proto.CompactTextString(m) }
func (*ListRequest) ProtoMessage() {}
func (*ListRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_pointerdb_138b3541b1f95735, []int{7}
}
func (m *ListRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_ListRequest.Unmarshal(m, b)
}
func (m *ListRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_ListRequest.Marshal(b, m, deterministic)
}
func (dst *ListRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_ListRequest.Merge(dst, src)
}
func (m *ListRequest) XXX_Size() int {
return xxx_messageInfo_ListRequest.Size(m)
}
func (m *ListRequest) XXX_DiscardUnknown() {
xxx_messageInfo_ListRequest.DiscardUnknown(m)
}
var xxx_messageInfo_ListRequest proto.InternalMessageInfo
func (m *ListRequest) GetPrefix() string {
if m != nil {
@ -451,22 +608,66 @@ func (m *ListRequest) GetAPIKey() []byte {
// PutResponse is a response message for the Put rpc call
type PutResponse struct {
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *PutResponse) Reset() { *m = PutResponse{} }
func (m *PutResponse) String() string { return proto.CompactTextString(m) }
func (*PutResponse) ProtoMessage() {}
func (*PutResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
func (m *PutResponse) Reset() { *m = PutResponse{} }
func (m *PutResponse) String() string { return proto.CompactTextString(m) }
func (*PutResponse) ProtoMessage() {}
func (*PutResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_pointerdb_138b3541b1f95735, []int{8}
}
func (m *PutResponse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_PutResponse.Unmarshal(m, b)
}
func (m *PutResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_PutResponse.Marshal(b, m, deterministic)
}
func (dst *PutResponse) XXX_Merge(src proto.Message) {
xxx_messageInfo_PutResponse.Merge(dst, src)
}
func (m *PutResponse) XXX_Size() int {
return xxx_messageInfo_PutResponse.Size(m)
}
func (m *PutResponse) XXX_DiscardUnknown() {
xxx_messageInfo_PutResponse.DiscardUnknown(m)
}
var xxx_messageInfo_PutResponse proto.InternalMessageInfo
// GetResponse is a response message for the Get rpc call
type GetResponse struct {
Pointer []byte `protobuf:"bytes,1,opt,name=pointer,proto3" json:"pointer,omitempty"`
Pointer []byte `protobuf:"bytes,1,opt,name=pointer,proto3" json:"pointer,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *GetResponse) Reset() { *m = GetResponse{} }
func (m *GetResponse) String() string { return proto.CompactTextString(m) }
func (*GetResponse) ProtoMessage() {}
func (*GetResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
func (m *GetResponse) Reset() { *m = GetResponse{} }
func (m *GetResponse) String() string { return proto.CompactTextString(m) }
func (*GetResponse) ProtoMessage() {}
func (*GetResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_pointerdb_138b3541b1f95735, []int{9}
}
func (m *GetResponse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_GetResponse.Unmarshal(m, b)
}
func (m *GetResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_GetResponse.Marshal(b, m, deterministic)
}
func (dst *GetResponse) XXX_Merge(src proto.Message) {
xxx_messageInfo_GetResponse.Merge(dst, src)
}
func (m *GetResponse) XXX_Size() int {
return xxx_messageInfo_GetResponse.Size(m)
}
func (m *GetResponse) XXX_DiscardUnknown() {
xxx_messageInfo_GetResponse.DiscardUnknown(m)
}
var xxx_messageInfo_GetResponse proto.InternalMessageInfo
func (m *GetResponse) GetPointer() []byte {
if m != nil {
@ -477,14 +678,36 @@ func (m *GetResponse) GetPointer() []byte {
// ListResponse is a response message for the List rpc call
type ListResponse struct {
Items []*ListResponse_Item `protobuf:"bytes,1,rep,name=items" json:"items,omitempty"`
More bool `protobuf:"varint,2,opt,name=more" json:"more,omitempty"`
Items []*ListResponse_Item `protobuf:"bytes,1,rep,name=items,proto3" json:"items,omitempty"`
More bool `protobuf:"varint,2,opt,name=more,proto3" json:"more,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *ListResponse) Reset() { *m = ListResponse{} }
func (m *ListResponse) String() string { return proto.CompactTextString(m) }
func (*ListResponse) ProtoMessage() {}
func (*ListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
func (m *ListResponse) Reset() { *m = ListResponse{} }
func (m *ListResponse) String() string { return proto.CompactTextString(m) }
func (*ListResponse) ProtoMessage() {}
func (*ListResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_pointerdb_138b3541b1f95735, []int{10}
}
func (m *ListResponse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_ListResponse.Unmarshal(m, b)
}
func (m *ListResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_ListResponse.Marshal(b, m, deterministic)
}
func (dst *ListResponse) XXX_Merge(src proto.Message) {
xxx_messageInfo_ListResponse.Merge(dst, src)
}
func (m *ListResponse) XXX_Size() int {
return xxx_messageInfo_ListResponse.Size(m)
}
func (m *ListResponse) XXX_DiscardUnknown() {
xxx_messageInfo_ListResponse.DiscardUnknown(m)
}
var xxx_messageInfo_ListResponse proto.InternalMessageInfo
func (m *ListResponse) GetItems() []*ListResponse_Item {
if m != nil {
@ -501,14 +724,37 @@ func (m *ListResponse) GetMore() bool {
}
type ListResponse_Item struct {
Path string `protobuf:"bytes,1,opt,name=path" json:"path,omitempty"`
Pointer *Pointer `protobuf:"bytes,2,opt,name=pointer" json:"pointer,omitempty"`
Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
Pointer *Pointer `protobuf:"bytes,2,opt,name=pointer,proto3" json:"pointer,omitempty"`
IsPrefix bool `protobuf:"varint,3,opt,name=is_prefix,json=isPrefix,proto3" json:"is_prefix,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *ListResponse_Item) Reset() { *m = ListResponse_Item{} }
func (m *ListResponse_Item) String() string { return proto.CompactTextString(m) }
func (*ListResponse_Item) ProtoMessage() {}
func (*ListResponse_Item) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10, 0} }
func (m *ListResponse_Item) Reset() { *m = ListResponse_Item{} }
func (m *ListResponse_Item) String() string { return proto.CompactTextString(m) }
func (*ListResponse_Item) ProtoMessage() {}
func (*ListResponse_Item) Descriptor() ([]byte, []int) {
return fileDescriptor_pointerdb_138b3541b1f95735, []int{10, 0}
}
func (m *ListResponse_Item) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_ListResponse_Item.Unmarshal(m, b)
}
func (m *ListResponse_Item) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_ListResponse_Item.Marshal(b, m, deterministic)
}
func (dst *ListResponse_Item) XXX_Merge(src proto.Message) {
xxx_messageInfo_ListResponse_Item.Merge(dst, src)
}
func (m *ListResponse_Item) XXX_Size() int {
return xxx_messageInfo_ListResponse_Item.Size(m)
}
func (m *ListResponse_Item) XXX_DiscardUnknown() {
xxx_messageInfo_ListResponse_Item.DiscardUnknown(m)
}
var xxx_messageInfo_ListResponse_Item proto.InternalMessageInfo
func (m *ListResponse_Item) GetPath() string {
if m != nil {
@ -524,15 +770,44 @@ func (m *ListResponse_Item) GetPointer() *Pointer {
return nil
}
type DeleteRequest struct {
Path string `protobuf:"bytes,1,opt,name=path" json:"path,omitempty"`
APIKey []byte `protobuf:"bytes,2,opt,name=API_key,json=APIKey,proto3" json:"API_key,omitempty"`
func (m *ListResponse_Item) GetIsPrefix() bool {
if m != nil {
return m.IsPrefix
}
return false
}
func (m *DeleteRequest) Reset() { *m = DeleteRequest{} }
func (m *DeleteRequest) String() string { return proto.CompactTextString(m) }
func (*DeleteRequest) ProtoMessage() {}
func (*DeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
type DeleteRequest struct {
Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
APIKey []byte `protobuf:"bytes,2,opt,name=API_key,json=APIKey,proto3" json:"API_key,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *DeleteRequest) Reset() { *m = DeleteRequest{} }
func (m *DeleteRequest) String() string { return proto.CompactTextString(m) }
func (*DeleteRequest) ProtoMessage() {}
func (*DeleteRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_pointerdb_138b3541b1f95735, []int{11}
}
func (m *DeleteRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_DeleteRequest.Unmarshal(m, b)
}
func (m *DeleteRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_DeleteRequest.Marshal(b, m, deterministic)
}
func (dst *DeleteRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_DeleteRequest.Merge(dst, src)
}
func (m *DeleteRequest) XXX_Size() int {
return xxx_messageInfo_DeleteRequest.Size(m)
}
func (m *DeleteRequest) XXX_DiscardUnknown() {
xxx_messageInfo_DeleteRequest.DiscardUnknown(m)
}
var xxx_messageInfo_DeleteRequest proto.InternalMessageInfo
func (m *DeleteRequest) GetPath() string {
if m != nil {
@ -550,12 +825,34 @@ func (m *DeleteRequest) GetAPIKey() []byte {
// DeleteResponse is a response message for the Delete rpc call
type DeleteResponse struct {
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *DeleteResponse) Reset() { *m = DeleteResponse{} }
func (m *DeleteResponse) String() string { return proto.CompactTextString(m) }
func (*DeleteResponse) ProtoMessage() {}
func (*DeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
func (m *DeleteResponse) Reset() { *m = DeleteResponse{} }
func (m *DeleteResponse) String() string { return proto.CompactTextString(m) }
func (*DeleteResponse) ProtoMessage() {}
func (*DeleteResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_pointerdb_138b3541b1f95735, []int{12}
}
func (m *DeleteResponse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_DeleteResponse.Unmarshal(m, b)
}
func (m *DeleteResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_DeleteResponse.Marshal(b, m, deterministic)
}
func (dst *DeleteResponse) XXX_Merge(src proto.Message) {
xxx_messageInfo_DeleteResponse.Merge(dst, src)
}
func (m *DeleteResponse) XXX_Size() int {
return xxx_messageInfo_DeleteResponse.Size(m)
}
func (m *DeleteResponse) XXX_DiscardUnknown() {
xxx_messageInfo_DeleteResponse.DiscardUnknown(m)
}
var xxx_messageInfo_DeleteResponse proto.InternalMessageInfo
func init() {
proto.RegisterType((*RedundancyScheme)(nil), "pointerdb.RedundancyScheme")
@ -585,8 +882,9 @@ var _ grpc.ClientConn
// is compatible with the grpc package it is being compiled against.
const _ = grpc.SupportPackageIsVersion4
// Client API for PointerDB service
// PointerDBClient is the client API for PointerDB service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
type PointerDBClient interface {
// Put formats and hands off a file path to be saved to boltdb
Put(ctx context.Context, in *PutRequest, opts ...grpc.CallOption) (*PutResponse, error)
@ -608,7 +906,7 @@ func NewPointerDBClient(cc *grpc.ClientConn) PointerDBClient {
func (c *pointerDBClient) Put(ctx context.Context, in *PutRequest, opts ...grpc.CallOption) (*PutResponse, error) {
out := new(PutResponse)
err := grpc.Invoke(ctx, "/pointerdb.PointerDB/Put", in, out, c.cc, opts...)
err := c.cc.Invoke(ctx, "/pointerdb.PointerDB/Put", in, out, opts...)
if err != nil {
return nil, err
}
@ -617,7 +915,7 @@ func (c *pointerDBClient) Put(ctx context.Context, in *PutRequest, opts ...grpc.
func (c *pointerDBClient) Get(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetResponse, error) {
out := new(GetResponse)
err := grpc.Invoke(ctx, "/pointerdb.PointerDB/Get", in, out, c.cc, opts...)
err := c.cc.Invoke(ctx, "/pointerdb.PointerDB/Get", in, out, opts...)
if err != nil {
return nil, err
}
@ -626,7 +924,7 @@ func (c *pointerDBClient) Get(ctx context.Context, in *GetRequest, opts ...grpc.
func (c *pointerDBClient) List(ctx context.Context, in *ListRequest, opts ...grpc.CallOption) (*ListResponse, error) {
out := new(ListResponse)
err := grpc.Invoke(ctx, "/pointerdb.PointerDB/List", in, out, c.cc, opts...)
err := c.cc.Invoke(ctx, "/pointerdb.PointerDB/List", in, out, opts...)
if err != nil {
return nil, err
}
@ -635,15 +933,14 @@ func (c *pointerDBClient) List(ctx context.Context, in *ListRequest, opts ...grp
func (c *pointerDBClient) Delete(ctx context.Context, in *DeleteRequest, opts ...grpc.CallOption) (*DeleteResponse, error) {
out := new(DeleteResponse)
err := grpc.Invoke(ctx, "/pointerdb.PointerDB/Delete", in, out, c.cc, opts...)
err := c.cc.Invoke(ctx, "/pointerdb.PointerDB/Delete", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// Server API for PointerDB service
// PointerDBServer is the server API for PointerDB service.
type PointerDBServer interface {
// Put formats and hands off a file path to be saved to boltdb
Put(context.Context, *PutRequest) (*PutResponse, error)
@ -756,68 +1053,70 @@ var _PointerDB_serviceDesc = grpc.ServiceDesc{
Metadata: "pointerdb.proto",
}
func init() { proto.RegisterFile("pointerdb.proto", fileDescriptor0) }
func init() { proto.RegisterFile("pointerdb.proto", fileDescriptor_pointerdb_138b3541b1f95735) }
var fileDescriptor0 = []byte{
// 960 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x54, 0xdd, 0x8e, 0xdb, 0x44,
0x14, 0x5e, 0x27, 0xbb, 0xce, 0xfa, 0x78, 0x93, 0x35, 0xa3, 0x92, 0xba, 0x69, 0x11, 0x2b, 0x4b,
0xc0, 0x42, 0xab, 0x80, 0x42, 0x25, 0x7e, 0xca, 0x8f, 0xf6, 0x27, 0x2c, 0x51, 0xdb, 0x6d, 0x34,
0xc9, 0x05, 0x77, 0x96, 0x37, 0x3e, 0x49, 0x46, 0x8d, 0x7f, 0x76, 0x3c, 0x46, 0x4d, 0xdf, 0x84,
0x3b, 0x5e, 0x86, 0x07, 0xe0, 0x41, 0x10, 0x17, 0xbc, 0x00, 0x9a, 0x19, 0x27, 0xb6, 0x77, 0xa1,
0x48, 0xf4, 0x26, 0x99, 0xf3, 0xcd, 0x77, 0xce, 0xcc, 0xf9, 0xce, 0x37, 0x86, 0xc3, 0x34, 0x61,
0xb1, 0x40, 0x1e, 0x5e, 0xf5, 0x53, 0x9e, 0x88, 0x84, 0x58, 0x5b, 0xa0, 0x77, 0x28, 0x58, 0x84,
0x99, 0x08, 0xa2, 0x54, 0xef, 0x79, 0xbf, 0x34, 0xc0, 0xa1, 0x18, 0xe6, 0x71, 0x18, 0xc4, 0xb3,
0xf5, 0x64, 0xb6, 0xc4, 0x08, 0xc9, 0xd7, 0xb0, 0x2b, 0xd6, 0x29, 0xba, 0xc6, 0x91, 0x71, 0xdc,
0x19, 0x7c, 0xd8, 0x2f, 0x0b, 0xde, 0xa4, 0xf6, 0xf5, 0xdf, 0x74, 0x9d, 0x22, 0x55, 0x39, 0xe4,
0x2e, 0xb4, 0x22, 0x16, 0xfb, 0x1c, 0xaf, 0xdd, 0xc6, 0x91, 0x71, 0xbc, 0x47, 0xcd, 0x88, 0xc5,
0x14, 0xaf, 0xc9, 0x1d, 0xd8, 0x13, 0x89, 0x08, 0x56, 0x6e, 0x53, 0xc1, 0x3a, 0x20, 0x1f, 0x83,
0xc3, 0x31, 0x0d, 0x18, 0xf7, 0xc5, 0x92, 0x63, 0xb6, 0x4c, 0x56, 0xa1, 0xbb, 0xab, 0x08, 0x87,
0x1a, 0x9f, 0x6e, 0x60, 0xf2, 0x10, 0xde, 0xc9, 0xf2, 0xd9, 0x0c, 0xb3, 0xac, 0xc2, 0xdd, 0x53,
0x5c, 0xa7, 0xd8, 0x28, 0xc9, 0x8f, 0x80, 0x20, 0x0f, 0xb2, 0x9c, 0xa3, 0x9f, 0x2d, 0x03, 0xf9,
0xcb, 0x5e, 0xa3, 0x6b, 0x6a, 0x76, 0xb1, 0x33, 0x91, 0x1b, 0x13, 0xf6, 0x1a, 0xbd, 0x3b, 0x00,
0x65, 0x23, 0xc4, 0x84, 0x06, 0x9d, 0x38, 0x3b, 0xde, 0x5f, 0x06, 0x38, 0xc3, 0x78, 0xc6, 0xd7,
0xa9, 0x60, 0x49, 0x5c, 0x68, 0xf3, 0x5d, 0x4d, 0x9b, 0x4f, 0x2a, 0xda, 0xdc, 0xa4, 0x56, 0x80,
0x8a, 0x3e, 0x5f, 0x82, 0x8b, 0x1a, 0xc7, 0xd0, 0xc7, 0x2d, 0xc3, 0x7f, 0x89, 0x6b, 0x25, 0xd8,
0x01, 0xed, 0x6e, 0xf7, 0xcb, 0x02, 0x4f, 0x71, 0x5d, 0xcf, 0xcc, 0x44, 0xc0, 0x05, 0x8b, 0x17,
0x7e, 0x9c, 0xc4, 0x33, 0x54, 0x9a, 0x56, 0x33, 0x27, 0xc5, 0xf6, 0xa5, 0xdc, 0xf5, 0x1e, 0x42,
0xa7, 0x7e, 0x17, 0x02, 0x60, 0x9e, 0x0c, 0x27, 0x17, 0x67, 0xcf, 0x9d, 0x1d, 0xd2, 0x06, 0x6b,
0x32, 0x3c, 0xa3, 0xc3, 0xe9, 0xe9, 0x8b, 0x9f, 0x1c, 0xc3, 0x3b, 0x03, 0x9b, 0x62, 0x94, 0x08,
0x1c, 0x33, 0x9c, 0x21, 0xb9, 0x0f, 0x56, 0x2a, 0x17, 0x7e, 0x9c, 0x47, 0xaa, 0xe9, 0x3d, 0xba,
0xaf, 0x80, 0xcb, 0x3c, 0x92, 0xc3, 0x8e, 0x93, 0x10, 0x7d, 0x16, 0xaa, 0xbb, 0x5b, 0xd4, 0x94,
0xe1, 0x28, 0xf4, 0x7e, 0x33, 0xa0, 0xad, 0xab, 0x4c, 0x70, 0x11, 0x61, 0x2c, 0xc8, 0x13, 0x00,
0xbe, 0x35, 0x8f, 0x2a, 0x64, 0x0f, 0xee, 0xbf, 0xc1, 0x59, 0xb4, 0x42, 0x27, 0xf7, 0x40, 0x9f,
0x59, 0x1e, 0xd4, 0x52, 0xf1, 0x28, 0x24, 0x4f, 0xa0, 0xcd, 0xd5, 0x41, 0xbe, 0x42, 0x32, 0xb7,
0x79, 0xd4, 0x3c, 0xb6, 0x07, 0xdd, 0x5a, 0xe9, 0x6d, 0x3b, 0xf4, 0x80, 0x97, 0x41, 0x46, 0xde,
0x07, 0x3b, 0x42, 0xfe, 0x72, 0x85, 0x3e, 0x4f, 0x12, 0xa1, 0x8c, 0x77, 0x40, 0x41, 0x43, 0x34,
0x49, 0x84, 0xf7, 0x47, 0x03, 0x5a, 0x63, 0x5d, 0x88, 0x7c, 0x5a, 0x9b, 0x7c, 0xf5, 0xee, 0x05,
0xa3, 0x7f, 0x1e, 0x88, 0xa0, 0x32, 0xea, 0x0f, 0xa0, 0xc3, 0xe2, 0x15, 0x8b, 0xd1, 0xcf, 0xb4,
0x08, 0xc5, 0x98, 0xda, 0x1a, 0xdd, 0x28, 0xf3, 0x19, 0x98, 0xfa, 0x52, 0xea, 0x7c, 0x7b, 0xe0,
0xde, 0xba, 0x7a, 0xc1, 0xa4, 0x05, 0x8f, 0x10, 0xd8, 0x55, 0x76, 0x96, 0xe6, 0x6f, 0x52, 0xb5,
0x26, 0xdf, 0x43, 0x7b, 0xc6, 0x31, 0x50, 0x5e, 0x0a, 0x03, 0xa1, 0xbd, 0x6e, 0x0f, 0x7a, 0xfd,
0x45, 0x92, 0x2c, 0x56, 0xa8, 0x9f, 0xfb, 0x55, 0x3e, 0xef, 0x4f, 0x37, 0x5f, 0x00, 0x7a, 0xb0,
0x49, 0x38, 0x0f, 0x04, 0x92, 0x33, 0x38, 0xc4, 0x57, 0x29, 0xe3, 0x95, 0x12, 0xad, 0xff, 0x2c,
0xd1, 0x29, 0x53, 0x54, 0x91, 0x1e, 0xec, 0x47, 0x28, 0x82, 0x30, 0x10, 0x81, 0xbb, 0xaf, 0x9a,
0xdd, 0xc6, 0x9e, 0x07, 0xfb, 0x1b, 0x81, 0xa4, 0xff, 0x46, 0x97, 0xcf, 0x46, 0x97, 0x43, 0x67,
0x47, 0xae, 0xe9, 0xf0, 0xf9, 0x8b, 0xe9, 0xd0, 0x31, 0xbc, 0x05, 0xc0, 0x38, 0x17, 0x14, 0xaf,
0x73, 0xcc, 0x84, 0xec, 0x33, 0x0d, 0xc4, 0x52, 0x29, 0x6e, 0x51, 0xb5, 0x26, 0x8f, 0xa0, 0x55,
0xc8, 0xa3, 0x9c, 0x60, 0x0f, 0xc8, 0xed, 0x41, 0xd0, 0x0d, 0x45, 0x1a, 0xf4, 0x64, 0x3c, 0x52,
0x8f, 0x4b, 0x6b, 0x6f, 0x9e, 0x8c, 0x47, 0x4f, 0x71, 0xed, 0x7d, 0x05, 0x70, 0x81, 0x6f, 0x3c,
0xa8, 0x92, 0xda, 0xa8, 0xa5, 0xfe, 0x6e, 0x80, 0xfd, 0x8c, 0x65, 0xdb, 0xe4, 0x2e, 0x98, 0x29,
0xc7, 0x39, 0x7b, 0x55, 0xa4, 0x17, 0x91, 0x34, 0x97, 0x7a, 0xa5, 0x7e, 0x30, 0xdf, 0xdc, 0xd6,
0xa2, 0xa0, 0xa0, 0x13, 0x89, 0x90, 0xf7, 0x00, 0x30, 0x0e, 0xfd, 0x2b, 0x9c, 0x27, 0x5c, 0x3f,
0x61, 0x8b, 0x5a, 0x18, 0x87, 0xa7, 0x0a, 0x20, 0x0f, 0xc0, 0xe2, 0x38, 0xcb, 0x79, 0xc6, 0x7e,
0xd6, 0xd6, 0xd8, 0xa7, 0x25, 0x20, 0x3f, 0xa7, 0x2b, 0x16, 0x31, 0x51, 0x7c, 0x01, 0x75, 0x20,
0x4b, 0x4a, 0xbd, 0xfd, 0xf9, 0x2a, 0x58, 0x64, 0xca, 0x02, 0x2d, 0x6a, 0x49, 0xe4, 0x07, 0x09,
0x54, 0x7b, 0x6a, 0xd5, 0x7a, 0x6a, 0x83, 0xad, 0x74, 0xcf, 0xd2, 0x24, 0xce, 0xd0, 0xfb, 0x08,
0x6c, 0xa5, 0x8e, 0x0e, 0x89, 0x5b, 0x6a, 0x6e, 0xa8, 0xb4, 0x4d, 0xe8, 0xfd, 0x6a, 0xc0, 0x81,
0xd6, 0xa2, 0xa0, 0x0e, 0x60, 0x8f, 0x09, 0x8c, 0x32, 0xd7, 0x50, 0xcf, 0xf0, 0x41, 0x65, 0x38,
0x55, 0x5e, 0x7f, 0x24, 0x30, 0xa2, 0x9a, 0x2a, 0xd5, 0x8f, 0xa4, 0x02, 0x0d, 0xd5, 0xa3, 0x5a,
0xf7, 0x7e, 0x84, 0x5d, 0x49, 0x79, 0x7b, 0x0b, 0x78, 0xdf, 0x40, 0xfb, 0x1c, 0x57, 0x28, 0xf0,
0x7f, 0x0d, 0xdb, 0x81, 0xce, 0x26, 0x5b, 0xdf, 0x7c, 0xf0, 0xa7, 0x01, 0x56, 0x71, 0xc8, 0xf9,
0x29, 0x79, 0x0c, 0xcd, 0x71, 0x2e, 0xc8, 0xbb, 0xd5, 0x1b, 0x6c, 0x0d, 0xdc, 0xeb, 0xde, 0x84,
0x0b, 0x95, 0x1e, 0x43, 0xf3, 0x02, 0xeb, 0x59, 0xa5, 0x1b, 0x6b, 0x59, 0xd5, 0x31, 0x7c, 0x01,
0xbb, 0x52, 0x43, 0xd2, 0xbd, 0x25, 0xaa, 0xce, 0xbb, 0xfb, 0x2f, 0x62, 0x93, 0x6f, 0xc1, 0xd4,
0x4d, 0x90, 0xea, 0xb7, 0xa5, 0xa6, 0x4a, 0xef, 0xde, 0x3f, 0xec, 0xe8, 0xf4, 0x2b, 0x53, 0x3d,
0xfc, 0xcf, 0xff, 0x0e, 0x00, 0x00, 0xff, 0xff, 0x0d, 0x6e, 0x14, 0xa6, 0x59, 0x08, 0x00, 0x00,
var fileDescriptor_pointerdb_138b3541b1f95735 = []byte{
// 982 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xdb, 0x6e, 0xdb, 0x46,
0x10, 0x35, 0x25, 0x5b, 0x12, 0x47, 0x96, 0xcd, 0x2e, 0x52, 0x87, 0x91, 0x53, 0xc4, 0x20, 0xd0,
0xd6, 0x6d, 0x02, 0xb9, 0x50, 0x03, 0xf4, 0x92, 0x5e, 0xe0, 0x8b, 0x6a, 0x08, 0x49, 0x1c, 0x61,
0xe5, 0x87, 0xbe, 0x11, 0xb4, 0x38, 0x96, 0x16, 0x11, 0x2f, 0xde, 0x5d, 0x16, 0x51, 0xfe, 0xa4,
0x1f, 0xd3, 0xc7, 0x7e, 0x40, 0x3f, 0xa4, 0xe8, 0x43, 0x7f, 0xa0, 0xd8, 0x5d, 0x52, 0x24, 0xed,
0x36, 0x05, 0x8a, 0xbe, 0x48, 0x3b, 0x67, 0xcf, 0xcc, 0xee, 0x9c, 0x39, 0x5c, 0xd8, 0x4d, 0x13,
0x16, 0x4b, 0xe4, 0xe1, 0xd5, 0x20, 0xe5, 0x89, 0x4c, 0x88, 0xbd, 0x06, 0xfa, 0x8f, 0xe6, 0x49,
0x32, 0x5f, 0xe2, 0x91, 0xde, 0xb8, 0xca, 0xae, 0x8f, 0x24, 0x8b, 0x50, 0xc8, 0x20, 0x4a, 0x0d,
0xd7, 0xfb, 0xb9, 0x01, 0x0e, 0xc5, 0x30, 0x8b, 0xc3, 0x20, 0x9e, 0xad, 0xa6, 0xb3, 0x05, 0x46,
0x48, 0xbe, 0x86, 0x4d, 0xb9, 0x4a, 0xd1, 0xb5, 0x0e, 0xac, 0xc3, 0x9d, 0xe1, 0x47, 0x83, 0xf2,
0x80, 0xdb, 0xd4, 0x81, 0xf9, 0xbb, 0x5c, 0xa5, 0x48, 0x75, 0x0e, 0xb9, 0x0f, 0xed, 0x88, 0xc5,
0x3e, 0xc7, 0x1b, 0xb7, 0x71, 0x60, 0x1d, 0x6e, 0xd1, 0x56, 0xc4, 0x62, 0x8a, 0x37, 0xe4, 0x1e,
0x6c, 0xc9, 0x44, 0x06, 0x4b, 0xb7, 0xa9, 0x61, 0x13, 0x90, 0x4f, 0xc0, 0xe1, 0x98, 0x06, 0x8c,
0xfb, 0x72, 0xc1, 0x51, 0x2c, 0x92, 0x65, 0xe8, 0x6e, 0x6a, 0xc2, 0xae, 0xc1, 0x2f, 0x0b, 0x98,
0x3c, 0x86, 0xf7, 0x44, 0x36, 0x9b, 0xa1, 0x10, 0x15, 0xee, 0x96, 0xe6, 0x3a, 0xf9, 0x46, 0x49,
0x7e, 0x02, 0x04, 0x79, 0x20, 0x32, 0x8e, 0xbe, 0x58, 0x04, 0xea, 0x97, 0xbd, 0x45, 0xb7, 0x65,
0xd8, 0xf9, 0xce, 0x54, 0x6d, 0x4c, 0xd9, 0x5b, 0xf4, 0xee, 0x01, 0x94, 0x8d, 0x90, 0x16, 0x34,
0xe8, 0xd4, 0xd9, 0xf0, 0xfe, 0xb4, 0xc0, 0x19, 0xc5, 0x33, 0xbe, 0x4a, 0x25, 0x4b, 0xe2, 0x5c,
0x9b, 0xef, 0x6a, 0xda, 0x7c, 0x5a, 0xd1, 0xe6, 0x36, 0xb5, 0x02, 0x54, 0xf4, 0xf9, 0x12, 0x5c,
0x34, 0x38, 0x86, 0x3e, 0xae, 0x19, 0xfe, 0x6b, 0x5c, 0x69, 0xc1, 0xb6, 0xe9, 0xde, 0x7a, 0xbf,
0x2c, 0xf0, 0x1c, 0x57, 0xf5, 0x4c, 0x21, 0x03, 0x2e, 0x59, 0x3c, 0xf7, 0xe3, 0x24, 0x9e, 0xa1,
0xd6, 0xb4, 0x9a, 0x39, 0xcd, 0xb7, 0x2f, 0xd4, 0xae, 0xf7, 0x18, 0x76, 0xea, 0x77, 0x21, 0x00,
0xad, 0xe3, 0xd1, 0xf4, 0xfc, 0xf4, 0xa5, 0xb3, 0x41, 0x7a, 0x60, 0x4f, 0x47, 0xa7, 0x74, 0x74,
0x79, 0xf2, 0xea, 0x47, 0xc7, 0xf2, 0x4e, 0xa1, 0x4b, 0x31, 0x4a, 0x24, 0x4e, 0x18, 0xce, 0x90,
0xec, 0x83, 0x9d, 0xaa, 0x85, 0x1f, 0x67, 0x91, 0x6e, 0x7a, 0x8b, 0x76, 0x34, 0x70, 0x91, 0x45,
0x6a, 0xd8, 0x71, 0x12, 0xa2, 0xcf, 0x42, 0x7d, 0x77, 0x9b, 0xb6, 0x54, 0x38, 0x0e, 0xbd, 0x5f,
0x2d, 0xe8, 0x99, 0x2a, 0x53, 0x9c, 0x47, 0x18, 0x4b, 0xf2, 0x0c, 0x80, 0xaf, 0xcd, 0xa3, 0x0b,
0x75, 0x87, 0xfb, 0xef, 0x70, 0x16, 0xad, 0xd0, 0xc9, 0x03, 0x30, 0x67, 0x96, 0x07, 0xb5, 0x75,
0x3c, 0x0e, 0xc9, 0x33, 0xe8, 0x71, 0x7d, 0x90, 0xaf, 0x11, 0xe1, 0x36, 0x0f, 0x9a, 0x87, 0xdd,
0xe1, 0x5e, 0xad, 0xf4, 0xba, 0x1d, 0xba, 0xcd, 0xcb, 0x40, 0x90, 0x47, 0xd0, 0x8d, 0x90, 0xbf,
0x5e, 0xa2, 0xcf, 0x93, 0x44, 0x6a, 0xe3, 0x6d, 0x53, 0x30, 0x10, 0x4d, 0x12, 0xe9, 0xfd, 0xde,
0x80, 0xf6, 0xc4, 0x14, 0x22, 0x47, 0xb5, 0xc9, 0x57, 0xef, 0x9e, 0x33, 0x06, 0x67, 0x81, 0x0c,
0x2a, 0xa3, 0xfe, 0x10, 0x76, 0x58, 0xbc, 0x64, 0x31, 0xfa, 0xc2, 0x88, 0x90, 0x8f, 0xa9, 0x67,
0xd0, 0x42, 0x99, 0xcf, 0xa0, 0x65, 0x2e, 0xa5, 0xcf, 0xef, 0x0e, 0xdd, 0x3b, 0x57, 0xcf, 0x99,
0x34, 0xe7, 0x11, 0x02, 0x9b, 0xda, 0xce, 0xca, 0xfc, 0x4d, 0xaa, 0xd7, 0xe4, 0x7b, 0xe8, 0xcd,
0x38, 0x06, 0xda, 0x4b, 0x61, 0x20, 0x8d, 0xd7, 0xbb, 0xc3, 0xfe, 0xc0, 0xbc, 0x00, 0x83, 0xe2,
0x05, 0x18, 0x5c, 0x16, 0x2f, 0x00, 0xdd, 0x2e, 0x12, 0xce, 0x02, 0x89, 0xe4, 0x14, 0x76, 0xf1,
0x4d, 0xca, 0x78, 0xa5, 0x44, 0xfb, 0x5f, 0x4b, 0xec, 0x94, 0x29, 0xba, 0x48, 0x1f, 0x3a, 0x11,
0xca, 0x20, 0x0c, 0x64, 0xe0, 0x76, 0x74, 0xb3, 0xeb, 0xd8, 0xf3, 0xa0, 0x53, 0x08, 0xa4, 0xfc,
0x37, 0xbe, 0x78, 0x31, 0xbe, 0x18, 0x39, 0x1b, 0x6a, 0x4d, 0x47, 0x2f, 0x5f, 0x5d, 0x8e, 0x1c,
0xcb, 0x9b, 0x03, 0x4c, 0x32, 0x49, 0xf1, 0x26, 0x43, 0x21, 0x55, 0x9f, 0x69, 0x20, 0x17, 0x5a,
0x71, 0x9b, 0xea, 0x35, 0x79, 0x02, 0xed, 0x5c, 0x1e, 0xed, 0x84, 0xee, 0x90, 0xdc, 0x1d, 0x04,
0x2d, 0x28, 0xca, 0xa0, 0xc7, 0x93, 0xb1, 0xfe, 0xb8, 0x8c, 0xf6, 0xad, 0xe3, 0xc9, 0xf8, 0x39,
0xae, 0xbc, 0xaf, 0x00, 0xce, 0xf1, 0x9d, 0x07, 0x55, 0x52, 0x1b, 0xb5, 0xd4, 0xdf, 0x2c, 0xe8,
0xbe, 0x60, 0x62, 0x9d, 0xbc, 0x07, 0xad, 0x94, 0xe3, 0x35, 0x7b, 0x93, 0xa7, 0xe7, 0x91, 0x32,
0x97, 0xfe, 0x4a, 0xfd, 0xe0, 0xba, 0xb8, 0xad, 0x4d, 0x41, 0x43, 0xc7, 0x0a, 0x21, 0x1f, 0x00,
0x60, 0x1c, 0xfa, 0x57, 0x78, 0x9d, 0x70, 0xf3, 0x09, 0xdb, 0xd4, 0xc6, 0x38, 0x3c, 0xd1, 0x00,
0x79, 0x08, 0x36, 0xc7, 0x59, 0xc6, 0x05, 0xfb, 0xc9, 0x58, 0xa3, 0x43, 0x4b, 0x40, 0x3d, 0xa7,
0x4b, 0x16, 0x31, 0x99, 0xbf, 0x80, 0x26, 0x50, 0x25, 0x95, 0xde, 0xfe, 0xf5, 0x32, 0x98, 0x0b,
0x6d, 0x81, 0x36, 0xb5, 0x15, 0xf2, 0x83, 0x02, 0xaa, 0x3d, 0xb5, 0x6b, 0x3d, 0xf5, 0xa0, 0xab,
0x75, 0x17, 0x69, 0x12, 0x0b, 0xf4, 0x3e, 0x86, 0xae, 0x56, 0xc7, 0x84, 0xc4, 0x2d, 0x35, 0xb7,
0x74, 0x5a, 0x11, 0x7a, 0xbf, 0x58, 0xb0, 0x6d, 0xb4, 0xc8, 0xa9, 0x43, 0xd8, 0x62, 0x12, 0x23,
0xe1, 0x5a, 0xfa, 0x33, 0x7c, 0x58, 0x19, 0x4e, 0x95, 0x37, 0x18, 0x4b, 0x8c, 0xa8, 0xa1, 0x2a,
0xf5, 0x23, 0xa5, 0x40, 0x43, 0xf7, 0xa8, 0xd7, 0x7d, 0x84, 0x4d, 0x45, 0xf9, 0x1f, 0x2c, 0xb0,
0x0f, 0x36, 0x13, 0x7e, 0x3e, 0xa1, 0xa6, 0x3e, 0xa2, 0xc3, 0xc4, 0x44, 0xc7, 0xde, 0x37, 0xd0,
0x3b, 0xc3, 0x25, 0x4a, 0xfc, 0x4f, 0x4e, 0x70, 0x60, 0xa7, 0xc8, 0x36, 0x6d, 0x0d, 0xff, 0xb0,
0xc0, 0xce, 0x6f, 0x70, 0x76, 0x42, 0x9e, 0x42, 0x73, 0x92, 0x49, 0xf2, 0x7e, 0xf5, 0x7a, 0x6b,
0x77, 0xf7, 0xf7, 0x6e, 0xc3, 0xb9, 0x84, 0x4f, 0xa1, 0x79, 0x8e, 0xf5, 0xac, 0xd2, 0xaa, 0xb5,
0xac, 0xea, 0x8c, 0xbe, 0x80, 0x4d, 0x25, 0x30, 0xd9, 0xbb, 0xa3, 0xb8, 0xc9, 0xbb, 0xff, 0x0f,
0x93, 0x20, 0xdf, 0x42, 0xcb, 0x34, 0x41, 0xaa, 0x0f, 0x4f, 0x4d, 0x95, 0xfe, 0x83, 0xbf, 0xd9,
0x31, 0xe9, 0x57, 0x2d, 0xfd, 0x2a, 0x7c, 0xfe, 0x57, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x61,
0xbc, 0x67, 0x86, 0x08, 0x00, 0x00,
}

3
protos/pointerdb/pointerdb.proto
View File

@ -111,8 +111,9 @@ message GetResponse {
// ListResponse is a response message for the List rpc call
message ListResponse {
message Item {
string path = 1;
string path = 1;
Pointer pointer = 2;
bool is_prefix = 3;
}
repeated Item items = 1;

14
storage/boltdb/client.go
View File

@ -96,13 +96,13 @@ func (client *Client) Delete(key storage.Key) error {
}
// List returns either a list of keys for which boltdb has values or an error.
func (client *Client) List(first storage.Key, limit storage.Limit) (storage.Keys, error) {
func (client *Client) List(first storage.Key, limit int) (storage.Keys, error) {
return storage.ListKeys(client, first, limit)
}
// ReverseList returns either a list of keys for which boltdb has values or an error.
// Starts from first and iterates backwards
func (client *Client) ReverseList(first storage.Key, limit storage.Limit) (storage.Keys, error) {
func (client *Client) ReverseList(first storage.Key, limit int) (storage.Keys, error) {
return storage.ReverseListKeys(client, first, limit)
}
@ -161,7 +161,7 @@ func (client *Client) Iterate(opts storage.IterateOptions, fn func(storage.Itera
}
}
if key == nil || !bytes.HasPrefix(key, opts.Prefix) {
if len(key) == 0 || !bytes.HasPrefix(key, opts.Prefix) {
return false
}
@ -200,7 +200,7 @@ type forward struct {
}
func (cursor forward) PositionToFirst(prefix, first storage.Key) (key, value []byte) {
if first == nil || first.Less(prefix) {
if first.IsZero() || first.Less(prefix) {
return cursor.Seek([]byte(prefix))
}
return cursor.Seek([]byte(first))
@ -219,15 +219,15 @@ type backward struct {
}
func (cursor backward) PositionToFirst(prefix, first storage.Key) (key, value []byte) {
if prefix == nil {
if prefix.IsZero() {
// there's no prefix
if first == nil {
if first.IsZero() {
// and no first item, so start from the end
return cursor.Last()
}
} else {
// there's a prefix
if first == nil || storage.AfterPrefix(prefix).Less(first) {
if first.IsZero() || storage.AfterPrefix(prefix).Less(first) {
// there's no first, or it's after our prefix
// storage.AfterPrefix("axxx/") is the next item after prefixes
// so we position to the item before

7
storage/common.go
View File

@ -34,9 +34,6 @@ type Keys []Key
// Values is the type for a slice of Values in a `KeyValueStore`
type Values []Value
// Limit indicates how many keys to return when calling List
type Limit int
// Items keeps all ListItem
type Items []ListItem
@ -61,9 +58,9 @@ type KeyValueStore interface {
// Delete deletes key and the value
Delete(Key) error
// List lists all keys starting from start and upto limit items
List(start Key, limit Limit) (Keys, error)
List(start Key, limit int) (Keys, error)
// ReverseList lists all keys in revers order
ReverseList(Key, Limit) (Keys, error)
ReverseList(Key, int) (Keys, error)
// Iterate iterates over items based on opts
Iterate(opts IterateOptions, fn func(Iterator) error) error
// Close closes the store

4
storage/listkeys.go
View File

@ -5,7 +5,7 @@ package storage
// ListKeys returns keys starting from first and upto limit
// limit is capped to LookupLimit
func ListKeys(store KeyValueStore, first Key, limit Limit) (Keys, error) {
func ListKeys(store KeyValueStore, first Key, limit int) (Keys, error) {
if limit <= 0 || limit > LookupLimit {
limit = LookupLimit
}
@ -30,7 +30,7 @@ func ListKeys(store KeyValueStore, first Key, limit Limit) (Keys, error) {
// ReverseListKeys returns keys starting from first and upto limit in reverse order
// limit is capped to LookupLimit
func ReverseListKeys(store KeyValueStore, first Key, limit Limit) (Keys, error) {
func ReverseListKeys(store KeyValueStore, first Key, limit int) (Keys, error) {
if limit <= 0 || limit > LookupLimit {
limit = LookupLimit
}

42
storage/listv2.go
View File

@ -3,13 +3,9 @@
package storage
import "errors"
// More indicates if the result was truncated. If false
// then the result []ListItem includes all requested keys.
// If true then the caller must call List again to get more
// results by setting `StartAfter` or `EndBefore` appropriately.
type More bool
import (
"errors"
)
// ListOptions are items that are optional for the LIST method
type ListOptions struct {
@ -18,28 +14,29 @@ type ListOptions struct {
EndBefore Key // EndBefore is relative to Prefix
Recursive bool
IncludeValue bool
Limit Limit
Limit int
}
// ListV2 lists all keys corresponding to ListOptions
// limit is capped to LookupLimit
func ListV2(store KeyValueStore, opts ListOptions) (result Items, more More, err error) {
if opts.Limit <= 0 || opts.Limit > LookupLimit {
opts.Limit = LookupLimit
}
if opts.StartAfter != nil && opts.EndBefore != nil {
//
// more indicates if the result was truncated. If false
// then the result []ListItem includes all requested keys.
// If true then the caller must call List again to get more
// results by setting `StartAfter` or `EndBefore` appropriately.
func ListV2(store KeyValueStore, opts ListOptions) (result Items, more bool, err error) {
if !opts.StartAfter.IsZero() && !opts.EndBefore.IsZero() {
return nil, false, errors.New("start-after and end-before cannot be combined")
}
reverse := opts.EndBefore != nil
more = More(true)
limit := opts.Limit
if limit == 0 {
limit = Limit(1 << 31)
if limit <= 0 || limit > LookupLimit {
limit = LookupLimit
}
more = true
reverse := !opts.EndBefore.IsZero()
var first Key
if !reverse {
first = opts.StartAfter
@ -80,14 +77,17 @@ func ListV2(store KeyValueStore, opts ListOptions) (result Items, more More, err
})
}
}
// we still need to consume one item for the more flag
more = it.Next(&item)
return nil
}
var firstFull Key
if !reverse && opts.StartAfter != nil {
if !reverse && !opts.StartAfter.IsZero() {
firstFull = joinKey(opts.Prefix, opts.StartAfter)
}
if reverse && opts.EndBefore != nil {
if reverse && !opts.EndBefore.IsZero() {
firstFull = joinKey(opts.Prefix, opts.EndBefore)
}
err = store.Iterate(IterateOptions{

8
storage/redis/client.go
View File

@ -69,13 +69,13 @@ func (client *Client) Put(key storage.Key, value storage.Value) error {
}
// List returns either a list of keys for which boltdb has values or an error.
func (client *Client) List(first storage.Key, limit storage.Limit) (storage.Keys, error) {
func (client *Client) List(first storage.Key, limit int) (storage.Keys, error) {
return storage.ListKeys(client, first, limit)
}
// ReverseList returns either a list of keys for which redis has values or an error.
// Starts from first and iterates backwards
func (client *Client) ReverseList(first storage.Key, limit storage.Limit) (storage.Keys, error) {
func (client *Client) ReverseList(first storage.Key, limit int) (storage.Keys, error) {
return storage.ReverseListKeys(client, first, limit)
}
@ -153,10 +153,10 @@ func (client *Client) allPrefixedItems(prefix, first, last storage.Key) (storage
it := client.db.Scan(0, match, 0).Iterator()
for it.Next() {
key := it.Val()
if first != nil && storage.Key(key).Less(first) {
if !first.IsZero() && storage.Key(key).Less(first) {
continue
}
if last != nil && last.Less(storage.Key(key)) {
if !last.IsZero() && last.Less(storage.Key(key)) {
continue
}

8
storage/storelogger/logger.go
View File

@ -59,16 +59,16 @@ func (store *Logger) Delete(key storage.Key) error {
}
// List lists all keys starting from first and upto limit items
func (store *Logger) List(first storage.Key, limit storage.Limit) (storage.Keys, error) {
func (store *Logger) List(first storage.Key, limit int) (storage.Keys, error) {
keys, err := store.store.List(first, limit)
store.log.Debug("List", zap.String("first", string(first)), zap.Int("limit", int(limit)), zap.Any("keys", keys.Strings()))
store.log.Debug("List", zap.String("first", string(first)), zap.Int("limit", limit), zap.Any("keys", keys.Strings()))
return keys, err
}
// ReverseList lists all keys in reverse order, starting from first
func (store *Logger) ReverseList(first storage.Key, limit storage.Limit) (storage.Keys, error) {
func (store *Logger) ReverseList(first storage.Key, limit int) (storage.Keys, error) {
keys, err := store.store.ReverseList(first, limit)
store.log.Debug("ReverseList", zap.String("first", string(first)), zap.Int("limit", int(limit)), zap.Any("keys", keys.Strings()))
store.log.Debug("ReverseList", zap.String("first", string(first)), zap.Int("limit", limit), zap.Any("keys", keys.Strings()))
return keys, err
}

14
storage/teststore/store.go
View File

@ -52,7 +52,7 @@ func (store *Client) indexOf(key storage.Key) (int, bool) {
func (store *Client) Put(key storage.Key, value storage.Value) error {
store.version++
store.CallCount.Put++
if key == nil {
if key.IsZero() {
return storage.ErrEmptyKey
}
@ -118,13 +118,13 @@ func (store *Client) Delete(key storage.Key) error {
}
// List lists all keys starting from start and upto limit items
func (store *Client) List(first storage.Key, limit storage.Limit) (storage.Keys, error) {
func (store *Client) List(first storage.Key, limit int) (storage.Keys, error) {
store.CallCount.List++
return storage.ListKeys(store, first, limit)
}
// ReverseList lists all keys in revers order
func (store *Client) ReverseList(first storage.Key, limit storage.Limit) (storage.Keys, error) {
func (store *Client) ReverseList(first storage.Key, limit int) (storage.Keys, error) {
store.CallCount.ReverseList++
return storage.ReverseListKeys(store, first, limit)
}
@ -202,7 +202,7 @@ type advancer interface {
type forward struct{ cursor }
func (cursor *forward) PositionToFirst(prefix, first storage.Key) {
if first == nil || first.Less(prefix) {
if first.IsZero() || first.Less(prefix) {
cursor.positionForward(prefix)
} else {
cursor.positionForward(first)
@ -221,9 +221,9 @@ func (cursor *forward) Advance() (*storage.ListItem, bool) {
type backward struct{ cursor }
func (cursor *backward) PositionToFirst(prefix, first storage.Key) {
if prefix == nil {
if prefix.IsZero() {
// there's no prefix
if first == nil {
if first.IsZero() {
// and no first item, so start from the end
cursor.positionLast()
} else {
@ -232,7 +232,7 @@ func (cursor *backward) PositionToFirst(prefix, first storage.Key) {
}
} else {
// there's a prefix
if first == nil || storage.AfterPrefix(prefix).Less(first) {
if first.IsZero() || storage.AfterPrefix(prefix).Less(first) {
// there's no first, or it's after our prefix
// storage.AfterPrefix("axxx/") is the next item after prefixes
// so we position to the item before

7
storage/testsuite/test_iterate.go
View File

@ -25,11 +25,8 @@ func testIterate(t *testing.T, store storage.KeyValueStore) {
}
rand.Shuffle(len(items), items.Swap)
defer cleanupItems(store, items)
for _, item := range items {
if err := store.Put(item.Key, item.Value); err != nil {
t.Fatalf("failed to put: %v", err)
}
if err := storage.PutAll(store, items...); err != nil {
t.Fatalf("failed to setup: %v", err)
}
testIterations(t, store, []iterationTest{

23
storage/testsuite/test_iterate_all.go
View File

@ -25,11 +25,8 @@ func testIterateAll(t *testing.T, store storage.KeyValueStore) {
}
rand.Shuffle(len(items), items.Swap)
defer cleanupItems(store, items)
for _, item := range items {
if err := store.Put(item.Key, item.Value); err != nil {
t.Fatalf("failed to put: %v", err)
}
if err := storage.PutAll(store, items...); err != nil {
t.Fatalf("failed to setup: %v", err)
}
testIterations(t, store, []iterationTest{
@ -48,6 +45,22 @@ func testIterateAll(t *testing.T, store storage.KeyValueStore) {
newItem("g", "g", false),
newItem("h", "h", false),
}},
{"no limits with non-nil first",
storage.IterateOptions{
Recurse: true,
First: storage.Key(""),
}, storage.Items{
newItem("a", "a", false),
newItem("b/1", "b/1", false),
newItem("b/2", "b/2", false),
newItem("b/3", "b/3", false),
newItem("c", "c", false),
newItem("c/", "c/", false),
newItem("c//", "c//", false),
newItem("c/1", "c/1", false),
newItem("g", "g", false),
newItem("h", "h", false),
}},
{"no limits reverse",
storage.IterateOptions{
Recurse: true, Reverse: true,

9
storage/testsuite/test_list.go
View File

@ -25,18 +25,15 @@ func testList(t *testing.T, store storage.KeyValueStore) {
rand.Shuffle(len(items), items.Swap)
defer cleanupItems(store, items)
for _, item := range items {
if err := store.Put(item.Key, item.Value); err != nil {
t.Fatalf("failed to put: %v", err)
}
if err := storage.PutAll(store, items...); err != nil {
t.Fatalf("failed to setup: %v", err)
}
type Test struct {
Name string
Reverse bool
First storage.Key
Limit storage.Limit
Limit int
Expected storage.Keys
}

11
storage/testsuite/test_listv2.go
View File

@ -25,20 +25,17 @@ func testListV2(t *testing.T, store storage.KeyValueStore) {
newItem("videos/movie.mkv", "7", false),
}
rand.Shuffle(len(items), items.Swap)
defer cleanupItems(store, items)
for _, item := range items {
if err := store.Put(item.Key, item.Value); err != nil {
t.Fatalf("failed to put: %v", err)
}
if err := storage.PutAll(store, items...); err != nil {
t.Fatalf("failed to setup: %v", err)
}
sort.Sort(items)
type Test struct {
Name string
Options storage.ListOptions
More storage.More
More bool
Expected storage.Items
}

7
storage/testsuite/test_prefix.go
View File

@ -25,11 +25,8 @@ func testPrefix(t *testing.T, store storage.KeyValueStore) {
}
rand.Shuffle(len(items), items.Swap)
defer cleanupItems(store, items)
for _, item := range items {
if err := store.Put(item.Key, item.Value); err != nil {
t.Fatalf("failed to put: %v", err)
}
if err := storage.PutAll(store, items...); err != nil {
t.Fatalf("failed to setup: %v", err)
}
testIterations(t, store, []iterationTest{

13
storage/util.go
View File

@ -3,6 +3,8 @@
package storage
import "fmt"
// NextKey returns the successive key
func NextKey(key Key) Key {
return append(CloneKey(key), 0)
@ -38,3 +40,14 @@ func CloneItems(items Items) Items {
}
return result
}
// PutAll adds multiple values to the store
func PutAll(store KeyValueStore, items ...ListItem) error {
for _, item := range items {
err := store.Put(item.Key, item.Value)
if err != nil {
return fmt.Errorf("failed to put %v: %v", item, err)
}
}
return nil
}