diff --git a/cmd/netstate-http/README.md b/cmd/netstate-http/README.md deleted file mode 100644 index 8517f162a..000000000 --- a/cmd/netstate-http/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# HTTP + BoltDB Crud Interface - -This is an http server that provides a CRUD (create, read, update, delete) interface for storing file paths and small values with BoltDB. -For example, you can store a value (i.e. "hello world") at `/my/test/file` and interact with `/my/test/file` through an API, backed by BoltDB. - -To run: -``` -go run cmd/netstate-http/main.go -``` -You can also run using these flags: `-port= -prod= -db=` - -Then you can use http methods (Put, Get, List, and Delete) to interact with small values stored on BoltDB. -To store a value, put it into the PUT request body using this format: -``` -{ - "value": "here's my value" -} -``` -If you're using Postman, select the `raw` request setting to do this. - -Afterward, you can also use [Bolter](https://github.com/hasit/bolter) or a similar BoltDB viewer to make sure your files were changed as expected. \ No newline at end of file diff --git a/cmd/netstate-http/main.go b/cmd/netstate-http/main.go deleted file mode 100644 index 49f9e00df..000000000 --- a/cmd/netstate-http/main.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (C) 2018 Storj Labs, Inc. -// See LICENSE for copying information. - -package main - -import ( - "flag" - "fmt" - "log" - "net/http" - "os" - - "github.com/julienschmidt/httprouter" - "go.uber.org/zap" - - "storj.io/storj/netstate/routes" - "storj.io/storj/storage/boltdb" -) - -var ( - port int - dbPath string - prod bool -) - -func initializeFlags() { - flag.IntVar(&port, "port", 3000, "port") - flag.StringVar(&dbPath, "db", "netstate.db", "db path") - flag.BoolVar(&prod, "prod", false, "The environment this service is running in") - flag.Parse() -} - -func main() { - err := Main() - if err != nil { - log.Fatalf("fatal error: %v", err) - os.Exit(1) - } -} - -// Main allows simplified error handling -func Main() error { - initializeFlags() - - // No err here because no vars passed into NewDevelopment(). - // The default won't return an error, but if args are passed in, - // then there will need to be error handling. - logger, _ := zap.NewDevelopment() - if prod { - logger, _ = zap.NewProduction() - } - defer logger.Sync() - logger.Info(fmt.Sprintf("serving on %d", port)) - - bdb, err := boltdb.New(logger, dbPath) - if err != nil { - return err - } - defer bdb.Close() - - routes := routes.NewNetStateRoutes(logger, bdb) - - return http.ListenAndServe(fmt.Sprintf(":%d", port), start(routes)) -} - -func start(f *routes.NetStateRoutes) *httprouter.Router { - router := httprouter.New() - - router.PUT("/file/*path", f.Put) - router.GET("/file/*path", f.Get) - router.GET("/file", f.List) - router.DELETE("/file/*path", f.Delete) - - return router -} diff --git a/cmd/netstate-rpc/server/README.md b/cmd/netstate/README.md similarity index 54% rename from cmd/netstate-rpc/server/README.md rename to cmd/netstate/README.md index 9fc5bbfce..2dc9e9e22 100644 --- a/cmd/netstate-rpc/server/README.md +++ b/cmd/netstate/README.md @@ -1,15 +1,28 @@ # gRPC Server + BoltDB Crud Interface -This is a gRPC server which handles CRUD (create, read, update, delete) requests for storing file paths and small values to BoltDB. -For example, you can store a value (i.e. "hello world") at `/my/test/file` and interact with `/my/test/file` through an API, backed by BoltDB. +This is a gRPC server which handles CRUD (create, read, update, delete) requests for storing pointers at given paths in BoltDB. +An example Put Request to store a path and pointer would look like this: +``` + pr := proto.PutRequest{ + Path: []byte("here's/a/pointer/path"), + Pointer: &proto.Pointer{ + Type: proto.Pointer_INLINE, + Encryption: &proto.EncryptionScheme{ + EncryptedEncryptionKey: []byte("key"), + EncryptedStartingNonce: []byte("nonce"), + }, + InlineSegment: []byte("littledata"), + }, + } +``` To run the server: ``` -go run cmd/netstate-rpc/server/main.go +go run cmd/netstate/main.go ``` You can also run using these flags: `-port= -prod= -db=` -You can then write a client program using the client library to access the Put, Get, List, and Delete methods to create/interact with small values stored in a BoltDB. +You can then write a client program using the client library to access the Put, Get, List, and Delete methods to create and interact with pointer entries stored in BoltDB. An example program utilizing these functions can be found at `storj.io/storj/examples/netstate-client/main.go`. Afterward, you can also use [Bolter](https://github.com/hasit/bolter) or a similar BoltDB viewer to make sure your files were changed as expected. diff --git a/cmd/netstate-rpc/server/main.go b/cmd/netstate/main.go similarity index 91% rename from cmd/netstate-rpc/server/main.go rename to cmd/netstate/main.go index e0426a88c..10c97e5f7 100644 --- a/cmd/netstate-rpc/server/main.go +++ b/cmd/netstate/main.go @@ -43,6 +43,7 @@ func main() { bdb, err := boltdb.New(logger, dbPath) if err != nil { + logger.Fatal("failed to initiate boltdb", zap.Error(err)) return } defer bdb.Close() @@ -55,6 +56,7 @@ func main() { grpcServer := grpc.NewServer() proto.RegisterNetStateServer(grpcServer, netstate.NewServer(bdb, logger)) + logger.Debug(fmt.Sprintf("server listening on port %d", port)) defer grpcServer.GracefulStop() err = grpcServer.Serve(lis) diff --git a/examples/netstate-client/README.md b/examples/netstate-client/README.md index 7b87990bc..3863129ce 100644 --- a/examples/netstate-client/README.md +++ b/examples/netstate-client/README.md @@ -1,8 +1,8 @@ # gRPC Client + BoltDB Crud Interface -This is an example gRPC client which makes CRUD requests (create, read, update, delete) for storing file paths and small values to BoltDB. +This is an example gRPC client which makes CRUD requests (create, read, update, delete) for storing pointers at given paths in BoltDB. -The gRPC server at `storj.io/storj/cmd/netstate-rpc/server/main.go` needs to be running for this to work. +The gRPC server at `storj.io/storj/cmd/netstate/main.go` needs to be running for this to work. To run the client: ``` @@ -10,7 +10,7 @@ go run examples/netstate-client/main.go ``` You can change the port number with a flag if necessary: `-port=` -Afterward, you can use [Bolter](https://github.com/hasit/bolter) or a similar BoltDB viewer to make sure your files were changed as expected. +Afterward, you can use [Bolter](https://github.com/hasit/bolter) or a similar BoltDB viewer to make sure your pointer entries were changed as expected. If changes are made to `storj.io/storj/protos/netstate/netstate.proto`, the protobuf file will need to be regenerated by running `go generate` inside `protos/netstate`. diff --git a/examples/netstate-client/main.go b/examples/netstate-client/main.go index 1f74659dd..c26c62c35 100644 --- a/examples/netstate-client/main.go +++ b/examples/netstate-client/main.go @@ -6,10 +6,13 @@ package main import ( "context" "flag" + "fmt" "strings" "go.uber.org/zap" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" proto "storj.io/storj/protos/netstate" ) @@ -18,10 +21,6 @@ var ( port string ) -const ( - success string = "success" -) - func initializeFlags() { flag.StringVar(&port, "port", ":8080", "port") flag.Parse() @@ -40,73 +39,93 @@ func main() { client := proto.NewNetStateClient(conn) + logger.Debug(fmt.Sprintf("client dialed port %s", port)) + ctx := context.Background() - // Examples file paths to be saved - fp := proto.FilePath{ - Path: []byte("welcome/to/my/file/journey"), - SmallValue: []byte("granola"), + // Example pointer paths to put + pr1 := proto.PutRequest{ + Path: []byte("welcome/to/my/pointer/journey"), + Pointer: &proto.Pointer{ + Type: proto.Pointer_INLINE, + Encryption: &proto.EncryptionScheme{ + EncryptedEncryptionKey: []byte("key"), + EncryptedStartingNonce: []byte("nonce"), + }, + InlineSegment: []byte("granola"), + }, } - fp2 := proto.FilePath{ - Path: []byte("so/many/file/paths"), - SmallValue: []byte("m&ms"), + pr2 := proto.PutRequest{ + Path: []byte("so/many/pointers"), + Pointer: &proto.Pointer{ + Type: proto.Pointer_INLINE, + Encryption: &proto.EncryptionScheme{ + EncryptedEncryptionKey: []byte("key"), + EncryptedStartingNonce: []byte("nonce"), + }, + InlineSegment: []byte("m&ms"), + }, } - fp3 := proto.FilePath{ - Path: []byte("another/file/path/for/the/pile"), - SmallValue: []byte("popcorn"), + pr3 := proto.PutRequest{ + Path: []byte("another/pointer/for/the/pile"), + Pointer: &proto.Pointer{ + Type: proto.Pointer_INLINE, + Encryption: &proto.EncryptionScheme{ + EncryptedEncryptionKey: []byte("key"), + EncryptedStartingNonce: []byte("nonce"), + }, + InlineSegment: []byte("popcorn"), + }, } // Example Puts - putRes, err := client.Put(ctx, &fp) - if err != nil || putRes.Confirmation != success { + _, err = client.Put(ctx, &pr1) + if err != nil || status.Code(err) == codes.Internal { logger.Error("failed to put", zap.Error(err)) } - putRes2, err := client.Put(ctx, &fp2) - if err != nil || putRes2.Confirmation != success { + _, err = client.Put(ctx, &pr2) + if err != nil || status.Code(err) == codes.Internal { logger.Error("failed to put", zap.Error(err)) } - putRes3, err := client.Put(ctx, &fp3) - if err != nil || putRes3.Confirmation != success { + _, err = client.Put(ctx, &pr3) + if err != nil || status.Code(err) == codes.Internal { logger.Error("failed to put", zap.Error(err)) } // Example Get getReq := proto.GetRequest{ - Path: []byte("so/many/file/paths"), + Path: []byte("so/many/pointers"), } getRes, err := client.Get(ctx, &getReq) - if err != nil { + if err != nil || status.Code(err) == codes.Internal { logger.Error("failed to get", zap.Error(err)) } - value := string(getRes.SmallValue) - logger.Debug("get response: " + value) + pointer := string(getRes.Pointer) + logger.Debug("get response: " + pointer) // Example List listReq := proto.ListRequest{ - // This Bucket value isn't actually used by List() now, - // but in the future could be used to select specific - // buckets to list from. - Bucket: []byte("files"), + // This pagination functionality doesn't work yet. + // The given arguments are placeholders. + StartingPathKey: []byte("test/pointer/path"), + Limit: 5, } listRes, err := client.List(ctx, &listReq) - if err != nil { + if err != nil || status.Code(err) == codes.Internal { logger.Error("failed to list file paths") } var stringList []string - for _, pathByte := range listRes.Filepaths { + for _, pathByte := range listRes.Paths { stringList = append(stringList, string(pathByte)) } logger.Debug("listed paths: " + strings.Join(stringList, ", ")) // Example Delete delReq := proto.DeleteRequest{ - Path: []byte("welcome/to/my/file/journey"), + Path: []byte("welcome/to/my/pointer/journey"), } - delRes, err := client.Delete(ctx, &delReq) - if err != nil { - logger.Error("failed to delete: 'welcome/to/my/file/journey'") - } - if delRes.Confirmation == "success" { - logger.Debug("deleted: welcome/to/my/file/journey") + _, err = client.Delete(ctx, &delReq) + if err != nil || status.Code(err) == codes.Internal { + logger.Error("failed to delete: " + string(delReq.Path)) } } diff --git a/netstate/routes/netstate-routes.go b/netstate/routes/netstate-routes.go deleted file mode 100644 index aaf52a966..000000000 --- a/netstate/routes/netstate-routes.go +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright (C) 2018 Storj Labs, Inc. -// See LICENSE for copying information. - -package routes - -import ( - "encoding/json" - "fmt" - "net/http" - "strings" - - "github.com/julienschmidt/httprouter" - "go.uber.org/zap" - - "storj.io/storj/storage/boltdb" -) - -// NetStateRoutes maintains access to a boltdb client and zap logger -type NetStateRoutes struct { - DB *boltdb.Client - logger *zap.Logger -} - -// Message contains the small value provided by the user to be stored -type Message struct { - Value string `json:"value"` -} - -// NewNetStateRoutes instantiates NetStateRoutes -func NewNetStateRoutes(logger *zap.Logger, db *boltdb.Client) *NetStateRoutes { - return &NetStateRoutes{ - DB: db, - logger: logger, - } -} - -// Put takes the given path and small value from the user and formats the values -// to be given to boltdb.Put -func (n *NetStateRoutes) Put(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { - n.logger.Debug("entering netstate http put") - - givenPath := ps.ByName("path") - var msg Message - - decoder := json.NewDecoder(r.Body) - err := decoder.Decode(&msg) - if err != nil { - http.Error(w, "bad request: err decoding response", http.StatusBadRequest) - n.logger.Error("err decoding response", zap.Error(err)) - return - } - - file := boltdb.File{ - Path: []byte(givenPath), - Value: []byte(msg.Value), - } - - if err := n.DB.Put(file); err != nil { - http.Error(w, "err putting file", http.StatusInternalServerError) - n.logger.Error("err putting file", zap.Error(err)) - return - } - - n.logger.Debug("put to the db: " + givenPath) - w.WriteHeader(http.StatusCreated) - fmt.Fprintf(w, "PUT to %s\n", givenPath) -} - -// Get takes the given file path from the user and calls the bolt client's Get function -func (n *NetStateRoutes) Get(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { - n.logger.Debug("entering netstate http get") - - fileKey := ps.ByName("path") - - fileValue, err := n.DB.Get([]byte(fileKey)) - if err != nil { - http.Error(w, "err getting file", http.StatusInternalServerError) - n.logger.Error("err getting file", zap.Error(err)) - return - } - - w.Header().Set("Content-Type", "application/octet-stream") - _, err = w.Write(fileValue) - if err != nil { - n.logger.Error("err writing response", zap.Error(err)) - } - w.WriteHeader(http.StatusOK) - n.logger.Debug("response written: " + string(fileValue)) -} - -// List calls the bolt client's List function and responds with a list of all saved file paths -// or "filekeys" -func (n *NetStateRoutes) List(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { - n.logger.Debug("entering netstate http list") - - filePaths, err := n.DB.List() - if err != nil { - http.Error(w, "internal error: unable to list paths", http.StatusInternalServerError) - n.logger.Error("err listing file paths", zap.Error(err)) - return - } - - var pathList []string - for _, path := range filePaths { - pathList = append(pathList, string(path)) - } - - bytes, err := json.Marshal(pathList) - if err != nil { - http.Error(w, "internal error: unable to marshal path list", http.StatusInternalServerError) - n.logger.Error("err marshaling path list", zap.Error(err)) - return - } - - _, err = w.Write(bytes) - if err != nil { - n.logger.Error("err writing response", zap.Error(err)) - } - w.WriteHeader(http.StatusOK) - n.logger.Debug("response written: " + strings.Join(pathList, ", ")) -} - -// Delete takes a given file path and calls the bolt client's Delete function -func (n *NetStateRoutes) Delete(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { - n.logger.Debug("entering netstate http delete") - - fileKey := ps.ByName("path") - if err := n.DB.Delete([]byte(fileKey)); err != nil { - http.Error(w, "internal error: unable to delete file", http.StatusInternalServerError) - n.logger.Error("err deleting file", zap.Error(err)) - return - } - n.logger.Debug("deleted file: " + fileKey) - w.WriteHeader(204) - fmt.Fprintf(w, "Deleted file key: %s", fileKey) -} diff --git a/pkg/netstate/client_test.go b/pkg/netstate/client_test.go index 730ced7eb..4a42c5154 100644 --- a/pkg/netstate/client_test.go +++ b/pkg/netstate/client_test.go @@ -10,10 +10,13 @@ import ( "net" "testing" + "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" "go.uber.org/zap" "google.golang.org/grpc" - proto "storj.io/storj/protos/netstate" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + pb "storj.io/storj/protos/netstate" ) func TestNetStateClient(t *testing.T) { @@ -22,12 +25,12 @@ func TestNetStateClient(t *testing.T) { lis, err := net.Listen("tcp", fmt.Sprintf(":%d", 9000)) assert.NoError(t, err) - mdb := &mockDB{ + mdb := &MockDB{ timesCalled: 0, } grpcServer := grpc.NewServer() - proto.RegisterNetStateServer(grpcServer, NewServer(mdb, logger)) + pb.RegisterNetStateServer(grpcServer, NewServer(mdb, logger)) defer grpcServer.GracefulStop() go grpcServer.Serve(lis) @@ -36,14 +39,21 @@ func TestNetStateClient(t *testing.T) { conn, err := grpc.Dial(address, grpc.WithInsecure()) assert.NoError(t, err) - c := proto.NewNetStateClient(conn) + c := pb.NewNetStateClient(conn) ctx := context.Background() // example file path to put/get - fp := proto.FilePath{ - Path: []byte("here/is/a/path"), - SmallValue: []byte("oatmeal"), + pr1 := pb.PutRequest{ + Path: []byte("here/is/a/path"), + Pointer: &pb.Pointer{ + Type: pb.Pointer_INLINE, + Encryption: &pb.EncryptionScheme{ + EncryptedEncryptionKey: []byte("key"), + EncryptedStartingNonce: []byte("nonce"), + }, + InlineSegment: []byte("oatmeal"), + }, } if mdb.timesCalled != 0 { @@ -51,34 +61,37 @@ func TestNetStateClient(t *testing.T) { } // Tests Server.Put - putRes, err := c.Put(ctx, &fp) - assert.NoError(t, err) - - if putRes.Confirmation != "success" { - t.Error("Failed to receive success Put response") + _, err = c.Put(ctx, &pr1) + if err != nil || status.Code(err) == codes.Internal { + t.Error("Failed to Put") } if mdb.timesCalled != 1 { t.Error("Failed to call mockdb correctly") } - if !bytes.Equal(mdb.puts[0].Path, fp.Path) { + if !bytes.Equal(mdb.puts[0].Path, pr1.Path) { t.Error("Expected saved path to equal given path") } - if !bytes.Equal(mdb.puts[0].Value, fp.SmallValue) { + pointerBytes, err := proto.Marshal(pr1.Pointer) + if err != nil { + t.Error("failed to marshal test pointer") + } + + if !bytes.Equal(mdb.puts[0].Pointer, pointerBytes) { t.Error("Expected saved value to equal given value") } // Tests Server.Get - getReq := proto.GetRequest{ + getReq := pb.GetRequest{ Path: []byte("here/is/a/path"), } getRes, err := c.Get(ctx, &getReq) assert.NoError(t, err) - if !bytes.Equal(getRes.SmallValue, fp.SmallValue) { + if !bytes.Equal(getRes.Pointer, pointerBytes) { t.Error("Expected to get same content that was put") } @@ -86,17 +99,22 @@ func TestNetStateClient(t *testing.T) { t.Error("Failed to call mockdb correct number of times") } - // Puts another file path to test delete and list - fp2 := proto.FilePath{ - Path: []byte("here/is/another/path"), - SmallValue: []byte("raisins"), + // Puts another pointer entry to test delete and list + pr2 := pb.PutRequest{ + Path: []byte("here/is/another/path"), + Pointer: &pb.Pointer{ + Type: pb.Pointer_INLINE, + Encryption: &pb.EncryptionScheme{ + EncryptedEncryptionKey: []byte("key"), + EncryptedStartingNonce: []byte("nonce"), + }, + InlineSegment: []byte("raisins"), + }, } - putRes2, err := c.Put(ctx, &fp2) - assert.NoError(t, err) - - if putRes2.Confirmation != "success" { - t.Error("Failed to receive success Put response") + _, err = c.Put(ctx, &pr2) + if err != nil || status.Code(err) == codes.Internal { + t.Error("Failed to Put") } if mdb.timesCalled != 3 { @@ -104,17 +122,13 @@ func TestNetStateClient(t *testing.T) { } // Test Server.Delete - delReq := proto.DeleteRequest{ + delReq := pb.DeleteRequest{ Path: []byte("here/is/a/path"), } - delRes, err := c.Delete(ctx, &delReq) - if err != nil { - t.Error("Failed to delete file path") - } - - if delRes.Confirmation != "success" { - t.Error("Failed to receive success delete response") + _, err = c.Delete(ctx, &delReq) + if err != nil || status.Code(err) == codes.Internal { + t.Error("Failed to delete") } if mdb.timesCalled != 4 { @@ -122,8 +136,11 @@ func TestNetStateClient(t *testing.T) { } // Tests Server.List - listReq := proto.ListRequest{ - Bucket: []byte("files"), + listReq := pb.ListRequest{ + // This pagination functionality doesn't work yet. + // The given arguments are placeholders. + StartingPathKey: []byte("test/pointer/path"), + Limit: 5, } listRes, err := c.List(ctx, &listReq) @@ -131,7 +148,7 @@ func TestNetStateClient(t *testing.T) { t.Error("Failed to list file paths") } - if !bytes.Equal(listRes.Filepaths[0], []byte("here/is/another/path")) { + if !bytes.Equal(listRes.Paths[0], []byte("here/is/another/path")) { t.Error("Failed to list correct file path") } diff --git a/pkg/netstate/netstate.go b/pkg/netstate/netstate.go index b631f9aac..e2968ff85 100644 --- a/pkg/netstate/netstate.go +++ b/pkg/netstate/netstate.go @@ -6,9 +6,12 @@ package netstate import ( "context" + "github.com/golang/protobuf/proto" "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" - proto "storj.io/storj/protos/netstate" + pb "storj.io/storj/protos/netstate" "storj.io/storj/storage/boltdb" ) @@ -30,75 +33,77 @@ func NewServer(db DB, logger *zap.Logger) *Server { // and makes it easier in the future to substitute // db clients other than bolt type DB interface { - Put(boltdb.File) error + Put(boltdb.PointerEntry) error Get([]byte) ([]byte, error) List() ([][]byte, error) Delete([]byte) error } // Put formats and hands off a file path to be saved to boltdb -func (s *Server) Put(ctx context.Context, filepath *proto.FilePath) (*proto.PutResponse, error) { +func (s *Server) Put(ctx context.Context, putReq *pb.PutRequest) (*pb.PutResponse, error) { s.logger.Debug("entering netstate put") - file := boltdb.File{ - Path: []byte(filepath.Path), - Value: []byte(filepath.SmallValue), + pointerBytes, err := proto.Marshal(putReq.Pointer) + if err != nil { + s.logger.Error("err marshaling pointer", zap.Error(err)) + return nil, status.Errorf(codes.Internal, err.Error()) } - if err := s.DB.Put(file); err != nil { - s.logger.Error("err putting file", zap.Error(err)) - return nil, err + pe := boltdb.PointerEntry{ + Path: putReq.Path, + Pointer: pointerBytes, } - s.logger.Debug("put to the db: " + string(file.Path)) - return &proto.PutResponse{ - Confirmation: "success", - }, nil + if err := s.DB.Put(pe); err != nil { + s.logger.Error("err putting pointer", zap.Error(err)) + return nil, status.Errorf(codes.Internal, err.Error()) + } + s.logger.Debug("put to the db: " + string(pe.Path)) + + return &pb.PutResponse{}, nil } // Get formats and hands off a file path to get from boltdb -func (s *Server) Get(ctx context.Context, req *proto.GetRequest) (*proto.GetResponse, error) { +func (s *Server) Get(ctx context.Context, req *pb.GetRequest) (*pb.GetResponse, error) { s.logger.Debug("entering netstate get") - fileValue, err := s.DB.Get(req.Path) + pointerBytes, err := s.DB.Get(req.Path) if err != nil { s.logger.Error("err getting file", zap.Error(err)) - return nil, err + return nil, status.Errorf(codes.Internal, err.Error()) } - return &proto.GetResponse{ - SmallValue: fileValue, + return &pb.GetResponse{ + Pointer: pointerBytes, }, nil } -// List calls the bolt client's List function and returns all file paths -func (s *Server) List(ctx context.Context, req *proto.ListRequest) (*proto.ListResponse, error) { +// List calls the bolt client's List function and returns all Path keys in the Pointers bucket +func (s *Server) List(ctx context.Context, req *pb.ListRequest) (*pb.ListResponse, error) { s.logger.Debug("entering netstate list") - filePaths, err := s.DB.List() + pathKeys, err := s.DB.List() if err != nil { - s.logger.Error("err listing file paths", zap.Error(err)) - return nil, err + s.logger.Error("err listing path keys", zap.Error(err)) + return nil, status.Errorf(codes.Internal, err.Error()) } - s.logger.Debug("file paths retrieved") - return &proto.ListResponse{ - // filePaths is an array of byte arrays - Filepaths: filePaths, + s.logger.Debug("path keys retrieved") + return &pb.ListResponse{ + // pathKeys is an array of byte arrays + Paths: pathKeys, }, nil } // Delete formats and hands off a file path to delete from boltdb -func (s *Server) Delete(ctx context.Context, req *proto.DeleteRequest) (*proto.DeleteResponse, error) { +func (s *Server) Delete(ctx context.Context, req *pb.DeleteRequest) (*pb.DeleteResponse, error) { s.logger.Debug("entering netstate delete") err := s.DB.Delete(req.Path) if err != nil { - s.logger.Error("err deleting file", zap.Error(err)) - return nil, err + s.logger.Error("err deleting pointer entry", zap.Error(err)) + return nil, status.Errorf(codes.Internal, err.Error()) } - s.logger.Debug("deleted: " + string(req.Path)) - return &proto.DeleteResponse{ - Confirmation: "success", - }, nil + s.logger.Debug("deleted pointer at path: " + string(req.Path)) + return &pb.DeleteResponse{}, nil } diff --git a/pkg/netstate/server_test.go b/pkg/netstate/server_test.go index 1059f8d5e..cdec5ee64 100644 --- a/pkg/netstate/server_test.go +++ b/pkg/netstate/server_test.go @@ -9,44 +9,45 @@ import ( "storj.io/storj/storage/boltdb" ) -type mockDB struct { +// MockDB mocks db functionality for testing +type MockDB struct { timesCalled int - puts []boltdb.File - filePaths [][]byte + puts []boltdb.PointerEntry + pathKeys [][]byte } -func (m *mockDB) Put(f boltdb.File) error { +func (m *MockDB) Put(f boltdb.PointerEntry) error { m.timesCalled++ m.puts = append(m.puts, f) return nil } -func (m *mockDB) Get(path []byte) ([]byte, error) { +func (m *MockDB) Get(path []byte) ([]byte, error) { m.timesCalled++ - for _, file := range m.puts { - if bytes.Equal(path, file.Path) { - return file.Value, nil + for _, pointerEntry := range m.puts { + if bytes.Equal(path, pointerEntry.Path) { + return pointerEntry.Pointer, nil } } panic("failed to get the given file") } -func (m *mockDB) List() ([][]byte, error) { +func (m *MockDB) List() ([][]byte, error) { m.timesCalled++ - for _, file := range m.puts { - m.filePaths = append(m.filePaths, file.Path) + for _, putReq := range m.puts { + m.pathKeys = append(m.pathKeys, putReq.Path) } - return m.filePaths, nil + return m.pathKeys, nil } -func (m *mockDB) Delete(path []byte) error { +func (m *MockDB) Delete(path []byte) error { m.timesCalled++ - for i, file := range m.puts { - if bytes.Equal(path, file.Path) { + for i, pointerEntry := range m.puts { + if bytes.Equal(path, pointerEntry.Path) { m.puts = append(m.puts[:i], m.puts[i+1:]...) } } diff --git a/protos/netstate/netstate.pb.go b/protos/netstate/netstate.pb.go index cd7950b26..9f313d867 100644 --- a/protos/netstate/netstate.pb.go +++ b/protos/netstate/netstate.pb.go @@ -8,7 +8,12 @@ It is generated from these files: netstate.proto It has these top-level messages: - FilePath + RedundancyScheme + EncryptionScheme + RemotePiece + RemoteSegment + Pointer + PutRequest GetRequest ListRequest PutResponse @@ -22,6 +27,7 @@ package netstate import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" +import google_protobuf "github.com/golang/protobuf/ptypes/timestamp" import ( context "golang.org/x/net/context" @@ -39,29 +45,316 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package -// FilePath is a request message for the Put rpc call -type FilePath struct { - Path []byte `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` - // smallValue is a value too small to be broken up and stored - // in different places - SmallValue []byte `protobuf:"bytes,2,opt,name=smallValue,proto3" json:"smallValue,omitempty"` +type RedundancyScheme_SchemeType int32 + +const ( + RedundancyScheme_RS RedundancyScheme_SchemeType = 0 +) + +var RedundancyScheme_SchemeType_name = map[int32]string{ + 0: "RS", +} +var RedundancyScheme_SchemeType_value = map[string]int32{ + "RS": 0, } -func (m *FilePath) Reset() { *m = FilePath{} } -func (m *FilePath) String() string { return proto.CompactTextString(m) } -func (*FilePath) ProtoMessage() {} -func (*FilePath) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } +func (x RedundancyScheme_SchemeType) String() string { + return proto.EnumName(RedundancyScheme_SchemeType_name, int32(x)) +} +func (RedundancyScheme_SchemeType) EnumDescriptor() ([]byte, []int) { + return fileDescriptor0, []int{0, 0} +} -func (m *FilePath) GetPath() []byte { +type EncryptionScheme_EncryptionType int32 + +const ( + EncryptionScheme_AESGCM EncryptionScheme_EncryptionType = 0 + EncryptionScheme_SECRETBOX EncryptionScheme_EncryptionType = 1 +) + +var EncryptionScheme_EncryptionType_name = map[int32]string{ + 0: "AESGCM", + 1: "SECRETBOX", +} +var EncryptionScheme_EncryptionType_value = map[string]int32{ + "AESGCM": 0, + "SECRETBOX": 1, +} + +func (x EncryptionScheme_EncryptionType) String() string { + return proto.EnumName(EncryptionScheme_EncryptionType_name, int32(x)) +} +func (EncryptionScheme_EncryptionType) EnumDescriptor() ([]byte, []int) { + return fileDescriptor0, []int{1, 0} +} + +type Pointer_DataType int32 + +const ( + Pointer_INLINE Pointer_DataType = 0 + Pointer_REMOTE Pointer_DataType = 1 +) + +var Pointer_DataType_name = map[int32]string{ + 0: "INLINE", + 1: "REMOTE", +} +var Pointer_DataType_value = map[string]int32{ + "INLINE": 0, + "REMOTE": 1, +} + +func (x Pointer_DataType) String() string { + return proto.EnumName(Pointer_DataType_name, int32(x)) +} +func (Pointer_DataType) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{4, 0} } + +type RedundancyScheme struct { + Type RedundancyScheme_SchemeType `protobuf:"varint,1,opt,name=type,enum=netstate.RedundancyScheme_SchemeType" json:"type,omitempty"` + // these values apply to RS encoding + MinReq int64 `protobuf:"varint,2,opt,name=min_req,json=minReq" json:"min_req,omitempty"` + Total int64 `protobuf:"varint,3,opt,name=total" json:"total,omitempty"` + RepairThreshold int64 `protobuf:"varint,4,opt,name=repair_threshold,json=repairThreshold" json:"repair_threshold,omitempty"` + SuccessThreshold int64 `protobuf:"varint,5,opt,name=success_threshold,json=successThreshold" json:"success_threshold,omitempty"` +} + +func (m *RedundancyScheme) Reset() { *m = RedundancyScheme{} } +func (m *RedundancyScheme) String() string { return proto.CompactTextString(m) } +func (*RedundancyScheme) ProtoMessage() {} +func (*RedundancyScheme) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func (m *RedundancyScheme) GetType() RedundancyScheme_SchemeType { + if m != nil { + return m.Type + } + return RedundancyScheme_RS +} + +func (m *RedundancyScheme) GetMinReq() int64 { + if m != nil { + return m.MinReq + } + return 0 +} + +func (m *RedundancyScheme) GetTotal() int64 { + if m != nil { + return m.Total + } + return 0 +} + +func (m *RedundancyScheme) GetRepairThreshold() int64 { + if m != nil { + return m.RepairThreshold + } + return 0 +} + +func (m *RedundancyScheme) GetSuccessThreshold() int64 { + if m != nil { + return m.SuccessThreshold + } + return 0 +} + +type EncryptionScheme struct { + Type EncryptionScheme_EncryptionType `protobuf:"varint,1,opt,name=type,enum=netstate.EncryptionScheme_EncryptionType" json:"type,omitempty"` + EncryptedEncryptionKey []byte `protobuf:"bytes,2,opt,name=encrypted_encryption_key,json=encryptedEncryptionKey,proto3" json:"encrypted_encryption_key,omitempty"` + EncryptedStartingNonce []byte `protobuf:"bytes,3,opt,name=encrypted_starting_nonce,json=encryptedStartingNonce,proto3" json:"encrypted_starting_nonce,omitempty"` +} + +func (m *EncryptionScheme) Reset() { *m = EncryptionScheme{} } +func (m *EncryptionScheme) String() string { return proto.CompactTextString(m) } +func (*EncryptionScheme) ProtoMessage() {} +func (*EncryptionScheme) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +func (m *EncryptionScheme) GetType() EncryptionScheme_EncryptionType { + if m != nil { + return m.Type + } + return EncryptionScheme_AESGCM +} + +func (m *EncryptionScheme) GetEncryptedEncryptionKey() []byte { + if m != nil { + return m.EncryptedEncryptionKey + } + return nil +} + +func (m *EncryptionScheme) GetEncryptedStartingNonce() []byte { + if m != nil { + return m.EncryptedStartingNonce + } + return nil +} + +type RemotePiece struct { + PieceNum int64 `protobuf:"varint,1,opt,name=piece_num,json=pieceNum" json:"piece_num,omitempty"` + NodeId string `protobuf:"bytes,2,opt,name=node_id,json=nodeId" json:"node_id,omitempty"` + Size int64 `protobuf:"varint,3,opt,name=size" json:"size,omitempty"` +} + +func (m *RemotePiece) Reset() { *m = RemotePiece{} } +func (m *RemotePiece) String() string { return proto.CompactTextString(m) } +func (*RemotePiece) ProtoMessage() {} +func (*RemotePiece) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } + +func (m *RemotePiece) GetPieceNum() int64 { + if m != nil { + return m.PieceNum + } + return 0 +} + +func (m *RemotePiece) GetNodeId() string { + if m != nil { + return m.NodeId + } + return "" +} + +func (m *RemotePiece) GetSize() int64 { + if m != nil { + return m.Size + } + return 0 +} + +type RemoteSegment struct { + Redundancy *RedundancyScheme `protobuf:"bytes,1,opt,name=redundancy" json:"redundancy,omitempty"` + PieceName string `protobuf:"bytes,2,opt,name=piece_name,json=pieceName" json:"piece_name,omitempty"` + RemotePieces []*RemotePiece `protobuf:"bytes,3,rep,name=remote_pieces,json=remotePieces" json:"remote_pieces,omitempty"` + MerkleRoot []byte `protobuf:"bytes,4,opt,name=merkle_root,json=merkleRoot,proto3" json:"merkle_root,omitempty"` + MerkleSize int64 `protobuf:"varint,5,opt,name=merkle_size,json=merkleSize" json:"merkle_size,omitempty"` +} + +func (m *RemoteSegment) Reset() { *m = RemoteSegment{} } +func (m *RemoteSegment) String() string { return proto.CompactTextString(m) } +func (*RemoteSegment) ProtoMessage() {} +func (*RemoteSegment) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } + +func (m *RemoteSegment) GetRedundancy() *RedundancyScheme { + if m != nil { + return m.Redundancy + } + return nil +} + +func (m *RemoteSegment) GetPieceName() string { + if m != nil { + return m.PieceName + } + return "" +} + +func (m *RemoteSegment) GetRemotePieces() []*RemotePiece { + if m != nil { + return m.RemotePieces + } + return nil +} + +func (m *RemoteSegment) GetMerkleRoot() []byte { + if m != nil { + return m.MerkleRoot + } + return nil +} + +func (m *RemoteSegment) GetMerkleSize() int64 { + if m != nil { + return m.MerkleSize + } + return 0 +} + +type Pointer struct { + Type Pointer_DataType `protobuf:"varint,1,opt,name=type,enum=netstate.Pointer_DataType" json:"type,omitempty"` + Encryption *EncryptionScheme `protobuf:"bytes,2,opt,name=encryption" json:"encryption,omitempty"` + InlineSegment []byte `protobuf:"bytes,3,opt,name=inline_segment,json=inlineSegment,proto3" json:"inline_segment,omitempty"` + Remote *RemoteSegment `protobuf:"bytes,4,opt,name=remote" json:"remote,omitempty"` + EncryptedUnencryptedSize []byte `protobuf:"bytes,5,opt,name=encrypted_unencrypted_size,json=encryptedUnencryptedSize,proto3" json:"encrypted_unencrypted_size,omitempty"` + CreationDate *google_protobuf.Timestamp `protobuf:"bytes,6,opt,name=creation_date,json=creationDate" json:"creation_date,omitempty"` + ExpirationDate *google_protobuf.Timestamp `protobuf:"bytes,7,opt,name=expiration_date,json=expirationDate" json:"expiration_date,omitempty"` +} + +func (m *Pointer) Reset() { *m = Pointer{} } +func (m *Pointer) String() string { return proto.CompactTextString(m) } +func (*Pointer) ProtoMessage() {} +func (*Pointer) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } + +func (m *Pointer) GetType() Pointer_DataType { + if m != nil { + return m.Type + } + return Pointer_INLINE +} + +func (m *Pointer) GetEncryption() *EncryptionScheme { + if m != nil { + return m.Encryption + } + return nil +} + +func (m *Pointer) GetInlineSegment() []byte { + if m != nil { + return m.InlineSegment + } + return nil +} + +func (m *Pointer) GetRemote() *RemoteSegment { + if m != nil { + return m.Remote + } + return nil +} + +func (m *Pointer) GetEncryptedUnencryptedSize() []byte { + if m != nil { + return m.EncryptedUnencryptedSize + } + return nil +} + +func (m *Pointer) GetCreationDate() *google_protobuf.Timestamp { + if m != nil { + return m.CreationDate + } + return nil +} + +func (m *Pointer) GetExpirationDate() *google_protobuf.Timestamp { + if m != nil { + return m.ExpirationDate + } + return nil +} + +// PutRequest is a request message for the Put rpc call +type PutRequest struct { + Path []byte `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + Pointer *Pointer `protobuf:"bytes,2,opt,name=pointer" json:"pointer,omitempty"` +} + +func (m *PutRequest) Reset() { *m = PutRequest{} } +func (m *PutRequest) String() string { return proto.CompactTextString(m) } +func (*PutRequest) ProtoMessage() {} +func (*PutRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } + +func (m *PutRequest) GetPath() []byte { if m != nil { return m.Path } return nil } -func (m *FilePath) GetSmallValue() []byte { +func (m *PutRequest) GetPointer() *Pointer { if m != nil { - return m.SmallValue + return m.Pointer } return nil } @@ -74,7 +367,7 @@ type GetRequest struct { func (m *GetRequest) Reset() { *m = GetRequest{} } func (m *GetRequest) String() string { return proto.CompactTextString(m) } func (*GetRequest) ProtoMessage() {} -func (*GetRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } +func (*GetRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } func (m *GetRequest) GetPath() []byte { if m != nil { @@ -85,68 +378,68 @@ func (m *GetRequest) GetPath() []byte { // ListRequest is a request message for the List rpc call type ListRequest struct { - Bucket []byte `protobuf:"bytes,1,opt,name=bucket,proto3" json:"bucket,omitempty"` + StartingPathKey []byte `protobuf:"bytes,1,opt,name=starting_path_key,json=startingPathKey,proto3" json:"starting_path_key,omitempty"` + Limit int64 `protobuf:"varint,2,opt,name=limit" json:"limit,omitempty"` } func (m *ListRequest) Reset() { *m = ListRequest{} } func (m *ListRequest) String() string { return proto.CompactTextString(m) } func (*ListRequest) ProtoMessage() {} -func (*ListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } +func (*ListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } -func (m *ListRequest) GetBucket() []byte { +func (m *ListRequest) GetStartingPathKey() []byte { if m != nil { - return m.Bucket + return m.StartingPathKey } return nil } +func (m *ListRequest) GetLimit() int64 { + if m != nil { + return m.Limit + } + return 0 +} + // PutResponse is a response message for the Put rpc call type PutResponse struct { - Confirmation string `protobuf:"bytes,1,opt,name=confirmation" json:"confirmation,omitempty"` } func (m *PutResponse) Reset() { *m = PutResponse{} } func (m *PutResponse) String() string { return proto.CompactTextString(m) } func (*PutResponse) ProtoMessage() {} -func (*PutResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } - -func (m *PutResponse) GetConfirmation() string { - if m != nil { - return m.Confirmation - } - return "" -} +func (*PutResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } // GetResponse is a response message for the Get rpc call type GetResponse struct { - SmallValue []byte `protobuf:"bytes,1,opt,name=smallValue,proto3" json:"smallValue,omitempty"` + Pointer []byte `protobuf:"bytes,1,opt,name=pointer,proto3" json:"pointer,omitempty"` } func (m *GetResponse) Reset() { *m = GetResponse{} } func (m *GetResponse) String() string { return proto.CompactTextString(m) } func (*GetResponse) ProtoMessage() {} -func (*GetResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } +func (*GetResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } -func (m *GetResponse) GetSmallValue() []byte { +func (m *GetResponse) GetPointer() []byte { if m != nil { - return m.SmallValue + return m.Pointer } return nil } // ListResponse is a response message for the List rpc call type ListResponse struct { - Filepaths [][]byte `protobuf:"bytes,1,rep,name=filepaths,proto3" json:"filepaths,omitempty"` + Paths [][]byte `protobuf:"bytes,1,rep,name=paths,proto3" json:"paths,omitempty"` } func (m *ListResponse) Reset() { *m = ListResponse{} } func (m *ListResponse) String() string { return proto.CompactTextString(m) } func (*ListResponse) ProtoMessage() {} -func (*ListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } +func (*ListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } -func (m *ListResponse) GetFilepaths() [][]byte { +func (m *ListResponse) GetPaths() [][]byte { if m != nil { - return m.Filepaths + return m.Paths } return nil } @@ -158,7 +451,7 @@ type DeleteRequest struct { func (m *DeleteRequest) Reset() { *m = DeleteRequest{} } func (m *DeleteRequest) String() string { return proto.CompactTextString(m) } func (*DeleteRequest) ProtoMessage() {} -func (*DeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } +func (*DeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} } func (m *DeleteRequest) GetPath() []byte { if m != nil { @@ -169,23 +462,20 @@ func (m *DeleteRequest) GetPath() []byte { // DeleteResponse is a response message for the Delete rpc call type DeleteResponse struct { - Confirmation string `protobuf:"bytes,1,opt,name=confirmation" json:"confirmation,omitempty"` } func (m *DeleteResponse) Reset() { *m = DeleteResponse{} } func (m *DeleteResponse) String() string { return proto.CompactTextString(m) } func (*DeleteResponse) ProtoMessage() {} -func (*DeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } - -func (m *DeleteResponse) GetConfirmation() string { - if m != nil { - return m.Confirmation - } - return "" -} +func (*DeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} } func init() { - proto.RegisterType((*FilePath)(nil), "netstate.FilePath") + proto.RegisterType((*RedundancyScheme)(nil), "netstate.RedundancyScheme") + proto.RegisterType((*EncryptionScheme)(nil), "netstate.EncryptionScheme") + proto.RegisterType((*RemotePiece)(nil), "netstate.RemotePiece") + proto.RegisterType((*RemoteSegment)(nil), "netstate.RemoteSegment") + proto.RegisterType((*Pointer)(nil), "netstate.Pointer") + proto.RegisterType((*PutRequest)(nil), "netstate.PutRequest") proto.RegisterType((*GetRequest)(nil), "netstate.GetRequest") proto.RegisterType((*ListRequest)(nil), "netstate.ListRequest") proto.RegisterType((*PutResponse)(nil), "netstate.PutResponse") @@ -193,6 +483,9 @@ func init() { proto.RegisterType((*ListResponse)(nil), "netstate.ListResponse") proto.RegisterType((*DeleteRequest)(nil), "netstate.DeleteRequest") proto.RegisterType((*DeleteResponse)(nil), "netstate.DeleteResponse") + proto.RegisterEnum("netstate.RedundancyScheme_SchemeType", RedundancyScheme_SchemeType_name, RedundancyScheme_SchemeType_value) + proto.RegisterEnum("netstate.EncryptionScheme_EncryptionType", EncryptionScheme_EncryptionType_name, EncryptionScheme_EncryptionType_value) + proto.RegisterEnum("netstate.Pointer_DataType", Pointer_DataType_name, Pointer_DataType_value) } // Reference imports to suppress errors if they are not otherwise used. @@ -207,7 +500,7 @@ const _ = grpc.SupportPackageIsVersion4 type NetStateClient interface { // Put formats and hands off a file path to be saved to boltdb - Put(ctx context.Context, in *FilePath, opts ...grpc.CallOption) (*PutResponse, error) + Put(ctx context.Context, in *PutRequest, opts ...grpc.CallOption) (*PutResponse, error) // Get formats and hands off a file path to get a small value from boltdb Get(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetResponse, error) // List calls the bolt client's List function and returns all file paths @@ -224,7 +517,7 @@ func NewNetStateClient(cc *grpc.ClientConn) NetStateClient { return &netStateClient{cc} } -func (c *netStateClient) Put(ctx context.Context, in *FilePath, opts ...grpc.CallOption) (*PutResponse, error) { +func (c *netStateClient) Put(ctx context.Context, in *PutRequest, opts ...grpc.CallOption) (*PutResponse, error) { out := new(PutResponse) err := grpc.Invoke(ctx, "/netstate.NetState/Put", in, out, c.cc, opts...) if err != nil { @@ -264,7 +557,7 @@ func (c *netStateClient) Delete(ctx context.Context, in *DeleteRequest, opts ... type NetStateServer interface { // Put formats and hands off a file path to be saved to boltdb - Put(context.Context, *FilePath) (*PutResponse, error) + Put(context.Context, *PutRequest) (*PutResponse, error) // Get formats and hands off a file path to get a small value from boltdb Get(context.Context, *GetRequest) (*GetResponse, error) // List calls the bolt client's List function and returns all file paths @@ -278,7 +571,7 @@ func RegisterNetStateServer(s *grpc.Server, srv NetStateServer) { } func _NetState_Put_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(FilePath) + in := new(PutRequest) if err := dec(in); err != nil { return nil, err } @@ -290,7 +583,7 @@ func _NetState_Put_Handler(srv interface{}, ctx context.Context, dec func(interf FullMethod: "/netstate.NetState/Put", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(NetStateServer).Put(ctx, req.(*FilePath)) + return srv.(NetStateServer).Put(ctx, req.(*PutRequest)) } return interceptor(ctx, in, info, handler) } @@ -377,24 +670,58 @@ var _NetState_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("netstate.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 302 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x52, 0x4d, 0x4b, 0xc3, 0x40, - 0x10, 0x25, 0xb6, 0x94, 0x74, 0x12, 0x7b, 0x18, 0xb4, 0x86, 0x20, 0x12, 0x56, 0x84, 0x1e, 0xb4, - 0x68, 0xd5, 0x93, 0xe0, 0x49, 0xec, 0x45, 0x24, 0x44, 0xf0, 0xbe, 0x2d, 0x53, 0x1a, 0xdc, 0x26, - 0xb1, 0x3b, 0xf9, 0xc7, 0xfe, 0x10, 0xc9, 0x57, 0xb3, 0xb1, 0x22, 0xde, 0x32, 0xef, 0x23, 0x79, - 0x33, 0x2f, 0x30, 0x4a, 0x88, 0x35, 0x4b, 0xa6, 0x69, 0xb6, 0x4d, 0x39, 0x45, 0xbb, 0x99, 0xc5, - 0x23, 0xd8, 0xcf, 0xb1, 0xa2, 0x50, 0xf2, 0x1a, 0x11, 0xfa, 0x99, 0xe4, 0xb5, 0x67, 0x05, 0xd6, - 0xc4, 0x8d, 0xca, 0x67, 0x3c, 0x03, 0xd0, 0x1b, 0xa9, 0xd4, 0xbb, 0x54, 0x39, 0x79, 0x07, 0x25, - 0x63, 0x20, 0x22, 0x00, 0x98, 0x13, 0x47, 0xf4, 0x99, 0x93, 0xe6, 0xdf, 0xde, 0x20, 0x2e, 0xc0, - 0x79, 0x89, 0xf5, 0x4e, 0x32, 0x86, 0xc1, 0x22, 0x5f, 0x7e, 0x10, 0xd7, 0xa2, 0x7a, 0x12, 0x37, - 0xe0, 0x84, 0x39, 0x47, 0xa4, 0xb3, 0x34, 0xd1, 0x84, 0x02, 0xdc, 0x65, 0x9a, 0xac, 0xe2, 0xed, - 0x46, 0x72, 0x9c, 0x26, 0xa5, 0x78, 0x18, 0x75, 0x30, 0x71, 0x05, 0x4e, 0xf9, 0xed, 0xda, 0xd2, - 0x8d, 0x6a, 0xed, 0x45, 0xbd, 0x04, 0xb7, 0x0a, 0x52, 0xeb, 0x4f, 0x61, 0xb8, 0x8a, 0x15, 0x15, - 0x21, 0xb5, 0x67, 0x05, 0xbd, 0x89, 0x1b, 0xb5, 0x80, 0x38, 0x87, 0xc3, 0x27, 0x52, 0xc4, 0xf4, - 0xd7, 0x6e, 0x77, 0x30, 0x6a, 0x44, 0xff, 0xcf, 0x3d, 0xfb, 0xb2, 0xc0, 0x7e, 0x25, 0x7e, 0x2b, - 0x0a, 0xc0, 0x6b, 0xe8, 0x85, 0x39, 0x23, 0x4e, 0x77, 0x15, 0x35, 0x7d, 0xf8, 0xc7, 0x2d, 0x66, - 0x9e, 0x66, 0x06, 0xbd, 0x39, 0x31, 0x1e, 0xb5, 0x6c, 0xdb, 0x80, 0xe9, 0x31, 0x6f, 0x73, 0x0f, - 0xfd, 0x62, 0x77, 0x34, 0x68, 0xa3, 0x14, 0x7f, 0xfc, 0x13, 0xae, 0x6d, 0x0f, 0x30, 0xa8, 0xf6, - 0xc3, 0x93, 0x56, 0xd1, 0x39, 0x8b, 0xef, 0xed, 0x13, 0x95, 0x79, 0x31, 0x28, 0xff, 0xb5, 0xdb, - 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x0c, 0x07, 0x2f, 0xb8, 0x7d, 0x02, 0x00, 0x00, + // 847 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x54, 0x7f, 0x6f, 0xe3, 0x44, + 0x10, 0xad, 0xcf, 0x6d, 0xda, 0x4e, 0x7e, 0xd4, 0x5d, 0xe5, 0xee, 0xac, 0x20, 0x44, 0x64, 0x38, + 0xd1, 0xa3, 0x52, 0x90, 0x82, 0x90, 0xa0, 0x80, 0x10, 0xb4, 0x51, 0x55, 0x71, 0x4d, 0xa3, 0x4d, + 0x10, 0xfc, 0x67, 0xf9, 0xe2, 0x21, 0xb1, 0x2e, 0xde, 0x75, 0xbd, 0x6b, 0x89, 0xf0, 0xbd, 0xf8, + 0x4a, 0x08, 0xf1, 0x27, 0x9f, 0x00, 0x79, 0x77, 0x6d, 0x6f, 0x72, 0xe2, 0xee, 0xaf, 0xec, 0xcc, + 0xbc, 0x99, 0xcc, 0x7b, 0xfb, 0xbc, 0xd0, 0x63, 0x28, 0x85, 0x8c, 0x24, 0x8e, 0xb2, 0x9c, 0x4b, + 0x4e, 0x4e, 0xaa, 0x78, 0x70, 0x26, 0x93, 0x14, 0x85, 0x8c, 0xd2, 0x4c, 0x97, 0x82, 0x7f, 0x1c, + 0xf0, 0x28, 0xc6, 0x05, 0x8b, 0x23, 0xb6, 0xdc, 0xce, 0x97, 0x6b, 0x4c, 0x91, 0x7c, 0x0d, 0x87, + 0x72, 0x9b, 0xa1, 0xef, 0x0c, 0x9d, 0x8b, 0xde, 0xf8, 0xc5, 0xa8, 0x1e, 0xb7, 0x8f, 0x1c, 0xe9, + 0x9f, 0xc5, 0x36, 0x43, 0xaa, 0x5a, 0xc8, 0x73, 0x38, 0x4e, 0x13, 0x16, 0xe6, 0xf8, 0xe8, 0x3f, + 0x19, 0x3a, 0x17, 0x2e, 0x6d, 0xa5, 0x09, 0xa3, 0xf8, 0x48, 0xfa, 0x70, 0x24, 0xb9, 0x8c, 0x36, + 0xbe, 0xab, 0xd2, 0x3a, 0x20, 0x2f, 0xc1, 0xcb, 0x31, 0x8b, 0x92, 0x3c, 0x94, 0xeb, 0x1c, 0xc5, + 0x9a, 0x6f, 0x62, 0xff, 0x50, 0x01, 0xce, 0x74, 0x7e, 0x51, 0xa5, 0xc9, 0x25, 0x9c, 0x8b, 0x62, + 0xb9, 0x44, 0x21, 0x2c, 0xec, 0x91, 0xc2, 0x7a, 0xa6, 0x50, 0x83, 0x83, 0x3e, 0x40, 0xb3, 0x1a, + 0x69, 0xc1, 0x13, 0x3a, 0xf7, 0x0e, 0x82, 0x7f, 0x1d, 0xf0, 0x26, 0x6c, 0x99, 0x6f, 0x33, 0x99, + 0x70, 0x66, 0xc8, 0x7e, 0xb7, 0x43, 0xf6, 0x65, 0x43, 0x76, 0x1f, 0x69, 0x25, 0x2c, 0xc2, 0x5f, + 0x81, 0x8f, 0x3a, 0x8f, 0x71, 0x88, 0x35, 0x22, 0x7c, 0x83, 0x5b, 0xa5, 0x40, 0x87, 0x3e, 0xab, + 0xeb, 0xcd, 0x80, 0x9f, 0x70, 0xbb, 0xdb, 0x29, 0x64, 0x94, 0xcb, 0x84, 0xad, 0x42, 0xc6, 0xd9, + 0x12, 0x95, 0x48, 0x76, 0xe7, 0xdc, 0x94, 0xa7, 0x65, 0x35, 0xb8, 0x84, 0xde, 0xee, 0x2e, 0x04, + 0xa0, 0xf5, 0xc3, 0x64, 0x7e, 0x7b, 0x7d, 0xef, 0x1d, 0x90, 0x2e, 0x9c, 0xce, 0x27, 0xd7, 0x74, + 0xb2, 0xf8, 0xf1, 0xe1, 0x57, 0xcf, 0x09, 0x7e, 0x81, 0x36, 0xc5, 0x94, 0x4b, 0x9c, 0x25, 0xb8, + 0x44, 0xf2, 0x01, 0x9c, 0x66, 0xe5, 0x21, 0x64, 0x45, 0xaa, 0x38, 0xbb, 0xf4, 0x44, 0x25, 0xa6, + 0x45, 0x5a, 0xde, 0x1e, 0xe3, 0x31, 0x86, 0x49, 0xac, 0x76, 0x3f, 0xa5, 0xad, 0x32, 0xbc, 0x8b, + 0x09, 0x81, 0x43, 0x91, 0xfc, 0x81, 0xe6, 0xf2, 0xd4, 0x39, 0xf8, 0xdb, 0x81, 0xae, 0x9e, 0x3c, + 0xc7, 0x55, 0x8a, 0x4c, 0x92, 0x2b, 0x80, 0xbc, 0x76, 0x88, 0x1a, 0xde, 0x1e, 0x0f, 0xfe, 0xdf, + 0x3d, 0xd4, 0x42, 0x93, 0x0f, 0x01, 0xcc, 0x5e, 0x51, 0x8a, 0xe6, 0xdf, 0xf5, 0xa6, 0xd3, 0x28, + 0x45, 0x72, 0x05, 0xdd, 0x5c, 0xfd, 0x57, 0xa8, 0x72, 0xc2, 0x77, 0x87, 0xee, 0x45, 0x7b, 0xfc, + 0xd4, 0x9e, 0x5e, 0x93, 0xa4, 0x9d, 0xbc, 0x09, 0x04, 0xf9, 0x08, 0xda, 0x29, 0xe6, 0x6f, 0x36, + 0x18, 0xe6, 0x9c, 0x4b, 0xe5, 0xaf, 0x0e, 0x05, 0x9d, 0xa2, 0x9c, 0x4b, 0x0b, 0xa0, 0x48, 0x6a, + 0x53, 0x19, 0xc0, 0xbc, 0xa4, 0xfa, 0xa7, 0x0b, 0xc7, 0x33, 0x9e, 0x30, 0x89, 0x39, 0x19, 0xed, + 0xf8, 0xc5, 0xa2, 0x67, 0x00, 0xa3, 0x9b, 0x48, 0x46, 0x96, 0x41, 0xae, 0x00, 0x1a, 0x5b, 0x28, + 0x62, 0x3b, 0xa2, 0xec, 0xbb, 0x8c, 0x5a, 0x68, 0xf2, 0x02, 0x7a, 0x09, 0xdb, 0x24, 0x0c, 0x43, + 0xa1, 0x25, 0x36, 0xc6, 0xe8, 0xea, 0x6c, 0xa5, 0xfb, 0xe7, 0xd0, 0xd2, 0x84, 0x15, 0xb7, 0xf6, + 0xf8, 0xf9, 0xbe, 0x2a, 0x06, 0x48, 0x0d, 0x8c, 0x7c, 0x0b, 0x83, 0xc6, 0x7a, 0x05, 0xb3, 0x6c, + 0x58, 0xf1, 0xef, 0xd0, 0xc6, 0x9c, 0x3f, 0x37, 0x80, 0x52, 0x0d, 0xf2, 0x3d, 0x74, 0x97, 0x39, + 0x46, 0xca, 0xe6, 0x71, 0x24, 0xd1, 0x6f, 0x19, 0x52, 0x2b, 0xce, 0x57, 0x1b, 0xf3, 0xe8, 0xbc, + 0x2e, 0x7e, 0x1b, 0x2d, 0xaa, 0xc7, 0x86, 0x76, 0xaa, 0x86, 0x9b, 0x48, 0x22, 0xb9, 0x86, 0x33, + 0xfc, 0x3d, 0x4b, 0x72, 0x6b, 0xc4, 0xf1, 0x7b, 0x47, 0xf4, 0x9a, 0x96, 0x72, 0x48, 0x10, 0xc0, + 0x49, 0xa5, 0x74, 0x69, 0xff, 0xbb, 0xe9, 0xab, 0xbb, 0xe9, 0xc4, 0x3b, 0x28, 0xcf, 0x74, 0x72, + 0xff, 0xb0, 0x98, 0x78, 0x4e, 0x70, 0x0f, 0x30, 0x2b, 0x24, 0xc5, 0xc7, 0x02, 0x85, 0x2c, 0x4d, + 0x9c, 0x45, 0x72, 0xad, 0x6e, 0xae, 0x43, 0xd5, 0x99, 0x5c, 0xc2, 0x71, 0xa6, 0xef, 0xcd, 0x5c, + 0xcd, 0xf9, 0x5b, 0x17, 0x4a, 0x2b, 0x44, 0x30, 0x04, 0xb8, 0xc5, 0x77, 0x8d, 0x0b, 0x1e, 0xa0, + 0xfd, 0x2a, 0x11, 0x35, 0xe4, 0x33, 0x38, 0xaf, 0x3f, 0xec, 0xb2, 0xae, 0x5e, 0x05, 0x8d, 0x3f, + 0xab, 0x0a, 0xb3, 0x48, 0xae, 0xcb, 0xe7, 0xa0, 0x0f, 0x47, 0x9b, 0x24, 0x4d, 0xa4, 0x79, 0x37, + 0x75, 0x10, 0x74, 0xa1, 0xad, 0x18, 0x88, 0x8c, 0x33, 0x81, 0xc1, 0xa7, 0xd0, 0x56, 0x1b, 0xe8, + 0x90, 0xf8, 0xcd, 0xf6, 0x7a, 0x6a, 0xbd, 0xea, 0x27, 0xd0, 0xd1, 0x8b, 0x18, 0x64, 0x1f, 0x8e, + 0xca, 0x05, 0x84, 0xef, 0x0c, 0xdd, 0x8b, 0x0e, 0xd5, 0x41, 0xf0, 0x31, 0x74, 0x6f, 0x70, 0x83, + 0x12, 0xdf, 0xc5, 0xc9, 0x83, 0x5e, 0x05, 0xd2, 0xc3, 0xc6, 0x7f, 0x39, 0x70, 0x32, 0x45, 0x39, + 0x2f, 0x55, 0x22, 0x63, 0x70, 0x67, 0x85, 0x24, 0x7d, 0x4b, 0xb7, 0x5a, 0xf2, 0xc1, 0xd3, 0xbd, + 0xac, 0xd9, 0x66, 0x0c, 0xee, 0x2d, 0xee, 0xf4, 0x34, 0xba, 0xda, 0x3d, 0x36, 0xd7, 0x2f, 0xe1, + 0xb0, 0x64, 0x44, 0xac, 0xb2, 0x25, 0xf5, 0xe0, 0xd9, 0x7e, 0xda, 0xb4, 0x7d, 0x03, 0x2d, 0xbd, + 0x3d, 0xb1, 0xbe, 0x8a, 0x1d, 0xd2, 0x03, 0xff, 0xed, 0x82, 0x6e, 0x7e, 0xdd, 0x52, 0x3e, 0xfc, + 0xe2, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xab, 0x26, 0xfc, 0xe1, 0x51, 0x07, 0x00, 0x00, } diff --git a/protos/netstate/netstate.proto b/protos/netstate/netstate.proto index faec219ad..abf1a62d9 100644 --- a/protos/netstate/netstate.proto +++ b/protos/netstate/netstate.proto @@ -4,10 +4,12 @@ syntax = "proto3"; package netstate; +import "timestamp.proto"; + // NetState defines the interface for interacting with the network state persistence layer service NetState { // Put formats and hands off a file path to be saved to boltdb - rpc Put(FilePath) returns (PutResponse); + rpc Put(PutRequest) returns (PutResponse); // Get formats and hands off a file path to get a small value from boltdb rpc Get(GetRequest) returns (GetResponse); // List calls the bolt client's List function and returns all file paths @@ -16,12 +18,66 @@ service NetState { rpc Delete(DeleteRequest) returns (DeleteResponse); } -// FilePath is a request message for the Put rpc call -message FilePath { +message RedundancyScheme { + enum SchemeType { + RS = 0; + } + SchemeType type = 1; + + // these values apply to RS encoding + int64 min_req = 2; // minimum required for reconstruction + int64 total = 3; // total amount of pieces we generated + int64 repair_threshold = 4; // amount of pieces we need to drop to before triggering repair + int64 success_threshold = 5; // amount of pieces we need to store to call it a success +} + +message EncryptionScheme { + enum EncryptionType { + AESGCM = 0; + SECRETBOX = 1; + // only allow authenticated encryption schemes + } + EncryptionType type = 1; + bytes encrypted_encryption_key = 2; + bytes encrypted_starting_nonce = 3; +} + +message RemotePiece { + int64 piece_num = 1; + string node_id = 2; + int64 size = 3; // full size including merkle tree +} + +message RemoteSegment { + RedundancyScheme redundancy = 1; + string piece_name = 2; + repeated RemotePiece remote_pieces = 3; + + bytes merkle_root = 4; // root hash of the hashes of all of these pieces + int64 merkle_size = 5; // amount of space the merkle tree takes up at the end of each chunk +} + +message Pointer { + enum DataType { + INLINE = 0; + REMOTE = 1; + } + + DataType type = 1; + EncryptionScheme encryption = 2; + + bytes inline_segment = 3; + RemoteSegment remote = 4; + bytes encrypted_unencrypted_size = 5; // size of the unencrypted object (!) + + google.protobuf.Timestamp creation_date = 6; + google.protobuf.Timestamp expiration_date = 7; +} + +// PutRequest is a request message for the Put rpc call +message PutRequest { bytes path = 1; - // smallValue is a value too small to be broken up and stored - // in different places - bytes smallValue = 2; + Pointer pointer = 2; } // GetRequest is a request message for the Get rpc call @@ -31,22 +87,22 @@ message GetRequest { // ListRequest is a request message for the List rpc call message ListRequest { - bytes bucket = 1; + bytes starting_path_key = 1; // the Path key in the bucket to start listing + int64 limit = 2; // how many keys to list } // PutResponse is a response message for the Put rpc call message PutResponse { - string confirmation = 1; } // GetResponse is a response message for the Get rpc call message GetResponse { - bytes smallValue = 1; + bytes pointer = 1; // this is a Pointer type marshalled into bytes } // ListResponse is a response message for the List rpc call message ListResponse { - repeated bytes filepaths = 1; + repeated bytes paths = 1; } message DeleteRequest { @@ -55,5 +111,4 @@ message DeleteRequest { // DeleteResponse is a response message for the Delete rpc call message DeleteResponse { - string confirmation = 1; } diff --git a/storage/boltdb/netstate.go b/storage/boltdb/netstate.go index 86ea9ffc1..ecd0ca548 100644 --- a/storage/boltdb/netstate.go +++ b/storage/boltdb/netstate.go @@ -7,53 +7,54 @@ import ( "github.com/boltdb/bolt" ) -// File Path and Value are saved to boltdb -type File struct { - Path []byte `json:"path"` - Value []byte `json:"value"` +// PointerEntry - Path and Pointer are saved as a kv pair to boltdb. +// The following boltdb methods handle the pointer type (defined in +// the protobuf file) after it has been marshalled into bytes. +type PointerEntry struct { + Path []byte + Pointer []byte } const ( - fileBucketName = "files" + pointerBucket = "pointers" ) -// Put saves the file path and value as a kv pair in the "files" bucket -func (client *Client) Put(file File) error { +// Put saves the Path and Pointer as a kv entry in the "pointers" bucket +func (client *Client) Put(pe PointerEntry) error { client.logger.Debug("entering bolt put") return client.db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucketIfNotExists([]byte(fileBucketName)) + b, err := tx.CreateBucketIfNotExists([]byte(pointerBucket)) if err != nil { return err } - fileKey := []byte(file.Path) - return b.Put(fileKey, file.Value) + return b.Put(pe.Path, pe.Pointer) }) } -// Get retrieves the value stored at the file path key -func (client *Client) Get(fileKey []byte) ([]byte, error) { - client.logger.Debug("entering bolt get: " + string(fileKey)) - var fileValue []byte +// Get retrieves the Pointer value stored at the Path key +func (client *Client) Get(pathKey []byte) ([]byte, error) { + client.logger.Debug("entering bolt get: " + string(pathKey)) + var pointerBytes []byte err := client.db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte(fileBucketName)) - v := b.Get(fileKey) + b := tx.Bucket([]byte(pointerBucket)) + v := b.Get(pathKey) if v == nil { - return Error.New("file %#v not found", string(fileKey)) + return Error.New("pointer at %#v not found", string(pathKey)) } - fileValue = v + pointerBytes = v return nil }) - return fileValue, err + return pointerBytes, err } -// List creates a string array of all keys in in the "files" bucket +// List creates a byte array of all path keys in in the "pointers" bucket func (client *Client) List() ([][]byte, error) { client.logger.Debug("entering bolt list") var paths [][]byte err := client.db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte(fileBucketName)) + b := tx.Bucket([]byte(pointerBucket)) err := b.ForEach(func(key, value []byte) error { paths = append(paths, key) @@ -65,10 +66,10 @@ func (client *Client) List() ([][]byte, error) { return paths, err } -// Delete deletes a kv pair from the "files" bucket, given the key -func (client *Client) Delete(fileKey []byte) error { - client.logger.Debug("entering bolt delete: " + string(fileKey)) +// Delete deletes a kv pair from the "pointers" bucket, given the Path key +func (client *Client) Delete(pathKey []byte) error { + client.logger.Debug("entering bolt delete: " + string(pathKey)) return client.db.Update(func(tx *bolt.Tx) error { - return tx.Bucket([]byte(fileBucketName)).Delete(fileKey) + return tx.Bucket([]byte(pointerBucket)).Delete(pathKey) }) } diff --git a/storage/boltdb/netstate_test.go b/storage/boltdb/netstate_test.go index 516ec99e9..39197c5d8 100644 --- a/storage/boltdb/netstate_test.go +++ b/storage/boltdb/netstate_test.go @@ -36,46 +36,46 @@ func TestNetState(t *testing.T) { os.Remove(c.Path) }() - testFile := File{ - Path: []byte(`test/path`), - Value: []byte(`test value`), + testEntry1 := PointerEntry{ + Path: []byte(`test/path`), + Pointer: []byte(`pointer1`), } - testFile2 := File{ - Path: []byte(`test/path2`), - Value: []byte(`value2`), + testEntry2 := PointerEntry{ + Path: []byte(`test/path2`), + Pointer: []byte(`pointer2`), } // tests Put function - if err := c.Put(testFile); err != nil { - t.Error("Failed to save testFile to files bucket") + if err := c.Put(testEntry1); err != nil { + t.Error("Failed to save testFile to pointers bucket") } // tests Get function retrvValue, err := c.Get([]byte("test/path")) if err != nil { - t.Error("Failed to get saved test value") + t.Error("Failed to get saved test pointer") } - if !bytes.Equal(retrvValue, testFile.Value) { - t.Error("Retrieved file was not same as original file") + if !bytes.Equal(retrvValue, testEntry1.Pointer) { + t.Error("Retrieved pointer was not same as put pointer") } // tests Delete function if err := c.Delete([]byte("test/path")); err != nil { - t.Error("Failed to delete testfile") + t.Error("Failed to delete test entry") } // tests List function - if err := c.Put(testFile2); err != nil { - t.Error("Failed to save testFile2 to files bucket") + if err := c.Put(testEntry2); err != nil { + t.Error("Failed to put testEntry2 to pointers bucket") } - testFiles, err := c.List() + testPaths, err := c.List() if err != nil { - t.Error("Failed to list file keys") + t.Error("Failed to list Path keys in pointers bucket") } // tests List + Delete function - if !bytes.Equal(testFiles[0], []byte("test/path2")) { - t.Error("Expected only testFile2 in list") + if !bytes.Equal(testPaths[0], []byte("test/path2")) { + t.Error("Expected only testEntry2 path in list") } }