Enables netstate service to save pointers (#49)
* adds pointer to netstate proto file * generated updated netstate proto * changes boltdb netstate to save pointers as values * updates netstate Put to save Pointers, updates client example to put a pointer, adds grpc status errors, updates tests, changes boltdb 'File' struct to 'PointerEntry' * updates netstate client example and client test to save pointers, updates netstate List and Delete * begins adding netstate-http tests * removes netstate http service * re-adds netstate auth * updates boltdb netstate test * changes encrypted_unencrypted_size from int64 to bytes in netstate proto * updates READMEs
This commit is contained in:
Natalie Villasana
GitHub
parent
9aeff0ab57
commit
6723064bfb
@ -1,21 +0,0 @@
|
||||
# HTTP + BoltDB Crud Interface
|
||||
|
||||
This is an http server that provides a CRUD (create, read, update, delete) interface for storing file paths and small values with BoltDB.
|
||||
For example, you can store a value (i.e. "hello world") at `/my/test/file` and interact with `/my/test/file` through an API, backed by BoltDB.
|
||||
|
||||
To run:
|
||||
```
|
||||
go run cmd/netstate-http/main.go
|
||||
```
|
||||
You can also run using these flags: `-port=<port-number> -prod=<bool> -db=<db-name>`
|
||||
|
||||
Then you can use http methods (Put, Get, List, and Delete) to interact with small values stored on BoltDB.
|
||||
To store a value, put it into the PUT request body using this format:
|
||||
```
|
||||
{
|
||||
"value": "here's my value"
|
||||
}
|
||||
```
|
||||
If you're using Postman, select the `raw` request setting to do this.
|
||||
|
||||
Afterward, you can also use [Bolter](https://github.com/hasit/bolter) or a similar BoltDB viewer to make sure your files were changed as expected.
|
||||
@ -1,75 +0,0 @@
|
||||
// Copyright (C) 2018 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/julienschmidt/httprouter"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"storj.io/storj/netstate/routes"
|
||||
"storj.io/storj/storage/boltdb"
|
||||
)
|
||||
|
||||
var (
|
||||
port int
|
||||
dbPath string
|
||||
prod bool
|
||||
)
|
||||
|
||||
func initializeFlags() {
|
||||
flag.IntVar(&port, "port", 3000, "port")
|
||||
flag.StringVar(&dbPath, "db", "netstate.db", "db path")
|
||||
flag.BoolVar(&prod, "prod", false, "The environment this service is running in")
|
||||
flag.Parse()
|
||||
}
|
||||
|
||||
func main() {
|
||||
err := Main()
|
||||
if err != nil {
|
||||
log.Fatalf("fatal error: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Main allows simplified error handling
|
||||
func Main() error {
|
||||
initializeFlags()
|
||||
|
||||
// No err here because no vars passed into NewDevelopment().
|
||||
// The default won't return an error, but if args are passed in,
|
||||
// then there will need to be error handling.
|
||||
logger, _ := zap.NewDevelopment()
|
||||
if prod {
|
||||
logger, _ = zap.NewProduction()
|
||||
}
|
||||
defer logger.Sync()
|
||||
logger.Info(fmt.Sprintf("serving on %d", port))
|
||||
|
||||
bdb, err := boltdb.New(logger, dbPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer bdb.Close()
|
||||
|
||||
routes := routes.NewNetStateRoutes(logger, bdb)
|
||||
|
||||
return http.ListenAndServe(fmt.Sprintf(":%d", port), start(routes))
|
||||
}
|
||||
|
||||
func start(f *routes.NetStateRoutes) *httprouter.Router {
|
||||
router := httprouter.New()
|
||||
|
||||
router.PUT("/file/*path", f.Put)
|
||||
router.GET("/file/*path", f.Get)
|
||||
router.GET("/file", f.List)
|
||||
router.DELETE("/file/*path", f.Delete)
|
||||
|
||||
return router
|
||||
}
|
||||
@ -1,15 +1,28 @@
|
||||
# gRPC Server + BoltDB Crud Interface
|
||||
|
||||
This is a gRPC server which handles CRUD (create, read, update, delete) requests for storing file paths and small values to BoltDB.
|
||||
For example, you can store a value (i.e. "hello world") at `/my/test/file` and interact with `/my/test/file` through an API, backed by BoltDB.
|
||||
This is a gRPC server which handles CRUD (create, read, update, delete) requests for storing pointers at given paths in BoltDB.
|
||||
An example Put Request to store a path and pointer would look like this:
|
||||
```
|
||||
pr := proto.PutRequest{
|
||||
Path: []byte("here's/a/pointer/path"),
|
||||
Pointer: &proto.Pointer{
|
||||
Type: proto.Pointer_INLINE,
|
||||
Encryption: &proto.EncryptionScheme{
|
||||
EncryptedEncryptionKey: []byte("key"),
|
||||
EncryptedStartingNonce: []byte("nonce"),
|
||||
},
|
||||
InlineSegment: []byte("littledata"),
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
To run the server:
|
||||
```
|
||||
go run cmd/netstate-rpc/server/main.go
|
||||
go run cmd/netstate/main.go
|
||||
```
|
||||
You can also run using these flags: `-port=<port-number> -prod=<bool> -db=<db-name>`
|
||||
|
||||
You can then write a client program using the client library to access the Put, Get, List, and Delete methods to create/interact with small values stored in a BoltDB.
|
||||
You can then write a client program using the client library to access the Put, Get, List, and Delete methods to create and interact with pointer entries stored in BoltDB.
|
||||
An example program utilizing these functions can be found at `storj.io/storj/examples/netstate-client/main.go`.
|
||||
|
||||
Afterward, you can also use [Bolter](https://github.com/hasit/bolter) or a similar BoltDB viewer to make sure your files were changed as expected.
|
||||
@ -43,6 +43,7 @@ func main() {
|
||||
|
||||
bdb, err := boltdb.New(logger, dbPath)
|
||||
if err != nil {
|
||||
logger.Fatal("failed to initiate boltdb", zap.Error(err))
|
||||
return
|
||||
}
|
||||
defer bdb.Close()
|
||||
@ -55,6 +56,7 @@ func main() {
|
||||
|
||||
grpcServer := grpc.NewServer()
|
||||
proto.RegisterNetStateServer(grpcServer, netstate.NewServer(bdb, logger))
|
||||
logger.Debug(fmt.Sprintf("server listening on port %d", port))
|
||||
|
||||
defer grpcServer.GracefulStop()
|
||||
err = grpcServer.Serve(lis)
|
||||
@ -1,8 +1,8 @@
|
||||
# gRPC Client + BoltDB Crud Interface
|
||||
|
||||
This is an example gRPC client which makes CRUD requests (create, read, update, delete) for storing file paths and small values to BoltDB.
|
||||
This is an example gRPC client which makes CRUD requests (create, read, update, delete) for storing pointers at given paths in BoltDB.
|
||||
|
||||
The gRPC server at `storj.io/storj/cmd/netstate-rpc/server/main.go` needs to be running for this to work.
|
||||
The gRPC server at `storj.io/storj/cmd/netstate/main.go` needs to be running for this to work.
|
||||
|
||||
To run the client:
|
||||
```
|
||||
@ -10,7 +10,7 @@ go run examples/netstate-client/main.go
|
||||
```
|
||||
You can change the port number with a flag if necessary: `-port=<port-number>`
|
||||
|
||||
Afterward, you can use [Bolter](https://github.com/hasit/bolter) or a similar BoltDB viewer to make sure your files were changed as expected.
|
||||
Afterward, you can use [Bolter](https://github.com/hasit/bolter) or a similar BoltDB viewer to make sure your pointer entries were changed as expected.
|
||||
|
||||
If changes are made to `storj.io/storj/protos/netstate/netstate.proto`, the protobuf file will need to be regenerated by running `go generate` inside `protos/netstate`.
|
||||
|
||||
|
||||
@ -6,10 +6,13 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
proto "storj.io/storj/protos/netstate"
|
||||
)
|
||||
@ -18,10 +21,6 @@ var (
|
||||
port string
|
||||
)
|
||||
|
||||
const (
|
||||
success string = "success"
|
||||
)
|
||||
|
||||
func initializeFlags() {
|
||||
flag.StringVar(&port, "port", ":8080", "port")
|
||||
flag.Parse()
|
||||
@ -40,73 +39,93 @@ func main() {
|
||||
|
||||
client := proto.NewNetStateClient(conn)
|
||||
|
||||
logger.Debug(fmt.Sprintf("client dialed port %s", port))
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Examples file paths to be saved
|
||||
fp := proto.FilePath{
|
||||
Path: []byte("welcome/to/my/file/journey"),
|
||||
SmallValue: []byte("granola"),
|
||||
// Example pointer paths to put
|
||||
pr1 := proto.PutRequest{
|
||||
Path: []byte("welcome/to/my/pointer/journey"),
|
||||
Pointer: &proto.Pointer{
|
||||
Type: proto.Pointer_INLINE,
|
||||
Encryption: &proto.EncryptionScheme{
|
||||
EncryptedEncryptionKey: []byte("key"),
|
||||
EncryptedStartingNonce: []byte("nonce"),
|
||||
},
|
||||
InlineSegment: []byte("granola"),
|
||||
},
|
||||
}
|
||||
fp2 := proto.FilePath{
|
||||
Path: []byte("so/many/file/paths"),
|
||||
SmallValue: []byte("m&ms"),
|
||||
pr2 := proto.PutRequest{
|
||||
Path: []byte("so/many/pointers"),
|
||||
Pointer: &proto.Pointer{
|
||||
Type: proto.Pointer_INLINE,
|
||||
Encryption: &proto.EncryptionScheme{
|
||||
EncryptedEncryptionKey: []byte("key"),
|
||||
EncryptedStartingNonce: []byte("nonce"),
|
||||
},
|
||||
InlineSegment: []byte("m&ms"),
|
||||
},
|
||||
}
|
||||
fp3 := proto.FilePath{
|
||||
Path: []byte("another/file/path/for/the/pile"),
|
||||
SmallValue: []byte("popcorn"),
|
||||
pr3 := proto.PutRequest{
|
||||
Path: []byte("another/pointer/for/the/pile"),
|
||||
Pointer: &proto.Pointer{
|
||||
Type: proto.Pointer_INLINE,
|
||||
Encryption: &proto.EncryptionScheme{
|
||||
EncryptedEncryptionKey: []byte("key"),
|
||||
EncryptedStartingNonce: []byte("nonce"),
|
||||
},
|
||||
InlineSegment: []byte("popcorn"),
|
||||
},
|
||||
}
|
||||
|
||||
// Example Puts
|
||||
putRes, err := client.Put(ctx, &fp)
|
||||
if err != nil || putRes.Confirmation != success {
|
||||
_, err = client.Put(ctx, &pr1)
|
||||
if err != nil || status.Code(err) == codes.Internal {
|
||||
logger.Error("failed to put", zap.Error(err))
|
||||
}
|
||||
putRes2, err := client.Put(ctx, &fp2)
|
||||
if err != nil || putRes2.Confirmation != success {
|
||||
_, err = client.Put(ctx, &pr2)
|
||||
if err != nil || status.Code(err) == codes.Internal {
|
||||
logger.Error("failed to put", zap.Error(err))
|
||||
}
|
||||
putRes3, err := client.Put(ctx, &fp3)
|
||||
if err != nil || putRes3.Confirmation != success {
|
||||
_, err = client.Put(ctx, &pr3)
|
||||
if err != nil || status.Code(err) == codes.Internal {
|
||||
logger.Error("failed to put", zap.Error(err))
|
||||
}
|
||||
|
||||
// Example Get
|
||||
getReq := proto.GetRequest{
|
||||
Path: []byte("so/many/file/paths"),
|
||||
Path: []byte("so/many/pointers"),
|
||||
}
|
||||
getRes, err := client.Get(ctx, &getReq)
|
||||
if err != nil {
|
||||
if err != nil || status.Code(err) == codes.Internal {
|
||||
logger.Error("failed to get", zap.Error(err))
|
||||
}
|
||||
value := string(getRes.SmallValue)
|
||||
logger.Debug("get response: " + value)
|
||||
pointer := string(getRes.Pointer)
|
||||
logger.Debug("get response: " + pointer)
|
||||
|
||||
// Example List
|
||||
listReq := proto.ListRequest{
|
||||
// This Bucket value isn't actually used by List() now,
|
||||
// but in the future could be used to select specific
|
||||
// buckets to list from.
|
||||
Bucket: []byte("files"),
|
||||
// This pagination functionality doesn't work yet.
|
||||
// The given arguments are placeholders.
|
||||
StartingPathKey: []byte("test/pointer/path"),
|
||||
Limit: 5,
|
||||
}
|
||||
listRes, err := client.List(ctx, &listReq)
|
||||
if err != nil {
|
||||
if err != nil || status.Code(err) == codes.Internal {
|
||||
logger.Error("failed to list file paths")
|
||||
}
|
||||
var stringList []string
|
||||
for _, pathByte := range listRes.Filepaths {
|
||||
for _, pathByte := range listRes.Paths {
|
||||
stringList = append(stringList, string(pathByte))
|
||||
}
|
||||
logger.Debug("listed paths: " + strings.Join(stringList, ", "))
|
||||
|
||||
// Example Delete
|
||||
delReq := proto.DeleteRequest{
|
||||
Path: []byte("welcome/to/my/file/journey"),
|
||||
Path: []byte("welcome/to/my/pointer/journey"),
|
||||
}
|
||||
delRes, err := client.Delete(ctx, &delReq)
|
||||
if err != nil {
|
||||
logger.Error("failed to delete: 'welcome/to/my/file/journey'")
|
||||
}
|
||||
if delRes.Confirmation == "success" {
|
||||
logger.Debug("deleted: welcome/to/my/file/journey")
|
||||
_, err = client.Delete(ctx, &delReq)
|
||||
if err != nil || status.Code(err) == codes.Internal {
|
||||
logger.Error("failed to delete: " + string(delReq.Path))
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,136 +0,0 @@
|
||||
// Copyright (C) 2018 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package routes
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/julienschmidt/httprouter"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"storj.io/storj/storage/boltdb"
|
||||
)
|
||||
|
||||
// NetStateRoutes maintains access to a boltdb client and zap logger
|
||||
type NetStateRoutes struct {
|
||||
DB *boltdb.Client
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// Message contains the small value provided by the user to be stored
|
||||
type Message struct {
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
// NewNetStateRoutes instantiates NetStateRoutes
|
||||
func NewNetStateRoutes(logger *zap.Logger, db *boltdb.Client) *NetStateRoutes {
|
||||
return &NetStateRoutes{
|
||||
DB: db,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// Put takes the given path and small value from the user and formats the values
|
||||
// to be given to boltdb.Put
|
||||
func (n *NetStateRoutes) Put(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
|
||||
n.logger.Debug("entering netstate http put")
|
||||
|
||||
givenPath := ps.ByName("path")
|
||||
var msg Message
|
||||
|
||||
decoder := json.NewDecoder(r.Body)
|
||||
err := decoder.Decode(&msg)
|
||||
if err != nil {
|
||||
http.Error(w, "bad request: err decoding response", http.StatusBadRequest)
|
||||
n.logger.Error("err decoding response", zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
file := boltdb.File{
|
||||
Path: []byte(givenPath),
|
||||
Value: []byte(msg.Value),
|
||||
}
|
||||
|
||||
if err := n.DB.Put(file); err != nil {
|
||||
http.Error(w, "err putting file", http.StatusInternalServerError)
|
||||
n.logger.Error("err putting file", zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
n.logger.Debug("put to the db: " + givenPath)
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
fmt.Fprintf(w, "PUT to %s\n", givenPath)
|
||||
}
|
||||
|
||||
// Get takes the given file path from the user and calls the bolt client's Get function
|
||||
func (n *NetStateRoutes) Get(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
|
||||
n.logger.Debug("entering netstate http get")
|
||||
|
||||
fileKey := ps.ByName("path")
|
||||
|
||||
fileValue, err := n.DB.Get([]byte(fileKey))
|
||||
if err != nil {
|
||||
http.Error(w, "err getting file", http.StatusInternalServerError)
|
||||
n.logger.Error("err getting file", zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
_, err = w.Write(fileValue)
|
||||
if err != nil {
|
||||
n.logger.Error("err writing response", zap.Error(err))
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
n.logger.Debug("response written: " + string(fileValue))
|
||||
}
|
||||
|
||||
// List calls the bolt client's List function and responds with a list of all saved file paths
|
||||
// or "filekeys"
|
||||
func (n *NetStateRoutes) List(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
|
||||
n.logger.Debug("entering netstate http list")
|
||||
|
||||
filePaths, err := n.DB.List()
|
||||
if err != nil {
|
||||
http.Error(w, "internal error: unable to list paths", http.StatusInternalServerError)
|
||||
n.logger.Error("err listing file paths", zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
var pathList []string
|
||||
for _, path := range filePaths {
|
||||
pathList = append(pathList, string(path))
|
||||
}
|
||||
|
||||
bytes, err := json.Marshal(pathList)
|
||||
if err != nil {
|
||||
http.Error(w, "internal error: unable to marshal path list", http.StatusInternalServerError)
|
||||
n.logger.Error("err marshaling path list", zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
_, err = w.Write(bytes)
|
||||
if err != nil {
|
||||
n.logger.Error("err writing response", zap.Error(err))
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
n.logger.Debug("response written: " + strings.Join(pathList, ", "))
|
||||
}
|
||||
|
||||
// Delete takes a given file path and calls the bolt client's Delete function
|
||||
func (n *NetStateRoutes) Delete(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
|
||||
n.logger.Debug("entering netstate http delete")
|
||||
|
||||
fileKey := ps.ByName("path")
|
||||
if err := n.DB.Delete([]byte(fileKey)); err != nil {
|
||||
http.Error(w, "internal error: unable to delete file", http.StatusInternalServerError)
|
||||
n.logger.Error("err deleting file", zap.Error(err))
|
||||
return
|
||||
}
|
||||
n.logger.Debug("deleted file: " + fileKey)
|
||||
w.WriteHeader(204)
|
||||
fmt.Fprintf(w, "Deleted file key: %s", fileKey)
|
||||
}
|
||||
@ -10,10 +10,13 @@ import (
|
||||
"net"
|
||||
"testing"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"go.uber.org/zap"
|
||||
"google.golang.org/grpc"
|
||||
proto "storj.io/storj/protos/netstate"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
pb "storj.io/storj/protos/netstate"
|
||||
)
|
||||
|
||||
func TestNetStateClient(t *testing.T) {
|
||||
@ -22,12 +25,12 @@ func TestNetStateClient(t *testing.T) {
|
||||
lis, err := net.Listen("tcp", fmt.Sprintf(":%d", 9000))
|
||||
assert.NoError(t, err)
|
||||
|
||||
mdb := &mockDB{
|
||||
mdb := &MockDB{
|
||||
timesCalled: 0,
|
||||
}
|
||||
|
||||
grpcServer := grpc.NewServer()
|
||||
proto.RegisterNetStateServer(grpcServer, NewServer(mdb, logger))
|
||||
pb.RegisterNetStateServer(grpcServer, NewServer(mdb, logger))
|
||||
|
||||
defer grpcServer.GracefulStop()
|
||||
go grpcServer.Serve(lis)
|
||||
@ -36,14 +39,21 @@ func TestNetStateClient(t *testing.T) {
|
||||
conn, err := grpc.Dial(address, grpc.WithInsecure())
|
||||
assert.NoError(t, err)
|
||||
|
||||
c := proto.NewNetStateClient(conn)
|
||||
c := pb.NewNetStateClient(conn)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// example file path to put/get
|
||||
fp := proto.FilePath{
|
||||
Path: []byte("here/is/a/path"),
|
||||
SmallValue: []byte("oatmeal"),
|
||||
pr1 := pb.PutRequest{
|
||||
Path: []byte("here/is/a/path"),
|
||||
Pointer: &pb.Pointer{
|
||||
Type: pb.Pointer_INLINE,
|
||||
Encryption: &pb.EncryptionScheme{
|
||||
EncryptedEncryptionKey: []byte("key"),
|
||||
EncryptedStartingNonce: []byte("nonce"),
|
||||
},
|
||||
InlineSegment: []byte("oatmeal"),
|
||||
},
|
||||
}
|
||||
|
||||
if mdb.timesCalled != 0 {
|
||||
@ -51,34 +61,37 @@ func TestNetStateClient(t *testing.T) {
|
||||
}
|
||||
|
||||
// Tests Server.Put
|
||||
putRes, err := c.Put(ctx, &fp)
|
||||
assert.NoError(t, err)
|
||||
|
||||
if putRes.Confirmation != "success" {
|
||||
t.Error("Failed to receive success Put response")
|
||||
_, err = c.Put(ctx, &pr1)
|
||||
if err != nil || status.Code(err) == codes.Internal {
|
||||
t.Error("Failed to Put")
|
||||
}
|
||||
|
||||
if mdb.timesCalled != 1 {
|
||||
t.Error("Failed to call mockdb correctly")
|
||||
}
|
||||
|
||||
if !bytes.Equal(mdb.puts[0].Path, fp.Path) {
|
||||
if !bytes.Equal(mdb.puts[0].Path, pr1.Path) {
|
||||
t.Error("Expected saved path to equal given path")
|
||||
}
|
||||
|
||||
if !bytes.Equal(mdb.puts[0].Value, fp.SmallValue) {
|
||||
pointerBytes, err := proto.Marshal(pr1.Pointer)
|
||||
if err != nil {
|
||||
t.Error("failed to marshal test pointer")
|
||||
}
|
||||
|
||||
if !bytes.Equal(mdb.puts[0].Pointer, pointerBytes) {
|
||||
t.Error("Expected saved value to equal given value")
|
||||
}
|
||||
|
||||
// Tests Server.Get
|
||||
getReq := proto.GetRequest{
|
||||
getReq := pb.GetRequest{
|
||||
Path: []byte("here/is/a/path"),
|
||||
}
|
||||
|
||||
getRes, err := c.Get(ctx, &getReq)
|
||||
assert.NoError(t, err)
|
||||
|
||||
if !bytes.Equal(getRes.SmallValue, fp.SmallValue) {
|
||||
if !bytes.Equal(getRes.Pointer, pointerBytes) {
|
||||
t.Error("Expected to get same content that was put")
|
||||
}
|
||||
|
||||
@ -86,17 +99,22 @@ func TestNetStateClient(t *testing.T) {
|
||||
t.Error("Failed to call mockdb correct number of times")
|
||||
}
|
||||
|
||||
// Puts another file path to test delete and list
|
||||
fp2 := proto.FilePath{
|
||||
Path: []byte("here/is/another/path"),
|
||||
SmallValue: []byte("raisins"),
|
||||
// Puts another pointer entry to test delete and list
|
||||
pr2 := pb.PutRequest{
|
||||
Path: []byte("here/is/another/path"),
|
||||
Pointer: &pb.Pointer{
|
||||
Type: pb.Pointer_INLINE,
|
||||
Encryption: &pb.EncryptionScheme{
|
||||
EncryptedEncryptionKey: []byte("key"),
|
||||
EncryptedStartingNonce: []byte("nonce"),
|
||||
},
|
||||
InlineSegment: []byte("raisins"),
|
||||
},
|
||||
}
|
||||
|
||||
putRes2, err := c.Put(ctx, &fp2)
|
||||
assert.NoError(t, err)
|
||||
|
||||
if putRes2.Confirmation != "success" {
|
||||
t.Error("Failed to receive success Put response")
|
||||
_, err = c.Put(ctx, &pr2)
|
||||
if err != nil || status.Code(err) == codes.Internal {
|
||||
t.Error("Failed to Put")
|
||||
}
|
||||
|
||||
if mdb.timesCalled != 3 {
|
||||
@ -104,17 +122,13 @@ func TestNetStateClient(t *testing.T) {
|
||||
}
|
||||
|
||||
// Test Server.Delete
|
||||
delReq := proto.DeleteRequest{
|
||||
delReq := pb.DeleteRequest{
|
||||
Path: []byte("here/is/a/path"),
|
||||
}
|
||||
|
||||
delRes, err := c.Delete(ctx, &delReq)
|
||||
if err != nil {
|
||||
t.Error("Failed to delete file path")
|
||||
}
|
||||
|
||||
if delRes.Confirmation != "success" {
|
||||
t.Error("Failed to receive success delete response")
|
||||
_, err = c.Delete(ctx, &delReq)
|
||||
if err != nil || status.Code(err) == codes.Internal {
|
||||
t.Error("Failed to delete")
|
||||
}
|
||||
|
||||
if mdb.timesCalled != 4 {
|
||||
@ -122,8 +136,11 @@ func TestNetStateClient(t *testing.T) {
|
||||
}
|
||||
|
||||
// Tests Server.List
|
||||
listReq := proto.ListRequest{
|
||||
Bucket: []byte("files"),
|
||||
listReq := pb.ListRequest{
|
||||
// This pagination functionality doesn't work yet.
|
||||
// The given arguments are placeholders.
|
||||
StartingPathKey: []byte("test/pointer/path"),
|
||||
Limit: 5,
|
||||
}
|
||||
|
||||
listRes, err := c.List(ctx, &listReq)
|
||||
@ -131,7 +148,7 @@ func TestNetStateClient(t *testing.T) {
|
||||
t.Error("Failed to list file paths")
|
||||
}
|
||||
|
||||
if !bytes.Equal(listRes.Filepaths[0], []byte("here/is/another/path")) {
|
||||
if !bytes.Equal(listRes.Paths[0], []byte("here/is/another/path")) {
|
||||
t.Error("Failed to list correct file path")
|
||||
}
|
||||
|
||||
|
||||
@ -6,9 +6,12 @@ package netstate
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
"go.uber.org/zap"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
proto "storj.io/storj/protos/netstate"
|
||||
pb "storj.io/storj/protos/netstate"
|
||||
"storj.io/storj/storage/boltdb"
|
||||
)
|
||||
|
||||
@ -30,75 +33,77 @@ func NewServer(db DB, logger *zap.Logger) *Server {
|
||||
// and makes it easier in the future to substitute
|
||||
// db clients other than bolt
|
||||
type DB interface {
|
||||
Put(boltdb.File) error
|
||||
Put(boltdb.PointerEntry) error
|
||||
Get([]byte) ([]byte, error)
|
||||
List() ([][]byte, error)
|
||||
Delete([]byte) error
|
||||
}
|
||||
|
||||
// Put formats and hands off a file path to be saved to boltdb
|
||||
func (s *Server) Put(ctx context.Context, filepath *proto.FilePath) (*proto.PutResponse, error) {
|
||||
func (s *Server) Put(ctx context.Context, putReq *pb.PutRequest) (*pb.PutResponse, error) {
|
||||
s.logger.Debug("entering netstate put")
|
||||
|
||||
file := boltdb.File{
|
||||
Path: []byte(filepath.Path),
|
||||
Value: []byte(filepath.SmallValue),
|
||||
pointerBytes, err := proto.Marshal(putReq.Pointer)
|
||||
if err != nil {
|
||||
s.logger.Error("err marshaling pointer", zap.Error(err))
|
||||
return nil, status.Errorf(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
if err := s.DB.Put(file); err != nil {
|
||||
s.logger.Error("err putting file", zap.Error(err))
|
||||
return nil, err
|
||||
pe := boltdb.PointerEntry{
|
||||
Path: putReq.Path,
|
||||
Pointer: pointerBytes,
|
||||
}
|
||||
s.logger.Debug("put to the db: " + string(file.Path))
|
||||
|
||||
return &proto.PutResponse{
|
||||
Confirmation: "success",
|
||||
}, nil
|
||||
if err := s.DB.Put(pe); err != nil {
|
||||
s.logger.Error("err putting pointer", zap.Error(err))
|
||||
return nil, status.Errorf(codes.Internal, err.Error())
|
||||
}
|
||||
s.logger.Debug("put to the db: " + string(pe.Path))
|
||||
|
||||
return &pb.PutResponse{}, nil
|
||||
}
|
||||
|
||||
// Get formats and hands off a file path to get from boltdb
|
||||
func (s *Server) Get(ctx context.Context, req *proto.GetRequest) (*proto.GetResponse, error) {
|
||||
func (s *Server) Get(ctx context.Context, req *pb.GetRequest) (*pb.GetResponse, error) {
|
||||
s.logger.Debug("entering netstate get")
|
||||
|
||||
fileValue, err := s.DB.Get(req.Path)
|
||||
pointerBytes, err := s.DB.Get(req.Path)
|
||||
if err != nil {
|
||||
s.logger.Error("err getting file", zap.Error(err))
|
||||
return nil, err
|
||||
return nil, status.Errorf(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
return &proto.GetResponse{
|
||||
SmallValue: fileValue,
|
||||
return &pb.GetResponse{
|
||||
Pointer: pointerBytes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// List calls the bolt client's List function and returns all file paths
|
||||
func (s *Server) List(ctx context.Context, req *proto.ListRequest) (*proto.ListResponse, error) {
|
||||
// List calls the bolt client's List function and returns all Path keys in the Pointers bucket
|
||||
func (s *Server) List(ctx context.Context, req *pb.ListRequest) (*pb.ListResponse, error) {
|
||||
s.logger.Debug("entering netstate list")
|
||||
|
||||
filePaths, err := s.DB.List()
|
||||
pathKeys, err := s.DB.List()
|
||||
if err != nil {
|
||||
s.logger.Error("err listing file paths", zap.Error(err))
|
||||
return nil, err
|
||||
s.logger.Error("err listing path keys", zap.Error(err))
|
||||
return nil, status.Errorf(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
s.logger.Debug("file paths retrieved")
|
||||
return &proto.ListResponse{
|
||||
// filePaths is an array of byte arrays
|
||||
Filepaths: filePaths,
|
||||
s.logger.Debug("path keys retrieved")
|
||||
return &pb.ListResponse{
|
||||
// pathKeys is an array of byte arrays
|
||||
Paths: pathKeys,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Delete formats and hands off a file path to delete from boltdb
|
||||
func (s *Server) Delete(ctx context.Context, req *proto.DeleteRequest) (*proto.DeleteResponse, error) {
|
||||
func (s *Server) Delete(ctx context.Context, req *pb.DeleteRequest) (*pb.DeleteResponse, error) {
|
||||
s.logger.Debug("entering netstate delete")
|
||||
|
||||
err := s.DB.Delete(req.Path)
|
||||
if err != nil {
|
||||
s.logger.Error("err deleting file", zap.Error(err))
|
||||
return nil, err
|
||||
s.logger.Error("err deleting pointer entry", zap.Error(err))
|
||||
return nil, status.Errorf(codes.Internal, err.Error())
|
||||
}
|
||||
s.logger.Debug("deleted: " + string(req.Path))
|
||||
return &proto.DeleteResponse{
|
||||
Confirmation: "success",
|
||||
}, nil
|
||||
s.logger.Debug("deleted pointer at path: " + string(req.Path))
|
||||
return &pb.DeleteResponse{}, nil
|
||||
}
|
||||
|
||||
@ -9,44 +9,45 @@ import (
|
||||
"storj.io/storj/storage/boltdb"
|
||||
)
|
||||
|
||||
type mockDB struct {
|
||||
// MockDB mocks db functionality for testing
|
||||
type MockDB struct {
|
||||
timesCalled int
|
||||
puts []boltdb.File
|
||||
filePaths [][]byte
|
||||
puts []boltdb.PointerEntry
|
||||
pathKeys [][]byte
|
||||
}
|
||||
|
||||
func (m *mockDB) Put(f boltdb.File) error {
|
||||
func (m *MockDB) Put(f boltdb.PointerEntry) error {
|
||||
m.timesCalled++
|
||||
m.puts = append(m.puts, f)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockDB) Get(path []byte) ([]byte, error) {
|
||||
func (m *MockDB) Get(path []byte) ([]byte, error) {
|
||||
m.timesCalled++
|
||||
|
||||
for _, file := range m.puts {
|
||||
if bytes.Equal(path, file.Path) {
|
||||
return file.Value, nil
|
||||
for _, pointerEntry := range m.puts {
|
||||
if bytes.Equal(path, pointerEntry.Path) {
|
||||
return pointerEntry.Pointer, nil
|
||||
}
|
||||
}
|
||||
panic("failed to get the given file")
|
||||
}
|
||||
|
||||
func (m *mockDB) List() ([][]byte, error) {
|
||||
func (m *MockDB) List() ([][]byte, error) {
|
||||
m.timesCalled++
|
||||
|
||||
for _, file := range m.puts {
|
||||
m.filePaths = append(m.filePaths, file.Path)
|
||||
for _, putReq := range m.puts {
|
||||
m.pathKeys = append(m.pathKeys, putReq.Path)
|
||||
}
|
||||
|
||||
return m.filePaths, nil
|
||||
return m.pathKeys, nil
|
||||
}
|
||||
|
||||
func (m *mockDB) Delete(path []byte) error {
|
||||
func (m *MockDB) Delete(path []byte) error {
|
||||
m.timesCalled++
|
||||
|
||||
for i, file := range m.puts {
|
||||
if bytes.Equal(path, file.Path) {
|
||||
for i, pointerEntry := range m.puts {
|
||||
if bytes.Equal(path, pointerEntry.Path) {
|
||||
m.puts = append(m.puts[:i], m.puts[i+1:]...)
|
||||
}
|
||||
}
|
||||
|
||||
@ -8,7 +8,12 @@ It is generated from these files:
|
||||
netstate.proto
|
||||
|
||||
It has these top-level messages:
|
||||
FilePath
|
||||
RedundancyScheme
|
||||
EncryptionScheme
|
||||
RemotePiece
|
||||
RemoteSegment
|
||||
Pointer
|
||||
PutRequest
|
||||
GetRequest
|
||||
ListRequest
|
||||
PutResponse
|
||||
@ -22,6 +27,7 @@ package netstate
|
||||
import proto "github.com/golang/protobuf/proto"
|
||||
import fmt "fmt"
|
||||
import math "math"
|
||||
import google_protobuf "github.com/golang/protobuf/ptypes/timestamp"
|
||||
|
||||
import (
|
||||
context "golang.org/x/net/context"
|
||||
@ -39,29 +45,316 @@ var _ = math.Inf
|
||||
// proto package needs to be updated.
|
||||
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
|
||||
|
||||
// FilePath is a request message for the Put rpc call
|
||||
type FilePath struct {
|
||||
Path []byte `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
|
||||
// smallValue is a value too small to be broken up and stored
|
||||
// in different places
|
||||
SmallValue []byte `protobuf:"bytes,2,opt,name=smallValue,proto3" json:"smallValue,omitempty"`
|
||||
type RedundancyScheme_SchemeType int32
|
||||
|
||||
const (
|
||||
RedundancyScheme_RS RedundancyScheme_SchemeType = 0
|
||||
)
|
||||
|
||||
var RedundancyScheme_SchemeType_name = map[int32]string{
|
||||
0: "RS",
|
||||
}
|
||||
var RedundancyScheme_SchemeType_value = map[string]int32{
|
||||
"RS": 0,
|
||||
}
|
||||
|
||||
func (m *FilePath) Reset() { *m = FilePath{} }
|
||||
func (m *FilePath) String() string { return proto.CompactTextString(m) }
|
||||
func (*FilePath) ProtoMessage() {}
|
||||
func (*FilePath) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
|
||||
func (x RedundancyScheme_SchemeType) String() string {
|
||||
return proto.EnumName(RedundancyScheme_SchemeType_name, int32(x))
|
||||
}
|
||||
func (RedundancyScheme_SchemeType) EnumDescriptor() ([]byte, []int) {
|
||||
return fileDescriptor0, []int{0, 0}
|
||||
}
|
||||
|
||||
func (m *FilePath) GetPath() []byte {
|
||||
type EncryptionScheme_EncryptionType int32
|
||||
|
||||
const (
|
||||
EncryptionScheme_AESGCM EncryptionScheme_EncryptionType = 0
|
||||
EncryptionScheme_SECRETBOX EncryptionScheme_EncryptionType = 1
|
||||
)
|
||||
|
||||
var EncryptionScheme_EncryptionType_name = map[int32]string{
|
||||
0: "AESGCM",
|
||||
1: "SECRETBOX",
|
||||
}
|
||||
var EncryptionScheme_EncryptionType_value = map[string]int32{
|
||||
"AESGCM": 0,
|
||||
"SECRETBOX": 1,
|
||||
}
|
||||
|
||||
func (x EncryptionScheme_EncryptionType) String() string {
|
||||
return proto.EnumName(EncryptionScheme_EncryptionType_name, int32(x))
|
||||
}
|
||||
func (EncryptionScheme_EncryptionType) EnumDescriptor() ([]byte, []int) {
|
||||
return fileDescriptor0, []int{1, 0}
|
||||
}
|
||||
|
||||
type Pointer_DataType int32
|
||||
|
||||
const (
|
||||
Pointer_INLINE Pointer_DataType = 0
|
||||
Pointer_REMOTE Pointer_DataType = 1
|
||||
)
|
||||
|
||||
var Pointer_DataType_name = map[int32]string{
|
||||
0: "INLINE",
|
||||
1: "REMOTE",
|
||||
}
|
||||
var Pointer_DataType_value = map[string]int32{
|
||||
"INLINE": 0,
|
||||
"REMOTE": 1,
|
||||
}
|
||||
|
||||
func (x Pointer_DataType) String() string {
|
||||
return proto.EnumName(Pointer_DataType_name, int32(x))
|
||||
}
|
||||
func (Pointer_DataType) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{4, 0} }
|
||||
|
||||
type RedundancyScheme struct {
|
||||
Type RedundancyScheme_SchemeType `protobuf:"varint,1,opt,name=type,enum=netstate.RedundancyScheme_SchemeType" json:"type,omitempty"`
|
||||
// these values apply to RS encoding
|
||||
MinReq int64 `protobuf:"varint,2,opt,name=min_req,json=minReq" json:"min_req,omitempty"`
|
||||
Total int64 `protobuf:"varint,3,opt,name=total" json:"total,omitempty"`
|
||||
RepairThreshold int64 `protobuf:"varint,4,opt,name=repair_threshold,json=repairThreshold" json:"repair_threshold,omitempty"`
|
||||
SuccessThreshold int64 `protobuf:"varint,5,opt,name=success_threshold,json=successThreshold" json:"success_threshold,omitempty"`
|
||||
}
|
||||
|
||||
func (m *RedundancyScheme) Reset() { *m = RedundancyScheme{} }
|
||||
func (m *RedundancyScheme) String() string { return proto.CompactTextString(m) }
|
||||
func (*RedundancyScheme) ProtoMessage() {}
|
||||
func (*RedundancyScheme) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
|
||||
|
||||
func (m *RedundancyScheme) GetType() RedundancyScheme_SchemeType {
|
||||
if m != nil {
|
||||
return m.Type
|
||||
}
|
||||
return RedundancyScheme_RS
|
||||
}
|
||||
|
||||
func (m *RedundancyScheme) GetMinReq() int64 {
|
||||
if m != nil {
|
||||
return m.MinReq
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (m *RedundancyScheme) GetTotal() int64 {
|
||||
if m != nil {
|
||||
return m.Total
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (m *RedundancyScheme) GetRepairThreshold() int64 {
|
||||
if m != nil {
|
||||
return m.RepairThreshold
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (m *RedundancyScheme) GetSuccessThreshold() int64 {
|
||||
if m != nil {
|
||||
return m.SuccessThreshold
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type EncryptionScheme struct {
|
||||
Type EncryptionScheme_EncryptionType `protobuf:"varint,1,opt,name=type,enum=netstate.EncryptionScheme_EncryptionType" json:"type,omitempty"`
|
||||
EncryptedEncryptionKey []byte `protobuf:"bytes,2,opt,name=encrypted_encryption_key,json=encryptedEncryptionKey,proto3" json:"encrypted_encryption_key,omitempty"`
|
||||
EncryptedStartingNonce []byte `protobuf:"bytes,3,opt,name=encrypted_starting_nonce,json=encryptedStartingNonce,proto3" json:"encrypted_starting_nonce,omitempty"`
|
||||
}
|
||||
|
||||
func (m *EncryptionScheme) Reset() { *m = EncryptionScheme{} }
|
||||
func (m *EncryptionScheme) String() string { return proto.CompactTextString(m) }
|
||||
func (*EncryptionScheme) ProtoMessage() {}
|
||||
func (*EncryptionScheme) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
|
||||
|
||||
func (m *EncryptionScheme) GetType() EncryptionScheme_EncryptionType {
|
||||
if m != nil {
|
||||
return m.Type
|
||||
}
|
||||
return EncryptionScheme_AESGCM
|
||||
}
|
||||
|
||||
func (m *EncryptionScheme) GetEncryptedEncryptionKey() []byte {
|
||||
if m != nil {
|
||||
return m.EncryptedEncryptionKey
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *EncryptionScheme) GetEncryptedStartingNonce() []byte {
|
||||
if m != nil {
|
||||
return m.EncryptedStartingNonce
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type RemotePiece struct {
|
||||
PieceNum int64 `protobuf:"varint,1,opt,name=piece_num,json=pieceNum" json:"piece_num,omitempty"`
|
||||
NodeId string `protobuf:"bytes,2,opt,name=node_id,json=nodeId" json:"node_id,omitempty"`
|
||||
Size int64 `protobuf:"varint,3,opt,name=size" json:"size,omitempty"`
|
||||
}
|
||||
|
||||
func (m *RemotePiece) Reset() { *m = RemotePiece{} }
|
||||
func (m *RemotePiece) String() string { return proto.CompactTextString(m) }
|
||||
func (*RemotePiece) ProtoMessage() {}
|
||||
func (*RemotePiece) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
|
||||
|
||||
func (m *RemotePiece) GetPieceNum() int64 {
|
||||
if m != nil {
|
||||
return m.PieceNum
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (m *RemotePiece) GetNodeId() string {
|
||||
if m != nil {
|
||||
return m.NodeId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (m *RemotePiece) GetSize() int64 {
|
||||
if m != nil {
|
||||
return m.Size
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type RemoteSegment struct {
|
||||
Redundancy *RedundancyScheme `protobuf:"bytes,1,opt,name=redundancy" json:"redundancy,omitempty"`
|
||||
PieceName string `protobuf:"bytes,2,opt,name=piece_name,json=pieceName" json:"piece_name,omitempty"`
|
||||
RemotePieces []*RemotePiece `protobuf:"bytes,3,rep,name=remote_pieces,json=remotePieces" json:"remote_pieces,omitempty"`
|
||||
MerkleRoot []byte `protobuf:"bytes,4,opt,name=merkle_root,json=merkleRoot,proto3" json:"merkle_root,omitempty"`
|
||||
MerkleSize int64 `protobuf:"varint,5,opt,name=merkle_size,json=merkleSize" json:"merkle_size,omitempty"`
|
||||
}
|
||||
|
||||
func (m *RemoteSegment) Reset() { *m = RemoteSegment{} }
|
||||
func (m *RemoteSegment) String() string { return proto.CompactTextString(m) }
|
||||
func (*RemoteSegment) ProtoMessage() {}
|
||||
func (*RemoteSegment) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
|
||||
|
||||
func (m *RemoteSegment) GetRedundancy() *RedundancyScheme {
|
||||
if m != nil {
|
||||
return m.Redundancy
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *RemoteSegment) GetPieceName() string {
|
||||
if m != nil {
|
||||
return m.PieceName
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (m *RemoteSegment) GetRemotePieces() []*RemotePiece {
|
||||
if m != nil {
|
||||
return m.RemotePieces
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *RemoteSegment) GetMerkleRoot() []byte {
|
||||
if m != nil {
|
||||
return m.MerkleRoot
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *RemoteSegment) GetMerkleSize() int64 {
|
||||
if m != nil {
|
||||
return m.MerkleSize
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type Pointer struct {
|
||||
Type Pointer_DataType `protobuf:"varint,1,opt,name=type,enum=netstate.Pointer_DataType" json:"type,omitempty"`
|
||||
Encryption *EncryptionScheme `protobuf:"bytes,2,opt,name=encryption" json:"encryption,omitempty"`
|
||||
InlineSegment []byte `protobuf:"bytes,3,opt,name=inline_segment,json=inlineSegment,proto3" json:"inline_segment,omitempty"`
|
||||
Remote *RemoteSegment `protobuf:"bytes,4,opt,name=remote" json:"remote,omitempty"`
|
||||
EncryptedUnencryptedSize []byte `protobuf:"bytes,5,opt,name=encrypted_unencrypted_size,json=encryptedUnencryptedSize,proto3" json:"encrypted_unencrypted_size,omitempty"`
|
||||
CreationDate *google_protobuf.Timestamp `protobuf:"bytes,6,opt,name=creation_date,json=creationDate" json:"creation_date,omitempty"`
|
||||
ExpirationDate *google_protobuf.Timestamp `protobuf:"bytes,7,opt,name=expiration_date,json=expirationDate" json:"expiration_date,omitempty"`
|
||||
}
|
||||
|
||||
func (m *Pointer) Reset() { *m = Pointer{} }
|
||||
func (m *Pointer) String() string { return proto.CompactTextString(m) }
|
||||
func (*Pointer) ProtoMessage() {}
|
||||
func (*Pointer) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
|
||||
|
||||
func (m *Pointer) GetType() Pointer_DataType {
|
||||
if m != nil {
|
||||
return m.Type
|
||||
}
|
||||
return Pointer_INLINE
|
||||
}
|
||||
|
||||
func (m *Pointer) GetEncryption() *EncryptionScheme {
|
||||
if m != nil {
|
||||
return m.Encryption
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Pointer) GetInlineSegment() []byte {
|
||||
if m != nil {
|
||||
return m.InlineSegment
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Pointer) GetRemote() *RemoteSegment {
|
||||
if m != nil {
|
||||
return m.Remote
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Pointer) GetEncryptedUnencryptedSize() []byte {
|
||||
if m != nil {
|
||||
return m.EncryptedUnencryptedSize
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Pointer) GetCreationDate() *google_protobuf.Timestamp {
|
||||
if m != nil {
|
||||
return m.CreationDate
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Pointer) GetExpirationDate() *google_protobuf.Timestamp {
|
||||
if m != nil {
|
||||
return m.ExpirationDate
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PutRequest is a request message for the Put rpc call
|
||||
type PutRequest struct {
|
||||
Path []byte `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
|
||||
Pointer *Pointer `protobuf:"bytes,2,opt,name=pointer" json:"pointer,omitempty"`
|
||||
}
|
||||
|
||||
func (m *PutRequest) Reset() { *m = PutRequest{} }
|
||||
func (m *PutRequest) String() string { return proto.CompactTextString(m) }
|
||||
func (*PutRequest) ProtoMessage() {}
|
||||
func (*PutRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
|
||||
|
||||
func (m *PutRequest) GetPath() []byte {
|
||||
if m != nil {
|
||||
return m.Path
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *FilePath) GetSmallValue() []byte {
|
||||
func (m *PutRequest) GetPointer() *Pointer {
|
||||
if m != nil {
|
||||
return m.SmallValue
|
||||
return m.Pointer
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -74,7 +367,7 @@ type GetRequest struct {
|
||||
func (m *GetRequest) Reset() { *m = GetRequest{} }
|
||||
func (m *GetRequest) String() string { return proto.CompactTextString(m) }
|
||||
func (*GetRequest) ProtoMessage() {}
|
||||
func (*GetRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
|
||||
func (*GetRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
|
||||
|
||||
func (m *GetRequest) GetPath() []byte {
|
||||
if m != nil {
|
||||
@ -85,68 +378,68 @@ func (m *GetRequest) GetPath() []byte {
|
||||
|
||||
// ListRequest is a request message for the List rpc call
|
||||
type ListRequest struct {
|
||||
Bucket []byte `protobuf:"bytes,1,opt,name=bucket,proto3" json:"bucket,omitempty"`
|
||||
StartingPathKey []byte `protobuf:"bytes,1,opt,name=starting_path_key,json=startingPathKey,proto3" json:"starting_path_key,omitempty"`
|
||||
Limit int64 `protobuf:"varint,2,opt,name=limit" json:"limit,omitempty"`
|
||||
}
|
||||
|
||||
func (m *ListRequest) Reset() { *m = ListRequest{} }
|
||||
func (m *ListRequest) String() string { return proto.CompactTextString(m) }
|
||||
func (*ListRequest) ProtoMessage() {}
|
||||
func (*ListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
|
||||
func (*ListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
|
||||
|
||||
func (m *ListRequest) GetBucket() []byte {
|
||||
func (m *ListRequest) GetStartingPathKey() []byte {
|
||||
if m != nil {
|
||||
return m.Bucket
|
||||
return m.StartingPathKey
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *ListRequest) GetLimit() int64 {
|
||||
if m != nil {
|
||||
return m.Limit
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// PutResponse is a response message for the Put rpc call
|
||||
type PutResponse struct {
|
||||
Confirmation string `protobuf:"bytes,1,opt,name=confirmation" json:"confirmation,omitempty"`
|
||||
}
|
||||
|
||||
func (m *PutResponse) Reset() { *m = PutResponse{} }
|
||||
func (m *PutResponse) String() string { return proto.CompactTextString(m) }
|
||||
func (*PutResponse) ProtoMessage() {}
|
||||
func (*PutResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
|
||||
|
||||
func (m *PutResponse) GetConfirmation() string {
|
||||
if m != nil {
|
||||
return m.Confirmation
|
||||
}
|
||||
return ""
|
||||
}
|
||||
func (*PutResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
|
||||
|
||||
// GetResponse is a response message for the Get rpc call
|
||||
type GetResponse struct {
|
||||
SmallValue []byte `protobuf:"bytes,1,opt,name=smallValue,proto3" json:"smallValue,omitempty"`
|
||||
Pointer []byte `protobuf:"bytes,1,opt,name=pointer,proto3" json:"pointer,omitempty"`
|
||||
}
|
||||
|
||||
func (m *GetResponse) Reset() { *m = GetResponse{} }
|
||||
func (m *GetResponse) String() string { return proto.CompactTextString(m) }
|
||||
func (*GetResponse) ProtoMessage() {}
|
||||
func (*GetResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
|
||||
func (*GetResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
|
||||
|
||||
func (m *GetResponse) GetSmallValue() []byte {
|
||||
func (m *GetResponse) GetPointer() []byte {
|
||||
if m != nil {
|
||||
return m.SmallValue
|
||||
return m.Pointer
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListResponse is a response message for the List rpc call
|
||||
type ListResponse struct {
|
||||
Filepaths [][]byte `protobuf:"bytes,1,rep,name=filepaths,proto3" json:"filepaths,omitempty"`
|
||||
Paths [][]byte `protobuf:"bytes,1,rep,name=paths,proto3" json:"paths,omitempty"`
|
||||
}
|
||||
|
||||
func (m *ListResponse) Reset() { *m = ListResponse{} }
|
||||
func (m *ListResponse) String() string { return proto.CompactTextString(m) }
|
||||
func (*ListResponse) ProtoMessage() {}
|
||||
func (*ListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
|
||||
func (*ListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
|
||||
|
||||
func (m *ListResponse) GetFilepaths() [][]byte {
|
||||
func (m *ListResponse) GetPaths() [][]byte {
|
||||
if m != nil {
|
||||
return m.Filepaths
|
||||
return m.Paths
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -158,7 +451,7 @@ type DeleteRequest struct {
|
||||
func (m *DeleteRequest) Reset() { *m = DeleteRequest{} }
|
||||
func (m *DeleteRequest) String() string { return proto.CompactTextString(m) }
|
||||
func (*DeleteRequest) ProtoMessage() {}
|
||||
func (*DeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
|
||||
func (*DeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
|
||||
|
||||
func (m *DeleteRequest) GetPath() []byte {
|
||||
if m != nil {
|
||||
@ -169,23 +462,20 @@ func (m *DeleteRequest) GetPath() []byte {
|
||||
|
||||
// DeleteResponse is a response message for the Delete rpc call
|
||||
type DeleteResponse struct {
|
||||
Confirmation string `protobuf:"bytes,1,opt,name=confirmation" json:"confirmation,omitempty"`
|
||||
}
|
||||
|
||||
func (m *DeleteResponse) Reset() { *m = DeleteResponse{} }
|
||||
func (m *DeleteResponse) String() string { return proto.CompactTextString(m) }
|
||||
func (*DeleteResponse) ProtoMessage() {}
|
||||
func (*DeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
|
||||
|
||||
func (m *DeleteResponse) GetConfirmation() string {
|
||||
if m != nil {
|
||||
return m.Confirmation
|
||||
}
|
||||
return ""
|
||||
}
|
||||
func (*DeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
|
||||
|
||||
func init() {
|
||||
proto.RegisterType((*FilePath)(nil), "netstate.FilePath")
|
||||
proto.RegisterType((*RedundancyScheme)(nil), "netstate.RedundancyScheme")
|
||||
proto.RegisterType((*EncryptionScheme)(nil), "netstate.EncryptionScheme")
|
||||
proto.RegisterType((*RemotePiece)(nil), "netstate.RemotePiece")
|
||||
proto.RegisterType((*RemoteSegment)(nil), "netstate.RemoteSegment")
|
||||
proto.RegisterType((*Pointer)(nil), "netstate.Pointer")
|
||||
proto.RegisterType((*PutRequest)(nil), "netstate.PutRequest")
|
||||
proto.RegisterType((*GetRequest)(nil), "netstate.GetRequest")
|
||||
proto.RegisterType((*ListRequest)(nil), "netstate.ListRequest")
|
||||
proto.RegisterType((*PutResponse)(nil), "netstate.PutResponse")
|
||||
@ -193,6 +483,9 @@ func init() {
|
||||
proto.RegisterType((*ListResponse)(nil), "netstate.ListResponse")
|
||||
proto.RegisterType((*DeleteRequest)(nil), "netstate.DeleteRequest")
|
||||
proto.RegisterType((*DeleteResponse)(nil), "netstate.DeleteResponse")
|
||||
proto.RegisterEnum("netstate.RedundancyScheme_SchemeType", RedundancyScheme_SchemeType_name, RedundancyScheme_SchemeType_value)
|
||||
proto.RegisterEnum("netstate.EncryptionScheme_EncryptionType", EncryptionScheme_EncryptionType_name, EncryptionScheme_EncryptionType_value)
|
||||
proto.RegisterEnum("netstate.Pointer_DataType", Pointer_DataType_name, Pointer_DataType_value)
|
||||
}
|
||||
|
||||
// Reference imports to suppress errors if they are not otherwise used.
|
||||
@ -207,7 +500,7 @@ const _ = grpc.SupportPackageIsVersion4
|
||||
|
||||
type NetStateClient interface {
|
||||
// Put formats and hands off a file path to be saved to boltdb
|
||||
Put(ctx context.Context, in *FilePath, opts ...grpc.CallOption) (*PutResponse, error)
|
||||
Put(ctx context.Context, in *PutRequest, opts ...grpc.CallOption) (*PutResponse, error)
|
||||
// Get formats and hands off a file path to get a small value from boltdb
|
||||
Get(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetResponse, error)
|
||||
// List calls the bolt client's List function and returns all file paths
|
||||
@ -224,7 +517,7 @@ func NewNetStateClient(cc *grpc.ClientConn) NetStateClient {
|
||||
return &netStateClient{cc}
|
||||
}
|
||||
|
||||
func (c *netStateClient) Put(ctx context.Context, in *FilePath, opts ...grpc.CallOption) (*PutResponse, error) {
|
||||
func (c *netStateClient) Put(ctx context.Context, in *PutRequest, opts ...grpc.CallOption) (*PutResponse, error) {
|
||||
out := new(PutResponse)
|
||||
err := grpc.Invoke(ctx, "/netstate.NetState/Put", in, out, c.cc, opts...)
|
||||
if err != nil {
|
||||
@ -264,7 +557,7 @@ func (c *netStateClient) Delete(ctx context.Context, in *DeleteRequest, opts ...
|
||||
|
||||
type NetStateServer interface {
|
||||
// Put formats and hands off a file path to be saved to boltdb
|
||||
Put(context.Context, *FilePath) (*PutResponse, error)
|
||||
Put(context.Context, *PutRequest) (*PutResponse, error)
|
||||
// Get formats and hands off a file path to get a small value from boltdb
|
||||
Get(context.Context, *GetRequest) (*GetResponse, error)
|
||||
// List calls the bolt client's List function and returns all file paths
|
||||
@ -278,7 +571,7 @@ func RegisterNetStateServer(s *grpc.Server, srv NetStateServer) {
|
||||
}
|
||||
|
||||
func _NetState_Put_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(FilePath)
|
||||
in := new(PutRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -290,7 +583,7 @@ func _NetState_Put_Handler(srv interface{}, ctx context.Context, dec func(interf
|
||||
FullMethod: "/netstate.NetState/Put",
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(NetStateServer).Put(ctx, req.(*FilePath))
|
||||
return srv.(NetStateServer).Put(ctx, req.(*PutRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
@ -377,24 +670,58 @@ var _NetState_serviceDesc = grpc.ServiceDesc{
|
||||
func init() { proto.RegisterFile("netstate.proto", fileDescriptor0) }
|
||||
|
||||
var fileDescriptor0 = []byte{
|
||||
// 302 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x52, 0x4d, 0x4b, 0xc3, 0x40,
|
||||
0x10, 0x25, 0xb6, 0x94, 0x74, 0x12, 0x7b, 0x18, 0xb4, 0x86, 0x20, 0x12, 0x56, 0x84, 0x1e, 0xb4,
|
||||
0x68, 0xd5, 0x93, 0xe0, 0x49, 0xec, 0x45, 0x24, 0x44, 0xf0, 0xbe, 0x2d, 0x53, 0x1a, 0xdc, 0x26,
|
||||
0xb1, 0x3b, 0xf9, 0xc7, 0xfe, 0x10, 0xc9, 0x57, 0xb3, 0xb1, 0x22, 0xde, 0x32, 0xef, 0x23, 0x79,
|
||||
0x33, 0x2f, 0x30, 0x4a, 0x88, 0x35, 0x4b, 0xa6, 0x69, 0xb6, 0x4d, 0x39, 0x45, 0xbb, 0x99, 0xc5,
|
||||
0x23, 0xd8, 0xcf, 0xb1, 0xa2, 0x50, 0xf2, 0x1a, 0x11, 0xfa, 0x99, 0xe4, 0xb5, 0x67, 0x05, 0xd6,
|
||||
0xc4, 0x8d, 0xca, 0x67, 0x3c, 0x03, 0xd0, 0x1b, 0xa9, 0xd4, 0xbb, 0x54, 0x39, 0x79, 0x07, 0x25,
|
||||
0x63, 0x20, 0x22, 0x00, 0x98, 0x13, 0x47, 0xf4, 0x99, 0x93, 0xe6, 0xdf, 0xde, 0x20, 0x2e, 0xc0,
|
||||
0x79, 0x89, 0xf5, 0x4e, 0x32, 0x86, 0xc1, 0x22, 0x5f, 0x7e, 0x10, 0xd7, 0xa2, 0x7a, 0x12, 0x37,
|
||||
0xe0, 0x84, 0x39, 0x47, 0xa4, 0xb3, 0x34, 0xd1, 0x84, 0x02, 0xdc, 0x65, 0x9a, 0xac, 0xe2, 0xed,
|
||||
0x46, 0x72, 0x9c, 0x26, 0xa5, 0x78, 0x18, 0x75, 0x30, 0x71, 0x05, 0x4e, 0xf9, 0xed, 0xda, 0xd2,
|
||||
0x8d, 0x6a, 0xed, 0x45, 0xbd, 0x04, 0xb7, 0x0a, 0x52, 0xeb, 0x4f, 0x61, 0xb8, 0x8a, 0x15, 0x15,
|
||||
0x21, 0xb5, 0x67, 0x05, 0xbd, 0x89, 0x1b, 0xb5, 0x80, 0x38, 0x87, 0xc3, 0x27, 0x52, 0xc4, 0xf4,
|
||||
0xd7, 0x6e, 0x77, 0x30, 0x6a, 0x44, 0xff, 0xcf, 0x3d, 0xfb, 0xb2, 0xc0, 0x7e, 0x25, 0x7e, 0x2b,
|
||||
0x0a, 0xc0, 0x6b, 0xe8, 0x85, 0x39, 0x23, 0x4e, 0x77, 0x15, 0x35, 0x7d, 0xf8, 0xc7, 0x2d, 0x66,
|
||||
0x9e, 0x66, 0x06, 0xbd, 0x39, 0x31, 0x1e, 0xb5, 0x6c, 0xdb, 0x80, 0xe9, 0x31, 0x6f, 0x73, 0x0f,
|
||||
0xfd, 0x62, 0x77, 0x34, 0x68, 0xa3, 0x14, 0x7f, 0xfc, 0x13, 0xae, 0x6d, 0x0f, 0x30, 0xa8, 0xf6,
|
||||
0xc3, 0x93, 0x56, 0xd1, 0x39, 0x8b, 0xef, 0xed, 0x13, 0x95, 0x79, 0x31, 0x28, 0xff, 0xb5, 0xdb,
|
||||
0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x0c, 0x07, 0x2f, 0xb8, 0x7d, 0x02, 0x00, 0x00,
|
||||
// 847 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x54, 0x7f, 0x6f, 0xe3, 0x44,
|
||||
0x10, 0xad, 0xcf, 0x6d, 0xda, 0x4e, 0x7e, 0xd4, 0x5d, 0xe5, 0xee, 0xac, 0x20, 0x44, 0x64, 0x38,
|
||||
0xd1, 0xa3, 0x52, 0x90, 0x82, 0x90, 0xa0, 0x80, 0x10, 0xb4, 0x51, 0x55, 0x71, 0x4d, 0xa3, 0x4d,
|
||||
0x10, 0xfc, 0x67, 0xf9, 0xe2, 0x21, 0xb1, 0x2e, 0xde, 0x75, 0xbd, 0x6b, 0x89, 0xf0, 0xbd, 0xf8,
|
||||
0x4a, 0x08, 0xf1, 0x27, 0x9f, 0x00, 0x79, 0x77, 0x6d, 0x6f, 0x72, 0xe2, 0xee, 0xaf, 0xec, 0xcc,
|
||||
0xbc, 0x99, 0xcc, 0x7b, 0xfb, 0xbc, 0xd0, 0x63, 0x28, 0x85, 0x8c, 0x24, 0x8e, 0xb2, 0x9c, 0x4b,
|
||||
0x4e, 0x4e, 0xaa, 0x78, 0x70, 0x26, 0x93, 0x14, 0x85, 0x8c, 0xd2, 0x4c, 0x97, 0x82, 0x7f, 0x1c,
|
||||
0xf0, 0x28, 0xc6, 0x05, 0x8b, 0x23, 0xb6, 0xdc, 0xce, 0x97, 0x6b, 0x4c, 0x91, 0x7c, 0x0d, 0x87,
|
||||
0x72, 0x9b, 0xa1, 0xef, 0x0c, 0x9d, 0x8b, 0xde, 0xf8, 0xc5, 0xa8, 0x1e, 0xb7, 0x8f, 0x1c, 0xe9,
|
||||
0x9f, 0xc5, 0x36, 0x43, 0xaa, 0x5a, 0xc8, 0x73, 0x38, 0x4e, 0x13, 0x16, 0xe6, 0xf8, 0xe8, 0x3f,
|
||||
0x19, 0x3a, 0x17, 0x2e, 0x6d, 0xa5, 0x09, 0xa3, 0xf8, 0x48, 0xfa, 0x70, 0x24, 0xb9, 0x8c, 0x36,
|
||||
0xbe, 0xab, 0xd2, 0x3a, 0x20, 0x2f, 0xc1, 0xcb, 0x31, 0x8b, 0x92, 0x3c, 0x94, 0xeb, 0x1c, 0xc5,
|
||||
0x9a, 0x6f, 0x62, 0xff, 0x50, 0x01, 0xce, 0x74, 0x7e, 0x51, 0xa5, 0xc9, 0x25, 0x9c, 0x8b, 0x62,
|
||||
0xb9, 0x44, 0x21, 0x2c, 0xec, 0x91, 0xc2, 0x7a, 0xa6, 0x50, 0x83, 0x83, 0x3e, 0x40, 0xb3, 0x1a,
|
||||
0x69, 0xc1, 0x13, 0x3a, 0xf7, 0x0e, 0x82, 0x7f, 0x1d, 0xf0, 0x26, 0x6c, 0x99, 0x6f, 0x33, 0x99,
|
||||
0x70, 0x66, 0xc8, 0x7e, 0xb7, 0x43, 0xf6, 0x65, 0x43, 0x76, 0x1f, 0x69, 0x25, 0x2c, 0xc2, 0x5f,
|
||||
0x81, 0x8f, 0x3a, 0x8f, 0x71, 0x88, 0x35, 0x22, 0x7c, 0x83, 0x5b, 0xa5, 0x40, 0x87, 0x3e, 0xab,
|
||||
0xeb, 0xcd, 0x80, 0x9f, 0x70, 0xbb, 0xdb, 0x29, 0x64, 0x94, 0xcb, 0x84, 0xad, 0x42, 0xc6, 0xd9,
|
||||
0x12, 0x95, 0x48, 0x76, 0xe7, 0xdc, 0x94, 0xa7, 0x65, 0x35, 0xb8, 0x84, 0xde, 0xee, 0x2e, 0x04,
|
||||
0xa0, 0xf5, 0xc3, 0x64, 0x7e, 0x7b, 0x7d, 0xef, 0x1d, 0x90, 0x2e, 0x9c, 0xce, 0x27, 0xd7, 0x74,
|
||||
0xb2, 0xf8, 0xf1, 0xe1, 0x57, 0xcf, 0x09, 0x7e, 0x81, 0x36, 0xc5, 0x94, 0x4b, 0x9c, 0x25, 0xb8,
|
||||
0x44, 0xf2, 0x01, 0x9c, 0x66, 0xe5, 0x21, 0x64, 0x45, 0xaa, 0x38, 0xbb, 0xf4, 0x44, 0x25, 0xa6,
|
||||
0x45, 0x5a, 0xde, 0x1e, 0xe3, 0x31, 0x86, 0x49, 0xac, 0x76, 0x3f, 0xa5, 0xad, 0x32, 0xbc, 0x8b,
|
||||
0x09, 0x81, 0x43, 0x91, 0xfc, 0x81, 0xe6, 0xf2, 0xd4, 0x39, 0xf8, 0xdb, 0x81, 0xae, 0x9e, 0x3c,
|
||||
0xc7, 0x55, 0x8a, 0x4c, 0x92, 0x2b, 0x80, 0xbc, 0x76, 0x88, 0x1a, 0xde, 0x1e, 0x0f, 0xfe, 0xdf,
|
||||
0x3d, 0xd4, 0x42, 0x93, 0x0f, 0x01, 0xcc, 0x5e, 0x51, 0x8a, 0xe6, 0xdf, 0xf5, 0xa6, 0xd3, 0x28,
|
||||
0x45, 0x72, 0x05, 0xdd, 0x5c, 0xfd, 0x57, 0xa8, 0x72, 0xc2, 0x77, 0x87, 0xee, 0x45, 0x7b, 0xfc,
|
||||
0xd4, 0x9e, 0x5e, 0x93, 0xa4, 0x9d, 0xbc, 0x09, 0x04, 0xf9, 0x08, 0xda, 0x29, 0xe6, 0x6f, 0x36,
|
||||
0x18, 0xe6, 0x9c, 0x4b, 0xe5, 0xaf, 0x0e, 0x05, 0x9d, 0xa2, 0x9c, 0x4b, 0x0b, 0xa0, 0x48, 0x6a,
|
||||
0x53, 0x19, 0xc0, 0xbc, 0xa4, 0xfa, 0xa7, 0x0b, 0xc7, 0x33, 0x9e, 0x30, 0x89, 0x39, 0x19, 0xed,
|
||||
0xf8, 0xc5, 0xa2, 0x67, 0x00, 0xa3, 0x9b, 0x48, 0x46, 0x96, 0x41, 0xae, 0x00, 0x1a, 0x5b, 0x28,
|
||||
0x62, 0x3b, 0xa2, 0xec, 0xbb, 0x8c, 0x5a, 0x68, 0xf2, 0x02, 0x7a, 0x09, 0xdb, 0x24, 0x0c, 0x43,
|
||||
0xa1, 0x25, 0x36, 0xc6, 0xe8, 0xea, 0x6c, 0xa5, 0xfb, 0xe7, 0xd0, 0xd2, 0x84, 0x15, 0xb7, 0xf6,
|
||||
0xf8, 0xf9, 0xbe, 0x2a, 0x06, 0x48, 0x0d, 0x8c, 0x7c, 0x0b, 0x83, 0xc6, 0x7a, 0x05, 0xb3, 0x6c,
|
||||
0x58, 0xf1, 0xef, 0xd0, 0xc6, 0x9c, 0x3f, 0x37, 0x80, 0x52, 0x0d, 0xf2, 0x3d, 0x74, 0x97, 0x39,
|
||||
0x46, 0xca, 0xe6, 0x71, 0x24, 0xd1, 0x6f, 0x19, 0x52, 0x2b, 0xce, 0x57, 0x1b, 0xf3, 0xe8, 0xbc,
|
||||
0x2e, 0x7e, 0x1b, 0x2d, 0xaa, 0xc7, 0x86, 0x76, 0xaa, 0x86, 0x9b, 0x48, 0x22, 0xb9, 0x86, 0x33,
|
||||
0xfc, 0x3d, 0x4b, 0x72, 0x6b, 0xc4, 0xf1, 0x7b, 0x47, 0xf4, 0x9a, 0x96, 0x72, 0x48, 0x10, 0xc0,
|
||||
0x49, 0xa5, 0x74, 0x69, 0xff, 0xbb, 0xe9, 0xab, 0xbb, 0xe9, 0xc4, 0x3b, 0x28, 0xcf, 0x74, 0x72,
|
||||
0xff, 0xb0, 0x98, 0x78, 0x4e, 0x70, 0x0f, 0x30, 0x2b, 0x24, 0xc5, 0xc7, 0x02, 0x85, 0x2c, 0x4d,
|
||||
0x9c, 0x45, 0x72, 0xad, 0x6e, 0xae, 0x43, 0xd5, 0x99, 0x5c, 0xc2, 0x71, 0xa6, 0xef, 0xcd, 0x5c,
|
||||
0xcd, 0xf9, 0x5b, 0x17, 0x4a, 0x2b, 0x44, 0x30, 0x04, 0xb8, 0xc5, 0x77, 0x8d, 0x0b, 0x1e, 0xa0,
|
||||
0xfd, 0x2a, 0x11, 0x35, 0xe4, 0x33, 0x38, 0xaf, 0x3f, 0xec, 0xb2, 0xae, 0x5e, 0x05, 0x8d, 0x3f,
|
||||
0xab, 0x0a, 0xb3, 0x48, 0xae, 0xcb, 0xe7, 0xa0, 0x0f, 0x47, 0x9b, 0x24, 0x4d, 0xa4, 0x79, 0x37,
|
||||
0x75, 0x10, 0x74, 0xa1, 0xad, 0x18, 0x88, 0x8c, 0x33, 0x81, 0xc1, 0xa7, 0xd0, 0x56, 0x1b, 0xe8,
|
||||
0x90, 0xf8, 0xcd, 0xf6, 0x7a, 0x6a, 0xbd, 0xea, 0x27, 0xd0, 0xd1, 0x8b, 0x18, 0x64, 0x1f, 0x8e,
|
||||
0xca, 0x05, 0x84, 0xef, 0x0c, 0xdd, 0x8b, 0x0e, 0xd5, 0x41, 0xf0, 0x31, 0x74, 0x6f, 0x70, 0x83,
|
||||
0x12, 0xdf, 0xc5, 0xc9, 0x83, 0x5e, 0x05, 0xd2, 0xc3, 0xc6, 0x7f, 0x39, 0x70, 0x32, 0x45, 0x39,
|
||||
0x2f, 0x55, 0x22, 0x63, 0x70, 0x67, 0x85, 0x24, 0x7d, 0x4b, 0xb7, 0x5a, 0xf2, 0xc1, 0xd3, 0xbd,
|
||||
0xac, 0xd9, 0x66, 0x0c, 0xee, 0x2d, 0xee, 0xf4, 0x34, 0xba, 0xda, 0x3d, 0x36, 0xd7, 0x2f, 0xe1,
|
||||
0xb0, 0x64, 0x44, 0xac, 0xb2, 0x25, 0xf5, 0xe0, 0xd9, 0x7e, 0xda, 0xb4, 0x7d, 0x03, 0x2d, 0xbd,
|
||||
0x3d, 0xb1, 0xbe, 0x8a, 0x1d, 0xd2, 0x03, 0xff, 0xed, 0x82, 0x6e, 0x7e, 0xdd, 0x52, 0x3e, 0xfc,
|
||||
0xe2, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xab, 0x26, 0xfc, 0xe1, 0x51, 0x07, 0x00, 0x00,
|
||||
}
|
||||
|
||||
@ -4,10 +4,12 @@
|
||||
syntax = "proto3";
|
||||
package netstate;
|
||||
|
||||
import "timestamp.proto";
|
||||
|
||||
// NetState defines the interface for interacting with the network state persistence layer
|
||||
service NetState {
|
||||
// Put formats and hands off a file path to be saved to boltdb
|
||||
rpc Put(FilePath) returns (PutResponse);
|
||||
rpc Put(PutRequest) returns (PutResponse);
|
||||
// Get formats and hands off a file path to get a small value from boltdb
|
||||
rpc Get(GetRequest) returns (GetResponse);
|
||||
// List calls the bolt client's List function and returns all file paths
|
||||
@ -16,12 +18,66 @@ service NetState {
|
||||
rpc Delete(DeleteRequest) returns (DeleteResponse);
|
||||
}
|
||||
|
||||
// FilePath is a request message for the Put rpc call
|
||||
message FilePath {
|
||||
message RedundancyScheme {
|
||||
enum SchemeType {
|
||||
RS = 0;
|
||||
}
|
||||
SchemeType type = 1;
|
||||
|
||||
// these values apply to RS encoding
|
||||
int64 min_req = 2; // minimum required for reconstruction
|
||||
int64 total = 3; // total amount of pieces we generated
|
||||
int64 repair_threshold = 4; // amount of pieces we need to drop to before triggering repair
|
||||
int64 success_threshold = 5; // amount of pieces we need to store to call it a success
|
||||
}
|
||||
|
||||
message EncryptionScheme {
|
||||
enum EncryptionType {
|
||||
AESGCM = 0;
|
||||
SECRETBOX = 1;
|
||||
// only allow authenticated encryption schemes
|
||||
}
|
||||
EncryptionType type = 1;
|
||||
bytes encrypted_encryption_key = 2;
|
||||
bytes encrypted_starting_nonce = 3;
|
||||
}
|
||||
|
||||
message RemotePiece {
|
||||
int64 piece_num = 1;
|
||||
string node_id = 2;
|
||||
int64 size = 3; // full size including merkle tree
|
||||
}
|
||||
|
||||
message RemoteSegment {
|
||||
RedundancyScheme redundancy = 1;
|
||||
string piece_name = 2;
|
||||
repeated RemotePiece remote_pieces = 3;
|
||||
|
||||
bytes merkle_root = 4; // root hash of the hashes of all of these pieces
|
||||
int64 merkle_size = 5; // amount of space the merkle tree takes up at the end of each chunk
|
||||
}
|
||||
|
||||
message Pointer {
|
||||
enum DataType {
|
||||
INLINE = 0;
|
||||
REMOTE = 1;
|
||||
}
|
||||
|
||||
DataType type = 1;
|
||||
EncryptionScheme encryption = 2;
|
||||
|
||||
bytes inline_segment = 3;
|
||||
RemoteSegment remote = 4;
|
||||
bytes encrypted_unencrypted_size = 5; // size of the unencrypted object (!)
|
||||
|
||||
google.protobuf.Timestamp creation_date = 6;
|
||||
google.protobuf.Timestamp expiration_date = 7;
|
||||
}
|
||||
|
||||
// PutRequest is a request message for the Put rpc call
|
||||
message PutRequest {
|
||||
bytes path = 1;
|
||||
// smallValue is a value too small to be broken up and stored
|
||||
// in different places
|
||||
bytes smallValue = 2;
|
||||
Pointer pointer = 2;
|
||||
}
|
||||
|
||||
// GetRequest is a request message for the Get rpc call
|
||||
@ -31,22 +87,22 @@ message GetRequest {
|
||||
|
||||
// ListRequest is a request message for the List rpc call
|
||||
message ListRequest {
|
||||
bytes bucket = 1;
|
||||
bytes starting_path_key = 1; // the Path key in the bucket to start listing
|
||||
int64 limit = 2; // how many keys to list
|
||||
}
|
||||
|
||||
// PutResponse is a response message for the Put rpc call
|
||||
message PutResponse {
|
||||
string confirmation = 1;
|
||||
}
|
||||
|
||||
// GetResponse is a response message for the Get rpc call
|
||||
message GetResponse {
|
||||
bytes smallValue = 1;
|
||||
bytes pointer = 1; // this is a Pointer type marshalled into bytes
|
||||
}
|
||||
|
||||
// ListResponse is a response message for the List rpc call
|
||||
message ListResponse {
|
||||
repeated bytes filepaths = 1;
|
||||
repeated bytes paths = 1;
|
||||
}
|
||||
|
||||
message DeleteRequest {
|
||||
@ -55,5 +111,4 @@ message DeleteRequest {
|
||||
|
||||
// DeleteResponse is a response message for the Delete rpc call
|
||||
message DeleteResponse {
|
||||
string confirmation = 1;
|
||||
}
|
||||
|
||||
@ -7,53 +7,54 @@ import (
|
||||
"github.com/boltdb/bolt"
|
||||
)
|
||||
|
||||
// File Path and Value are saved to boltdb
|
||||
type File struct {
|
||||
Path []byte `json:"path"`
|
||||
Value []byte `json:"value"`
|
||||
// PointerEntry - Path and Pointer are saved as a kv pair to boltdb.
|
||||
// The following boltdb methods handle the pointer type (defined in
|
||||
// the protobuf file) after it has been marshalled into bytes.
|
||||
type PointerEntry struct {
|
||||
Path []byte
|
||||
Pointer []byte
|
||||
}
|
||||
|
||||
const (
|
||||
fileBucketName = "files"
|
||||
pointerBucket = "pointers"
|
||||
)
|
||||
|
||||
// Put saves the file path and value as a kv pair in the "files" bucket
|
||||
func (client *Client) Put(file File) error {
|
||||
// Put saves the Path and Pointer as a kv entry in the "pointers" bucket
|
||||
func (client *Client) Put(pe PointerEntry) error {
|
||||
client.logger.Debug("entering bolt put")
|
||||
return client.db.Update(func(tx *bolt.Tx) error {
|
||||
b, err := tx.CreateBucketIfNotExists([]byte(fileBucketName))
|
||||
b, err := tx.CreateBucketIfNotExists([]byte(pointerBucket))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fileKey := []byte(file.Path)
|
||||
return b.Put(fileKey, file.Value)
|
||||
return b.Put(pe.Path, pe.Pointer)
|
||||
})
|
||||
}
|
||||
|
||||
// Get retrieves the value stored at the file path key
|
||||
func (client *Client) Get(fileKey []byte) ([]byte, error) {
|
||||
client.logger.Debug("entering bolt get: " + string(fileKey))
|
||||
var fileValue []byte
|
||||
// Get retrieves the Pointer value stored at the Path key
|
||||
func (client *Client) Get(pathKey []byte) ([]byte, error) {
|
||||
client.logger.Debug("entering bolt get: " + string(pathKey))
|
||||
var pointerBytes []byte
|
||||
err := client.db.Update(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte(fileBucketName))
|
||||
v := b.Get(fileKey)
|
||||
b := tx.Bucket([]byte(pointerBucket))
|
||||
v := b.Get(pathKey)
|
||||
if v == nil {
|
||||
return Error.New("file %#v not found", string(fileKey))
|
||||
return Error.New("pointer at %#v not found", string(pathKey))
|
||||
}
|
||||
fileValue = v
|
||||
pointerBytes = v
|
||||
return nil
|
||||
})
|
||||
|
||||
return fileValue, err
|
||||
return pointerBytes, err
|
||||
}
|
||||
|
||||
// List creates a string array of all keys in in the "files" bucket
|
||||
// List creates a byte array of all path keys in in the "pointers" bucket
|
||||
func (client *Client) List() ([][]byte, error) {
|
||||
client.logger.Debug("entering bolt list")
|
||||
var paths [][]byte
|
||||
err := client.db.Update(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte(fileBucketName))
|
||||
b := tx.Bucket([]byte(pointerBucket))
|
||||
|
||||
err := b.ForEach(func(key, value []byte) error {
|
||||
paths = append(paths, key)
|
||||
@ -65,10 +66,10 @@ func (client *Client) List() ([][]byte, error) {
|
||||
return paths, err
|
||||
}
|
||||
|
||||
// Delete deletes a kv pair from the "files" bucket, given the key
|
||||
func (client *Client) Delete(fileKey []byte) error {
|
||||
client.logger.Debug("entering bolt delete: " + string(fileKey))
|
||||
// Delete deletes a kv pair from the "pointers" bucket, given the Path key
|
||||
func (client *Client) Delete(pathKey []byte) error {
|
||||
client.logger.Debug("entering bolt delete: " + string(pathKey))
|
||||
return client.db.Update(func(tx *bolt.Tx) error {
|
||||
return tx.Bucket([]byte(fileBucketName)).Delete(fileKey)
|
||||
return tx.Bucket([]byte(pointerBucket)).Delete(pathKey)
|
||||
})
|
||||
}
|
||||
|
||||
@ -36,46 +36,46 @@ func TestNetState(t *testing.T) {
|
||||
os.Remove(c.Path)
|
||||
}()
|
||||
|
||||
testFile := File{
|
||||
Path: []byte(`test/path`),
|
||||
Value: []byte(`test value`),
|
||||
testEntry1 := PointerEntry{
|
||||
Path: []byte(`test/path`),
|
||||
Pointer: []byte(`pointer1`),
|
||||
}
|
||||
|
||||
testFile2 := File{
|
||||
Path: []byte(`test/path2`),
|
||||
Value: []byte(`value2`),
|
||||
testEntry2 := PointerEntry{
|
||||
Path: []byte(`test/path2`),
|
||||
Pointer: []byte(`pointer2`),
|
||||
}
|
||||
|
||||
// tests Put function
|
||||
if err := c.Put(testFile); err != nil {
|
||||
t.Error("Failed to save testFile to files bucket")
|
||||
if err := c.Put(testEntry1); err != nil {
|
||||
t.Error("Failed to save testFile to pointers bucket")
|
||||
}
|
||||
|
||||
// tests Get function
|
||||
retrvValue, err := c.Get([]byte("test/path"))
|
||||
if err != nil {
|
||||
t.Error("Failed to get saved test value")
|
||||
t.Error("Failed to get saved test pointer")
|
||||
}
|
||||
if !bytes.Equal(retrvValue, testFile.Value) {
|
||||
t.Error("Retrieved file was not same as original file")
|
||||
if !bytes.Equal(retrvValue, testEntry1.Pointer) {
|
||||
t.Error("Retrieved pointer was not same as put pointer")
|
||||
}
|
||||
|
||||
// tests Delete function
|
||||
if err := c.Delete([]byte("test/path")); err != nil {
|
||||
t.Error("Failed to delete testfile")
|
||||
t.Error("Failed to delete test entry")
|
||||
}
|
||||
|
||||
// tests List function
|
||||
if err := c.Put(testFile2); err != nil {
|
||||
t.Error("Failed to save testFile2 to files bucket")
|
||||
if err := c.Put(testEntry2); err != nil {
|
||||
t.Error("Failed to put testEntry2 to pointers bucket")
|
||||
}
|
||||
testFiles, err := c.List()
|
||||
testPaths, err := c.List()
|
||||
if err != nil {
|
||||
t.Error("Failed to list file keys")
|
||||
t.Error("Failed to list Path keys in pointers bucket")
|
||||
}
|
||||
|
||||
// tests List + Delete function
|
||||
if !bytes.Equal(testFiles[0], []byte("test/path2")) {
|
||||
t.Error("Expected only testFile2 in list")
|
||||
if !bytes.Equal(testPaths[0], []byte("test/path2")) {
|
||||
t.Error("Expected only testEntry2 path in list")
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user