JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enables netstate service to save pointers (#49)

Browse Source 
* adds pointer to netstate proto file

* generated updated netstate proto

* changes boltdb netstate to save pointers as values

* updates netstate Put to save Pointers, updates client example to put a pointer, adds grpc status errors, updates tests, changes boltdb 'File' struct to 'PointerEntry'

* updates netstate client example and client test to save pointers, updates netstate List and Delete

* begins adding netstate-http tests

* removes netstate http service

* re-adds netstate auth

* updates boltdb netstate test

* changes encrypted_unencrypted_size from int64 to bytes in netstate proto

* updates READMEs
This commit is contained in:
Natalie Villasana 2018-05-29 22:47:40 -04:00 committed by GitHub
parent 9aeff0ab57
commit 6723064bfb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 695 additions and 487 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
cmd/netstate-http/README.md
View File

@ -1,21 +0,0 @@
# HTTP + BoltDB Crud Interface
This is an http server that provides a CRUD (create, read, update, delete) interface for storing file paths and small values with BoltDB.
For example, you can store a value (i.e. "hello world") at `/my/test/file` and interact with `/my/test/file` through an API, backed by BoltDB.
To run:
```
go run cmd/netstate-http/main.go
```
You can also run using these flags: `-port=<port-number> -prod=<bool> -db=<db-name>`
Then you can use http methods (Put, Get, List, and Delete) to interact with small values stored on BoltDB.
To store a value, put it into the PUT request body using this format:
```
{
"value": "here's my value"
}
```
If you're using Postman, select the `raw` request setting to do this.
Afterward, you can also use [Bolter](https://github.com/hasit/bolter) or a similar BoltDB viewer to make sure your files were changed as expected.

75
cmd/netstate-http/main.go
View File

@ -1,75 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package main
import (
"flag"
"fmt"
"log"
"net/http"
"os"
"github.com/julienschmidt/httprouter"
"go.uber.org/zap"
"storj.io/storj/netstate/routes"
"storj.io/storj/storage/boltdb"
)
var (
port int
dbPath string
prod bool
)
func initializeFlags() {
flag.IntVar(&port, "port", 3000, "port")
flag.StringVar(&dbPath, "db", "netstate.db", "db path")
flag.BoolVar(&prod, "prod", false, "The environment this service is running in")
flag.Parse()
}
func main() {
err := Main()
if err != nil {
log.Fatalf("fatal error: %v", err)
os.Exit(1)
}
}
// Main allows simplified error handling
func Main() error {
initializeFlags()
// No err here because no vars passed into NewDevelopment().
// The default won't return an error, but if args are passed in,
// then there will need to be error handling.
logger, _ := zap.NewDevelopment()
if prod {
logger, _ = zap.NewProduction()
}
defer logger.Sync()
logger.Info(fmt.Sprintf("serving on %d", port))
bdb, err := boltdb.New(logger, dbPath)
if err != nil {
return err
}
defer bdb.Close()
routes := routes.NewNetStateRoutes(logger, bdb)
return http.ListenAndServe(fmt.Sprintf(":%d", port), start(routes))
}
func start(f *routes.NetStateRoutes) *httprouter.Router {
router := httprouter.New()
router.PUT("/file/*path", f.Put)
router.GET("/file/*path", f.Get)
router.GET("/file", f.List)
router.DELETE("/file/*path", f.Delete)
return router
}

21
cmd/netstate-rpc/server/README.md → cmd/netstate/README.md
View File

@ -1,15 +1,28 @@
# gRPC Server + BoltDB Crud Interface # gRPC Server + BoltDB Crud Interface
This is a gRPC server which handles CRUD (create, read, update, delete) requests for storing file paths and small values to BoltDB. This is a gRPC server which handles CRUD (create, read, update, delete) requests for storing pointers at given paths in BoltDB.
For example, you can store a value (i.e. "hello world") at `/my/test/file` and interact with `/my/test/file` through an API, backed by BoltDB. An example Put Request to store a path and pointer would look like this:
```
pr := proto.PutRequest{
Path: []byte("here's/a/pointer/path"),
Pointer: &proto.Pointer{
Type: proto.Pointer_INLINE,
Encryption: &proto.EncryptionScheme{
EncryptedEncryptionKey: []byte("key"),
EncryptedStartingNonce: []byte("nonce"),
},
InlineSegment: []byte("littledata"),
},
}
```
To run the server: To run the server:
``` ```
go run cmd/netstate-rpc/server/main.go go run cmd/netstate/main.go
``` ```
You can also run using these flags: `-port=<port-number> -prod=<bool> -db=<db-name>` You can also run using these flags: `-port=<port-number> -prod=<bool> -db=<db-name>`
You can then write a client program using the client library to access the Put, Get, List, and Delete methods to create/interact with small values stored in a BoltDB. You can then write a client program using the client library to access the Put, Get, List, and Delete methods to create and interact with pointer entries stored in BoltDB.
An example program utilizing these functions can be found at `storj.io/storj/examples/netstate-client/main.go`. An example program utilizing these functions can be found at `storj.io/storj/examples/netstate-client/main.go`.
Afterward, you can also use [Bolter](https://github.com/hasit/bolter) or a similar BoltDB viewer to make sure your files were changed as expected. Afterward, you can also use [Bolter](https://github.com/hasit/bolter) or a similar BoltDB viewer to make sure your files were changed as expected.

2
cmd/netstate-rpc/server/main.go → cmd/netstate/main.go
View File

@ -43,6 +43,7 @@ func main() {
bdb, err := boltdb.New(logger, dbPath) bdb, err := boltdb.New(logger, dbPath)
if err != nil { if err != nil {
logger.Fatal("failed to initiate boltdb", zap.Error(err))
return return
} }
defer bdb.Close() defer bdb.Close()
@ -55,6 +56,7 @@ func main() {
grpcServer := grpc.NewServer() grpcServer := grpc.NewServer()
proto.RegisterNetStateServer(grpcServer, netstate.NewServer(bdb, logger)) proto.RegisterNetStateServer(grpcServer, netstate.NewServer(bdb, logger))
logger.Debug(fmt.Sprintf("server listening on port %d", port))
defer grpcServer.GracefulStop() defer grpcServer.GracefulStop()
err = grpcServer.Serve(lis) err = grpcServer.Serve(lis)

6
examples/netstate-client/README.md
View File

@ -1,8 +1,8 @@
# gRPC Client + BoltDB Crud Interface # gRPC Client + BoltDB Crud Interface
This is an example gRPC client which makes CRUD requests (create, read, update, delete) for storing file paths and small values to BoltDB. This is an example gRPC client which makes CRUD requests (create, read, update, delete) for storing pointers at given paths in BoltDB.
The gRPC server at `storj.io/storj/cmd/netstate-rpc/server/main.go` needs to be running for this to work. The gRPC server at `storj.io/storj/cmd/netstate/main.go` needs to be running for this to work.
To run the client: To run the client:
``` ```
@ -10,7 +10,7 @@ go run examples/netstate-client/main.go
``` ```
You can change the port number with a flag if necessary: `-port=<port-number>` You can change the port number with a flag if necessary: `-port=<port-number>`
Afterward, you can use [Bolter](https://github.com/hasit/bolter) or a similar BoltDB viewer to make sure your files were changed as expected. Afterward, you can use [Bolter](https://github.com/hasit/bolter) or a similar BoltDB viewer to make sure your pointer entries were changed as expected.
If changes are made to `storj.io/storj/protos/netstate/netstate.proto`, the protobuf file will need to be regenerated by running `go generate` inside `protos/netstate`. If changes are made to `storj.io/storj/protos/netstate/netstate.proto`, the protobuf file will need to be regenerated by running `go generate` inside `protos/netstate`.

93
examples/netstate-client/main.go
View File

@ -6,10 +6,13 @@ package main
import ( import (
"context" "context"
"flag" "flag"
"fmt"
"strings" "strings"
"go.uber.org/zap" "go.uber.org/zap"
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
proto "storj.io/storj/protos/netstate" proto "storj.io/storj/protos/netstate"
) )
@ -18,10 +21,6 @@ var (
port string port string
) )
const (
success string = "success"
)
func initializeFlags() { func initializeFlags() {
flag.StringVar(&port, "port", ":8080", "port") flag.StringVar(&port, "port", ":8080", "port")
flag.Parse() flag.Parse()
@ -40,73 +39,93 @@ func main() {
client := proto.NewNetStateClient(conn) client := proto.NewNetStateClient(conn)
logger.Debug(fmt.Sprintf("client dialed port %s", port))
ctx := context.Background() ctx := context.Background()
// Examples file paths to be saved // Example pointer paths to put
fp := proto.FilePath{ pr1 := proto.PutRequest{
Path: []byte("welcome/to/my/file/journey"), Path: []byte("welcome/to/my/pointer/journey"),
SmallValue: []byte("granola"), Pointer: &proto.Pointer{
Type: proto.Pointer_INLINE,
Encryption: &proto.EncryptionScheme{
EncryptedEncryptionKey: []byte("key"),
EncryptedStartingNonce: []byte("nonce"),
},
InlineSegment: []byte("granola"),
},
} }
fp2 := proto.FilePath{ pr2 := proto.PutRequest{
Path: []byte("so/many/file/paths"), Path: []byte("so/many/pointers"),
SmallValue: []byte("m&ms"), Pointer: &proto.Pointer{
Type: proto.Pointer_INLINE,
Encryption: &proto.EncryptionScheme{
EncryptedEncryptionKey: []byte("key"),
EncryptedStartingNonce: []byte("nonce"),
},
InlineSegment: []byte("m&ms"),
},
} }
fp3 := proto.FilePath{ pr3 := proto.PutRequest{
Path: []byte("another/file/path/for/the/pile"), Path: []byte("another/pointer/for/the/pile"),
SmallValue: []byte("popcorn"), Pointer: &proto.Pointer{
Type: proto.Pointer_INLINE,
Encryption: &proto.EncryptionScheme{
EncryptedEncryptionKey: []byte("key"),
EncryptedStartingNonce: []byte("nonce"),
},
InlineSegment: []byte("popcorn"),
},
} }
// Example Puts // Example Puts
putRes, err := client.Put(ctx, &fp) _, err = client.Put(ctx, &pr1)
if err != nil || putRes.Confirmation != success { if err != nil || status.Code(err) == codes.Internal {
logger.Error("failed to put", zap.Error(err)) logger.Error("failed to put", zap.Error(err))
} }
putRes2, err := client.Put(ctx, &fp2) _, err = client.Put(ctx, &pr2)
if err != nil || putRes2.Confirmation != success { if err != nil || status.Code(err) == codes.Internal {
logger.Error("failed to put", zap.Error(err)) logger.Error("failed to put", zap.Error(err))
} }
putRes3, err := client.Put(ctx, &fp3) _, err = client.Put(ctx, &pr3)
if err != nil || putRes3.Confirmation != success { if err != nil || status.Code(err) == codes.Internal {
logger.Error("failed to put", zap.Error(err)) logger.Error("failed to put", zap.Error(err))
} }
// Example Get // Example Get
getReq := proto.GetRequest{ getReq := proto.GetRequest{
Path: []byte("so/many/file/paths"), Path: []byte("so/many/pointers"),
} }
getRes, err := client.Get(ctx, &getReq) getRes, err := client.Get(ctx, &getReq)
if err != nil { if err != nil || status.Code(err) == codes.Internal {
logger.Error("failed to get", zap.Error(err)) logger.Error("failed to get", zap.Error(err))
} }
value := string(getRes.SmallValue) pointer := string(getRes.Pointer)
logger.Debug("get response: " + value) logger.Debug("get response: " + pointer)
// Example List // Example List
listReq := proto.ListRequest{ listReq := proto.ListRequest{
// This Bucket value isn't actually used by List() now, // This pagination functionality doesn't work yet.
// but in the future could be used to select specific // The given arguments are placeholders.
// buckets to list from. StartingPathKey: []byte("test/pointer/path"),
Bucket: []byte("files"), Limit: 5,
} }
listRes, err := client.List(ctx, &listReq) listRes, err := client.List(ctx, &listReq)
if err != nil { if err != nil || status.Code(err) == codes.Internal {
logger.Error("failed to list file paths") logger.Error("failed to list file paths")
} }
var stringList []string var stringList []string
for _, pathByte := range listRes.Filepaths { for _, pathByte := range listRes.Paths {
stringList = append(stringList, string(pathByte)) stringList = append(stringList, string(pathByte))
} }
logger.Debug("listed paths: " + strings.Join(stringList, ", ")) logger.Debug("listed paths: " + strings.Join(stringList, ", "))
// Example Delete // Example Delete
delReq := proto.DeleteRequest{ delReq := proto.DeleteRequest{
Path: []byte("welcome/to/my/file/journey"), Path: []byte("welcome/to/my/pointer/journey"),
} }
delRes, err := client.Delete(ctx, &delReq) _, err = client.Delete(ctx, &delReq)
if err != nil { if err != nil || status.Code(err) == codes.Internal {
logger.Error("failed to delete: 'welcome/to/my/file/journey'") logger.Error("failed to delete: " + string(delReq.Path))
}
if delRes.Confirmation == "success" {
logger.Debug("deleted: welcome/to/my/file/journey")
} }
} }

136
netstate/routes/netstate-routes.go
View File

@ -1,136 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package routes
import (
"encoding/json"
"fmt"
"net/http"
"strings"
"github.com/julienschmidt/httprouter"
"go.uber.org/zap"
"storj.io/storj/storage/boltdb"
)
// NetStateRoutes maintains access to a boltdb client and zap logger
type NetStateRoutes struct {
DB *boltdb.Client
logger *zap.Logger
}
// Message contains the small value provided by the user to be stored
type Message struct {
Value string `json:"value"`
}
// NewNetStateRoutes instantiates NetStateRoutes
func NewNetStateRoutes(logger *zap.Logger, db *boltdb.Client) *NetStateRoutes {
return &NetStateRoutes{
DB: db,
logger: logger,
}
}
// Put takes the given path and small value from the user and formats the values
// to be given to boltdb.Put
func (n *NetStateRoutes) Put(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
n.logger.Debug("entering netstate http put")
givenPath := ps.ByName("path")
var msg Message
decoder := json.NewDecoder(r.Body)
err := decoder.Decode(&msg)
if err != nil {
http.Error(w, "bad request: err decoding response", http.StatusBadRequest)
n.logger.Error("err decoding response", zap.Error(err))
return
}
file := boltdb.File{
Path: []byte(givenPath),
Value: []byte(msg.Value),
}
if err := n.DB.Put(file); err != nil {
http.Error(w, "err putting file", http.StatusInternalServerError)
n.logger.Error("err putting file", zap.Error(err))
return
}
n.logger.Debug("put to the db: " + givenPath)
w.WriteHeader(http.StatusCreated)
fmt.Fprintf(w, "PUT to %s\n", givenPath)
}
// Get takes the given file path from the user and calls the bolt client's Get function
func (n *NetStateRoutes) Get(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
n.logger.Debug("entering netstate http get")
fileKey := ps.ByName("path")
fileValue, err := n.DB.Get([]byte(fileKey))
if err != nil {
http.Error(w, "err getting file", http.StatusInternalServerError)
n.logger.Error("err getting file", zap.Error(err))
return
}
w.Header().Set("Content-Type", "application/octet-stream")
_, err = w.Write(fileValue)
if err != nil {
n.logger.Error("err writing response", zap.Error(err))
}
w.WriteHeader(http.StatusOK)
n.logger.Debug("response written: " + string(fileValue))
}
// List calls the bolt client's List function and responds with a list of all saved file paths
// or "filekeys"
func (n *NetStateRoutes) List(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
n.logger.Debug("entering netstate http list")
filePaths, err := n.DB.List()
if err != nil {
http.Error(w, "internal error: unable to list paths", http.StatusInternalServerError)
n.logger.Error("err listing file paths", zap.Error(err))
return
}
var pathList []string
for _, path := range filePaths {
pathList = append(pathList, string(path))
}
bytes, err := json.Marshal(pathList)
if err != nil {
http.Error(w, "internal error: unable to marshal path list", http.StatusInternalServerError)
n.logger.Error("err marshaling path list", zap.Error(err))
return
}
_, err = w.Write(bytes)
if err != nil {
n.logger.Error("err writing response", zap.Error(err))
}
w.WriteHeader(http.StatusOK)
n.logger.Debug("response written: " + strings.Join(pathList, ", "))
}
// Delete takes a given file path and calls the bolt client's Delete function
func (n *NetStateRoutes) Delete(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
n.logger.Debug("entering netstate http delete")
fileKey := ps.ByName("path")
if err := n.DB.Delete([]byte(fileKey)); err != nil {
http.Error(w, "internal error: unable to delete file", http.StatusInternalServerError)
n.logger.Error("err deleting file", zap.Error(err))
return
}
n.logger.Debug("deleted file: " + fileKey)
w.WriteHeader(204)
fmt.Fprintf(w, "Deleted file key: %s", fileKey)
}

89
pkg/netstate/client_test.go
View File

@ -10,10 +10,13 @@ import (
"net" "net"
"testing" "testing"
"github.com/golang/protobuf/proto"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"go.uber.org/zap" "go.uber.org/zap"
"google.golang.org/grpc" "google.golang.org/grpc"
proto "storj.io/storj/protos/netstate" "google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
pb "storj.io/storj/protos/netstate"
) )
func TestNetStateClient(t *testing.T) { func TestNetStateClient(t *testing.T) {
@ -22,12 +25,12 @@ func TestNetStateClient(t *testing.T) {
lis, err := net.Listen("tcp", fmt.Sprintf(":%d", 9000)) lis, err := net.Listen("tcp", fmt.Sprintf(":%d", 9000))
assert.NoError(t, err) assert.NoError(t, err)
mdb := &mockDB{ mdb := &MockDB{
timesCalled: 0, timesCalled: 0,
} }
grpcServer := grpc.NewServer() grpcServer := grpc.NewServer()
proto.RegisterNetStateServer(grpcServer, NewServer(mdb, logger)) pb.RegisterNetStateServer(grpcServer, NewServer(mdb, logger))
defer grpcServer.GracefulStop() defer grpcServer.GracefulStop()
go grpcServer.Serve(lis) go grpcServer.Serve(lis)
@ -36,14 +39,21 @@ func TestNetStateClient(t *testing.T) {
conn, err := grpc.Dial(address, grpc.WithInsecure()) conn, err := grpc.Dial(address, grpc.WithInsecure())
assert.NoError(t, err) assert.NoError(t, err)
c := proto.NewNetStateClient(conn) c := pb.NewNetStateClient(conn)
ctx := context.Background() ctx := context.Background()
// example file path to put/get // example file path to put/get
fp := proto.FilePath{ pr1 := pb.PutRequest{
Path: []byte("here/is/a/path"), Path: []byte("here/is/a/path"),
SmallValue: []byte("oatmeal"), Pointer: &pb.Pointer{
Type: pb.Pointer_INLINE,
Encryption: &pb.EncryptionScheme{
EncryptedEncryptionKey: []byte("key"),
EncryptedStartingNonce: []byte("nonce"),
},
InlineSegment: []byte("oatmeal"),
},
} }
if mdb.timesCalled != 0 { if mdb.timesCalled != 0 {
@ -51,34 +61,37 @@ func TestNetStateClient(t *testing.T) {
} }
// Tests Server.Put // Tests Server.Put
putRes, err := c.Put(ctx, &fp) _, err = c.Put(ctx, &pr1)
assert.NoError(t, err) if err != nil || status.Code(err) == codes.Internal {
t.Error("Failed to Put")
if putRes.Confirmation != "success" {
t.Error("Failed to receive success Put response")
} }
if mdb.timesCalled != 1 { if mdb.timesCalled != 1 {
t.Error("Failed to call mockdb correctly") t.Error("Failed to call mockdb correctly")
} }
if !bytes.Equal(mdb.puts[0].Path, fp.Path) { if !bytes.Equal(mdb.puts[0].Path, pr1.Path) {
t.Error("Expected saved path to equal given path") t.Error("Expected saved path to equal given path")
} }
if !bytes.Equal(mdb.puts[0].Value, fp.SmallValue) { pointerBytes, err := proto.Marshal(pr1.Pointer)
if err != nil {
t.Error("failed to marshal test pointer")
}
if !bytes.Equal(mdb.puts[0].Pointer, pointerBytes) {
t.Error("Expected saved value to equal given value") t.Error("Expected saved value to equal given value")
} }
// Tests Server.Get // Tests Server.Get
getReq := proto.GetRequest{ getReq := pb.GetRequest{
Path: []byte("here/is/a/path"), Path: []byte("here/is/a/path"),
} }
getRes, err := c.Get(ctx, &getReq) getRes, err := c.Get(ctx, &getReq)
assert.NoError(t, err) assert.NoError(t, err)
if !bytes.Equal(getRes.SmallValue, fp.SmallValue) { if !bytes.Equal(getRes.Pointer, pointerBytes) {
t.Error("Expected to get same content that was put") t.Error("Expected to get same content that was put")
} }
@ -86,17 +99,22 @@ func TestNetStateClient(t *testing.T) {
t.Error("Failed to call mockdb correct number of times") t.Error("Failed to call mockdb correct number of times")
} }
// Puts another file path to test delete and list // Puts another pointer entry to test delete and list
fp2 := proto.FilePath{ pr2 := pb.PutRequest{
Path: []byte("here/is/another/path"), Path: []byte("here/is/another/path"),
SmallValue: []byte("raisins"), Pointer: &pb.Pointer{
Type: pb.Pointer_INLINE,
Encryption: &pb.EncryptionScheme{
EncryptedEncryptionKey: []byte("key"),
EncryptedStartingNonce: []byte("nonce"),
},
InlineSegment: []byte("raisins"),
},
} }
putRes2, err := c.Put(ctx, &fp2) _, err = c.Put(ctx, &pr2)
assert.NoError(t, err) if err != nil || status.Code(err) == codes.Internal {
t.Error("Failed to Put")
if putRes2.Confirmation != "success" {
t.Error("Failed to receive success Put response")
} }
if mdb.timesCalled != 3 { if mdb.timesCalled != 3 {
@ -104,17 +122,13 @@ func TestNetStateClient(t *testing.T) {
} }
// Test Server.Delete // Test Server.Delete
delReq := proto.DeleteRequest{ delReq := pb.DeleteRequest{
Path: []byte("here/is/a/path"), Path: []byte("here/is/a/path"),
} }
delRes, err := c.Delete(ctx, &delReq) _, err = c.Delete(ctx, &delReq)
if err != nil { if err != nil || status.Code(err) == codes.Internal {
t.Error("Failed to delete file path") t.Error("Failed to delete")
}
if delRes.Confirmation != "success" {
t.Error("Failed to receive success delete response")
} }
if mdb.timesCalled != 4 { if mdb.timesCalled != 4 {
@ -122,8 +136,11 @@ func TestNetStateClient(t *testing.T) {
} }
// Tests Server.List // Tests Server.List
listReq := proto.ListRequest{ listReq := pb.ListRequest{
Bucket: []byte("files"), // This pagination functionality doesn't work yet.
// The given arguments are placeholders.
StartingPathKey: []byte("test/pointer/path"),
Limit: 5,
} }
listRes, err := c.List(ctx, &listReq) listRes, err := c.List(ctx, &listReq)
@ -131,7 +148,7 @@ func TestNetStateClient(t *testing.T) {
t.Error("Failed to list file paths") t.Error("Failed to list file paths")
} }
if !bytes.Equal(listRes.Filepaths[0], []byte("here/is/another/path")) { if !bytes.Equal(listRes.Paths[0], []byte("here/is/another/path")) {
t.Error("Failed to list correct file path") t.Error("Failed to list correct file path")
} }

73
pkg/netstate/netstate.go
View File

@ -6,9 +6,12 @@ package netstate
import ( import (
"context" "context"
"github.com/golang/protobuf/proto"
"go.uber.org/zap" "go.uber.org/zap"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
proto "storj.io/storj/protos/netstate" pb "storj.io/storj/protos/netstate"
"storj.io/storj/storage/boltdb" "storj.io/storj/storage/boltdb"
) )
@ -30,75 +33,77 @@ func NewServer(db DB, logger *zap.Logger) *Server {
// and makes it easier in the future to substitute // and makes it easier in the future to substitute
// db clients other than bolt // db clients other than bolt
type DB interface { type DB interface {
Put(boltdb.File) error Put(boltdb.PointerEntry) error
Get([]byte) ([]byte, error) Get([]byte) ([]byte, error)
List() ([][]byte, error) List() ([][]byte, error)
Delete([]byte) error Delete([]byte) error
} }
// Put formats and hands off a file path to be saved to boltdb // Put formats and hands off a file path to be saved to boltdb
func (s *Server) Put(ctx context.Context, filepath *proto.FilePath) (*proto.PutResponse, error) { func (s *Server) Put(ctx context.Context, putReq *pb.PutRequest) (*pb.PutResponse, error) {
s.logger.Debug("entering netstate put") s.logger.Debug("entering netstate put")
file := boltdb.File{ pointerBytes, err := proto.Marshal(putReq.Pointer)
Path: []byte(filepath.Path), if err != nil {
Value: []byte(filepath.SmallValue), s.logger.Error("err marshaling pointer", zap.Error(err))
return nil, status.Errorf(codes.Internal, err.Error())
} }
if err := s.DB.Put(file); err != nil { pe := boltdb.PointerEntry{
s.logger.Error("err putting file", zap.Error(err)) Path: putReq.Path,
return nil, err Pointer: pointerBytes,
} }
s.logger.Debug("put to the db: " + string(file.Path))
return &proto.PutResponse{ if err := s.DB.Put(pe); err != nil {
Confirmation: "success", s.logger.Error("err putting pointer", zap.Error(err))
}, nil return nil, status.Errorf(codes.Internal, err.Error())
}
s.logger.Debug("put to the db: " + string(pe.Path))
return &pb.PutResponse{}, nil
} }
// Get formats and hands off a file path to get from boltdb // Get formats and hands off a file path to get from boltdb
func (s *Server) Get(ctx context.Context, req *proto.GetRequest) (*proto.GetResponse, error) { func (s *Server) Get(ctx context.Context, req *pb.GetRequest) (*pb.GetResponse, error) {
s.logger.Debug("entering netstate get") s.logger.Debug("entering netstate get")
fileValue, err := s.DB.Get(req.Path) pointerBytes, err := s.DB.Get(req.Path)
if err != nil { if err != nil {
s.logger.Error("err getting file", zap.Error(err)) s.logger.Error("err getting file", zap.Error(err))
return nil, err return nil, status.Errorf(codes.Internal, err.Error())
} }
return &proto.GetResponse{ return &pb.GetResponse{
SmallValue: fileValue, Pointer: pointerBytes,
}, nil }, nil
} }
// List calls the bolt client's List function and returns all file paths // List calls the bolt client's List function and returns all Path keys in the Pointers bucket
func (s *Server) List(ctx context.Context, req *proto.ListRequest) (*proto.ListResponse, error) { func (s *Server) List(ctx context.Context, req *pb.ListRequest) (*pb.ListResponse, error) {
s.logger.Debug("entering netstate list") s.logger.Debug("entering netstate list")
filePaths, err := s.DB.List() pathKeys, err := s.DB.List()
if err != nil { if err != nil {
s.logger.Error("err listing file paths", zap.Error(err)) s.logger.Error("err listing path keys", zap.Error(err))
return nil, err return nil, status.Errorf(codes.Internal, err.Error())
} }
s.logger.Debug("file paths retrieved") s.logger.Debug("path keys retrieved")
return &proto.ListResponse{ return &pb.ListResponse{
// filePaths is an array of byte arrays // pathKeys is an array of byte arrays
Filepaths: filePaths, Paths: pathKeys,
}, nil }, nil
} }
// Delete formats and hands off a file path to delete from boltdb // Delete formats and hands off a file path to delete from boltdb
func (s *Server) Delete(ctx context.Context, req *proto.DeleteRequest) (*proto.DeleteResponse, error) { func (s *Server) Delete(ctx context.Context, req *pb.DeleteRequest) (*pb.DeleteResponse, error) {
s.logger.Debug("entering netstate delete") s.logger.Debug("entering netstate delete")
err := s.DB.Delete(req.Path) err := s.DB.Delete(req.Path)
if err != nil { if err != nil {
s.logger.Error("err deleting file", zap.Error(err)) s.logger.Error("err deleting pointer entry", zap.Error(err))
return nil, err return nil, status.Errorf(codes.Internal, err.Error())
} }
s.logger.Debug("deleted: " + string(req.Path)) s.logger.Debug("deleted pointer at path: " + string(req.Path))
return &proto.DeleteResponse{ return &pb.DeleteResponse{}, nil
Confirmation: "success",
}, nil
} }

31
pkg/netstate/server_test.go
View File

@ -9,44 +9,45 @@ import (
"storj.io/storj/storage/boltdb" "storj.io/storj/storage/boltdb"
) )
type mockDB struct { // MockDB mocks db functionality for testing
type MockDB struct {
timesCalled int timesCalled int
puts []boltdb.File puts []boltdb.PointerEntry
filePaths [][]byte pathKeys [][]byte
} }
func (m *mockDB) Put(f boltdb.File) error { func (m *MockDB) Put(f boltdb.PointerEntry) error {
m.timesCalled++ m.timesCalled++
m.puts = append(m.puts, f) m.puts = append(m.puts, f)
return nil return nil
} }
func (m *mockDB) Get(path []byte) ([]byte, error) { func (m *MockDB) Get(path []byte) ([]byte, error) {
m.timesCalled++ m.timesCalled++
for _, file := range m.puts { for _, pointerEntry := range m.puts {
if bytes.Equal(path, file.Path) { if bytes.Equal(path, pointerEntry.Path) {
return file.Value, nil return pointerEntry.Pointer, nil
} }
} }
panic("failed to get the given file") panic("failed to get the given file")
} }
func (m *mockDB) List() ([][]byte, error) { func (m *MockDB) List() ([][]byte, error) {
m.timesCalled++ m.timesCalled++
for _, file := range m.puts { for _, putReq := range m.puts {
m.filePaths = append(m.filePaths, file.Path) m.pathKeys = append(m.pathKeys, putReq.Path)
} }
return m.filePaths, nil return m.pathKeys, nil
} }
func (m *mockDB) Delete(path []byte) error { func (m *MockDB) Delete(path []byte) error {
m.timesCalled++ m.timesCalled++
for i, file := range m.puts { for i, pointerEntry := range m.puts {
if bytes.Equal(path, file.Path) { if bytes.Equal(path, pointerEntry.Path) {
m.puts = append(m.puts[:i], m.puts[i+1:]...) m.puts = append(m.puts[:i], m.puts[i+1:]...)
} }
} }

471
protos/netstate/netstate.pb.go
View File

@ -8,7 +8,12 @@ It is generated from these files:
netstate.proto netstate.proto
It has these top-level messages: It has these top-level messages:
FilePath RedundancyScheme
EncryptionScheme
RemotePiece
RemoteSegment
Pointer
PutRequest
GetRequest GetRequest
ListRequest ListRequest
PutResponse PutResponse
@ -22,6 +27,7 @@ package netstate
import proto "github.com/golang/protobuf/proto" import proto "github.com/golang/protobuf/proto"
import fmt "fmt" import fmt "fmt"
import math "math" import math "math"
import google_protobuf "github.com/golang/protobuf/ptypes/timestamp"
import ( import (
context "golang.org/x/net/context" context "golang.org/x/net/context"
@ -39,29 +45,316 @@ var _ = math.Inf
// proto package needs to be updated. // proto package needs to be updated.
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
// FilePath is a request message for the Put rpc call type RedundancyScheme_SchemeType int32
type FilePath struct {
Path []byte `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` const (
// smallValue is a value too small to be broken up and stored RedundancyScheme_RS RedundancyScheme_SchemeType = 0
// in different places )
SmallValue []byte `protobuf:"bytes,2,opt,name=smallValue,proto3" json:"smallValue,omitempty"`
var RedundancyScheme_SchemeType_name = map[int32]string{
0: "RS",
}
var RedundancyScheme_SchemeType_value = map[string]int32{
"RS": 0,
} }
func (m *FilePath) Reset() { *m = FilePath{} } func (x RedundancyScheme_SchemeType) String() string {
func (m *FilePath) String() string { return proto.CompactTextString(m) } return proto.EnumName(RedundancyScheme_SchemeType_name, int32(x))
func (*FilePath) ProtoMessage() {} }
func (*FilePath) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } func (RedundancyScheme_SchemeType) EnumDescriptor() ([]byte, []int) {
return fileDescriptor0, []int{0, 0}
}
func (m *FilePath) GetPath() []byte { type EncryptionScheme_EncryptionType int32
const (
EncryptionScheme_AESGCM EncryptionScheme_EncryptionType = 0
EncryptionScheme_SECRETBOX EncryptionScheme_EncryptionType = 1
)
var EncryptionScheme_EncryptionType_name = map[int32]string{
0: "AESGCM",
1: "SECRETBOX",
}
var EncryptionScheme_EncryptionType_value = map[string]int32{
"AESGCM": 0,
"SECRETBOX": 1,
}
func (x EncryptionScheme_EncryptionType) String() string {
return proto.EnumName(EncryptionScheme_EncryptionType_name, int32(x))
}
func (EncryptionScheme_EncryptionType) EnumDescriptor() ([]byte, []int) {
return fileDescriptor0, []int{1, 0}
}
type Pointer_DataType int32
const (
Pointer_INLINE Pointer_DataType = 0
Pointer_REMOTE Pointer_DataType = 1
)
var Pointer_DataType_name = map[int32]string{
0: "INLINE",
1: "REMOTE",
}
var Pointer_DataType_value = map[string]int32{
"INLINE": 0,
"REMOTE": 1,
}
func (x Pointer_DataType) String() string {
return proto.EnumName(Pointer_DataType_name, int32(x))
}
func (Pointer_DataType) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{4, 0} }
type RedundancyScheme struct {
Type RedundancyScheme_SchemeType `protobuf:"varint,1,opt,name=type,enum=netstate.RedundancyScheme_SchemeType" json:"type,omitempty"`
// these values apply to RS encoding
MinReq int64 `protobuf:"varint,2,opt,name=min_req,json=minReq" json:"min_req,omitempty"`
Total int64 `protobuf:"varint,3,opt,name=total" json:"total,omitempty"`
RepairThreshold int64 `protobuf:"varint,4,opt,name=repair_threshold,json=repairThreshold" json:"repair_threshold,omitempty"`
SuccessThreshold int64 `protobuf:"varint,5,opt,name=success_threshold,json=successThreshold" json:"success_threshold,omitempty"`
}
func (m *RedundancyScheme) Reset() { *m = RedundancyScheme{} }
func (m *RedundancyScheme) String() string { return proto.CompactTextString(m) }
func (*RedundancyScheme) ProtoMessage() {}
func (*RedundancyScheme) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
func (m *RedundancyScheme) GetType() RedundancyScheme_SchemeType {
if m != nil {
return m.Type
}
return RedundancyScheme_RS
}
func (m *RedundancyScheme) GetMinReq() int64 {
if m != nil {
return m.MinReq
}
return 0
}
func (m *RedundancyScheme) GetTotal() int64 {
if m != nil {
return m.Total
}
return 0
}
func (m *RedundancyScheme) GetRepairThreshold() int64 {
if m != nil {
return m.RepairThreshold
}
return 0
}
func (m *RedundancyScheme) GetSuccessThreshold() int64 {
if m != nil {
return m.SuccessThreshold
}
return 0
}
type EncryptionScheme struct {
Type EncryptionScheme_EncryptionType `protobuf:"varint,1,opt,name=type,enum=netstate.EncryptionScheme_EncryptionType" json:"type,omitempty"`
EncryptedEncryptionKey []byte `protobuf:"bytes,2,opt,name=encrypted_encryption_key,json=encryptedEncryptionKey,proto3" json:"encrypted_encryption_key,omitempty"`
EncryptedStartingNonce []byte `protobuf:"bytes,3,opt,name=encrypted_starting_nonce,json=encryptedStartingNonce,proto3" json:"encrypted_starting_nonce,omitempty"`
}
func (m *EncryptionScheme) Reset() { *m = EncryptionScheme{} }
func (m *EncryptionScheme) String() string { return proto.CompactTextString(m) }
func (*EncryptionScheme) ProtoMessage() {}
func (*EncryptionScheme) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
func (m *EncryptionScheme) GetType() EncryptionScheme_EncryptionType {
if m != nil {
return m.Type
}
return EncryptionScheme_AESGCM
}
func (m *EncryptionScheme) GetEncryptedEncryptionKey() []byte {
if m != nil {
return m.EncryptedEncryptionKey
}
return nil
}
func (m *EncryptionScheme) GetEncryptedStartingNonce() []byte {
if m != nil {
return m.EncryptedStartingNonce
}
return nil
}
type RemotePiece struct {
PieceNum int64 `protobuf:"varint,1,opt,name=piece_num,json=pieceNum" json:"piece_num,omitempty"`
NodeId string `protobuf:"bytes,2,opt,name=node_id,json=nodeId" json:"node_id,omitempty"`
Size int64 `protobuf:"varint,3,opt,name=size" json:"size,omitempty"`
}
func (m *RemotePiece) Reset() { *m = RemotePiece{} }
func (m *RemotePiece) String() string { return proto.CompactTextString(m) }
func (*RemotePiece) ProtoMessage() {}
func (*RemotePiece) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
func (m *RemotePiece) GetPieceNum() int64 {
if m != nil {
return m.PieceNum
}
return 0
}
func (m *RemotePiece) GetNodeId() string {
if m != nil {
return m.NodeId
}
return ""
}
func (m *RemotePiece) GetSize() int64 {
if m != nil {
return m.Size
}
return 0
}
type RemoteSegment struct {
Redundancy *RedundancyScheme `protobuf:"bytes,1,opt,name=redundancy" json:"redundancy,omitempty"`
PieceName string `protobuf:"bytes,2,opt,name=piece_name,json=pieceName" json:"piece_name,omitempty"`
RemotePieces []*RemotePiece `protobuf:"bytes,3,rep,name=remote_pieces,json=remotePieces" json:"remote_pieces,omitempty"`
MerkleRoot []byte `protobuf:"bytes,4,opt,name=merkle_root,json=merkleRoot,proto3" json:"merkle_root,omitempty"`
MerkleSize int64 `protobuf:"varint,5,opt,name=merkle_size,json=merkleSize" json:"merkle_size,omitempty"`
}
func (m *RemoteSegment) Reset() { *m = RemoteSegment{} }
func (m *RemoteSegment) String() string { return proto.CompactTextString(m) }
func (*RemoteSegment) ProtoMessage() {}
func (*RemoteSegment) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
func (m *RemoteSegment) GetRedundancy() *RedundancyScheme {
if m != nil {
return m.Redundancy
}
return nil
}
func (m *RemoteSegment) GetPieceName() string {
if m != nil {
return m.PieceName
}
return ""
}
func (m *RemoteSegment) GetRemotePieces() []*RemotePiece {
if m != nil {
return m.RemotePieces
}
return nil
}
func (m *RemoteSegment) GetMerkleRoot() []byte {
if m != nil {
return m.MerkleRoot
}
return nil
}
func (m *RemoteSegment) GetMerkleSize() int64 {
if m != nil {
return m.MerkleSize
}
return 0
}
type Pointer struct {
Type Pointer_DataType `protobuf:"varint,1,opt,name=type,enum=netstate.Pointer_DataType" json:"type,omitempty"`
Encryption *EncryptionScheme `protobuf:"bytes,2,opt,name=encryption" json:"encryption,omitempty"`
InlineSegment []byte `protobuf:"bytes,3,opt,name=inline_segment,json=inlineSegment,proto3" json:"inline_segment,omitempty"`
Remote *RemoteSegment `protobuf:"bytes,4,opt,name=remote" json:"remote,omitempty"`
EncryptedUnencryptedSize []byte `protobuf:"bytes,5,opt,name=encrypted_unencrypted_size,json=encryptedUnencryptedSize,proto3" json:"encrypted_unencrypted_size,omitempty"`
CreationDate *google_protobuf.Timestamp `protobuf:"bytes,6,opt,name=creation_date,json=creationDate" json:"creation_date,omitempty"`
ExpirationDate *google_protobuf.Timestamp `protobuf:"bytes,7,opt,name=expiration_date,json=expirationDate" json:"expiration_date,omitempty"`
}
func (m *Pointer) Reset() { *m = Pointer{} }
func (m *Pointer) String() string { return proto.CompactTextString(m) }
func (*Pointer) ProtoMessage() {}
func (*Pointer) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
func (m *Pointer) GetType() Pointer_DataType {
if m != nil {
return m.Type
}
return Pointer_INLINE
}
func (m *Pointer) GetEncryption() *EncryptionScheme {
if m != nil {
return m.Encryption
}
return nil
}
func (m *Pointer) GetInlineSegment() []byte {
if m != nil {
return m.InlineSegment
}
return nil
}
func (m *Pointer) GetRemote() *RemoteSegment {
if m != nil {
return m.Remote
}
return nil
}
func (m *Pointer) GetEncryptedUnencryptedSize() []byte {
if m != nil {
return m.EncryptedUnencryptedSize
}
return nil
}
func (m *Pointer) GetCreationDate() *google_protobuf.Timestamp {
if m != nil {
return m.CreationDate
}
return nil
}
func (m *Pointer) GetExpirationDate() *google_protobuf.Timestamp {
if m != nil {
return m.ExpirationDate
}
return nil
}
// PutRequest is a request message for the Put rpc call
type PutRequest struct {
Path []byte `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
Pointer *Pointer `protobuf:"bytes,2,opt,name=pointer" json:"pointer,omitempty"`
}
func (m *PutRequest) Reset() { *m = PutRequest{} }
func (m *PutRequest) String() string { return proto.CompactTextString(m) }
func (*PutRequest) ProtoMessage() {}
func (*PutRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
func (m *PutRequest) GetPath() []byte {
if m != nil { if m != nil {
return m.Path return m.Path
} }
return nil return nil
} }
func (m *FilePath) GetSmallValue() []byte { func (m *PutRequest) GetPointer() *Pointer {
if m != nil { if m != nil {
return m.SmallValue return m.Pointer
} }
return nil return nil
} }
@ -74,7 +367,7 @@ type GetRequest struct {
func (m *GetRequest) Reset() { *m = GetRequest{} } func (m *GetRequest) Reset() { *m = GetRequest{} }
func (m *GetRequest) String() string { return proto.CompactTextString(m) } func (m *GetRequest) String() string { return proto.CompactTextString(m) }
func (*GetRequest) ProtoMessage() {} func (*GetRequest) ProtoMessage() {}
func (*GetRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } func (*GetRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
func (m *GetRequest) GetPath() []byte { func (m *GetRequest) GetPath() []byte {
if m != nil { if m != nil {
@ -85,68 +378,68 @@ func (m *GetRequest) GetPath() []byte {
// ListRequest is a request message for the List rpc call // ListRequest is a request message for the List rpc call
type ListRequest struct { type ListRequest struct {
Bucket []byte `protobuf:"bytes,1,opt,name=bucket,proto3" json:"bucket,omitempty"` StartingPathKey []byte `protobuf:"bytes,1,opt,name=starting_path_key,json=startingPathKey,proto3" json:"starting_path_key,omitempty"`
Limit int64 `protobuf:"varint,2,opt,name=limit" json:"limit,omitempty"`
} }
func (m *ListRequest) Reset() { *m = ListRequest{} } func (m *ListRequest) Reset() { *m = ListRequest{} }
func (m *ListRequest) String() string { return proto.CompactTextString(m) } func (m *ListRequest) String() string { return proto.CompactTextString(m) }
func (*ListRequest) ProtoMessage() {} func (*ListRequest) ProtoMessage() {}
func (*ListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } func (*ListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
func (m *ListRequest) GetBucket() []byte { func (m *ListRequest) GetStartingPathKey() []byte {
if m != nil { if m != nil {
return m.Bucket return m.StartingPathKey
} }
return nil return nil
} }
func (m *ListRequest) GetLimit() int64 {
if m != nil {
return m.Limit
}
return 0
}
// PutResponse is a response message for the Put rpc call // PutResponse is a response message for the Put rpc call
type PutResponse struct { type PutResponse struct {
Confirmation string `protobuf:"bytes,1,opt,name=confirmation" json:"confirmation,omitempty"`
} }
func (m *PutResponse) Reset() { *m = PutResponse{} } func (m *PutResponse) Reset() { *m = PutResponse{} }
func (m *PutResponse) String() string { return proto.CompactTextString(m) } func (m *PutResponse) String() string { return proto.CompactTextString(m) }
func (*PutResponse) ProtoMessage() {} func (*PutResponse) ProtoMessage() {}
func (*PutResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } func (*PutResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
func (m *PutResponse) GetConfirmation() string {
if m != nil {
return m.Confirmation
}
return ""
}
// GetResponse is a response message for the Get rpc call // GetResponse is a response message for the Get rpc call
type GetResponse struct { type GetResponse struct {
SmallValue []byte `protobuf:"bytes,1,opt,name=smallValue,proto3" json:"smallValue,omitempty"` Pointer []byte `protobuf:"bytes,1,opt,name=pointer,proto3" json:"pointer,omitempty"`
} }
func (m *GetResponse) Reset() { *m = GetResponse{} } func (m *GetResponse) Reset() { *m = GetResponse{} }
func (m *GetResponse) String() string { return proto.CompactTextString(m) } func (m *GetResponse) String() string { return proto.CompactTextString(m) }
func (*GetResponse) ProtoMessage() {} func (*GetResponse) ProtoMessage() {}
func (*GetResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } func (*GetResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
func (m *GetResponse) GetSmallValue() []byte { func (m *GetResponse) GetPointer() []byte {
if m != nil { if m != nil {
return m.SmallValue return m.Pointer
} }
return nil return nil
} }
// ListResponse is a response message for the List rpc call // ListResponse is a response message for the List rpc call
type ListResponse struct { type ListResponse struct {
Filepaths [][]byte `protobuf:"bytes,1,rep,name=filepaths,proto3" json:"filepaths,omitempty"` Paths [][]byte `protobuf:"bytes,1,rep,name=paths,proto3" json:"paths,omitempty"`
} }
func (m *ListResponse) Reset() { *m = ListResponse{} } func (m *ListResponse) Reset() { *m = ListResponse{} }
func (m *ListResponse) String() string { return proto.CompactTextString(m) } func (m *ListResponse) String() string { return proto.CompactTextString(m) }
func (*ListResponse) ProtoMessage() {} func (*ListResponse) ProtoMessage() {}
func (*ListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } func (*ListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
func (m *ListResponse) GetFilepaths() [][]byte { func (m *ListResponse) GetPaths() [][]byte {
if m != nil { if m != nil {
return m.Filepaths return m.Paths
} }
return nil return nil
} }
@ -158,7 +451,7 @@ type DeleteRequest struct {
func (m *DeleteRequest) Reset() { *m = DeleteRequest{} } func (m *DeleteRequest) Reset() { *m = DeleteRequest{} }
func (m *DeleteRequest) String() string { return proto.CompactTextString(m) } func (m *DeleteRequest) String() string { return proto.CompactTextString(m) }
func (*DeleteRequest) ProtoMessage() {} func (*DeleteRequest) ProtoMessage() {}
func (*DeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } func (*DeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
func (m *DeleteRequest) GetPath() []byte { func (m *DeleteRequest) GetPath() []byte {
if m != nil { if m != nil {
@ -169,23 +462,20 @@ func (m *DeleteRequest) GetPath() []byte {
// DeleteResponse is a response message for the Delete rpc call // DeleteResponse is a response message for the Delete rpc call
type DeleteResponse struct { type DeleteResponse struct {
Confirmation string `protobuf:"bytes,1,opt,name=confirmation" json:"confirmation,omitempty"`
} }
func (m *DeleteResponse) Reset() { *m = DeleteResponse{} } func (m *DeleteResponse) Reset() { *m = DeleteResponse{} }
func (m *DeleteResponse) String() string { return proto.CompactTextString(m) } func (m *DeleteResponse) String() string { return proto.CompactTextString(m) }
func (*DeleteResponse) ProtoMessage() {} func (*DeleteResponse) ProtoMessage() {}
func (*DeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } func (*DeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
func (m *DeleteResponse) GetConfirmation() string {
if m != nil {
return m.Confirmation
}
return ""
}
func init() { func init() {
proto.RegisterType((*FilePath)(nil), "netstate.FilePath") proto.RegisterType((*RedundancyScheme)(nil), "netstate.RedundancyScheme")
proto.RegisterType((*EncryptionScheme)(nil), "netstate.EncryptionScheme")
proto.RegisterType((*RemotePiece)(nil), "netstate.RemotePiece")
proto.RegisterType((*RemoteSegment)(nil), "netstate.RemoteSegment")
proto.RegisterType((*Pointer)(nil), "netstate.Pointer")
proto.RegisterType((*PutRequest)(nil), "netstate.PutRequest")
proto.RegisterType((*GetRequest)(nil), "netstate.GetRequest") proto.RegisterType((*GetRequest)(nil), "netstate.GetRequest")
proto.RegisterType((*ListRequest)(nil), "netstate.ListRequest") proto.RegisterType((*ListRequest)(nil), "netstate.ListRequest")
proto.RegisterType((*PutResponse)(nil), "netstate.PutResponse") proto.RegisterType((*PutResponse)(nil), "netstate.PutResponse")
@ -193,6 +483,9 @@ func init() {
proto.RegisterType((*ListResponse)(nil), "netstate.ListResponse") proto.RegisterType((*ListResponse)(nil), "netstate.ListResponse")
proto.RegisterType((*DeleteRequest)(nil), "netstate.DeleteRequest") proto.RegisterType((*DeleteRequest)(nil), "netstate.DeleteRequest")
proto.RegisterType((*DeleteResponse)(nil), "netstate.DeleteResponse") proto.RegisterType((*DeleteResponse)(nil), "netstate.DeleteResponse")
proto.RegisterEnum("netstate.RedundancyScheme_SchemeType", RedundancyScheme_SchemeType_name, RedundancyScheme_SchemeType_value)
proto.RegisterEnum("netstate.EncryptionScheme_EncryptionType", EncryptionScheme_EncryptionType_name, EncryptionScheme_EncryptionType_value)
proto.RegisterEnum("netstate.Pointer_DataType", Pointer_DataType_name, Pointer_DataType_value)
} }
// Reference imports to suppress errors if they are not otherwise used. // Reference imports to suppress errors if they are not otherwise used.
@ -207,7 +500,7 @@ const _ = grpc.SupportPackageIsVersion4
type NetStateClient interface { type NetStateClient interface {
// Put formats and hands off a file path to be saved to boltdb // Put formats and hands off a file path to be saved to boltdb
Put(ctx context.Context, in *FilePath, opts ...grpc.CallOption) (*PutResponse, error) Put(ctx context.Context, in *PutRequest, opts ...grpc.CallOption) (*PutResponse, error)
// Get formats and hands off a file path to get a small value from boltdb // Get formats and hands off a file path to get a small value from boltdb
Get(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetResponse, error) Get(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetResponse, error)
// List calls the bolt client's List function and returns all file paths // List calls the bolt client's List function and returns all file paths
@ -224,7 +517,7 @@ func NewNetStateClient(cc *grpc.ClientConn) NetStateClient {
return &netStateClient{cc} return &netStateClient{cc}
} }
func (c *netStateClient) Put(ctx context.Context, in *FilePath, opts ...grpc.CallOption) (*PutResponse, error) { func (c *netStateClient) Put(ctx context.Context, in *PutRequest, opts ...grpc.CallOption) (*PutResponse, error) {
out := new(PutResponse) out := new(PutResponse)
err := grpc.Invoke(ctx, "/netstate.NetState/Put", in, out, c.cc, opts...) err := grpc.Invoke(ctx, "/netstate.NetState/Put", in, out, c.cc, opts...)
if err != nil { if err != nil {
@ -264,7 +557,7 @@ func (c *netStateClient) Delete(ctx context.Context, in *DeleteRequest, opts ...
type NetStateServer interface { type NetStateServer interface {
// Put formats and hands off a file path to be saved to boltdb // Put formats and hands off a file path to be saved to boltdb
Put(context.Context, *FilePath) (*PutResponse, error) Put(context.Context, *PutRequest) (*PutResponse, error)
// Get formats and hands off a file path to get a small value from boltdb // Get formats and hands off a file path to get a small value from boltdb
Get(context.Context, *GetRequest) (*GetResponse, error) Get(context.Context, *GetRequest) (*GetResponse, error)
// List calls the bolt client's List function and returns all file paths // List calls the bolt client's List function and returns all file paths
@ -278,7 +571,7 @@ func RegisterNetStateServer(s *grpc.Server, srv NetStateServer) {
} }
func _NetState_Put_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { func _NetState_Put_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(FilePath) in := new(PutRequest)
if err := dec(in); err != nil { if err := dec(in); err != nil {
return nil, err return nil, err
} }
@ -290,7 +583,7 @@ func _NetState_Put_Handler(srv interface{}, ctx context.Context, dec func(interf
FullMethod: "/netstate.NetState/Put", FullMethod: "/netstate.NetState/Put",
} }
handler := func(ctx context.Context, req interface{}) (interface{}, error) { handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(NetStateServer).Put(ctx, req.(*FilePath)) return srv.(NetStateServer).Put(ctx, req.(*PutRequest))
} }
return interceptor(ctx, in, info, handler) return interceptor(ctx, in, info, handler)
} }
@ -377,24 +670,58 @@ var _NetState_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("netstate.proto", fileDescriptor0) } func init() { proto.RegisterFile("netstate.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{ var fileDescriptor0 = []byte{
// 302 bytes of a gzipped FileDescriptorProto // 847 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x52, 0x4d, 0x4b, 0xc3, 0x40, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x54, 0x7f, 0x6f, 0xe3, 0x44,
0x10, 0x25, 0xb6, 0x94, 0x74, 0x12, 0x7b, 0x18, 0xb4, 0x86, 0x20, 0x12, 0x56, 0x84, 0x1e, 0xb4, 0x10, 0xad, 0xcf, 0x6d, 0xda, 0x4e, 0x7e, 0xd4, 0x5d, 0xe5, 0xee, 0xac, 0x20, 0x44, 0x64, 0x38,
0x68, 0xd5, 0x93, 0xe0, 0x49, 0xec, 0x45, 0x24, 0x44, 0xf0, 0xbe, 0x2d, 0x53, 0x1a, 0xdc, 0x26, 0xd1, 0xa3, 0x52, 0x90, 0x82, 0x90, 0xa0, 0x80, 0x10, 0xb4, 0x51, 0x55, 0x71, 0x4d, 0xa3, 0x4d,
0xb1, 0x3b, 0xf9, 0xc7, 0xfe, 0x10, 0xc9, 0x57, 0xb3, 0xb1, 0x22, 0xde, 0x32, 0xef, 0x23, 0x79, 0x10, 0xfc, 0x67, 0xf9, 0xe2, 0x21, 0xb1, 0x2e, 0xde, 0x75, 0xbd, 0x6b, 0x89, 0xf0, 0xbd, 0xf8,
0x33, 0x2f, 0x30, 0x4a, 0x88, 0x35, 0x4b, 0xa6, 0x69, 0xb6, 0x4d, 0x39, 0x45, 0xbb, 0x99, 0xc5, 0x4a, 0x08, 0xf1, 0x27, 0x9f, 0x00, 0x79, 0x77, 0x6d, 0x6f, 0x72, 0xe2, 0xee, 0xaf, 0xec, 0xcc,
0x23, 0xd8, 0xcf, 0xb1, 0xa2, 0x50, 0xf2, 0x1a, 0x11, 0xfa, 0x99, 0xe4, 0xb5, 0x67, 0x05, 0xd6, 0xbc, 0x99, 0xcc, 0x7b, 0xfb, 0xbc, 0xd0, 0x63, 0x28, 0x85, 0x8c, 0x24, 0x8e, 0xb2, 0x9c, 0x4b,
0xc4, 0x8d, 0xca, 0x67, 0x3c, 0x03, 0xd0, 0x1b, 0xa9, 0xd4, 0xbb, 0x54, 0x39, 0x79, 0x07, 0x25, 0x4e, 0x4e, 0xaa, 0x78, 0x70, 0x26, 0x93, 0x14, 0x85, 0x8c, 0xd2, 0x4c, 0x97, 0x82, 0x7f, 0x1c,
0x63, 0x20, 0x22, 0x00, 0x98, 0x13, 0x47, 0xf4, 0x99, 0x93, 0xe6, 0xdf, 0xde, 0x20, 0x2e, 0xc0, 0xf0, 0x28, 0xc6, 0x05, 0x8b, 0x23, 0xb6, 0xdc, 0xce, 0x97, 0x6b, 0x4c, 0x91, 0x7c, 0x0d, 0x87,
0x79, 0x89, 0xf5, 0x4e, 0x32, 0x86, 0xc1, 0x22, 0x5f, 0x7e, 0x10, 0xd7, 0xa2, 0x7a, 0x12, 0x37, 0x72, 0x9b, 0xa1, 0xef, 0x0c, 0x9d, 0x8b, 0xde, 0xf8, 0xc5, 0xa8, 0x1e, 0xb7, 0x8f, 0x1c, 0xe9,
0xe0, 0x84, 0x39, 0x47, 0xa4, 0xb3, 0x34, 0xd1, 0x84, 0x02, 0xdc, 0x65, 0x9a, 0xac, 0xe2, 0xed, 0x9f, 0xc5, 0x36, 0x43, 0xaa, 0x5a, 0xc8, 0x73, 0x38, 0x4e, 0x13, 0x16, 0xe6, 0xf8, 0xe8, 0x3f,
0x46, 0x72, 0x9c, 0x26, 0xa5, 0x78, 0x18, 0x75, 0x30, 0x71, 0x05, 0x4e, 0xf9, 0xed, 0xda, 0xd2, 0x19, 0x3a, 0x17, 0x2e, 0x6d, 0xa5, 0x09, 0xa3, 0xf8, 0x48, 0xfa, 0x70, 0x24, 0xb9, 0x8c, 0x36,
0x8d, 0x6a, 0xed, 0x45, 0xbd, 0x04, 0xb7, 0x0a, 0x52, 0xeb, 0x4f, 0x61, 0xb8, 0x8a, 0x15, 0x15, 0xbe, 0xab, 0xd2, 0x3a, 0x20, 0x2f, 0xc1, 0xcb, 0x31, 0x8b, 0x92, 0x3c, 0x94, 0xeb, 0x1c, 0xc5,
0x21, 0xb5, 0x67, 0x05, 0xbd, 0x89, 0x1b, 0xb5, 0x80, 0x38, 0x87, 0xc3, 0x27, 0x52, 0xc4, 0xf4, 0x9a, 0x6f, 0x62, 0xff, 0x50, 0x01, 0xce, 0x74, 0x7e, 0x51, 0xa5, 0xc9, 0x25, 0x9c, 0x8b, 0x62,
0xd7, 0x6e, 0x77, 0x30, 0x6a, 0x44, 0xff, 0xcf, 0x3d, 0xfb, 0xb2, 0xc0, 0x7e, 0x25, 0x7e, 0x2b, 0xb9, 0x44, 0x21, 0x2c, 0xec, 0x91, 0xc2, 0x7a, 0xa6, 0x50, 0x83, 0x83, 0x3e, 0x40, 0xb3, 0x1a,
0x0a, 0xc0, 0x6b, 0xe8, 0x85, 0x39, 0x23, 0x4e, 0x77, 0x15, 0x35, 0x7d, 0xf8, 0xc7, 0x2d, 0x66, 0x69, 0xc1, 0x13, 0x3a, 0xf7, 0x0e, 0x82, 0x7f, 0x1d, 0xf0, 0x26, 0x6c, 0x99, 0x6f, 0x33, 0x99,
0x9e, 0x66, 0x06, 0xbd, 0x39, 0x31, 0x1e, 0xb5, 0x6c, 0xdb, 0x80, 0xe9, 0x31, 0x6f, 0x73, 0x0f, 0x70, 0x66, 0xc8, 0x7e, 0xb7, 0x43, 0xf6, 0x65, 0x43, 0x76, 0x1f, 0x69, 0x25, 0x2c, 0xc2, 0x5f,
0xfd, 0x62, 0x77, 0x34, 0x68, 0xa3, 0x14, 0x7f, 0xfc, 0x13, 0xae, 0x6d, 0x0f, 0x30, 0xa8, 0xf6, 0x81, 0x8f, 0x3a, 0x8f, 0x71, 0x88, 0x35, 0x22, 0x7c, 0x83, 0x5b, 0xa5, 0x40, 0x87, 0x3e, 0xab,
0xc3, 0x93, 0x56, 0xd1, 0x39, 0x8b, 0xef, 0xed, 0x13, 0x95, 0x79, 0x31, 0x28, 0xff, 0xb5, 0xdb, 0xeb, 0xcd, 0x80, 0x9f, 0x70, 0xbb, 0xdb, 0x29, 0x64, 0x94, 0xcb, 0x84, 0xad, 0x42, 0xc6, 0xd9,
0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x0c, 0x07, 0x2f, 0xb8, 0x7d, 0x02, 0x00, 0x00, 0x12, 0x95, 0x48, 0x76, 0xe7, 0xdc, 0x94, 0xa7, 0x65, 0x35, 0xb8, 0x84, 0xde, 0xee, 0x2e, 0x04,
0xa0, 0xf5, 0xc3, 0x64, 0x7e, 0x7b, 0x7d, 0xef, 0x1d, 0x90, 0x2e, 0x9c, 0xce, 0x27, 0xd7, 0x74,
0xb2, 0xf8, 0xf1, 0xe1, 0x57, 0xcf, 0x09, 0x7e, 0x81, 0x36, 0xc5, 0x94, 0x4b, 0x9c, 0x25, 0xb8,
0x44, 0xf2, 0x01, 0x9c, 0x66, 0xe5, 0x21, 0x64, 0x45, 0xaa, 0x38, 0xbb, 0xf4, 0x44, 0x25, 0xa6,
0x45, 0x5a, 0xde, 0x1e, 0xe3, 0x31, 0x86, 0x49, 0xac, 0x76, 0x3f, 0xa5, 0xad, 0x32, 0xbc, 0x8b,
0x09, 0x81, 0x43, 0x91, 0xfc, 0x81, 0xe6, 0xf2, 0xd4, 0x39, 0xf8, 0xdb, 0x81, 0xae, 0x9e, 0x3c,
0xc7, 0x55, 0x8a, 0x4c, 0x92, 0x2b, 0x80, 0xbc, 0x76, 0x88, 0x1a, 0xde, 0x1e, 0x0f, 0xfe, 0xdf,
0x3d, 0xd4, 0x42, 0x93, 0x0f, 0x01, 0xcc, 0x5e, 0x51, 0x8a, 0xe6, 0xdf, 0xf5, 0xa6, 0xd3, 0x28,
0x45, 0x72, 0x05, 0xdd, 0x5c, 0xfd, 0x57, 0xa8, 0x72, 0xc2, 0x77, 0x87, 0xee, 0x45, 0x7b, 0xfc,
0xd4, 0x9e, 0x5e, 0x93, 0xa4, 0x9d, 0xbc, 0x09, 0x04, 0xf9, 0x08, 0xda, 0x29, 0xe6, 0x6f, 0x36,
0x18, 0xe6, 0x9c, 0x4b, 0xe5, 0xaf, 0x0e, 0x05, 0x9d, 0xa2, 0x9c, 0x4b, 0x0b, 0xa0, 0x48, 0x6a,
0x53, 0x19, 0xc0, 0xbc, 0xa4, 0xfa, 0xa7, 0x0b, 0xc7, 0x33, 0x9e, 0x30, 0x89, 0x39, 0x19, 0xed,
0xf8, 0xc5, 0xa2, 0x67, 0x00, 0xa3, 0x9b, 0x48, 0x46, 0x96, 0x41, 0xae, 0x00, 0x1a, 0x5b, 0x28,
0x62, 0x3b, 0xa2, 0xec, 0xbb, 0x8c, 0x5a, 0x68, 0xf2, 0x02, 0x7a, 0x09, 0xdb, 0x24, 0x0c, 0x43,
0xa1, 0x25, 0x36, 0xc6, 0xe8, 0xea, 0x6c, 0xa5, 0xfb, 0xe7, 0xd0, 0xd2, 0x84, 0x15, 0xb7, 0xf6,
0xf8, 0xf9, 0xbe, 0x2a, 0x06, 0x48, 0x0d, 0x8c, 0x7c, 0x0b, 0x83, 0xc6, 0x7a, 0x05, 0xb3, 0x6c,
0x58, 0xf1, 0xef, 0xd0, 0xc6, 0x9c, 0x3f, 0x37, 0x80, 0x52, 0x0d, 0xf2, 0x3d, 0x74, 0x97, 0x39,
0x46, 0xca, 0xe6, 0x71, 0x24, 0xd1, 0x6f, 0x19, 0x52, 0x2b, 0xce, 0x57, 0x1b, 0xf3, 0xe8, 0xbc,
0x2e, 0x7e, 0x1b, 0x2d, 0xaa, 0xc7, 0x86, 0x76, 0xaa, 0x86, 0x9b, 0x48, 0x22, 0xb9, 0x86, 0x33,
0xfc, 0x3d, 0x4b, 0x72, 0x6b, 0xc4, 0xf1, 0x7b, 0x47, 0xf4, 0x9a, 0x96, 0x72, 0x48, 0x10, 0xc0,
0x49, 0xa5, 0x74, 0x69, 0xff, 0xbb, 0xe9, 0xab, 0xbb, 0xe9, 0xc4, 0x3b, 0x28, 0xcf, 0x74, 0x72,
0xff, 0xb0, 0x98, 0x78, 0x4e, 0x70, 0x0f, 0x30, 0x2b, 0x24, 0xc5, 0xc7, 0x02, 0x85, 0x2c, 0x4d,
0x9c, 0x45, 0x72, 0xad, 0x6e, 0xae, 0x43, 0xd5, 0x99, 0x5c, 0xc2, 0x71, 0xa6, 0xef, 0xcd, 0x5c,
0xcd, 0xf9, 0x5b, 0x17, 0x4a, 0x2b, 0x44, 0x30, 0x04, 0xb8, 0xc5, 0x77, 0x8d, 0x0b, 0x1e, 0xa0,
0xfd, 0x2a, 0x11, 0x35, 0xe4, 0x33, 0x38, 0xaf, 0x3f, 0xec, 0xb2, 0xae, 0x5e, 0x05, 0x8d, 0x3f,
0xab, 0x0a, 0xb3, 0x48, 0xae, 0xcb, 0xe7, 0xa0, 0x0f, 0x47, 0x9b, 0x24, 0x4d, 0xa4, 0x79, 0x37,
0x75, 0x10, 0x74, 0xa1, 0xad, 0x18, 0x88, 0x8c, 0x33, 0x81, 0xc1, 0xa7, 0xd0, 0x56, 0x1b, 0xe8,
0x90, 0xf8, 0xcd, 0xf6, 0x7a, 0x6a, 0xbd, 0xea, 0x27, 0xd0, 0xd1, 0x8b, 0x18, 0x64, 0x1f, 0x8e,
0xca, 0x05, 0x84, 0xef, 0x0c, 0xdd, 0x8b, 0x0e, 0xd5, 0x41, 0xf0, 0x31, 0x74, 0x6f, 0x70, 0x83,
0x12, 0xdf, 0xc5, 0xc9, 0x83, 0x5e, 0x05, 0xd2, 0xc3, 0xc6, 0x7f, 0x39, 0x70, 0x32, 0x45, 0x39,
0x2f, 0x55, 0x22, 0x63, 0x70, 0x67, 0x85, 0x24, 0x7d, 0x4b, 0xb7, 0x5a, 0xf2, 0xc1, 0xd3, 0xbd,
0xac, 0xd9, 0x66, 0x0c, 0xee, 0x2d, 0xee, 0xf4, 0x34, 0xba, 0xda, 0x3d, 0x36, 0xd7, 0x2f, 0xe1,
0xb0, 0x64, 0x44, 0xac, 0xb2, 0x25, 0xf5, 0xe0, 0xd9, 0x7e, 0xda, 0xb4, 0x7d, 0x03, 0x2d, 0xbd,
0x3d, 0xb1, 0xbe, 0x8a, 0x1d, 0xd2, 0x03, 0xff, 0xed, 0x82, 0x6e, 0x7e, 0xdd, 0x52, 0x3e, 0xfc,
0xe2, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xab, 0x26, 0xfc, 0xe1, 0x51, 0x07, 0x00, 0x00,
} }

77
protos/netstate/netstate.proto
View File

@ -4,10 +4,12 @@
syntax = "proto3"; syntax = "proto3";
package netstate; package netstate;
import "timestamp.proto";
// NetState defines the interface for interacting with the network state persistence layer // NetState defines the interface for interacting with the network state persistence layer
service NetState { service NetState {
// Put formats and hands off a file path to be saved to boltdb // Put formats and hands off a file path to be saved to boltdb
rpc Put(FilePath) returns (PutResponse); rpc Put(PutRequest) returns (PutResponse);
// Get formats and hands off a file path to get a small value from boltdb // Get formats and hands off a file path to get a small value from boltdb
rpc Get(GetRequest) returns (GetResponse); rpc Get(GetRequest) returns (GetResponse);
// List calls the bolt client's List function and returns all file paths // List calls the bolt client's List function and returns all file paths
@ -16,12 +18,66 @@ service NetState {
rpc Delete(DeleteRequest) returns (DeleteResponse); rpc Delete(DeleteRequest) returns (DeleteResponse);
} }
// FilePath is a request message for the Put rpc call message RedundancyScheme {
message FilePath { enum SchemeType {
RS = 0;
}
SchemeType type = 1;
// these values apply to RS encoding
int64 min_req = 2; // minimum required for reconstruction
int64 total = 3; // total amount of pieces we generated
int64 repair_threshold = 4; // amount of pieces we need to drop to before triggering repair
int64 success_threshold = 5; // amount of pieces we need to store to call it a success
}
message EncryptionScheme {
enum EncryptionType {
AESGCM = 0;
SECRETBOX = 1;
// only allow authenticated encryption schemes
}
EncryptionType type = 1;
bytes encrypted_encryption_key = 2;
bytes encrypted_starting_nonce = 3;
}
message RemotePiece {
int64 piece_num = 1;
string node_id = 2;
int64 size = 3; // full size including merkle tree
}
message RemoteSegment {
RedundancyScheme redundancy = 1;
string piece_name = 2;
repeated RemotePiece remote_pieces = 3;
bytes merkle_root = 4; // root hash of the hashes of all of these pieces
int64 merkle_size = 5; // amount of space the merkle tree takes up at the end of each chunk
}
message Pointer {
enum DataType {
INLINE = 0;
REMOTE = 1;
}
DataType type = 1;
EncryptionScheme encryption = 2;
bytes inline_segment = 3;
RemoteSegment remote = 4;
bytes encrypted_unencrypted_size = 5; // size of the unencrypted object (!)
google.protobuf.Timestamp creation_date = 6;
google.protobuf.Timestamp expiration_date = 7;
}
// PutRequest is a request message for the Put rpc call
message PutRequest {
bytes path = 1; bytes path = 1;
// smallValue is a value too small to be broken up and stored Pointer pointer = 2;
// in different places
bytes smallValue = 2;
} }
// GetRequest is a request message for the Get rpc call // GetRequest is a request message for the Get rpc call
@ -31,22 +87,22 @@ message GetRequest {
// ListRequest is a request message for the List rpc call // ListRequest is a request message for the List rpc call
message ListRequest { message ListRequest {
bytes bucket = 1; bytes starting_path_key = 1; // the Path key in the bucket to start listing
int64 limit = 2; // how many keys to list
} }
// PutResponse is a response message for the Put rpc call // PutResponse is a response message for the Put rpc call
message PutResponse { message PutResponse {
string confirmation = 1;
} }
// GetResponse is a response message for the Get rpc call // GetResponse is a response message for the Get rpc call
message GetResponse { message GetResponse {
bytes smallValue = 1; bytes pointer = 1; // this is a Pointer type marshalled into bytes
} }
// ListResponse is a response message for the List rpc call // ListResponse is a response message for the List rpc call
message ListResponse { message ListResponse {
repeated bytes filepaths = 1; repeated bytes paths = 1;
} }
message DeleteRequest { message DeleteRequest {
@ -55,5 +111,4 @@ message DeleteRequest {
// DeleteResponse is a response message for the Delete rpc call // DeleteResponse is a response message for the Delete rpc call
message DeleteResponse { message DeleteResponse {
string confirmation = 1;
} }

51
storage/boltdb/netstate.go
View File

@ -7,53 +7,54 @@ import (
"github.com/boltdb/bolt" "github.com/boltdb/bolt"
) )
// File Path and Value are saved to boltdb // PointerEntry - Path and Pointer are saved as a kv pair to boltdb.
type File struct { // The following boltdb methods handle the pointer type (defined in
Path []byte `json:"path"` // the protobuf file) after it has been marshalled into bytes.
Value []byte `json:"value"` type PointerEntry struct {
Path []byte
Pointer []byte
} }
const ( const (
fileBucketName = "files" pointerBucket = "pointers"
) )
// Put saves the file path and value as a kv pair in the "files" bucket // Put saves the Path and Pointer as a kv entry in the "pointers" bucket
func (client *Client) Put(file File) error { func (client *Client) Put(pe PointerEntry) error {
client.logger.Debug("entering bolt put") client.logger.Debug("entering bolt put")
return client.db.Update(func(tx *bolt.Tx) error { return client.db.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(fileBucketName)) b, err := tx.CreateBucketIfNotExists([]byte(pointerBucket))
if err != nil { if err != nil {
return err return err
} }
fileKey := []byte(file.Path) return b.Put(pe.Path, pe.Pointer)
return b.Put(fileKey, file.Value)
}) })
} }
// Get retrieves the value stored at the file path key // Get retrieves the Pointer value stored at the Path key
func (client *Client) Get(fileKey []byte) ([]byte, error) { func (client *Client) Get(pathKey []byte) ([]byte, error) {
client.logger.Debug("entering bolt get: " + string(fileKey)) client.logger.Debug("entering bolt get: " + string(pathKey))
var fileValue []byte var pointerBytes []byte
err := client.db.Update(func(tx *bolt.Tx) error { err := client.db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(fileBucketName)) b := tx.Bucket([]byte(pointerBucket))
v := b.Get(fileKey) v := b.Get(pathKey)
if v == nil { if v == nil {
return Error.New("file %#v not found", string(fileKey)) return Error.New("pointer at %#v not found", string(pathKey))
} }
fileValue = v pointerBytes = v
return nil return nil
}) })
return fileValue, err return pointerBytes, err
} }
// List creates a string array of all keys in in the "files" bucket // List creates a byte array of all path keys in in the "pointers" bucket
func (client *Client) List() ([][]byte, error) { func (client *Client) List() ([][]byte, error) {
client.logger.Debug("entering bolt list") client.logger.Debug("entering bolt list")
var paths [][]byte var paths [][]byte
err := client.db.Update(func(tx *bolt.Tx) error { err := client.db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(fileBucketName)) b := tx.Bucket([]byte(pointerBucket))
err := b.ForEach(func(key, value []byte) error { err := b.ForEach(func(key, value []byte) error {
paths = append(paths, key) paths = append(paths, key)
@ -65,10 +66,10 @@ func (client *Client) List() ([][]byte, error) {
return paths, err return paths, err
} }
// Delete deletes a kv pair from the "files" bucket, given the key // Delete deletes a kv pair from the "pointers" bucket, given the Path key
func (client *Client) Delete(fileKey []byte) error { func (client *Client) Delete(pathKey []byte) error {
client.logger.Debug("entering bolt delete: " + string(fileKey)) client.logger.Debug("entering bolt delete: " + string(pathKey))
return client.db.Update(func(tx *bolt.Tx) error { return client.db.Update(func(tx *bolt.Tx) error {
return tx.Bucket([]byte(fileBucketName)).Delete(fileKey) return tx.Bucket([]byte(pointerBucket)).Delete(pathKey)
}) })
} }

36
storage/boltdb/netstate_test.go
View File

@ -36,46 +36,46 @@ func TestNetState(t *testing.T) {
os.Remove(c.Path) os.Remove(c.Path)
}() }()
testFile := File{ testEntry1 := PointerEntry{
Path: []byte(`test/path`), Path: []byte(`test/path`),
Value: []byte(`test value`), Pointer: []byte(`pointer1`),
} }
testFile2 := File{ testEntry2 := PointerEntry{
Path: []byte(`test/path2`), Path: []byte(`test/path2`),
Value: []byte(`value2`), Pointer: []byte(`pointer2`),
} }
// tests Put function // tests Put function
if err := c.Put(testFile); err != nil { if err := c.Put(testEntry1); err != nil {
t.Error("Failed to save testFile to files bucket") t.Error("Failed to save testFile to pointers bucket")
} }
// tests Get function // tests Get function
retrvValue, err := c.Get([]byte("test/path")) retrvValue, err := c.Get([]byte("test/path"))
if err != nil { if err != nil {
t.Error("Failed to get saved test value") t.Error("Failed to get saved test pointer")
} }
if !bytes.Equal(retrvValue, testFile.Value) { if !bytes.Equal(retrvValue, testEntry1.Pointer) {
t.Error("Retrieved file was not same as original file") t.Error("Retrieved pointer was not same as put pointer")
} }
// tests Delete function // tests Delete function
if err := c.Delete([]byte("test/path")); err != nil { if err := c.Delete([]byte("test/path")); err != nil {
t.Error("Failed to delete testfile") t.Error("Failed to delete test entry")
} }
// tests List function // tests List function
if err := c.Put(testFile2); err != nil { if err := c.Put(testEntry2); err != nil {
t.Error("Failed to save testFile2 to files bucket") t.Error("Failed to put testEntry2 to pointers bucket")
} }
testFiles, err := c.List() testPaths, err := c.List()
if err != nil { if err != nil {
t.Error("Failed to list file keys") t.Error("Failed to list Path keys in pointers bucket")
} }
// tests List + Delete function // tests List + Delete function
if !bytes.Equal(testFiles[0], []byte("test/path2")) { if !bytes.Equal(testPaths[0], []byte("test/path2")) {
t.Error("Expected only testFile2 in list") t.Error("Expected only testEntry2 path in list")
} }
} }