JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

adds zap logger, some flags, some nit fixes

Browse Source
This commit is contained in:
Natalie Ventura Villasana 2018-04-20 19:54:18 -04:00 committed by JT Olds
parent d93dacb53e
commit 4e8336f2ab
11 changed files with 242 additions and 395 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
cmd/net-state/main.go
View File

@ -1,54 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package main
import (
"log"
"net/http"
"os"
"github.com/julienschmidt/httprouter"
"go.uber.org/zap"
"storj.io/storj/netstate/routes"
"storj.io/storj/storage/boltdb"
)
func main() {
err := Main()
if err != nil {
log.Fatalf("fatal error: %v", err)
os.Exit(1)
}
}
func Main() error {
logger, err := zap.NewDevelopment()
if err != nil {
return err
}
defer logger.Sync()
logger.Info("serving on localhost:3000")
bdb, err := boltdb.New("netstate.db")
if err != nil {
return err
}
defer bdb.Close()
routes := routes.NewNetStateRoutes(bdb)
return http.ListenAndServe(":3000", start(routes))
}
func start(f *routes.NetStateRoutes) *httprouter.Router {
router := httprouter.New()
router.PUT("/file/*path", f.Put)
router.GET("/file/*path", f.Get)
router.GET("/file", f.List)
router.DELETE("/file/*path", f.Delete)
return router
}

6
cmd/net-state/README.md → cmd/netstate/README.md
View File

@ -11,6 +11,8 @@ To run:
go run cmd/net-state/main.go
```
You can also run using these flags: `-port=<port-number> -prod=<bool> -db=<db-name>`
Then you can use http methods (Put, Get, List, and Delete) to interact with small values stored on BoltDB.
To store a value to a PUT request body, use the format:
```
@ -20,6 +22,4 @@ To store a value to a PUT request body, use the format:
```
TODO:
- add zap logger throughout
- add functions for grpc + protobufs
- add http tests
- add functions for grpc + protobufs

75
cmd/netstate/main.go Normal file
View File

@ -0,0 +1,75 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package main
import (
"flag"
"fmt"
"log"
"net/http"
"os"
"github.com/julienschmidt/httprouter"
"go.uber.org/zap"
"github.com/storj/netstate/routes"
"github.com/storj/storage/boltdb"
)
var (
port int
dbPath string
prod bool
)
func initializeFlags() {
flag.IntVar(&port, "port", 3000, "port")
flag.StringVar(&dbPath, "db", "netstate.db", "db path")
flag.BoolVar(&prod, "prod", false, "The environment this service is running in")
flag.Parse()
}
func main() {
err := Main()
if err != nil {
log.Fatalf("fatal error: %v", err)
os.Exit(1)
}
}
// Main allows simplified error handling
func Main() error {
initializeFlags()
// No err here because no vars passed into NewDevelopment().
// The default won't return an error, but if args are passed in,
// then there will need to be error handling.
logger, _ := zap.NewDevelopment()
if prod {
logger, _ = zap.NewProduction()
}
defer logger.Sync()
logger.Info(fmt.Sprintf("serving on %d", port))
bdb, err := boltdb.New(logger, dbPath)
if err != nil {
return err
}
defer bdb.Close()
routes := routes.NewNetStateRoutes(logger, bdb)
return http.ListenAndServe(fmt.Sprintf(":%d", port), start(routes))
}
func start(f *routes.NetStateRoutes) *httprouter.Router {
router := httprouter.New()
router.PUT("/file/*path", f.Put)
router.GET("/file/*path", f.Get)
router.GET("/file", f.List)
router.DELETE("/file/*path", f.Delete)
return router
}

46
cmd/storj-node/main.go
View File

@ -1,46 +0,0 @@
package main
import (
"fmt"
"github.com/kataras/iris"
"storj.io/storj/routes"
"storj.io/storj/storage/boltdb"
)
func main() {
bdb, err := boltdb.New()
if err != nil {
fmt.Println(err)
return
}
defer bdb.DB.Close()
users := routes.Users{DB: bdb}
app := iris.Default()
SetRoutes(app, users)
app.Run(iris.Addr(":8080"))
}
// SetRoutes defines all restful routes on the service
func SetRoutes(app *iris.Application, users routes.Users) {
app.Post("/users/:id", users.CreateUser)
app.Get("/users/:id", users.GetUser)
app.Put("/users/:id/:email", users.UpdateUser)
app.Delete("/users/:id", users.DeleteUser)
// app.Get("/users/confirmations/:token", users.Confirm)
// app.Get("/files?startDate=<timestamp>?tag=<tag>", files.ListFiles)
// app.Get("/file-ids/:name", files.GetFileId)
// app.Get("/files/:file?skip=<number>&limit=<number>&exclude=<node-ids>", files.GetPointers)
// app.Delete("/files/:file", files.DeleteFile)
// app.Post("/files", files.NewFile)
// app.Put("/files/:file/shards/:index", files.AddShardToFile)
// app.Post("/reports", reports.CreateReport)
// app.Get("/contacts?address=<address>&skip=<number>&limit=<number>", contacts.GetContacts)
}

104
netstate/routes/net-state-files.go
View File

@ -1,104 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package routes
import (
"encoding/json"
"fmt"
"log"
"net/http"
"github.com/julienschmidt/httprouter"
"storj.io/storj/storage/boltdb"
)
type NetStateRoutes struct {
DB *boltdb.Client
}
type Message struct {
Value string `json:"value"`
}
func NewNetStateRoutes(db *boltdb.Client) *NetStateRoutes {
return &NetStateRoutes{DB: db}
}
func (n *NetStateRoutes) Put(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
givenPath := ps.ByName("path")
var msg Message
decoder := json.NewDecoder(r.Body)
err := decoder.Decode(&msg)
if err != nil {
http.Error(w, "bad request: err decoding response", http.StatusBadRequest)
log.Printf("err decoding response: %v", err)
return
}
file := boltdb.File{
Path: givenPath,
Value: msg.Value,
}
if err := n.DB.Put(file); err != nil {
http.Error(w, "err saving file", http.StatusInternalServerError)
log.Println(err)
return
}
fmt.Fprintf(w, "PUT to %s\n", givenPath)
}
func (n *NetStateRoutes) Get(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
fileKey := ps.ByName("path")
fileInfo, err := n.DB.Get([]byte(fileKey))
if err != nil {
http.Error(w, "err getting file", http.StatusInternalServerError)
log.Println(err)
return
}
bytes, err := json.Marshal(fileInfo)
if err != nil {
http.Error(w, "internal error: unable to get value", http.StatusInternalServerError)
return
}
_, err = w.Write(bytes)
if err != nil {
log.Printf("failed writing response: %v", err)
}
}
func (n *NetStateRoutes) List(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
fileKeys, err := n.DB.List()
if err != nil {
http.Error(w, "internal error: unable to list paths", http.StatusInternalServerError)
log.Println(err)
return
}
bytes, err := json.Marshal(fileKeys)
if err != nil {
http.Error(w, "internal error: unable to marshal path list", http.StatusInternalServerError)
log.Println(err)
return
}
_, err = w.Write(bytes)
if err != nil {
log.Printf("failed writing response: %v", err)
}
}
func (n *NetStateRoutes) Delete(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
fileKey := ps.ByName("path")
if err := n.DB.Delete([]byte(fileKey)); err != nil {
http.Error(w, "internal error: unable to delete file", http.StatusInternalServerError)
log.Printf("err deleting file %v", err)
return
}
fmt.Fprintf(w, "Deleted file key: %s", fileKey)
}

125
netstate/routes/netstate-routes.go Normal file
View File

@ -0,0 +1,125 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package routes
import (
"encoding/json"
"fmt"
"net/http"
"github.com/julienschmidt/httprouter"
"go.uber.org/zap"
"github.com/storj/storage/boltdb"
)
// NetStateRoutes maintains access to a boltdb client and zap logger
type NetStateRoutes struct {
DB *boltdb.Client
logger *zap.Logger
}
// Message contains the small value provided by the user to be stored
type Message struct {
Value string `json:"value"`
}
// NewNetStateRoutes instantiates NetStateRoutes
func NewNetStateRoutes(logger *zap.Logger, db *boltdb.Client) *NetStateRoutes {
return &NetStateRoutes{
DB: db,
logger: logger,
}
}
// Put takes the given path and small value from the user and formats the values
// to be given to boltdb.Put
func (n *NetStateRoutes) Put(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
n.logger.Debug("entering NetStateRoutes.Put(...)")
givenPath := ps.ByName("path")
var msg Message
decoder := json.NewDecoder(r.Body)
err := decoder.Decode(&msg)
if err != nil {
http.Error(w, "bad request: err decoding response", http.StatusBadRequest)
n.logger.Error("err decoding response", zap.Error(err))
return
}
file := boltdb.File{
Path: givenPath,
Value: []byte(msg.Value),
}
if err := n.DB.Put(file); err != nil {
http.Error(w, "err putting file", http.StatusInternalServerError)
n.logger.Error("err putting file", zap.Error(err))
return
}
n.logger.Debug("the file was put to the db")
fmt.Fprintf(w, "PUT to %s\n", givenPath)
}
// Get takes the given file path from the user and calls the bolt client's Get function
func (n *NetStateRoutes) Get(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
n.logger.Debug("entering NetStateRoutes.Get(...)")
fileKey := ps.ByName("path")
fileInfo, err := n.DB.Get([]byte(fileKey))
if err != nil {
http.Error(w, "err getting file", http.StatusInternalServerError)
n.logger.Error("err getting file", zap.Error(err))
return
}
w.Header().Set("Content-Type", "application/octet-stream")
_, err = w.Write(fileInfo.Value)
if err != nil {
n.logger.Error("err writing response", zap.Error(err))
}
n.logger.Debug("response written")
}
// List calls the bolt client's List function and responds with a list of all saved file paths
// or "filekeys"
func (n *NetStateRoutes) List(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
n.logger.Debug("entering NetStateRoutes.List(...)")
fileKeys, err := n.DB.List()
if err != nil {
http.Error(w, "internal error: unable to list paths", http.StatusInternalServerError)
n.logger.Error("err listing file paths", zap.Error(err))
return
}
bytes, err := json.Marshal(fileKeys)
if err != nil {
http.Error(w, "internal error: unable to marshal path list", http.StatusInternalServerError)
n.logger.Error("err marshaling path list", zap.Error(err))
return
}
_, err = w.Write(bytes)
if err != nil {
n.logger.Error("err writing response", zap.Error(err))
}
n.logger.Debug("response written")
}
// Delete takes a given file path and calls the bolt client's Delete function
func (n *NetStateRoutes) Delete(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
n.logger.Debug("entering NetStateRoutes.Delete(...)")
fileKey := ps.ByName("path")
if err := n.DB.Delete([]byte(fileKey)); err != nil {
http.Error(w, "internal error: unable to delete file", http.StatusInternalServerError)
n.logger.Error("err deleting file", zap.Error(err))
return
}
n.logger.Debug("file deleted")
fmt.Fprintf(w, "Deleted file key: %s", fileKey)
}

68
routes/users.go
View File

@ -1,68 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package routes
import (
"log"
"github.com/google/uuid"
"github.com/kataras/iris"
"storj.io/storj/storage/boltdb"
)
// Users contains items needed to process requests to the user namespace
type Users struct {
DB *boltdb.Client
}
func (u *Users) CreateUser(ctx iris.Context) {
user := boltdb.User{
Id: uuid.New(),
Username: ctx.Params().Get("id"),
Email: `dece@trali.zzd`,
}
if err := ctx.ReadJSON(user); err != nil {
ctx.JSON(iris.StatusNotAcceptable)
}
u.DB.CreateUser(user)
}
func (u *Users) GetUser(ctx iris.Context) {
userId := ctx.Params().Get("id")
userInfo, err := u.DB.GetUser([]byte(userId))
if err != nil {
log.Println(err)
}
ctx.Writef("%s's info is: %s", userId, userInfo)
}
// Updates only email for now
// Uses two db queries now, can refactor
func (u *Users) UpdateUser(ctx iris.Context) {
userId := ctx.Params().Get("id")
userInfo, err := u.DB.GetUser([]byte(userId))
if err != nil {
log.Println(err)
}
updated := boltdb.User{
Id: userInfo.Id,
Username: userInfo.Username,
Email: ctx.Params().Get("email"),
}
err1 := u.DB.UpdateUser(updated)
if err1 != nil {
log.Println(err)
}
}
func (u *Users) DeleteUser(ctx iris.Context) {
userId := ctx.Params().Get("id")
u.DB.DeleteUser([]byte(userId))
}

21
storage/boltdb/client.go
View File

@ -7,31 +7,40 @@ import (
"time"
"github.com/boltdb/bolt"
"go.uber.org/zap"
)
var (
defaultTimeout = 1 * time.Second
)
const (
// fileMode sets permissions so owner can read and write
fileMode = 0600
)
// Client is the storage interface for the Bolt database
type Client struct {
db *bolt.DB
Path string
logger *zap.Logger
db *bolt.DB
Path string
}
// New instantiates a new BoltDB client
func New(path string) (*Client, error) {
db, err := bolt.Open(path, 0600, &bolt.Options{Timeout: defaultTimeout})
func New(logger *zap.Logger, path string) (*Client, error) {
db, err := bolt.Open(path, fileMode, &bolt.Options{Timeout: defaultTimeout})
if err != nil {
return nil, err
}
return &Client{
db: db,
Path: path,
logger: logger,
db: db,
Path: path,
}, nil
}
// Close closes a BoltDB client
func (c *Client) Close() error {
return c.db.Close()
}

43
storage/boltdb/net-state.go → storage/boltdb/netstate.go
View File

@ -4,27 +4,22 @@
package boltdb
import (
"encoding/json"
"log"
"github.com/boltdb/bolt"
)
// File Path and Value are saved to boltdb
type File struct {
Path string `json:"path"`
Value string `json:"value"`
Value []byte `json:"value"`
}
const (
fileBucketName = "files"
)
var (
errFileNotFound = Error.New("error file not found")
)
// Put saves the file path and value as a kv pair in the "files" bucket
func (client *Client) Put(file File) error {
client.logger.Debug("entering Client.Put(File)")
return client.db.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(fileBucketName))
if err != nil {
@ -32,33 +27,31 @@ func (client *Client) Put(file File) error {
}
fileKey := []byte(file.Path)
fileBytes, err := json.Marshal(file.Value)
if err != nil {
log.Println(err)
}
return b.Put(fileKey, fileBytes)
return b.Put(fileKey, file.Value)
})
}
// Get retrieves the value stored at the file path key
func (client *Client) Get(fileKey []byte) (File, error) {
client.logger.Debug("entering Client.Get(fileKey)")
var fileInfo File
err := client.db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(fileBucketName))
v := b.Get(fileKey)
if v == nil {
return errFileNotFound
return Error.New("file %#v not found", string(fileKey))
}
unmarshalErr := json.Unmarshal(v, &fileInfo.Value)
return unmarshalErr
fileInfo.Value = v
return nil
})
fileInfo.Path = string(fileKey)
return fileInfo, err
}
// List creates a string array of all keys in in the "files" bucket
func (client *Client) List() ([]string, error) {
client.logger.Debug("entering Client.List()")
var paths []string
err := client.db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(fileBucketName))
@ -67,20 +60,16 @@ func (client *Client) List() ([]string, error) {
paths = append(paths, string(key))
return nil
})
if err != nil {
return err
}
return nil
return err
})
return paths, err
}
// Delete deletes a kv pair from the "files" bucket, given the key
func (client *Client) Delete(fileKey []byte) error {
if err := client.db.Update(func(tx *bolt.Tx) error {
client.logger.Debug("entering Client.Delete(fileKey)")
return client.db.Update(func(tx *bolt.Tx) error {
return tx.Bucket([]byte(fileBucketName)).Delete(fileKey)
}); err != nil {
return err
}
return nil
})
}

12
storage/boltdb/net-state_test.go → storage/boltdb/netstate_test.go
View File

@ -4,10 +4,13 @@
package boltdb
import (
"bytes"
"io/ioutil"
"os"
"strings"
"testing"
"go.uber.org/zap"
)
func tempfile() string {
@ -18,7 +21,8 @@ func tempfile() string {
}
func TestNetState(t *testing.T) {
c, err := New(tempfile())
logger, _ := zap.NewDevelopment()
c, err := New(logger, tempfile())
if err != nil {
t.Error("Failed to create test db")
}
@ -29,12 +33,12 @@ func TestNetState(t *testing.T) {
testFile := File{
Path: `test/path`,
Value: `test value`,
Value: []byte(`test value`),
}
testFile2 := File{
Path: `test/path2`,
Value: `value2`,
Value: []byte(`value2`),
}
// tests Put function
@ -47,7 +51,7 @@ func TestNetState(t *testing.T) {
if err != nil {
t.Error("Failed to get saved test value")
}
if retrvFile.Value != testFile.Value {
if !bytes.Equal(retrvFile.Value, testFile.Value) {
t.Error("Retrieved file was not same as original file")
}

83
storage/boltdb/user.go
View File

@ -1,83 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package boltdb
import (
"encoding/json"
"errors"
"log"
"github.com/boltdb/bolt"
"github.com/google/uuid"
)
const (
userBucketName = "users"
)
var (
errCreatingUserBucket = errors.New("error creating user bucket")
)
type User struct {
Id uuid.UUID `json:"id"`
Email string `json:"email"`
Username string `json:"username"`
}
// CreateUser calls bolt database instance to create user
func (bdb *Client) CreateUser(user User) error {
return bdb.db.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(userBucketName))
if err != nil {
return errCreatingUserBucket
}
usernameKey := []byte(user.Username)
userBytes, err := json.Marshal(user)
if err != nil {
log.Println(err)
}
return b.Put(usernameKey, userBytes)
})
}
func (bdb *Client) GetUser(key []byte) (User, error) {
var userInfo User
err := bdb.db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(userBucketName))
v := b.Get(key)
if v == nil {
log.Println("user not found")
return nil
}
err1 := json.Unmarshal(v, &userInfo)
return err1
})
return userInfo, err
}
func (bdb *Client) UpdateUser(user User) error {
return bdb.db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(userBucketName))
usernameKey := []byte(user.Username)
userBytes, err := json.Marshal(user)
if err != nil {
log.Println(err)
}
return b.Put(usernameKey, userBytes)
})
}
func (bdb *Client) DeleteUser(key []byte) {
if err := bdb.db.Update(func(tx *bolt.Tx) error {
return tx.Bucket([]byte(userBucketName)).Delete(key)
}); err != nil {
log.Println(err)
}
}