JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Clean up Storage node setup (#1013)

Browse Source 
* Edit config on Setup

* Default to 1TiB storage space and 500GiB bandwidth

* Use human readable formats

* Use memory

* units of 1024 are measured with KiB/MiB etc

* pkg/cfgstruct: allow values to be configured with human readable sizes

Change-Id: Ic4e9ae461516d1d26fb81f6e44c5ac5cfccf777f

* Modify tests

* Removed comments

* More merge conflict stuff resolved

* Fix lint

* test fixin

Change-Id: I3a008206bf03a4446da19f642a2f9c1f9acaae36

* Remove commented code but secretly leave it in the histroy forever

* Move flag definition to struct
This commit is contained in:
Alexander Leitner 2019-01-14 16:19:15 -05:00 committed by GitHub
parent 87925789de
commit bfde515391
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 115 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
cmd/storagenode/main.go
View File

@ -33,6 +33,7 @@ import (
type StorageNode struct {
CA identity.CASetupConfig `setup:"true"`
Identity identity.SetupConfig `setup:"true"`
EditConf bool `default:"false" help:"open config in default editor"`
Server server.Config
Kademlia kademlia.StorageNodeConfig
@ -76,7 +77,7 @@ var (
defaultConfDir string
defaultDiagDir string
confDir *string
confDir string
)
const (
@ -91,7 +92,7 @@ func init() {
defaultConfDir = dirParam
}
confDir = rootCmd.PersistentFlags().String("config-dir", defaultConfDir, "main directory for storagenode configuration")
rootCmd.PersistentFlags().StringVar(&confDir, "config-dir", defaultConfDir, "main directory for storagenode configuration")
defaultDiagDir = filepath.Join(defaultConfDir, "storage")
rootCmd.AddCommand(runCmd)
@ -100,6 +101,7 @@ func init() {
rootCmd.AddCommand(diagCmd)
cfgstruct.Bind(runCmd.Flags(), &runCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.BindSetup(configCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(diagCmd.Flags(), &diagCfg, cfgstruct.ConfDir(defaultDiagDir))
}
@ -120,7 +122,7 @@ func cmdRun(cmd *cobra.Command, args []string) (err error) {
}
func cmdSetup(cmd *cobra.Command, args []string) (err error) {
setupDir, err := filepath.Abs(*confDir)
setupDir, err := filepath.Abs(confDir)
if err != nil {
return err
}
@ -157,28 +159,39 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
"identity.key-path": setupCfg.Identity.KeyPath,
"identity.server.address": defaultServerAddr,
"storage.path": filepath.Join(setupDir, "storage"),
"log.level": "info",
}
return process.SaveConfig(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), overrides)
configFile := filepath.Join(setupDir, "config.yaml")
err = process.SaveConfig(cmd.Flags(), configFile, overrides)
if err != nil {
return err
}
if setupCfg.EditConf {
return fpath.EditFile(configFile)
}
return err
}
func cmdConfig(cmd *cobra.Command, args []string) (err error) {
setupDir, err := filepath.Abs(*confDir)
setupDir, err := filepath.Abs(confDir)
if err != nil {
return err
}
//run setup if we can't access the config file
conf := filepath.Join(setupDir, "config.yaml")
if _, err := os.Stat(conf); err != nil {
if err = cmdSetup(cmd, args); err != nil {
return err
}
return cmdSetup(cmd, args)
}
return fpath.EditFile(conf)
}
func cmdDiag(cmd *cobra.Command, args []string) (err error) {
diagDir, err := filepath.Abs(*confDir)
diagDir, err := filepath.Abs(confDir)
if err != nil {
return err
}

8
examples/grpc-debug/main.go
View File

@ -5,9 +5,9 @@ package main
import (
"context"
"flag"
"fmt"
"github.com/spf13/pflag"
"google.golang.org/grpc"
"storj.io/storj/pkg/cfgstruct"
@ -16,18 +16,18 @@ import (
)
var (
targetAddr = flag.String("target", "satellite.staging.storj.io:7777", "address of target")
targetAddr = pflag.String("target", "satellite.staging.storj.io:7777", "address of target")
identityConfig provider.IdentityConfig
)
func init() {
cfgstruct.Bind(flag.CommandLine, &identityConfig, cfgstruct.ConfDir("$HOME/.storj/gw"))
cfgstruct.Bind(pflag.CommandLine, &identityConfig, cfgstruct.ConfDir("$HOME/.storj/gw"))
}
func main() {
ctx := context.Background()
flag.Parse()
pflag.Parse()
identity, err := identityConfig.Load()
if err != nil {
panic(err)

22
internal/memory/size.go
View File

@ -89,10 +89,11 @@ func (size *Size) Set(s string) error {
}
p := len(s)
if isLetter(s[len(s)-1]) {
p--
if len(s)-2 >= 0 && isLetter(s[len(s)-2]) {
for isLetter(s[p-1]) {
p--
if p < 0 {
return errors.New("p out of bounds")
}
}
@ -109,17 +110,17 @@ func (size *Size) Set(s string) error {
}
switch suffix {
case "EB":
case "EB", "EIB":
*size = Size(v * EB.Float64())
case "PB":
case "PB", "PIB":
*size = Size(v * PB.Float64())
case "TB":
case "TB", "TIB":
*size = Size(v * TB.Float64())
case "GB":
case "GB", "GIB":
*size = Size(v * GB.Float64())
case "MB":
case "MB", "MIB":
*size = Size(v * MB.Float64())
case "KB":
case "KB", "KIB":
*size = Size(v * KB.Float64())
case "B", "":
*size = Size(v)
@ -129,3 +130,6 @@ func (size *Size) Set(s string) error {
return nil
}
// Type implements pflag.Value
func (Size) Type() string { return "memory.Size" }

5
internal/memory/size_test.go
View File

@ -10,6 +10,8 @@ import (
)
const (
eb = 1 << 60
pb = 1 << 50
tb = 1 << 40
gb = 1 << 30
mb = 1 << 20
@ -67,6 +69,9 @@ func TestParse(t *testing.T) {
{1 * gb, "1.0 gB"},
{1 * mb, "1.0 Mb"},
{1 * kb, "1.0 kb"},
{1 * kb, "1.0kib"},
{1 * pb, "1.0pib"},
{1 * eb, "1.0eib"},
{1, "1.00"},
// without B suffix
{1 * tb, "1.00T"},

4
internal/testplanet/planet.go
View File

@ -293,8 +293,8 @@ func (planet *Planet) newStorageNodes(count int) ([]*storagenode.Peer, error) {
},
Storage: psserver.Config{
Path: "", // TODO: this argument won't be needed with master storagenodedb
AllocatedDiskSpace: memory.TB.Int64(),
AllocatedBandwidth: memory.TB.Int64(),
AllocatedDiskSpace: memory.TB,
AllocatedBandwidth: memory.TB,
KBucketRefreshInterval: time.Minute,
},
}

5
pkg/cfgstruct/bind.go
View File

@ -13,6 +13,8 @@ import (
"time"
"github.com/spf13/pflag"
"storj.io/storj/internal/memory"
)
// BindOpt is an option for the Bind method
@ -120,6 +122,9 @@ func bindConfig(flags FlagSet, prefix string, val reflect.Value, vars map[string
}
}
switch field.Type {
case reflect.TypeOf(memory.Size(0)):
check(fieldaddr.(*memory.Size).Set(def))
flags.Var(fieldaddr.(*memory.Size), flagname, help)
case reflect.TypeOf(int(0)):
val, err := strconv.ParseInt(def, 0, strconv.IntSize)
check(err)

9
pkg/cfgstruct/bind_test.go
View File

@ -4,12 +4,13 @@
package cfgstruct
import (
"flag"
"fmt"
"path/filepath"
"reflect"
"testing"
"time"
"github.com/spf13/pflag"
)
func assertEqual(actual, expected interface{}) {
@ -19,7 +20,7 @@ func assertEqual(actual, expected interface{}) {
}
func TestBind(t *testing.T) {
f := flag.NewFlagSet("test", flag.PanicOnError)
f := pflag.NewFlagSet("test", pflag.PanicOnError)
var c struct {
String string `default:""`
Bool bool `default:"false"`
@ -77,7 +78,7 @@ func TestBind(t *testing.T) {
}
func TestConfDir(t *testing.T) {
f := flag.NewFlagSet("test", flag.PanicOnError)
f := pflag.NewFlagSet("test", pflag.PanicOnError)
var c struct {
String string `default:"-$CONFDIR+"`
MyStruct1 struct {
@ -94,7 +95,7 @@ func TestConfDir(t *testing.T) {
}
func TestNesting(t *testing.T) {
f := flag.NewFlagSet("test", flag.PanicOnError)
f := pflag.NewFlagSet("test", pflag.PanicOnError)
var c struct {
String string `default:"-$CONFDIR+"`
MyStruct1 struct {

8
pkg/cfgstruct/flags.go
View File

@ -4,11 +4,12 @@
package cfgstruct
import (
"flag"
"time"
"github.com/spf13/pflag"
)
// FlagSet is an interface that matches both *flag.FlagSet and *pflag.FlagSet
// FlagSet is an interface that matches *pflag.FlagSet
type FlagSet interface {
BoolVar(p *bool, name string, value bool, usage string)
IntVar(p *int, name string, value int, usage string)
@ -18,6 +19,7 @@ type FlagSet interface {
DurationVar(p *time.Duration, name string, value time.Duration, usage string)
Float64Var(p *float64, name string, value float64, usage string)
StringVar(p *string, name string, value string, usage string)
Var(val pflag.Value, name string, usage string)
}
var _ FlagSet = (*flag.FlagSet)(nil)
var _ FlagSet = (*pflag.FlagSet)(nil)

5
pkg/datarepair/repairer/config.go
View File

@ -9,6 +9,7 @@ import (
"go.uber.org/zap"
"storj.io/storj/internal/memory"
"storj.io/storj/pkg/datarepair/queue"
"storj.io/storj/pkg/overlay"
"storj.io/storj/pkg/pointerdb/pdbclient"
@ -23,7 +24,7 @@ type Config struct {
Interval time.Duration `help:"how frequently checker should audit segments" default:"3600s"`
OverlayAddr string `help:"Address to contact overlay server through"`
PointerDBAddr string `help:"Address to contact pointerdb server through"`
MaxBufferMem int `help:"maximum buffer memory (in bytes) to be allocated for read buffers" default:"0x400000"`
MaxBufferMem memory.Size `help:"maximum buffer memory (in bytes) to be allocated for read buffers" default:"4M"`
APIKey string `help:"repairer-specific pointerdb access credential"`
}
@ -71,7 +72,7 @@ func (c Config) getSegmentRepairer(ctx context.Context, identity *provider.FullI
return nil, err
}
ec := ecclient.NewClient(identity, c.MaxBufferMem)
ec := ecclient.NewClient(identity, c.MaxBufferMem.Int())
return segments.NewSegmentRepairer(oc, ec, pdb), nil
}

21
pkg/miniogw/config.go
View File

@ -14,6 +14,7 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap"
"storj.io/storj/internal/memory"
"storj.io/storj/pkg/eestream"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/metainfo/kvmetainfo"
@ -30,8 +31,8 @@ import (
// RSConfig is a configuration struct that keeps details about default
// redundancy strategy information
type RSConfig struct {
MaxBufferMem int `help:"maximum buffer memory (in bytes) to be allocated for read buffers" default:"0x400000"`
ErasureShareSize int `help:"the size of each new erasure sure in bytes" default:"1024"`
MaxBufferMem memory.Size `help:"maximum buffer memory (in bytes) to be allocated for read buffers" default:"4M"`
ErasureShareSize memory.Size `help:"the size of each new erasure sure in bytes" default:"1K"`
MinThreshold int `help:"the minimum pieces required to recover a segment. k." default:"29"`
RepairThreshold int `help:"the minimum safe pieces before a repair is triggered. m." default:"35"`
SuccessThreshold int `help:"the desired total pieces for a segment. o." default:"80"`
@ -42,7 +43,7 @@ type RSConfig struct {
// encrypting segments
type EncryptionConfig struct {
Key string `help:"root key for encrypting the data"`
BlockSize int `help:"size (in bytes) of encrypted blocks" default:"1024"`
BlockSize memory.Size `help:"size (in bytes) of encrypted blocks" default:"1K"`
DataType int `help:"Type of encryption to use for content and metadata (1=AES-GCM, 2=SecretBox)" default:"1"`
PathType int `help:"Type of encryption to use for paths (0=Unencrypted, 1=AES-GCM, 2=SecretBox)" default:"1"`
}
@ -63,8 +64,8 @@ type ClientConfig struct {
PointerDBAddr string `help:"Address to contact pointerdb server through"`
APIKey string `help:"API Key (TODO: this needs to change to macaroons somehow)"`
MaxInlineSize int `help:"max inline segment size in bytes" default:"4096"`
SegmentSize int64 `help:"the size of a segment in bytes" default:"64000000"`
MaxInlineSize memory.Size `help:"max inline segment size in bytes" default:"4K"`
SegmentSize memory.Size `help:"the size of a segment in bytes" default:"64M"`
}
// ServerConfig determines how minio listens for requests
@ -156,19 +157,19 @@ func (c Config) GetMetainfo(ctx context.Context, identity *provider.FullIdentity
return nil, nil, Error.New("failed to connect to pointer DB: %v", err)
}
ec := ecclient.NewClient(identity, c.RS.MaxBufferMem)
ec := ecclient.NewClient(identity, c.RS.MaxBufferMem.Int())
fc, err := infectious.NewFEC(c.RS.MinThreshold, c.RS.MaxThreshold)
if err != nil {
return nil, nil, Error.New("failed to create erasure coding client: %v", err)
}
rs, err := eestream.NewRedundancyStrategy(eestream.NewRSScheme(fc, c.RS.ErasureShareSize), c.RS.RepairThreshold, c.RS.SuccessThreshold)
rs, err := eestream.NewRedundancyStrategy(eestream.NewRSScheme(fc, c.RS.ErasureShareSize.Int()), c.RS.RepairThreshold, c.RS.SuccessThreshold)
if err != nil {
return nil, nil, Error.New("failed to create redundancy strategy: %v", err)
}
segments := segments.NewSegmentStore(oc, ec, pdb, rs, c.Client.MaxInlineSize)
segments := segments.NewSegmentStore(oc, ec, pdb, rs, c.Client.MaxInlineSize.Int())
if c.RS.ErasureShareSize*c.RS.MinThreshold%c.Enc.BlockSize != 0 {
if c.RS.ErasureShareSize.Int()*c.RS.MinThreshold%c.Enc.BlockSize.Int() != 0 {
err = Error.New("EncryptionBlockSize must be a multiple of ErasureShareSize * RS MinThreshold")
return nil, nil, err
}
@ -176,7 +177,7 @@ func (c Config) GetMetainfo(ctx context.Context, identity *provider.FullIdentity
key := new(storj.Key)
copy(key[:], c.Enc.Key)
streams, err := streams.NewStreamStore(segments, c.Client.SegmentSize, key, c.Enc.BlockSize, storj.Cipher(c.Enc.DataType))
streams, err := streams.NewStreamStore(segments, c.Client.SegmentSize.Int64(), key, c.Enc.BlockSize.Int(), storj.Cipher(c.Enc.DataType))
if err != nil {
return nil, nil, Error.New("failed to create stream store: %v", err)
}

3
pkg/miniogw/integration_test.go
View File

@ -13,6 +13,7 @@ import (
"github.com/minio/cli"
minio "github.com/minio/minio/cmd"
"github.com/spf13/pflag"
"github.com/stretchr/testify/assert"
"go.uber.org/zap"
"go.uber.org/zap/zaptest"
@ -42,7 +43,7 @@ func TestUploadDownload(t *testing.T) {
// bind default values to config
var gwCfg miniogw.Config
cfgstruct.Bind(&flag.FlagSet{}, &gwCfg)
cfgstruct.Bind(&pflag.FlagSet{}, &gwCfg)
// minio config directory
gwCfg.Minio.Dir = ctx.Dir("minio")

26
pkg/piecestore/psclient/client.go
View File

@ -18,6 +18,7 @@ import (
"go.uber.org/zap"
"golang.org/x/net/context"
"storj.io/storj/internal/memory"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/ranger"
"storj.io/storj/pkg/storj"
@ -28,14 +29,19 @@ import (
var ClientError = errs.Class("piecestore client error")
var (
defaultBandwidthMsgSize = flag.Int(
"piecestore.rpc.client.default-bandwidth-msg-size", 32*1024,
"default bandwidth message size in bytes")
maxBandwidthMsgSize = flag.Int(
"piecestore.rpc.client.max-bandwidth-msg-size", 64*1024,
"max bandwidth message size in bytes")
defaultBandwidthMsgSize memory.Size = 32 * memory.KB
maxBandwidthMsgSize memory.Size = 64 * memory.KB
)
func init() {
flag.Var(&defaultBandwidthMsgSize,
"piecestore.rpc.client.default-bandwidth-msg-size",
"default bandwidth message size in bytes")
flag.Var(&maxBandwidthMsgSize,
"piecestore.rpc.client.max-bandwidth-msg-size",
"max bandwidth message size in bytes")
}
// Client is an interface describing the functions for interacting with piecestore nodes
type Client interface {
Meta(ctx context.Context, id PieceID) (*pb.PieceSummary, error)
@ -63,12 +69,12 @@ func NewPSClient(ctx context.Context, tc transport.Client, n *pb.Node, bandwidth
return nil, err
}
if bandwidthMsgSize < 0 || bandwidthMsgSize > *maxBandwidthMsgSize {
if bandwidthMsgSize < 0 || bandwidthMsgSize > maxBandwidthMsgSize.Int() {
return nil, ClientError.New("invalid Bandwidth Message Size: %v", bandwidthMsgSize)
}
if bandwidthMsgSize == 0 {
bandwidthMsgSize = *defaultBandwidthMsgSize
bandwidthMsgSize = defaultBandwidthMsgSize.Int()
}
return &PieceStore{
@ -83,12 +89,12 @@ func NewPSClient(ctx context.Context, tc transport.Client, n *pb.Node, bandwidth
// NewCustomRoute creates new PieceStore with custom client interface
func NewCustomRoute(client pb.PieceStoreRoutesClient, target *pb.Node, bandwidthMsgSize int, prikey crypto.PrivateKey) (*PieceStore, error) {
target.Type.DPanicOnInvalid("new custom route")
if bandwidthMsgSize < 0 || bandwidthMsgSize > *maxBandwidthMsgSize {
if bandwidthMsgSize < 0 || bandwidthMsgSize > maxBandwidthMsgSize.Int() {
return nil, ClientError.New("invalid Bandwidth Message Size: %v", bandwidthMsgSize)
}
if bandwidthMsgSize == 0 {
bandwidthMsgSize = *defaultBandwidthMsgSize
bandwidthMsgSize = defaultBandwidthMsgSize.Int()
}
return &PieceStore{

5
pkg/piecestore/psserver/config.go
View File

@ -13,6 +13,7 @@ import (
"golang.org/x/net/context"
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/internal/memory"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/pb"
pstore "storj.io/storj/pkg/piecestore"
@ -28,8 +29,8 @@ var (
// Config contains everything necessary for a server
type Config struct {
Path string `help:"path to store data in" default:"$CONFDIR"`
AllocatedDiskSpace int64 `help:"total allocated disk space in bytes, default(1GB)" default:"1073741824"`
AllocatedBandwidth int64 `help:"total allocated bandwidth in bytes, default(100GB)" default:"107374182400"`
AllocatedDiskSpace memory.Size `help:"total allocated disk space in bytes" default:"1TiB"`
AllocatedBandwidth memory.Size `help:"total allocated bandwidth in bytes" default:"500GiB"`
KBucketRefreshInterval time.Duration `help:"how frequently Kademlia bucket should be refreshed with node stats" default:"1h0m0s"`
AgreementSenderCheckInterval time.Duration `help:"duration between agreement checks" default:"1h0m0s"`
}

8
pkg/piecestore/psserver/server.go
View File

@ -68,8 +68,8 @@ type Server struct {
// NewEndpoint -- initializes a new endpoint for a piecestore server
func NewEndpoint(log *zap.Logger, config Config, storage *pstore.Storage, db *psdb.DB, pkey crypto.PrivateKey) (*Server, error) {
// read the allocated disk space from the config file
allocatedDiskSpace := config.AllocatedDiskSpace
allocatedBandwidth := config.AllocatedBandwidth
allocatedDiskSpace := config.AllocatedDiskSpace.Int64()
allocatedBandwidth := config.AllocatedBandwidth.Int64()
// get the disk space details
// The returned path ends in a slash only if it represents a root directory, such as "/" on Unix or `C:\` on Windows.
@ -137,8 +137,8 @@ func New(log *zap.Logger, storage *pstore.Storage, db *psdb.DB, config Config, p
storage: storage,
DB: db,
pkey: pkey,
totalAllocated: config.AllocatedDiskSpace,
totalBwAllocated: config.AllocatedBandwidth,
totalAllocated: config.AllocatedDiskSpace.Int64(),
totalBwAllocated: config.AllocatedBandwidth.Int64(),
verifier: auth.NewSignedMessageVerifier(),
}
}

5
pkg/pointerdb/config.go
View File

@ -8,6 +8,7 @@ import (
"go.uber.org/zap"
"storj.io/storj/internal/memory"
"storj.io/storj/pkg/overlay"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/provider"
@ -31,8 +32,8 @@ const (
// PointerDB responsibility
type Config struct {
DatabaseURL string `help:"the database connection string to use" default:"bolt://$CONFDIR/pointerdb.db"`
MinRemoteSegmentSize int `default:"1240" help:"minimum remote segment size"`
MaxInlineSegmentSize int `default:"8000" help:"maximum inline segment size"`
MinRemoteSegmentSize memory.Size `default:"1240" help:"minimum remote segment size"`
MaxInlineSegmentSize memory.Size `default:"8000" help:"maximum inline segment size"`
Overlay bool `default:"true" help:"toggle flag if overlay is enabled"`
BwExpiration int `default:"45" help:"lifespan of bandwidth agreements in days"`
}

2
pkg/pointerdb/pointerdb.go
View File

@ -71,7 +71,7 @@ func (s *Server) validateSegment(req *pb.PutRequest) error {
return segmentError.New("remote segment size %d less than minimum allowed %d", remoteSize, min)
}
max := s.config.MaxInlineSegmentSize
max := s.config.MaxInlineSegmentSize.Int()
inlineSize := len(req.GetPointer().InlineSegment)
if inlineSize > max {