Clean up Storage node setup (#1013)

* Edit config on Setup

* Default to 1TiB storage space and 500GiB bandwidth

* Use human readable formats

* Use memory

* units of 1024 are measured with KiB/MiB etc

* pkg/cfgstruct: allow values to be configured with human readable sizes

Change-Id: Ic4e9ae461516d1d26fb81f6e44c5ac5cfccf777f

* Modify tests

* Removed comments

* More merge conflict stuff resolved

* Fix lint

* test fixin

Change-Id: I3a008206bf03a4446da19f642a2f9c1f9acaae36

* Remove commented code but secretly leave it in the histroy forever

* Move flag definition to struct
This commit is contained in:
Alexander Leitner 2019-01-14 16:19:15 -05:00 committed by GitHub
parent 87925789de
commit bfde515391
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 115 additions and 74 deletions

View File

@ -33,6 +33,7 @@ import (
type StorageNode struct { type StorageNode struct {
CA identity.CASetupConfig `setup:"true"` CA identity.CASetupConfig `setup:"true"`
Identity identity.SetupConfig `setup:"true"` Identity identity.SetupConfig `setup:"true"`
EditConf bool `default:"false" help:"open config in default editor"`
Server server.Config Server server.Config
Kademlia kademlia.StorageNodeConfig Kademlia kademlia.StorageNodeConfig
@ -76,7 +77,7 @@ var (
defaultConfDir string defaultConfDir string
defaultDiagDir string defaultDiagDir string
confDir *string confDir string
) )
const ( const (
@ -91,7 +92,7 @@ func init() {
defaultConfDir = dirParam defaultConfDir = dirParam
} }
confDir = rootCmd.PersistentFlags().String("config-dir", defaultConfDir, "main directory for storagenode configuration") rootCmd.PersistentFlags().StringVar(&confDir, "config-dir", defaultConfDir, "main directory for storagenode configuration")
defaultDiagDir = filepath.Join(defaultConfDir, "storage") defaultDiagDir = filepath.Join(defaultConfDir, "storage")
rootCmd.AddCommand(runCmd) rootCmd.AddCommand(runCmd)
@ -100,6 +101,7 @@ func init() {
rootCmd.AddCommand(diagCmd) rootCmd.AddCommand(diagCmd)
cfgstruct.Bind(runCmd.Flags(), &runCfg, cfgstruct.ConfDir(defaultConfDir)) cfgstruct.Bind(runCmd.Flags(), &runCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir)) cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.BindSetup(configCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(diagCmd.Flags(), &diagCfg, cfgstruct.ConfDir(defaultDiagDir)) cfgstruct.Bind(diagCmd.Flags(), &diagCfg, cfgstruct.ConfDir(defaultDiagDir))
} }
@ -120,7 +122,7 @@ func cmdRun(cmd *cobra.Command, args []string) (err error) {
} }
func cmdSetup(cmd *cobra.Command, args []string) (err error) { func cmdSetup(cmd *cobra.Command, args []string) (err error) {
setupDir, err := filepath.Abs(*confDir) setupDir, err := filepath.Abs(confDir)
if err != nil { if err != nil {
return err return err
} }
@ -157,28 +159,39 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
"identity.key-path": setupCfg.Identity.KeyPath, "identity.key-path": setupCfg.Identity.KeyPath,
"identity.server.address": defaultServerAddr, "identity.server.address": defaultServerAddr,
"storage.path": filepath.Join(setupDir, "storage"), "storage.path": filepath.Join(setupDir, "storage"),
"log.level": "info",
} }
return process.SaveConfig(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), overrides) configFile := filepath.Join(setupDir, "config.yaml")
err = process.SaveConfig(cmd.Flags(), configFile, overrides)
if err != nil {
return err
}
if setupCfg.EditConf {
return fpath.EditFile(configFile)
}
return err
} }
func cmdConfig(cmd *cobra.Command, args []string) (err error) { func cmdConfig(cmd *cobra.Command, args []string) (err error) {
setupDir, err := filepath.Abs(*confDir) setupDir, err := filepath.Abs(confDir)
if err != nil { if err != nil {
return err return err
} }
//run setup if we can't access the config file //run setup if we can't access the config file
conf := filepath.Join(setupDir, "config.yaml") conf := filepath.Join(setupDir, "config.yaml")
if _, err := os.Stat(conf); err != nil { if _, err := os.Stat(conf); err != nil {
if err = cmdSetup(cmd, args); err != nil { return cmdSetup(cmd, args)
return err
}
} }
return fpath.EditFile(conf) return fpath.EditFile(conf)
} }
func cmdDiag(cmd *cobra.Command, args []string) (err error) { func cmdDiag(cmd *cobra.Command, args []string) (err error) {
diagDir, err := filepath.Abs(*confDir) diagDir, err := filepath.Abs(confDir)
if err != nil { if err != nil {
return err return err
} }

View File

@ -5,9 +5,9 @@ package main
import ( import (
"context" "context"
"flag"
"fmt" "fmt"
"github.com/spf13/pflag"
"google.golang.org/grpc" "google.golang.org/grpc"
"storj.io/storj/pkg/cfgstruct" "storj.io/storj/pkg/cfgstruct"
@ -16,18 +16,18 @@ import (
) )
var ( var (
targetAddr = flag.String("target", "satellite.staging.storj.io:7777", "address of target") targetAddr = pflag.String("target", "satellite.staging.storj.io:7777", "address of target")
identityConfig provider.IdentityConfig identityConfig provider.IdentityConfig
) )
func init() { func init() {
cfgstruct.Bind(flag.CommandLine, &identityConfig, cfgstruct.ConfDir("$HOME/.storj/gw")) cfgstruct.Bind(pflag.CommandLine, &identityConfig, cfgstruct.ConfDir("$HOME/.storj/gw"))
} }
func main() { func main() {
ctx := context.Background() ctx := context.Background()
flag.Parse() pflag.Parse()
identity, err := identityConfig.Load() identity, err := identityConfig.Load()
if err != nil { if err != nil {
panic(err) panic(err)

View File

@ -89,10 +89,11 @@ func (size *Size) Set(s string) error {
} }
p := len(s) p := len(s)
if isLetter(s[len(s)-1]) { for isLetter(s[p-1]) {
p--
if len(s)-2 >= 0 && isLetter(s[len(s)-2]) {
p-- p--
if p < 0 {
return errors.New("p out of bounds")
} }
} }
@ -109,17 +110,17 @@ func (size *Size) Set(s string) error {
} }
switch suffix { switch suffix {
case "EB": case "EB", "EIB":
*size = Size(v * EB.Float64()) *size = Size(v * EB.Float64())
case "PB": case "PB", "PIB":
*size = Size(v * PB.Float64()) *size = Size(v * PB.Float64())
case "TB": case "TB", "TIB":
*size = Size(v * TB.Float64()) *size = Size(v * TB.Float64())
case "GB": case "GB", "GIB":
*size = Size(v * GB.Float64()) *size = Size(v * GB.Float64())
case "MB": case "MB", "MIB":
*size = Size(v * MB.Float64()) *size = Size(v * MB.Float64())
case "KB": case "KB", "KIB":
*size = Size(v * KB.Float64()) *size = Size(v * KB.Float64())
case "B", "": case "B", "":
*size = Size(v) *size = Size(v)
@ -129,3 +130,6 @@ func (size *Size) Set(s string) error {
return nil return nil
} }
// Type implements pflag.Value
func (Size) Type() string { return "memory.Size" }

View File

@ -10,6 +10,8 @@ import (
) )
const ( const (
eb = 1 << 60
pb = 1 << 50
tb = 1 << 40 tb = 1 << 40
gb = 1 << 30 gb = 1 << 30
mb = 1 << 20 mb = 1 << 20
@ -67,6 +69,9 @@ func TestParse(t *testing.T) {
{1 * gb, "1.0 gB"}, {1 * gb, "1.0 gB"},
{1 * mb, "1.0 Mb"}, {1 * mb, "1.0 Mb"},
{1 * kb, "1.0 kb"}, {1 * kb, "1.0 kb"},
{1 * kb, "1.0kib"},
{1 * pb, "1.0pib"},
{1 * eb, "1.0eib"},
{1, "1.00"}, {1, "1.00"},
// without B suffix // without B suffix
{1 * tb, "1.00T"}, {1 * tb, "1.00T"},

View File

@ -293,8 +293,8 @@ func (planet *Planet) newStorageNodes(count int) ([]*storagenode.Peer, error) {
}, },
Storage: psserver.Config{ Storage: psserver.Config{
Path: "", // TODO: this argument won't be needed with master storagenodedb Path: "", // TODO: this argument won't be needed with master storagenodedb
AllocatedDiskSpace: memory.TB.Int64(), AllocatedDiskSpace: memory.TB,
AllocatedBandwidth: memory.TB.Int64(), AllocatedBandwidth: memory.TB,
KBucketRefreshInterval: time.Minute, KBucketRefreshInterval: time.Minute,
}, },
} }

View File

@ -13,6 +13,8 @@ import (
"time" "time"
"github.com/spf13/pflag" "github.com/spf13/pflag"
"storj.io/storj/internal/memory"
) )
// BindOpt is an option for the Bind method // BindOpt is an option for the Bind method
@ -120,6 +122,9 @@ func bindConfig(flags FlagSet, prefix string, val reflect.Value, vars map[string
} }
} }
switch field.Type { switch field.Type {
case reflect.TypeOf(memory.Size(0)):
check(fieldaddr.(*memory.Size).Set(def))
flags.Var(fieldaddr.(*memory.Size), flagname, help)
case reflect.TypeOf(int(0)): case reflect.TypeOf(int(0)):
val, err := strconv.ParseInt(def, 0, strconv.IntSize) val, err := strconv.ParseInt(def, 0, strconv.IntSize)
check(err) check(err)

View File

@ -4,12 +4,13 @@
package cfgstruct package cfgstruct
import ( import (
"flag"
"fmt" "fmt"
"path/filepath" "path/filepath"
"reflect" "reflect"
"testing" "testing"
"time" "time"
"github.com/spf13/pflag"
) )
func assertEqual(actual, expected interface{}) { func assertEqual(actual, expected interface{}) {
@ -19,7 +20,7 @@ func assertEqual(actual, expected interface{}) {
} }
func TestBind(t *testing.T) { func TestBind(t *testing.T) {
f := flag.NewFlagSet("test", flag.PanicOnError) f := pflag.NewFlagSet("test", pflag.PanicOnError)
var c struct { var c struct {
String string `default:""` String string `default:""`
Bool bool `default:"false"` Bool bool `default:"false"`
@ -77,7 +78,7 @@ func TestBind(t *testing.T) {
} }
func TestConfDir(t *testing.T) { func TestConfDir(t *testing.T) {
f := flag.NewFlagSet("test", flag.PanicOnError) f := pflag.NewFlagSet("test", pflag.PanicOnError)
var c struct { var c struct {
String string `default:"-$CONFDIR+"` String string `default:"-$CONFDIR+"`
MyStruct1 struct { MyStruct1 struct {
@ -94,7 +95,7 @@ func TestConfDir(t *testing.T) {
} }
func TestNesting(t *testing.T) { func TestNesting(t *testing.T) {
f := flag.NewFlagSet("test", flag.PanicOnError) f := pflag.NewFlagSet("test", pflag.PanicOnError)
var c struct { var c struct {
String string `default:"-$CONFDIR+"` String string `default:"-$CONFDIR+"`
MyStruct1 struct { MyStruct1 struct {

View File

@ -4,11 +4,12 @@
package cfgstruct package cfgstruct
import ( import (
"flag"
"time" "time"
"github.com/spf13/pflag"
) )
// FlagSet is an interface that matches both *flag.FlagSet and *pflag.FlagSet // FlagSet is an interface that matches *pflag.FlagSet
type FlagSet interface { type FlagSet interface {
BoolVar(p *bool, name string, value bool, usage string) BoolVar(p *bool, name string, value bool, usage string)
IntVar(p *int, name string, value int, usage string) IntVar(p *int, name string, value int, usage string)
@ -18,6 +19,7 @@ type FlagSet interface {
DurationVar(p *time.Duration, name string, value time.Duration, usage string) DurationVar(p *time.Duration, name string, value time.Duration, usage string)
Float64Var(p *float64, name string, value float64, usage string) Float64Var(p *float64, name string, value float64, usage string)
StringVar(p *string, name string, value string, usage string) StringVar(p *string, name string, value string, usage string)
Var(val pflag.Value, name string, usage string)
} }
var _ FlagSet = (*flag.FlagSet)(nil) var _ FlagSet = (*pflag.FlagSet)(nil)

View File

@ -9,6 +9,7 @@ import (
"go.uber.org/zap" "go.uber.org/zap"
"storj.io/storj/internal/memory"
"storj.io/storj/pkg/datarepair/queue" "storj.io/storj/pkg/datarepair/queue"
"storj.io/storj/pkg/overlay" "storj.io/storj/pkg/overlay"
"storj.io/storj/pkg/pointerdb/pdbclient" "storj.io/storj/pkg/pointerdb/pdbclient"
@ -23,7 +24,7 @@ type Config struct {
Interval time.Duration `help:"how frequently checker should audit segments" default:"3600s"` Interval time.Duration `help:"how frequently checker should audit segments" default:"3600s"`
OverlayAddr string `help:"Address to contact overlay server through"` OverlayAddr string `help:"Address to contact overlay server through"`
PointerDBAddr string `help:"Address to contact pointerdb server through"` PointerDBAddr string `help:"Address to contact pointerdb server through"`
MaxBufferMem int `help:"maximum buffer memory (in bytes) to be allocated for read buffers" default:"0x400000"` MaxBufferMem memory.Size `help:"maximum buffer memory (in bytes) to be allocated for read buffers" default:"4M"`
APIKey string `help:"repairer-specific pointerdb access credential"` APIKey string `help:"repairer-specific pointerdb access credential"`
} }
@ -71,7 +72,7 @@ func (c Config) getSegmentRepairer(ctx context.Context, identity *provider.FullI
return nil, err return nil, err
} }
ec := ecclient.NewClient(identity, c.MaxBufferMem) ec := ecclient.NewClient(identity, c.MaxBufferMem.Int())
return segments.NewSegmentRepairer(oc, ec, pdb), nil return segments.NewSegmentRepairer(oc, ec, pdb), nil
} }

View File

@ -14,6 +14,7 @@ import (
"github.com/zeebo/errs" "github.com/zeebo/errs"
"go.uber.org/zap" "go.uber.org/zap"
"storj.io/storj/internal/memory"
"storj.io/storj/pkg/eestream" "storj.io/storj/pkg/eestream"
"storj.io/storj/pkg/identity" "storj.io/storj/pkg/identity"
"storj.io/storj/pkg/metainfo/kvmetainfo" "storj.io/storj/pkg/metainfo/kvmetainfo"
@ -30,8 +31,8 @@ import (
// RSConfig is a configuration struct that keeps details about default // RSConfig is a configuration struct that keeps details about default
// redundancy strategy information // redundancy strategy information
type RSConfig struct { type RSConfig struct {
MaxBufferMem int `help:"maximum buffer memory (in bytes) to be allocated for read buffers" default:"0x400000"` MaxBufferMem memory.Size `help:"maximum buffer memory (in bytes) to be allocated for read buffers" default:"4M"`
ErasureShareSize int `help:"the size of each new erasure sure in bytes" default:"1024"` ErasureShareSize memory.Size `help:"the size of each new erasure sure in bytes" default:"1K"`
MinThreshold int `help:"the minimum pieces required to recover a segment. k." default:"29"` MinThreshold int `help:"the minimum pieces required to recover a segment. k." default:"29"`
RepairThreshold int `help:"the minimum safe pieces before a repair is triggered. m." default:"35"` RepairThreshold int `help:"the minimum safe pieces before a repair is triggered. m." default:"35"`
SuccessThreshold int `help:"the desired total pieces for a segment. o." default:"80"` SuccessThreshold int `help:"the desired total pieces for a segment. o." default:"80"`
@ -42,7 +43,7 @@ type RSConfig struct {
// encrypting segments // encrypting segments
type EncryptionConfig struct { type EncryptionConfig struct {
Key string `help:"root key for encrypting the data"` Key string `help:"root key for encrypting the data"`
BlockSize int `help:"size (in bytes) of encrypted blocks" default:"1024"` BlockSize memory.Size `help:"size (in bytes) of encrypted blocks" default:"1K"`
DataType int `help:"Type of encryption to use for content and metadata (1=AES-GCM, 2=SecretBox)" default:"1"` DataType int `help:"Type of encryption to use for content and metadata (1=AES-GCM, 2=SecretBox)" default:"1"`
PathType int `help:"Type of encryption to use for paths (0=Unencrypted, 1=AES-GCM, 2=SecretBox)" default:"1"` PathType int `help:"Type of encryption to use for paths (0=Unencrypted, 1=AES-GCM, 2=SecretBox)" default:"1"`
} }
@ -63,8 +64,8 @@ type ClientConfig struct {
PointerDBAddr string `help:"Address to contact pointerdb server through"` PointerDBAddr string `help:"Address to contact pointerdb server through"`
APIKey string `help:"API Key (TODO: this needs to change to macaroons somehow)"` APIKey string `help:"API Key (TODO: this needs to change to macaroons somehow)"`
MaxInlineSize int `help:"max inline segment size in bytes" default:"4096"` MaxInlineSize memory.Size `help:"max inline segment size in bytes" default:"4K"`
SegmentSize int64 `help:"the size of a segment in bytes" default:"64000000"` SegmentSize memory.Size `help:"the size of a segment in bytes" default:"64M"`
} }
// ServerConfig determines how minio listens for requests // ServerConfig determines how minio listens for requests
@ -156,19 +157,19 @@ func (c Config) GetMetainfo(ctx context.Context, identity *provider.FullIdentity
return nil, nil, Error.New("failed to connect to pointer DB: %v", err) return nil, nil, Error.New("failed to connect to pointer DB: %v", err)
} }
ec := ecclient.NewClient(identity, c.RS.MaxBufferMem) ec := ecclient.NewClient(identity, c.RS.MaxBufferMem.Int())
fc, err := infectious.NewFEC(c.RS.MinThreshold, c.RS.MaxThreshold) fc, err := infectious.NewFEC(c.RS.MinThreshold, c.RS.MaxThreshold)
if err != nil { if err != nil {
return nil, nil, Error.New("failed to create erasure coding client: %v", err) return nil, nil, Error.New("failed to create erasure coding client: %v", err)
} }
rs, err := eestream.NewRedundancyStrategy(eestream.NewRSScheme(fc, c.RS.ErasureShareSize), c.RS.RepairThreshold, c.RS.SuccessThreshold) rs, err := eestream.NewRedundancyStrategy(eestream.NewRSScheme(fc, c.RS.ErasureShareSize.Int()), c.RS.RepairThreshold, c.RS.SuccessThreshold)
if err != nil { if err != nil {
return nil, nil, Error.New("failed to create redundancy strategy: %v", err) return nil, nil, Error.New("failed to create redundancy strategy: %v", err)
} }
segments := segments.NewSegmentStore(oc, ec, pdb, rs, c.Client.MaxInlineSize) segments := segments.NewSegmentStore(oc, ec, pdb, rs, c.Client.MaxInlineSize.Int())
if c.RS.ErasureShareSize*c.RS.MinThreshold%c.Enc.BlockSize != 0 { if c.RS.ErasureShareSize.Int()*c.RS.MinThreshold%c.Enc.BlockSize.Int() != 0 {
err = Error.New("EncryptionBlockSize must be a multiple of ErasureShareSize * RS MinThreshold") err = Error.New("EncryptionBlockSize must be a multiple of ErasureShareSize * RS MinThreshold")
return nil, nil, err return nil, nil, err
} }
@ -176,7 +177,7 @@ func (c Config) GetMetainfo(ctx context.Context, identity *provider.FullIdentity
key := new(storj.Key) key := new(storj.Key)
copy(key[:], c.Enc.Key) copy(key[:], c.Enc.Key)
streams, err := streams.NewStreamStore(segments, c.Client.SegmentSize, key, c.Enc.BlockSize, storj.Cipher(c.Enc.DataType)) streams, err := streams.NewStreamStore(segments, c.Client.SegmentSize.Int64(), key, c.Enc.BlockSize.Int(), storj.Cipher(c.Enc.DataType))
if err != nil { if err != nil {
return nil, nil, Error.New("failed to create stream store: %v", err) return nil, nil, Error.New("failed to create stream store: %v", err)
} }

View File

@ -13,6 +13,7 @@ import (
"github.com/minio/cli" "github.com/minio/cli"
minio "github.com/minio/minio/cmd" minio "github.com/minio/minio/cmd"
"github.com/spf13/pflag"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"go.uber.org/zap" "go.uber.org/zap"
"go.uber.org/zap/zaptest" "go.uber.org/zap/zaptest"
@ -42,7 +43,7 @@ func TestUploadDownload(t *testing.T) {
// bind default values to config // bind default values to config
var gwCfg miniogw.Config var gwCfg miniogw.Config
cfgstruct.Bind(&flag.FlagSet{}, &gwCfg) cfgstruct.Bind(&pflag.FlagSet{}, &gwCfg)
// minio config directory // minio config directory
gwCfg.Minio.Dir = ctx.Dir("minio") gwCfg.Minio.Dir = ctx.Dir("minio")

View File

@ -18,6 +18,7 @@ import (
"go.uber.org/zap" "go.uber.org/zap"
"golang.org/x/net/context" "golang.org/x/net/context"
"storj.io/storj/internal/memory"
"storj.io/storj/pkg/pb" "storj.io/storj/pkg/pb"
"storj.io/storj/pkg/ranger" "storj.io/storj/pkg/ranger"
"storj.io/storj/pkg/storj" "storj.io/storj/pkg/storj"
@ -28,14 +29,19 @@ import (
var ClientError = errs.Class("piecestore client error") var ClientError = errs.Class("piecestore client error")
var ( var (
defaultBandwidthMsgSize = flag.Int( defaultBandwidthMsgSize memory.Size = 32 * memory.KB
"piecestore.rpc.client.default-bandwidth-msg-size", 32*1024, maxBandwidthMsgSize memory.Size = 64 * memory.KB
"default bandwidth message size in bytes")
maxBandwidthMsgSize = flag.Int(
"piecestore.rpc.client.max-bandwidth-msg-size", 64*1024,
"max bandwidth message size in bytes")
) )
func init() {
flag.Var(&defaultBandwidthMsgSize,
"piecestore.rpc.client.default-bandwidth-msg-size",
"default bandwidth message size in bytes")
flag.Var(&maxBandwidthMsgSize,
"piecestore.rpc.client.max-bandwidth-msg-size",
"max bandwidth message size in bytes")
}
// Client is an interface describing the functions for interacting with piecestore nodes // Client is an interface describing the functions for interacting with piecestore nodes
type Client interface { type Client interface {
Meta(ctx context.Context, id PieceID) (*pb.PieceSummary, error) Meta(ctx context.Context, id PieceID) (*pb.PieceSummary, error)
@ -63,12 +69,12 @@ func NewPSClient(ctx context.Context, tc transport.Client, n *pb.Node, bandwidth
return nil, err return nil, err
} }
if bandwidthMsgSize < 0 || bandwidthMsgSize > *maxBandwidthMsgSize { if bandwidthMsgSize < 0 || bandwidthMsgSize > maxBandwidthMsgSize.Int() {
return nil, ClientError.New("invalid Bandwidth Message Size: %v", bandwidthMsgSize) return nil, ClientError.New("invalid Bandwidth Message Size: %v", bandwidthMsgSize)
} }
if bandwidthMsgSize == 0 { if bandwidthMsgSize == 0 {
bandwidthMsgSize = *defaultBandwidthMsgSize bandwidthMsgSize = defaultBandwidthMsgSize.Int()
} }
return &PieceStore{ return &PieceStore{
@ -83,12 +89,12 @@ func NewPSClient(ctx context.Context, tc transport.Client, n *pb.Node, bandwidth
// NewCustomRoute creates new PieceStore with custom client interface // NewCustomRoute creates new PieceStore with custom client interface
func NewCustomRoute(client pb.PieceStoreRoutesClient, target *pb.Node, bandwidthMsgSize int, prikey crypto.PrivateKey) (*PieceStore, error) { func NewCustomRoute(client pb.PieceStoreRoutesClient, target *pb.Node, bandwidthMsgSize int, prikey crypto.PrivateKey) (*PieceStore, error) {
target.Type.DPanicOnInvalid("new custom route") target.Type.DPanicOnInvalid("new custom route")
if bandwidthMsgSize < 0 || bandwidthMsgSize > *maxBandwidthMsgSize { if bandwidthMsgSize < 0 || bandwidthMsgSize > maxBandwidthMsgSize.Int() {
return nil, ClientError.New("invalid Bandwidth Message Size: %v", bandwidthMsgSize) return nil, ClientError.New("invalid Bandwidth Message Size: %v", bandwidthMsgSize)
} }
if bandwidthMsgSize == 0 { if bandwidthMsgSize == 0 {
bandwidthMsgSize = *defaultBandwidthMsgSize bandwidthMsgSize = defaultBandwidthMsgSize.Int()
} }
return &PieceStore{ return &PieceStore{

View File

@ -13,6 +13,7 @@ import (
"golang.org/x/net/context" "golang.org/x/net/context"
monkit "gopkg.in/spacemonkeygo/monkit.v2" monkit "gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/internal/memory"
"storj.io/storj/pkg/kademlia" "storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/pb" "storj.io/storj/pkg/pb"
pstore "storj.io/storj/pkg/piecestore" pstore "storj.io/storj/pkg/piecestore"
@ -28,8 +29,8 @@ var (
// Config contains everything necessary for a server // Config contains everything necessary for a server
type Config struct { type Config struct {
Path string `help:"path to store data in" default:"$CONFDIR"` Path string `help:"path to store data in" default:"$CONFDIR"`
AllocatedDiskSpace int64 `help:"total allocated disk space in bytes, default(1GB)" default:"1073741824"` AllocatedDiskSpace memory.Size `help:"total allocated disk space in bytes" default:"1TiB"`
AllocatedBandwidth int64 `help:"total allocated bandwidth in bytes, default(100GB)" default:"107374182400"` AllocatedBandwidth memory.Size `help:"total allocated bandwidth in bytes" default:"500GiB"`
KBucketRefreshInterval time.Duration `help:"how frequently Kademlia bucket should be refreshed with node stats" default:"1h0m0s"` KBucketRefreshInterval time.Duration `help:"how frequently Kademlia bucket should be refreshed with node stats" default:"1h0m0s"`
AgreementSenderCheckInterval time.Duration `help:"duration between agreement checks" default:"1h0m0s"` AgreementSenderCheckInterval time.Duration `help:"duration between agreement checks" default:"1h0m0s"`
} }

View File

@ -68,8 +68,8 @@ type Server struct {
// NewEndpoint -- initializes a new endpoint for a piecestore server // NewEndpoint -- initializes a new endpoint for a piecestore server
func NewEndpoint(log *zap.Logger, config Config, storage *pstore.Storage, db *psdb.DB, pkey crypto.PrivateKey) (*Server, error) { func NewEndpoint(log *zap.Logger, config Config, storage *pstore.Storage, db *psdb.DB, pkey crypto.PrivateKey) (*Server, error) {
// read the allocated disk space from the config file // read the allocated disk space from the config file
allocatedDiskSpace := config.AllocatedDiskSpace allocatedDiskSpace := config.AllocatedDiskSpace.Int64()
allocatedBandwidth := config.AllocatedBandwidth allocatedBandwidth := config.AllocatedBandwidth.Int64()
// get the disk space details // get the disk space details
// The returned path ends in a slash only if it represents a root directory, such as "/" on Unix or `C:\` on Windows. // The returned path ends in a slash only if it represents a root directory, such as "/" on Unix or `C:\` on Windows.
@ -137,8 +137,8 @@ func New(log *zap.Logger, storage *pstore.Storage, db *psdb.DB, config Config, p
storage: storage, storage: storage,
DB: db, DB: db,
pkey: pkey, pkey: pkey,
totalAllocated: config.AllocatedDiskSpace, totalAllocated: config.AllocatedDiskSpace.Int64(),
totalBwAllocated: config.AllocatedBandwidth, totalBwAllocated: config.AllocatedBandwidth.Int64(),
verifier: auth.NewSignedMessageVerifier(), verifier: auth.NewSignedMessageVerifier(),
} }
} }

View File

@ -8,6 +8,7 @@ import (
"go.uber.org/zap" "go.uber.org/zap"
"storj.io/storj/internal/memory"
"storj.io/storj/pkg/overlay" "storj.io/storj/pkg/overlay"
"storj.io/storj/pkg/pb" "storj.io/storj/pkg/pb"
"storj.io/storj/pkg/provider" "storj.io/storj/pkg/provider"
@ -31,8 +32,8 @@ const (
// PointerDB responsibility // PointerDB responsibility
type Config struct { type Config struct {
DatabaseURL string `help:"the database connection string to use" default:"bolt://$CONFDIR/pointerdb.db"` DatabaseURL string `help:"the database connection string to use" default:"bolt://$CONFDIR/pointerdb.db"`
MinRemoteSegmentSize int `default:"1240" help:"minimum remote segment size"` MinRemoteSegmentSize memory.Size `default:"1240" help:"minimum remote segment size"`
MaxInlineSegmentSize int `default:"8000" help:"maximum inline segment size"` MaxInlineSegmentSize memory.Size `default:"8000" help:"maximum inline segment size"`
Overlay bool `default:"true" help:"toggle flag if overlay is enabled"` Overlay bool `default:"true" help:"toggle flag if overlay is enabled"`
BwExpiration int `default:"45" help:"lifespan of bandwidth agreements in days"` BwExpiration int `default:"45" help:"lifespan of bandwidth agreements in days"`
} }

View File

@ -71,7 +71,7 @@ func (s *Server) validateSegment(req *pb.PutRequest) error {
return segmentError.New("remote segment size %d less than minimum allowed %d", remoteSize, min) return segmentError.New("remote segment size %d less than minimum allowed %d", remoteSize, min)
} }
max := s.config.MaxInlineSegmentSize max := s.config.MaxInlineSegmentSize.Int()
inlineSize := len(req.GetPointer().InlineSegment) inlineSize := len(req.GetPointer().InlineSegment)
if inlineSize > max { if inlineSize > max {