From bfde5153917e1eeed1bf21467dc0ce2faa8e8196 Mon Sep 17 00:00:00 2001 From: Alexander Leitner Date: Mon, 14 Jan 2019 16:19:15 -0500 Subject: [PATCH] Clean up Storage node setup (#1013) * Edit config on Setup * Default to 1TiB storage space and 500GiB bandwidth * Use human readable formats * Use memory * units of 1024 are measured with KiB/MiB etc * pkg/cfgstruct: allow values to be configured with human readable sizes Change-Id: Ic4e9ae461516d1d26fb81f6e44c5ac5cfccf777f * Modify tests * Removed comments * More merge conflict stuff resolved * Fix lint * test fixin Change-Id: I3a008206bf03a4446da19f642a2f9c1f9acaae36 * Remove commented code but secretly leave it in the histroy forever * Move flag definition to struct --- cmd/storagenode/main.go | 31 ++++++++++++++++++-------- examples/grpc-debug/main.go | 8 +++---- internal/memory/size.go | 22 ++++++++++-------- internal/memory/size_test.go | 5 +++++ internal/testplanet/planet.go | 4 ++-- pkg/cfgstruct/bind.go | 5 +++++ pkg/cfgstruct/bind_test.go | 9 ++++---- pkg/cfgstruct/flags.go | 8 ++++--- pkg/datarepair/repairer/config.go | 5 +++-- pkg/miniogw/config.go | 37 ++++++++++++++++--------------- pkg/miniogw/integration_test.go | 3 ++- pkg/piecestore/psclient/client.go | 26 +++++++++++++--------- pkg/piecestore/psserver/config.go | 5 +++-- pkg/piecestore/psserver/server.go | 8 +++---- pkg/pointerdb/config.go | 11 ++++----- pkg/pointerdb/pointerdb.go | 2 +- 16 files changed, 115 insertions(+), 74 deletions(-) diff --git a/cmd/storagenode/main.go b/cmd/storagenode/main.go index 4c18adce0..16bf42696 100644 --- a/cmd/storagenode/main.go +++ b/cmd/storagenode/main.go @@ -33,6 +33,7 @@ import ( type StorageNode struct { CA identity.CASetupConfig `setup:"true"` Identity identity.SetupConfig `setup:"true"` + EditConf bool `default:"false" help:"open config in default editor"` Server server.Config Kademlia kademlia.StorageNodeConfig @@ -76,7 +77,7 @@ var ( defaultConfDir string defaultDiagDir string - confDir *string + confDir string ) const ( @@ -91,7 +92,7 @@ func init() { defaultConfDir = dirParam } - confDir = rootCmd.PersistentFlags().String("config-dir", defaultConfDir, "main directory for storagenode configuration") + rootCmd.PersistentFlags().StringVar(&confDir, "config-dir", defaultConfDir, "main directory for storagenode configuration") defaultDiagDir = filepath.Join(defaultConfDir, "storage") rootCmd.AddCommand(runCmd) @@ -100,6 +101,7 @@ func init() { rootCmd.AddCommand(diagCmd) cfgstruct.Bind(runCmd.Flags(), &runCfg, cfgstruct.ConfDir(defaultConfDir)) cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir)) + cfgstruct.BindSetup(configCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir)) cfgstruct.Bind(diagCmd.Flags(), &diagCfg, cfgstruct.ConfDir(defaultDiagDir)) } @@ -120,7 +122,7 @@ func cmdRun(cmd *cobra.Command, args []string) (err error) { } func cmdSetup(cmd *cobra.Command, args []string) (err error) { - setupDir, err := filepath.Abs(*confDir) + setupDir, err := filepath.Abs(confDir) if err != nil { return err } @@ -157,28 +159,39 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) { "identity.key-path": setupCfg.Identity.KeyPath, "identity.server.address": defaultServerAddr, "storage.path": filepath.Join(setupDir, "storage"), + "log.level": "info", } - return process.SaveConfig(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), overrides) + configFile := filepath.Join(setupDir, "config.yaml") + + err = process.SaveConfig(cmd.Flags(), configFile, overrides) + if err != nil { + return err + } + + if setupCfg.EditConf { + return fpath.EditFile(configFile) + } + + return err } func cmdConfig(cmd *cobra.Command, args []string) (err error) { - setupDir, err := filepath.Abs(*confDir) + setupDir, err := filepath.Abs(confDir) if err != nil { return err } //run setup if we can't access the config file conf := filepath.Join(setupDir, "config.yaml") if _, err := os.Stat(conf); err != nil { - if err = cmdSetup(cmd, args); err != nil { - return err - } + return cmdSetup(cmd, args) } + return fpath.EditFile(conf) } func cmdDiag(cmd *cobra.Command, args []string) (err error) { - diagDir, err := filepath.Abs(*confDir) + diagDir, err := filepath.Abs(confDir) if err != nil { return err } diff --git a/examples/grpc-debug/main.go b/examples/grpc-debug/main.go index fe42dac94..7929251ad 100644 --- a/examples/grpc-debug/main.go +++ b/examples/grpc-debug/main.go @@ -5,9 +5,9 @@ package main import ( "context" - "flag" "fmt" + "github.com/spf13/pflag" "google.golang.org/grpc" "storj.io/storj/pkg/cfgstruct" @@ -16,18 +16,18 @@ import ( ) var ( - targetAddr = flag.String("target", "satellite.staging.storj.io:7777", "address of target") + targetAddr = pflag.String("target", "satellite.staging.storj.io:7777", "address of target") identityConfig provider.IdentityConfig ) func init() { - cfgstruct.Bind(flag.CommandLine, &identityConfig, cfgstruct.ConfDir("$HOME/.storj/gw")) + cfgstruct.Bind(pflag.CommandLine, &identityConfig, cfgstruct.ConfDir("$HOME/.storj/gw")) } func main() { ctx := context.Background() - flag.Parse() + pflag.Parse() identity, err := identityConfig.Load() if err != nil { panic(err) diff --git a/internal/memory/size.go b/internal/memory/size.go index 7753d84c2..efe6454f0 100644 --- a/internal/memory/size.go +++ b/internal/memory/size.go @@ -89,10 +89,11 @@ func (size *Size) Set(s string) error { } p := len(s) - if isLetter(s[len(s)-1]) { + for isLetter(s[p-1]) { p-- - if len(s)-2 >= 0 && isLetter(s[len(s)-2]) { - p-- + + if p < 0 { + return errors.New("p out of bounds") } } @@ -109,17 +110,17 @@ func (size *Size) Set(s string) error { } switch suffix { - case "EB": + case "EB", "EIB": *size = Size(v * EB.Float64()) - case "PB": + case "PB", "PIB": *size = Size(v * PB.Float64()) - case "TB": + case "TB", "TIB": *size = Size(v * TB.Float64()) - case "GB": + case "GB", "GIB": *size = Size(v * GB.Float64()) - case "MB": + case "MB", "MIB": *size = Size(v * MB.Float64()) - case "KB": + case "KB", "KIB": *size = Size(v * KB.Float64()) case "B", "": *size = Size(v) @@ -129,3 +130,6 @@ func (size *Size) Set(s string) error { return nil } + +// Type implements pflag.Value +func (Size) Type() string { return "memory.Size" } diff --git a/internal/memory/size_test.go b/internal/memory/size_test.go index ff5db032e..3baced274 100644 --- a/internal/memory/size_test.go +++ b/internal/memory/size_test.go @@ -10,6 +10,8 @@ import ( ) const ( + eb = 1 << 60 + pb = 1 << 50 tb = 1 << 40 gb = 1 << 30 mb = 1 << 20 @@ -67,6 +69,9 @@ func TestParse(t *testing.T) { {1 * gb, "1.0 gB"}, {1 * mb, "1.0 Mb"}, {1 * kb, "1.0 kb"}, + {1 * kb, "1.0kib"}, + {1 * pb, "1.0pib"}, + {1 * eb, "1.0eib"}, {1, "1.00"}, // without B suffix {1 * tb, "1.00T"}, diff --git a/internal/testplanet/planet.go b/internal/testplanet/planet.go index 3645e1eb3..0c2c4c602 100644 --- a/internal/testplanet/planet.go +++ b/internal/testplanet/planet.go @@ -293,8 +293,8 @@ func (planet *Planet) newStorageNodes(count int) ([]*storagenode.Peer, error) { }, Storage: psserver.Config{ Path: "", // TODO: this argument won't be needed with master storagenodedb - AllocatedDiskSpace: memory.TB.Int64(), - AllocatedBandwidth: memory.TB.Int64(), + AllocatedDiskSpace: memory.TB, + AllocatedBandwidth: memory.TB, KBucketRefreshInterval: time.Minute, }, } diff --git a/pkg/cfgstruct/bind.go b/pkg/cfgstruct/bind.go index 8b1be0833..365d35cbc 100644 --- a/pkg/cfgstruct/bind.go +++ b/pkg/cfgstruct/bind.go @@ -13,6 +13,8 @@ import ( "time" "github.com/spf13/pflag" + + "storj.io/storj/internal/memory" ) // BindOpt is an option for the Bind method @@ -120,6 +122,9 @@ func bindConfig(flags FlagSet, prefix string, val reflect.Value, vars map[string } } switch field.Type { + case reflect.TypeOf(memory.Size(0)): + check(fieldaddr.(*memory.Size).Set(def)) + flags.Var(fieldaddr.(*memory.Size), flagname, help) case reflect.TypeOf(int(0)): val, err := strconv.ParseInt(def, 0, strconv.IntSize) check(err) diff --git a/pkg/cfgstruct/bind_test.go b/pkg/cfgstruct/bind_test.go index e0d717cbf..56504031a 100644 --- a/pkg/cfgstruct/bind_test.go +++ b/pkg/cfgstruct/bind_test.go @@ -4,12 +4,13 @@ package cfgstruct import ( - "flag" "fmt" "path/filepath" "reflect" "testing" "time" + + "github.com/spf13/pflag" ) func assertEqual(actual, expected interface{}) { @@ -19,7 +20,7 @@ func assertEqual(actual, expected interface{}) { } func TestBind(t *testing.T) { - f := flag.NewFlagSet("test", flag.PanicOnError) + f := pflag.NewFlagSet("test", pflag.PanicOnError) var c struct { String string `default:""` Bool bool `default:"false"` @@ -77,7 +78,7 @@ func TestBind(t *testing.T) { } func TestConfDir(t *testing.T) { - f := flag.NewFlagSet("test", flag.PanicOnError) + f := pflag.NewFlagSet("test", pflag.PanicOnError) var c struct { String string `default:"-$CONFDIR+"` MyStruct1 struct { @@ -94,7 +95,7 @@ func TestConfDir(t *testing.T) { } func TestNesting(t *testing.T) { - f := flag.NewFlagSet("test", flag.PanicOnError) + f := pflag.NewFlagSet("test", pflag.PanicOnError) var c struct { String string `default:"-$CONFDIR+"` MyStruct1 struct { diff --git a/pkg/cfgstruct/flags.go b/pkg/cfgstruct/flags.go index 094867f8f..f09d75c36 100644 --- a/pkg/cfgstruct/flags.go +++ b/pkg/cfgstruct/flags.go @@ -4,11 +4,12 @@ package cfgstruct import ( - "flag" "time" + + "github.com/spf13/pflag" ) -// FlagSet is an interface that matches both *flag.FlagSet and *pflag.FlagSet +// FlagSet is an interface that matches *pflag.FlagSet type FlagSet interface { BoolVar(p *bool, name string, value bool, usage string) IntVar(p *int, name string, value int, usage string) @@ -18,6 +19,7 @@ type FlagSet interface { DurationVar(p *time.Duration, name string, value time.Duration, usage string) Float64Var(p *float64, name string, value float64, usage string) StringVar(p *string, name string, value string, usage string) + Var(val pflag.Value, name string, usage string) } -var _ FlagSet = (*flag.FlagSet)(nil) +var _ FlagSet = (*pflag.FlagSet)(nil) diff --git a/pkg/datarepair/repairer/config.go b/pkg/datarepair/repairer/config.go index bbc774c3e..fa2a076b2 100644 --- a/pkg/datarepair/repairer/config.go +++ b/pkg/datarepair/repairer/config.go @@ -9,6 +9,7 @@ import ( "go.uber.org/zap" + "storj.io/storj/internal/memory" "storj.io/storj/pkg/datarepair/queue" "storj.io/storj/pkg/overlay" "storj.io/storj/pkg/pointerdb/pdbclient" @@ -23,7 +24,7 @@ type Config struct { Interval time.Duration `help:"how frequently checker should audit segments" default:"3600s"` OverlayAddr string `help:"Address to contact overlay server through"` PointerDBAddr string `help:"Address to contact pointerdb server through"` - MaxBufferMem int `help:"maximum buffer memory (in bytes) to be allocated for read buffers" default:"0x400000"` + MaxBufferMem memory.Size `help:"maximum buffer memory (in bytes) to be allocated for read buffers" default:"4M"` APIKey string `help:"repairer-specific pointerdb access credential"` } @@ -71,7 +72,7 @@ func (c Config) getSegmentRepairer(ctx context.Context, identity *provider.FullI return nil, err } - ec := ecclient.NewClient(identity, c.MaxBufferMem) + ec := ecclient.NewClient(identity, c.MaxBufferMem.Int()) return segments.NewSegmentRepairer(oc, ec, pdb), nil } diff --git a/pkg/miniogw/config.go b/pkg/miniogw/config.go index 12c5c84b9..7cfcf879f 100644 --- a/pkg/miniogw/config.go +++ b/pkg/miniogw/config.go @@ -14,6 +14,7 @@ import ( "github.com/zeebo/errs" "go.uber.org/zap" + "storj.io/storj/internal/memory" "storj.io/storj/pkg/eestream" "storj.io/storj/pkg/identity" "storj.io/storj/pkg/metainfo/kvmetainfo" @@ -30,21 +31,21 @@ import ( // RSConfig is a configuration struct that keeps details about default // redundancy strategy information type RSConfig struct { - MaxBufferMem int `help:"maximum buffer memory (in bytes) to be allocated for read buffers" default:"0x400000"` - ErasureShareSize int `help:"the size of each new erasure sure in bytes" default:"1024"` - MinThreshold int `help:"the minimum pieces required to recover a segment. k." default:"29"` - RepairThreshold int `help:"the minimum safe pieces before a repair is triggered. m." default:"35"` - SuccessThreshold int `help:"the desired total pieces for a segment. o." default:"80"` - MaxThreshold int `help:"the largest amount of pieces to encode to. n." default:"95"` + MaxBufferMem memory.Size `help:"maximum buffer memory (in bytes) to be allocated for read buffers" default:"4M"` + ErasureShareSize memory.Size `help:"the size of each new erasure sure in bytes" default:"1K"` + MinThreshold int `help:"the minimum pieces required to recover a segment. k." default:"29"` + RepairThreshold int `help:"the minimum safe pieces before a repair is triggered. m." default:"35"` + SuccessThreshold int `help:"the desired total pieces for a segment. o." default:"80"` + MaxThreshold int `help:"the largest amount of pieces to encode to. n." default:"95"` } // EncryptionConfig is a configuration struct that keeps details about // encrypting segments type EncryptionConfig struct { - Key string `help:"root key for encrypting the data"` - BlockSize int `help:"size (in bytes) of encrypted blocks" default:"1024"` - DataType int `help:"Type of encryption to use for content and metadata (1=AES-GCM, 2=SecretBox)" default:"1"` - PathType int `help:"Type of encryption to use for paths (0=Unencrypted, 1=AES-GCM, 2=SecretBox)" default:"1"` + Key string `help:"root key for encrypting the data"` + BlockSize memory.Size `help:"size (in bytes) of encrypted blocks" default:"1K"` + DataType int `help:"Type of encryption to use for content and metadata (1=AES-GCM, 2=SecretBox)" default:"1"` + PathType int `help:"Type of encryption to use for paths (0=Unencrypted, 1=AES-GCM, 2=SecretBox)" default:"1"` } // MinioConfig is a configuration struct that keeps details about starting @@ -62,9 +63,9 @@ type ClientConfig struct { OverlayAddr string `help:"Address to contact overlay server through"` PointerDBAddr string `help:"Address to contact pointerdb server through"` - APIKey string `help:"API Key (TODO: this needs to change to macaroons somehow)"` - MaxInlineSize int `help:"max inline segment size in bytes" default:"4096"` - SegmentSize int64 `help:"the size of a segment in bytes" default:"64000000"` + APIKey string `help:"API Key (TODO: this needs to change to macaroons somehow)"` + MaxInlineSize memory.Size `help:"max inline segment size in bytes" default:"4K"` + SegmentSize memory.Size `help:"the size of a segment in bytes" default:"64M"` } // ServerConfig determines how minio listens for requests @@ -156,19 +157,19 @@ func (c Config) GetMetainfo(ctx context.Context, identity *provider.FullIdentity return nil, nil, Error.New("failed to connect to pointer DB: %v", err) } - ec := ecclient.NewClient(identity, c.RS.MaxBufferMem) + ec := ecclient.NewClient(identity, c.RS.MaxBufferMem.Int()) fc, err := infectious.NewFEC(c.RS.MinThreshold, c.RS.MaxThreshold) if err != nil { return nil, nil, Error.New("failed to create erasure coding client: %v", err) } - rs, err := eestream.NewRedundancyStrategy(eestream.NewRSScheme(fc, c.RS.ErasureShareSize), c.RS.RepairThreshold, c.RS.SuccessThreshold) + rs, err := eestream.NewRedundancyStrategy(eestream.NewRSScheme(fc, c.RS.ErasureShareSize.Int()), c.RS.RepairThreshold, c.RS.SuccessThreshold) if err != nil { return nil, nil, Error.New("failed to create redundancy strategy: %v", err) } - segments := segments.NewSegmentStore(oc, ec, pdb, rs, c.Client.MaxInlineSize) + segments := segments.NewSegmentStore(oc, ec, pdb, rs, c.Client.MaxInlineSize.Int()) - if c.RS.ErasureShareSize*c.RS.MinThreshold%c.Enc.BlockSize != 0 { + if c.RS.ErasureShareSize.Int()*c.RS.MinThreshold%c.Enc.BlockSize.Int() != 0 { err = Error.New("EncryptionBlockSize must be a multiple of ErasureShareSize * RS MinThreshold") return nil, nil, err } @@ -176,7 +177,7 @@ func (c Config) GetMetainfo(ctx context.Context, identity *provider.FullIdentity key := new(storj.Key) copy(key[:], c.Enc.Key) - streams, err := streams.NewStreamStore(segments, c.Client.SegmentSize, key, c.Enc.BlockSize, storj.Cipher(c.Enc.DataType)) + streams, err := streams.NewStreamStore(segments, c.Client.SegmentSize.Int64(), key, c.Enc.BlockSize.Int(), storj.Cipher(c.Enc.DataType)) if err != nil { return nil, nil, Error.New("failed to create stream store: %v", err) } diff --git a/pkg/miniogw/integration_test.go b/pkg/miniogw/integration_test.go index 2e0cf1f4d..faaff944e 100644 --- a/pkg/miniogw/integration_test.go +++ b/pkg/miniogw/integration_test.go @@ -13,6 +13,7 @@ import ( "github.com/minio/cli" minio "github.com/minio/minio/cmd" + "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "go.uber.org/zap" "go.uber.org/zap/zaptest" @@ -42,7 +43,7 @@ func TestUploadDownload(t *testing.T) { // bind default values to config var gwCfg miniogw.Config - cfgstruct.Bind(&flag.FlagSet{}, &gwCfg) + cfgstruct.Bind(&pflag.FlagSet{}, &gwCfg) // minio config directory gwCfg.Minio.Dir = ctx.Dir("minio") diff --git a/pkg/piecestore/psclient/client.go b/pkg/piecestore/psclient/client.go index 81b0ce32d..c9cf384de 100644 --- a/pkg/piecestore/psclient/client.go +++ b/pkg/piecestore/psclient/client.go @@ -18,6 +18,7 @@ import ( "go.uber.org/zap" "golang.org/x/net/context" + "storj.io/storj/internal/memory" "storj.io/storj/pkg/pb" "storj.io/storj/pkg/ranger" "storj.io/storj/pkg/storj" @@ -28,14 +29,19 @@ import ( var ClientError = errs.Class("piecestore client error") var ( - defaultBandwidthMsgSize = flag.Int( - "piecestore.rpc.client.default-bandwidth-msg-size", 32*1024, - "default bandwidth message size in bytes") - maxBandwidthMsgSize = flag.Int( - "piecestore.rpc.client.max-bandwidth-msg-size", 64*1024, - "max bandwidth message size in bytes") + defaultBandwidthMsgSize memory.Size = 32 * memory.KB + maxBandwidthMsgSize memory.Size = 64 * memory.KB ) +func init() { + flag.Var(&defaultBandwidthMsgSize, + "piecestore.rpc.client.default-bandwidth-msg-size", + "default bandwidth message size in bytes") + flag.Var(&maxBandwidthMsgSize, + "piecestore.rpc.client.max-bandwidth-msg-size", + "max bandwidth message size in bytes") +} + // Client is an interface describing the functions for interacting with piecestore nodes type Client interface { Meta(ctx context.Context, id PieceID) (*pb.PieceSummary, error) @@ -63,12 +69,12 @@ func NewPSClient(ctx context.Context, tc transport.Client, n *pb.Node, bandwidth return nil, err } - if bandwidthMsgSize < 0 || bandwidthMsgSize > *maxBandwidthMsgSize { + if bandwidthMsgSize < 0 || bandwidthMsgSize > maxBandwidthMsgSize.Int() { return nil, ClientError.New("invalid Bandwidth Message Size: %v", bandwidthMsgSize) } if bandwidthMsgSize == 0 { - bandwidthMsgSize = *defaultBandwidthMsgSize + bandwidthMsgSize = defaultBandwidthMsgSize.Int() } return &PieceStore{ @@ -83,12 +89,12 @@ func NewPSClient(ctx context.Context, tc transport.Client, n *pb.Node, bandwidth // NewCustomRoute creates new PieceStore with custom client interface func NewCustomRoute(client pb.PieceStoreRoutesClient, target *pb.Node, bandwidthMsgSize int, prikey crypto.PrivateKey) (*PieceStore, error) { target.Type.DPanicOnInvalid("new custom route") - if bandwidthMsgSize < 0 || bandwidthMsgSize > *maxBandwidthMsgSize { + if bandwidthMsgSize < 0 || bandwidthMsgSize > maxBandwidthMsgSize.Int() { return nil, ClientError.New("invalid Bandwidth Message Size: %v", bandwidthMsgSize) } if bandwidthMsgSize == 0 { - bandwidthMsgSize = *defaultBandwidthMsgSize + bandwidthMsgSize = defaultBandwidthMsgSize.Int() } return &PieceStore{ diff --git a/pkg/piecestore/psserver/config.go b/pkg/piecestore/psserver/config.go index 83cd9c049..ff5434242 100644 --- a/pkg/piecestore/psserver/config.go +++ b/pkg/piecestore/psserver/config.go @@ -13,6 +13,7 @@ import ( "golang.org/x/net/context" monkit "gopkg.in/spacemonkeygo/monkit.v2" + "storj.io/storj/internal/memory" "storj.io/storj/pkg/kademlia" "storj.io/storj/pkg/pb" pstore "storj.io/storj/pkg/piecestore" @@ -28,8 +29,8 @@ var ( // Config contains everything necessary for a server type Config struct { Path string `help:"path to store data in" default:"$CONFDIR"` - AllocatedDiskSpace int64 `help:"total allocated disk space in bytes, default(1GB)" default:"1073741824"` - AllocatedBandwidth int64 `help:"total allocated bandwidth in bytes, default(100GB)" default:"107374182400"` + AllocatedDiskSpace memory.Size `help:"total allocated disk space in bytes" default:"1TiB"` + AllocatedBandwidth memory.Size `help:"total allocated bandwidth in bytes" default:"500GiB"` KBucketRefreshInterval time.Duration `help:"how frequently Kademlia bucket should be refreshed with node stats" default:"1h0m0s"` AgreementSenderCheckInterval time.Duration `help:"duration between agreement checks" default:"1h0m0s"` } diff --git a/pkg/piecestore/psserver/server.go b/pkg/piecestore/psserver/server.go index 73abb4e63..bff3d3e84 100644 --- a/pkg/piecestore/psserver/server.go +++ b/pkg/piecestore/psserver/server.go @@ -68,8 +68,8 @@ type Server struct { // NewEndpoint -- initializes a new endpoint for a piecestore server func NewEndpoint(log *zap.Logger, config Config, storage *pstore.Storage, db *psdb.DB, pkey crypto.PrivateKey) (*Server, error) { // read the allocated disk space from the config file - allocatedDiskSpace := config.AllocatedDiskSpace - allocatedBandwidth := config.AllocatedBandwidth + allocatedDiskSpace := config.AllocatedDiskSpace.Int64() + allocatedBandwidth := config.AllocatedBandwidth.Int64() // get the disk space details // The returned path ends in a slash only if it represents a root directory, such as "/" on Unix or `C:\` on Windows. @@ -137,8 +137,8 @@ func New(log *zap.Logger, storage *pstore.Storage, db *psdb.DB, config Config, p storage: storage, DB: db, pkey: pkey, - totalAllocated: config.AllocatedDiskSpace, - totalBwAllocated: config.AllocatedBandwidth, + totalAllocated: config.AllocatedDiskSpace.Int64(), + totalBwAllocated: config.AllocatedBandwidth.Int64(), verifier: auth.NewSignedMessageVerifier(), } } diff --git a/pkg/pointerdb/config.go b/pkg/pointerdb/config.go index 18a872ed3..fc613bac9 100644 --- a/pkg/pointerdb/config.go +++ b/pkg/pointerdb/config.go @@ -8,6 +8,7 @@ import ( "go.uber.org/zap" + "storj.io/storj/internal/memory" "storj.io/storj/pkg/overlay" "storj.io/storj/pkg/pb" "storj.io/storj/pkg/provider" @@ -30,11 +31,11 @@ const ( // Config is a configuration struct that is everything you need to start a // PointerDB responsibility type Config struct { - DatabaseURL string `help:"the database connection string to use" default:"bolt://$CONFDIR/pointerdb.db"` - MinRemoteSegmentSize int `default:"1240" help:"minimum remote segment size"` - MaxInlineSegmentSize int `default:"8000" help:"maximum inline segment size"` - Overlay bool `default:"true" help:"toggle flag if overlay is enabled"` - BwExpiration int `default:"45" help:"lifespan of bandwidth agreements in days"` + DatabaseURL string `help:"the database connection string to use" default:"bolt://$CONFDIR/pointerdb.db"` + MinRemoteSegmentSize memory.Size `default:"1240" help:"minimum remote segment size"` + MaxInlineSegmentSize memory.Size `default:"8000" help:"maximum inline segment size"` + Overlay bool `default:"true" help:"toggle flag if overlay is enabled"` + BwExpiration int `default:"45" help:"lifespan of bandwidth agreements in days"` } func newKeyValueStore(dbURLString string) (db storage.KeyValueStore, err error) { diff --git a/pkg/pointerdb/pointerdb.go b/pkg/pointerdb/pointerdb.go index 9ca3d4bd0..cd4a5a836 100644 --- a/pkg/pointerdb/pointerdb.go +++ b/pkg/pointerdb/pointerdb.go @@ -71,7 +71,7 @@ func (s *Server) validateSegment(req *pb.PutRequest) error { return segmentError.New("remote segment size %d less than minimum allowed %d", remoteSize, min) } - max := s.config.MaxInlineSegmentSize + max := s.config.MaxInlineSegmentSize.Int() inlineSize := len(req.GetPointer().InlineSegment) if inlineSize > max {