From d7b5df70d34e208ae72b831c584d8cf3e6b9c1d2 Mon Sep 17 00:00:00 2001 From: Michal Niewrzal Date: Mon, 9 Mar 2020 11:25:15 +0100 Subject: [PATCH] cmd/uplink: remove unused flag New API has limited number of options to configure at the moment. We should remove unused flags from Uplink CLI and add if needed in the future. Change-Id: Icf3f3dadd43cb61a3b408b02d0762aef34425dbf --- cmd/uplink/cmd/config.go | 61 +---- private/testplanet/uplink.go | 13 +- private/testplanet/uplink_config.go | 223 ++++++++++++++++++ .../metainfo/delete_pieces_service_test.go | 21 +- satellite/metainfo/endpoint_test.go | 17 +- 5 files changed, 248 insertions(+), 87 deletions(-) create mode 100644 private/testplanet/uplink_config.go diff --git a/cmd/uplink/cmd/config.go b/cmd/uplink/cmd/config.go index 5d8cf1d4d..c4fbc7203 100644 --- a/cmd/uplink/cmd/config.go +++ b/cmd/uplink/cmd/config.go @@ -11,8 +11,6 @@ import ( "github.com/spacemonkeygo/monkit/v3" "github.com/zeebo/errs" - "storj.io/common/memory" - "storj.io/common/peertls/tlsopts" "storj.io/common/storj" libuplink "storj.io/storj/lib/uplink" "storj.io/uplink" @@ -20,39 +18,16 @@ import ( var mon = monkit.Package() -// RSConfig is a configuration struct that keeps details about default -// redundancy strategy information -type RSConfig struct { - MaxBufferMem memory.Size `help:"maximum buffer memory (in bytes) to be allocated for read buffers" default:"4MiB" hidden:"true"` - ErasureShareSize memory.Size `help:"the size of each new erasure share in bytes" default:"256B" hidden:"true"` - MinThreshold int `help:"the minimum pieces required to recover a segment. k." releaseDefault:"29" devDefault:"4" hidden:"true"` - RepairThreshold int `help:"the minimum safe pieces before a repair is triggered. m." releaseDefault:"35" devDefault:"6" hidden:"true"` - SuccessThreshold int `help:"the desired total pieces for a segment. o." releaseDefault:"80" devDefault:"8" hidden:"true"` - MaxThreshold int `help:"the largest amount of pieces to encode to. n." releaseDefault:"110" devDefault:"10" hidden:"true"` -} - -// EncryptionConfig is a configuration struct that keeps details about -// encrypting segments -type EncryptionConfig struct { - DataType int `help:"Type of encryption to use for content and metadata (2=AES-GCM, 3=SecretBox)" default:"2"` - PathType int `help:"Type of encryption to use for paths (1=Unencrypted, 2=AES-GCM, 3=SecretBox)" default:"2"` -} - // ClientConfig is a configuration struct for the uplink that controls how // to talk to the rest of the network. type ClientConfig struct { - MaxInlineSize memory.Size `help:"max inline segment size in bytes" default:"4KiB"` - SegmentSize memory.Size `help:"the size of a segment in bytes" default:"64MiB"` - DialTimeout time.Duration `help:"timeout for dials" default:"0h2m00s"` + DialTimeout time.Duration `help:"timeout for dials" default:"0h2m00s"` } // Config uplink configuration type Config struct { AccessConfig Client ClientConfig - RS RSConfig - Enc EncryptionConfig - TLS tlsopts.Config } // AccessConfig holds information about which accesses exist and are selected. @@ -195,40 +170,6 @@ func (a AccessConfig) GetNamedAccess(name string) (_ *libuplink.Scope, err error return nil, nil } -// GetRedundancyScheme returns the configured redundancy scheme for new uploads -func (c Config) GetRedundancyScheme() storj.RedundancyScheme { - return storj.RedundancyScheme{ - Algorithm: storj.ReedSolomon, - ShareSize: c.RS.ErasureShareSize.Int32(), - RequiredShares: int16(c.RS.MinThreshold), - RepairShares: int16(c.RS.RepairThreshold), - OptimalShares: int16(c.RS.SuccessThreshold), - TotalShares: int16(c.RS.MaxThreshold), - } -} - -// GetPathCipherSuite returns the cipher suite used for path encryption for bucket objects -func (c Config) GetPathCipherSuite() storj.CipherSuite { - return storj.CipherSuite(c.Enc.PathType) -} - -// GetEncryptionParameters returns the configured encryption scheme for new uploads -// Blocksize should align with the stripe size therefore multiples of stripes -// should fit in every encryption block. Instead of lettings users configure this -// multiple value, we hardcode stripesPerBlock as 2 for simplicity. -func (c Config) GetEncryptionParameters() storj.EncryptionParameters { - const stripesPerBlock = 2 - return storj.EncryptionParameters{ - CipherSuite: storj.CipherSuite(c.Enc.DataType), - BlockSize: c.GetRedundancyScheme().StripeSize() * stripesPerBlock, - } -} - -// GetSegmentSize returns the segment size set in uplink config -func (c Config) GetSegmentSize() memory.Size { - return c.Client.SegmentSize -} - // IsSerializedAccess returns whether the passed access is a serialized // access string or not. func IsSerializedAccess(access string) bool { diff --git a/private/testplanet/uplink.go b/private/testplanet/uplink.go index 0906c58dd..4e9be24e1 100644 --- a/private/testplanet/uplink.go +++ b/private/testplanet/uplink.go @@ -23,7 +23,6 @@ import ( "storj.io/common/peertls/tlsopts" "storj.io/common/rpc" "storj.io/common/storj" - "storj.io/storj/cmd/uplink/cmd" libuplink "storj.io/storj/lib/uplink" "storj.io/storj/pkg/cfgstruct" "storj.io/storj/satellite/console" @@ -186,7 +185,7 @@ func (client *Uplink) UploadWithExpiration(ctx context.Context, satellite *Satel } // UploadWithClientConfig uploads data to specific satellite with custom client configuration -func (client *Uplink) UploadWithClientConfig(ctx context.Context, satellite *SatelliteSystem, clientConfig cmd.Config, bucketName string, path storj.Path, data []byte) (err error) { +func (client *Uplink) UploadWithClientConfig(ctx context.Context, satellite *SatelliteSystem, clientConfig UplinkConfig, bucketName string, path storj.Path, data []byte) (err error) { project, bucket, err := client.GetProjectAndBucket(ctx, satellite, bucketName, clientConfig) if err != nil { return err @@ -322,7 +321,7 @@ func (client *Uplink) DeleteBucket(ctx context.Context, satellite *SatelliteSyst } // GetConfig returns a default config for a given satellite. -func (client *Uplink) GetConfig(satellite *SatelliteSystem) cmd.Config { +func (client *Uplink) GetConfig(satellite *SatelliteSystem) UplinkConfig { config := getDefaultConfig() // client.APIKey[satellite.ID()] is a *macaroon.APIKey, but we want a @@ -365,8 +364,8 @@ func (client *Uplink) GetConfig(satellite *SatelliteSystem) cmd.Config { return config } -func getDefaultConfig() cmd.Config { - config := cmd.Config{} +func getDefaultConfig() UplinkConfig { + config := UplinkConfig{} cfgstruct.Bind(&pflag.FlagSet{}, &config, cfgstruct.UseDevDefaults()) return config } @@ -415,7 +414,7 @@ func (client *Uplink) GetProject(ctx context.Context, satellite *SatelliteSystem } // GetProjectAndBucket returns a libuplink.Project and Bucket which allows interactions with a specific project and its buckets -func (client *Uplink) GetProjectAndBucket(ctx context.Context, satellite *SatelliteSystem, bucketName string, clientCfg cmd.Config) (_ *libuplink.Project, _ *libuplink.Bucket, err error) { +func (client *Uplink) GetProjectAndBucket(ctx context.Context, satellite *SatelliteSystem, bucketName string, clientCfg UplinkConfig) (_ *libuplink.Project, _ *libuplink.Bucket, err error) { project, err := client.GetProject(ctx, satellite) if err != nil { return nil, nil, err @@ -451,7 +450,7 @@ func (client *Uplink) GetProjectAndBucket(ctx context.Context, satellite *Satell return project, bucket, nil } -func createBucket(ctx context.Context, config cmd.Config, project libuplink.Project, bucketName string) error { +func createBucket(ctx context.Context, config UplinkConfig, project libuplink.Project, bucketName string) error { bucketCfg := &libuplink.BucketConfig{} bucketCfg.PathCipher = config.GetPathCipherSuite() bucketCfg.EncryptionParameters = config.GetEncryptionParameters() diff --git a/private/testplanet/uplink_config.go b/private/testplanet/uplink_config.go new file mode 100644 index 000000000..62b2a4c13 --- /dev/null +++ b/private/testplanet/uplink_config.go @@ -0,0 +1,223 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package testplanet + +import ( + "io/ioutil" + "strings" + "time" + + "github.com/zeebo/errs" + + "storj.io/common/memory" + "storj.io/common/peertls/tlsopts" + "storj.io/common/storj" + libuplink "storj.io/storj/lib/uplink" + "storj.io/uplink" +) + +// RSConfig is a configuration struct that keeps details about default +// redundancy strategy information +type RSConfig struct { + MaxBufferMem memory.Size `help:"maximum buffer memory (in bytes) to be allocated for read buffers" default:"4MiB" hidden:"true"` + ErasureShareSize memory.Size `help:"the size of each new erasure share in bytes" default:"256B" hidden:"true"` + MinThreshold int `help:"the minimum pieces required to recover a segment. k." releaseDefault:"29" devDefault:"4" hidden:"true"` + RepairThreshold int `help:"the minimum safe pieces before a repair is triggered. m." releaseDefault:"35" devDefault:"6" hidden:"true"` + SuccessThreshold int `help:"the desired total pieces for a segment. o." releaseDefault:"80" devDefault:"8" hidden:"true"` + MaxThreshold int `help:"the largest amount of pieces to encode to. n." releaseDefault:"110" devDefault:"10" hidden:"true"` +} + +// EncryptionConfig is a configuration struct that keeps details about +// encrypting segments +type EncryptionConfig struct { + DataType int `help:"Type of encryption to use for content and metadata (2=AES-GCM, 3=SecretBox)" default:"2"` + PathType int `help:"Type of encryption to use for paths (1=Unencrypted, 2=AES-GCM, 3=SecretBox)" default:"2"` +} + +// ClientConfig is a configuration struct for the uplink that controls how +// to talk to the rest of the network. +type ClientConfig struct { + MaxInlineSize memory.Size `help:"max inline segment size in bytes" default:"4KiB"` + SegmentSize memory.Size `help:"the size of a segment in bytes" default:"64MiB"` + DialTimeout time.Duration `help:"timeout for dials" default:"0h2m00s"` +} + +// UplinkConfig uplink configuration +type UplinkConfig struct { + AccessConfig + Client ClientConfig + RS RSConfig + Enc EncryptionConfig + TLS tlsopts.Config +} + +// AccessConfig holds information about which accesses exist and are selected. +type AccessConfig struct { + Accesses map[string]string `internal:"true"` + Access string `help:"the serialized access, or name of the access to use" default:"" basic-help:"true"` + + // used for backward compatibility + Scopes map[string]string `internal:"true"` // deprecated + Scope string `internal:"true"` // deprecated + + Legacy // Holds on to legacy configuration values +} + +// Legacy holds deprecated configuration values +type Legacy struct { + Client struct { + APIKey string `default:"" help:"the api key to use for the satellite (deprecated)" noprefix:"true" deprecated:"true"` + SatelliteAddr string `releaseDefault:"127.0.0.1:7777" devDefault:"127.0.0.1:10000" help:"the address to use for the satellite (deprecated)" noprefix:"true"` + } + Enc struct { + EncryptionKey string `help:"the root key for encrypting the data which will be stored in KeyFilePath (deprecated)" setup:"true" deprecated:"true"` + KeyFilepath string `help:"the path to the file which contains the root key for encrypting the data (deprecated)" deprecated:"true"` + EncAccessFilepath string `help:"the path to a file containing a serialized encryption access (deprecated)" deprecated:"true"` + } +} + +// normalize looks for usage of deprecated config values and sets the respective +// non-deprecated config values accordingly and returns them in a copy of the config. +func (a AccessConfig) normalize() (_ AccessConfig) { + // fallback to scope if access not found + if a.Access == "" { + a.Access = a.Scope + } + + if a.Accesses == nil { + a.Accesses = make(map[string]string) + } + + // fallback to scopes if accesses not found + if len(a.Accesses) == 0 { + for name, access := range a.Scopes { + a.Accesses[name] = access + } + } + + return a +} + +// GetAccess returns the appropriate access for the config. +func (a AccessConfig) GetAccess() (_ *libuplink.Scope, err error) { + a = a.normalize() + + access, err := a.GetNamedAccess(a.Access) + if err != nil { + return nil, err + } + if access != nil { + return access, nil + } + + // Otherwise, try to load the access name as a serialized access. + if access, err := libuplink.ParseScope(a.Access); err == nil { + return access, nil + } + + // fall back to trying to load the legacy values. + apiKey, err := libuplink.ParseAPIKey(a.Legacy.Client.APIKey) + if err != nil { + return nil, err + } + + satelliteAddr := a.Legacy.Client.SatelliteAddr + if satelliteAddr == "" { + return nil, errs.New("must specify a satellite address") + } + + var encAccess *libuplink.EncryptionAccess + if a.Legacy.Enc.EncAccessFilepath != "" { + data, err := ioutil.ReadFile(a.Legacy.Enc.EncAccessFilepath) + if err != nil { + return nil, errs.Wrap(err) + } + encAccess, err = libuplink.ParseEncryptionAccess(strings.TrimSpace(string(data))) + if err != nil { + return nil, err + } + } else { + data := []byte(a.Legacy.Enc.EncryptionKey) + if a.Legacy.Enc.KeyFilepath != "" { + data, err = ioutil.ReadFile(a.Legacy.Enc.KeyFilepath) + if err != nil { + return nil, errs.Wrap(err) + } + } + key, err := storj.NewKey(data) + if err != nil { + return nil, errs.Wrap(err) + } + encAccess = libuplink.NewEncryptionAccessWithDefaultKey(*key) + encAccess.SetDefaultPathCipher(storj.EncAESGCM) + } + + return &libuplink.Scope{ + APIKey: apiKey, + SatelliteAddr: satelliteAddr, + EncryptionAccess: encAccess, + }, nil +} + +// GetNewAccess returns the appropriate access for the config. +func (a AccessConfig) GetNewAccess() (_ *uplink.Access, err error) { + oldAccess, err := a.GetAccess() + if err != nil { + return nil, err + } + + serializedOldAccess, err := oldAccess.Serialize() + if err != nil { + return nil, err + } + + access, err := uplink.ParseAccess(serializedOldAccess) + if err != nil { + return nil, err + } + return access, nil +} + +// GetNamedAccess returns named access if exists. +func (a AccessConfig) GetNamedAccess(name string) (_ *libuplink.Scope, err error) { + // if an access exists for that name, try to load it. + if data, ok := a.Accesses[name]; ok { + return libuplink.ParseScope(data) + } + return nil, nil +} + +// GetRedundancyScheme returns the configured redundancy scheme for new uploads +func (c UplinkConfig) GetRedundancyScheme() storj.RedundancyScheme { + return storj.RedundancyScheme{ + Algorithm: storj.ReedSolomon, + ShareSize: c.RS.ErasureShareSize.Int32(), + RequiredShares: int16(c.RS.MinThreshold), + RepairShares: int16(c.RS.RepairThreshold), + OptimalShares: int16(c.RS.SuccessThreshold), + TotalShares: int16(c.RS.MaxThreshold), + } +} + +// GetPathCipherSuite returns the cipher suite used for path encryption for bucket objects +func (c UplinkConfig) GetPathCipherSuite() storj.CipherSuite { + return storj.CipherSuite(c.Enc.PathType) +} + +// GetEncryptionParameters returns the configured encryption scheme for new uploads +// Blocksize should align with the stripe size therefore multiples of stripes +// should fit in every encryption block. Instead of lettings users configure this +// multiple value, we hardcode stripesPerBlock as 2 for simplicity. +func (c UplinkConfig) GetEncryptionParameters() storj.EncryptionParameters { + const stripesPerBlock = 2 + return storj.EncryptionParameters{ + CipherSuite: storj.CipherSuite(c.Enc.DataType), + BlockSize: c.GetRedundancyScheme().StripeSize() * stripesPerBlock, + } +} + +// GetSegmentSize returns the segment size set in uplink config +func (c UplinkConfig) GetSegmentSize() memory.Size { + return c.Client.SegmentSize +} diff --git a/satellite/metainfo/delete_pieces_service_test.go b/satellite/metainfo/delete_pieces_service_test.go index a2984d458..3c0ee62ec 100644 --- a/satellite/metainfo/delete_pieces_service_test.go +++ b/satellite/metainfo/delete_pieces_service_test.go @@ -19,7 +19,6 @@ import ( "storj.io/common/storj" "storj.io/common/testcontext" "storj.io/common/testrand" - "storj.io/storj/cmd/uplink/cmd" "storj.io/storj/private/testblobs" "storj.io/storj/private/testplanet" "storj.io/storj/satellite" @@ -89,8 +88,8 @@ func TestDeletePiecesService_DeletePieces_AllNodesUp(t *testing.T) { { data := testrand.Bytes(10 * memory.KiB) - err := uplnk.UploadWithClientConfig(ctx, satelliteSys, cmd.Config{ - Client: cmd.ClientConfig{ + err := uplnk.UploadWithClientConfig(ctx, satelliteSys, testplanet.UplinkConfig{ + Client: testplanet.ClientConfig{ SegmentSize: 10 * memory.KiB, }, }, @@ -162,8 +161,8 @@ func TestDeletePiecesService_DeletePieces_SomeNodesDown(t *testing.T) { { data := testrand.Bytes(10 * memory.KiB) - err := uplnk.UploadWithClientConfig(ctx, satelliteSys, cmd.Config{ - Client: cmd.ClientConfig{ + err := uplnk.UploadWithClientConfig(ctx, satelliteSys, testplanet.UplinkConfig{ + Client: testplanet.ClientConfig{ SegmentSize: 10 * memory.KiB, }, }, @@ -228,8 +227,8 @@ func TestDeletePiecesService_DeletePieces_AllNodesDown(t *testing.T) { { data := testrand.Bytes(10 * memory.KiB) - err := uplnk.UploadWithClientConfig(ctx, satelliteSys, cmd.Config{ - Client: cmd.ClientConfig{ + err := uplnk.UploadWithClientConfig(ctx, satelliteSys, testplanet.UplinkConfig{ + Client: testplanet.ClientConfig{ SegmentSize: 10 * memory.KiB, }, }, @@ -297,8 +296,8 @@ func TestDeletePiecesService_DeletePieces_InvalidDialer(t *testing.T) { data := testrand.Bytes(10 * memory.KiB) // Use RSConfig for ensuring that we don't have long-tail cancellations // and the upload doesn't leave garbage in the SNs - err := uplnk.UploadWithClientConfig(ctx, satelliteSys, cmd.Config{ - Client: cmd.ClientConfig{ + err := uplnk.UploadWithClientConfig(ctx, satelliteSys, testplanet.UplinkConfig{ + Client: testplanet.ClientConfig{ SegmentSize: 10 * memory.KiB, }, }, @@ -414,8 +413,8 @@ func TestDeletePiecesService_DeletePieces_Timeout(t *testing.T) { { data := testrand.Bytes(10 * memory.KiB) - err := uplnk.UploadWithClientConfig(ctx, satelliteSys, cmd.Config{ - Client: cmd.ClientConfig{ + err := uplnk.UploadWithClientConfig(ctx, satelliteSys, testplanet.UplinkConfig{ + Client: testplanet.ClientConfig{ SegmentSize: 10 * memory.KiB, }, }, diff --git a/satellite/metainfo/endpoint_test.go b/satellite/metainfo/endpoint_test.go index 055f8ed58..7ed2c83ba 100644 --- a/satellite/metainfo/endpoint_test.go +++ b/satellite/metainfo/endpoint_test.go @@ -16,7 +16,6 @@ import ( "storj.io/common/storj" "storj.io/common/testcontext" "storj.io/common/testrand" - "storj.io/storj/cmd/uplink/cmd" "storj.io/storj/private/testplanet" "storj.io/storj/satellite/metainfo" "storj.io/storj/storage" @@ -58,8 +57,8 @@ func TestEndpoint_DeleteObjectPieces(t *testing.T) { objectName = "object-filename" + strconv.Itoa(i) ) - err := uplnk.UploadWithClientConfig(ctx, satelliteSys, cmd.Config{ - Client: cmd.ClientConfig{ + err := uplnk.UploadWithClientConfig(ctx, satelliteSys, testplanet.UplinkConfig{ + Client: testplanet.ClientConfig{ SegmentSize: 10 * memory.KiB, }, }, @@ -133,8 +132,8 @@ func TestEndpoint_DeleteObjectPieces(t *testing.T) { uplnk = planet.Uplinks[0] satelliteSys = planet.Satellites[0] ) - err := uplnk.UploadWithClientConfig(ctx, satelliteSys, cmd.Config{ - Client: cmd.ClientConfig{ + err := uplnk.UploadWithClientConfig(ctx, satelliteSys, testplanet.UplinkConfig{ + Client: testplanet.ClientConfig{ SegmentSize: 10 * memory.KiB, }, }, bucketName, objectName, tc.objData) @@ -208,8 +207,8 @@ func TestEndpoint_DeleteObjectPieces(t *testing.T) { satelliteSys = planet.Satellites[0] ) - err := uplnk.UploadWithClientConfig(ctx, satelliteSys, cmd.Config{ - Client: cmd.ClientConfig{ + err := uplnk.UploadWithClientConfig(ctx, satelliteSys, testplanet.UplinkConfig{ + Client: testplanet.ClientConfig{ SegmentSize: 10 * memory.KiB, }, }, bucketName, objectName, tc.objData) @@ -485,8 +484,8 @@ func uploadFirstObjectWithoutSomeSegmentsPointers( t.Fatal("noSegments list must have at least one segment") } - err := uplnk.UploadWithClientConfig(ctx, satelliteSys, cmd.Config{ - Client: cmd.ClientConfig{ + err := uplnk.UploadWithClientConfig(ctx, satelliteSys, testplanet.UplinkConfig{ + Client: testplanet.ClientConfig{ SegmentSize: segmentSize, }, },