JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cmd/uplink: remove unused flag

Browse Source 
New API has limited number of options to configure at the moment. We
should remove unused flags from Uplink CLI and add if needed in the
future.

Change-Id: Icf3f3dadd43cb61a3b408b02d0762aef34425dbf
This commit is contained in:
Michal Niewrzal 2020-03-09 11:25:15 +01:00
parent c20cf25f35
commit d7b5df70d3
5 changed files with 248 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
cmd/uplink/cmd/config.go
View File

@ -11,8 +11,6 @@ import (
"github.com/spacemonkeygo/monkit/v3"
"github.com/zeebo/errs"
"storj.io/common/memory"
"storj.io/common/peertls/tlsopts"
"storj.io/common/storj"
libuplink "storj.io/storj/lib/uplink"
"storj.io/uplink"
@ -20,39 +18,16 @@ import (
var mon = monkit.Package()
// RSConfig is a configuration struct that keeps details about default
// redundancy strategy information
type RSConfig struct {
MaxBufferMem memory.Size `help:"maximum buffer memory (in bytes) to be allocated for read buffers" default:"4MiB" hidden:"true"`
ErasureShareSize memory.Size `help:"the size of each new erasure share in bytes" default:"256B" hidden:"true"`
MinThreshold int `help:"the minimum pieces required to recover a segment. k." releaseDefault:"29" devDefault:"4" hidden:"true"`
RepairThreshold int `help:"the minimum safe pieces before a repair is triggered. m." releaseDefault:"35" devDefault:"6" hidden:"true"`
SuccessThreshold int `help:"the desired total pieces for a segment. o." releaseDefault:"80" devDefault:"8" hidden:"true"`
MaxThreshold int `help:"the largest amount of pieces to encode to. n." releaseDefault:"110" devDefault:"10" hidden:"true"`
}
// EncryptionConfig is a configuration struct that keeps details about
// encrypting segments
type EncryptionConfig struct {
DataType int `help:"Type of encryption to use for content and metadata (2=AES-GCM, 3=SecretBox)" default:"2"`
PathType int `help:"Type of encryption to use for paths (1=Unencrypted, 2=AES-GCM, 3=SecretBox)" default:"2"`
}
// ClientConfig is a configuration struct for the uplink that controls how
// to talk to the rest of the network.
type ClientConfig struct {
MaxInlineSize memory.Size `help:"max inline segment size in bytes" default:"4KiB"`
SegmentSize memory.Size `help:"the size of a segment in bytes" default:"64MiB"`
DialTimeout time.Duration `help:"timeout for dials" default:"0h2m00s"`
DialTimeout time.Duration `help:"timeout for dials" default:"0h2m00s"`
}
// Config uplink configuration
type Config struct {
AccessConfig
Client ClientConfig
RS RSConfig
Enc EncryptionConfig
TLS tlsopts.Config
}
// AccessConfig holds information about which accesses exist and are selected.
@ -195,40 +170,6 @@ func (a AccessConfig) GetNamedAccess(name string) (_ *libuplink.Scope, err error
return nil, nil
}
// GetRedundancyScheme returns the configured redundancy scheme for new uploads
func (c Config) GetRedundancyScheme() storj.RedundancyScheme {
return storj.RedundancyScheme{
Algorithm: storj.ReedSolomon,
ShareSize: c.RS.ErasureShareSize.Int32(),
RequiredShares: int16(c.RS.MinThreshold),
RepairShares: int16(c.RS.RepairThreshold),
OptimalShares: int16(c.RS.SuccessThreshold),
TotalShares: int16(c.RS.MaxThreshold),
}
}
// GetPathCipherSuite returns the cipher suite used for path encryption for bucket objects
func (c Config) GetPathCipherSuite() storj.CipherSuite {
return storj.CipherSuite(c.Enc.PathType)
}
// GetEncryptionParameters returns the configured encryption scheme for new uploads
// Blocksize should align with the stripe size therefore multiples of stripes
// should fit in every encryption block. Instead of lettings users configure this
// multiple value, we hardcode stripesPerBlock as 2 for simplicity.
func (c Config) GetEncryptionParameters() storj.EncryptionParameters {
const stripesPerBlock = 2
return storj.EncryptionParameters{
CipherSuite: storj.CipherSuite(c.Enc.DataType),
BlockSize: c.GetRedundancyScheme().StripeSize() * stripesPerBlock,
}
}
// GetSegmentSize returns the segment size set in uplink config
func (c Config) GetSegmentSize() memory.Size {
return c.Client.SegmentSize
}
// IsSerializedAccess returns whether the passed access is a serialized
// access string or not.
func IsSerializedAccess(access string) bool {

13
private/testplanet/uplink.go
View File

@ -23,7 +23,6 @@ import (
"storj.io/common/peertls/tlsopts"
"storj.io/common/rpc"
"storj.io/common/storj"
"storj.io/storj/cmd/uplink/cmd"
libuplink "storj.io/storj/lib/uplink"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/satellite/console"
@ -186,7 +185,7 @@ func (client *Uplink) UploadWithExpiration(ctx context.Context, satellite *Satel
}
// UploadWithClientConfig uploads data to specific satellite with custom client configuration
func (client *Uplink) UploadWithClientConfig(ctx context.Context, satellite *SatelliteSystem, clientConfig cmd.Config, bucketName string, path storj.Path, data []byte) (err error) {
func (client *Uplink) UploadWithClientConfig(ctx context.Context, satellite *SatelliteSystem, clientConfig UplinkConfig, bucketName string, path storj.Path, data []byte) (err error) {
project, bucket, err := client.GetProjectAndBucket(ctx, satellite, bucketName, clientConfig)
if err != nil {
return err
@ -322,7 +321,7 @@ func (client *Uplink) DeleteBucket(ctx context.Context, satellite *SatelliteSyst
}
// GetConfig returns a default config for a given satellite.
func (client *Uplink) GetConfig(satellite *SatelliteSystem) cmd.Config {
func (client *Uplink) GetConfig(satellite *SatelliteSystem) UplinkConfig {
config := getDefaultConfig()
// client.APIKey[satellite.ID()] is a *macaroon.APIKey, but we want a
@ -365,8 +364,8 @@ func (client *Uplink) GetConfig(satellite *SatelliteSystem) cmd.Config {
return config
}
func getDefaultConfig() cmd.Config {
config := cmd.Config{}
func getDefaultConfig() UplinkConfig {
config := UplinkConfig{}
cfgstruct.Bind(&pflag.FlagSet{}, &config, cfgstruct.UseDevDefaults())
return config
}
@ -415,7 +414,7 @@ func (client *Uplink) GetProject(ctx context.Context, satellite *SatelliteSystem
}
// GetProjectAndBucket returns a libuplink.Project and Bucket which allows interactions with a specific project and its buckets
func (client *Uplink) GetProjectAndBucket(ctx context.Context, satellite *SatelliteSystem, bucketName string, clientCfg cmd.Config) (_ *libuplink.Project, _ *libuplink.Bucket, err error) {
func (client *Uplink) GetProjectAndBucket(ctx context.Context, satellite *SatelliteSystem, bucketName string, clientCfg UplinkConfig) (_ *libuplink.Project, _ *libuplink.Bucket, err error) {
project, err := client.GetProject(ctx, satellite)
if err != nil {
return nil, nil, err
@ -451,7 +450,7 @@ func (client *Uplink) GetProjectAndBucket(ctx context.Context, satellite *Satell
return project, bucket, nil
}
func createBucket(ctx context.Context, config cmd.Config, project libuplink.Project, bucketName string) error {
func createBucket(ctx context.Context, config UplinkConfig, project libuplink.Project, bucketName string) error {
bucketCfg := &libuplink.BucketConfig{}
bucketCfg.PathCipher = config.GetPathCipherSuite()
bucketCfg.EncryptionParameters = config.GetEncryptionParameters()

223
private/testplanet/uplink_config.go Normal file
View File

@ -0,0 +1,223 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package testplanet
import (
"io/ioutil"
"strings"
"time"
"github.com/zeebo/errs"
"storj.io/common/memory"
"storj.io/common/peertls/tlsopts"
"storj.io/common/storj"
libuplink "storj.io/storj/lib/uplink"
"storj.io/uplink"
)
// RSConfig is a configuration struct that keeps details about default
// redundancy strategy information
type RSConfig struct {
MaxBufferMem memory.Size `help:"maximum buffer memory (in bytes) to be allocated for read buffers" default:"4MiB" hidden:"true"`
ErasureShareSize memory.Size `help:"the size of each new erasure share in bytes" default:"256B" hidden:"true"`
MinThreshold int `help:"the minimum pieces required to recover a segment. k." releaseDefault:"29" devDefault:"4" hidden:"true"`
RepairThreshold int `help:"the minimum safe pieces before a repair is triggered. m." releaseDefault:"35" devDefault:"6" hidden:"true"`
SuccessThreshold int `help:"the desired total pieces for a segment. o." releaseDefault:"80" devDefault:"8" hidden:"true"`
MaxThreshold int `help:"the largest amount of pieces to encode to. n." releaseDefault:"110" devDefault:"10" hidden:"true"`
}
// EncryptionConfig is a configuration struct that keeps details about
// encrypting segments
type EncryptionConfig struct {
DataType int `help:"Type of encryption to use for content and metadata (2=AES-GCM, 3=SecretBox)" default:"2"`
PathType int `help:"Type of encryption to use for paths (1=Unencrypted, 2=AES-GCM, 3=SecretBox)" default:"2"`
}
// ClientConfig is a configuration struct for the uplink that controls how
// to talk to the rest of the network.
type ClientConfig struct {
MaxInlineSize memory.Size `help:"max inline segment size in bytes" default:"4KiB"`
SegmentSize memory.Size `help:"the size of a segment in bytes" default:"64MiB"`
DialTimeout time.Duration `help:"timeout for dials" default:"0h2m00s"`
}
// UplinkConfig uplink configuration
type UplinkConfig struct {
AccessConfig
Client ClientConfig
RS RSConfig
Enc EncryptionConfig
TLS tlsopts.Config
}
// AccessConfig holds information about which accesses exist and are selected.
type AccessConfig struct {
Accesses map[string]string `internal:"true"`
Access string `help:"the serialized access, or name of the access to use" default:"" basic-help:"true"`
// used for backward compatibility
Scopes map[string]string `internal:"true"` // deprecated
Scope string `internal:"true"` // deprecated
Legacy // Holds on to legacy configuration values
}
// Legacy holds deprecated configuration values
type Legacy struct {
Client struct {
APIKey string `default:"" help:"the api key to use for the satellite (deprecated)" noprefix:"true" deprecated:"true"`
SatelliteAddr string `releaseDefault:"127.0.0.1:7777" devDefault:"127.0.0.1:10000" help:"the address to use for the satellite (deprecated)" noprefix:"true"`
}
Enc struct {
EncryptionKey string `help:"the root key for encrypting the data which will be stored in KeyFilePath (deprecated)" setup:"true" deprecated:"true"`
KeyFilepath string `help:"the path to the file which contains the root key for encrypting the data (deprecated)" deprecated:"true"`
EncAccessFilepath string `help:"the path to a file containing a serialized encryption access (deprecated)" deprecated:"true"`
}
}
// normalize looks for usage of deprecated config values and sets the respective
// non-deprecated config values accordingly and returns them in a copy of the config.
func (a AccessConfig) normalize() (_ AccessConfig) {
// fallback to scope if access not found
if a.Access == "" {
a.Access = a.Scope
}
if a.Accesses == nil {
a.Accesses = make(map[string]string)
}
// fallback to scopes if accesses not found
if len(a.Accesses) == 0 {
for name, access := range a.Scopes {
a.Accesses[name] = access
}
}
return a
}
// GetAccess returns the appropriate access for the config.
func (a AccessConfig) GetAccess() (_ *libuplink.Scope, err error) {
a = a.normalize()
access, err := a.GetNamedAccess(a.Access)
if err != nil {
return nil, err
}
if access != nil {
return access, nil
}
// Otherwise, try to load the access name as a serialized access.
if access, err := libuplink.ParseScope(a.Access); err == nil {
return access, nil
}
// fall back to trying to load the legacy values.
apiKey, err := libuplink.ParseAPIKey(a.Legacy.Client.APIKey)
if err != nil {
return nil, err
}
satelliteAddr := a.Legacy.Client.SatelliteAddr
if satelliteAddr == "" {
return nil, errs.New("must specify a satellite address")
}
var encAccess *libuplink.EncryptionAccess
if a.Legacy.Enc.EncAccessFilepath != "" {
data, err := ioutil.ReadFile(a.Legacy.Enc.EncAccessFilepath)
if err != nil {
return nil, errs.Wrap(err)
}
encAccess, err = libuplink.ParseEncryptionAccess(strings.TrimSpace(string(data)))
if err != nil {
return nil, err
}
} else {
data := []byte(a.Legacy.Enc.EncryptionKey)
if a.Legacy.Enc.KeyFilepath != "" {
data, err = ioutil.ReadFile(a.Legacy.Enc.KeyFilepath)
if err != nil {
return nil, errs.Wrap(err)
}
}
key, err := storj.NewKey(data)
if err != nil {
return nil, errs.Wrap(err)
}
encAccess = libuplink.NewEncryptionAccessWithDefaultKey(*key)
encAccess.SetDefaultPathCipher(storj.EncAESGCM)
}
return &libuplink.Scope{
APIKey: apiKey,
SatelliteAddr: satelliteAddr,
EncryptionAccess: encAccess,
}, nil
}
// GetNewAccess returns the appropriate access for the config.
func (a AccessConfig) GetNewAccess() (_ *uplink.Access, err error) {
oldAccess, err := a.GetAccess()
if err != nil {
return nil, err
}
serializedOldAccess, err := oldAccess.Serialize()
if err != nil {
return nil, err
}
access, err := uplink.ParseAccess(serializedOldAccess)
if err != nil {
return nil, err
}
return access, nil
}
// GetNamedAccess returns named access if exists.
func (a AccessConfig) GetNamedAccess(name string) (_ *libuplink.Scope, err error) {
// if an access exists for that name, try to load it.
if data, ok := a.Accesses[name]; ok {
return libuplink.ParseScope(data)
}
return nil, nil
}
// GetRedundancyScheme returns the configured redundancy scheme for new uploads
func (c UplinkConfig) GetRedundancyScheme() storj.RedundancyScheme {
return storj.RedundancyScheme{
Algorithm: storj.ReedSolomon,
ShareSize: c.RS.ErasureShareSize.Int32(),
RequiredShares: int16(c.RS.MinThreshold),
RepairShares: int16(c.RS.RepairThreshold),
OptimalShares: int16(c.RS.SuccessThreshold),
TotalShares: int16(c.RS.MaxThreshold),
}
}
// GetPathCipherSuite returns the cipher suite used for path encryption for bucket objects
func (c UplinkConfig) GetPathCipherSuite() storj.CipherSuite {
return storj.CipherSuite(c.Enc.PathType)
}
// GetEncryptionParameters returns the configured encryption scheme for new uploads
// Blocksize should align with the stripe size therefore multiples of stripes
// should fit in every encryption block. Instead of lettings users configure this
// multiple value, we hardcode stripesPerBlock as 2 for simplicity.
func (c UplinkConfig) GetEncryptionParameters() storj.EncryptionParameters {
const stripesPerBlock = 2
return storj.EncryptionParameters{
CipherSuite: storj.CipherSuite(c.Enc.DataType),
BlockSize: c.GetRedundancyScheme().StripeSize() * stripesPerBlock,
}
}
// GetSegmentSize returns the segment size set in uplink config
func (c UplinkConfig) GetSegmentSize() memory.Size {
return c.Client.SegmentSize
}

21
satellite/metainfo/delete_pieces_service_test.go
View File

@ -19,7 +19,6 @@ import (
"storj.io/common/storj"
"storj.io/common/testcontext"
"storj.io/common/testrand"
"storj.io/storj/cmd/uplink/cmd"
"storj.io/storj/private/testblobs"
"storj.io/storj/private/testplanet"
"storj.io/storj/satellite"
@ -89,8 +88,8 @@ func TestDeletePiecesService_DeletePieces_AllNodesUp(t *testing.T) {
{
data := testrand.Bytes(10 * memory.KiB)
err := uplnk.UploadWithClientConfig(ctx, satelliteSys, cmd.Config{
Client: cmd.ClientConfig{
err := uplnk.UploadWithClientConfig(ctx, satelliteSys, testplanet.UplinkConfig{
Client: testplanet.ClientConfig{
SegmentSize: 10 * memory.KiB,
},
},
@ -162,8 +161,8 @@ func TestDeletePiecesService_DeletePieces_SomeNodesDown(t *testing.T) {
{
data := testrand.Bytes(10 * memory.KiB)
err := uplnk.UploadWithClientConfig(ctx, satelliteSys, cmd.Config{
Client: cmd.ClientConfig{
err := uplnk.UploadWithClientConfig(ctx, satelliteSys, testplanet.UplinkConfig{
Client: testplanet.ClientConfig{
SegmentSize: 10 * memory.KiB,
},
},
@ -228,8 +227,8 @@ func TestDeletePiecesService_DeletePieces_AllNodesDown(t *testing.T) {
{
data := testrand.Bytes(10 * memory.KiB)
err := uplnk.UploadWithClientConfig(ctx, satelliteSys, cmd.Config{
Client: cmd.ClientConfig{
err := uplnk.UploadWithClientConfig(ctx, satelliteSys, testplanet.UplinkConfig{
Client: testplanet.ClientConfig{
SegmentSize: 10 * memory.KiB,
},
},
@ -297,8 +296,8 @@ func TestDeletePiecesService_DeletePieces_InvalidDialer(t *testing.T) {
data := testrand.Bytes(10 * memory.KiB)
// Use RSConfig for ensuring that we don't have long-tail cancellations
// and the upload doesn't leave garbage in the SNs
err := uplnk.UploadWithClientConfig(ctx, satelliteSys, cmd.Config{
Client: cmd.ClientConfig{
err := uplnk.UploadWithClientConfig(ctx, satelliteSys, testplanet.UplinkConfig{
Client: testplanet.ClientConfig{
SegmentSize: 10 * memory.KiB,
},
},
@ -414,8 +413,8 @@ func TestDeletePiecesService_DeletePieces_Timeout(t *testing.T) {
{
data := testrand.Bytes(10 * memory.KiB)
err := uplnk.UploadWithClientConfig(ctx, satelliteSys, cmd.Config{
Client: cmd.ClientConfig{
err := uplnk.UploadWithClientConfig(ctx, satelliteSys, testplanet.UplinkConfig{
Client: testplanet.ClientConfig{
SegmentSize: 10 * memory.KiB,
},
},

17
satellite/metainfo/endpoint_test.go
View File

@ -16,7 +16,6 @@ import (
"storj.io/common/storj"
"storj.io/common/testcontext"
"storj.io/common/testrand"
"storj.io/storj/cmd/uplink/cmd"
"storj.io/storj/private/testplanet"
"storj.io/storj/satellite/metainfo"
"storj.io/storj/storage"
@ -58,8 +57,8 @@ func TestEndpoint_DeleteObjectPieces(t *testing.T) {
objectName = "object-filename" + strconv.Itoa(i)
)
err := uplnk.UploadWithClientConfig(ctx, satelliteSys, cmd.Config{
Client: cmd.ClientConfig{
err := uplnk.UploadWithClientConfig(ctx, satelliteSys, testplanet.UplinkConfig{
Client: testplanet.ClientConfig{
SegmentSize: 10 * memory.KiB,
},
},
@ -133,8 +132,8 @@ func TestEndpoint_DeleteObjectPieces(t *testing.T) {
uplnk = planet.Uplinks[0]
satelliteSys = planet.Satellites[0]
)
err := uplnk.UploadWithClientConfig(ctx, satelliteSys, cmd.Config{
Client: cmd.ClientConfig{
err := uplnk.UploadWithClientConfig(ctx, satelliteSys, testplanet.UplinkConfig{
Client: testplanet.ClientConfig{
SegmentSize: 10 * memory.KiB,
},
}, bucketName, objectName, tc.objData)
@ -208,8 +207,8 @@ func TestEndpoint_DeleteObjectPieces(t *testing.T) {
satelliteSys = planet.Satellites[0]
)
err := uplnk.UploadWithClientConfig(ctx, satelliteSys, cmd.Config{
Client: cmd.ClientConfig{
err := uplnk.UploadWithClientConfig(ctx, satelliteSys, testplanet.UplinkConfig{
Client: testplanet.ClientConfig{
SegmentSize: 10 * memory.KiB,
},
}, bucketName, objectName, tc.objData)
@ -485,8 +484,8 @@ func uploadFirstObjectWithoutSomeSegmentsPointers(
t.Fatal("noSegments list must have at least one segment")
}
err := uplnk.UploadWithClientConfig(ctx, satelliteSys, cmd.Config{
Client: cmd.ClientConfig{
err := uplnk.UploadWithClientConfig(ctx, satelliteSys, testplanet.UplinkConfig{
Client: testplanet.ClientConfig{
SegmentSize: segmentSize,
},
},