storj/uplink/config.go

191 lines
7.2 KiB
Go
Raw Normal View History

// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package uplink
import (
"context"
"errors"
"io/ioutil"
"time"
"github.com/vivint/infectious"
"github.com/zeebo/errs"
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/internal/memory"
"storj.io/storj/pkg/eestream"
"storj.io/storj/pkg/encryption"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/metainfo/kvmetainfo"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/storage/buckets"
ecclient "storj.io/storj/pkg/storage/ec"
"storj.io/storj/pkg/storage/segments"
"storj.io/storj/pkg/storage/streams"
"storj.io/storj/pkg/storj"
"storj.io/storj/pkg/transport"
"storj.io/storj/uplink/metainfo"
)
// RSConfig is a configuration struct that keeps details about default
// redundancy strategy information
type RSConfig struct {
MaxBufferMem memory.Size `help:"maximum buffer memory (in bytes) to be allocated for read buffers" default:"4MiB" hidden:"true"`
ErasureShareSize memory.Size `help:"the size of each new erasure share in bytes" default:"256B" hidden:"true"`
MinThreshold int `help:"the minimum pieces required to recover a segment. k." releaseDefault:"29" devDefault:"4" hidden:"true"`
RepairThreshold int `help:"the minimum safe pieces before a repair is triggered. m." releaseDefault:"35" devDefault:"6" hidden:"true"`
SuccessThreshold int `help:"the desired total pieces for a segment. o." releaseDefault:"80" devDefault:"8" hidden:"true"`
MaxThreshold int `help:"the largest amount of pieces to encode to. n." releaseDefault:"130" devDefault:"10" hidden:"true"`
}
// EncryptionConfig is a configuration struct that keeps details about
// encrypting segments
type EncryptionConfig struct {
uplink: enc.encryption-key flag is only available for setup command (#2090) * uplink: Mark encryption key config field for setup Set the "setup" property to the `EncryptionConfig.EncrptionKey` for avoiding to save it in the configuration file. This field is only meant for using in the command line parameters which need to use a different encryption key than the one present in the key file or use it when there is not set any encryption key file path. * cmd/uplink: Setup non-interactive accept enc key Change the uplink CLI setup command non-interactive to save the encryption key into a file when it's passed through the flag --enc.encryption-key Previous to this change it wasn't possible to create an key file despite of that the flag was provided, so it was useless on the setup command. * cmd/uplink: Reuse logic to read pwd from terminal Reuse the logic which is already implemented in the pkg/cfgstruct for reading a password from the terminal on interactive mode, rather than duplicating it in the setup command. * cmd/gateway: Use encryption key file flags The cmd/gateway was still using the `enc.key` configuration field which doesn't exist anymore and its setup command wasn't using the `enc.key-filepath` with combination of the `enc.encryption-key` for generating a file with the encryption key. This commit update the cmd/gateway appropriately and move to the uplink package the function used by cmd/uplink to save the encryption key for allowing to also be used by the cmd/gateway without duplicating the logic. * cmd/storj-sim: Adapt gateway config cmd changes Adapt the cmd/storj-sim to correctly pass the parameters to the cmd/gateway setup and run command. * scripts: Don't pass the --enc.encryption-key flag uplink configuration has changed to only support the `--enc.encryption-key` flag for setup commands and consequently the cmd/uplink and cmd/gateway don't accept this flag over other commands, hence the test for the uplink had to be updated for no passing the flag on the multiples calls that the test do to cmd/uplink. * uplink: Remove func which aren't useful anymore Remove the function which allows to user or load an encryption key because it isn't needed anymore since the `--enc.encryption-key` flag is only available for the setup command. Consequently remove its usage from cmd/uplink and cmd/gateway, because such flag will always be empty because in case that's passed Cobra will return an error due to a "unknown flag".
2019-06-07 17:14:40 +01:00
EncryptionKey string `help:"the root key for encrypting the data which will be stored in KeyFilePath" setup:"true"`
KeyFilepath string `help:"the path to the file which contains the root key for encrypting the data"`
DataType int `help:"Type of encryption to use for content and metadata (1=AES-GCM, 2=SecretBox)" default:"1"`
PathType int `help:"Type of encryption to use for paths (0=Unencrypted, 1=AES-GCM, 2=SecretBox)" default:"1"`
}
// ClientConfig is a configuration struct for the uplink that controls how
// to talk to the rest of the network.
type ClientConfig struct {
APIKey string `default:"" help:"the api key to use for the satellite" noprefix:"true"`
SatelliteAddr string `releaseDefault:"127.0.0.1:7777" devDefault:"127.0.0.1:10000" help:"the address to use for the satellite" noprefix:"true"`
MaxInlineSize memory.Size `help:"max inline segment size in bytes" default:"4KiB"`
SegmentSize memory.Size `help:"the size of a segment in bytes" default:"64MiB"`
RequestTimeout time.Duration `help:"timeout for request" default:"0h0m20s"`
DialTimeout time.Duration `help:"timeout for dials" default:"0h0m20s"`
}
// Config uplink configuration
type Config struct {
Client ClientConfig
RS RSConfig
Enc EncryptionConfig
TLS tlsopts.Config
}
var (
mon = monkit.Package()
// Error is the errs class of standard End User Client errors
Error = errs.Class("Uplink configuration error")
)
// GetMetainfo returns an implementation of storj.Metainfo
func (c Config) GetMetainfo(ctx context.Context, identity *identity.FullIdentity) (db storj.Metainfo, ss streams.Store, err error) {
defer mon.Task()(&ctx)(&err)
tlsOpts, err := tlsopts.NewOptions(identity, c.TLS)
if err != nil {
return nil, nil, err
}
Add Versioning Server (#1576) * Initial Webserver Draft for Version Controlling * Rename type to avoid confusion * Move Function Calls into Version Package * Fix Linting and Language Typos * Fix Linting and Spelling Mistakes * Include Copyright * Include Copyright * Adjust Version-Control Server to return list of Versions * Linting * Improve Request Handling and Readability * Add Configuration File Option Add Systemd Service file * Add Logging to File * Smaller Changes * Add Semantic Versioning and refuses outdated Software from Startup (#1612) * implements internal Semantic Version library * adds version logging + reporting to process * Advance SemVer struct for easier handling * Add Accepted Version Store * Fix Function * Restructure * Type Conversion * Handle Version String properly * Add Note about array index * Set temporary Default Version * Add Copyright * Adding Version to Dashboard * Adding Version Info Log * Renaming and adding CheckerProcess * Iteration Sync * Iteration V2 * linting * made LogAndReportVersion a go routine * Refactor to Go Routine * Add Context to Go Routine and allow Operation if Lookup to Control Server fails * Handle Unmarshal properly * Linting * Relocate Version Checks * Relocating Version Check and specified default Version for now * Linting Error Prevention * Refuse Startup on outdated Version * Add Startup Check Function * Straighten Logging * Dont force Shutdown if --dev flag is set * Create full Service/Peer Structure for ControlServer * Linting * Straighting Naming * Finish VersionControl Service Layout * Improve Error Handling * Change Listening Address * Move Checker Function * Remove VersionControl Peer * Linting * Linting * Create VersionClient Service * Renaming * Add Version Client to Peer Definitions * Linting and Renaming * Linting * Remove Transport Checks for now * Move to Client Side Flag * Remove check * Linting * Transport Client Version Intro * Adding Version Client to Transport Client * Add missing parameter * Adding Version Check, to set Allowed = true * Set Default to true, testing * Restructuring Code * Uplink Changes * Add more proper Defaults * Renaming of Version struct * Dont pass Service use Pointer * Set Defaults for Versioning Checks * Put HTTP Server in go routine * Add Versioncontrol to Storj-Sim * Testplanet Fixes * Linting * Add Error Handling and new Server Struct * Move Lock slightly * Reduce Race Potentials * Remove unnecessary files * Linting * Add Proper Transport Handling * small fixes * add fence for allowed check * Add Startup Version Check and Service Naming * make errormessage private * Add Comments about VersionedClient * Linting * Remove Checks that refuse outgoing connections * Remove release cmd * Add Release Script * Linting * Update to use correct Values * Move vars private and set minimum default versions for testing builds * Remove VersionedClient * Better Error Handling and naked return removal * Straighten the Regex and string conversion * Change Check to allows testplanet and storj-sim to run without the need to pass an LDFlag * Cosmetic Change to Dashboard * Cleanup Returns and remove commented code * Remove Version Check if no build options are passed in * Pass in Config Values instead of Pointers * Handle missed Error * Update Endpoint URL * Change Type of Release Flag * Add additional Logging * Remove Versions Logging of other Services * minor fixes Change-Id: I5cc04a410ea6b2008d14dffd63eb5f36dd348a8b
2019-04-03 20:13:39 +01:00
// ToDo: Handle Versioning for Uplinks here
tc := transport.NewClientWithTimeouts(tlsOpts, transport.Timeouts{
Request: c.Client.RequestTimeout,
Dial: c.Client.DialTimeout,
})
2019-03-22 09:01:49 +00:00
if c.Client.SatelliteAddr == "" {
return nil, nil, errors.New("satellite address not specified")
}
2019-03-22 09:01:49 +00:00
metainfo, err := metainfo.NewClient(ctx, tc, c.Client.SatelliteAddr, c.Client.APIKey)
if err != nil {
return nil, nil, Error.New("failed to connect to metainfo service: %v", err)
}
ec := ecclient.NewClient(tc, c.RS.MaxBufferMem.Int())
fc, err := infectious.NewFEC(c.RS.MinThreshold, c.RS.MaxThreshold)
if err != nil {
return nil, nil, Error.New("failed to create erasure coding client: %v", err)
}
rs, err := eestream.NewRedundancyStrategy(eestream.NewRSScheme(fc, c.RS.ErasureShareSize.Int()), c.RS.RepairThreshold, c.RS.SuccessThreshold)
if err != nil {
return nil, nil, Error.New("failed to create redundancy strategy: %v", err)
}
maxEncryptedSegmentSize, err := encryption.CalcEncryptedSize(c.Client.SegmentSize.Int64(), c.GetEncryptionScheme())
if err != nil {
return nil, nil, Error.New("failed to calculate max encrypted segment size: %v", err)
}
segments := segments.NewSegmentStore(metainfo, ec, rs, c.Client.MaxInlineSize.Int(), maxEncryptedSegmentSize)
blockSize := c.GetEncryptionScheme().BlockSize
if int(blockSize)%c.RS.ErasureShareSize.Int()*c.RS.MinThreshold != 0 {
err = Error.New("EncryptionBlockSize must be a multiple of ErasureShareSize * RS MinThreshold")
return nil, nil, err
}
uplink: enc.encryption-key flag is only available for setup command (#2090) * uplink: Mark encryption key config field for setup Set the "setup" property to the `EncryptionConfig.EncrptionKey` for avoiding to save it in the configuration file. This field is only meant for using in the command line parameters which need to use a different encryption key than the one present in the key file or use it when there is not set any encryption key file path. * cmd/uplink: Setup non-interactive accept enc key Change the uplink CLI setup command non-interactive to save the encryption key into a file when it's passed through the flag --enc.encryption-key Previous to this change it wasn't possible to create an key file despite of that the flag was provided, so it was useless on the setup command. * cmd/uplink: Reuse logic to read pwd from terminal Reuse the logic which is already implemented in the pkg/cfgstruct for reading a password from the terminal on interactive mode, rather than duplicating it in the setup command. * cmd/gateway: Use encryption key file flags The cmd/gateway was still using the `enc.key` configuration field which doesn't exist anymore and its setup command wasn't using the `enc.key-filepath` with combination of the `enc.encryption-key` for generating a file with the encryption key. This commit update the cmd/gateway appropriately and move to the uplink package the function used by cmd/uplink to save the encryption key for allowing to also be used by the cmd/gateway without duplicating the logic. * cmd/storj-sim: Adapt gateway config cmd changes Adapt the cmd/storj-sim to correctly pass the parameters to the cmd/gateway setup and run command. * scripts: Don't pass the --enc.encryption-key flag uplink configuration has changed to only support the `--enc.encryption-key` flag for setup commands and consequently the cmd/uplink and cmd/gateway don't accept this flag over other commands, hence the test for the uplink had to be updated for no passing the flag on the multiples calls that the test do to cmd/uplink. * uplink: Remove func which aren't useful anymore Remove the function which allows to user or load an encryption key because it isn't needed anymore since the `--enc.encryption-key` flag is only available for the setup command. Consequently remove its usage from cmd/uplink and cmd/gateway, because such flag will always be empty because in case that's passed Cobra will return an error due to a "unknown flag".
2019-06-07 17:14:40 +01:00
key, err := LoadEncryptionKey(c.Enc.KeyFilepath)
if err != nil {
return nil, nil, Error.Wrap(err)
}
streams, err := streams.NewStreamStore(segments, c.Client.SegmentSize.Int64(), key,
int(blockSize), storj.Cipher(c.Enc.DataType), c.Client.MaxInlineSize.Int(),
)
if err != nil {
return nil, nil, Error.New("failed to create stream store: %v", err)
}
buckets := buckets.NewStore(streams)
return kvmetainfo.New(metainfo, buckets, streams, segments, key, blockSize, rs, c.Client.SegmentSize.Int64()), streams, nil
}
// GetRedundancyScheme returns the configured redundancy scheme for new uploads
func (c Config) GetRedundancyScheme() storj.RedundancyScheme {
return storj.RedundancyScheme{
Algorithm: storj.ReedSolomon,
ShareSize: c.RS.ErasureShareSize.Int32(),
RequiredShares: int16(c.RS.MinThreshold),
RepairShares: int16(c.RS.RepairThreshold),
OptimalShares: int16(c.RS.SuccessThreshold),
TotalShares: int16(c.RS.MaxThreshold),
}
}
// GetPathCipherSuite returns the cipher suite used for path encryption for bucket objects
func (c Config) GetPathCipherSuite() storj.CipherSuite {
return storj.Cipher(c.Enc.PathType).ToCipherSuite()
}
// GetEncryptionScheme returns the configured encryption scheme for new uploads
// Blocksize should align with the stripe size therefore multiples of stripes
// should fit in every encryption block. Instead of lettings users configure this
// multiple value, we hardcode stripesPerBlock as 2 for simplicity.
func (c Config) GetEncryptionScheme() storj.EncryptionScheme {
const stripesPerBlock = 2
return storj.EncryptionScheme{
Cipher: storj.Cipher(c.Enc.DataType),
BlockSize: c.GetRedundancyScheme().StripeSize() * stripesPerBlock,
}
}
// GetSegmentSize returns the segment size set in uplink config
func (c Config) GetSegmentSize() memory.Size {
return c.Client.SegmentSize
}
// LoadEncryptionKey loads the encryption key stored in the file pointed by
// filepath.
//
// An error is file is not found or there is an I/O error.
func LoadEncryptionKey(filepath string) (key *storj.Key, error error) {
if filepath == "" {
return &storj.Key{}, nil
}
rawKey, err := ioutil.ReadFile(filepath)
if err != nil {
return nil, err
}
return storj.NewKey(rawKey)
}