cmd/uplink: use scopes to open (#2501)

What: Change cmd/uplink to use scopes

It moves the fields that will be subsumed by scopes into an explicit legacy section and hides their configuration flags.

Why: So that it can read scopes in from files and stuff

cmd/bootstrap/main.go

@@ -142,7 +142,8 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
 		overrides[kademliaBootstrapAddr.Name] = "127.0.0.1" + defaultServerAddr
 	}
 
-	return process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), overrides)
+	return process.SaveConfig(cmd, filepath.Join(setupDir, "config.yaml"),
+		process.SaveConfigWithOverrides(overrides))
 }
 
 func main() {

cmd/certificates/setup.go

@@ -63,11 +63,11 @@ func cmdSetup(cmd *cobra.Command, args []string) error {
 		return errors.New("identity is missing")
 	}
 
-	overrides := map[string]interface{}{
+	return process.SaveConfig(cmd, filepath.Join(setupDir, "config.yaml"),
-		"ca.cert-path":       config.CA.CertPath,
+		process.SaveConfigWithOverrides(map[string]interface{}{
-		"ca.key-path":        config.CA.KeyPath,
+			"ca.cert-path":       config.CA.CertPath,
-		"identity.cert-path": config.Identity.CertPath,
+			"ca.key-path":        config.CA.KeyPath,
-		"identity.key-path":  config.Identity.KeyPath,
+			"identity.cert-path": config.Identity.CertPath,
-	}
+			"identity.key-path":  config.Identity.KeyPath,
-	return process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), overrides)
+		}))
 }

cmd/gateway/main.go

@@ -27,7 +27,6 @@ import (
 	"storj.io/storj/pkg/process"
 	"storj.io/storj/pkg/storj"
 	"storj.io/storj/uplink"
-	"storj.io/storj/uplink/setup"
 )
 
 // GatewayFlags configuration flags
@@ -107,7 +106,6 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
 		if err != nil {
 			return err
 		}
-
 		overrides[accessKeyFlag.Name] = accessKey
 	}
 
@@ -117,24 +115,13 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
 		if err != nil {
 			return err
 		}
-
 		overrides[secretKeyFlag.Name] = secretKey
 	}
 
-	// override is required because the default value of Enc.KeyFilepath is ""
-	// and setting the value directly in setupCfg.Enc.KeyFiletpath will set the
-	// value in the config file but commented out.
-	encryptionKeyFilepath := setupCfg.Enc.KeyFilepath
-	if encryptionKeyFilepath == "" {
-		encryptionKeyFilepath = filepath.Join(setupDir, ".encryption.key")
-		overrides["enc.key-filepath"] = encryptionKeyFilepath
-	}
-
 	if setupCfg.NonInteractive {
-		return setupCfg.nonInteractive(cmd, setupDir, encryptionKeyFilepath, overrides)
+		return setupCfg.nonInteractive(cmd, setupDir, overrides)
 	}
-
+	return setupCfg.interactive(cmd, setupDir, overrides)
-	return setupCfg.interactive(cmd, setupDir, encryptionKeyFilepath, overrides)
 }
 
 func cmdRun(cmd *cobra.Command, args []string) (err error) {
@@ -233,7 +220,7 @@ func (flags GatewayFlags) action(ctx context.Context, cliCtx *cli.Context) (err
 
 // NewGateway creates a new minio Gateway
 func (flags GatewayFlags) NewGateway(ctx context.Context) (gw minio.Gateway, err error) {
-	access, err := setup.LoadEncryptionAccess(ctx, flags.Enc)
+	scope, err := flags.GetScope()
 	if err != nil {
 		return nil, err
 	}
@@ -245,7 +232,7 @@ func (flags GatewayFlags) NewGateway(ctx context.Context) (gw minio.Gateway, err
 
 	return miniogw.NewStorjGateway(
 		project,
-		access,
+		scope.EncryptionAccess,
 		storj.CipherSuite(flags.Enc.PathType),
 		flags.GetEncryptionParameters(),
 		flags.GetRedundancyScheme(),
@@ -269,26 +256,24 @@ func (flags *GatewayFlags) newUplink(ctx context.Context) (*libuplink.Uplink, er
 }
 
 func (flags GatewayFlags) openProject(ctx context.Context) (*libuplink.Project, error) {
-	apiKey, err := libuplink.ParseAPIKey(flags.Client.APIKey)
+	scope, err := flags.GetScope()
 	if err != nil {
-		return nil, err
+		return nil, Error.Wrap(err)
 	}
-
+	// TODO(jeff): this leaks the uplink and project :(
-	uplk, err := flags.newUplink(ctx)
+	uplink, err := flags.newUplink(ctx)
 	if err != nil {
-		return nil, err
+		return nil, Error.Wrap(err)
 	}
-
+	project, err := uplink.OpenProject(ctx, scope.SatelliteAddr, scope.APIKey)
-	return uplk.OpenProject(ctx, flags.Client.SatelliteAddr, apiKey)
+	if err != nil {
+		return nil, Error.Wrap(err)
+	}
+	return project, nil
 }
 
 // interactive creates the configuration of the gateway interactively.
-//
+func (flags GatewayFlags) interactive(cmd *cobra.Command, setupDir string, overrides map[string]interface{}) error {
-// encryptionKeyFilepath should be set to the filepath indicated by the user or
-// or to a default path whose directory tree exists.
-func (flags GatewayFlags) interactive(
-	cmd *cobra.Command, setupDir string, encryptionKeyFilepath string, overrides map[string]interface{},
-) error {
 	ctx := process.Ctx(cmd)
 
 	satelliteAddress, err := wizard.PromptForSatellite(cmd)
@@ -311,13 +296,13 @@ func (flags GatewayFlags) interactive(
 		return Error.Wrap(err)
 	}
 
-	uplk, err := flags.newUplink(ctx)
+	uplink, err := flags.newUplink(ctx)
 	if err != nil {
 		return Error.Wrap(err)
 	}
-	defer func() { err = errs.Combine(err, uplk.Close()) }()
+	defer func() { err = errs.Combine(err, uplink.Close()) }()
 
-	project, err := uplk.OpenProject(ctx, satelliteAddress, apiKey)
+	project, err := uplink.OpenProject(ctx, satelliteAddress, apiKey)
 	if err != nil {
 		return Error.Wrap(err)
 	}
@@ -328,63 +313,51 @@ func (flags GatewayFlags) interactive(
 		return Error.Wrap(err)
 	}
 
-	err = setup.SaveEncryptionKey(string(key[:]), encryptionKeyFilepath)
+	scopeData, err := (&libuplink.Scope{
+		SatelliteAddr:    satelliteAddress,
+		APIKey:           apiKey,
+		EncryptionAccess: libuplink.NewEncryptionAccessWithDefaultKey(*key),
+	}).Serialize()
+	if err != nil {
+		return Error.Wrap(err)
+	}
+	overrides["scope"] = scopeData
+
+	err = process.SaveConfig(cmd, filepath.Join(setupDir, "config.yaml"),
+		process.SaveConfigWithOverrides(overrides),
+		process.SaveConfigRemovingDeprecated())
 	if err != nil {
 		return Error.Wrap(err)
 	}
 
-	overrides["satellite-addr"] = satelliteAddress
+	fmt.Println(`
-	overrides["api-key"] = apiKeyString
-	overrides["enc.key-filepath"] = encryptionKeyFilepath
-
-	err = process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), overrides)
-	if err != nil {
-		return nil
-	}
-
-	_, err = fmt.Printf(`
-Your encryption key is saved to: %s
-
 Your S3 Gateway is configured and ready to use!
 
 Some things to try next:
 
 * Run 'gateway --help' to see the operations that can be performed
 
-* See https://github.com/storj/docs/blob/master/S3-Gateway.md#using-the-aws-s3-commandline-interface for some example commands
+* See https://github.com/storj/docs/blob/master/S3-Gateway.md#using-the-aws-s3-commandline-interface for some example commands`)
-	`, encryptionKeyFilepath)
-	if err != nil {
-		return nil
-	}
 
 	return nil
-
 }
 
 // nonInteractive creates the configuration of the gateway non-interactively.
-//
+func (flags GatewayFlags) nonInteractive(cmd *cobra.Command, setupDir string, overrides map[string]interface{}) error {
-// encryptionKeyFilepath should be set to the filepath indicated by the user or
+	// ensure we're using the scope for the setup
-// or to a default path whose directory tree exists.
+	scope, err := setupCfg.GetScope()
-func (flags GatewayFlags) nonInteractive(
-	cmd *cobra.Command, setupDir string, encryptionKeyFilepath string, overrides map[string]interface{},
-) error {
-	if setupCfg.Enc.EncryptionKey != "" {
-		err := setup.SaveEncryptionKey(setupCfg.Enc.EncryptionKey, encryptionKeyFilepath)
-		if err != nil {
-			return Error.Wrap(err)
-		}
-	}
-
-	err := process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), overrides)
 	if err != nil {
-		return Error.Wrap(err)
+		return err
 	}
-
+	scopeData, err := scope.Serialize()
-	if setupCfg.Enc.EncryptionKey != "" {
+	if err != nil {
-		_, _ = fmt.Printf("Your encryption key is saved to: %s\n", encryptionKeyFilepath)
+		return err
 	}
+	overrides["scope"] = scopeData
 
-	return nil
+	return Error.Wrap(process.SaveConfig(cmd, filepath.Join(setupDir, "config.yaml"),
+		process.SaveConfigWithOverrides(overrides),
+		process.SaveConfigRemovingDeprecated()))
 }
 
 func main() {

cmd/linksharing/main.go

@@ -114,7 +114,7 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
 		return err
 	}
 
-	return process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), nil)
+	return process.SaveConfig(cmd, filepath.Join(setupDir, "config.yaml"))
 }
 
 func configureTLS(certFile, keyFile string) (*tls.Config, error) {

cmd/satellite/main.go

@@ -172,7 +172,7 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
 		return err
 	}
 
-	return process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), nil)
+	return process.SaveConfig(cmd, filepath.Join(setupDir, "config.yaml"))
 }
 
 func cmdQDiag(cmd *cobra.Command, args []string) (err error) {

cmd/storagenode/main.go

@@ -27,8 +27,7 @@ import (
 
 // StorageNodeFlags defines storage node configuration
 type StorageNodeFlags struct {
-	EditConf        bool `default:"false" help:"open config in default editor"`
+	EditConf bool `default:"false" help:"open config in default editor"`
-	SaveAllDefaults bool `default:"false" help:"save all default values to config.yaml file" setup:"true"`
 
 	storagenode.Config
 }
@@ -197,11 +196,7 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
 	}
 
 	configFile := filepath.Join(setupDir, "config.yaml")
-	if setupCfg.SaveAllDefaults {
+	err = process.SaveConfig(cmd, configFile, process.SaveConfigWithOverrides(overrides))
-		err = process.SaveConfigWithAllDefaults(cmd.Flags(), configFile, overrides)
-	} else {
-		err = process.SaveConfig(cmd.Flags(), configFile, overrides)
-	}
 	if err != nil {
 		return err
 	}

cmd/storj-sim/network.go

@@ -24,7 +24,9 @@ import (
 	"storj.io/storj/internal/dbutil/pgutil"
 	"storj.io/storj/internal/fpath"
 	"storj.io/storj/internal/processgroup"
+	"storj.io/storj/lib/uplink"
 	"storj.io/storj/pkg/identity"
+	"storj.io/storj/pkg/storj"
 )
 
 const (
@@ -34,6 +36,11 @@ const (
 	folderPermissions = 0744
 )
 
+var (
+	defaultAPIKeyData = "13YqgH45XZLg7nm6KsQ72QgXfjbDu2uhTaeSdMVP2A85QuANthM9K58ww5Y4nhMowrZDoqdA4Kyqt1ioQghQcm9fT5uR2drPHpFEqeb"
+	defaultAPIKey, _  = uplink.ParseAPIKey(defaultAPIKeyData)
+)
+
 const (
 	// The following values of peer class and endpoints are used
 	// to create a port with a consistent format for storj-sim services.
@@ -176,7 +183,7 @@ func newNetwork(flags *Flags) (*Processes, error) {
 	withCommon := func(dir string, all Arguments) Arguments {
 		common := []string{"--metrics.app-suffix", "sim", "--log.level", "debug", "--config-dir", dir}
 		if flags.IsDev {
-			common = append(common, "--dev")
+			common = append(common, "--defaults", "dev")
 		}
 		for command, args := range all {
 			all[command] = append(append(common, command), args...)
@@ -311,21 +318,25 @@ func newNetwork(flags *Flags) (*Processes, error) {
 			Extra:      []string{},
 		})
 
+		scopeData, err := (&uplink.Scope{
+			SatelliteAddr:    satellite.Address,
+			APIKey:           defaultAPIKey,
+			EncryptionAccess: uplink.NewEncryptionAccessWithDefaultKey(storj.Key{}),
+		}).Serialize()
+		if err != nil {
+			return nil, err
+		}
+
 		// gateway must wait for the corresponding satellite to start up
 		process.WaitForStart(satellite)
 		process.Arguments = withCommon(process.Directory, Arguments{
 			"setup": {
 				"--non-interactive",
 
-				"--enc.encryption-key=TestEncryptionKey",
+				"--scope", scopeData,
-
 				"--identity-dir", process.Directory,
-				"--satellite-addr", satellite.Address,
-
 				"--server.address", process.Address,
 
-				"--satellite-addr", satellite.Address,
-
 				"--rs.min-threshold", strconv.Itoa(1 * flags.StorageNodeCount / 5),
 				"--rs.repair-threshold", strconv.Itoa(2 * flags.StorageNodeCount / 5),
 				"--rs.success-threshold", strconv.Itoa(3 * flags.StorageNodeCount / 5),
@@ -336,6 +347,7 @@ func newNetwork(flags *Flags) (*Processes, error) {
 
 				"--debug.addr", net.JoinHostPort(host, port(gatewayPeer, i, debugHTTP)),
 			},
+
 			"run": {},
 		})
 
@@ -356,12 +368,11 @@ func newNetwork(flags *Flags) (*Processes, error) {
 			// check if gateway config has an api key, if it's not
 			// create example project with key and add it to the config
 			// so that gateway can have access to the satellite
-			apiKey := vip.GetString("api-key")
+			if runScopeData := vip.GetString("scope"); !flags.OnlyEnv && runScopeData == scopeData {
-			if !flags.OnlyEnv && apiKey == "" {
 				var consoleAddress string
-				satelliteConfigErr := readConfigString(&consoleAddress, satellite.Directory, "console.address")
+				err := readConfigString(&consoleAddress, satellite.Directory, "console.address")
-				if satelliteConfigErr != nil {
+				if err != nil {
-					return satelliteConfigErr
+					return err
 				}
 
 				host := "http://" + consoleAddress
@@ -372,19 +383,32 @@ func newNetwork(flags *Flags) (*Processes, error) {
 				// wait for console server to start
 				time.Sleep(3 * time.Second)
 
+				var apiKey string
 				if err := addExampleProjectWithKey(&apiKey, createRegistrationTokenAddress, consoleActivationAddress, consoleAPIAddress); err != nil {
 					return err
 				}
 
-				vip.Set("api-key", apiKey)
+				scope, err := uplink.ParseScope(runScopeData)
+				if err != nil {
+					return err
+				}
+				scope.APIKey, err = uplink.ParseAPIKey(apiKey)
+				if err != nil {
+					return err
+				}
+				scopeData, err := scope.Serialize()
+				if err != nil {
+					return err
+				}
+				vip.Set("scope", scopeData)
 
 				if err := vip.WriteConfig(); err != nil {
 					return err
 				}
 			}
 
-			if apiKey != "" {
+			if runScopeData := vip.GetString("scope"); runScopeData != scopeData {
-				process.Extra = append(process.Extra, "API_KEY="+apiKey)
+				process.Extra = append(process.Extra, "SCOPE="+runScopeData)
 			}
 
 			accessKey := vip.GetString("minio.access-key")
@@ -429,8 +453,8 @@ func newNetwork(flags *Flags) (*Processes, error) {
 				"--kademlia.operator.email", fmt.Sprintf("storage%d@mail.test", i),
 				"--kademlia.operator.wallet", "0x0123456789012345678901234567890123456789",
 
-				"--storage2.monitor.minimum-disk-space", "10GB",
+				"--storage2.monitor.minimum-disk-space", "0",
-				"--storage2.monitor.minimum-bandwidth", "10GB",
+				"--storage2.monitor.minimum-bandwidth", "0",
 
 				"--server.extensions.revocation=false",
 				"--server.use-peer-ca-whitelist=false",
@@ -461,7 +485,6 @@ func newNetwork(flags *Flags) (*Processes, error) {
 		}
 
 		process.ExecBefore["run"] = func(process *Process) error {
-
 			return readConfigString(&process.Address, process.Directory, "server.address")
 		}
 	}

cmd/uplink/cmd/cp.go

@@ -19,7 +19,6 @@ import (
 	"storj.io/storj/internal/fpath"
 	libuplink "storj.io/storj/lib/uplink"
 	"storj.io/storj/pkg/process"
-	"storj.io/storj/uplink/setup"
 )
 
 var (
@@ -83,16 +82,10 @@ func upload(ctx context.Context, src fpath.FPath, dst fpath.FPath, showProgress
 		return fmt.Errorf("source cannot be a directory: %s", src)
 	}
 
-	access, err := setup.LoadEncryptionAccess(ctx, cfg.Enc)
+	project, bucket, err := cfg.GetProjectAndBucket(ctx, dst.Bucket())
 	if err != nil {
 		return err
 	}
-
-	project, bucket, err := cfg.GetProjectAndBucket(ctx, dst.Bucket(), access)
-	if err != nil {
-		return err
-	}
-
 	defer closeProjectAndBucket(project, bucket)
 
 	reader := io.Reader(file)
@@ -136,16 +129,10 @@ func download(ctx context.Context, src fpath.FPath, dst fpath.FPath, showProgres
 		return fmt.Errorf("destination must be local path: %s", dst)
 	}
 
-	access, err := setup.LoadEncryptionAccess(ctx, cfg.Enc)
+	project, bucket, err := cfg.GetProjectAndBucket(ctx, src.Bucket())
 	if err != nil {
 		return err
 	}
-
-	project, bucket, err := cfg.GetProjectAndBucket(ctx, src.Bucket(), access)
-	if err != nil {
-		return err
-	}
-
 	defer closeProjectAndBucket(project, bucket)
 
 	object, err := bucket.OpenObject(ctx, src.Path())
@@ -215,16 +202,10 @@ func copyObject(ctx context.Context, src fpath.FPath, dst fpath.FPath) (err erro
 		return fmt.Errorf("destination must be Storj URL: %s", dst)
 	}
 
-	access, err := setup.LoadEncryptionAccess(ctx, cfg.Enc)
+	project, bucket, err := cfg.GetProjectAndBucket(ctx, dst.Bucket())
 	if err != nil {
 		return err
 	}
-
-	project, bucket, err := cfg.GetProjectAndBucket(ctx, dst.Bucket(), access)
-	if err != nil {
-		return err
-	}
-
 	defer closeProjectAndBucket(project, bucket)
 
 	object, err := bucket.OpenObject(ctx, src.Path())

cmd/uplink/cmd/ls.go

@@ -14,7 +14,6 @@ import (
 	libuplink "storj.io/storj/lib/uplink"
 	"storj.io/storj/pkg/process"
 	"storj.io/storj/pkg/storj"
-	"storj.io/storj/uplink/setup"
 )
 
 var (
@@ -43,7 +42,7 @@ func list(cmd *cobra.Command, args []string) error {
 		}
 	}()
 
-	access, err := setup.LoadEncryptionAccess(ctx, cfg.Enc)
+	scope, err := cfg.GetScope()
 	if err != nil {
 		return err
 	}
@@ -59,11 +58,10 @@ func list(cmd *cobra.Command, args []string) error {
 			return fmt.Errorf("No bucket specified, use format sj://bucket/")
 		}
 
-		bucket, err := project.OpenBucket(ctx, src.Bucket(), access)
+		bucket, err := project.OpenBucket(ctx, src.Bucket(), scope.EncryptionAccess)
 		if err != nil {
 			return err
 		}
-
 		defer func() {
 			if err := bucket.Close(); err != nil {
 				fmt.Printf("error closing bucket: %+v\n", err)
@@ -71,7 +69,6 @@ func list(cmd *cobra.Command, args []string) error {
 		}()
 
 		err = listFiles(ctx, bucket, src, false)
-
 		return convertError(err, src)
 	}
 
@@ -92,7 +89,7 @@ func list(cmd *cobra.Command, args []string) error {
 			for _, bucket := range list.Items {
 				fmt.Println("BKT", formatTime(bucket.Created), bucket.Name)
 				if *recursiveFlag {
-					if err := listFilesFromBucket(ctx, project, bucket.Name, access); err != nil {
+					if err := listFilesFromBucket(ctx, project, bucket.Name, scope.EncryptionAccess); err != nil {
 						return err
 					}
 				}
@@ -122,7 +119,6 @@ func listFilesFromBucket(ctx context.Context, project *libuplink.Project, bucket
 	if err != nil {
 		return err
 	}
-
 	defer func() {
 		if err := bucket.Close(); err != nil {
 			fmt.Printf("error closing bucket: %+v\n", err)

cmd/uplink/cmd/rb.go

@@ -11,7 +11,6 @@ import (
 	"storj.io/storj/internal/fpath"
 	"storj.io/storj/pkg/process"
 	"storj.io/storj/pkg/storj"
-	"storj.io/storj/uplink/setup"
 )
 
 func init() {
@@ -42,16 +41,10 @@ func deleteBucket(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("Nested buckets not supported, use format sj://bucket/")
 	}
 
-	access, err := setup.LoadEncryptionAccess(ctx, cfg.Enc)
+	project, bucket, err := cfg.GetProjectAndBucket(ctx, dst.Bucket())
-	if err != nil {
-		return err
-	}
-
-	project, bucket, err := cfg.GetProjectAndBucket(ctx, dst.Bucket(), access)
 	if err != nil {
 		return convertError(err, dst)
 	}
-
 	defer closeProjectAndBucket(project, bucket)
 
 	list, err := bucket.ListObjects(ctx, &storj.ListOptions{Direction: storj.After, Recursive: true, Limit: 1})

cmd/uplink/cmd/rm.go

@@ -10,7 +10,6 @@ import (
 
 	"storj.io/storj/internal/fpath"
 	"storj.io/storj/pkg/process"
-	"storj.io/storj/uplink/setup"
 )
 
 func init() {
@@ -37,16 +36,10 @@ func deleteObject(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("No bucket specified, use format sj://bucket/")
 	}
 
-	access, err := setup.LoadEncryptionAccess(ctx, cfg.Enc)
+	project, bucket, err := cfg.GetProjectAndBucket(ctx, dst.Bucket())
-	if err != nil {
-		return err
-	}
-
-	project, bucket, err := cfg.GetProjectAndBucket(ctx, dst.Bucket(), access)
 	if err != nil {
 		return convertError(err, dst)
 	}
-
 	defer closeProjectAndBucket(project, bucket)
 
 	err = bucket.DeleteObject(ctx, dst.Path())

cmd/uplink/cmd/root.go

@@ -63,7 +63,6 @@ func addCmd(cmd *cobra.Command, root *cobra.Command) *cobra.Command {
 	root.AddCommand(cmd)
 
 	defaultConfDir := fpath.ApplicationDir("storj", "uplink")
-
 	confDirParam := cfgstruct.FindConfigDirParam()
 	if confDirParam != "" {
 		defaultConfDir = confDirParam
@@ -92,13 +91,13 @@ func (cliCfg *UplinkFlags) NewUplink(ctx context.Context) (*libuplink.Uplink, er
 }
 
 // GetProject returns a *libuplink.Project for interacting with a specific project
-func (cliCfg *UplinkFlags) GetProject(ctx context.Context) (*libuplink.Project, error) {
+func (cliCfg *UplinkFlags) GetProject(ctx context.Context) (_ *libuplink.Project, err error) {
-	err := version.CheckProcessVersion(ctx, zap.L(), cliCfg.Version, version.Build, "Uplink")
+	err = version.CheckProcessVersion(ctx, zap.L(), cliCfg.Version, version.Build, "Uplink")
 	if err != nil {
 		return nil, err
 	}
 
-	apiKey, err := libuplink.ParseAPIKey(cliCfg.Client.APIKey)
+	scope, err := cliCfg.GetScope()
 	if err != nil {
 		return nil, err
 	}
@@ -107,24 +106,40 @@ func (cliCfg *UplinkFlags) GetProject(ctx context.Context) (*libuplink.Project,
 	if err != nil {
 		return nil, err
 	}
-
+	defer func() {
-	project, err := uplk.OpenProject(ctx, cliCfg.Client.SatelliteAddr, apiKey)
+		if err != nil {
-	if err != nil {
+			if err := uplk.Close(); err != nil {
-		if err := uplk.Close(); err != nil {
+				fmt.Printf("error closing uplink: %+v\n", err)
-			fmt.Printf("error closing uplink: %+v\n", err)
+			}
 		}
-	}
+	}()
 
-	return project, err
+	return uplk.OpenProject(ctx, scope.SatelliteAddr, scope.APIKey)
 }
 
 // GetProjectAndBucket returns a *libuplink.Bucket for interacting with a specific project's bucket
-func (cliCfg *UplinkFlags) GetProjectAndBucket(ctx context.Context, bucketName string, access *libuplink.EncryptionAccess) (project *libuplink.Project, bucket *libuplink.Bucket, err error) {
+func (cliCfg *UplinkFlags) GetProjectAndBucket(ctx context.Context, bucketName string) (project *libuplink.Project, bucket *libuplink.Bucket, err error) {
-	project, err = cliCfg.GetProject(ctx)
+	scope, err := cliCfg.GetScope()
 	if err != nil {
-		return project, bucket, err
+		return nil, nil, err
 	}
 
+	uplk, err := cliCfg.NewUplink(ctx)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer func() {
+		if err != nil {
+			if err := uplk.Close(); err != nil {
+				fmt.Printf("error closing uplink: %+v\n", err)
+			}
+		}
+	}()
+
+	project, err = uplk.OpenProject(ctx, scope.SatelliteAddr, scope.APIKey)
+	if err != nil {
+		return nil, nil, err
+	}
 	defer func() {
 		if err != nil {
 			if err := project.Close(); err != nil {
@@ -133,11 +148,7 @@ func (cliCfg *UplinkFlags) GetProjectAndBucket(ctx context.Context, bucketName s
 		}
 	}()
 
-	bucket, err = project.OpenBucket(ctx, bucketName, access)
+	bucket, err = project.OpenBucket(ctx, bucketName, scope.EncryptionAccess)
-	if err != nil {
-		return project, bucket, err
-	}
-
 	return project, bucket, err
 }
 

cmd/uplink/cmd/setup.go

@@ -18,7 +18,6 @@ import (
 	libuplink "storj.io/storj/lib/uplink"
 	"storj.io/storj/pkg/cfgstruct"
 	"storj.io/storj/pkg/process"
-	"storj.io/storj/uplink/setup"
 )
 
 var (
@@ -37,12 +36,6 @@ func init() {
 }
 
 func cmdSetup(cmd *cobra.Command, args []string) (err error) {
-	// Ensure use the default port if the user only specifies a host.
-	err = ApplyDefaultHostAndPortToAddrFlag(cmd, "satellite-addr")
-	if err != nil {
-		return err
-	}
-
 	setupDir, err := filepath.Abs(confDir)
 	if err != nil {
 		return err
@@ -58,61 +51,59 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
 		return err
 	}
 
-	// override is required because the default value of Enc.KeyFilepath is ""
-	// and setting the value directly in setupCfg.Enc.KeyFiletpathon will set the
-	// value in the config file but commented out.
-	usedEncryptionKeyFilepath := setupCfg.Enc.KeyFilepath
-	if usedEncryptionKeyFilepath == "" {
-		usedEncryptionKeyFilepath = filepath.Join(setupDir, ".encryption.key")
-	}
-
 	if setupCfg.NonInteractive {
-		return cmdSetupNonInteractive(cmd, setupDir, usedEncryptionKeyFilepath)
+		return cmdSetupNonInteractive(cmd, setupDir)
 	}
-
+	return cmdSetupInteractive(cmd, setupDir)
-	return cmdSetupInteractive(cmd, setupDir, usedEncryptionKeyFilepath)
 }
 
 // cmdSetupNonInteractive sets up uplink non-interactively.
-//
+func cmdSetupNonInteractive(cmd *cobra.Command, setupDir string) error {
-// encryptionKeyFilepath should be set to the filepath indicated by the user or
+	// ensure we're using the scope for the setup
-// or to a default path whose directory tree exists.
+	scope, err := setupCfg.GetScope()
-func cmdSetupNonInteractive(cmd *cobra.Command, setupDir string, encryptionKeyFilepath string) error {
-	if setupCfg.Enc.EncryptionKey != "" {
-		err := setup.SaveEncryptionKey(setupCfg.Enc.EncryptionKey, encryptionKeyFilepath)
-		if err != nil {
-			return err
-		}
-	}
-
-	override := map[string]interface{}{
-		"enc.key-filepath": encryptionKeyFilepath,
-	}
-
-	err := process.SaveConfigWithAllDefaults(
-		cmd.Flags(), filepath.Join(setupDir, process.DefaultCfgFilename), override)
 	if err != nil {
 		return err
 	}
 
-	if setupCfg.Enc.EncryptionKey != "" {
+	// apply helpful default host and port to the address
-		_, _ = fmt.Printf("Your encryption key is saved to: %s\n", encryptionKeyFilepath)
+	vip, err := process.Viper(cmd)
+	if err != nil {
+		return err
+	}
+	scope.SatelliteAddr, err = ApplyDefaultHostAndPortToAddr(
+		scope.SatelliteAddr, vip.GetString("satellite-addr"))
+	if err != nil {
+		return err
 	}
 
-	return nil
+	scopeData, err := scope.Serialize()
+	if err != nil {
+		return err
+	}
+	return Error.Wrap(process.SaveConfig(cmd, filepath.Join(setupDir, process.DefaultCfgFilename),
+		process.SaveConfigWithOverride("scope", scopeData),
+		process.SaveConfigRemovingDeprecated()))
 }
 
 // cmdSetupInteractive sets up uplink interactively.
-//
+func cmdSetupInteractive(cmd *cobra.Command, setupDir string) error {
-// encryptionKeyFilepath should be set to the filepath indicated by the user or
-// or to a default path whose directory tree exists.
-func cmdSetupInteractive(cmd *cobra.Command, setupDir string, encryptionKeyFilepath string) error {
 	ctx := process.Ctx(cmd)
 
 	satelliteAddress, err := wizard.PromptForSatellite(cmd)
+	if err != nil {
+		return Error.Wrap(err)
+	}
+
+	// apply helpful default host and port to the address
+	vip, err := process.Viper(cmd)
 	if err != nil {
 		return err
 	}
+	satelliteAddress, err = ApplyDefaultHostAndPortToAddr(
+		satelliteAddress, vip.GetString("satellite-addr"))
+	if err != nil {
+		return Error.Wrap(err)
+	}
 
 	apiKeyString, err := wizard.PromptForAPIKey()
 	if err != nil {
@@ -129,13 +120,13 @@ func cmdSetupInteractive(cmd *cobra.Command, setupDir string, encryptionKeyFilep
 		return Error.Wrap(err)
 	}
 
-	uplk, err := setupCfg.NewUplink(ctx)
+	uplink, err := libuplink.NewUplink(ctx, nil)
 	if err != nil {
 		return Error.Wrap(err)
 	}
-	defer func() { err = errs.Combine(err, uplk.Close()) }()
+	defer func() { err = errs.Combine(err, uplink.Close()) }()
 
-	project, err := uplk.OpenProject(ctx, satelliteAddress, apiKey)
+	project, err := uplink.OpenProject(ctx, satelliteAddress, apiKey)
 	if err != nil {
 		return Error.Wrap(err)
 	}
@@ -146,61 +137,36 @@ func cmdSetupInteractive(cmd *cobra.Command, setupDir string, encryptionKeyFilep
 		return Error.Wrap(err)
 	}
 
-	err = setup.SaveEncryptionKey(string(key[:]), encryptionKeyFilepath)
+	scopeData, err := (&libuplink.Scope{
+		SatelliteAddr:    satelliteAddress,
+		APIKey:           apiKey,
+		EncryptionAccess: libuplink.NewEncryptionAccessWithDefaultKey(*key),
+	}).Serialize()
 	if err != nil {
-		return err
+		return Error.Wrap(err)
 	}
 
-	var override = map[string]interface{}{
+	err = process.SaveConfig(cmd, filepath.Join(setupDir, "config.yaml"),
-		"api-key":          apiKeyString,
+		process.SaveConfigWithOverride("scope", scopeData),
-		"satellite-addr":   satelliteAddress,
+		process.SaveConfigRemovingDeprecated())
-		"enc.key-filepath": encryptionKeyFilepath,
-	}
-
-	err = process.SaveConfigWithAllDefaults(
-		cmd.Flags(), filepath.Join(setupDir, process.DefaultCfgFilename), override)
 	if err != nil {
-		return nil
+		return Error.Wrap(err)
 	}
 
 	// if there is an error with this we cannot do that much and the setup process
 	// has ended OK, so we ignore it.
-	_, _ = fmt.Printf(`
+	fmt.Println(`
-Your encryption key is saved to: %s
-
 Your Uplink CLI is configured and ready to use!
 
 Some things to try next:
 
 * Run 'uplink --help' to see the operations that can be performed
 
-* See https://github.com/storj/docs/blob/master/Uplink-CLI.md#usage for some example commands
+* See https://github.com/storj/docs/blob/master/Uplink-CLI.md#usage for some example commands`)
-	`, encryptionKeyFilepath)
 
 	return nil
 }
 
-// ApplyDefaultHostAndPortToAddrFlag applies the default host and/or port if either is missing in the specified flag name.
-func ApplyDefaultHostAndPortToAddrFlag(cmd *cobra.Command, flagName string) error {
-	flag := cmd.Flags().Lookup(flagName)
-	if flag == nil {
-		// No flag found for us to handle.
-		return nil
-	}
-
-	address, err := ApplyDefaultHostAndPortToAddr(flag.Value.String(), flag.DefValue)
-	if err != nil {
-		return Error.Wrap(err)
-	}
-
-	if flag.Value.String() == address {
-		// Don't trip the flag set bit
-		return nil
-	}
-
-	return Error.Wrap(flag.Value.Set(address))
-}
-
 // ApplyDefaultHostAndPortToAddr applies the default host and/or port if either is missing in the specified address.
 func ApplyDefaultHostAndPortToAddr(address, defaultAddress string) (string, error) {
 	defaultHost, defaultPort, err := net.SplitHostPort(defaultAddress)

cmd/uplink/cmd/setup_test.go

@@ -6,113 +6,44 @@ package cmd_test
 import (
 	"testing"
 
-	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
 
 	"storj.io/storj/cmd/uplink/cmd"
 )
 
-func TestDefaultHostAndPortAppliedToSatelliteAddrWithNoHostOrPort(t *testing.T) {
+func TestApplyDefaultHostAndPortToAddr(t *testing.T) {
-	setupCmd := &cobra.Command{
+	{
-		Use:         "setup",
+		got, err := cmd.ApplyDefaultHostAndPortToAddr("", "localhost:7777")
-		Short:       "Create an uplink config file",
+		assert.NoError(t, err)
-		RunE:        nil,
+		assert.Equal(t, "localhost:7777", got,
-		Annotations: map[string]string{"type": "setup"},
+			"satellite-addr should contain default port when no port specified")
 	}
-	flagName := "satellite-addr"
-	defaultValue := "localhost:7777"
-	setupCmd.Flags().String(flagName, defaultValue, "")
 
-	err := setupCmd.Flags().Set(flagName, "")
+	{
-	assert.NoError(t, err)
+		got, err := cmd.ApplyDefaultHostAndPortToAddr("ahost", "localhost:7777")
-
+		assert.NoError(t, err)
-	err = cmd.ApplyDefaultHostAndPortToAddrFlag(setupCmd, flagName)
+		assert.Equal(t, "ahost:7777", got,
-	assert.NoError(t, err)
+			"satellite-addr should contain default port when no port specified")
-
-	assert.Equal(t, "localhost:7777", setupCmd.Flags().Lookup("satellite-addr").Value.String(),
-		"satellite-addr should contain default port when no port specified")
-}
-
-func TestDefaultPortAppliedToSatelliteAddrWithNoPort(t *testing.T) {
-	setupCmd := &cobra.Command{
-		Use:         "setup",
-		Short:       "Create an uplink config file",
-		RunE:        nil,
-		Annotations: map[string]string{"type": "setup"},
 	}
-	flagName := "satellite-addr"
-	defaultValue := "localhost:7777"
-	setupCmd.Flags().String(flagName, defaultValue, "")
 
-	err := setupCmd.Flags().Set(flagName, "ahost")
+	{
-	assert.NoError(t, err)
+		got, err := cmd.ApplyDefaultHostAndPortToAddr("ahost:7778", "localhost:7777")
-
+		assert.NoError(t, err)
-	err = cmd.ApplyDefaultHostAndPortToAddrFlag(setupCmd, flagName)
+		assert.Equal(t, "ahost:7778", got,
-	assert.NoError(t, err)
+			"satellite-addr should contain default port when no port specified")
-
-	assert.Equal(t, "ahost:7777", setupCmd.Flags().Lookup("satellite-addr").Value.String(),
-		"satellite-addr should contain default port when no port specified")
-}
-
-func TestNoDefaultPortAppliedToSatelliteAddrWithPort(t *testing.T) {
-	setupCmd := &cobra.Command{
-		Use:         "setup",
-		Short:       "Create an uplink config file",
-		RunE:        nil,
-		Annotations: map[string]string{"type": "setup"},
 	}
-	flagName := "satellite-addr"
-	defaultValue := "localhost:7777"
-	setupCmd.Flags().String(flagName, defaultValue, "")
 
-	err := setupCmd.Flags().Set(flagName, "ahost:7778")
+	{
-	assert.NoError(t, err)
+		got, err := cmd.ApplyDefaultHostAndPortToAddr(":7778", "localhost:7777")
-
+		assert.NoError(t, err)
-	err = cmd.ApplyDefaultHostAndPortToAddrFlag(setupCmd, flagName)
+		assert.Equal(t, "localhost:7778", got,
-	assert.NoError(t, err)
+			"satellite-addr should contain default port when no port specified")
-
-	assert.Equal(t, "ahost:7778", setupCmd.Flags().Lookup(flagName).Value.String(),
-		"satellite-addr should contain default port when no port specified")
-}
-
-func TestDefaultHostAppliedToSatelliteAddrWithNoHost(t *testing.T) {
-	setupCmd := &cobra.Command{
-		Use:         "setup",
-		Short:       "Create an uplink config file",
-		RunE:        nil,
-		Annotations: map[string]string{"type": "setup"},
 	}
-	flagName := "satellite-addr"
-	defaultValue := "localhost:7777"
-	setupCmd.Flags().String(flagName, defaultValue, "")
 
-	err := setupCmd.Flags().Set(flagName, ":7778")
+	{
-	assert.NoError(t, err)
+		got, err := cmd.ApplyDefaultHostAndPortToAddr("ahost:", "localhost:7777")
-
+		assert.NoError(t, err)
-	err = cmd.ApplyDefaultHostAndPortToAddrFlag(setupCmd, flagName)
+		assert.Equal(t, "ahost:7777", got,
-	assert.NoError(t, err)
+			"satellite-addr should contain default port when no port specified")
-
-	assert.Equal(t, "localhost:7778", setupCmd.Flags().Lookup("satellite-addr").Value.String(),
-		"satellite-addr should contain default port when no port specified")
-}
-
-func TestDefaultPortAppliedToSatelliteAddrWithPortColonButNoPort(t *testing.T) {
-	setupCmd := &cobra.Command{
-		Use:         "setup",
-		Short:       "Create an uplink config file",
-		RunE:        nil,
-		Annotations: map[string]string{"type": "setup"},
 	}
-	flagName := "satellite-addr"
-	defaultValue := "localhost:7777"
-	setupCmd.Flags().String(flagName, defaultValue, "")
-
-	err := setupCmd.Flags().Set(flagName, "ahost:")
-	assert.NoError(t, err)
-
-	err = cmd.ApplyDefaultHostAndPortToAddrFlag(setupCmd, flagName)
-	assert.NoError(t, err)
-
-	assert.Equal(t, "ahost:7777", setupCmd.Flags().Lookup("satellite-addr").Value.String(),
-		"satellite-addr should contain default port when no port specified")
 }

cmd/uplink/cmd/share.go

@@ -12,11 +12,11 @@ import (
 	"github.com/zeebo/errs"
 
 	"storj.io/storj/internal/fpath"
-	"storj.io/storj/lib/uplink"
 	libuplink "storj.io/storj/lib/uplink"
+	"storj.io/storj/pkg/cfgstruct"
 	"storj.io/storj/pkg/macaroon"
 	"storj.io/storj/pkg/process"
-	"storj.io/storj/uplink/setup"
+	"storj.io/storj/uplink"
 )
 
 var shareCfg struct {
@@ -29,19 +29,29 @@ var shareCfg struct {
 	NotBefore         string   `help:"disallow access before this time"`
 	NotAfter          string   `help:"disallow access after this time"`
 	AllowedPathPrefix []string `help:"whitelist of bucket path prefixes to require"`
+
+	// Share requires information about the current scope
+	uplink.ScopeConfig
 }
 
 func init() {
-	// sadly, we have to use addCmd so that it adds the cfg struct to the flags
+	// We skip the use of addCmd here because we only want the configuration options listed
-	// so that we can open projects and buckets. that pulls in so many unnecessary
+	// above, and addCmd adds a whole lot more than we want.
-	// flags which makes figuring out the share command really hard. oh well.
+
-	shareCmd := addCmd(&cobra.Command{
+	shareCmd := &cobra.Command{
 		Use:   "share",
 		Short: "Creates a possibly restricted api key",
 		RunE:  shareMain,
-	}, RootCmd)
+	}
+	RootCmd.AddCommand(shareCmd)
 
-	process.Bind(shareCmd, &shareCfg)
+	defaultConfDir := fpath.ApplicationDir("storj", "uplink")
+	confDirParam := cfgstruct.FindConfigDirParam()
+	if confDirParam != "" {
+		defaultConfDir = confDirParam
+	}
+
+	process.Bind(shareCmd, &shareCfg, defaults, cfgstruct.ConfDir(defaultConfDir))
 }
 
 const shareISO8601 = "2006-01-02T15:04:05-0700"
@@ -68,7 +78,6 @@ func parseHumanDate(date string, now time.Time) (*time.Time, error) {
 
 // shareMain is the function executed when shareCmd is called
 func shareMain(cmd *cobra.Command, args []string) (err error) {
-	ctx := process.Ctx(cmd)
 	now := time.Now()
 
 	notBefore, err := parseHumanDate(shareCfg.NotBefore, now)
@@ -96,15 +105,11 @@ func shareMain(cmd *cobra.Command, args []string) (err error) {
 		})
 	}
 
-	key, err := libuplink.ParseAPIKey(cfg.Client.APIKey)
+	scope, err := shareCfg.GetScope()
-	if err != nil {
-		return err
-	}
-
-	access, err := setup.LoadEncryptionAccess(ctx, cfg.Enc)
 	if err != nil {
 		return err
 	}
+	key, access := scope.APIKey, scope.EncryptionAccess
 
 	if len(restrictions) > 0 {
 		key, access, err = access.Restrict(key, restrictions...)
@@ -152,13 +157,13 @@ func shareMain(cmd *cobra.Command, args []string) (err error) {
 		return err
 	}
 
-	scope := &uplink.Scope{
+	newScope := &libuplink.Scope{
-		SatelliteAddr:    cfg.Client.SatelliteAddr,
+		SatelliteAddr:    scope.SatelliteAddr,
 		APIKey:           key,
 		EncryptionAccess: access,
 	}
 
-	scopeData, err := scope.Serialize()
+	scopeData, err := newScope.Serialize()
 	if err != nil {
 		return err
 	}

cmd/versioncontrol/main.go

@@ -88,7 +88,8 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
 		overrides[serverAddress.Name] = defaultServerAddr
 	}
 
-	return process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), overrides)
+	return process.SaveConfig(cmd, filepath.Join(setupDir, "config.yaml"),
+		process.SaveConfigWithOverrides(overrides))
 }
 
 func main() {

go.mod

@@ -11,7 +11,6 @@ require (
 	github.com/mattn/go-colorable v0.0.9 // indirect
 
 	github.com/minio/minio v0.0.0-20180508161510-54cd29b51c38
-	github.com/mitchellh/mapstructure v1.1.1 // indirect
 	github.com/segmentio/go-prompt v1.2.1-0.20161017233205-f0d19b6901ad
 )
 
@@ -25,7 +24,6 @@ require (
 	github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect
 	github.com/boltdb/bolt v1.3.1
 	github.com/cheggaaa/pb v1.0.5-0.20160713104425-73ae1d68fe0b
-	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
 	github.com/djherbis/atime v1.0.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/eapache/go-resiliency v1.1.0 // indirect
@@ -87,16 +85,16 @@ require (
 	github.com/pierrec/lz4 v2.0.5+incompatible // indirect
 	github.com/pkg/errors v0.8.1 // indirect
 	github.com/pkg/profile v1.2.1 // indirect
-	github.com/prometheus/client_golang v0.9.3 // indirect
 	github.com/prometheus/procfs v0.0.0-20190517135640-51af30a78b0e // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a // indirect
 	github.com/rs/cors v1.5.0 // indirect
 	github.com/sirupsen/logrus v1.3.0 // indirect
 	github.com/skyrings/skyring-common v0.0.0-20160929130248-d1c0bb1cbd5e
 	github.com/spacemonkeygo/errors v0.0.0-20171212215202-9064522e9fd1 // indirect
+	github.com/spf13/cast v1.3.0
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/pflag v1.0.3
-	github.com/spf13/viper v1.2.1
+	github.com/spf13/viper v1.4.0
 	github.com/streadway/amqp v0.0.0-20180806233856-70e15c650864 // indirect
 	github.com/stretchr/testify v1.3.0
 	github.com/stripe/stripe-go v60.17.0+incompatible
@@ -109,16 +107,12 @@ require (
 	github.com/zeebo/float16 v0.1.0 // indirect
 	github.com/zeebo/incenc v0.0.0-20180505221441-0d92902eec54 // indirect
 	github.com/zeebo/structs v1.0.2
-	go.etcd.io/bbolt v1.3.2 // indirect
-	go.uber.org/atomic v1.3.2 // indirect
-	go.uber.org/multierr v1.1.0 // indirect
 	go.uber.org/zap v1.10.0
 	golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8
 	golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 // indirect
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/sys v0.0.0-20190730183949-1393eb018365
 	golang.org/x/text v0.3.2 // indirect
-	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
 	golang.org/x/tools v0.0.0-20190614152001-1edc8e83c897
 	google.golang.org/appengine v1.6.0 // indirect
 	google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610 // indirect
@@ -127,5 +121,5 @@ require (
 	gopkg.in/cheggaaa/pb.v1 v1.0.25 // indirect
 	gopkg.in/olivere/elastic.v5 v5.0.76 // indirect
 	gopkg.in/spacemonkeygo/monkit.v2 v2.0.0-20190612171030-cf5a9e6f8fd2
-	gopkg.in/yaml.v2 v2.2.2 // indirect
+	gopkg.in/yaml.v2 v2.2.2
 )

go.sum

@@ -20,6 +20,7 @@ github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 h1:45bxf7AZMw
 github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
 github.com/alicebob/miniredis v0.0.0-20180911162847-3657542c8629 h1:gLoh8jzwIxdisBnHiWRIuReqtH9cpslSE2564UWXun0=
 github.com/alicebob/miniredis v0.0.0-20180911162847-3657542c8629/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/aws/aws-sdk-go v1.15.34/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
@@ -40,6 +41,11 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cloudfoundry/gosigar v1.1.0 h1:V/dVCzhKOdIU3WRB5inQU20s4yIgL9Dxx/Mhi0SF8eM=
 github.com/cloudfoundry/gosigar v1.1.0/go.mod h1:3qLfc2GlfmwOx2+ZDaRGH3Y9fwQ0sQeaAleo2GV5pH0=
 github.com/cockroachdb/cockroach-go v0.0.0-20180212155653-59c0560478b7/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8=
 github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f/go.mod h1:8S58EK26zhXSxzv7NQFpnliaOQsmDUxvoQO3rt154Vg=
 github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc=
@@ -92,6 +98,7 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/fsouza/fake-gcs-server v1.2.0/go.mod h1:rM69NBSmfAkTlKDhzXC41OeX9lQxQXnkimkiGWDU1Ek=
 github.com/garyburd/redigo v1.0.1-0.20170216214944-0d253a66e6e1 h1:YmyuMm99D7kezPc0ZVWYnaUIWfMKR81lVVXttKTnDbw=
 github.com/garyburd/redigo v1.0.1-0.20170216214944-0d253a66e6e1/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-ini/ini v1.38.2 h1:6Hl/z3p3iFkA0dlDfzYxuFuUGD+kaweypF6btsR2/Q4=
 github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
@@ -111,6 +118,7 @@ github.com/golang-migrate/migrate/v3 v3.5.2 h1:SUWSv6PD8Lr2TGx1lmVW7W2lRoQiVny3s
 github.com/golang-migrate/migrate/v3 v3.5.2/go.mod h1:QDa9JH6Bdwbi+1jIEpOIWnTNoYRbtaVgByErXU0844E=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -127,6 +135,7 @@ github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
 github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
@@ -153,8 +162,12 @@ github.com/gorilla/rpc v1.1.0 h1:marKfvVP0Gpd/jHlVBKCQ8RAoUPdX7K1Nuh6l1BNh7A=
 github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
 github.com/gorilla/schema v1.1.0 h1:CamqUDOFUBqzrvxuz2vEwo8+SUdwsluFh7IlzJh30LY=
 github.com/gorilla/schema v1.1.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/graphql-go/graphql v0.7.9-0.20190403165646-199d20bbfed7 h1:E45QFM7IqRdFnuyFk8GSamb42EckUSyJ55rtVB/w8VQ=
 github.com/graphql-go/graphql v0.7.9-0.20190403165646-199d20bbfed7/go.mod h1:k6yrAYQaSP59DC5UVxbgxESlmVyojThKdORUqGDGmrI=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
 github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
@@ -182,6 +195,7 @@ github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6 h1:4zOlv2my+vf98j
 github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6/go.mod h1:r/8JmuR0qjuCiEhAolkfvdZgmPiHTnJaG0UXCSeR1Zo=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/jtolds/go-luar v0.0.0-20170419063437-0786921db8c0 h1:UyVaeqfY1fLPMt1iUTaWsxUNxYAzZVyK+7G+a3sRfhk=
@@ -247,10 +261,8 @@ github.com/minio/sio v0.0.0-20180327104954-6a41828a60f0 h1:ys4bbOlPvaUBlA0byjm6T
 github.com/minio/sio v0.0.0-20180327104954-6a41828a60f0/go.mod h1:PDJGYr8GXjiOTIst0hQMOSK5FdXLwObr2cGbiMddDPc=
 github.com/mitchellh/go-homedir v0.0.0-20180801233206-58046073cbff h1:jM4Eo4qMmmcqePS3u6X2lcEELtVuXWkWJIS/pRI3oSk=
 github.com/mitchellh/go-homedir v0.0.0-20180801233206-58046073cbff/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/mapstructure v1.0.0 h1:vVpGvMXJPqSDh2VYHF7gsfQj8Ncx+Xw5Y1KHeTRY+7I=
+github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
-github.com/mitchellh/mapstructure v1.0.0/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.1 h1:0fcGQkeJPHl7DauilpdNG27ZxXHDSg+rbbTpfpniZd8=
-github.com/mitchellh/mapstructure v1.1.1/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nats-io/gnatsd v1.3.0 h1:+5d80klu3QaJgNbdavVBjWJP7cHd11U2CLnRTFM9ICI=
 github.com/nats-io/gnatsd v1.3.0/go.mod h1:nqco77VO78hLCJpIcVfygDP2rPGfsEHkGTUk94uh5DQ=
@@ -317,6 +329,7 @@ github.com/prometheus/procfs v0.0.0-20190517135640-51af30a78b0e/go.mod h1:TjEm7z
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rs/cors v1.5.0 h1:dgSHE6+ia18arGOTIYQKKGWLvEbGvmbNE6NfxhoNHUY=
 github.com/rs/cors v1.5.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/segmentio/go-prompt v1.2.1-0.20161017233205-f0d19b6901ad h1:EqOdoSJGI7CsBQczPcIgmpm3hJE7X8Hj3jrgI002whs=
@@ -333,6 +346,7 @@ github.com/smartystreets/assertions v0.0.0-20180820201707-7c9eb446e3cf/go.mod h1
 github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
 github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a h1:JSvGDIbmil4Ui/dDdFBExb7/cmkNjyX5F97oglmvCDo=
 github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spacemonkeygo/errors v0.0.0-20171212215202-9064522e9fd1 h1:xHQewZjohU9/wUsyC99navCjQDNHtTgUOM/J1jAbzfw=
 github.com/spacemonkeygo/errors v0.0.0-20171212215202-9064522e9fd1/go.mod h1:7NL9UAYQnRM5iKHUCld3tf02fKb5Dft+41+VckASUy0=
 github.com/spacemonkeygo/monotime v0.0.0-20180824235756-e3f48a95f98a h1:8+cCjxhToanKmxLIbuyBNe2EnpgwhiivsIaRJstDRFA=
@@ -342,20 +356,16 @@ github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg=
-github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
 github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc=
-github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/viper v1.2.1 h1:bIcUwXqLseLF3BDAZduuNfekWG87ibtFxi59Bq+oI9M=
+github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
-github.com/spf13/viper v1.2.1/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI=
+github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/streadway/amqp v0.0.0-20180806233856-70e15c650864 h1:Oj3PUEs+OUSYUpn35O+BE/ivHGirKixA3+vqA0Atu9A=
 github.com/streadway/amqp v0.0.0-20180806233856-70e15c650864/go.mod h1:1WNBiOZtZQLpVAyu0iTduoJL9hEsMloAK5XWrtW0xdY=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -371,8 +381,12 @@ github.com/tidwall/gjson v1.1.3 h1:u4mspaByxY+Qk4U1QYYVzGFI8qxN/3jtEV0ZDb2vRic=
 github.com/tidwall/gjson v1.1.3/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA=
 github.com/tidwall/match v0.0.0-20171002075945-1731857f09b1 h1:pWIN9LOlFRCJFqWIOEbHLvY0WWJddsjH2FQ6N0HKZdU=
 github.com/tidwall/match v0.0.0-20171002075945-1731857f09b1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/vivint/infectious v0.0.0-20190108171102-2455b059135b h1:dLkqBELopfQNhe8S9ucnSf+HhiUCgK/hPIjVG0f9GlY=
 github.com/vivint/infectious v0.0.0-20190108171102-2455b059135b/go.mod h1:5oyMAv4hrBEKqBwORFsiqIrCNCmL2qcZLQTdJLYeYIc=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/gopher-lua v0.0.0-20180918061612-799fa34954fb h1:Jmfk7z2f/+gxVFAgPsJMuczO1uEIxZy6wytTdeZ49lg=
 github.com/yuin/gopher-lua v0.0.0-20180918061612-799fa34954fb/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU=
 github.com/zeebo/admission v0.0.0-20180821192747-f24f2a94a40c h1:WoYvMZp+keiJz+ZogLAhwsUZvWe81W+mCnpfdgEUOl4=
@@ -393,8 +407,8 @@ go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=
 go.opencensus.io v0.16.0/go.mod h1:0TeCCqcQSLNZtiq/62+vUzqwnjqF5el6hjmuZaFtyNk=
-go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
+go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
-go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
@@ -415,9 +429,13 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
@@ -433,7 +451,6 @@ golang.org/x/sys v0.0.0-20180824143301-4910a1d54f87 h1:GqwDwfvIpC33dK9bA1fD+JiDU
 golang.org/x/sys v0.0.0-20180824143301-4910a1d54f87/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -479,6 +496,9 @@ google.golang.org/grpc v1.15.0 h1:Az/KuahOM4NAidTEuJCv/RonAA7rYsTPkqXVjr+8OOw=
 google.golang.org/grpc v1.15.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.22.0 h1:J0UbZOIrCAl+fpTOf8YLs4dJo8L/owV4LYVtAXQoPkw=
+google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.22.1 h1:/7cs52RnTJmD43s3uxzlq2U7nqVTd/37viQwMrMNlOM=
 google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 gopkg.in/Shopify/sarama.v1 v1.18.0 h1:f9aTXuIEFEjVvLG9p+kMSk01dMfFumHsySRk1okTdqU=
@@ -500,12 +520,14 @@ gopkg.in/ini.v1 v1.38.2 h1:dGcbywv4RufeGeiMycPT/plKB5FtmLKLnWKwBiLhUA4=
 gopkg.in/ini.v1 v1.38.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/olivere/elastic.v5 v5.0.76 h1:A6W7X4yLPQDINHiYAqIwqev+rD5hIQ4G0e1d5H//VXk=
 gopkg.in/olivere/elastic.v5 v5.0.76/go.mod h1:uhHoB4o3bvX5sorxBU29rPcmBQdV2Qfg0FBrx5D6pV0=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/spacemonkeygo/monkit.v2 v2.0.0-20180827161543-6ebf5a752f9b h1:zkhw+LuPvld5/ZwIlpYzd4DX/r0KxTvhEpo+2lcrp9k=
 gopkg.in/spacemonkeygo/monkit.v2 v2.0.0-20180827161543-6ebf5a752f9b/go.mod h1:6UQdi0rNB4YDNwVYP8fJNmjLNaU8MA84WCdXStx3Spo=
 gopkg.in/spacemonkeygo/monkit.v2 v2.0.0-20190612171030-cf5a9e6f8fd2 h1:Hzf9SARsnWv8oWl9TWmQ3vo3lSqOMByumL6aJ7ehw7g=
 gopkg.in/spacemonkeygo/monkit.v2 v2.0.0-20190612171030-cf5a9e6f8fd2/go.mod h1:6UQdi0rNB4YDNwVYP8fJNmjLNaU8MA84WCdXStx3Spo=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=

internal/testplanet/uplink.go

@@ -28,7 +28,6 @@ import (
 	"storj.io/storj/uplink"
 	"storj.io/storj/uplink/metainfo"
 	"storj.io/storj/uplink/piecestore"
-	"storj.io/storj/uplink/setup"
 )
 
 // Uplink is a general purpose
@@ -295,8 +294,30 @@ func (client *Uplink) CreateBucket(ctx context.Context, satellite *satellite.Pee
 // GetConfig returns a default config for a given satellite.
 func (client *Uplink) GetConfig(satellite *satellite.Peer) uplink.Config {
 	config := getDefaultConfig()
-	config.Client.SatelliteAddr = satellite.Addr()
+
-	config.Client.APIKey = client.APIKey[satellite.ID()]
+	apiKey, err := libuplink.ParseAPIKey(client.APIKey[satellite.ID()])
+	if err != nil {
+		panic(err)
+	}
+
+	encAccess := libuplink.NewEncryptionAccess()
+	encAccess.SetDefaultKey(storj.Key{})
+
+	scopeData, err := (&libuplink.Scope{
+		SatelliteAddr:    satellite.Addr(),
+		APIKey:           apiKey,
+		EncryptionAccess: encAccess,
+	}).Serialize()
+	if err != nil {
+		panic(err)
+	}
+
+	config.Scope = scopeData
+
+	// Support some legacy stuff
+	config.Legacy.Client.APIKey = apiKey.Serialize()
+	config.Legacy.Client.SatelliteAddr = satellite.Addr()
+
 	config.Client.RequestTimeout = 10 * time.Second
 	config.Client.DialTimeout = 10 * time.Second
 
@@ -350,13 +371,12 @@ func (client *Uplink) GetProject(ctx context.Context, satellite *satellite.Peer)
 	}
 	defer func() { err = errs.Combine(err, testLibuplink.Close()) }()
 
-	clientAPIKey := client.APIKey[satellite.ID()]
+	scope, err := client.GetConfig(satellite).GetScope()
-	key, err := libuplink.ParseAPIKey(clientAPIKey)
 	if err != nil {
 		return nil, err
 	}
 
-	project, err := testLibuplink.OpenProject(ctx, satellite.Addr(), key)
+	project, err := testLibuplink.OpenProject(ctx, scope.SatelliteAddr, scope.APIKey)
 	if err != nil {
 		return nil, err
 	}
@@ -369,14 +389,13 @@ func (client *Uplink) GetProjectAndBucket(ctx context.Context, satellite *satell
 	if err != nil {
 		return nil, nil, err
 	}
-
 	defer func() {
 		if err != nil {
 			err = errs.Combine(err, project.Close())
 		}
 	}()
 
-	access, err := setup.LoadEncryptionAccess(ctx, clientCfg.Enc)
+	scope, err := client.GetConfig(satellite).GetScope()
 	if err != nil {
 		return nil, nil, err
 	}
@@ -394,11 +413,10 @@ func (client *Uplink) GetProjectAndBucket(ctx context.Context, satellite *satell
 		}
 	}
 
-	bucket, err := project.OpenBucket(ctx, bucketName, access)
+	bucket, err := project.OpenBucket(ctx, bucketName, scope.EncryptionAccess)
 	if err != nil {
 		return nil, nil, err
 	}
-
 	return project, bucket, nil
 }
 

lib/uplink/project_test.go

@@ -14,9 +14,7 @@ import (
 	"storj.io/storj/internal/testplanet"
 	"storj.io/storj/lib/uplink"
 	libuplink "storj.io/storj/lib/uplink"
-	"storj.io/storj/pkg/macaroon"
 	"storj.io/storj/pkg/storj"
-	"storj.io/storj/uplink/setup"
 )
 
 func TestProjectListBuckets(t *testing.T) {
@@ -29,17 +27,14 @@ func TestProjectListBuckets(t *testing.T) {
 			cfg.Volatile.Log = zaptest.NewLogger(t)
 			cfg.Volatile.TLS.SkipPeerCAWhitelist = true
 
-			satelliteAddr := planet.Satellites[0].Local().Address.Address
+			scope, err := planet.Uplinks[0].GetConfig(planet.Satellites[0]).GetScope()
-			apiKey := planet.Uplinks[0].APIKey[planet.Satellites[0].ID()]
+			require.NoError(t, err)
 
 			ul, err := uplink.NewUplink(ctx, &cfg)
 			require.NoError(t, err)
 			defer ctx.Check(ul.Close)
 
-			key, err := uplink.ParseAPIKey(apiKey)
+			p, err := ul.OpenProject(ctx, scope.SatelliteAddr, scope.APIKey)
-			require.NoError(t, err)
-
-			p, err := ul.OpenProject(ctx, satelliteAddr, key)
 			require.NoError(t, err)
 
 			// create 6 test buckets
@@ -73,28 +68,13 @@ func TestProjectListBuckets(t *testing.T) {
 			require.False(t, result.More)
 
 			// List with restrictions
-			restriction := libuplink.EncryptionRestriction{
+			scope.APIKey, scope.EncryptionAccess, err =
-				Bucket: "test0",
+				scope.EncryptionAccess.Restrict(scope.APIKey,
-			}
+					libuplink.EncryptionRestriction{Bucket: "test0"},
-			access, err := setup.LoadEncryptionAccess(ctx,
+					libuplink.EncryptionRestriction{Bucket: "test1"})
-				planet.Uplinks[0].GetConfig(planet.Satellites[0]).Enc,
-			)
-			require.NoError(t, err)
-			key, access, err = access.Restrict(key, restriction)
 			require.NoError(t, err)
 
-			caveat := macaroon.Caveat{}
+			p, err = ul.OpenProject(ctx, scope.SatelliteAddr, scope.APIKey)
-			caveat.DisallowReads = true
-			caveat.AllowedPaths = append(caveat.AllowedPaths,
-				&macaroon.Caveat_Path{
-					Bucket: []byte("test1"),
-				},
-			)
-
-			key, err = key.Restrict(caveat)
-			require.NoError(t, err)
-
-			p, err = ul.OpenProject(ctx, satelliteAddr, key)
 			require.NoError(t, err)
 			defer ctx.Check(p.Close)
 

pkg/cfgstruct/bind.go

@@ -154,6 +154,7 @@ func bindConfig(flags FlagSet, prefix string, val reflect.Value, vars map[string
 			}
 			flags.Var(fieldvalue, flagname, help)
 
+			markHidden := false
 			if onlyForSetup {
 				setBoolAnnotation(flags, flagname, "setup")
 			}
@@ -161,6 +162,14 @@ func bindConfig(flags FlagSet, prefix string, val reflect.Value, vars map[string
 				setBoolAnnotation(flags, flagname, "user")
 			}
 			if field.Tag.Get("hidden") == "true" {
+				markHidden = true
+				setBoolAnnotation(flags, flagname, "hidden")
+			}
+			if field.Tag.Get("deprecated") == "true" {
+				markHidden = true
+				setBoolAnnotation(flags, flagname, "deprecated")
+			}
+			if markHidden {
 				err := flags.MarkHidden(flagname)
 				if err != nil {
 					panic(fmt.Sprintf("mark hidden failed %s: %v", flagname, err))
@@ -239,9 +248,21 @@ func bindConfig(flags FlagSet, prefix string, val reflect.Value, vars map[string
 			if field.Tag.Get("user") == "true" {
 				setBoolAnnotation(flags, flagname, "user")
 			}
+
+			markHidden := false
 			if field.Tag.Get("hidden") == "true" {
+				markHidden = true
+				setBoolAnnotation(flags, flagname, "hidden")
+			}
+			if field.Tag.Get("deprecated") == "true" {
+				markHidden = true
+				setBoolAnnotation(flags, flagname, "deprecated")
+			}
+			if markHidden {
 				err := flags.MarkHidden(flagname)
-				check(err)
+				if err != nil {
+					panic(fmt.Sprintf("mark hidden failed %s: %v", flagname, err))
+				}
 			}
 		}
 	}

pkg/miniogw/integration_test.go

@@ -7,16 +7,13 @@ import (
 	"context"
 	"errors"
 	"flag"
-	"io/ioutil"
 	"os"
 	"testing"
 	"time"
 
 	"github.com/minio/cli"
 	minio "github.com/minio/minio/cmd"
-	"github.com/spf13/pflag"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zaptest"
 
@@ -24,11 +21,8 @@ import (
 	"storj.io/storj/internal/testcontext"
 	"storj.io/storj/internal/testidentity"
 	"storj.io/storj/internal/testplanet"
-	"storj.io/storj/internal/testrand"
 	libuplink "storj.io/storj/lib/uplink"
-	"storj.io/storj/pkg/cfgstruct"
 	"storj.io/storj/pkg/identity"
-	"storj.io/storj/pkg/macaroon"
 	"storj.io/storj/pkg/miniogw"
 	"storj.io/storj/pkg/storj"
 	"storj.io/storj/satellite/console"
@@ -46,62 +40,22 @@ func TestUploadDownload(t *testing.T) {
 	ctx := testcontext.New(t)
 	defer ctx.Cleanup()
 
-	planet, err := testplanet.New(t, 1, 30, 0)
+	planet, err := testplanet.New(t, 1, 30, 1)
 	assert.NoError(t, err)
 
 	defer ctx.Check(planet.Shutdown)
 
 	// add project to satisfy constraint
-	project, err := planet.Satellites[0].DB.Console().Projects().Insert(context.Background(), &console.Project{
+	_, err = planet.Satellites[0].DB.Console().Projects().Insert(context.Background(), &console.Project{
 		Name: "testProject",
 	})
 	assert.NoError(t, err)
 
-	apiKey, err := macaroon.NewAPIKey([]byte("testSecret"))
-	assert.NoError(t, err)
-
-	apiKeyInfo := console.APIKeyInfo{
-		ProjectID: project.ID,
-		Name:      "testKey",
-		Secret:    []byte("testSecret"),
-	}
-
-	// add api key to db
-	_, err = planet.Satellites[0].DB.Console().APIKeys().Create(context.Background(), apiKey.Head(), apiKeyInfo)
-	assert.NoError(t, err)
-
-	// bind default values to config
 	var gwCfg config
-	cfgstruct.Bind(&pflag.FlagSet{}, &gwCfg, cfgstruct.UseDevDefaults())
-
-	var uplinkCfg uplink.Config
-	cfgstruct.Bind(&pflag.FlagSet{}, &uplinkCfg, cfgstruct.UseDevDefaults())
-
-	// minio config directory
 	gwCfg.Minio.Dir = ctx.Dir("minio")
-
-	// addresses
 	gwCfg.Server.Address = "127.0.0.1:7777"
-	uplinkCfg.Client.SatelliteAddr = planet.Satellites[0].Addr()
 
-	// keys
+	uplinkCfg := planet.Uplinks[0].GetConfig(planet.Satellites[0])
-	uplinkCfg.Client.APIKey = "apiKey"
-
-	// Encryption key
-	passphrase := testrand.BytesInt(testrand.Intn(100) + 1)
-
-	encryptionKey, err := storj.NewKey(passphrase)
-	require.NoError(t, err)
-	filename := ctx.File("encryption.key")
-	err = ioutil.WriteFile(filename, encryptionKey[:], os.FileMode(0400))
-	require.NoError(t, err)
-	uplinkCfg.Enc.KeyFilepath = filename
-
-	// redundancy
-	uplinkCfg.RS.MinThreshold = 7
-	uplinkCfg.RS.RepairThreshold = 8
-	uplinkCfg.RS.SuccessThreshold = 9
-	uplinkCfg.RS.MaxThreshold = 10
 
 	planet.Start(ctx)
 
@@ -127,8 +81,8 @@ func TestUploadDownload(t *testing.T) {
 		Satellite:     planet.Satellites[0].Addr(),
 		AccessKey:     gwCfg.Minio.AccessKey,
 		SecretKey:     gwCfg.Minio.SecretKey,
-		APIKey:        uplinkCfg.Client.APIKey,
+		APIKey:        uplinkCfg.Legacy.Client.APIKey,
-		EncryptionKey: string(encryptionKey[:]),
+		EncryptionKey: "fake-encryption-key",
 		NoSSL:         true,
 	})
 	assert.NoError(t, err)
@@ -197,12 +151,12 @@ func runGateway(ctx context.Context, gwCfg config, uplinkCfg uplink.Config, log
 		return err
 	}
 
-	apiKey, err := libuplink.ParseAPIKey(uplinkCfg.Client.APIKey)
+	apiKey, err := libuplink.ParseAPIKey(uplinkCfg.Legacy.Client.APIKey)
 	if err != nil {
 		return err
 	}
 
-	project, err := uplink.OpenProject(ctx, uplinkCfg.Client.SatelliteAddr, apiKey)
+	project, err := uplink.OpenProject(ctx, uplinkCfg.Legacy.Client.SatelliteAddr, apiKey)
 	if err != nil {
 		return err
 	}

pkg/process/config.go

@@ -0,0 +1,223 @@
+// Copyright (C) 2019 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package process
+
+import (
+	"bytes"
+	"flag"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/spf13/cast"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"github.com/zeebo/errs"
+	yaml "gopkg.in/yaml.v2"
+)
+
+// SaveConfigOption is a function that updates the options for SaveConfig.
+type SaveConfigOption func(*SaveConfigOptions)
+
+// SaveConfigOptions controls the behavior of SaveConfig.
+type SaveConfigOptions struct {
+	Overrides        map[string]interface{}
+	RemoveDeprecated bool
+}
+
+// SaveConfigWithOverrides sets the overrides to the provided map.
+func SaveConfigWithOverrides(overrides map[string]interface{}) SaveConfigOption {
+	return func(opts *SaveConfigOptions) {
+		opts.Overrides = overrides
+	}
+}
+
+// SaveConfigWithOverride adds a single override to SaveConfig.
+func SaveConfigWithOverride(name string, value interface{}) SaveConfigOption {
+	return func(opts *SaveConfigOptions) {
+		if opts.Overrides == nil {
+			opts.Overrides = make(map[string]interface{})
+		}
+		opts.Overrides[name] = value
+	}
+}
+
+// SaveConfigRemovingDeprecated tells SaveConfig to not store deprecated flags.
+func SaveConfigRemovingDeprecated() SaveConfigOption {
+	return func(opts *SaveConfigOptions) {
+		opts.RemoveDeprecated = true
+	}
+}
+
+// SaveConfig will save only the user-specific flags with default values to
+// outfile with specific values specified in 'overrides' overridden.
+func SaveConfig(cmd *cobra.Command, outfile string, opts ...SaveConfigOption) error {
+	// step 0. apply any options to change the behavior
+	//
+
+	var options SaveConfigOptions
+	for _, opt := range opts {
+		opt(&options)
+	}
+
+	// step 1. load all of the configuration settings we are going to save
+	//
+
+	flags := cmd.Flags()
+	vip, err := Viper(cmd)
+	if err != nil {
+		return errs.Wrap(err)
+	}
+	if err := vip.MergeConfigMap(options.Overrides); err != nil {
+		return errs.Wrap(err)
+	}
+	settings := vip.AllSettings()
+
+	// step 2. construct some data describing what exactly we're saving to the
+	// config file, and how they're saved.
+	//
+
+	type configValue struct {
+		value   interface{}
+		comment string
+		set     bool
+	}
+	flat := make(map[string]configValue)
+	flatKeys := make([]string, 0)
+
+	// N.B. we have to pre-declare the function so that it can make recursive calls.
+	var filterAndFlatten func(string, map[string]interface{})
+	filterAndFlatten = func(base string, settings map[string]interface{}) {
+		for key, value := range settings {
+			if value, ok := value.(map[string]interface{}); ok {
+				filterAndFlatten(base+key+".", value)
+				continue
+			}
+			fullKey := base + key
+
+			// since this key can't affect anything from the config file and must be present
+			// on the command line, remove it so as to not mislead anyone
+			if fullKey == "defaults" {
+				continue
+			}
+
+			// gather information about the flag under consideration
+			var (
+				changed    bool
+				setup      bool
+				hidden     bool
+				user       bool
+				deprecated bool
+				comment    string
+				typ        string
+
+				_, overrideExists = options.Overrides[fullKey]
+			)
+			if f := flags.Lookup(fullKey); f != nil { // first check pflags
+				changed = f.Changed
+				setup = readBoolAnnotation(f, "setup")
+				hidden = readBoolAnnotation(f, "hidden")
+				user = readBoolAnnotation(f, "user")
+				deprecated = readBoolAnnotation(f, "deprecated")
+				comment = f.Usage
+				typ = f.Value.Type()
+			} else if f := flag.Lookup(fullKey); f != nil { // then stdlib flags
+				changed = f.Value.String() != f.DefValue
+				comment = f.Usage
+			} else {
+				// by default we store config values we know nothing about. we
+				// absue the meaning of "changed" to include this case.
+				changed = true
+			}
+
+			// in any of these cases, don't store the key in the config file
+			if setup || hidden || options.RemoveDeprecated && deprecated {
+				continue
+			}
+
+			// viper is super cool and doesn't cast floats automatically, so we
+			// handle that ourselves.
+			if typ == "float64" {
+				value = cast.ToFloat64(value)
+			}
+
+			flatKeys = append(flatKeys, fullKey)
+			flat[fullKey] = configValue{
+				value:   value,
+				comment: comment,
+				set:     user || changed || overrideExists,
+			}
+		}
+	}
+	filterAndFlatten("", settings)
+	sort.Strings(flatKeys)
+
+	// step 3. write out the configuration file
+	//
+
+	var nl = []byte("\n")
+	var lines [][]byte
+	for _, key := range flatKeys {
+		config := flat[key]
+
+		if config.comment != "" {
+			lines = append(lines, []byte("# "+config.comment))
+		}
+
+		data, err := yaml.Marshal(map[string]interface{}{key: config.value})
+		if err != nil {
+			return errs.Wrap(err)
+		}
+		dataLines := bytes.Split(bytes.TrimSpace(data), nl)
+
+		// if the config value is set, concat in the yaml lines
+		if config.set {
+			lines = append(lines, dataLines...)
+		} else {
+			// otherwise, add them in but commented out
+			for _, line := range dataLines {
+				lines = append(lines, append([]byte("# "), line...))
+			}
+		}
+
+		// add a blank line separator
+		lines = append(lines, nil)
+	}
+
+	return errs.Wrap(atomicWrite(outfile, 0600, bytes.Join(lines, nl)))
+}
+
+// readBoolAnnotation is a helper to see if a boolean annotation is set to true on the flag.
+func readBoolAnnotation(flag *pflag.Flag, key string) bool {
+	annotation := flag.Annotations[key]
+	return len(annotation) > 0 && annotation[0] == "true"
+}
+
+// atomicWrite is a helper to atomically write the data to the outfile.
+func atomicWrite(outfile string, mode os.FileMode, data []byte) (err error) {
+	fh, err := ioutil.TempFile(filepath.Dir(outfile), filepath.Base(outfile))
+	if err != nil {
+		return errs.Wrap(err)
+	}
+	defer func() {
+		if err != nil {
+			err = errs.Combine(err, fh.Close())
+			err = errs.Combine(err, os.Remove(fh.Name()))
+		}
+	}()
+	if _, err := fh.Write(data); err != nil {
+		return errs.Wrap(err)
+	}
+	if err := fh.Sync(); err != nil {
+		return errs.Wrap(err)
+	}
+	if err := fh.Close(); err != nil {
+		return errs.Wrap(err)
+	}
+	if err := os.Rename(fh.Name(), outfile); err != nil {
+		return errs.Wrap(err)
+	}
+	return nil
+}

pkg/process/exec_conf.go

@@ -7,12 +7,10 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"os/signal"
 	"path/filepath"
-	"sort"
 	"strings"
 	"sync"
 	"syscall"
@@ -38,19 +36,18 @@ const DefaultCfgFilename = "config.yaml"
 var (
 	mon = monkit.Package()
 
-	contextMtx sync.Mutex
+	commandMtx sync.Mutex
 	contexts   = map[*cobra.Command]context.Context{}
-
+	configs    = map[*cobra.Command][]interface{}{}
-	configMtx sync.Mutex
+	vipers     = map[*cobra.Command]*viper.Viper{}
-	configs   = map[*cobra.Command][]interface{}{}
 )
 
 // Bind sets flags on a command that match the configuration struct
 // 'config'. It ensures that the config has all of the values loaded into it
 // when the command runs.
 func Bind(cmd *cobra.Command, config interface{}, opts ...cfgstruct.BindOpt) {
-	configMtx.Lock()
+	commandMtx.Lock()
-	defer configMtx.Unlock()
+	defer commandMtx.Unlock()
 
 	cfgstruct.Bind(cmd.Flags(), config, opts...)
 	configs[cmd] = append(configs[cmd], config)
@@ -75,91 +72,11 @@ func Exec(cmd *cobra.Command) {
 	_ = cmd.Execute()
 }
 
-// SaveConfig will save only the user-specific flags with default values to
-// outfile with specific values specified in 'overrides' overridden.
-func SaveConfig(flagset *pflag.FlagSet, outfile string, overrides map[string]interface{}) error {
-	return saveConfig(flagset, outfile, overrides, false)
-}
-
-// SaveConfigWithAllDefaults will save all flags with default values to outfile
-// with specific values specified in 'overrides' overridden.
-func SaveConfigWithAllDefaults(flagset *pflag.FlagSet, outfile string, overrides map[string]interface{}) error {
-	return saveConfig(flagset, outfile, overrides, true)
-}
-
-func saveConfig(flagset *pflag.FlagSet, outfile string, overrides map[string]interface{}, saveAllDefaults bool) error {
-	// we previously used Viper here, but switched to a custom serializer to allow comments
-	//todo:  switch back to Viper once go-yaml v3 is released and its supports writing comments?
-	flagset.AddFlagSet(pflag.CommandLine)
-	//sort keys
-	var keys []string
-	flagset.VisitAll(func(f *pflag.Flag) { keys = append(keys, f.Name) })
-	sort.Strings(keys)
-	//serialize
-	var sb strings.Builder
-	w := &sb
-	for _, k := range keys {
-		f := flagset.Lookup(k)
-		if readBoolAnnotation(f, "setup") {
-			continue
-		}
-
-		if f.Hidden == true {
-			continue
-		}
-
-		var overriddenValue interface{}
-		var overrideExist bool
-		if overrides != nil {
-			overriddenValue, overrideExist = overrides[k]
-		}
-
-		if !saveAllDefaults && !readBoolAnnotation(f, "user") && !f.Changed && !overrideExist {
-			continue
-		}
-
-		value := f.Value.String()
-		if overriddenValue != nil {
-			value = fmt.Sprintf("%v", overriddenValue)
-		}
-		//print usage info
-		if f.Usage != "" {
-			fmt.Fprintf(w, "# %s\n", f.Usage)
-		}
-		//print commented key (beginning of value assignement line)
-		if readBoolAnnotation(f, "user") || f.Changed || overrideExist {
-			fmt.Fprintf(w, "%s: ", k)
-		} else {
-			fmt.Fprintf(w, "# %s: ", k)
-		}
-		//print value (remainder of value assignement line)
-		switch f.Value.Type() {
-		case "string":
-			// save ourselves 250+ lines of code and just double quote strings
-			fmt.Fprintf(w, "%q\n\n", value)
-		default:
-			//assume that everything else doesn't have fancy control characters
-			fmt.Fprintf(w, "%s\n\n", value)
-		}
-	}
-
-	err := ioutil.WriteFile(outfile, []byte(sb.String()), os.FileMode(0644))
-	if err != nil {
-		return err
-	}
-	fmt.Println("Your configuration is saved to:", outfile)
-	return nil
-}
-
-func readBoolAnnotation(flag *pflag.Flag, key string) bool {
-	annotation := flag.Annotations[key]
-	return len(annotation) > 0 && annotation[0] == "true"
-}
-
 // Ctx returns the appropriate context.Context for ExecuteWithConfig commands
 func Ctx(cmd *cobra.Command) context.Context {
-	contextMtx.Lock()
+	commandMtx.Lock()
-	defer contextMtx.Unlock()
+	defer commandMtx.Unlock()
+
 	ctx := contexts[cmd]
 	if ctx == nil {
 		ctx = context.Background()
@@ -176,6 +93,38 @@ func Ctx(cmd *cobra.Command) context.Context {
 	return ctx
 }
 
+// Viper returns the appropriate *viper.Viper for the command, creating if necessary.
+func Viper(cmd *cobra.Command) (*viper.Viper, error) {
+	commandMtx.Lock()
+	defer commandMtx.Unlock()
+
+	if vip := vipers[cmd]; vip != nil {
+		return vip, nil
+	}
+
+	vip := viper.New()
+	if err := vip.BindPFlags(cmd.Flags()); err != nil {
+		return nil, err
+	}
+	vip.SetEnvPrefix("storj")
+	vip.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))
+	vip.AutomaticEnv()
+
+	cfgFlag := cmd.Flags().Lookup("config-dir")
+	if cfgFlag != nil && cfgFlag.Value.String() != "" {
+		path := filepath.Join(os.ExpandEnv(cfgFlag.Value.String()), DefaultCfgFilename)
+		if cmd.Annotations["type"] != "setup" || fileExists(path) {
+			vip.SetConfigFile(path)
+			if err := vip.ReadInConfig(); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	vipers[cmd] = vip
+	return vip, nil
+}
+
 var traceOut = flag.String("debug.trace-out", "", "If set, a path to write a process trace SVG to")
 
 func cleanup(cmd *cobra.Command) {
@@ -194,45 +143,30 @@ func cleanup(cmd *cobra.Command) {
 		ctx := context.Background()
 		defer mon.TaskNamed("root")(&ctx)(&err)
 
-		vip := viper.New()
+		vip, err := Viper(cmd)
-		err = vip.BindPFlags(cmd.Flags())
 		if err != nil {
 			return err
 		}
-		vip.SetEnvPrefix("storj")
-		vip.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))
-		vip.AutomaticEnv()
 
-		cfgFlag := cmd.Flags().Lookup("config-dir")
+		commandMtx.Lock()
-		if cfgFlag != nil && cfgFlag.Value.String() != "" {
-			path := filepath.Join(os.ExpandEnv(cfgFlag.Value.String()), DefaultCfgFilename)
-			if cmd.Annotations["type"] != "setup" || fileExists(path) {
-				vip.SetConfigFile(path)
-				err = vip.ReadInConfig()
-				if err != nil {
-					return err
-				}
-			}
-		}
-
-		configMtx.Lock()
 		configValues := configs[cmd]
-		configMtx.Unlock()
+		commandMtx.Unlock()
 
 		var (
 			brokenKeys  = map[string]struct{}{}
 			missingKeys = map[string]struct{}{}
 			usedKeys    = map[string]struct{}{}
+			allKeys     = map[string]struct{}{}
 			allSettings = vip.AllSettings()
 		)
 
 		// Hacky hack: these two keys are noprefix which breaks all scoping
 		if val, ok := allSettings["api-key"]; ok {
-			allSettings["client.api-key"] = val
+			allSettings["legacy.client.api-key"] = val
 			delete(allSettings, "api-key")
 		}
 		if val, ok := allSettings["satellite-addr"]; ok {
-			allSettings["client.satellite-addr"] = val
+			allSettings["legacy.client.satellite-addr"] = val
 			delete(allSettings, "satellite-addr")
 		}
 
@@ -241,36 +175,44 @@ func cleanup(cmd *cobra.Command) {
 			res := structs.Decode(allSettings, config)
 			for key := range res.Used {
 				usedKeys[key] = struct{}{}
+				allKeys[key] = struct{}{}
 			}
 			for key := range res.Missing {
 				missingKeys[key] = struct{}{}
+				allKeys[key] = struct{}{}
 			}
 			for key := range res.Broken {
 				brokenKeys[key] = struct{}{}
+				allKeys[key] = struct{}{}
 			}
 		}
 
-		for key := range missingKeys {
+		for key := range allKeys {
+			// Check if the key is a flag, and if so, propagate it.
+			if f := cmd.Flags().Lookup(key); f != nil {
+				val := vip.GetString(key)
+				err := f.Value.Set(val)
+				f.Changed = val != f.DefValue
+				if err != nil {
+					brokenKeys[key] = struct{}{}
+				} else {
+					usedKeys[key] = struct{}{}
+				}
+			} else if f := flag.Lookup(key); f != nil {
+				err := f.Value.Set(vip.GetString(key))
+				if err != nil {
+					brokenKeys[key] = struct{}{}
+				} else {
+					usedKeys[key] = struct{}{}
+				}
+			}
+
 			// A key is only missing if it was missing from every single config struct, so
 			// remove all of the used keys from it.
 			if _, ok := usedKeys[key]; ok {
 				delete(missingKeys, key)
 				continue
 			}
-
-			// Attempt to set through the flags any keys that were missing from all of the
-			// config structs.
-			flag := cmd.Flags().Lookup(key)
-			if flag == nil {
-				continue
-			}
-
-			changed := flag.Changed
-			if err := flag.Value.Set(vip.GetString(key)); err != nil {
-				brokenKeys[key] = struct{}{}
-			}
-			flag.Changed = changed
-			delete(missingKeys, key)
 		}
 
 		logger, err := newLogger()
@@ -309,13 +251,13 @@ func cleanup(cmd *cobra.Command) {
 
 		var workErr error
 		work := func(ctx context.Context) {
-			contextMtx.Lock()
+			commandMtx.Lock()
 			contexts[cmd] = ctx
-			contextMtx.Unlock()
+			commandMtx.Unlock()
 			defer func() {
-				contextMtx.Lock()
+				commandMtx.Lock()
 				delete(contexts, cmd)
-				contextMtx.Unlock()
+				commandMtx.Unlock()
 			}()
 
 			workErr = internalRun(cmd, args)

pkg/process/exec_conf_test.go

@@ -21,6 +21,8 @@ func setenv(key, value string) func() {
 	return func() { _ = os.Setenv(key, old) }
 }
 
+var testZ = flag.Int("z", 0, "z flag (stdlib)")
+
 func TestExec_PropagatesSettings(t *testing.T) {
 	// Set up a command that does nothing.
 	cmd := &cobra.Command{RunE: func(cmd *cobra.Command, args []string) error { return nil }}
@@ -31,7 +33,6 @@ func TestExec_PropagatesSettings(t *testing.T) {
 	}
 	Bind(cmd, &config)
 	y := cmd.Flags().Int("y", 0, "y flag (command)")
-	z := flag.Int("z", 0, "z flag (stdlib)")
 
 	// Set some environment variables for viper.
 	defer setenv("STORJ_X", "1")()
@@ -44,14 +45,14 @@ func TestExec_PropagatesSettings(t *testing.T) {
 	// Check that the variables are now bound.
 	require.Equal(t, 1, config.X)
 	require.Equal(t, 2, *y)
-	require.Equal(t, 3, *z)
+	require.Equal(t, 3, *testZ)
 }
 
 func TestHidden(t *testing.T) {
 	// Set up a command that does nothing.
 	cmd := &cobra.Command{RunE: func(cmd *cobra.Command, args []string) error { return nil }}
 
-	// Define a config struct and some flags.
+	// Define a config struct with a hidden field.
 	var config struct {
 		W int `default:"0" hidden:"false"`
 		X int `default:"0" hidden:"true"`
@@ -64,10 +65,12 @@ func TestHidden(t *testing.T) {
 	ctx := testcontext.New(t)
 	testConfigFile := ctx.File("testconfig.yaml")
 	defer ctx.Cleanup()
-	overrides := map[string]interface{}{}
 
-	// Test that only the configs that are not hidden show up in config file
+	// Run the command through the exec call.
-	err := SaveConfigWithAllDefaults(cmd.Flags(), testConfigFile, overrides)
+	Exec(cmd)
+
+	// Ensure that the file saves only the necessary data.
+	err := SaveConfig(cmd, testConfigFile)
 	require.NoError(t, err)
 
 	actualConfigFile, err := ioutil.ReadFile(testConfigFile)

scripts/testdata/satellite-config.yaml.lock

@@ -23,7 +23,7 @@
 # checker.reliability-cache-staleness: 5m0s
 
 # server address of the graphql api gateway and frontend app
-# console.address: ":10100"
+# console.address: :10100
 
 # auth token needed for access to registration token creation endpoint
 # console.auth-token: ""
@@ -41,7 +41,7 @@
 # console.stripe-key: ""
 
 # satellite database connection string
-# database: "postgres://"
+# database: postgres://
 
 # Maximum Database Connection Lifetime, -1ns means the stdlib default
 # db.conn_max_lifetime: -1ns
@@ -53,14 +53,11 @@
 # db.max_open_conns: 25
 
 # address to listen on for debug endpoints
-# debug.addr: "127.0.0.1:0"
+# debug.addr: 127.0.0.1:0
 
 # If set, a path to write a process trace SVG to
 # debug.trace-out: ""
 
-# determines which set of configuration defaults to use. can either be 'dev' or 'release'
-defaults: "release"
-
 # the interval at which the satellite attempts to find new nodes via random node ID lookups
 # discovery.discovery-interval: 1s
 
@@ -92,16 +89,16 @@ defaults: "release"
 # help: false
 
 # path to the certificate chain for this identity
-identity.cert-path: "/root/.local/share/storj/identity/satellite/identity.cert"
+identity.cert-path: /root/.local/share/storj/identity/satellite/identity.cert
 
 # path to the private key for this identity
-identity.key-path: "/root/.local/share/storj/identity/satellite/identity.key"
+identity.key-path: /root/.local/share/storj/identity/satellite/identity.key
 
 # alpha is a system wide concurrency parameter
 # kademlia.alpha: 5
 
 # the Kademlia node to bootstrap against
-# kademlia.bootstrap-addr: "bootstrap.storj.io:8888"
+# kademlia.bootstrap-addr: bootstrap.storj.io:8888
 
 # the base interval to wait when retrying bootstrap
 # kademlia.bootstrap-backoff-base: 1s
@@ -113,7 +110,7 @@ identity.key-path: "/root/.local/share/storj/identity/satellite/identity.key"
 # kademlia.bucket-size: 20
 
 # the path for storage node db services to be created on
-# kademlia.db-path: "testdata/kademlia"
+# kademlia.db-path: testdata/kademlia
 
 # the public address of the Kademlia node, useful for nodes behind NAT
 kademlia.external-address: ""
@@ -128,7 +125,7 @@ kademlia.operator.wallet: ""
 # kademlia.replacement-cache-size: 5
 
 # what to use for storing real-time accounting data
-# live-accounting.storage-backend: "plainmemory"
+# live-accounting.storage-backend: plainmemory
 
 # if true, log function filename and line number
 # log.caller: false
@@ -137,19 +134,19 @@ kademlia.operator.wallet: ""
 # log.development: false
 
 # configures log encoding. can either be 'console' or 'json'
-# log.encoding: "console"
+# log.encoding: console
 
 # the minimum log level to log
 # log.level: info
 
 # can be stdout, stderr, or a filename
-# log.output: "stderr"
+# log.output: stderr
 
 # if true, log stack traces
 # log.stack: false
 
 # smtp authentication type
-# mail.auth-type: "login"
+# mail.auth-type: login
 
 # oauth2 app's client id
 # mail.client-id: ""
@@ -179,13 +176,13 @@ kademlia.operator.wallet: ""
 # mail.token-uri: ""
 
 # server address of the marketing Admin GUI
-# marketing.address: "127.0.0.1:8090"
+# marketing.address: 127.0.0.1:8090
 
 # path to static resources
 # marketing.static-dir: ""
 
 # the database connection string to use
-# metainfo.database-url: "postgres://"
+# metainfo.database-url: postgres://
 
 # how long to wait for new observers before starting iteration
 # metainfo.loop.coalesce-duration: 5s
@@ -224,13 +221,13 @@ kademlia.operator.wallet: ""
 # metainfo.rs.validate: true
 
 # address to send telemetry to
-# metrics.addr: "collectora.storj.io:9000"
+# metrics.addr: collectora.storj.io:9000
 
 # application name for telemetry identification
-# metrics.app: "satellite"
+# metrics.app: satellite
 
 # application suffix
-# metrics.app-suffix: "-release"
+# metrics.app-suffix: -release
 
 # instance id prefix
 # metrics.instance-prefix: ""
@@ -239,7 +236,7 @@ kademlia.operator.wallet: ""
 # metrics.interval: 1m0s
 
 # path to log for oom notices
-# monkit.hw.oomlog: "/var/log/kern.log"
+# monkit.hw.oomlog: /var/log/kern.log
 
 # how long until an order expires
 # orders.expiration: 168h0m0s
@@ -332,7 +329,7 @@ kademlia.operator.wallet: ""
 # rollup.max-alpha-usage: 25.0 GB
 
 # public address to listen on
-server.address: ":7777"
+server.address: :7777
 
 # log all GRPC traffic to zap logger
 server.debug-log-traffic: false
@@ -347,13 +344,13 @@ server.debug-log-traffic: false
 # server.peer-ca-whitelist-path: ""
 
 # identity version(s) the server will be allowed to talk to
-# server.peer-id-versions: "latest"
+# server.peer-id-versions: latest
 
 # private address to listen on
-server.private-address: "127.0.0.1:7778"
+server.private-address: 127.0.0.1:7778
 
 # url for revocation database (e.g. bolt://some.db OR redis://127.0.0.1:6378?db=2&password=abc123)
-# server.revocation-dburl: "bolt://testdata/revocations.db"
+# server.revocation-dburl: bolt://testdata/revocations.db
 
 # if true, uses peer ca whitelist checking
 # server.use-peer-ca-whitelist: true
@@ -368,8 +365,7 @@ server.private-address: "127.0.0.1:7778"
 # version.request-timeout: 1m0s
 
 # server address to check its version against
-# version.server-address: "https://version.alpha.storj.io"
+# version.server-address: https://version.alpha.storj.io
 
 # length of time before a voucher expires
 # vouchers.expiration: 720h0m0s
-

uplink/config.go

@@ -4,13 +4,21 @@
 package uplink
 
 import (
+	"io/ioutil"
+	"strings"
 	"time"
 
+	"github.com/zeebo/errs"
+	"gopkg.in/spacemonkeygo/monkit.v2"
+
 	"storj.io/storj/internal/memory"
+	libuplink "storj.io/storj/lib/uplink"
 	"storj.io/storj/pkg/peertls/tlsopts"
 	"storj.io/storj/pkg/storj"
 )
 
+var mon = monkit.Package()
+
 // RSConfig is a configuration struct that keeps details about default
 // redundancy strategy information
 type RSConfig struct {
@@ -25,18 +33,13 @@ type RSConfig struct {
 // EncryptionConfig is a configuration struct that keeps details about
 // encrypting segments
 type EncryptionConfig struct {
-	EncryptionKey     string `help:"the root key for encrypting the data which will be stored in KeyFilePath" setup:"true"`
+	DataType int `help:"Type of encryption to use for content and metadata (2=AES-GCM, 3=SecretBox)" default:"2"`
-	KeyFilepath       string `help:"the path to the file which contains the root key for encrypting the data"`
+	PathType int `help:"Type of encryption to use for paths (1=Unencrypted, 2=AES-GCM, 3=SecretBox)" default:"2"`
-	EncAccessFilepath string `help:"the path to a file containing a serialized encryption access"`
-	DataType          int    `help:"Type of encryption to use for content and metadata (2=AES-GCM, 3=SecretBox)" default:"2"`
-	PathType          int    `help:"Type of encryption to use for paths (1=Unencrypted, 2=AES-GCM, 3=SecretBox)" default:"2"`
 }
 
 // ClientConfig is a configuration struct for the uplink that controls how
 // to talk to the rest of the network.
 type ClientConfig struct {
-	APIKey         string        `default:"" help:"the api key to use for the satellite" noprefix:"true"`
-	SatelliteAddr  string        `releaseDefault:"127.0.0.1:7777" devDefault:"127.0.0.1:10000" help:"the address to use for the satellite" noprefix:"true"`
 	MaxInlineSize  memory.Size   `help:"max inline segment size in bytes" default:"4KiB"`
 	SegmentSize    memory.Size   `help:"the size of a segment in bytes" default:"64MiB"`
 	RequestTimeout time.Duration `help:"timeout for request" default:"0h2m00s"`
@@ -45,12 +48,91 @@ type ClientConfig struct {
 
 // Config uplink configuration
 type Config struct {
+	ScopeConfig
 	Client ClientConfig
 	RS     RSConfig
 	Enc    EncryptionConfig
 	TLS    tlsopts.Config
 }
 
+// ScopeConfig holds information about which scopes exist and are selected.
+type ScopeConfig struct {
+	Scopes map[string]string `internal:"true"`
+	Scope  string            `help:"the serialized scope, or name of the scope to use" default:""`
+
+	Legacy // Holds on to legacy configuration values
+}
+
+// Legacy holds deprecated configuration values
+type Legacy struct {
+	Client struct {
+		APIKey        string `default:"" help:"the api key to use for the satellite (deprecated)" noprefix:"true" deprecated:"true"`
+		SatelliteAddr string `releaseDefault:"127.0.0.1:7777" devDefault:"127.0.0.1:10000" help:"the address to use for the satellite (deprecated)" noprefix:"true"`
+	}
+	Enc struct {
+		EncryptionKey     string `help:"the root key for encrypting the data which will be stored in KeyFilePath (deprecated)" setup:"true" deprecated:"true"`
+		KeyFilepath       string `help:"the path to the file which contains the root key for encrypting the data (deprecated)" deprecated:"true"`
+		EncAccessFilepath string `help:"the path to a file containing a serialized encryption access (deprecated)" deprecated:"true"`
+	}
+}
+
+// GetScope returns the appropriate scope for the config.
+func (c ScopeConfig) GetScope() (_ *libuplink.Scope, err error) {
+	defer mon.Task()(nil)(&err)
+
+	// if a scope exists for that name, try to load it.
+	if data, ok := c.Scopes[c.Scope]; ok && c.Scope != "" {
+		return libuplink.ParseScope(data)
+	}
+
+	// Otherwise, try to load the scope name as a serialized scope.
+	if scope, err := libuplink.ParseScope(c.Scope); err == nil {
+		return scope, nil
+	}
+
+	// fall back to trying to load the legacy values.
+	apiKey, err := libuplink.ParseAPIKey(c.Legacy.Client.APIKey)
+	if err != nil {
+		return nil, err
+	}
+
+	satelliteAddr := c.Legacy.Client.SatelliteAddr
+	if satelliteAddr == "" {
+		return nil, errs.New("must specify a satellite address")
+	}
+
+	var encAccess *libuplink.EncryptionAccess
+	if c.Legacy.Enc.EncAccessFilepath != "" {
+		data, err := ioutil.ReadFile(c.Legacy.Enc.EncAccessFilepath)
+		if err != nil {
+			return nil, errs.Wrap(err)
+		}
+		encAccess, err = libuplink.ParseEncryptionAccess(strings.TrimSpace(string(data)))
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		data := []byte(c.Legacy.Enc.EncryptionKey)
+		if c.Legacy.Enc.KeyFilepath != "" {
+			data, err = ioutil.ReadFile(c.Legacy.Enc.KeyFilepath)
+			if err != nil {
+				return nil, errs.Wrap(err)
+			}
+		}
+		key, err := storj.NewKey(data)
+		if err != nil {
+			return nil, errs.Wrap(err)
+		}
+		encAccess = libuplink.NewEncryptionAccessWithDefaultKey(*key)
+	}
+
+	return &libuplink.Scope{
+		APIKey:           apiKey,
+		SatelliteAddr:    satelliteAddr,
+		EncryptionAccess: encAccess,
+	}, nil
+}
+
 // GetRedundancyScheme returns the configured redundancy scheme for new uploads
 func (c Config) GetRedundancyScheme() storj.RedundancyScheme {
 	return storj.RedundancyScheme{

uplink/setup/encryption_key.go

@@ -1,41 +0,0 @@
-// Copyright (C) 2019 Storj Labs, Inc.
-// See LICENSE for copying information.
-
-package setup
-
-import (
-	"os"
-
-	"github.com/zeebo/errs"
-)
-
-// SaveEncryptionKey generates a Storj key from the inputKey and save it into a
-// new file created in filepath.
-func SaveEncryptionKey(inputKey string, filepath string) error {
-	switch {
-	case len(inputKey) == 0:
-		return Error.New("inputKey is empty")
-	case filepath == "":
-		return Error.New("filepath is empty")
-	}
-
-	file, err := os.OpenFile(filepath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return Error.New("directory path doesn't exist. %+v", err)
-		}
-
-		if os.IsExist(err) {
-			return Error.New("file key already exists. %+v", err)
-		}
-
-		return Error.Wrap(err)
-	}
-
-	defer func() {
-		err = Error.Wrap(errs.Combine(err, file.Close()))
-	}()
-
-	_, err = file.Write([]byte(inputKey))
-	return err
-}

uplink/setup/encryption_key_test.go

@@ -1,82 +0,0 @@
-// Copyright (C) 2019 Storj Labs, Inc.
-// See LICENSE for copying information.
-
-package setup_test
-
-import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"storj.io/storj/internal/testcontext"
-	"storj.io/storj/internal/testrand"
-	"storj.io/storj/pkg/storj"
-	"storj.io/storj/uplink/setup"
-)
-
-func TestSaveEncryptionKey(t *testing.T) {
-	generateInputKey := func() string {
-		return string(testrand.BytesInt(testrand.Intn(storj.KeySize*3) + 1))
-	}
-
-	t.Run("ok", func(t *testing.T) {
-		ctx := testcontext.New(t)
-		defer ctx.Cleanup()
-
-		inputKey := generateInputKey()
-		filename := ctx.File("storj-test-cmd-uplink", "encryption.key")
-		err := setup.SaveEncryptionKey(inputKey, filename)
-		require.NoError(t, err)
-
-		savedKey, err := ioutil.ReadFile(filename)
-		require.NoError(t, err)
-
-		assert.Equal(t, inputKey, string(savedKey))
-	})
-
-	t.Run("error: empty input key", func(t *testing.T) {
-		ctx := testcontext.New(t)
-		defer ctx.Cleanup()
-
-		filename := ctx.File("storj-test-cmd-uplink", "encryption.key")
-
-		err := setup.SaveEncryptionKey("", filename)
-		require.Error(t, err)
-	})
-
-	t.Run("error: empty filepath", func(t *testing.T) {
-		inputKey := generateInputKey()
-
-		err := setup.SaveEncryptionKey(inputKey, "")
-		require.Error(t, err)
-	})
-
-	t.Run("error: unexisting dir", func(t *testing.T) {
-		// Create a directory and remove it for making sure that the path doesn't
-		// exist
-		ctx := testcontext.New(t)
-		dir := ctx.Dir("storj-test-cmd-uplink")
-		ctx.Cleanup()
-
-		inputKey := generateInputKey()
-		filename := filepath.Join(dir, "enc.key")
-		err := setup.SaveEncryptionKey(inputKey, filename)
-		require.Errorf(t, err, "directory path doesn't exist")
-	})
-
-	t.Run("error: file already exists", func(t *testing.T) {
-		ctx := testcontext.New(t)
-		defer ctx.Cleanup()
-
-		inputKey := generateInputKey()
-		filename := ctx.File("encryption.key")
-		require.NoError(t, ioutil.WriteFile(filename, nil, os.FileMode(0600)))
-
-		err := setup.SaveEncryptionKey(inputKey, filename)
-		require.Errorf(t, err, "file key already exists")
-	})
-}

uplink/setup/setup.go

@@ -1,51 +0,0 @@
-// Copyright (C) 2019 Storj Labs, Inc.
-// See LICENSE for copying information.
-
-package setup
-
-import (
-	"context"
-	"io/ioutil"
-	"strings"
-
-	"github.com/zeebo/errs"
-	"gopkg.in/spacemonkeygo/monkit.v2"
-
-	libuplink "storj.io/storj/lib/uplink"
-	"storj.io/storj/pkg/storj"
-	"storj.io/storj/uplink"
-)
-
-var (
-	mon = monkit.Package()
-
-	// Error is the class of errors returned by this package
-	Error = errs.Class("uplink setup")
-)
-
-// LoadEncryptionAccess loads an EncryptionAccess from the values specified in the encryption config.
-func LoadEncryptionAccess(ctx context.Context, cfg uplink.EncryptionConfig) (_ *libuplink.EncryptionAccess, err error) {
-	defer mon.Task()(&ctx)(&err)
-
-	if cfg.EncAccessFilepath != "" {
-		data, err := ioutil.ReadFile(cfg.EncAccessFilepath)
-		if err != nil {
-			return nil, errs.Wrap(err)
-		}
-		return libuplink.ParseEncryptionAccess(strings.TrimSpace(string(data)))
-	}
-
-	data := []byte(cfg.EncryptionKey)
-	if cfg.KeyFilepath != "" {
-		data, err = ioutil.ReadFile(cfg.KeyFilepath)
-		if err != nil {
-			return nil, errs.Wrap(err)
-		}
-	}
-
-	key, err := storj.NewKey(data)
-	if err != nil {
-		return nil, errs.Wrap(err)
-	}
-	return libuplink.NewEncryptionAccessWithDefaultKey(*key), nil
-}

uplink/setup/setup_test.go

@@ -1,71 +0,0 @@
-// Copyright (C) 2019 Storj Labs, Inc.
-// See LICENSE for copying information.
-
-package setup_test
-
-import (
-	"context"
-	"io/ioutil"
-	"os"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-
-	"storj.io/storj/internal/testcontext"
-	"storj.io/storj/internal/testrand"
-	libuplink "storj.io/storj/lib/uplink"
-	"storj.io/storj/pkg/storj"
-	"storj.io/storj/uplink"
-	"storj.io/storj/uplink/setup"
-)
-
-func TestLoadEncryptionAccess(t *testing.T) {
-	saveRawCtx := func(access *libuplink.EncryptionAccess) (filepath string, clenaup func()) {
-		t.Helper()
-
-		ctx := testcontext.New(t)
-		filename := ctx.File("encryption.ctx")
-		data, err := access.Serialize()
-		require.NoError(t, err)
-		err = ioutil.WriteFile(filename, []byte(data), os.FileMode(0400))
-		require.NoError(t, err)
-
-		return filename, ctx.Cleanup
-	}
-
-	t.Run("ok: reading from file", func(t *testing.T) {
-		passphrase := testrand.BytesInt(1 + testrand.Intn(100))
-
-		key, err := storj.NewKey(passphrase)
-		require.NoError(t, err)
-		access := libuplink.NewEncryptionAccessWithDefaultKey(*key)
-		filename, cleanup := saveRawCtx(access)
-		defer cleanup()
-
-		gotCtx, err := setup.LoadEncryptionAccess(context.Background(), uplink.EncryptionConfig{
-			EncAccessFilepath: filename,
-		})
-		require.NoError(t, err)
-		require.Equal(t, access, gotCtx)
-	})
-
-	t.Run("ok: empty filepath", func(t *testing.T) {
-		gotCtx, err := setup.LoadEncryptionAccess(context.Background(), uplink.EncryptionConfig{
-			EncAccessFilepath: "",
-		})
-
-		require.NoError(t, err)
-		require.NotNil(t, gotCtx)
-	})
-
-	t.Run("error: file not found", func(t *testing.T) {
-		ctx := testcontext.New(t)
-		defer ctx.Cleanup()
-		filename := ctx.File("encryption.ctx")
-
-		_, err := setup.LoadEncryptionAccess(context.Background(), uplink.EncryptionConfig{
-			EncAccessFilepath: filename,
-		})
-		require.Error(t, err)
-	})
-}