diff --git a/Makefile b/Makefile index 2de3be8ce..fe5383b05 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,7 @@ goimports-fix: ## Applies goimports to every go file (excluding vendored files) .PHONY: goimports-st goimports-st: ## Applies goimports to every go file in `git status` (ignores untracked files) - git status --porcelain -uno|grep .go|grep -v "^D"|sed -E 's,\w+\s+(.+->\s+)?,,g'|xargs -I {} goimports -w -local storj.io {} + @git status --porcelain -uno|grep .go|grep -v "^D"|sed -E 's,\w+\s+(.+->\s+)?,,g'|xargs -I {} goimports -w -local storj.io {} .PHONY: proto proto: ## Rebuild protobuf files diff --git a/cmd/bootstrap/main.go b/cmd/bootstrap/main.go index cdb841605..364db4eb1 100644 --- a/cmd/bootstrap/main.go +++ b/cmd/bootstrap/main.go @@ -4,7 +4,6 @@ package main import ( - "errors" "fmt" "os" "path/filepath" @@ -14,7 +13,6 @@ import ( "storj.io/storj/internal/fpath" "storj.io/storj/pkg/cfgstruct" - "storj.io/storj/pkg/identity" "storj.io/storj/pkg/kademlia" "storj.io/storj/pkg/process" "storj.io/storj/pkg/server" @@ -22,9 +20,6 @@ import ( // Bootstrap defines a bootstrap node configuration type Bootstrap struct { - CA identity.CASetupConfig `setup:"true"` - Identity identity.SetupConfig `setup:"true"` - Server server.Config Kademlia kademlia.BootstrapConfig } @@ -48,8 +43,10 @@ var ( cfg Bootstrap - defaultConfDir string - confDir *string + defaultConfDir = fpath.ApplicationDir("storj", "bootstrap") + defaultIdentityDir = fpath.ApplicationDir("storj", "identity", "bootstrap") + confDir string + identityDir string ) const ( @@ -57,31 +54,46 @@ const ( ) func init() { - defaultConfDir = fpath.ApplicationDir("storj", "bootstrap") - - dirParam := cfgstruct.FindConfigDirParam() - if dirParam != "" { - defaultConfDir = dirParam + confDirParam := cfgstruct.FindConfigDirParam() + if confDirParam != "" { + defaultConfDir = confDirParam + } + identityDirParam := cfgstruct.FindCredsDirParam() + if identityDirParam != "" { + defaultIdentityDir = identityDirParam } - confDir = rootCmd.PersistentFlags().String("config-dir", defaultConfDir, "main directory for bootstrap configuration") + rootCmd.PersistentFlags().StringVar(&confDir, "config-dir", defaultConfDir, "main directory for bootstrap configuration") + err := rootCmd.PersistentFlags().SetAnnotation("config-dir", "setup", []string{"true"}) + if err != nil { + zap.S().Error("Failed to set 'setup' annotation for 'config-dir'") + } + rootCmd.PersistentFlags().StringVar(&identityDir, "identity-dir", defaultIdentityDir, "main directory for bootstrap identity credentials") + err = rootCmd.PersistentFlags().SetAnnotation("identity-dir", "setup", []string{"true"}) + if err != nil { + zap.S().Error("Failed to set 'setup' annotation for 'config-dir'") + } rootCmd.AddCommand(runCmd) rootCmd.AddCommand(setupCmd) - cfgstruct.Bind(runCmd.Flags(), &cfg, cfgstruct.ConfDir(defaultConfDir)) - cfgstruct.BindSetup(setupCmd.Flags(), &cfg, cfgstruct.ConfDir(defaultConfDir)) + cfgstruct.Bind(runCmd.Flags(), &cfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir)) + cfgstruct.BindSetup(setupCmd.Flags(), &cfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir)) } func cmdRun(cmd *cobra.Command, args []string) (err error) { ctx := process.Ctx(cmd) - if err := process.InitMetricsWithCertPath(ctx, nil, cfg.Identity.CertPath); err != nil { + if _, err := cfg.Server.Identity.Load(); err != nil { + zap.S().Fatal(err) + } + + if err := process.InitMetricsWithCertPath(ctx, nil, cfg.Server.Identity.CertPath); err != nil { zap.S().Errorf("Failed to initialize telemetry batcher: %+v", err) } return cfg.Server.Run(ctx, nil, cfg.Kademlia) } func cmdSetup(cmd *cobra.Command, args []string) (err error) { - setupDir, err := filepath.Abs(*confDir) + setupDir, err := filepath.Abs(confDir) if err != nil { return err } @@ -96,20 +108,7 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) { return err } - if setupDir != defaultConfDir { - cfg.CA.CertPath = filepath.Join(setupDir, "ca.cert") - cfg.CA.KeyPath = filepath.Join(setupDir, "ca.key") - cfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert") - cfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key") - } - - if cfg.Identity.Status() != identity.CertKey { - return errors.New("identity is missing") - } - overrides := map[string]interface{}{ - "identity.cert-path": cfg.Identity.CertPath, - "identity.key-path": cfg.Identity.KeyPath, "identity.server.address": defaultServerAddr, "kademlia.bootstrap-addr": "localhost" + defaultServerAddr, } diff --git a/cmd/certificates/auth.go b/cmd/certificates/auth.go index 822c9c9c2..1ccaf58e7 100644 --- a/cmd/certificates/auth.go +++ b/cmd/certificates/auth.go @@ -46,35 +46,16 @@ var ( Short: "Export authorization(s) from CSR authorization DB to a CSV file (or stdout)", RunE: cmdExportAuth, } - - authCreateCfg struct { - Signer certificates.CertServerConfig - batchCfg - } - - authInfoCfg struct { - All bool `help:"print the info for all authorizations" default:"false"` - ShowTokens bool `help:"if true, token strings will be printed" default:"false"` - Signer certificates.CertServerConfig - batchCfg - } - - authExportCfg struct { - All bool `help:"export all authorizations" default:"false"` - Out string `help:"output file path; if \"-\", will use STDOUT" default:"-"` - Signer certificates.CertServerConfig - batchCfg - } ) func init() { rootCmd.AddCommand(authCmd) authCmd.AddCommand(authCreateCmd) - cfgstruct.Bind(authCreateCmd.Flags(), &authCreateCfg, cfgstruct.ConfDir(defaultConfDir)) + cfgstruct.Bind(authCreateCmd.Flags(), &config, cfgstruct.ConfDir(defaultConfDir)) authCmd.AddCommand(authInfoCmd) - cfgstruct.Bind(authInfoCmd.Flags(), &authInfoCfg, cfgstruct.ConfDir(defaultConfDir)) + cfgstruct.Bind(authInfoCmd.Flags(), &config, cfgstruct.ConfDir(defaultConfDir)) authCmd.AddCommand(authExportCmd) - cfgstruct.Bind(authExportCmd.Flags(), &authExportCfg, cfgstruct.ConfDir(defaultConfDir)) + cfgstruct.Bind(authExportCmd.Flags(), &config, cfgstruct.ConfDir(defaultConfDir)) } func parseEmailsList(fileName, delimiter string) (emails []string, err error) { @@ -98,19 +79,19 @@ func cmdCreateAuth(cmd *cobra.Command, args []string) error { if err != nil { return errs.New("Count couldn't be parsed: %s", args[0]) } - authDB, err := authCreateCfg.Signer.NewAuthDB() + authDB, err := config.Signer.NewAuthDB() if err != nil { return err } var emails []string if len(args) > 1 { - if authCreateCfg.EmailsPath != "" { + if config.EmailsPath != "" { return errs.New("Either use `--emails-path` or positional args, not both.") } emails = args[1:] } else { - emails, err = parseEmailsList(authCreateCfg.EmailsPath, authCreateCfg.Delimiter) + emails, err = parseEmailsList(config.EmailsPath, config.Delimiter) if err != nil { return errs.Wrap(err) } @@ -126,26 +107,26 @@ func cmdCreateAuth(cmd *cobra.Command, args []string) error { } func cmdInfoAuth(cmd *cobra.Command, args []string) error { - authDB, err := authInfoCfg.Signer.NewAuthDB() + authDB, err := config.Signer.NewAuthDB() if err != nil { return err } var emails []string if len(args) > 0 { - if authInfoCfg.EmailsPath != "" && !authInfoCfg.All { + if config.EmailsPath != "" && !config.All { return errs.New("Either use `--emails-path` or positional args, not both.") } emails = args - } else if len(args) == 0 || authInfoCfg.All { + } else if len(args) == 0 || config.All { emails, err = authDB.UserIDs() if err != nil { return err } - } else if _, err := os.Stat(authInfoCfg.EmailsPath); err != nil { + } else if _, err := os.Stat(config.EmailsPath); err != nil { return errs.New("Emails path error: %s", err) } else { - emails, err = parseEmailsList(authInfoCfg.EmailsPath, authInfoCfg.Delimiter) + emails, err = parseEmailsList(config.EmailsPath, config.Delimiter) if err != nil { return errs.Wrap(err) } @@ -189,7 +170,7 @@ func writeAuthInfo(authDB *certificates.AuthorizationDB, email string, w io.Writ return err } - if authInfoCfg.ShowTokens { + if config.ShowTokens { if err := writeTokenInfo(claimed, open, w); err != nil { return err } @@ -222,24 +203,24 @@ func writeTokenInfo(claimed, open certificates.Authorizations, w io.Writer) erro } func cmdExportAuth(cmd *cobra.Command, args []string) error { - authDB, err := authExportCfg.Signer.NewAuthDB() + authDB, err := config.Signer.NewAuthDB() if err != nil { return err } var emails []string - if len(args) > 0 && !authExportCfg.All { - if authExportCfg.EmailsPath != "" { + if len(args) > 0 && !config.All { + if config.EmailsPath != "" { return errs.New("Either use `--emails-path` or positional args, not both.") } emails = args - } else if len(args) == 0 || authExportCfg.All { + } else if len(args) == 0 || config.All { emails, err = authDB.UserIDs() if err != nil { return err } } else { - emails, err = parseEmailsList(authExportCfg.EmailsPath, authExportCfg.Delimiter) + emails, err = parseEmailsList(config.EmailsPath, config.Delimiter) if err != nil { return errs.Wrap(err) } @@ -249,14 +230,14 @@ func cmdExportAuth(cmd *cobra.Command, args []string) error { emailErrs, csvErrs utils.ErrorGroup output io.Writer ) - switch authExportCfg.Out { + switch config.Out { case "-": output = os.Stdout default: - if err := os.MkdirAll(filepath.Dir(authExportCfg.Out), 0600); err != nil { + if err := os.MkdirAll(filepath.Dir(config.Out), 0600); err != nil { return errs.Wrap(err) } - output, err = os.OpenFile(authExportCfg.Out, os.O_CREATE, 0600) + output, err = os.OpenFile(config.Out, os.O_CREATE, 0600) if err != nil { return errs.Wrap(err) } diff --git a/cmd/certificates/main.go b/cmd/certificates/main.go index 0126ee037..9b3a06f81 100644 --- a/cmd/certificates/main.go +++ b/cmd/certificates/main.go @@ -5,10 +5,12 @@ package main import ( "github.com/spf13/cobra" + "go.uber.org/zap" "storj.io/storj/internal/fpath" "storj.io/storj/pkg/certificates" "storj.io/storj/pkg/cfgstruct" + "storj.io/storj/pkg/identity" "storj.io/storj/pkg/process" "storj.io/storj/pkg/server" ) @@ -30,30 +32,49 @@ var ( RunE: cmdRun, } - runCfg struct { - Signer certificates.CertServerConfig - Server server.Config + config struct { + batchCfg + CA identity.CASetupConfig + Identity identity.SetupConfig + Server server.Config + Signer certificates.CertServerConfig + All bool `help:"print the all authorizations for auth info/export subcommands" default:"false"` + Out string `help:"output file path for auth export subcommand; if \"-\", will use STDOUT" default:"-"` + ShowTokens bool `help:"if true, token strings will be printed for auth info command" default:"false"` + Overwrite bool `default:"false" help:"if true ca, identity, and authorization db will be overwritten/truncated"` } - defaultConfDir = fpath.ApplicationDir("storj", "cert-signing") - confDir *string + defaultConfDir = fpath.ApplicationDir("storj", "cert-signing") + defaultIdentityDir = fpath.ApplicationDir("storj", "identity", "certificates") + confDir string + identityDir string ) func init() { - dirParam := cfgstruct.FindConfigDirParam() - if dirParam != "" { - defaultConfDir = dirParam + confDirParam := cfgstruct.FindConfigDirParam() + if confDirParam != "" { + defaultIdentityDir = confDirParam } - confDir = rootCmd.PersistentFlags().String("config-dir", defaultConfDir, "main directory for certificates configuration") + identityDirParam := cfgstruct.FindCredsDirParam() + if identityDirParam != "" { + defaultIdentityDir = identityDirParam + } + + rootCmd.PersistentFlags().StringVar(&confDir, "config-dir", defaultConfDir, "main directory for certificates configuration") + err := rootCmd.PersistentFlags().SetAnnotation("config-dir", "setup", []string{"true"}) + if err != nil { + zap.S().Error("Failed to set 'setup' annotation for 'config-dir'") + } + rootCmd.PersistentFlags().StringVar(&identityDir, "identity-dir", defaultIdentityDir, "main directory for storagenode identity credentials") rootCmd.AddCommand(runCmd) - cfgstruct.Bind(runCmd.Flags(), &runCfg, cfgstruct.ConfDir(defaultConfDir)) + cfgstruct.Bind(runCmd.Flags(), &config, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir)) } func cmdRun(cmd *cobra.Command, args []string) error { ctx := process.Ctx(cmd) - return runCfg.Server.Run(ctx, nil, runCfg.Signer) + return config.Server.Run(ctx, nil, config.Signer) } func main() { diff --git a/cmd/certificates/setup.go b/cmd/certificates/setup.go index 9920392da..c4bbbe1b8 100644 --- a/cmd/certificates/setup.go +++ b/cmd/certificates/setup.go @@ -12,7 +12,6 @@ import ( "github.com/spf13/cobra" "storj.io/storj/internal/fpath" - "storj.io/storj/pkg/certificates" "storj.io/storj/pkg/cfgstruct" "storj.io/storj/pkg/identity" "storj.io/storj/pkg/process" @@ -25,24 +24,15 @@ var ( RunE: cmdSetup, Annotations: map[string]string{"type": "setup"}, } - - setupCfg struct { - // NB: cert and key paths overridden in setup - CA identity.CASetupConfig - // NB: cert and key paths overridden in setup - Identity identity.SetupConfig - Signer certificates.CertServerConfig - Overwrite bool `default:"false" help:"if true ca, identity, and authorization db will be overwritten/truncated"` - } ) func init() { rootCmd.AddCommand(setupCmd) - cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir)) + cfgstruct.BindSetup(setupCmd.Flags(), &config, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir)) } func cmdSetup(cmd *cobra.Command, args []string) error { - setupDir, err := filepath.Abs(*confDir) + setupDir, err := filepath.Abs(confDir) if err != nil { return err } @@ -56,36 +46,30 @@ func cmdSetup(cmd *cobra.Command, args []string) error { if err != nil { return err } - if !setupCfg.Overwrite && !valid { + if !config.Overwrite && !valid { fmt.Printf("certificate signer configuration already exists (%v). rerun with --overwrite\n", setupDir) return nil } - if setupCfg.Overwrite { - setupCfg.CA.Overwrite = true - setupCfg.Identity.Overwrite = true - setupCfg.Signer.Overwrite = true + if config.Overwrite { + config.CA.Overwrite = true + config.Identity.Overwrite = true + config.Signer.Overwrite = true } - if _, err := setupCfg.Signer.NewAuthDB(); err != nil { + if _, err := config.Signer.NewAuthDB(); err != nil { return err } - if setupDir != defaultConfDir { - setupCfg.CA.CertPath = filepath.Join(setupDir, "ca.cert") - setupCfg.CA.KeyPath = filepath.Join(setupDir, "ca.key") - setupCfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert") - setupCfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key") - } - if setupCfg.Identity.Status() != identity.CertKey { + if config.Identity.Status() != identity.CertKey { return errors.New("identity is missing") } o := map[string]interface{}{ - "ca.cert-path": setupCfg.CA.CertPath, - "ca.key-path": setupCfg.CA.KeyPath, - "identity.cert-path": setupCfg.Identity.CertPath, - "identity.key-path": setupCfg.Identity.KeyPath, + "ca.cert-path": config.CA.CertPath, + "ca.key-path": config.CA.KeyPath, + "identity.cert-path": config.Identity.CertPath, + "identity.key-path": config.Identity.KeyPath, "log.level": "info", } return process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), o) diff --git a/cmd/identity/certificate_authority.go b/cmd/identity/certificate_authority.go index d044922aa..3ea5a25e7 100644 --- a/cmd/identity/certificate_authority.go +++ b/cmd/identity/certificate_authority.go @@ -74,10 +74,10 @@ func init() { caCmd.AddCommand(caExtCmd) caCmd.AddCommand(revokeCACmd) - cfgstruct.Bind(newCACmd.Flags(), &newCACfg, cfgstruct.ConfDir(defaultConfDir)) - cfgstruct.Bind(getIDCmd.Flags(), &getIDCfg, cfgstruct.ConfDir(defaultConfDir)) - cfgstruct.Bind(caExtCmd.Flags(), &caExtCfg, cfgstruct.ConfDir(defaultConfDir)) - cfgstruct.Bind(revokeCACmd.Flags(), &revokeCACfg, cfgstruct.ConfDir(defaultConfDir)) + cfgstruct.Bind(newCACmd.Flags(), &newCACfg, cfgstruct.CredsDir(defaultIdentityDir)) + cfgstruct.Bind(getIDCmd.Flags(), &getIDCfg, cfgstruct.CredsDir(defaultIdentityDir)) + cfgstruct.Bind(caExtCmd.Flags(), &caExtCfg, cfgstruct.CredsDir(defaultIdentityDir)) + cfgstruct.Bind(revokeCACmd.Flags(), &revokeCACfg, cfgstruct.CredsDir(defaultIdentityDir)) } func cmdNewCA(cmd *cobra.Command, args []string) error { diff --git a/cmd/identity/identity.go b/cmd/identity/identity.go index 9d2a27ced..b39249eb3 100644 --- a/cmd/identity/identity.go +++ b/cmd/identity/identity.go @@ -58,11 +58,12 @@ var ( func init() { rootCmd.AddCommand(idCmd) idCmd.AddCommand(newIDCmd) - cfgstruct.Bind(newIDCmd.Flags(), &newIDCfg, cfgstruct.ConfDir(defaultConfDir)) idCmd.AddCommand(leafExtCmd) - cfgstruct.Bind(leafExtCmd.Flags(), &leafExtCfg, cfgstruct.ConfDir(defaultConfDir)) idCmd.AddCommand(revokeLeafCmd) - cfgstruct.Bind(revokeLeafCmd.Flags(), &revokeLeafCfg, cfgstruct.ConfDir(defaultConfDir)) + + cfgstruct.Bind(newIDCmd.Flags(), &newIDCfg, cfgstruct.CredsDir(defaultIdentityDir)) + cfgstruct.Bind(leafExtCmd.Flags(), &leafExtCfg, cfgstruct.CredsDir(defaultIdentityDir)) + cfgstruct.Bind(revokeLeafCmd.Flags(), &revokeLeafCfg, cfgstruct.CredsDir(defaultIdentityDir)) } func cmdNewID(cmd *cobra.Command, args []string) (err error) { diff --git a/cmd/identity/main.go b/cmd/identity/main.go index a9062872e..df86088fb 100644 --- a/cmd/identity/main.go +++ b/cmd/identity/main.go @@ -11,7 +11,6 @@ import ( "github.com/spf13/cobra" "github.com/zeebo/errs" - "go.uber.org/zap" "storj.io/storj/internal/fpath" "storj.io/storj/pkg/certificates" @@ -52,27 +51,18 @@ var ( Signer certificates.CertClientConfig } - confDir string - defaultConfDir = fpath.ApplicationDir("storj", "identity") + identityDir string + defaultIdentityDir = fpath.ApplicationDir("storj", "identity") ) func init() { - dirParam := cfgstruct.FindConfigDirParam() - if dirParam != "" { - defaultConfDir = dirParam - } - - rootCmd.PersistentFlags().StringVar(&confDir, "config-dir", defaultConfDir, "main directory for storagenode configuration") - err := rootCmd.PersistentFlags().SetAnnotation("config-dir", "setup", []string{"true"}) - if err != nil { - zap.S().Error("Failed to set 'setup' annotation for 'config-dir'") - } + rootCmd.PersistentFlags().StringVar(&identityDir, "identity-dir", defaultIdentityDir, "root directory for identity output") rootCmd.AddCommand(newServiceCmd) rootCmd.AddCommand(csrCmd) - cfgstruct.Bind(newServiceCmd.Flags(), &config, cfgstruct.ConfDir(defaultConfDir)) - cfgstruct.Bind(csrCmd.Flags(), &config, cfgstruct.ConfDir(defaultConfDir)) + cfgstruct.Bind(newServiceCmd.Flags(), &config, cfgstruct.CredsDir(defaultIdentityDir)) + cfgstruct.Bind(csrCmd.Flags(), &config, cfgstruct.CredsDir(defaultIdentityDir)) } func main() { @@ -80,7 +70,7 @@ func main() { } func serviceDirectory(serviceName string) string { - return filepath.Join(confDir, serviceName) + return filepath.Join(identityDir, serviceName) } func cmdNewService(cmd *cobra.Command, args []string) error { diff --git a/cmd/satellite/main.go b/cmd/satellite/main.go index fb84b9b59..468e20fe6 100644 --- a/cmd/satellite/main.go +++ b/cmd/satellite/main.go @@ -5,7 +5,6 @@ package main import ( "context" - "errors" "fmt" "os" "path/filepath" @@ -27,7 +26,6 @@ import ( "storj.io/storj/pkg/datarepair/checker" "storj.io/storj/pkg/datarepair/repairer" "storj.io/storj/pkg/discovery" - "storj.io/storj/pkg/identity" "storj.io/storj/pkg/kademlia" "storj.io/storj/pkg/overlay" "storj.io/storj/pkg/payments" @@ -42,9 +40,6 @@ import ( // Satellite defines satellite configuration type Satellite struct { - CA identity.CASetupConfig `setup:"true"` - Identity identity.SetupConfig `setup:"true"` - Server server.Config Kademlia kademlia.SatelliteConfig PointerDB pointerdb.Config @@ -99,32 +94,49 @@ var ( QListLimit int `help:"maximum segments that can be requested" default:"1000"` } - defaultConfDir string - confDir *string + defaultConfDir = fpath.ApplicationDir("storj", "satellite") + // TODO: this path should be defined somewhere else + defaultIdentityDir = fpath.ApplicationDir("storj", "identity", "satellite") + confDir string + identityDir string ) func init() { - defaultConfDir = fpath.ApplicationDir("storj", "satellite") - - dirParam := cfgstruct.FindConfigDirParam() - if dirParam != "" { - defaultConfDir = dirParam + confDirParam := cfgstruct.FindConfigDirParam() + if confDirParam != "" { + defaultConfDir = confDirParam + } + identityDirParam := cfgstruct.FindCredsDirParam() + if identityDirParam != "" { + defaultIdentityDir = identityDirParam } - confDir = rootCmd.PersistentFlags().String("config-dir", defaultConfDir, "main directory for satellite configuration") + rootCmd.PersistentFlags().StringVar(&confDir, "config-dir", defaultConfDir, "main directory for satellite configuration") + err := rootCmd.PersistentFlags().SetAnnotation("config-dir", "setup", []string{"true"}) + if err != nil { + zap.S().Error("Failed to set 'setup' annotation for 'config-dir'") + } + rootCmd.PersistentFlags().StringVar(&identityDir, "identity-dir", defaultIdentityDir, "main directory for satellite identity credentials") + err = rootCmd.PersistentFlags().SetAnnotation("identity-dir", "setup", []string{"true"}) + if err != nil { + zap.S().Error("Failed to set 'setup' annotation for 'config-dir'") + } rootCmd.AddCommand(runCmd) rootCmd.AddCommand(setupCmd) rootCmd.AddCommand(diagCmd) rootCmd.AddCommand(qdiagCmd) - cfgstruct.Bind(runCmd.Flags(), &runCfg, cfgstruct.ConfDir(defaultConfDir)) - cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir)) - cfgstruct.Bind(diagCmd.Flags(), &diagCfg, cfgstruct.ConfDir(defaultConfDir)) - cfgstruct.Bind(qdiagCmd.Flags(), &qdiagCfg, cfgstruct.ConfDir(defaultConfDir)) + cfgstruct.Bind(runCmd.Flags(), &runCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir)) + cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir)) + cfgstruct.Bind(diagCmd.Flags(), &diagCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir)) + cfgstruct.Bind(qdiagCmd.Flags(), &qdiagCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir)) } func cmdRun(cmd *cobra.Command, args []string) (err error) { ctx := process.Ctx(cmd) + if _, err := runCfg.Server.Identity.Load(); err != nil { + zap.S().Fatal(err) + } database, err := satellitedb.New(runCfg.Database) if err != nil { @@ -161,7 +173,7 @@ func cmdRun(cmd *cobra.Command, args []string) (err error) { } func cmdSetup(cmd *cobra.Command, args []string) (err error) { - setupDir, err := filepath.Abs(*confDir) + setupDir, err := filepath.Abs(confDir) if err != nil { return err } @@ -176,24 +188,7 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) { return err } - // TODO: handle setting base path *and* identity file paths via args - // NB: if base path is set this overrides identity and CA path options - if setupDir != defaultConfDir { - setupCfg.CA.CertPath = filepath.Join(setupDir, "ca.cert") - setupCfg.CA.KeyPath = filepath.Join(setupDir, "ca.key") - setupCfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert") - setupCfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key") - } - if setupCfg.Identity.Status() != identity.CertKey { - return errors.New("identity is missing") - } - - o := map[string]interface{}{ - "identity.cert-path": setupCfg.Identity.CertPath, - "identity.key-path": setupCfg.Identity.KeyPath, - } - - return process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), o) + return process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), nil) } func cmdDiag(cmd *cobra.Command, args []string) (err error) { diff --git a/cmd/storagenode/main.go b/cmd/storagenode/main.go index 358129f75..edbe323e9 100644 --- a/cmd/storagenode/main.go +++ b/cmd/storagenode/main.go @@ -86,11 +86,12 @@ var ( diagCfg struct { } - defaultConfDir string - defaultDiagDir string - defaultCredsDir string - confDir string - credsDir string + defaultConfDir = fpath.ApplicationDir("storj", "storagenode") + // TODO: this path should be defined somewhere else + defaultIdentityDir = fpath.ApplicationDir("storj", "identity", "storagenode") + defaultDiagDir string + confDir string + identityDir string ) const ( @@ -98,13 +99,13 @@ const ( ) func init() { - defaultConfDir = fpath.ApplicationDir("storj", "storagenode") - // TODO: this path should be defined somewhere else - defaultCredsDir = fpath.ApplicationDir("storj", "identity") - - dirParam := cfgstruct.FindConfigDirParam() - if dirParam != "" { - defaultConfDir = dirParam + confDirParam := cfgstruct.FindConfigDirParam() + if confDirParam != "" { + defaultConfDir = confDirParam + } + identityDirParam := cfgstruct.FindCredsDirParam() + if identityDirParam != "" { + defaultIdentityDir = identityDirParam } rootCmd.PersistentFlags().StringVar(&confDir, "config-dir", defaultConfDir, "main directory for storagenode configuration") @@ -112,7 +113,11 @@ func init() { if err != nil { zap.S().Error("Failed to set 'setup' annotation for 'config-dir'") } - rootCmd.PersistentFlags().StringVar(&credsDir, "creds-dir", defaultCredsDir, "main directory for storagenode identity credentials") + rootCmd.PersistentFlags().StringVar(&identityDir, "identity-dir", defaultIdentityDir, "main directory for storagenode identity credentials") + err = rootCmd.PersistentFlags().SetAnnotation("identity-dir", "setup", []string{"true"}) + if err != nil { + zap.S().Error("Failed to set 'setup' annotation for 'config-dir'") + } defaultDiagDir = filepath.Join(defaultConfDir, "storage") rootCmd.AddCommand(runCmd) @@ -120,18 +125,16 @@ func init() { rootCmd.AddCommand(configCmd) rootCmd.AddCommand(diagCmd) rootCmd.AddCommand(dashboardCmd) - cfgstruct.Bind(runCmd.Flags(), &runCfg, cfgstruct.ConfDir(defaultConfDir)) - cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir)) - cfgstruct.BindSetup(configCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir)) - cfgstruct.Bind(diagCmd.Flags(), &diagCfg, cfgstruct.ConfDir(defaultDiagDir)) + cfgstruct.Bind(runCmd.Flags(), &runCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir)) + cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir)) + cfgstruct.BindSetup(configCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir)) + cfgstruct.Bind(diagCmd.Flags(), &diagCfg, cfgstruct.ConfDir(defaultDiagDir), cfgstruct.CredsDir(defaultIdentityDir)) cfgstruct.Bind(dashboardCmd.Flags(), &dashboardCfg, cfgstruct.ConfDir(defaultDiagDir)) } func cmdRun(cmd *cobra.Command, args []string) (err error) { - if ident, err := runCfg.Server.Identity.Load(); err != nil { + if _, err := runCfg.Server.Identity.Load(); err != nil { zap.S().Fatal(err) - } else { - zap.S().Info("Node ID: ", ident.ID) } operatorConfig := runCfg.Kademlia.Operator diff --git a/cmd/storj-sim/network.go b/cmd/storj-sim/network.go index 008ef9960..138444303 100644 --- a/cmd/storj-sim/network.go +++ b/cmd/storj-sim/network.go @@ -126,7 +126,9 @@ func newNetwork(flags *Flags) (*Processes, error) { }) bootstrap.Arguments = withCommon(Arguments{ - "setup": {}, + "setup": { + "--identity-dir", bootstrap.Directory, + }, "run": { "--kademlia.bootstrap-addr", bootstrap.Address, "--kademlia.operator.email", "bootstrap@example.com", @@ -150,7 +152,9 @@ func newNetwork(flags *Flags) (*Processes, error) { process.WaitForStart(bootstrap) process.Arguments = withCommon(Arguments{ - "setup": {}, + "setup": { + "--identity-dir", process.Directory, + }, "run": { "--kademlia.bootstrap-addr", bootstrap.Address, "--server.address", process.Address, @@ -181,6 +185,7 @@ func newNetwork(flags *Flags) (*Processes, error) { process.Arguments = withCommon(Arguments{ "setup": { + "--identity-dir", process.Directory, "--satellite-addr", satellite.Address, }, "run": { @@ -212,7 +217,9 @@ func newNetwork(flags *Flags) (*Processes, error) { process.WaitForStart(bootstrap) process.Arguments = withCommon(Arguments{ - "setup": {}, + "setup": { + "--identity-dir", process.Directory, + }, "run": { "--kademlia.bootstrap-addr", bootstrap.Address, "--kademlia.operator.email", fmt.Sprintf("storage%d@example.com", i), @@ -263,7 +270,7 @@ func identitySetup(network *Processes) (*Processes, error) { identity.Arguments = Arguments{ "setup": { - "--config-dir", process.Directory, + "--identity-dir", process.Directory, "new", ".", }, } diff --git a/cmd/uplink/cmd/root.go b/cmd/uplink/cmd/root.go index 8c474f1ec..f8af702b4 100644 --- a/cmd/uplink/cmd/root.go +++ b/cmd/uplink/cmd/root.go @@ -39,13 +39,18 @@ func addCmd(cmd *cobra.Command, root *cobra.Command) *cobra.Command { root.AddCommand(cmd) defaultConfDir := fpath.ApplicationDir("storj", "uplink") + defaultIdentityDir := fpath.ApplicationDir("storj", "identity", "uplink") - dirParam := cfgstruct.FindConfigDirParam() - if dirParam != "" { - defaultConfDir = dirParam + confDirParam := cfgstruct.FindConfigDirParam() + if confDirParam != "" { + defaultConfDir = confDirParam + } + identityDirParam := cfgstruct.FindCredsDirParam() + if identityDirParam != "" { + defaultIdentityDir = identityDirParam } - cfgstruct.Bind(cmd.Flags(), &cfg, cfgstruct.ConfDir(defaultConfDir)) + cfgstruct.Bind(cmd.Flags(), &cfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir)) return cmd } diff --git a/cmd/uplink/cmd/run.go b/cmd/uplink/cmd/run.go index f938e58b1..8e33ad866 100644 --- a/cmd/uplink/cmd/run.go +++ b/cmd/uplink/cmd/run.go @@ -23,6 +23,10 @@ func init() { } func cmdRun(cmd *cobra.Command, args []string) (err error) { + if _, err := cfg.Identity.Load(); err != nil { + zap.S().Fatal(err) + } + for _, flagname := range args { return fmt.Errorf("Invalid argument %#v. Try 'uplink run'", flagname) } diff --git a/cmd/uplink/cmd/setup.go b/cmd/uplink/cmd/setup.go index 49a510ced..d1749e972 100644 --- a/cmd/uplink/cmd/setup.go +++ b/cmd/uplink/cmd/setup.go @@ -11,13 +11,14 @@ import ( base58 "github.com/jbenet/go-base58" "github.com/spf13/cobra" + "go.uber.org/zap" + + "storj.io/storj/pkg/provider" "storj.io/storj/internal/fpath" "storj.io/storj/pkg/cfgstruct" - "storj.io/storj/pkg/identity" "storj.io/storj/pkg/miniogw" "storj.io/storj/pkg/process" - "storj.io/storj/pkg/provider" ) var ( @@ -28,12 +29,11 @@ var ( Annotations: map[string]string{"type": "setup"}, } setupCfg struct { - CA provider.CASetupConfig `setup:"true"` - Identity provider.IdentitySetupConfig `setup:"true"` - APIKey string `default:"" help:"the api key to use for the satellite" setup:"true"` - EncKey string `default:"" help:"your root encryption key" setup:"true"` - GenerateMinioCerts bool `default:"false" help:"generate sample TLS certs for Minio GW" setup:"true"` - SatelliteAddr string `default:"localhost:7778" help:"the address to use for the satellite" setup:"true"` + Identity provider.IdentitySetupConfig + APIKey string `default:"" help:"the api key to use for the satellite" setup:"true"` + EncKey string `default:"" help:"your root encryption key" setup:"true"` + GenerateMinioCerts bool `default:"false" help:"generate sample TLS certs for Minio GW" setup:"true"` + SatelliteAddr string `default:"localhost:7778" help:"the address to use for the satellite" setup:"true"` Server miniogw.ServerConfig Minio miniogw.MinioConfig @@ -42,28 +42,44 @@ var ( Enc miniogw.EncryptionConfig } - cliConfDir *string - gwConfDir *string + defaultConfDir = fpath.ApplicationDir("storj", "uplink") + defaultIdentityDir = fpath.ApplicationDir("storj", "identity", "uplink") + + cliConfDir string + gwConfDir string + identityDir string ) func init() { - defaultConfDir := fpath.ApplicationDir("storj", "uplink") - - dirParam := cfgstruct.FindConfigDirParam() - if dirParam != "" { - defaultConfDir = dirParam + confDirParam := cfgstruct.FindConfigDirParam() + if confDirParam != "" { + defaultConfDir = confDirParam + } + identityDirParam := cfgstruct.FindCredsDirParam() + if identityDirParam != "" { + defaultIdentityDir = identityDirParam } - cliConfDir = CLICmd.PersistentFlags().String("config-dir", defaultConfDir, "main directory for setup configuration") - gwConfDir = GWCmd.PersistentFlags().String("config-dir", defaultConfDir, "main directory for setup configuration") + CLICmd.PersistentFlags().StringVar(&cliConfDir, "config-dir", defaultConfDir, "main directory for setup configuration") + GWCmd.PersistentFlags().StringVar(&gwConfDir, "config-dir", defaultConfDir, "main directory for setup configuration") + CLICmd.PersistentFlags().StringVar(&identityDir, "identity-dir", defaultIdentityDir, "main directory for uplink identity credentials") + err := CLICmd.PersistentFlags().SetAnnotation("identity-dir", "setup", []string{"true"}) + if err != nil { + zap.S().Error("Failed to set 'setup' annotation for 'config-dir'") + } + GWCmd.PersistentFlags().StringVar(&identityDir, "identity-dir", defaultIdentityDir, "main directory for gateway identity credentials") + err = GWCmd.PersistentFlags().SetAnnotation("identity-dir", "setup", []string{"true"}) + if err != nil { + zap.S().Error("Failed to set 'setup' annotation for 'config-dir'") + } CLICmd.AddCommand(setupCmd) GWCmd.AddCommand(setupCmd) - cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir)) + cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir)) } func cmdSetup(cmd *cobra.Command, args []string) (err error) { - setupDir, err := filepath.Abs(*cliConfDir) + setupDir, err := filepath.Abs(cliConfDir) if err != nil { return err } @@ -82,25 +98,6 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) { return err } - defaultConfDir := fpath.ApplicationDir("storj", "uplink") - // TODO: handle setting base path *and* identity file paths via args - // NB: if base path is set this overrides identity and CA path options - if setupDir != defaultConfDir { - setupCfg.CA.CertPath = filepath.Join(setupDir, "ca.cert") - setupCfg.CA.KeyPath = filepath.Join(setupDir, "ca.key") - setupCfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert") - setupCfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key") - } - - if setupCfg.Identity.Status() == identity.CertKey { - // identity already exists - } else { - err = provider.SetupIdentity(process.Ctx(cmd), setupCfg.CA, setupCfg.Identity) - if err != nil { - return err - } - } - if setupCfg.GenerateMinioCerts { minioCerts := filepath.Join(setupDir, "minio", "certs") if err := os.MkdirAll(minioCerts, 0744); err != nil { @@ -125,8 +122,6 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) { } o := map[string]interface{}{ - "identity.cert-path": setupCfg.Identity.CertPath, - "identity.key-path": setupCfg.Identity.KeyPath, "client.api-key": setupCfg.APIKey, "client.pointer-db-addr": setupCfg.SatelliteAddr, "client.overlay-addr": setupCfg.SatelliteAddr, diff --git a/pkg/cfgstruct/bind.go b/pkg/cfgstruct/bind.go index 7c31ae9dc..e869789c9 100644 --- a/pkg/cfgstruct/bind.go +++ b/pkg/cfgstruct/bind.go @@ -29,6 +29,14 @@ func ConfDir(confdir string) BindOpt { }) } +// CredsDir sets a variable for the default option called $CREDSDIR. +func CredsDir(credsdir string) BindOpt { + val := filepath.Clean(os.ExpandEnv(credsdir)) + return BindOpt(func(vars map[string]confVar) { + vars["CREDSDIR"] = confVar{val: val, nested: false} + }) +} + // ConfDirNested sets variables for default options called $CONFDIR and $CONFNAME. // ConfDirNested also appends the parent struct field name to the paths before // descending into substructs. @@ -187,11 +195,21 @@ func expand(vars map[string]string, val string) string { // FindConfigDirParam returns '--config-dir' param from os.Args (if exists) func FindConfigDirParam() string { + return FindFlagEarly("config-dir") +} + +// FindCredsDirParam returns '--identity-dir' param from os.Args (if exists) +func FindCredsDirParam() string { + return FindFlagEarly("identity-dir") +} + +// FindFlagEarly retrieves the value of a flag before `flag.Parse` has been called +func FindFlagEarly(flagName string) string { // workaround to have early access to 'dir' param for i, arg := range os.Args { - if strings.HasPrefix(arg, "--config-dir=") { - return strings.TrimPrefix(arg, "--config-dir=") - } else if arg == "--config-dir" && i < len(os.Args)-1 { + if strings.HasPrefix(arg, fmt.Sprintf("--%s=", flagName)) { + return strings.TrimPrefix(arg, fmt.Sprintf("--%s=", flagName)) + } else if arg == fmt.Sprintf("--%s", flagName) && i < len(os.Args)-1 { return os.Args[i+1] } } diff --git a/pkg/identity/certificate_authority.go b/pkg/identity/certificate_authority.go index 5a42bcd47..bedf656e3 100644 --- a/pkg/identity/certificate_authority.go +++ b/pkg/identity/certificate_authority.go @@ -49,8 +49,8 @@ type FullCertificateAuthority struct { type CASetupConfig struct { ParentCertPath string `help:"path to the parent authority's certificate chain"` ParentKeyPath string `help:"path to the parent authority's private key"` - CertPath string `help:"path to the certificate chain for this identity" default:"$CONFDIR/ca.cert"` - KeyPath string `help:"path to the private key for this identity" default:"$CONFDIR/ca.key"` + CertPath string `help:"path to the certificate chain for this identity" default:"$CREDSDIR/ca.cert"` + KeyPath string `help:"path to the private key for this identity" default:"$CREDSDIR/ca.key"` Difficulty uint64 `help:"minimum difficulty for identity generation" default:"15"` Timeout string `help:"timeout for CA generation; golang duration string (0 no timeout)" default:"5m"` Overwrite bool `help:"if true, existing CA certs AND keys will overwritten" default:"false"` @@ -71,13 +71,13 @@ type NewCAOptions struct { // PeerCAConfig is for locating a CA certificate without a private key type PeerCAConfig struct { - CertPath string `help:"path to the certificate chain for this identity" default:"$CONFDIR/ca.cert"` + CertPath string `help:"path to the certificate chain for this identity" default:"$CREDSDIR/ca.cert"` } // FullCAConfig is for locating a CA certificate and it's private key type FullCAConfig struct { - CertPath string `help:"path to the certificate chain for this identity" default:"$CONFDIR/ca.cert"` - KeyPath string `help:"path to the private key for this identity" default:"$CONFDIR/ca.key"` + CertPath string `help:"path to the certificate chain for this identity" default:"$CREDSDIR/ca.cert"` + KeyPath string `help:"path to the private key for this identity" default:"$CREDSDIR/ca.key"` } // NewCA creates a new full identity with the given difficulty diff --git a/pkg/identity/identity.go b/pkg/identity/identity.go index 97e1cb535..1b0ac8a4f 100644 --- a/pkg/identity/identity.go +++ b/pkg/identity/identity.go @@ -58,8 +58,8 @@ type FullIdentity struct { // SetupConfig allows you to run a set of Responsibilities with the given // identity. You can also just load an Identity from disk. type SetupConfig struct { - CertPath string `help:"path to the certificate chain for this identity" default:"$CONFDIR/identity.cert"` - KeyPath string `help:"path to the private key for this identity" default:"$CONFDIR/identity.key"` + CertPath string `help:"path to the certificate chain for this identity" default:"$CREDSDIR/identity.cert"` + KeyPath string `help:"path to the private key for this identity" default:"$CREDSDIR/identity.key"` Overwrite bool `help:"if true, existing identity certs AND keys will overwritten for" default:"false"` Version string `help:"semantic version of identity storage format" default:"0"` } @@ -67,8 +67,8 @@ type SetupConfig struct { // Config allows you to run a set of Responsibilities with the given // identity. You can also just load an Identity from disk. type Config struct { - CertPath string `help:"path to the certificate chain for this identity" default:"$CONFDIR/identity.cert"` - KeyPath string `help:"path to the private key for this identity" default:"$CONFDIR/identity.key"` + CertPath string `help:"path to the certificate chain for this identity" default:"$CREDSDIR/identity.cert" user:"true"` + KeyPath string `help:"path to the private key for this identity" default:"$CREDSDIR/identity.key" user:"true"` } // FullIdentityFromPEM loads a FullIdentity from a certificate chain and diff --git a/scripts/cert-gating.sh b/scripts/cert-gating.sh index 015a7efc9..eefef314b 100755 --- a/scripts/cert-gating.sh +++ b/scripts/cert-gating.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash source $(dirname $0)/utils.sh -basepath=$HOME/.storj/capt +basepath=$HOME/.storj/local-network alpha_config=$basepath/config-alpha.yaml unauthorized_config=$basepath/config-unauthorized.yaml ca_whitelist=$basepath/ca-alpha-whitelist.cert @@ -21,9 +21,9 @@ case $1 in echo "usage: $(basename $0) [setup|alpha|unauthorized]" ;; setup) - temp_build captplanet identity - echo "setting up captplanet" - "$captplanet" setup + temp_build "storj-sim" identity + echo "setting up storj-sim" + ${storj_sim} network setup echo "clearing whitelist" echo > ${ca_whitelist} echo -n "generating alpha certificate authorities.." @@ -37,7 +37,7 @@ case $1 in done echo "done" echo -n "generating alpha identities" - for dir in ${basepath}/{f*,sat*,up*}; do + for dir in ${basepath}/{bootstrap/*,satellite/*,storagenode/*,gateway/*}; do echo -n "." _ca_basepath=$(rand_ca_basepath) _ca_cert=${dir}/ca-alpha.cert @@ -64,15 +64,15 @@ case $1 in cat ${basepath}/config.yaml | sed -E "s,peer-ca-whitelist-path: \"\",peer-ca-whitelist-path: $ca_whitelist,g" >"$unauthorized_config" ;; alpha) - build captplanet - ${captplanet} run --config ${alpha_config} + build "storj-sim" + ${storj_sim} network run --config ${alpha_config} ;; unauthorized) - build captplanet - ${captplanet} run --config ${unauthorized_config} + build "storj-sim" + ${storj_sim} network run --config ${unauthorized_config} ;; run) - ${captplanet} run + ${storj_sim} network run ;; *) $@ --help diff --git a/scripts/utils.sh b/scripts/utils.sh index 307f7fd51..b48042fcc 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -31,7 +31,7 @@ build() { echo -n " building $cmd..." dots_on local path=${tmp_dir}/${cmd} - declare -g ${cmd}=${path} + declare -g $(echo $cmd | sed s,-,_,g)=${path} build_out=$(go build -o ${path} storj.io/storj/cmd/${cmd} 2>&1) dots_off echo "done"