JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

certificates config and --certs-dir fixes (#1093)

Browse Source
This commit is contained in:
Bryan White 2019-01-22 13:35:48 +01:00 committed by GitHub
parent eb2d858ffc
commit 8edfd43abb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 263 additions and 260 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -57,7 +57,7 @@ goimports-fix: ## Applies goimports to every go file (excluding vendored files)
.PHONY: goimports-st
goimports-st: ## Applies goimports to every go file in `git status` (ignores untracked files)
git status --porcelain -uno|grep .go|grep -v "^D"|sed -E 's,\w+\s+(.+->\s+)?,,g'|xargs -I {} goimports -w -local storj.io {}
@git status --porcelain -uno|grep .go|grep -v "^D"|sed -E 's,\w+\s+(.+->\s+)?,,g'|xargs -I {} goimports -w -local storj.io {}
.PHONY: proto
proto: ## Rebuild protobuf files

59
cmd/bootstrap/main.go
View File

@ -4,7 +4,6 @@
package main
import (
"errors"
"fmt"
"os"
"path/filepath"
@ -14,7 +13,6 @@ import (
"storj.io/storj/internal/fpath"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/server"
@ -22,9 +20,6 @@ import (
// Bootstrap defines a bootstrap node configuration
type Bootstrap struct {
CA identity.CASetupConfig `setup:"true"`
Identity identity.SetupConfig `setup:"true"`
Server server.Config
Kademlia kademlia.BootstrapConfig
}
@ -48,8 +43,10 @@ var (
cfg Bootstrap
defaultConfDir string
confDir *string
defaultConfDir = fpath.ApplicationDir("storj", "bootstrap")
defaultIdentityDir = fpath.ApplicationDir("storj", "identity", "bootstrap")
confDir string
identityDir string
)
const (
@ -57,31 +54,46 @@ const (
)
func init() {
defaultConfDir = fpath.ApplicationDir("storj", "bootstrap")
dirParam := cfgstruct.FindConfigDirParam()
if dirParam != "" {
defaultConfDir = dirParam
confDirParam := cfgstruct.FindConfigDirParam()
if confDirParam != "" {
defaultConfDir = confDirParam
}
identityDirParam := cfgstruct.FindCredsDirParam()
if identityDirParam != "" {
defaultIdentityDir = identityDirParam
}
confDir = rootCmd.PersistentFlags().String("config-dir", defaultConfDir, "main directory for bootstrap configuration")
rootCmd.PersistentFlags().StringVar(&confDir, "config-dir", defaultConfDir, "main directory for bootstrap configuration")
err := rootCmd.PersistentFlags().SetAnnotation("config-dir", "setup", []string{"true"})
if err != nil {
zap.S().Error("Failed to set 'setup' annotation for 'config-dir'")
}
rootCmd.PersistentFlags().StringVar(&identityDir, "identity-dir", defaultIdentityDir, "main directory for bootstrap identity credentials")
err = rootCmd.PersistentFlags().SetAnnotation("identity-dir", "setup", []string{"true"})
if err != nil {
zap.S().Error("Failed to set 'setup' annotation for 'config-dir'")
}
rootCmd.AddCommand(runCmd)
rootCmd.AddCommand(setupCmd)
cfgstruct.Bind(runCmd.Flags(), &cfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.BindSetup(setupCmd.Flags(), &cfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(runCmd.Flags(), &cfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir))
cfgstruct.BindSetup(setupCmd.Flags(), &cfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir))
}
func cmdRun(cmd *cobra.Command, args []string) (err error) {
ctx := process.Ctx(cmd)
if err := process.InitMetricsWithCertPath(ctx, nil, cfg.Identity.CertPath); err != nil {
if _, err := cfg.Server.Identity.Load(); err != nil {
zap.S().Fatal(err)
}
if err := process.InitMetricsWithCertPath(ctx, nil, cfg.Server.Identity.CertPath); err != nil {
zap.S().Errorf("Failed to initialize telemetry batcher: %+v", err)
}
return cfg.Server.Run(ctx, nil, cfg.Kademlia)
}
func cmdSetup(cmd *cobra.Command, args []string) (err error) {
setupDir, err := filepath.Abs(*confDir)
setupDir, err := filepath.Abs(confDir)
if err != nil {
return err
}
@ -96,20 +108,7 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
return err
}
if setupDir != defaultConfDir {
cfg.CA.CertPath = filepath.Join(setupDir, "ca.cert")
cfg.CA.KeyPath = filepath.Join(setupDir, "ca.key")
cfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert")
cfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key")
}
if cfg.Identity.Status() != identity.CertKey {
return errors.New("identity is missing")
}
overrides := map[string]interface{}{
"identity.cert-path": cfg.Identity.CertPath,
"identity.key-path": cfg.Identity.KeyPath,
"identity.server.address": defaultServerAddr,
"kademlia.bootstrap-addr": "localhost" + defaultServerAddr,
}

59
cmd/certificates/auth.go
View File

@ -46,35 +46,16 @@ var (
Short: "Export authorization(s) from CSR authorization DB to a CSV file (or stdout)",
RunE: cmdExportAuth,
}
authCreateCfg struct {
Signer certificates.CertServerConfig
batchCfg
}
authInfoCfg struct {
All bool `help:"print the info for all authorizations" default:"false"`
ShowTokens bool `help:"if true, token strings will be printed" default:"false"`
Signer certificates.CertServerConfig
batchCfg
}
authExportCfg struct {
All bool `help:"export all authorizations" default:"false"`
Out string `help:"output file path; if \"-\", will use STDOUT" default:"-"`
Signer certificates.CertServerConfig
batchCfg
}
)
func init() {
rootCmd.AddCommand(authCmd)
authCmd.AddCommand(authCreateCmd)
cfgstruct.Bind(authCreateCmd.Flags(), &authCreateCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(authCreateCmd.Flags(), &config, cfgstruct.ConfDir(defaultConfDir))
authCmd.AddCommand(authInfoCmd)
cfgstruct.Bind(authInfoCmd.Flags(), &authInfoCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(authInfoCmd.Flags(), &config, cfgstruct.ConfDir(defaultConfDir))
authCmd.AddCommand(authExportCmd)
cfgstruct.Bind(authExportCmd.Flags(), &authExportCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(authExportCmd.Flags(), &config, cfgstruct.ConfDir(defaultConfDir))
}
func parseEmailsList(fileName, delimiter string) (emails []string, err error) {
@ -98,19 +79,19 @@ func cmdCreateAuth(cmd *cobra.Command, args []string) error {
if err != nil {
return errs.New("Count couldn't be parsed: %s", args[0])
}
authDB, err := authCreateCfg.Signer.NewAuthDB()
authDB, err := config.Signer.NewAuthDB()
if err != nil {
return err
}
var emails []string
if len(args) > 1 {
if authCreateCfg.EmailsPath != "" {
if config.EmailsPath != "" {
return errs.New("Either use `--emails-path` or positional args, not both.")
}
emails = args[1:]
} else {
emails, err = parseEmailsList(authCreateCfg.EmailsPath, authCreateCfg.Delimiter)
emails, err = parseEmailsList(config.EmailsPath, config.Delimiter)
if err != nil {
return errs.Wrap(err)
}
@ -126,26 +107,26 @@ func cmdCreateAuth(cmd *cobra.Command, args []string) error {
}
func cmdInfoAuth(cmd *cobra.Command, args []string) error {
authDB, err := authInfoCfg.Signer.NewAuthDB()
authDB, err := config.Signer.NewAuthDB()
if err != nil {
return err
}
var emails []string
if len(args) > 0 {
if authInfoCfg.EmailsPath != "" && !authInfoCfg.All {
if config.EmailsPath != "" && !config.All {
return errs.New("Either use `--emails-path` or positional args, not both.")
}
emails = args
} else if len(args) == 0 || authInfoCfg.All {
} else if len(args) == 0 || config.All {
emails, err = authDB.UserIDs()
if err != nil {
return err
}
} else if _, err := os.Stat(authInfoCfg.EmailsPath); err != nil {
} else if _, err := os.Stat(config.EmailsPath); err != nil {
return errs.New("Emails path error: %s", err)
} else {
emails, err = parseEmailsList(authInfoCfg.EmailsPath, authInfoCfg.Delimiter)
emails, err = parseEmailsList(config.EmailsPath, config.Delimiter)
if err != nil {
return errs.Wrap(err)
}
@ -189,7 +170,7 @@ func writeAuthInfo(authDB *certificates.AuthorizationDB, email string, w io.Writ
return err
}
if authInfoCfg.ShowTokens {
if config.ShowTokens {
if err := writeTokenInfo(claimed, open, w); err != nil {
return err
}
@ -222,24 +203,24 @@ func writeTokenInfo(claimed, open certificates.Authorizations, w io.Writer) erro
}
func cmdExportAuth(cmd *cobra.Command, args []string) error {
authDB, err := authExportCfg.Signer.NewAuthDB()
authDB, err := config.Signer.NewAuthDB()
if err != nil {
return err
}
var emails []string
if len(args) > 0 && !authExportCfg.All {
if authExportCfg.EmailsPath != "" {
if len(args) > 0 && !config.All {
if config.EmailsPath != "" {
return errs.New("Either use `--emails-path` or positional args, not both.")
}
emails = args
} else if len(args) == 0 || authExportCfg.All {
} else if len(args) == 0 || config.All {
emails, err = authDB.UserIDs()
if err != nil {
return err
}
} else {
emails, err = parseEmailsList(authExportCfg.EmailsPath, authExportCfg.Delimiter)
emails, err = parseEmailsList(config.EmailsPath, config.Delimiter)
if err != nil {
return errs.Wrap(err)
}
@ -249,14 +230,14 @@ func cmdExportAuth(cmd *cobra.Command, args []string) error {
emailErrs, csvErrs utils.ErrorGroup
output io.Writer
)
switch authExportCfg.Out {
switch config.Out {
case "-":
output = os.Stdout
default:
if err := os.MkdirAll(filepath.Dir(authExportCfg.Out), 0600); err != nil {
if err := os.MkdirAll(filepath.Dir(config.Out), 0600); err != nil {
return errs.Wrap(err)
}
output, err = os.OpenFile(authExportCfg.Out, os.O_CREATE, 0600)
output, err = os.OpenFile(config.Out, os.O_CREATE, 0600)
if err != nil {
return errs.Wrap(err)
}

43
cmd/certificates/main.go
View File

@ -5,10 +5,12 @@ package main
import (
"github.com/spf13/cobra"
"go.uber.org/zap"
"storj.io/storj/internal/fpath"
"storj.io/storj/pkg/certificates"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/server"
)
@ -30,30 +32,49 @@ var (
RunE: cmdRun,
}
runCfg struct {
Signer certificates.CertServerConfig
Server server.Config
config struct {
batchCfg
CA identity.CASetupConfig
Identity identity.SetupConfig
Server server.Config
Signer certificates.CertServerConfig
All bool `help:"print the all authorizations for auth info/export subcommands" default:"false"`
Out string `help:"output file path for auth export subcommand; if \"-\", will use STDOUT" default:"-"`
ShowTokens bool `help:"if true, token strings will be printed for auth info command" default:"false"`
Overwrite bool `default:"false" help:"if true ca, identity, and authorization db will be overwritten/truncated"`
}
defaultConfDir = fpath.ApplicationDir("storj", "cert-signing")
confDir *string
defaultConfDir = fpath.ApplicationDir("storj", "cert-signing")
defaultIdentityDir = fpath.ApplicationDir("storj", "identity", "certificates")
confDir string
identityDir string
)
func init() {
dirParam := cfgstruct.FindConfigDirParam()
if dirParam != "" {
defaultConfDir = dirParam
confDirParam := cfgstruct.FindConfigDirParam()
if confDirParam != "" {
defaultIdentityDir = confDirParam
}
confDir = rootCmd.PersistentFlags().String("config-dir", defaultConfDir, "main directory for certificates configuration")
identityDirParam := cfgstruct.FindCredsDirParam()
if identityDirParam != "" {
defaultIdentityDir = identityDirParam
}
rootCmd.PersistentFlags().StringVar(&confDir, "config-dir", defaultConfDir, "main directory for certificates configuration")
err := rootCmd.PersistentFlags().SetAnnotation("config-dir", "setup", []string{"true"})
if err != nil {
zap.S().Error("Failed to set 'setup' annotation for 'config-dir'")
}
rootCmd.PersistentFlags().StringVar(&identityDir, "identity-dir", defaultIdentityDir, "main directory for storagenode identity credentials")
rootCmd.AddCommand(runCmd)
cfgstruct.Bind(runCmd.Flags(), &runCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(runCmd.Flags(), &config, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir))
}
func cmdRun(cmd *cobra.Command, args []string) error {
ctx := process.Ctx(cmd)
return runCfg.Server.Run(ctx, nil, runCfg.Signer)
return config.Server.Run(ctx, nil, config.Signer)
}
func main() {

42
cmd/certificates/setup.go
View File

@ -12,7 +12,6 @@ import (
"github.com/spf13/cobra"
"storj.io/storj/internal/fpath"
"storj.io/storj/pkg/certificates"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/process"
@ -25,24 +24,15 @@ var (
RunE: cmdSetup,
Annotations: map[string]string{"type": "setup"},
}
setupCfg struct {
// NB: cert and key paths overridden in setup
CA identity.CASetupConfig
// NB: cert and key paths overridden in setup
Identity identity.SetupConfig
Signer certificates.CertServerConfig
Overwrite bool `default:"false" help:"if true ca, identity, and authorization db will be overwritten/truncated"`
}
)
func init() {
rootCmd.AddCommand(setupCmd)
cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.BindSetup(setupCmd.Flags(), &config, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir))
}
func cmdSetup(cmd *cobra.Command, args []string) error {
setupDir, err := filepath.Abs(*confDir)
setupDir, err := filepath.Abs(confDir)
if err != nil {
return err
}
@ -56,36 +46,30 @@ func cmdSetup(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
if !setupCfg.Overwrite && !valid {
if !config.Overwrite && !valid {
fmt.Printf("certificate signer configuration already exists (%v). rerun with --overwrite\n", setupDir)
return nil
}
if setupCfg.Overwrite {
setupCfg.CA.Overwrite = true
setupCfg.Identity.Overwrite = true
setupCfg.Signer.Overwrite = true
if config.Overwrite {
config.CA.Overwrite = true
config.Identity.Overwrite = true
config.Signer.Overwrite = true
}
if _, err := setupCfg.Signer.NewAuthDB(); err != nil {
if _, err := config.Signer.NewAuthDB(); err != nil {
return err
}
if setupDir != defaultConfDir {
setupCfg.CA.CertPath = filepath.Join(setupDir, "ca.cert")
setupCfg.CA.KeyPath = filepath.Join(setupDir, "ca.key")
setupCfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert")
setupCfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key")
}
if setupCfg.Identity.Status() != identity.CertKey {
if config.Identity.Status() != identity.CertKey {
return errors.New("identity is missing")
}
o := map[string]interface{}{
"ca.cert-path": setupCfg.CA.CertPath,
"ca.key-path": setupCfg.CA.KeyPath,
"identity.cert-path": setupCfg.Identity.CertPath,
"identity.key-path": setupCfg.Identity.KeyPath,
"ca.cert-path": config.CA.CertPath,
"ca.key-path": config.CA.KeyPath,
"identity.cert-path": config.Identity.CertPath,
"identity.key-path": config.Identity.KeyPath,
"log.level": "info",
}
return process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), o)

8
cmd/identity/certificate_authority.go
View File

@ -74,10 +74,10 @@ func init() {
caCmd.AddCommand(caExtCmd)
caCmd.AddCommand(revokeCACmd)
cfgstruct.Bind(newCACmd.Flags(), &newCACfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(getIDCmd.Flags(), &getIDCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(caExtCmd.Flags(), &caExtCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(revokeCACmd.Flags(), &revokeCACfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(newCACmd.Flags(), &newCACfg, cfgstruct.CredsDir(defaultIdentityDir))
cfgstruct.Bind(getIDCmd.Flags(), &getIDCfg, cfgstruct.CredsDir(defaultIdentityDir))
cfgstruct.Bind(caExtCmd.Flags(), &caExtCfg, cfgstruct.CredsDir(defaultIdentityDir))
cfgstruct.Bind(revokeCACmd.Flags(), &revokeCACfg, cfgstruct.CredsDir(defaultIdentityDir))
}
func cmdNewCA(cmd *cobra.Command, args []string) error {

7
cmd/identity/identity.go
View File

@ -58,11 +58,12 @@ var (
func init() {
rootCmd.AddCommand(idCmd)
idCmd.AddCommand(newIDCmd)
cfgstruct.Bind(newIDCmd.Flags(), &newIDCfg, cfgstruct.ConfDir(defaultConfDir))
idCmd.AddCommand(leafExtCmd)
cfgstruct.Bind(leafExtCmd.Flags(), &leafExtCfg, cfgstruct.ConfDir(defaultConfDir))
idCmd.AddCommand(revokeLeafCmd)
cfgstruct.Bind(revokeLeafCmd.Flags(), &revokeLeafCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(newIDCmd.Flags(), &newIDCfg, cfgstruct.CredsDir(defaultIdentityDir))
cfgstruct.Bind(leafExtCmd.Flags(), &leafExtCfg, cfgstruct.CredsDir(defaultIdentityDir))
cfgstruct.Bind(revokeLeafCmd.Flags(), &revokeLeafCfg, cfgstruct.CredsDir(defaultIdentityDir))
}
func cmdNewID(cmd *cobra.Command, args []string) (err error) {

22
cmd/identity/main.go
View File

@ -11,7 +11,6 @@ import (
"github.com/spf13/cobra"
"github.com/zeebo/errs"
"go.uber.org/zap"
"storj.io/storj/internal/fpath"
"storj.io/storj/pkg/certificates"
@ -52,27 +51,18 @@ var (
Signer certificates.CertClientConfig
}
confDir string
defaultConfDir = fpath.ApplicationDir("storj", "identity")
identityDir string
defaultIdentityDir = fpath.ApplicationDir("storj", "identity")
)
func init() {
dirParam := cfgstruct.FindConfigDirParam()
if dirParam != "" {
defaultConfDir = dirParam
}
rootCmd.PersistentFlags().StringVar(&confDir, "config-dir", defaultConfDir, "main directory for storagenode configuration")
err := rootCmd.PersistentFlags().SetAnnotation("config-dir", "setup", []string{"true"})
if err != nil {
zap.S().Error("Failed to set 'setup' annotation for 'config-dir'")
}
rootCmd.PersistentFlags().StringVar(&identityDir, "identity-dir", defaultIdentityDir, "root directory for identity output")
rootCmd.AddCommand(newServiceCmd)
rootCmd.AddCommand(csrCmd)
cfgstruct.Bind(newServiceCmd.Flags(), &config, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(csrCmd.Flags(), &config, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(newServiceCmd.Flags(), &config, cfgstruct.CredsDir(defaultIdentityDir))
cfgstruct.Bind(csrCmd.Flags(), &config, cfgstruct.CredsDir(defaultIdentityDir))
}
func main() {
@ -80,7 +70,7 @@ func main() {
}
func serviceDirectory(serviceName string) string {
return filepath.Join(confDir, serviceName)
return filepath.Join(identityDir, serviceName)
}
func cmdNewService(cmd *cobra.Command, args []string) error {

67
cmd/satellite/main.go
View File

@ -5,7 +5,6 @@ package main
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
@ -27,7 +26,6 @@ import (
"storj.io/storj/pkg/datarepair/checker"
"storj.io/storj/pkg/datarepair/repairer"
"storj.io/storj/pkg/discovery"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/overlay"
"storj.io/storj/pkg/payments"
@ -42,9 +40,6 @@ import (
// Satellite defines satellite configuration
type Satellite struct {
CA identity.CASetupConfig `setup:"true"`
Identity identity.SetupConfig `setup:"true"`
Server server.Config
Kademlia kademlia.SatelliteConfig
PointerDB pointerdb.Config
@ -99,32 +94,49 @@ var (
QListLimit int `help:"maximum segments that can be requested" default:"1000"`
}
defaultConfDir string
confDir *string
defaultConfDir = fpath.ApplicationDir("storj", "satellite")
// TODO: this path should be defined somewhere else
defaultIdentityDir = fpath.ApplicationDir("storj", "identity", "satellite")
confDir string
identityDir string
)
func init() {
defaultConfDir = fpath.ApplicationDir("storj", "satellite")
dirParam := cfgstruct.FindConfigDirParam()
if dirParam != "" {
defaultConfDir = dirParam
confDirParam := cfgstruct.FindConfigDirParam()
if confDirParam != "" {
defaultConfDir = confDirParam
}
identityDirParam := cfgstruct.FindCredsDirParam()
if identityDirParam != "" {
defaultIdentityDir = identityDirParam
}
confDir = rootCmd.PersistentFlags().String("config-dir", defaultConfDir, "main directory for satellite configuration")
rootCmd.PersistentFlags().StringVar(&confDir, "config-dir", defaultConfDir, "main directory for satellite configuration")
err := rootCmd.PersistentFlags().SetAnnotation("config-dir", "setup", []string{"true"})
if err != nil {
zap.S().Error("Failed to set 'setup' annotation for 'config-dir'")
}
rootCmd.PersistentFlags().StringVar(&identityDir, "identity-dir", defaultIdentityDir, "main directory for satellite identity credentials")
err = rootCmd.PersistentFlags().SetAnnotation("identity-dir", "setup", []string{"true"})
if err != nil {
zap.S().Error("Failed to set 'setup' annotation for 'config-dir'")
}
rootCmd.AddCommand(runCmd)
rootCmd.AddCommand(setupCmd)
rootCmd.AddCommand(diagCmd)
rootCmd.AddCommand(qdiagCmd)
cfgstruct.Bind(runCmd.Flags(), &runCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(diagCmd.Flags(), &diagCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(qdiagCmd.Flags(), &qdiagCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(runCmd.Flags(), &runCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir))
cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir))
cfgstruct.Bind(diagCmd.Flags(), &diagCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir))
cfgstruct.Bind(qdiagCmd.Flags(), &qdiagCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir))
}
func cmdRun(cmd *cobra.Command, args []string) (err error) {
ctx := process.Ctx(cmd)
if _, err := runCfg.Server.Identity.Load(); err != nil {
zap.S().Fatal(err)
}
database, err := satellitedb.New(runCfg.Database)
if err != nil {
@ -161,7 +173,7 @@ func cmdRun(cmd *cobra.Command, args []string) (err error) {
}
func cmdSetup(cmd *cobra.Command, args []string) (err error) {
setupDir, err := filepath.Abs(*confDir)
setupDir, err := filepath.Abs(confDir)
if err != nil {
return err
}
@ -176,24 +188,7 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
return err
}
// TODO: handle setting base path *and* identity file paths via args
// NB: if base path is set this overrides identity and CA path options
if setupDir != defaultConfDir {
setupCfg.CA.CertPath = filepath.Join(setupDir, "ca.cert")
setupCfg.CA.KeyPath = filepath.Join(setupDir, "ca.key")
setupCfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert")
setupCfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key")
}
if setupCfg.Identity.Status() != identity.CertKey {
return errors.New("identity is missing")
}
o := map[string]interface{}{
"identity.cert-path": setupCfg.Identity.CertPath,
"identity.key-path": setupCfg.Identity.KeyPath,
}
return process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), o)
return process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), nil)
}
func cmdDiag(cmd *cobra.Command, args []string) (err error) {

43
cmd/storagenode/main.go
View File

@ -86,11 +86,12 @@ var (
diagCfg struct {
}
defaultConfDir string
defaultDiagDir string
defaultCredsDir string
confDir string
credsDir string
defaultConfDir = fpath.ApplicationDir("storj", "storagenode")
// TODO: this path should be defined somewhere else
defaultIdentityDir = fpath.ApplicationDir("storj", "identity", "storagenode")
defaultDiagDir string
confDir string
identityDir string
)
const (
@ -98,13 +99,13 @@ const (
)
func init() {
defaultConfDir = fpath.ApplicationDir("storj", "storagenode")
// TODO: this path should be defined somewhere else
defaultCredsDir = fpath.ApplicationDir("storj", "identity")
dirParam := cfgstruct.FindConfigDirParam()
if dirParam != "" {
defaultConfDir = dirParam
confDirParam := cfgstruct.FindConfigDirParam()
if confDirParam != "" {
defaultConfDir = confDirParam
}
identityDirParam := cfgstruct.FindCredsDirParam()
if identityDirParam != "" {
defaultIdentityDir = identityDirParam
}
rootCmd.PersistentFlags().StringVar(&confDir, "config-dir", defaultConfDir, "main directory for storagenode configuration")
@ -112,7 +113,11 @@ func init() {
if err != nil {
zap.S().Error("Failed to set 'setup' annotation for 'config-dir'")
}
rootCmd.PersistentFlags().StringVar(&credsDir, "creds-dir", defaultCredsDir, "main directory for storagenode identity credentials")
rootCmd.PersistentFlags().StringVar(&identityDir, "identity-dir", defaultIdentityDir, "main directory for storagenode identity credentials")
err = rootCmd.PersistentFlags().SetAnnotation("identity-dir", "setup", []string{"true"})
if err != nil {
zap.S().Error("Failed to set 'setup' annotation for 'config-dir'")
}
defaultDiagDir = filepath.Join(defaultConfDir, "storage")
rootCmd.AddCommand(runCmd)
@ -120,18 +125,16 @@ func init() {
rootCmd.AddCommand(configCmd)
rootCmd.AddCommand(diagCmd)
rootCmd.AddCommand(dashboardCmd)
cfgstruct.Bind(runCmd.Flags(), &runCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.BindSetup(configCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(diagCmd.Flags(), &diagCfg, cfgstruct.ConfDir(defaultDiagDir))
cfgstruct.Bind(runCmd.Flags(), &runCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir))
cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir))
cfgstruct.BindSetup(configCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir))
cfgstruct.Bind(diagCmd.Flags(), &diagCfg, cfgstruct.ConfDir(defaultDiagDir), cfgstruct.CredsDir(defaultIdentityDir))
cfgstruct.Bind(dashboardCmd.Flags(), &dashboardCfg, cfgstruct.ConfDir(defaultDiagDir))
}
func cmdRun(cmd *cobra.Command, args []string) (err error) {
if ident, err := runCfg.Server.Identity.Load(); err != nil {
if _, err := runCfg.Server.Identity.Load(); err != nil {
zap.S().Fatal(err)
} else {
zap.S().Info("Node ID: ", ident.ID)
}
operatorConfig := runCfg.Kademlia.Operator

15
cmd/storj-sim/network.go
View File

@ -126,7 +126,9 @@ func newNetwork(flags *Flags) (*Processes, error) {
})
bootstrap.Arguments = withCommon(Arguments{
"setup": {},
"setup": {
"--identity-dir", bootstrap.Directory,
},
"run": {
"--kademlia.bootstrap-addr", bootstrap.Address,
"--kademlia.operator.email", "bootstrap@example.com",
@ -150,7 +152,9 @@ func newNetwork(flags *Flags) (*Processes, error) {
process.WaitForStart(bootstrap)
process.Arguments = withCommon(Arguments{
"setup": {},
"setup": {
"--identity-dir", process.Directory,
},
"run": {
"--kademlia.bootstrap-addr", bootstrap.Address,
"--server.address", process.Address,
@ -181,6 +185,7 @@ func newNetwork(flags *Flags) (*Processes, error) {
process.Arguments = withCommon(Arguments{
"setup": {
"--identity-dir", process.Directory,
"--satellite-addr", satellite.Address,
},
"run": {
@ -212,7 +217,9 @@ func newNetwork(flags *Flags) (*Processes, error) {
process.WaitForStart(bootstrap)
process.Arguments = withCommon(Arguments{
"setup": {},
"setup": {
"--identity-dir", process.Directory,
},
"run": {
"--kademlia.bootstrap-addr", bootstrap.Address,
"--kademlia.operator.email", fmt.Sprintf("storage%d@example.com", i),
@ -263,7 +270,7 @@ func identitySetup(network *Processes) (*Processes, error) {
identity.Arguments = Arguments{
"setup": {
"--config-dir", process.Directory,
"--identity-dir", process.Directory,
"new", ".",
},
}

13
cmd/uplink/cmd/root.go
View File

@ -39,13 +39,18 @@ func addCmd(cmd *cobra.Command, root *cobra.Command) *cobra.Command {
root.AddCommand(cmd)
defaultConfDir := fpath.ApplicationDir("storj", "uplink")
defaultIdentityDir := fpath.ApplicationDir("storj", "identity", "uplink")
dirParam := cfgstruct.FindConfigDirParam()
if dirParam != "" {
defaultConfDir = dirParam
confDirParam := cfgstruct.FindConfigDirParam()
if confDirParam != "" {
defaultConfDir = confDirParam
}
identityDirParam := cfgstruct.FindCredsDirParam()
if identityDirParam != "" {
defaultIdentityDir = identityDirParam
}
cfgstruct.Bind(cmd.Flags(), &cfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(cmd.Flags(), &cfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir))
return cmd
}

4
cmd/uplink/cmd/run.go
View File

@ -23,6 +23,10 @@ func init() {
}
func cmdRun(cmd *cobra.Command, args []string) (err error) {
if _, err := cfg.Identity.Load(); err != nil {
zap.S().Fatal(err)
}
for _, flagname := range args {
return fmt.Errorf("Invalid argument %#v. Try 'uplink run'", flagname)
}

75
cmd/uplink/cmd/setup.go
View File

@ -11,13 +11,14 @@ import (
base58 "github.com/jbenet/go-base58"
"github.com/spf13/cobra"
"go.uber.org/zap"
"storj.io/storj/pkg/provider"
"storj.io/storj/internal/fpath"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/miniogw"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/provider"
)
var (
@ -28,12 +29,11 @@ var (
Annotations: map[string]string{"type": "setup"},
}
setupCfg struct {
CA provider.CASetupConfig `setup:"true"`
Identity provider.IdentitySetupConfig `setup:"true"`
APIKey string `default:"" help:"the api key to use for the satellite" setup:"true"`
EncKey string `default:"" help:"your root encryption key" setup:"true"`
GenerateMinioCerts bool `default:"false" help:"generate sample TLS certs for Minio GW" setup:"true"`
SatelliteAddr string `default:"localhost:7778" help:"the address to use for the satellite" setup:"true"`
Identity provider.IdentitySetupConfig
APIKey string `default:"" help:"the api key to use for the satellite" setup:"true"`
EncKey string `default:"" help:"your root encryption key" setup:"true"`
GenerateMinioCerts bool `default:"false" help:"generate sample TLS certs for Minio GW" setup:"true"`
SatelliteAddr string `default:"localhost:7778" help:"the address to use for the satellite" setup:"true"`
Server miniogw.ServerConfig
Minio miniogw.MinioConfig
@ -42,28 +42,44 @@ var (
Enc miniogw.EncryptionConfig
}
cliConfDir *string
gwConfDir *string
defaultConfDir = fpath.ApplicationDir("storj", "uplink")
defaultIdentityDir = fpath.ApplicationDir("storj", "identity", "uplink")
cliConfDir string
gwConfDir string
identityDir string
)
func init() {
defaultConfDir := fpath.ApplicationDir("storj", "uplink")
dirParam := cfgstruct.FindConfigDirParam()
if dirParam != "" {
defaultConfDir = dirParam
confDirParam := cfgstruct.FindConfigDirParam()
if confDirParam != "" {
defaultConfDir = confDirParam
}
identityDirParam := cfgstruct.FindCredsDirParam()
if identityDirParam != "" {
defaultIdentityDir = identityDirParam
}
cliConfDir = CLICmd.PersistentFlags().String("config-dir", defaultConfDir, "main directory for setup configuration")
gwConfDir = GWCmd.PersistentFlags().String("config-dir", defaultConfDir, "main directory for setup configuration")
CLICmd.PersistentFlags().StringVar(&cliConfDir, "config-dir", defaultConfDir, "main directory for setup configuration")
GWCmd.PersistentFlags().StringVar(&gwConfDir, "config-dir", defaultConfDir, "main directory for setup configuration")
CLICmd.PersistentFlags().StringVar(&identityDir, "identity-dir", defaultIdentityDir, "main directory for uplink identity credentials")
err := CLICmd.PersistentFlags().SetAnnotation("identity-dir", "setup", []string{"true"})
if err != nil {
zap.S().Error("Failed to set 'setup' annotation for 'config-dir'")
}
GWCmd.PersistentFlags().StringVar(&identityDir, "identity-dir", defaultIdentityDir, "main directory for gateway identity credentials")
err = GWCmd.PersistentFlags().SetAnnotation("identity-dir", "setup", []string{"true"})
if err != nil {
zap.S().Error("Failed to set 'setup' annotation for 'config-dir'")
}
CLICmd.AddCommand(setupCmd)
GWCmd.AddCommand(setupCmd)
cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.CredsDir(defaultIdentityDir))
}
func cmdSetup(cmd *cobra.Command, args []string) (err error) {
setupDir, err := filepath.Abs(*cliConfDir)
setupDir, err := filepath.Abs(cliConfDir)
if err != nil {
return err
}
@ -82,25 +98,6 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
return err
}
defaultConfDir := fpath.ApplicationDir("storj", "uplink")
// TODO: handle setting base path *and* identity file paths via args
// NB: if base path is set this overrides identity and CA path options
if setupDir != defaultConfDir {
setupCfg.CA.CertPath = filepath.Join(setupDir, "ca.cert")
setupCfg.CA.KeyPath = filepath.Join(setupDir, "ca.key")
setupCfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert")
setupCfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key")
}
if setupCfg.Identity.Status() == identity.CertKey {
// identity already exists
} else {
err = provider.SetupIdentity(process.Ctx(cmd), setupCfg.CA, setupCfg.Identity)
if err != nil {
return err
}
}
if setupCfg.GenerateMinioCerts {
minioCerts := filepath.Join(setupDir, "minio", "certs")
if err := os.MkdirAll(minioCerts, 0744); err != nil {
@ -125,8 +122,6 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
}
o := map[string]interface{}{
"identity.cert-path": setupCfg.Identity.CertPath,
"identity.key-path": setupCfg.Identity.KeyPath,
"client.api-key": setupCfg.APIKey,
"client.pointer-db-addr": setupCfg.SatelliteAddr,
"client.overlay-addr": setupCfg.SatelliteAddr,

24
pkg/cfgstruct/bind.go
View File

@ -29,6 +29,14 @@ func ConfDir(confdir string) BindOpt {
})
}
// CredsDir sets a variable for the default option called $CREDSDIR.
func CredsDir(credsdir string) BindOpt {
val := filepath.Clean(os.ExpandEnv(credsdir))
return BindOpt(func(vars map[string]confVar) {
vars["CREDSDIR"] = confVar{val: val, nested: false}
})
}
// ConfDirNested sets variables for default options called $CONFDIR and $CONFNAME.
// ConfDirNested also appends the parent struct field name to the paths before
// descending into substructs.
@ -187,11 +195,21 @@ func expand(vars map[string]string, val string) string {
// FindConfigDirParam returns '--config-dir' param from os.Args (if exists)
func FindConfigDirParam() string {
return FindFlagEarly("config-dir")
}
// FindCredsDirParam returns '--identity-dir' param from os.Args (if exists)
func FindCredsDirParam() string {
return FindFlagEarly("identity-dir")
}
// FindFlagEarly retrieves the value of a flag before `flag.Parse` has been called
func FindFlagEarly(flagName string) string {
// workaround to have early access to 'dir' param
for i, arg := range os.Args {
if strings.HasPrefix(arg, "--config-dir=") {
return strings.TrimPrefix(arg, "--config-dir=")
} else if arg == "--config-dir" && i < len(os.Args)-1 {
if strings.HasPrefix(arg, fmt.Sprintf("--%s=", flagName)) {
return strings.TrimPrefix(arg, fmt.Sprintf("--%s=", flagName))
} else if arg == fmt.Sprintf("--%s", flagName) && i < len(os.Args)-1 {
return os.Args[i+1]
}
}

10
pkg/identity/certificate_authority.go
View File

@ -49,8 +49,8 @@ type FullCertificateAuthority struct {
type CASetupConfig struct {
ParentCertPath string `help:"path to the parent authority's certificate chain"`
ParentKeyPath string `help:"path to the parent authority's private key"`
CertPath string `help:"path to the certificate chain for this identity" default:"$CONFDIR/ca.cert"`
KeyPath string `help:"path to the private key for this identity" default:"$CONFDIR/ca.key"`
CertPath string `help:"path to the certificate chain for this identity" default:"$CREDSDIR/ca.cert"`
KeyPath string `help:"path to the private key for this identity" default:"$CREDSDIR/ca.key"`
Difficulty uint64 `help:"minimum difficulty for identity generation" default:"15"`
Timeout string `help:"timeout for CA generation; golang duration string (0 no timeout)" default:"5m"`
Overwrite bool `help:"if true, existing CA certs AND keys will overwritten" default:"false"`
@ -71,13 +71,13 @@ type NewCAOptions struct {
// PeerCAConfig is for locating a CA certificate without a private key
type PeerCAConfig struct {
CertPath string `help:"path to the certificate chain for this identity" default:"$CONFDIR/ca.cert"`
CertPath string `help:"path to the certificate chain for this identity" default:"$CREDSDIR/ca.cert"`
}
// FullCAConfig is for locating a CA certificate and it's private key
type FullCAConfig struct {
CertPath string `help:"path to the certificate chain for this identity" default:"$CONFDIR/ca.cert"`
KeyPath string `help:"path to the private key for this identity" default:"$CONFDIR/ca.key"`
CertPath string `help:"path to the certificate chain for this identity" default:"$CREDSDIR/ca.cert"`
KeyPath string `help:"path to the private key for this identity" default:"$CREDSDIR/ca.key"`
}
// NewCA creates a new full identity with the given difficulty

8
pkg/identity/identity.go
View File

@ -58,8 +58,8 @@ type FullIdentity struct {
// SetupConfig allows you to run a set of Responsibilities with the given
// identity. You can also just load an Identity from disk.
type SetupConfig struct {
CertPath string `help:"path to the certificate chain for this identity" default:"$CONFDIR/identity.cert"`
KeyPath string `help:"path to the private key for this identity" default:"$CONFDIR/identity.key"`
CertPath string `help:"path to the certificate chain for this identity" default:"$CREDSDIR/identity.cert"`
KeyPath string `help:"path to the private key for this identity" default:"$CREDSDIR/identity.key"`
Overwrite bool `help:"if true, existing identity certs AND keys will overwritten for" default:"false"`
Version string `help:"semantic version of identity storage format" default:"0"`
}
@ -67,8 +67,8 @@ type SetupConfig struct {
// Config allows you to run a set of Responsibilities with the given
// identity. You can also just load an Identity from disk.
type Config struct {
CertPath string `help:"path to the certificate chain for this identity" default:"$CONFDIR/identity.cert"`
KeyPath string `help:"path to the private key for this identity" default:"$CONFDIR/identity.key"`
CertPath string `help:"path to the certificate chain for this identity" default:"$CREDSDIR/identity.cert" user:"true"`
KeyPath string `help:"path to the private key for this identity" default:"$CREDSDIR/identity.key" user:"true"`
}
// FullIdentityFromPEM loads a FullIdentity from a certificate chain and

20
scripts/cert-gating.sh
View File

@ -1,6 +1,6 @@
#!/usr/bin/env bash
source $(dirname $0)/utils.sh
basepath=$HOME/.storj/capt
basepath=$HOME/.storj/local-network
alpha_config=$basepath/config-alpha.yaml
unauthorized_config=$basepath/config-unauthorized.yaml
ca_whitelist=$basepath/ca-alpha-whitelist.cert
@ -21,9 +21,9 @@ case $1 in
echo "usage: $(basename $0) [setup|alpha|unauthorized]"
;;
setup)
temp_build captplanet identity
echo "setting up captplanet"
"$captplanet" setup
temp_build "storj-sim" identity
echo "setting up storj-sim"
${storj_sim} network setup
echo "clearing whitelist"
echo > ${ca_whitelist}
echo -n "generating alpha certificate authorities.."
@ -37,7 +37,7 @@ case $1 in
done
echo "done"
echo -n "generating alpha identities"
for dir in ${basepath}/{f*,sat*,up*}; do
for dir in ${basepath}/{bootstrap/*,satellite/*,storagenode/*,gateway/*}; do
echo -n "."
_ca_basepath=$(rand_ca_basepath)
_ca_cert=${dir}/ca-alpha.cert
@ -64,15 +64,15 @@ case $1 in
cat ${basepath}/config.yaml | sed -E "s,peer-ca-whitelist-path: \"\",peer-ca-whitelist-path: $ca_whitelist,g" >"$unauthorized_config"
;;
alpha)
build captplanet
${captplanet} run --config ${alpha_config}
build "storj-sim"
${storj_sim} network run --config ${alpha_config}
;;
unauthorized)
build captplanet
${captplanet} run --config ${unauthorized_config}
build "storj-sim"
${storj_sim} network run --config ${unauthorized_config}
;;
run)
${captplanet} run
${storj_sim} network run
;;
*)
$@ --help

2
scripts/utils.sh
View File

@ -31,7 +31,7 @@ build() {
echo -n " building $cmd..."
dots_on
local path=${tmp_dir}/${cmd}
declare -g ${cmd}=${path}
declare -g $(echo $cmd | sed s,-,_,g)=${path}
build_out=$(go build -o ${path} storj.io/storj/cmd/${cmd} 2>&1)
dots_off
echo "done"