automate certificate signing in storage node setup (#954)

.travis.yml

@@ -73,6 +73,11 @@ matrix:
       script:
         - make test-captplanet
 
+    ### certificate signing tests ###
+    - env: MODE=integration
+      script:
+        - make test-certificate-signing
+
     ### docker tests ###
     - env: MODE=docker
       services:

Makefile

@@ -77,6 +77,11 @@ test-captplanet: ## Test source with captain planet (travis)
 	@echo "Running ${@}"
 	@./scripts/test-captplanet.sh
 
+.PHONY: test-certificate-signing
+test-certificate-signing: ## Test certificate signing service and storagenode setup (travis)
+	@echo "Running ${@}"
+	@./scripts/test-certificate-signing.sh
+
 .PHONY: test-docker
 test-docker: ## Run tests in Docker
 	docker-compose up -d --remove-orphans test

cmd/certificates/main.go

@@ -38,9 +38,10 @@ var (
 	}
 
 	setupCmd = &cobra.Command{
-		Use:   "setup",
+		Use:         "setup",
-		Short: "Setup a certificate signing server",
+		Short:       "Setup a certificate signing server",
-		RunE:  cmdSetup,
+		RunE:        cmdSetup,
+		Annotations: map[string]string{"type": "setup"},
 	}
 
 	runCmd = &cobra.Command{
@@ -77,8 +78,9 @@ var (
 		// NB: cert and key paths overridden in setup
 		CA identity.CASetupConfig
 		// NB: cert and key paths overridden in setup
-		Identity identity.SetupConfig
+		Identity  identity.SetupConfig
-		certificates.CertSignerConfig
+		Signer    certificates.CertSignerConfig
+		Overwrite bool `default:"false" help:"if true ca, identity, and authorization db will be overwritten/truncated"`
 	}
 
 	runCfg struct {
@@ -105,11 +107,20 @@ var (
 	}
 
 	defaultConfDir = fpath.ApplicationDir("storj", "cert-signing")
+	confDir        *string
 )
 
 func init() {
+	dirParam := cfgstruct.FindConfigDirParam()
+	if dirParam != "" {
+		defaultConfDir = dirParam
+	}
+	confDir = rootCmd.PersistentFlags().String("config-dir", defaultConfDir, "main directory for captplanet configuration")
+
 	rootCmd.AddCommand(setupCmd)
 	cfgstruct.Bind(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir))
+	rootCmd.AddCommand(runCmd)
+	cfgstruct.Bind(runCmd.Flags(), &runCfg, cfgstruct.ConfDir(defaultConfDir))
 	rootCmd.AddCommand(authCmd)
 	authCmd.AddCommand(authCreateCmd)
 	cfgstruct.Bind(authCreateCmd.Flags(), &authCreateCfg, cfgstruct.ConfDir(defaultConfDir))
@@ -120,7 +131,12 @@ func init() {
 }
 
 func cmdSetup(cmd *cobra.Command, args []string) error {
-	setupDir, err := filepath.Abs(defaultConfDir)
+	setupDir, err := filepath.Abs(*confDir)
+	if err != nil {
+		return err
+	}
+
+	err = os.MkdirAll(setupDir, 0700)
 	if err != nil {
 		return err
 	}
@@ -134,21 +150,21 @@ func cmdSetup(cmd *cobra.Command, args []string) error {
 		return nil
 	}
 
-	if _, err := setupCfg.NewAuthDB(); err != nil {
+	if setupCfg.Overwrite {
-		return err
+		setupCfg.CA.Overwrite = true
+		setupCfg.Identity.Overwrite = true
+		setupCfg.Signer.Overwrite = true
 	}
 
-	err = os.MkdirAll(setupDir, 0700)
+	if _, err := setupCfg.Signer.NewAuthDB(); err != nil {
-	if err != nil {
 		return err
 	}
-
 	setupCfg.CA.CertPath = filepath.Join(setupDir, "ca.cert")
 	setupCfg.CA.KeyPath = filepath.Join(setupDir, "ca.key")
 	setupCfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert")
 	setupCfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key")
 
-	err = identity.SetupCA(process.Ctx(cmd), setupCfg.CA)
+	err = identity.SetupIdentity(process.Ctx(cmd), setupCfg.CA, setupCfg.Identity)
 	if err != nil {
 		return err
 	}
@@ -158,6 +174,7 @@ func cmdSetup(cmd *cobra.Command, args []string) error {
 		"ca.key-path":        setupCfg.CA.KeyPath,
 		"identity.cert-path": setupCfg.Identity.CertPath,
 		"identity.key-path":  setupCfg.Identity.KeyPath,
+		"log.level":          "info",
 	}
 	return process.SaveConfig(runCmd.Flags(),
 		filepath.Join(setupDir, "config.yaml"), o)
@@ -306,7 +323,7 @@ func cmdExportAuth(cmd *cobra.Command, args []string) error {
 			return errs.New("Either use `--emails-path` or positional args, not both.")
 		}
 		emails = args
-	} else if authExportCfg.All {
+	} else if len(args) == 0 || authExportCfg.All {
 		emails, err = authDB.UserIDs()
 		if err != nil {
 			return err
@@ -327,6 +344,9 @@ func cmdExportAuth(cmd *cobra.Command, args []string) error {
 	case "-":
 		output = os.Stdout
 	default:
+		if err := os.MkdirAll(filepath.Dir(authExportCfg.Out), 0600); err != nil {
+			return errs.Wrap(err)
+		}
 		output, err = os.OpenFile(authExportCfg.Out, os.O_CREATE, 0600)
 		if err != nil {
 			return errs.Wrap(err)

cmd/storagenode/main.go

@@ -17,6 +17,7 @@ import (
 	"go.uber.org/zap"
 
 	"storj.io/storj/internal/fpath"
+	"storj.io/storj/pkg/certificates"
 	"storj.io/storj/pkg/cfgstruct"
 	"storj.io/storj/pkg/identity"
 	"storj.io/storj/pkg/kademlia"
@@ -64,6 +65,7 @@ var (
 	setupCfg struct {
 		CA        identity.CASetupConfig
 		Identity  identity.SetupConfig
+		Signer    certificates.CertSigningConfig
 		Overwrite bool `default:"false" help:"whether to overwrite pre-existing configuration files"`
 	}
 	diagCfg struct {
@@ -137,14 +139,28 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
 		return err
 	}
 
+	// TODO: this is only applicable once we stop deleting the entire config dir on overwrite
+	// (see https://storjlabs.atlassian.net/browse/V3-1013)
+	// (see https://storjlabs.atlassian.net/browse/V3-949)
+	if setupCfg.Overwrite {
+		setupCfg.CA.Overwrite = true
+		setupCfg.Identity.Overwrite = true
+	}
 	setupCfg.CA.CertPath = filepath.Join(setupDir, "ca.cert")
 	setupCfg.CA.KeyPath = filepath.Join(setupDir, "ca.key")
 	setupCfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert")
 	setupCfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key")
 
-	err = identity.SetupIdentity(process.Ctx(cmd), setupCfg.CA, setupCfg.Identity)
+	if setupCfg.Signer.AuthToken != "" && setupCfg.Signer.Address != "" {
-	if err != nil {
+		err = setupCfg.Signer.SetupIdentity(process.Ctx(cmd), setupCfg.CA, setupCfg.Identity)
-		return err
+		if err != nil {
+			zap.S().Warn(err)
+		}
+	} else {
+		err = identity.SetupIdentity(process.Ctx(cmd), setupCfg.CA, setupCfg.Identity)
+		if err != nil {
+			return err
+		}
 	}
 
 	overrides := map[string]interface{}{

pkg/certificates/certificates.go

@@ -64,16 +64,16 @@ type CertSigningConfig struct {
 
 // CertSignerConfig is a config struct for use with a certificate signing service server
 type CertSignerConfig struct {
-	Overwrite          bool   `help:"if true, overwrites config AND authorization db is truncated" default:"false"`
+	Overwrite          bool   `default:"false" help:"if true, overwrites config AND authorization db is truncated"`
-	AuthorizationDBURL string `help:"url to the certificate signing authorization database" default:"bolt://$CONFDIR/authorizations.db"`
+	AuthorizationDBURL string `default:"bolt://$CONFDIR/authorizations.db" help:"url to the certificate signing authorization database"`
-	MinDifficulty      uint   `help:"minimum difficulty of the requester's identity required to claim an authorization"`
+	MinDifficulty      uint   `default:"16" help:"minimum difficulty of the requester's identity required to claim an authorization"`
-	CA                 provider.FullCAConfig
+	CA                 identity.FullCAConfig
 }
 
 // CertificateSigner implements pb.CertificatesServer
 type CertificateSigner struct {
-	Logger        *zap.Logger
+	Log           *zap.Logger
-	Signer        *provider.FullCertificateAuthority
+	Signer        *identity.FullCertificateAuthority
 	AuthDB        *AuthorizationDB
 	MinDifficulty uint16
 }
@@ -115,7 +115,7 @@ type ClaimOpts struct {
 type Claim struct {
 	Addr             string
 	Timestamp        int64
-	Identity         *provider.PeerIdentity
+	Identity         *identity.PeerIdentity
 	SignedChainBytes [][]byte
 }
 
@@ -130,7 +130,7 @@ func init() {
 }
 
 // NewClient creates a new certificate signing grpc client
-func NewClient(ctx context.Context, ident *provider.FullIdentity, address string) (*Client, error) {
+func NewClient(ctx context.Context, ident *identity.FullIdentity, address string) (*Client, error) {
 	tc := transport.NewClient(ident)
 	conn, err := tc.DialAddress(ctx, address)
 	if err != nil {
@@ -191,6 +191,94 @@ func ParseToken(tokenString string) (*Token, error) {
 	return t, nil
 }
 
+// SetupIdentity loads or creates a CA and identity and submits a certificate
+// signing request request for the CA; if successful, updated chains are saved.
+func (c CertSigningConfig) SetupIdentity(
+	ctx context.Context,
+	caConfig identity.CASetupConfig,
+	identConfig identity.SetupConfig,
+) error {
+	caStatus := caConfig.Status()
+	var (
+		ca    *identity.FullCertificateAuthority
+		ident *identity.FullIdentity
+		err   error
+	)
+	if caStatus == identity.CertKey && !caConfig.Overwrite {
+		ca, err = caConfig.FullConfig().Load()
+		if err != nil {
+			return err
+		}
+	} else if caStatus != identity.NoCertNoKey && !caConfig.Overwrite {
+		return identity.ErrSetup.New("certificate authority file(s) exist: %s", caStatus)
+	} else {
+		t, err := time.ParseDuration(caConfig.Timeout)
+		if err != nil {
+			return errs.Wrap(err)
+		}
+		ctx, cancel := context.WithTimeout(ctx, t)
+		defer cancel()
+
+		ca, err = caConfig.Create(ctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	identStatus := identConfig.Status()
+	if identStatus == identity.CertKey && !identConfig.Overwrite {
+		ident, err = identConfig.FullConfig().Load()
+		if err != nil {
+			return err
+		}
+	} else if identStatus != identity.NoCertNoKey && !identConfig.Overwrite {
+		return identity.ErrSetup.New("identity file(s) exist: %s", identStatus)
+	} else {
+		ident, err = identConfig.Create(ca)
+		if err != nil {
+			return err
+		}
+	}
+
+	signedChainBytes, err := c.Sign(ctx, ident)
+	if err != nil {
+		return errs.New("error occured while signing certificate: %s\n(identity files were still generated and saved, if you try again existnig files will be loaded)", err)
+	}
+
+	signedChain, err := identity.ParseCertChain(signedChainBytes)
+	if err != nil {
+		return nil
+	}
+
+	ca.Cert = signedChain[0]
+	ca.RestChain = signedChain[1:]
+	err = identity.FullCAConfig{
+		CertPath: caConfig.FullConfig().CertPath,
+	}.Save(ca)
+	if err != nil {
+		return err
+	}
+
+	ident.RestChain = signedChain[1:]
+	err = identity.Config{
+		CertPath: identConfig.FullConfig().CertPath,
+	}.Save(ident)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Sign submits a certificate signing request given the config
+func (c CertSigningConfig) Sign(ctx context.Context, ident *identity.FullIdentity) ([][]byte, error) {
+	client, err := NewClient(ctx, ident, c.Address)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.Sign(ctx, c.AuthToken)
+}
+
 // Sign claims an authorization using the token string and returns a signed
 // copy of the client's CA certificate
 func (c Client) Sign(ctx context.Context, tokenStr string) ([][]byte, error) {
@@ -216,7 +304,8 @@ func (c CertSignerConfig) NewAuthDB() (*AuthorizationDB, error) {
 	authDB := new(AuthorizationDB)
 	switch driver {
 	case "bolt":
-		if c.Overwrite {
+		_, err := os.Stat(source)
+		if c.Overwrite && err == nil {
 			if err := os.Remove(source); err != nil {
 				return nil, err
 			}
@@ -264,13 +353,26 @@ func (c CertSignerConfig) Run(ctx context.Context, server *provider.Provider) (e
 	}
 
 	srv := &CertificateSigner{
-		Logger:        zap.L(),
+		Log:           zap.L(),
 		Signer:        signer,
 		AuthDB:        authDB,
 		MinDifficulty: uint16(c.MinDifficulty),
 	}
 	pb.RegisterCertificatesServer(server.GRPC(), srv)
 
+	srv.Log.Info(
+		"Certificate signing server running",
+		zap.String("address", server.Addr().String()),
+	)
+
+	go func() {
+		done := ctx.Done()
+		<-done
+		if err := server.Close(); err != nil {
+			srv.Log.Error("closing server", zap.Error(err))
+		}
+	}()
+
 	return server.Run(ctx)
 }
 

pkg/certificates/certificates_test.go

@@ -872,7 +872,7 @@ func TestCertificateSigner_Sign(t *testing.T) {
 	peerCtx := peer.NewContext(ctx, grpcPeer)
 
 	certSigner := &CertificateSigner{
-		Logger: zap.L(),
+		Log:    zap.L(),
 		Signer: signingCA,
 		AuthDB: authDB,
 	}

pkg/identity/certificate_authority.go

@@ -149,6 +149,14 @@ func (caS CASetupConfig) Create(ctx context.Context) (*FullCertificateAuthority,
 	return ca, caC.Save(ca)
 }
 
+// FullConfig converts a `CASetupConfig` to `FullCAConfig`
+func (caS CASetupConfig) FullConfig() FullCAConfig {
+	return FullCAConfig{
+		CertPath: caS.CertPath,
+		KeyPath:  caS.KeyPath,
+	}
+}
+
 // Load loads a CA from the given configuration
 func (fc FullCAConfig) Load() (*FullCertificateAuthority, error) {
 	p, err := fc.PeerConfig().Load()

pkg/identity/identity.go

@@ -195,6 +195,14 @@ func (is SetupConfig) Create(ca *FullCertificateAuthority) (*FullIdentity, error
 	return fi, ic.Save(fi)
 }
 
+// FullConfig converts a `SetupConfig` to `Config`
+func (is SetupConfig) FullConfig() Config {
+	return Config{
+		CertPath: is.CertPath,
+		KeyPath:  is.KeyPath,
+	}
+}
+
 // Load loads a FullIdentity from the config
 func (ic Config) Load() (*FullIdentity, error) {
 	c, err := ioutil.ReadFile(ic.CertPath)

pkg/peertls/extensions.go

@@ -14,7 +14,6 @@ import (
 	"time"
 
 	"github.com/zeebo/errs"
-	"go.uber.org/zap"
 
 	"storj.io/storj/pkg/utils"
 	"storj.io/storj/storage"
@@ -421,13 +420,11 @@ func NewRevDB(revocationDBURL string) (*RevocationDB, error) {
 		if err != nil {
 			return nil, ErrRevocationDB.Wrap(err)
 		}
-		zap.S().Info("Starting overlay cache with BoltDB")
 	case "redis":
 		db, err = NewRevocationDBRedis(revocationDBURL)
 		if err != nil {
 			return nil, ErrRevocationDB.Wrap(err)
 		}
-		zap.S().Info("Starting overlay cache with Redis")
 	default:
 		return nil, ErrRevocationDB.New("database scheme not supported: %s", driver)
 	}

scripts/test-certificate-signing.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+set -o errexit
+
+trap "echo ERROR: exiting due to error; exit" ERR
+trap "exit" INT TERM
+trap "kill 0" EXIT
+
+. $(dirname $0)/utils.sh
+
+user_id="user@example.com"
+signer_address="127.0.0.1:8888"
+difficulty=16
+
+cleanup() {
+    if [[ -z ${bg+x} ]]; then
+        kill ${bg}
+    fi
+
+    dirs="$tmp $tmp_build_dir"
+    for dir in ${dirs}; do
+        if [[ ! -z ${dir+x} ]]; then
+            rm -rf ${dir}
+        fi
+    done
+}
+temp_build storagenode certificates
+tmp=$(mktemp -d)
+trap "rm -rf ${tmp} ${tmp_build_dir}; cleanup" EXIT
+
+certificates_dir=${tmp}/cert-signing
+storagenode_dir=${tmp}/storagenode
+
+# TODO: create separate signer CA and use `--signer.ca` options
+#                    --signer.ca.cert-path ${signer_cert} \
+#                    --signer.ca.key-path ${signer_key} \
+
+echo "setting up certificate signing server"
+$certificates setup --config-dir ${certificates_dir} \
+                    --signer.min-difficulty ${difficulty}
+
+echo "creating test authorization"
+$certificates auth create --config-dir ${certificates_dir} \
+                          1 ${user_id} >/dev/null 2>&1
+
+export_tokens() {
+    $certificates auth export --config-dir ${certificates_dir} \
+                              --out -
+}
+token=$(export_tokens 2>&1|cut -d , -f 2|grep -oE "$user_id:\w+")
+
+echo "starting certificate signing server"
+$certificates run --config-dir ${certificates_dir} \
+                  --server.address ${signer_address} &
+bg=$!
+sleep 1
+
+echo "setting up storage node"
+$storagenode setup --config-dir ${storagenode_dir} \
+                   --ca.difficulty ${difficulty} \
+                   --signer.address ${signer_address} \
+                   --signer.auth-token ${token}
+
+ca_chain_len=$(cat ${storagenode_dir}/ca.cert|grep "BEGIN CERTIFICATE"|wc -l)
+ident_chain_len=$(cat ${storagenode_dir}/identity.cert|grep "BEGIN CERTIFICATE"|wc -l)
+failures=0
+if [[ ! ${ca_chain_len} == 2 ]]; then
+    echo "FAIL: incorrect storage node CA chain length; expected: 2; actual: ${ca_chain_len}"
+    failures=$((failures+1))
+fi
+if [[ ! ${ident_chain_len} == 3 ]]; then
+    echo "FAIL: incorrect storage node identty chain length; expected: 2; actual: ${ident_chain_len}"
+    failures=$((failures+1))
+fi
+
+if [[ ${failures} == 0 ]]; then
+    echo "SUCCESS: all expectations met!"
+fi
+
+exit ${failures}

scripts/utils.sh

@@ -33,8 +33,8 @@ build() {
 }
 
 temp_build() {
-	tmp_dir=$(mktemp -d)
+    declare -g tmp_build_dir=$(mktemp -d)
-	build ${tmp_dir} $@
+	build ${tmp_build_dir} $@
 }
 
 check_help() {