JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Consolidate identity management to identity cli commands (#1083)

Browse Source 
* Consolidate identity management:

Move identity cretaion/signing out of storagenode setup command.

* fixes

* linters

* Consolidate identity management:

Move identity cretaion/signing out of storagenode setup command.

* fixes

* sava backups before saving signed certs

* add "-prebuilt-test-cmds" test flag

* linters

* prepare cli tests for travis

* linter fixes

* more fixes

* linter gods

* sp/sdk/sim

* remove ca.difficulty

* remove unused difficulty

* return setup to its rightful place

* wip travis

* Revert "wip travis"

This reverts commit 56834849dcf066d3cc0a4f139033fc3f6d7188ca.

* typo in travis.yaml

* remove tests

* remove more

* make it only create one identity at a time for consistency

* add config-dir for consitency

* add identity creation to storj-sim

* add flags

* simplify

* fix nolint and compile

* prevent overwrite and pass difficulty, concurrency, and parent creds

* goimports
This commit is contained in:
Egon Elbre 2019-01-18 05:36:58 -05:00 committed by Stefan Benten
parent 82f5bb6072
commit bbf81f2479
17 changed files with 299 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.travis.yml
View File

@ -64,14 +64,13 @@ matrix:
- golangci-lint run
# - gospace istidy
### service integration tests ###
### integration tests ###
- env: MODE=integration
services:
- redis
install:
- source scripts/install-awscli.sh
- make install-sim
- go install -race storj.io/storj/cmd/certificates
script:
- set -o pipefail && make test-sim |& go run scripts/fail-on-race.go
- set -o pipefail && make test-certificate-signing |& go run scripts/fail-on-race.go

2
Makefile
View File

@ -70,7 +70,7 @@ proto: ## Rebuild protobuf files
.PHONY: install-sim
install-sim: ## install storj-sim
@echo "Running ${@}"
@go install -race -v storj.io/storj/cmd/storj-sim storj.io/storj/cmd/bootstrap storj.io/storj/cmd/satellite storj.io/storj/cmd/storagenode storj.io/storj/cmd/uplink storj.io/storj/cmd/gateway
@go install -race -v storj.io/storj/cmd/storj-sim storj.io/storj/cmd/bootstrap storj.io/storj/cmd/satellite storj.io/storj/cmd/storagenode storj.io/storj/cmd/uplink storj.io/storj/cmd/gateway storj.io/storj/cmd/identity storj.io/storj/cmd/certificates
##@ Test

7
cmd/bootstrap/main.go
View File

@ -4,6 +4,7 @@
package main
import (
"errors"
"fmt"
"os"
"path/filepath"
@ -101,9 +102,9 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
cfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert")
cfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key")
}
err = identity.SetupIdentity(process.Ctx(cmd), cfg.CA, cfg.Identity)
if err != nil {
return err
if cfg.Identity.Status() != identity.CertKey {
return errors.New("identity is missing")
}
overrides := map[string]interface{}{

16
cmd/certificates/setup.go
View File

@ -4,6 +4,7 @@
package main
import (
"errors"
"fmt"
"os"
"path/filepath"
@ -69,14 +70,15 @@ func cmdSetup(cmd *cobra.Command, args []string) error {
if _, err := setupCfg.Signer.NewAuthDB(); err != nil {
return err
}
setupCfg.CA.CertPath = filepath.Join(setupDir, "ca.cert")
setupCfg.CA.KeyPath = filepath.Join(setupDir, "ca.key")
setupCfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert")
setupCfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key")
err = identity.SetupIdentity(process.Ctx(cmd), setupCfg.CA, setupCfg.Identity)
if err != nil {
return err
if setupDir != defaultConfDir {
setupCfg.CA.CertPath = filepath.Join(setupDir, "ca.cert")
setupCfg.CA.KeyPath = filepath.Join(setupDir, "ca.key")
setupCfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert")
setupCfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key")
}
if setupCfg.Identity.Status() != identity.CertKey {
return errors.New("identity is missing")
}
o := map[string]interface{}{

41
cmd/identity/certificate_authority.go
View File

@ -16,31 +16,36 @@ import (
var (
caCmd = &cobra.Command{
Use: "ca",
Short: "Manage certificate authorities",
Use: "ca",
Short: "Manage certificate authorities",
Annotations: map[string]string{"type": "setup"},
}
newCACmd = &cobra.Command{
Use: "new",
Short: "Create a new certificate authority",
RunE: cmdNewCA,
Use: "new",
Short: "Create a new certificate authority",
RunE: cmdNewCA,
Annotations: map[string]string{"type": "setup"},
}
getIDCmd = &cobra.Command{
Use: "id",
Short: "Get the id of a CA",
RunE: cmdGetID,
Use: "id",
Short: "Get the id of a CA",
RunE: cmdGetID,
Annotations: map[string]string{"type": "setup"},
}
caExtCmd = &cobra.Command{
Use: "extensions",
Short: "Prints the extensions attached to the identity CA certificate",
RunE: cmdCAExtensions,
Use: "extensions",
Short: "Prints the extensions attached to the identity CA certificate",
RunE: cmdCAExtensions,
Annotations: map[string]string{"type": "setup"},
}
revokeCACmd = &cobra.Command{
Use: "revoke",
Short: "Revoke the identity's CA certificate (creates backup)",
RunE: cmdRevokeCA,
Use: "revoke",
Short: "Revoke the identity's CA certificate (creates backup)",
RunE: cmdRevokeCA,
Annotations: map[string]string{"type": "setup"},
}
newCACfg struct {
@ -63,13 +68,15 @@ var (
func init() {
rootCmd.AddCommand(caCmd)
caCmd.AddCommand(newCACmd)
cfgstruct.Bind(newCACmd.Flags(), &newCACfg, cfgstruct.ConfDir(defaultConfDir))
caCmd.AddCommand(getIDCmd)
cfgstruct.Bind(getIDCmd.Flags(), &getIDCfg, cfgstruct.ConfDir(defaultConfDir))
caCmd.AddCommand(caExtCmd)
cfgstruct.Bind(caExtCmd.Flags(), &caExtCfg, cfgstruct.ConfDir(defaultConfDir))
caCmd.AddCommand(revokeCACmd)
cfgstruct.Bind(newCACmd.Flags(), &newCACfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(getIDCmd.Flags(), &getIDCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(caExtCmd.Flags(), &caExtCfg, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(revokeCACmd.Flags(), &revokeCACfg, cfgstruct.ConfDir(defaultConfDir))
}

26
cmd/identity/identity.go
View File

@ -13,26 +13,30 @@ import (
var (
idCmd = &cobra.Command{
Use: "id",
Short: "Manage identities",
Use: "id",
Short: "Manage identities",
Annotations: map[string]string{"type": "setup"},
}
newIDCmd = &cobra.Command{
Use: "new",
Short: "Creates a new identity from an existing certificate authority",
RunE: cmdNewID,
Use: "new",
Short: "Creates a new identity from an existing certificate authority",
RunE: cmdNewID,
Annotations: map[string]string{"type": "setup"},
}
leafExtCmd = &cobra.Command{
Use: "extensions",
Short: "Prints the extensions attached to the identity leaf certificate",
RunE: cmdLeafExtensions,
Use: "extensions",
Short: "Prints the extensions attached to the identity leaf certificate",
RunE: cmdLeafExtensions,
Annotations: map[string]string{"type": "setup"},
}
revokeLeafCmd = &cobra.Command{
Use: "revoke",
Short: "Revoke the identity's leaf certificate (creates backup)",
RunE: cmdRevokeLeaf,
Use: "revoke",
Short: "Revoke the identity's leaf certificate (creates backup)",
RunE: cmdRevokeLeaf,
Annotations: map[string]string{"type": "setup"},
}
newIDCfg struct {

7
cmd/identity/keys.go
View File

@ -26,9 +26,10 @@ var (
Short: "Manage keys",
}
keyGenerateCmd = &cobra.Command{
Use: "generate",
Short: "generate lots of keys",
RunE: cmdKeyGenerate,
Use: "generate",
Short: "generate lots of keys",
RunE: cmdKeyGenerate,
Annotations: map[string]string{"type": "setup"},
}
keyCfg struct {

159
cmd/identity/main.go
View File

@ -7,10 +7,16 @@ import (
"crypto/x509/pkix"
"encoding/json"
"fmt"
"path/filepath"
"github.com/spf13/cobra"
"github.com/zeebo/errs"
"go.uber.org/zap"
"storj.io/storj/internal/fpath"
"storj.io/storj/pkg/certificates"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/peertls"
"storj.io/storj/pkg/process"
)
@ -21,13 +27,166 @@ var (
Short: "Identity management",
}
newServiceCmd = &cobra.Command{
Use: "new <service>",
Short: "Create a new full identity for a service",
Args: cobra.ExactArgs(1),
RunE: cmdNewService,
Annotations: map[string]string{"type": "setup"},
}
csrCmd = &cobra.Command{
Use: "csr <service>",
Short: "Send a certificate signing request for a service's CA certificate",
Args: cobra.ExactArgs(1),
RunE: cmdCSR,
Annotations: map[string]string{"type": "setup"},
}
//nolint
config struct {
Difficulty uint64 `default:"15" help:"minimum difficulty for identity generation"`
Concurrency uint `default:"4" help:"number of concurrent workers for certificate authority generation"`
ParentCertPath string `help:"path to the parent authority's certificate chain"`
ParentKeyPath string `help:"path to the parent authority's private key"`
Signer certificates.CertClientConfig
}
confDir string
defaultConfDir = fpath.ApplicationDir("storj", "identity")
)
func init() {
dirParam := cfgstruct.FindConfigDirParam()
if dirParam != "" {
defaultConfDir = dirParam
}
rootCmd.PersistentFlags().StringVar(&confDir, "config-dir", defaultConfDir, "main directory for storagenode configuration")
err := rootCmd.PersistentFlags().SetAnnotation("config-dir", "setup", []string{"true"})
if err != nil {
zap.S().Error("Failed to set 'setup' annotation for 'config-dir'")
}
rootCmd.AddCommand(newServiceCmd)
rootCmd.AddCommand(csrCmd)
cfgstruct.Bind(newServiceCmd.Flags(), &config, cfgstruct.ConfDir(defaultConfDir))
cfgstruct.Bind(csrCmd.Flags(), &config, cfgstruct.ConfDir(defaultConfDir))
}
func main() {
process.Exec(rootCmd)
}
func serviceDirectory(serviceName string) string {
return filepath.Join(confDir, serviceName)
}
func cmdNewService(cmd *cobra.Command, args []string) error {
serviceDir := serviceDirectory(args[0])
caCertPath := filepath.Join(serviceDir, "ca.cert")
caKeyPath := filepath.Join(serviceDir, "ca.key")
identCertPath := filepath.Join(serviceDir, "identity.cert")
identKeyPath := filepath.Join(serviceDir, "identity.key")
caConfig := identity.CASetupConfig{
CertPath: caCertPath,
KeyPath: caKeyPath,
Difficulty: config.Difficulty,
Concurrency: config.Concurrency,
ParentCertPath: config.ParentCertPath,
ParentKeyPath: config.ParentKeyPath,
}
if caConfig.Status() != identity.NoCertNoKey {
return errs.New("CA certificate and/or key already exits, NOT overwriting!")
}
ca, caerr := caConfig.Create(process.Ctx(cmd))
identConfig := identity.SetupConfig{
CertPath: identCertPath,
KeyPath: identKeyPath,
}
if identConfig.Status() != identity.NoCertNoKey {
return errs.New("Identity certificate and/or key already exits, NOT overwriting!")
}
_, iderr := identConfig.Create(ca)
return errs.Combine(caerr, iderr)
}
func cmdCSR(cmd *cobra.Command, args []string) error {
ctx := process.Ctx(cmd)
serviceDir := serviceDirectory(args[0])
caCertPath := filepath.Join(serviceDir, "ca.cert")
caKeyPath := filepath.Join(serviceDir, "ca.key")
caConfig := identity.FullCAConfig{
CertPath: caCertPath,
KeyPath: caKeyPath,
}
identCertPath := filepath.Join(serviceDir, "identity.cert")
identKeyPath := filepath.Join(serviceDir, "identity.key")
identConfig := identity.Config{
CertPath: identCertPath,
KeyPath: identKeyPath,
}
ca, err := caConfig.Load()
if err != nil {
return err
}
ident, err := identConfig.Load()
if err != nil {
return err
}
signedChainBytes, err := config.Signer.Sign(ctx, ident)
if err != nil {
return errs.New("error occurred while signing certificate: %s\n(identity files were still generated and saved, if you try again existing files will be loaded)", err)
}
signedChain, err := identity.ParseCertChain(signedChainBytes)
if err != nil {
return nil
}
err = caConfig.SaveBackup(ca)
if err != nil {
return err
}
ca.Cert = signedChain[0]
ca.RestChain = signedChain[1:]
err = identity.FullCAConfig{
CertPath: caConfig.CertPath,
}.Save(ca)
if err != nil {
return err
}
err = identConfig.SaveBackup(ident)
if err != nil {
return err
}
ident.RestChain = signedChain[1:]
ident.CA = ca.Cert
err = identity.Config{
CertPath: identConfig.CertPath,
}.Save(ident)
if err != nil {
return err
}
return nil
}
func printExtensions(cert []byte, exts []pkix.Extension) error {
hash, err := peertls.SHA256Hash(cert)
if err != nil {

6
cmd/satellite/main.go
View File

@ -5,6 +5,7 @@ package main
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
@ -183,9 +184,8 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
setupCfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert")
setupCfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key")
}
err = identity.SetupIdentity(process.Ctx(cmd), setupCfg.CA, setupCfg.Identity)
if err != nil {
return err
if setupCfg.Identity.Status() != identity.CertKey {
return errors.New("identity is missing")
}
o := map[string]interface{}{

43
cmd/storagenode/main.go
View File

@ -22,9 +22,7 @@ import (
"go.uber.org/zap"
"storj.io/storj/internal/fpath"
"storj.io/storj/pkg/certificates"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/piecestore/psclient"
@ -37,15 +35,12 @@ import (
// StorageNode defines storage node configuration
type StorageNode struct {
CA identity.CASetupConfig `setup:"true"`
Identity identity.SetupConfig `setup:"true"`
EditConf bool `default:"false" help:"open config in default editor"`
SaveAllDefaults bool `default:"false" help:"save all default values to config.yaml file" setup:"true"`
EditConf bool `default:"false" help:"open config in default editor"`
SaveAllDefaults bool `default:"false" help:"save all default values to config.yaml file" setup:"true"`
Server server.Config
Kademlia kademlia.StorageNodeConfig
Storage psserver.Config
Signer certificates.CertClientConfig
}
var (
@ -89,9 +84,11 @@ var (
diagCfg struct {
}
defaultConfDir string
defaultDiagDir string
confDir string
defaultConfDir string
defaultDiagDir string
defaultCredsDir string
confDir string
credsDir string
)
const (
@ -100,6 +97,8 @@ const (
func init() {
defaultConfDir = fpath.ApplicationDir("storj", "storagenode")
// TODO: this path should be defined somewhere else
defaultCredsDir = fpath.ApplicationDir("storj", "identity")
dirParam := cfgstruct.FindConfigDirParam()
if dirParam != "" {
@ -111,6 +110,7 @@ func init() {
if err != nil {
zap.S().Error("Failed to set 'setup' annotation for 'config-dir'")
}
rootCmd.PersistentFlags().StringVar(&credsDir, "creds-dir", defaultCredsDir, "main directory for storagenode identity credentials")
defaultDiagDir = filepath.Join(defaultConfDir, "storage")
rootCmd.AddCommand(runCmd)
@ -126,6 +126,12 @@ func init() {
}
func cmdRun(cmd *cobra.Command, args []string) (err error) {
if ident, err := runCfg.Server.Identity.Load(); err != nil {
zap.S().Fatal(err)
} else {
zap.S().Info("Node ID: ", ident.ID)
}
operatorConfig := runCfg.Kademlia.Operator
if err := isOperatorEmailValid(operatorConfig.Email); err != nil {
zap.S().Warn(err)
@ -161,23 +167,6 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
return err
}
setupCfg.CA.CertPath = filepath.Join(setupDir, "ca.cert")
setupCfg.CA.KeyPath = filepath.Join(setupDir, "ca.key")
setupCfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert")
setupCfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key")
if setupCfg.Signer.AuthToken != "" && setupCfg.Signer.Address != "" {
err = setupCfg.Signer.SetupIdentity(process.Ctx(cmd), setupCfg.CA, setupCfg.Identity)
if err != nil {
zap.S().Warn(err)
}
} else {
err = identity.SetupIdentity(process.Ctx(cmd), setupCfg.CA, setupCfg.Identity)
if err != nil {
return err
}
}
overrides := map[string]interface{}{
"log.level": "info",
}

55
cmd/storj-sim/network.go
View File

@ -35,6 +35,18 @@ func networkExec(flags *Flags, args []string, command string) error {
ctx, cancel := NewCLIContext(context.Background())
defer cancel()
if command == "setup" {
identities, err := identitySetup(processes)
if err != nil {
return err
}
err = identities.Exec(ctx, command)
if err != nil {
return err
}
}
err = processes.Exec(ctx, command)
closeErr := processes.Close()
@ -104,7 +116,6 @@ func newNetwork(flags *Flags) (*Processes, error) {
bootstrapPort = 9999
satellitePort = 10000
storageNodePort = 11000
difficulty = "10"
)
bootstrap := processes.New(Info{
@ -115,9 +126,7 @@ func newNetwork(flags *Flags) (*Processes, error) {
})
bootstrap.Arguments = withCommon(Arguments{
"setup": {
"--ca.difficulty", difficulty,
},
"setup": {},
"run": {
"--kademlia.bootstrap-addr", bootstrap.Address,
"--kademlia.operator.email", "bootstrap@example.com",
@ -141,9 +150,7 @@ func newNetwork(flags *Flags) (*Processes, error) {
process.WaitForStart(bootstrap)
process.Arguments = withCommon(Arguments{
"setup": {
"--ca.difficulty", difficulty,
},
"setup": {},
"run": {
"--kademlia.bootstrap-addr", bootstrap.Address,
"--server.address", process.Address,
@ -175,7 +182,6 @@ func newNetwork(flags *Flags) (*Processes, error) {
process.Arguments = withCommon(Arguments{
"setup": {
"--satellite-addr", satellite.Address,
"--ca.difficulty", difficulty,
},
"run": {
"--server.address", process.Address,
@ -206,9 +212,7 @@ func newNetwork(flags *Flags) (*Processes, error) {
process.WaitForStart(bootstrap)
process.Arguments = withCommon(Arguments{
"setup": {
"--ca.difficulty", difficulty,
},
"setup": {},
"run": {
"--kademlia.bootstrap-addr", bootstrap.Address,
"--kademlia.operator.email", fmt.Sprintf("storage%d@example.com", i),
@ -246,6 +250,35 @@ func newNetwork(flags *Flags) (*Processes, error) {
return processes, nil
}
func identitySetup(network *Processes) (*Processes, error) {
processes := NewProcesses()
for _, process := range network.List {
identity := processes.New(Info{
Name: "identity/" + process.Info.Name,
Executable: "identity",
Directory: process.Directory,
Address: "",
})
identity.Arguments = Arguments{
"setup": {
"--config-dir", process.Directory,
"new", ".",
},
}
}
// create directories for all processes
for _, process := range processes.List {
if err := os.MkdirAll(process.Directory, folderPermissions); err != nil {
return nil, err
}
}
return processes, nil
}
func randomKey() string {
var data [10]byte
_, _ = rand.Read(data[:])

5
cmd/storj-sim/process.go
View File

@ -223,11 +223,10 @@ func (process *Process) Exec(ctx context.Context, command string) (err error) {
return err
}
switch command {
case "setup":
if command == "setup" || process.Address == "" {
// during setup we aren't starting the addresses, so we can release the dependencies immediately
process.Status.Started.Release()
default:
} else {
// release started when we are able to connect to the process address
go process.monitorAddress()
}

12
cmd/uplink/cmd/setup.go
View File

@ -14,6 +14,7 @@ import (
"storj.io/storj/internal/fpath"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/miniogw"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/provider"
@ -90,9 +91,14 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
setupCfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert")
setupCfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key")
}
err = provider.SetupIdentity(process.Ctx(cmd), setupCfg.CA, setupCfg.Identity)
if err != nil {
return err
if setupCfg.Identity.Status() == identity.CertKey {
// identity already exists
} else {
err = provider.SetupIdentity(process.Ctx(cmd), setupCfg.CA, setupCfg.Identity)
if err != nil {
return err
}
}
if setupCfg.GenerateMinioCerts {

7
internal/fpath/os.go
View File

@ -90,12 +90,9 @@ func IsValidSetupDir(name string) (ok bool, err error) {
}
for _, filename := range filenames {
// allow log files to exist in the folder
if strings.EqualFold(filepath.Ext(filename), ".log") {
continue
if filename == "config.yaml" {
return false, nil
}
return false, nil
}
}
}

2
pkg/certificates/config.go
View File

@ -87,7 +87,7 @@ func (c CertClientConfig) SetupIdentity(
signedChainBytes, err := c.Sign(ctx, ident)
if err != nil {
return errs.New("error occured while signing certificate: %s\n(identity files were still generated and saved, if you try again existnig files will be loaded)", err)
return errs.New("error occurred while signing certificate: %s\n(identity files were still generated and saved, if you try again existing files will be loaded)", err)
}
signedChain, err := identity.ParseCertChain(signedChainBytes)

6
pkg/identity/certificate_authority.go
View File

@ -162,9 +162,9 @@ func (caS CASetupConfig) Create(ctx context.Context) (*FullCertificateAuthority,
CertPath: caS.ParentCertPath,
KeyPath: caS.ParentKeyPath,
}.Load()
}
if err != nil {
return nil, err
if err != nil {
return nil, err
}
}
if parent == nil {

4
pkg/identity/utils.go
View File

@ -123,10 +123,10 @@ func writeFile(path string, dirmode, filemode os.FileMode, data []byte) error {
func statTLSFiles(certPath, keyPath string) TLSFilesStatus {
_, err := os.Stat(certPath)
hasCert := os.IsExist(err)
hasCert := !os.IsNotExist(err)
_, err = os.Stat(keyPath)
hasKey := os.IsExist(err)
hasKey := !os.IsNotExist(err)
if hasCert && hasKey {
return CertKey