JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

{cmd,pkg}/certificates: service refactor (#2938)

Browse Source
This commit is contained in:
Bryan White 2019-09-05 17:11:21 +02:00 committed by GitHub
parent b222936d19
commit 1fc0c63a1d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 1087 additions and 919 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
cmd/certificates/auth.go
View File

@ -18,7 +18,7 @@ import (
"github.com/spf13/cobra"
"github.com/zeebo/errs"
"storj.io/storj/pkg/certificates"
"storj.io/storj/pkg/certificate/authorization"
"storj.io/storj/pkg/process"
)
@ -70,7 +70,7 @@ func cmdCreateAuth(cmd *cobra.Command, args []string) error {
if err != nil {
return errs.New("Count couldn't be parsed: %s", args[0])
}
authDB, err := authCfg.Signer.NewAuthDB()
authDB, err := authorization.NewDBFromCfg(authCfg.Authorizations)
if err != nil {
return err
}
@ -99,7 +99,7 @@ func cmdCreateAuth(cmd *cobra.Command, args []string) error {
func cmdInfoAuth(cmd *cobra.Command, args []string) error {
ctx := process.Ctx(cmd)
authDB, err := authCfg.Signer.NewAuthDB()
authDB, err := authorization.NewDBFromCfg(authCfg.Authorizations)
if err != nil {
return err
}
@ -143,7 +143,7 @@ func cmdInfoAuth(cmd *cobra.Command, args []string) error {
return errs.Combine(emailErrs.Err(), printErrs.Err())
}
func writeAuthInfo(ctx context.Context, authDB *certificates.AuthorizationDB, email string, w io.Writer) error {
func writeAuthInfo(ctx context.Context, authDB *authorization.DB, email string, w io.Writer) error {
auths, err := authDB.Get(ctx, email)
if err != nil {
return err
@ -152,7 +152,7 @@ func writeAuthInfo(ctx context.Context, authDB *certificates.AuthorizationDB, em
return nil
}
claimed, open := auths.Group()
claimed, open := auths.GroupByClaimed()
if _, err := fmt.Fprintf(w,
"%s\t%d\t%d\t\n",
email,
@ -170,8 +170,8 @@ func writeAuthInfo(ctx context.Context, authDB *certificates.AuthorizationDB, em
return nil
}
func writeTokenInfo(claimed, open certificates.Authorizations, w io.Writer) error {
groups := map[string]certificates.Authorizations{
func writeTokenInfo(claimed, open authorization.Group, w io.Writer) error {
groups := map[string]authorization.Group{
"Claimed": claimed,
"Open": open,
}
@ -194,7 +194,7 @@ func writeTokenInfo(claimed, open certificates.Authorizations, w io.Writer) erro
func cmdExportAuth(cmd *cobra.Command, args []string) error {
ctx := process.Ctx(cmd)
authDB, err := authCfg.Signer.NewAuthDB()
authDB, err := authorization.NewDBFromCfg(authCfg.Authorizations)
if err != nil {
return err
}
@ -246,7 +246,7 @@ func cmdExportAuth(cmd *cobra.Command, args []string) error {
return errs.Combine(emailErrs.Err(), csvErrs.Err())
}
func writeAuthExport(ctx context.Context, authDB *certificates.AuthorizationDB, email string, w *csv.Writer) error {
func writeAuthExport(ctx context.Context, authDB *authorization.DB, email string, w *csv.Writer) error {
auths, err := authDB.Get(ctx, email)
if err != nil {
return err

20
cmd/certificates/claims.go
View File

@ -11,7 +11,7 @@ import (
"github.com/spf13/cobra"
"github.com/zeebo/errs"
"storj.io/storj/pkg/certificates"
"storj.io/storj/pkg/certificate/authorization"
"storj.io/storj/pkg/process"
)
@ -33,23 +33,15 @@ var (
Args: cobra.ExactArgs(1),
RunE: cmdDeleteClaim,
}
claimsExportCfg struct {
Signer certificates.CertServerConfig
Raw bool `default:"false" help:"if true, the raw data structures will be printed"`
}
claimsDeleteCfg struct {
Signer certificates.CertServerConfig
}
)
func cmdExportClaims(cmd *cobra.Command, args []string) (err error) {
ctx := process.Ctx(cmd)
authDB, err := claimsExportCfg.Signer.NewAuthDB()
authDB, err := authorization.NewDBFromCfg(claimsExportCfg.Authorizations)
if err != nil {
return err
}
defer func() {
err = errs.Combine(err, authDB.Close())
}()
@ -69,7 +61,7 @@ func cmdExportClaims(cmd *cobra.Command, args []string) (err error) {
}
if len(toPrint) == 0 {
fmt.Printf("no claims in database: %s\n", claimsExportCfg.Signer.AuthorizationDBURL)
fmt.Printf("no claims in database: %s\n", claimsExportCfg.Authorizations.DBURL)
return nil
}
@ -84,7 +76,7 @@ func cmdExportClaims(cmd *cobra.Command, args []string) (err error) {
func cmdDeleteClaim(cmd *cobra.Command, args []string) (err error) {
ctx := process.Ctx(cmd)
authDB, err := claimsDeleteCfg.Signer.NewAuthDB()
authDB, err := authorization.NewDBFromCfg(claimsDeleteCfg.Authorizations)
if err != nil {
return err
}
@ -109,7 +101,7 @@ type printableClaim struct {
NodeID string
}
func toPrintableAuth(auth *certificates.Authorization) *printableAuth {
func toPrintableAuth(auth *authorization.Authorization) *printableAuth {
pAuth := new(printableAuth)
pAuth.UserID = auth.Token.UserID

46
cmd/certificates/main.go
View File

@ -9,22 +9,13 @@ import (
"go.uber.org/zap"
"storj.io/storj/internal/fpath"
"storj.io/storj/pkg/certificates"
"storj.io/storj/pkg/certificate"
"storj.io/storj/pkg/certificate/authorization"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/revocation"
"storj.io/storj/pkg/server"
)
// CertificatesServerFlags defines certificate server configuration
type CertificatesServerFlags struct {
Identity identity.Config
Server server.Config
Signer certificates.CertServerConfig
}
var (
rootCmd = &cobra.Command{
Use: "certificates",
@ -37,11 +28,11 @@ var (
RunE: cmdRun,
}
runCfg CertificatesServerFlags
runCfg certificate.Config
setupCfg struct {
Overwrite bool `help:"if true ca, identity, and authorization db will be overwritten/truncated" default:"false"`
CertificatesServerFlags
certificate.Config
}
authCfg struct {
@ -51,9 +42,16 @@ var (
EmailsPath string `help:"optional path to a list of emails, delimited by <delimiter>, for batch processing"`
Delimiter string `help:"delimiter to split emails loaded from <emails-path> on (e.g. comma, new-line)" default:"\n"`
CertificatesServerFlags
certificate.Config
}
claimsExportCfg struct {
Raw bool `default:"false" help:"if true, the raw data structures will be printed"`
certificate.Config
}
claimsDeleteCfg certificate.Config
confDir string
identityDir string
)
@ -63,18 +61,32 @@ func cmdRun(cmd *cobra.Command, args []string) error {
identity, err := runCfg.Identity.Load()
if err != nil {
zap.S().Fatal(err)
return err
}
signer, err := runCfg.Signer.Load()
if err != nil {
return err
}
authorizationDB, err := authorization.NewDBFromCfg(runCfg.Authorizations)
if err != nil {
return errs.New("error opening authorizations database: %+v", err)
}
revocationDB, err := revocation.NewDBFromCfg(runCfg.Server.Config)
if err != nil {
return errs.New("Error creating revocation database: %+v", err)
return errs.New("error creating revocation database: %+v", err)
}
defer func() {
err = errs.Combine(err, revocationDB.Close())
}()
return runCfg.Server.Run(ctx, zap.L(), identity, revocationDB, nil, runCfg.Signer)
peer, err := certificate.New(zap.L(), identity, signer, authorizationDB, revocationDB, &runCfg)
if err != nil {
return err
}
return peer.Run(ctx)
}
func main() {

13
cmd/certificates/setup.go
View File

@ -11,6 +11,7 @@ import (
"github.com/spf13/cobra"
"storj.io/storj/internal/fpath"
"storj.io/storj/pkg/certificate/authorization"
"storj.io/storj/pkg/process"
)
@ -43,18 +44,18 @@ func cmdSetup(cmd *cobra.Command, args []string) error {
return nil
}
if setupCfg.Overwrite {
setupCfg.Signer.Overwrite = true
authorizationDB, err := authorization.NewDBFromCfg(setupCfg.Authorizations)
if err != nil {
return err
}
if _, err := setupCfg.Signer.NewAuthDB(); err != nil {
if err := authorizationDB.Close(); err != nil {
return err
}
return process.SaveConfig(cmd, filepath.Join(setupDir, "config.yaml"),
process.SaveConfigWithOverrides(map[string]interface{}{
"ca.cert-path": setupCfg.Signer.CA.CertPath,
"ca.key-path": setupCfg.Signer.CA.KeyPath,
"signer.cert-path": setupCfg.Signer.CertPath,
"signer.key-path": setupCfg.Signer.KeyPath,
"identity.cert-path": setupCfg.Identity.CertPath,
"identity.key-path": setupCfg.Identity.KeyPath,
}))

19
cmd/identity/main.go
View File

@ -16,13 +16,15 @@ import (
"storj.io/storj/internal/fpath"
"storj.io/storj/internal/version"
"storj.io/storj/pkg/certificates"
"storj.io/storj/pkg/certificate/certificateclient"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/pkcrypto"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/revocation"
"storj.io/storj/pkg/transport"
)
const (
@ -57,7 +59,7 @@ var (
Concurrency uint `default:"4" help:"number of concurrent workers for certificate authority generation"`
ParentCertPath string `help:"path to the parent authority's certificate chain"`
ParentKeyPath string `help:"path to the parent authority's private key"`
Signer certificates.CertClientConfig
Signer certificateclient.Config
// TODO: ideally the default is the latest version; can't interpolate struct tags
IdentityVersion uint `default:"0" help:"identity version to use when creating an identity or CA"`
@ -188,13 +190,22 @@ func cmdAuthorize(cmd *cobra.Command, args []string) error {
revocationDB, err := revocation.NewDBFromCfg(config.Signer.TLS)
if err != nil {
return errs.New("Error creating revocation database: %+v", err)
return errs.New("error creating revocation database: %+v", err)
}
defer func() {
err = errs.Combine(err, revocationDB.Close())
}()
signedChainBytes, err := config.Signer.Sign(ctx, ident, authToken, revocationDB)
tlsOpts, err := tlsopts.NewOptions(ident, config.Signer.TLS, nil)
if err != nil {
return err
}
client, err := certificateclient.New(ctx, transport.NewClient(tlsOpts), config.Signer.Address)
if err != nil {
return err
}
signedChainBytes, err := client.Sign(ctx, authToken)
if err != nil {
return errs.New("error occurred while signing certificate: %s\n(identity files were still generated and saved, if you try again existing files will be loaded)", err)
}

184
pkg/certificate/authorization/authorizations.go Normal file
View File

@ -0,0 +1,184 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package authorization
import (
"bytes"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/rsa"
"encoding/gob"
"fmt"
"strconv"
"strings"
"github.com/btcsuite/btcutil/base58"
"github.com/zeebo/errs"
"google.golang.org/grpc/peer"
"gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/pb"
)
const (
// Bucket is the bucket used with a bolt-backed authorizations DB.
Bucket = "authorizations"
// MaxClaimDelaySeconds is the max duration in seconds in the past or
// future that a claim timestamp is allowed to have and still be valid.
MaxClaimDelaySeconds = 15
tokenDataLength = 64 // 2^(64*8) =~ 1.34E+154
tokenDelimiter = ":"
tokenVersion = 0
)
var (
mon = monkit.Package()
// Error is the default authorizations error class.
Error = errs.Class("certificates error")
// ErrAuthorization is used when an error occurs involving an authorization.
ErrAuthorization = errs.Class("authorization error")
// ErrAuthorizationDB is used when an error occurs involving the authorization database.
ErrAuthorizationDB = errs.Class("authorization db error")
// ErrInvalidToken is used when a token is invalid.
ErrInvalidToken = errs.Class("invalid token error")
// ErrAuthorizationCount is used when attempting to create an invalid number of authorizations.
ErrAuthorizationCount = ErrAuthorizationDB.New("cannot add less than one authorizations")
)
// Group is a slice of authorizations for convenient de/serialization.
// and grouping.
type Group []*Authorization
// Authorization represents a single-use authorization token and its status.
type Authorization struct {
Token Token
Claim *Claim
}
// Token is a userID and a random byte array, when serialized, can be used like
// a pre-shared key for claiming certificate signatures.
type Token struct {
// NB: currently email address for convenience
UserID string
Data [tokenDataLength]byte
}
// ClaimOpts hold parameters for claiming an authorization.
type ClaimOpts struct {
Req *pb.SigningRequest
Peer *peer.Peer
ChainBytes [][]byte
MinDifficulty uint16
}
// Claim holds information about the circumstances under which an authorization
// token was claimed.
type Claim struct {
Addr string
Timestamp int64
Identity *identity.PeerIdentity
SignedChainBytes [][]byte
}
func init() {
gob.Register(&ecdsa.PublicKey{})
gob.Register(&rsa.PublicKey{})
gob.Register(elliptic.P256())
}
// NewAuthorization creates a new, unclaimed authorization with a random token value.
func NewAuthorization(userID string) (*Authorization, error) {
token := Token{UserID: userID}
_, err := rand.Read(token.Data[:])
if err != nil {
return nil, ErrAuthorization.Wrap(err)
}
return &Authorization{
Token: token,
}, nil
}
// ParseToken splits the token string on the delimiter to get a userID and data
// for a token and base58 decodes the data.
func ParseToken(tokenString string) (*Token, error) {
splitAt := strings.LastIndex(tokenString, tokenDelimiter)
if splitAt == -1 {
return nil, ErrInvalidToken.New("delimiter missing")
}
userID, b58Data := tokenString[:splitAt], tokenString[splitAt+1:]
if len(userID) == 0 {
return nil, ErrInvalidToken.New("user ID missing")
}
data, _, err := base58.CheckDecode(b58Data)
if err != nil {
return nil, ErrInvalidToken.Wrap(err)
}
if len(data) != tokenDataLength {
return nil, ErrInvalidToken.New("data size mismatch")
}
t := &Token{
UserID: userID,
}
copy(t.Data[:], data)
return t, nil
}
// Unmarshal deserializes a set of authorizations.
func (group *Group) Unmarshal(data []byte) error {
decoder := gob.NewDecoder(bytes.NewBuffer(data))
if err := decoder.Decode(group); err != nil {
return ErrAuthorization.Wrap(err)
}
return nil
}
// Marshal serializes a set of authorizations.
func (group Group) Marshal() ([]byte, error) {
data := new(bytes.Buffer)
encoder := gob.NewEncoder(data)
err := encoder.Encode(group)
if err != nil {
return nil, ErrAuthorization.Wrap(err)
}
return data.Bytes(), nil
}
// GroupByClaimed separates a group of authorizations into a group of claimed
// and a group of open authorizations.
func (group Group) GroupByClaimed() (claimed, open Group) {
for _, auth := range group {
if auth.Claim != nil {
// TODO: check if claim is valid? what if not?
claimed = append(claimed, auth)
} else {
open = append(open, auth)
}
}
return claimed, open
}
// String implements the stringer interface and prevents authorization data
// from completely leaking into logs and errors.
func (a Authorization) String() string {
fmtLen := strconv.Itoa(len(a.Token.UserID) + 7)
return fmt.Sprintf("%."+fmtLen+"s..", a.Token.String())
}
// Equal checks if two tokens have equal user IDs and data
func (t *Token) Equal(cmpToken *Token) bool {
return t.UserID == cmpToken.UserID && bytes.Equal(t.Data[:], cmpToken.Data[:])
}
// String implements the stringer interface. Base68 w/ version and checksum bytes
// are used for easy and reliable human transport.
func (t *Token) String() string {
return fmt.Sprintf("%s:%s", t.UserID, base58.CheckEncode(t.Data[:], tokenVersion))
}

239
pkg/certificates/certificates_test.go → pkg/certificate/authorization/authorizations_test.go
View File

@ -1,7 +1,7 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package certificates
package authorization
import (
"bytes"
@ -17,18 +17,15 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zeebo/errs"
"go.uber.org/zap/zaptest"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/peer"
"storj.io/storj/internal/testcontext"
"storj.io/storj/internal/testidentity"
"storj.io/storj/pkg/certificate/certificateclient"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/pkcrypto"
"storj.io/storj/pkg/revocation"
"storj.io/storj/pkg/server"
"storj.io/storj/pkg/storj"
"storj.io/storj/pkg/transport"
"storj.io/storj/storage"
@ -55,7 +52,7 @@ func TestCertSignerConfig_NewAuthDB(t *testing.T) {
defer ctx.Check(authDB.Close)
assert.NotNil(t, authDB)
assert.NotNil(t, authDB.DB)
assert.NotNil(t, authDB.db)
}
func TestAuthorizationDB_Create(t *testing.T) {
@ -108,14 +105,14 @@ func TestAuthorizationDB_Create(t *testing.T) {
emailKey := storage.Key(testCase.email)
if testCase.startCount == 0 {
_, err = authDB.DB.Get(ctx, emailKey)
_, err = authDB.db.Get(ctx, emailKey)
assert.Error(t, err)
} else {
v, err := authDB.DB.Get(ctx, emailKey)
v, err := authDB.db.Get(ctx, emailKey)
require.NoError(t, err)
require.NotEmpty(t, v)
var existingAuths Authorizations
var existingAuths Group
err = existingAuths.Unmarshal(v)
require.NoError(t, err)
require.Len(t, existingAuths, testCase.startCount)
@ -133,11 +130,11 @@ func TestAuthorizationDB_Create(t *testing.T) {
}
assert.Len(t, expectedAuths, testCase.newCount)
v, err := authDB.DB.Get(ctx, emailKey)
v, err := authDB.db.Get(ctx, emailKey)
assert.NoError(t, err)
assert.NotEmpty(t, v)
var actualAuths Authorizations
var actualAuths Group
err = actualAuths.Unmarshal(v)
assert.NoError(t, err)
assert.Len(t, actualAuths, testCase.endCount)
@ -153,7 +150,7 @@ func TestAuthorizationDB_Get(t *testing.T) {
require.NoError(t, err)
defer ctx.Check(authDB.Close)
var expectedAuths Authorizations
var expectedAuths Group
for i := 0; i < 5; i++ {
expectedAuths = append(expectedAuths, &Authorization{
Token: t1,
@ -163,13 +160,13 @@ func TestAuthorizationDB_Get(t *testing.T) {
authsBytes, err := expectedAuths.Marshal()
require.NoError(t, err)
err = authDB.DB.Put(ctx, storage.Key("user@mail.test"), authsBytes)
err = authDB.db.Put(ctx, storage.Key("user@mail.test"), authsBytes)
require.NoError(t, err)
cases := []struct {
testID,
email string
result Authorizations
result Group
}{
{
"Non-existent email",
@ -407,7 +404,7 @@ func TestNewAuthorization(t *testing.T) {
}
func TestAuthorizations_Marshal(t *testing.T) {
expectedAuths := Authorizations{
expectedAuths := Group{
{Token: t1},
{Token: t2},
}
@ -416,7 +413,7 @@ func TestAuthorizations_Marshal(t *testing.T) {
require.NoError(t, err)
require.NotEmpty(t, authsBytes)
var actualAuths Authorizations
var actualAuths Group
decoder := gob.NewDecoder(bytes.NewBuffer(authsBytes))
err = decoder.Decode(&actualAuths)
assert.NoError(t, err)
@ -425,7 +422,7 @@ func TestAuthorizations_Marshal(t *testing.T) {
}
func TestAuthorizations_Unmarshal(t *testing.T) {
expectedAuths := Authorizations{
expectedAuths := Group{
{Token: t1},
{Token: t2},
}
@ -434,7 +431,7 @@ func TestAuthorizations_Unmarshal(t *testing.T) {
require.NoError(t, err)
require.NotEmpty(t, authsBytes)
var actualAuths Authorizations
var actualAuths Group
err = actualAuths.Unmarshal(authsBytes)
assert.NoError(t, err)
assert.NotNil(t, actualAuths)
@ -442,7 +439,7 @@ func TestAuthorizations_Unmarshal(t *testing.T) {
}
func TestAuthorizations_Group(t *testing.T) {
auths := make(Authorizations, 10)
auths := make(Group, 10)
for i := 0; i < 10; i++ {
if i%2 == 0 {
auths[i] = &Authorization{
@ -458,7 +455,7 @@ func TestAuthorizations_Group(t *testing.T) {
}
}
claimed, open := auths.Group()
claimed, open := auths.GroupByClaimed()
for _, a := range claimed {
assert.NotNil(t, a.Claim)
}
@ -567,122 +564,6 @@ func TestToken_Equal(t *testing.T) {
assert.False(t, t1.Equal(&t2))
}
// TODO: test sad path
func TestCertificateSigner_Sign_E2E(t *testing.T) {
testidentity.SignerVersionsTest(t, func(t *testing.T, _ storj.IDVersion, signer *identity.FullCertificateAuthority) {
testidentity.CompleteIdentityVersionsTest(t, func(t *testing.T, _ storj.IDVersion, serverIdent *identity.FullIdentity) {
testidentity.CompleteIdentityVersionsTest(t, func(t *testing.T, _ storj.IDVersion, clientIdent *identity.FullIdentity) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
caCert := ctx.File("ca.cert")
caKey := ctx.File("ca.key")
userID := "user@mail.test"
signerCAConfig := identity.FullCAConfig{
CertPath: caCert,
KeyPath: caKey,
}
err := signerCAConfig.Save(signer)
require.NoError(t, err)
config := CertServerConfig{
AuthorizationDBURL: "bolt://" + ctx.File("authorizations.db"),
CA: signerCAConfig,
}
authDB, err := config.NewAuthDB()
require.NoError(t, err)
auths, err := authDB.Create(ctx, "user@mail.test", 1)
require.NoError(t, err)
require.NotEmpty(t, auths)
err = authDB.Close()
require.NoError(t, err)
sc := server.Config{
Config: tlsopts.Config{
PeerIDVersions: "*",
},
Address: "127.0.0.1:0",
PrivateAddress: "127.0.0.1:0",
}
revocationDB, err := revocation.NewDBFromCfg(sc.Config)
require.NoError(t, err)
defer ctx.Check(revocationDB.Close)
serverOpts, err := tlsopts.NewOptions(serverIdent, sc.Config, revocationDB)
require.NoError(t, err)
require.NotNil(t, serverOpts)
service, err := server.New(zaptest.NewLogger(t), serverOpts, sc.Address, sc.PrivateAddress, nil, config)
require.NoError(t, err)
require.NotNil(t, service)
ctx.Go(func() error {
err := service.Run(ctx)
assert.NoError(t, err)
return err
})
defer ctx.Check(service.Close)
clientOpts, err := tlsopts.NewOptions(clientIdent, tlsopts.Config{PeerIDVersions: "*"}, nil)
require.NoError(t, err)
clientTransport := transport.NewClient(clientOpts)
client, err := NewClient(ctx, clientTransport, service.Addr().String())
require.NoError(t, err)
require.NotNil(t, client)
signedChainBytes, err := client.Sign(ctx, auths[0].Token.String())
require.NoError(t, err)
require.NotEmpty(t, signedChainBytes)
signedChain, err := pkcrypto.CertsFromDER(signedChainBytes)
require.NoError(t, err)
assert.Equal(t, clientIdent.CA.RawTBSCertificate, signedChain[0].RawTBSCertificate)
assert.Equal(t, signer.Cert.Raw, signedChainBytes[1])
// TODO: test scenario with rest chain
//assert.Equal(t, signingCA.RawRestChain(), signedChainBytes[1:])
err = signedChain[0].CheckSignatureFrom(signer.Cert)
require.NoError(t, err)
err = service.Close()
assert.NoError(t, err)
// NB: re-open after closing for server
authDB, err = config.NewAuthDB()
require.NoError(t, err)
defer ctx.Check(authDB.Close)
require.NotNil(t, authDB)
updatedAuths, err := authDB.Get(ctx, userID)
require.NoError(t, err)
require.NotEmpty(t, updatedAuths)
require.NotNil(t, updatedAuths[0].Claim)
now := time.Now().Unix()
claim := updatedAuths[0].Claim
listenerHost, _, err := net.SplitHostPort(service.Addr().String())
require.NoError(t, err)
claimHost, _, err := net.SplitHostPort(claim.Addr)
require.NoError(t, err)
assert.Equal(t, listenerHost, claimHost)
assert.Equal(t, signedChainBytes, claim.SignedChainBytes)
assert.Condition(t, func() bool {
return now-10 < claim.Timestamp &&
claim.Timestamp < now+10
})
})
})
})
}
func TestNewClient(t *testing.T) {
t.Skip("needs proper grpc listener to work")
@ -716,7 +597,7 @@ func TestNewClient(t *testing.T) {
clientTransport := transport.NewClient(tlsOptions)
t.Run("Basic", func(t *testing.T) {
client, err := NewClient(ctx, clientTransport, listener.Addr().String())
client, err := certificateclient.New(ctx, clientTransport, listener.Addr().String())
assert.NoError(t, err)
assert.NotNil(t, client)
@ -733,7 +614,7 @@ func TestNewClient(t *testing.T) {
pbClient := pb.NewCertificatesClient(conn)
require.NotNil(t, pbClient)
client, err := NewClientFrom(pbClient)
client := certificateclient.NewClientFrom(pbClient)
assert.NoError(t, err)
assert.NotNil(t, client)
@ -741,83 +622,7 @@ func TestNewClient(t *testing.T) {
})
}
func TestCertificateSigner_Sign(t *testing.T) {
testidentity.SignerVersionsTest(t, func(t *testing.T, _ storj.IDVersion, signer *identity.FullCertificateAuthority) {
testidentity.CompleteIdentityVersionsTest(t, func(t *testing.T, _ storj.IDVersion, ident *identity.FullIdentity) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
userID := "user@mail.test"
// TODO: test with all types of authorization DBs (bolt, redis, etc.)
config := CertServerConfig{
AuthorizationDBURL: "bolt://" + ctx.File("authorizations.db"),
}
authDB, err := config.NewAuthDB()
require.NoError(t, err)
defer ctx.Check(authDB.Close)
require.NotNil(t, authDB)
auths, err := authDB.Create(ctx, userID, 1)
require.NoError(t, err)
require.NotEmpty(t, auths)
expectedAddr := &net.TCPAddr{
IP: net.ParseIP("1.2.3.4"),
Port: 5,
}
grpcPeer := &peer.Peer{
Addr: expectedAddr,
AuthInfo: credentials.TLSInfo{
State: tls.ConnectionState{
PeerCertificates: []*x509.Certificate{ident.Leaf, ident.CA},
},
},
}
peerCtx := peer.NewContext(ctx, grpcPeer)
certSigner := NewServer(zaptest.NewLogger(t), signer, authDB, 0)
req := pb.SigningRequest{
Timestamp: time.Now().Unix(),
AuthToken: auths[0].Token.String(),
}
res, err := certSigner.Sign(peerCtx, &req)
require.NoError(t, err)
require.NotNil(t, res)
require.NotEmpty(t, res.Chain)
signedChain, err := pkcrypto.CertsFromDER(res.Chain)
require.NoError(t, err)
assert.Equal(t, ident.CA.RawTBSCertificate, signedChain[0].RawTBSCertificate)
assert.Equal(t, signer.Cert.Raw, signedChain[1].Raw)
// TODO: test scenario with rest chain
//assert.Equal(t, signingCA.RawRestChain(), res.Chain[1:])
err = signedChain[0].CheckSignatureFrom(signer.Cert)
require.NoError(t, err)
updatedAuths, err := authDB.Get(ctx, userID)
require.NoError(t, err)
require.NotEmpty(t, updatedAuths)
require.NotNil(t, updatedAuths[0].Claim)
now := time.Now().Unix()
claim := updatedAuths[0].Claim
assert.Equal(t, expectedAddr.String(), claim.Addr)
assert.Equal(t, res.Chain, claim.SignedChainBytes)
assert.Condition(t, func() bool {
return now-MaxClaimDelaySeconds < claim.Timestamp &&
claim.Timestamp < now+MaxClaimDelaySeconds
})
})
})
}
func newTestAuthDB(ctx *testcontext.Context) (*AuthorizationDB, error) {
dbPath := "bolt://" + ctx.File("authorizations.db")
config := CertServerConfig{
AuthorizationDBURL: dbPath,
}
return config.NewAuthDB()
func newTestAuthDB(ctx *testcontext.Context) (*DB, error) {
dbURL := "bolt://" + ctx.File("authorizations.db")
return NewDB(dbURL, false)
}

276
pkg/certificate/authorization/db.go Normal file
View File

@ -0,0 +1,276 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package authorization
import (
"context"
"os"
"time"
"github.com/zeebo/errs"
"storj.io/storj/internal/dbutil"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/storage"
"storj.io/storj/storage/boltdb"
"storj.io/storj/storage/redis"
)
// DB stores authorizations which may be claimed in exchange for a
// certificate signature.
type DB struct {
db storage.KeyValueStore
}
// Config is the authorization db config.
type Config struct {
DBURL string `default:"bolt://$CONFDIR/authorizations.db" help:"url to the certificate signing authorization database"`
Overwrite bool `default:"false" help:"if true, overwrites config AND authorization db is truncated" setup:"true"`
}
// NewDBFromCfg creates and/or opens the authorization database specified by the config.
func NewDBFromCfg(config Config) (*DB, error) {
return NewDB(config.DBURL, config.Overwrite)
}
// NewDB creates and/or opens the authorization database.
func NewDB(dbURL string, overwrite bool) (*DB, error) {
driver, source, err := dbutil.SplitConnstr(dbURL)
if err != nil {
return nil, extensions.ErrRevocationDB.Wrap(err)
}
authDB := new(DB)
switch driver {
case "bolt":
_, err := os.Stat(source)
if overwrite && err == nil {
if err := os.Remove(source); err != nil {
return nil, err
}
}
authDB.db, err = boltdb.New(source, Bucket)
if err != nil {
return nil, ErrAuthorizationDB.Wrap(err)
}
case "redis":
redisClient, err := redis.NewClientFrom(dbURL)
if err != nil {
return nil, ErrAuthorizationDB.Wrap(err)
}
if overwrite {
if err := redisClient.FlushDB(); err != nil {
return nil, err
}
}
authDB.db = redisClient
default:
return nil, ErrAuthorizationDB.New("database scheme not supported: %s", driver)
}
return authDB, nil
}
// Close closes the authorization database's underlying store.
func (authDB *DB) Close() error {
return ErrAuthorizationDB.Wrap(authDB.db.Close())
}
// Create creates a new authorization and adds it to the authorization database.
func (authDB *DB) Create(ctx context.Context, userID string, count int) (_ Group, err error) {
defer mon.Task()(&ctx, userID, count)(&err)
if len(userID) == 0 {
return nil, ErrAuthorizationDB.New("userID cannot be empty")
}
if count < 1 {
return nil, ErrAuthorizationCount
}
var (
newAuths Group
authErrs errs.Group
)
for i := 0; i < count; i++ {
auth, err := NewAuthorization(userID)
if err != nil {
authErrs.Add(err)
continue
}
newAuths = append(newAuths, auth)
}
if authErrs.Err() != nil {
return nil, ErrAuthorizationDB.Wrap(authErrs.Err())
}
if err := authDB.add(ctx, userID, newAuths); err != nil {
return nil, err
}
return newAuths, nil
}
// Get retrieves authorizations by user ID.
func (authDB *DB) Get(ctx context.Context, userID string) (_ Group, err error) {
defer mon.Task()(&ctx, userID)(&err)
authsBytes, err := authDB.db.Get(ctx, storage.Key(userID))
if err != nil && !storage.ErrKeyNotFound.Has(err) {
return nil, ErrAuthorizationDB.Wrap(err)
}
if authsBytes == nil {
return nil, nil
}
var auths Group
if err := auths.Unmarshal(authsBytes); err != nil {
return nil, ErrAuthorizationDB.Wrap(err)
}
return auths, nil
}
// UserIDs returns a list of all userIDs present in the authorization database.
func (authDB *DB) UserIDs(ctx context.Context) (userIDs []string, err error) {
defer mon.Task()(&ctx)(&err)
err = authDB.db.Iterate(ctx, storage.IterateOptions{
Recurse: true,
}, func(ctx context.Context, iterator storage.Iterator) error {
var listItem storage.ListItem
for iterator.Next(ctx, &listItem) {
userIDs = append(userIDs, listItem.Key.String())
}
return nil
})
return userIDs, err
}
// List returns all authorizations in the database.
func (authDB *DB) List(ctx context.Context) (auths Group, err error) {
defer mon.Task()(&ctx)(&err)
err = authDB.db.Iterate(ctx, storage.IterateOptions{
Recurse: true,
}, func(ctx context.Context, iterator storage.Iterator) error {
var listErrs errs.Group
var listItem storage.ListItem
for iterator.Next(ctx, &listItem) {
var nextAuths Group
if err := nextAuths.Unmarshal(listItem.Value); err != nil {
listErrs.Add(err)
}
auths = append(auths, nextAuths...)
}
return listErrs.Err()
})
return auths, err
}
// Claim marks an authorization as claimed and records claim information.
func (authDB *DB) Claim(ctx context.Context, opts *ClaimOpts) (err error) {
defer mon.Task()(&ctx)(&err)
now := time.Now().Unix()
if !(now-MaxClaimDelaySeconds < opts.Req.Timestamp) ||
!(opts.Req.Timestamp < now+MaxClaimDelaySeconds) {
return ErrAuthorization.New("claim timestamp is outside of max delay window: %d", opts.Req.Timestamp)
}
ident, err := identity.PeerIdentityFromPeer(opts.Peer)
if err != nil {
return err
}
peerDifficulty, err := ident.ID.Difficulty()
if err != nil {
return err
}
if peerDifficulty < opts.MinDifficulty {
return ErrAuthorization.New("difficulty must be greater than: %d", opts.MinDifficulty)
}
token, err := ParseToken(opts.Req.AuthToken)
if err != nil {
return err
}
auths, err := authDB.Get(ctx, token.UserID)
if err != nil {
return err
}
for i, auth := range auths {
if auth.Token.Equal(token) {
if auth.Claim != nil {
return ErrAuthorization.New("authorization has already been claimed: %s", auth.String())
}
auths[i] = &Authorization{
Token: auth.Token,
Claim: &Claim{
Timestamp: now,
Addr: opts.Peer.Addr.String(),
Identity: ident,
SignedChainBytes: opts.ChainBytes,
},
}
if err := authDB.put(ctx, token.UserID, auths); err != nil {
return err
}
break
}
}
mon.Meter("authorization_claim").Mark(1)
return nil
}
// Unclaim removes a claim from an authorization.
func (authDB *DB) Unclaim(ctx context.Context, authToken string) (err error) {
defer mon.Task()(&ctx)(&err)
token, err := ParseToken(authToken)
if err != nil {
return err
}
auths, err := authDB.Get(ctx, token.UserID)
if err != nil {
return err
}
for i, auth := range auths {
if auth.Token.Equal(token) {
auths[i].Claim = nil
return authDB.put(ctx, token.UserID, auths)
}
}
mon.Meter("authorization_claim").Mark(1)
return errs.New("token not found in authorizations DB")
}
func (authDB *DB) add(ctx context.Context, userID string, newAuths Group) (err error) {
defer mon.Task()(&ctx, userID)(&err)
auths, err := authDB.Get(ctx, userID)
if err != nil {
return err
}
auths = append(auths, newAuths...)
return authDB.put(ctx, userID, auths)
}
func (authDB *DB) put(ctx context.Context, userID string, auths Group) (err error) {
defer mon.Task()(&ctx, userID)(&err)
authsBytes, err := auths.Marshal()
if err != nil {
return ErrAuthorizationDB.Wrap(err)
}
if err := authDB.db.Put(ctx, storage.Key(userID), authsBytes); err != nil {
return ErrAuthorizationDB.Wrap(err)
}
return nil
}

8
pkg/certificate/authorization/doc.go Normal file
View File

@ -0,0 +1,8 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
/*
Package authorization is used for managing one-time-use certificate-signing-
authorizations and claims.
*/
package authorization

94
pkg/certificate/certificateclient/client.go Normal file
View File

@ -0,0 +1,94 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package certificateclient
import (
"context"
"time"
"google.golang.org/grpc"
"gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/transport"
)
var mon = monkit.Package()
// Config is a config struct for use with a certificate signing service client.
type Config struct {
Address string `help:"address of the certificate signing rpc service"`
TLS tlsopts.Config
}
// Client implements pb.CertificateClient
type Client struct {
conn *grpc.ClientConn
client pb.CertificatesClient
}
// New creates a new certificate signing grpc client.
func New(ctx context.Context, tc transport.Client, address string) (_ *Client, err error) {
defer mon.Task()(&ctx, address)(&err)
conn, err := tc.DialAddress(ctx, address)
if err != nil {
return nil, err
}
return &Client{
conn: conn,
client: pb.NewCertificatesClient(conn),
}, nil
}
// NewClientFrom creates a new certificate signing gRPC client from an existing
// grpc cert signing client.
func NewClientFrom(client pb.CertificatesClient) *Client {
return &Client{
client: client,
}
}
// Sign submits a certificate signing request given the config.
func (config Config) Sign(ctx context.Context, ident *identity.FullIdentity, authToken string) (_ [][]byte, err error) {
defer mon.Task()(&ctx)(&err)
tlsOpts, err := tlsopts.NewOptions(ident, config.TLS, nil)
if err != nil {
return nil, err
}
client, err := New(ctx, transport.NewClient(tlsOpts), config.Address)
if err != nil {
return nil, err
}
return client.Sign(ctx, authToken)
}
// Sign claims an authorization using the token string and returns a signed
// copy of the client's CA certificate.
func (client *Client) Sign(ctx context.Context, tokenStr string) (_ [][]byte, err error) {
defer mon.Task()(&ctx)(&err)
res, err := client.client.Sign(ctx, &pb.SigningRequest{
AuthToken: tokenStr,
Timestamp: time.Now().Unix(),
})
if err != nil {
return nil, err
}
return res.Chain, nil
}
// Close closes the client.
func (client *Client) Close() error {
if client.conn != nil {
return client.conn.Close()
}
return nil
}

7
pkg/certificate/certificateclient/doc.go Normal file
View File

@ -0,0 +1,7 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
/*
Package certificateclient contains the client for the certificate endpoint.
*/
package certificateclient

8
pkg/certificate/doc.go Normal file
View File

@ -0,0 +1,8 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
/*
Package certificate is responsible for managing certificate signing operations
on peer identities' certificate chains.
*/
package certificate

104
pkg/certificate/endpoint.go Normal file
View File

@ -0,0 +1,104 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package certificate
import (
"context"
"go.uber.org/zap"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/peer"
"google.golang.org/grpc/status"
"storj.io/storj/pkg/certificate/authorization"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/pb"
)
// Endpoint implements pb.CertificatesServer.
type Endpoint struct {
log *zap.Logger
ca *identity.FullCertificateAuthority
authorizationDB *authorization.DB
minDifficulty uint16
}
// NewEndpoint creates a new certificate signing gRPC server.
func NewEndpoint(log *zap.Logger, ca *identity.FullCertificateAuthority, authorizationDB *authorization.DB, minDifficulty uint16) *Endpoint {
return &Endpoint{
log: log,
ca: ca,
authorizationDB: authorizationDB,
minDifficulty: minDifficulty,
}
}
// Sign signs the CA certificate of the remote peer's identity with the `certs.ca` certificate.
// Returns a certificate chain consisting of the remote peer's CA followed by the CA chain.
func (endpoint Endpoint) Sign(ctx context.Context, req *pb.SigningRequest) (_ *pb.SigningResponse, err error) {
defer mon.Task()(&ctx)(&err)
grpcPeer, ok := peer.FromContext(ctx)
if !ok {
msg := "error getting peer from context"
endpoint.log.Error(msg, zap.Error(err))
return nil, internalErr(msg)
}
peerIdent, err := identity.PeerIdentityFromPeer(grpcPeer)
if err != nil {
msg := "error getting peer identity"
endpoint.log.Error(msg, zap.Error(err))
return nil, internalErr(msg)
}
signedPeerCA, err := endpoint.ca.Sign(peerIdent.CA)
if err != nil {
msg := "error signing peer CA"
endpoint.log.Error(msg, zap.Error(err))
return nil, internalErr(msg)
}
signedChainBytes := [][]byte{signedPeerCA.Raw, endpoint.ca.Cert.Raw}
signedChainBytes = append(signedChainBytes, endpoint.ca.RawRestChain()...)
err = endpoint.authorizationDB.Claim(ctx, &authorization.ClaimOpts{
Req: req,
Peer: grpcPeer,
ChainBytes: signedChainBytes,
MinDifficulty: endpoint.minDifficulty,
})
if err != nil {
msg := "error claiming authorization"
endpoint.log.Error(msg, zap.Error(err))
return nil, internalErr(msg)
}
difficulty, err := peerIdent.ID.Difficulty()
if err != nil {
msg := "error checking difficulty"
endpoint.log.Error(msg, zap.Error(err))
return nil, internalErr(msg)
}
token, err := authorization.ParseToken(req.AuthToken)
if err != nil {
msg := "error parsing auth token"
endpoint.log.Error(msg, zap.Error(err))
return nil, internalErr(msg)
}
tokenFormatter := authorization.Authorization{
Token: *token,
}
endpoint.log.Info("certificate successfully signed",
zap.Stringer("node ID", peerIdent.ID),
zap.Uint16("difficulty", difficulty),
zap.Stringer("truncated token", tokenFormatter),
)
return &pb.SigningResponse{
Chain: signedChainBytes,
}, nil
}
func internalErr(msg string) error {
return status.Error(codes.Internal, Error.New(msg).Error())
}

104
pkg/certificate/peer.go Normal file
View File

@ -0,0 +1,104 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package certificate
import (
"context"
"github.com/zeebo/errs"
"go.uber.org/zap"
"gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/internal/errs2"
"storj.io/storj/pkg/certificate/authorization"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/revocation"
"storj.io/storj/pkg/server"
)
var (
mon = monkit.Package()
// Error is the default error class for the certificates peer.
Error = errs.Class("certificates peer error")
)
// Config is the global certificates config.
type Config struct {
Identity identity.Config
Server server.Config
Signer identity.FullCAConfig
Authorizations authorization.Config
MinDifficulty uint `default:"30" help:"minimum difficulty of the requester's identity required to claim an authorization"`
}
// Peer is the certificates server.
type Peer struct {
// core dependencies
Log *zap.Logger
Identity *identity.FullIdentity
Server *server.Server
// services and endpoints
Certificates struct {
AuthorizationDB *authorization.DB
Endpoint *Endpoint
}
}
// New creates a new certificates peer.
func New(log *zap.Logger, ident *identity.FullIdentity, ca *identity.FullCertificateAuthority, authorizationDB *authorization.DB, revocationDB *revocation.DB, config *Config) (*Peer, error) {
peer := &Peer{
Log: log,
Identity: ident,
}
{
log.Debug("Starting listener and server")
sc := config.Server
options, err := tlsopts.NewOptions(peer.Identity, sc.Config, revocationDB)
if err != nil {
return nil, Error.Wrap(errs.Combine(err, peer.Close()))
}
peer.Server, err = server.New(log.Named("server"), options, sc.Address, sc.PrivateAddress, nil)
if err != nil {
return nil, Error.Wrap(err)
}
}
peer.Certificates.AuthorizationDB = authorizationDB
peer.Certificates.Endpoint = NewEndpoint(log.Named("certificates"), ca, authorizationDB, uint16(config.MinDifficulty))
pb.RegisterCertificatesServer(peer.Server.GRPC(), peer.Certificates.Endpoint)
return peer, nil
}
// Run runs the certificates peer until it's either closed or it errors.
func (peer *Peer) Run(ctx context.Context) (err error) {
defer mon.Task()(&ctx)(&err)
return errs2.IgnoreCanceled(peer.Server.Run(ctx))
}
// Close closes all resources.
func (peer *Peer) Close() error {
var errlist errs.Group
if peer.Server != nil {
errlist.Add(peer.Server.Close())
}
if peer.Certificates.AuthorizationDB != nil {
errlist.Add(peer.Certificates.AuthorizationDB.Close())
}
return Error.Wrap(errlist.Err())
}

211
pkg/certificate/peer_test.go Normal file
View File

@ -0,0 +1,211 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package certificate_test
import (
"crypto/tls"
"crypto/x509"
"net"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/peer"
"storj.io/storj/internal/testcontext"
"storj.io/storj/internal/testidentity"
"storj.io/storj/pkg/certificate"
"storj.io/storj/pkg/certificate/authorization"
"storj.io/storj/pkg/certificate/certificateclient"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/pkcrypto"
"storj.io/storj/pkg/server"
"storj.io/storj/pkg/storj"
"storj.io/storj/pkg/transport"
)
// TODO: test sad path
func TestCertificateSigner_Sign_E2E(t *testing.T) {
testidentity.SignerVersionsTest(t, func(t *testing.T, _ storj.IDVersion, signer *identity.FullCertificateAuthority) {
testidentity.CompleteIdentityVersionsTest(t, func(t *testing.T, _ storj.IDVersion, serverIdent *identity.FullIdentity) {
testidentity.CompleteIdentityVersionsTest(t, func(t *testing.T, _ storj.IDVersion, clientIdent *identity.FullIdentity) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
caCert := ctx.File("ca.cert")
caKey := ctx.File("ca.key")
userID := "user@mail.test"
signerCAConfig := identity.FullCAConfig{
CertPath: caCert,
KeyPath: caKey,
}
err := signerCAConfig.Save(signer)
require.NoError(t, err)
authorizationsCfg := authorization.Config{
DBURL: "bolt://" + ctx.File("authorizations.db"),
}
authDB, err := authorization.NewDBFromCfg(authorizationsCfg)
require.NoError(t, err)
require.NotNil(t, authDB)
auths, err := authDB.Create(ctx, "user@mail.test", 1)
require.NoError(t, err)
require.NotEmpty(t, auths)
certificatesCfg := certificate.Config{
Authorizations: authorizationsCfg,
Signer: signerCAConfig,
Server: server.Config{
Address: "127.0.0.1:0",
Config: tlsopts.Config{
PeerIDVersions: "*",
},
},
}
peer, err := certificate.New(zaptest.NewLogger(t), serverIdent, signer, authDB, nil, &certificatesCfg)
require.NoError(t, err)
require.NotNil(t, peer)
ctx.Go(func() error {
err := peer.Run(ctx)
assert.NoError(t, err)
return err
})
defer ctx.Check(peer.Close)
clientOpts, err := tlsopts.NewOptions(clientIdent, tlsopts.Config{
PeerIDVersions: "*",
}, nil)
require.NoError(t, err)
clientTransport := transport.NewClient(clientOpts)
client, err := certificateclient.New(ctx, clientTransport, peer.Server.Addr().String())
require.NoError(t, err)
require.NotNil(t, client)
signedChainBytes, err := client.Sign(ctx, auths[0].Token.String())
require.NoError(t, err)
require.NotEmpty(t, signedChainBytes)
signedChain, err := pkcrypto.CertsFromDER(signedChainBytes)
require.NoError(t, err)
assert.Equal(t, clientIdent.CA.RawTBSCertificate, signedChain[0].RawTBSCertificate)
assert.Equal(t, signer.Cert.Raw, signedChainBytes[1])
// TODO: test scenario with rest chain
//assert.Equal(t, signingCA.RawRestChain(), signedChainBytes[1:])
err = signedChain[0].CheckSignatureFrom(signer.Cert)
require.NoError(t, err)
err = peer.Close()
assert.NoError(t, err)
// NB: re-open after closing for server
authDB, err = authorization.NewDBFromCfg(authorizationsCfg)
require.NoError(t, err)
defer ctx.Check(authDB.Close)
require.NotNil(t, authDB)
updatedAuths, err := authDB.Get(ctx, userID)
require.NoError(t, err)
require.NotEmpty(t, updatedAuths)
require.NotNil(t, updatedAuths[0].Claim)
now := time.Now().Unix()
claim := updatedAuths[0].Claim
listenerHost, _, err := net.SplitHostPort(peer.Server.Addr().String())
require.NoError(t, err)
claimHost, _, err := net.SplitHostPort(claim.Addr)
require.NoError(t, err)
assert.Equal(t, listenerHost, claimHost)
assert.Equal(t, signedChainBytes, claim.SignedChainBytes)
assert.Condition(t, func() bool {
return now-10 < claim.Timestamp &&
claim.Timestamp < now+10
})
})
})
})
}
func TestCertificateSigner_Sign(t *testing.T) {
testidentity.SignerVersionsTest(t, func(t *testing.T, _ storj.IDVersion, ca *identity.FullCertificateAuthority) {
testidentity.CompleteIdentityVersionsTest(t, func(t *testing.T, _ storj.IDVersion, ident *identity.FullIdentity) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
userID := "user@mail.test"
// TODO: test with all types of authorization DBs (bolt, redis, etc.)
authDB, err := authorization.NewDB("bolt://"+ctx.File("authorizations.db"), false)
require.NoError(t, err)
defer ctx.Check(authDB.Close)
require.NotNil(t, authDB)
auths, err := authDB.Create(ctx, userID, 1)
require.NoError(t, err)
require.NotEmpty(t, auths)
expectedAddr := &net.TCPAddr{
IP: net.ParseIP("1.2.3.4"),
Port: 5,
}
grpcPeer := &peer.Peer{
Addr: expectedAddr,
AuthInfo: credentials.TLSInfo{
State: tls.ConnectionState{
PeerCertificates: []*x509.Certificate{ident.Leaf, ident.CA},
},
},
}
peerCtx := peer.NewContext(ctx, grpcPeer)
certSigner := certificate.NewEndpoint(zaptest.NewLogger(t), ca, authDB, 0)
req := pb.SigningRequest{
Timestamp: time.Now().Unix(),
AuthToken: auths[0].Token.String(),
}
res, err := certSigner.Sign(peerCtx, &req)
require.NoError(t, err)
require.NotNil(t, res)
require.NotEmpty(t, res.Chain)
signedChain, err := pkcrypto.CertsFromDER(res.Chain)
require.NoError(t, err)
assert.Equal(t, ident.CA.RawTBSCertificate, signedChain[0].RawTBSCertificate)
assert.Equal(t, ca.Cert.Raw, signedChain[1].Raw)
// TODO: test scenario with rest chain
//assert.Equal(t, signingCA.RawRestChain(), res.Chain[1:])
err = signedChain[0].CheckSignatureFrom(ca.Cert)
require.NoError(t, err)
updatedAuths, err := authDB.Get(ctx, userID)
require.NoError(t, err)
require.NotEmpty(t, updatedAuths)
require.NotNil(t, updatedAuths[0].Claim)
now := time.Now().Unix()
claim := updatedAuths[0].Claim
assert.Equal(t, expectedAddr.String(), claim.Addr)
assert.Equal(t, res.Chain, claim.SignedChainBytes)
assert.Condition(t, func() bool {
return now-authorization.MaxClaimDelaySeconds < claim.Timestamp &&
claim.Timestamp < now+authorization.MaxClaimDelaySeconds
})
})
})
}

509
pkg/certificates/certificates.go
View File

@ -1,509 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package certificates
import (
"bytes"
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/rsa"
"encoding/gob"
"fmt"
"strconv"
"strings"
"time"
"github.com/btcsuite/btcutil/base58"
"github.com/zeebo/errs"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/peer"
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/transport"
"storj.io/storj/storage"
)
const (
// AuthorizationsBucket is the bucket used with a bolt-backed authorizations DB.
AuthorizationsBucket = "authorizations"
// MaxClaimDelaySeconds is the max duration in seconds in the past or
// future that a claim timestamp is allowed to have and still be valid.
MaxClaimDelaySeconds = 15
tokenDataLength = 64 // 2^(64*8) =~ 1.34E+154
tokenDelimiter = ":"
tokenVersion = 0
)
var (
mon = monkit.Package()
// ErrAuthorization is used when an error occurs involving an authorization.
ErrAuthorization = errs.Class("authorization error")
// ErrAuthorizationDB is used when an error occurs involving the authorization database.
ErrAuthorizationDB = errs.Class("authorization db error")
// ErrInvalidToken is used when a token is invalid
ErrInvalidToken = errs.Class("invalid token error")
// ErrAuthorizationCount is used when attempting to create an invalid number of authorizations.
ErrAuthorizationCount = ErrAuthorizationDB.New("cannot add less than one authorizations")
)
// CertificateSigner implements pb.CertificatesServer
type CertificateSigner struct {
log *zap.Logger
signer *identity.FullCertificateAuthority
authDB *AuthorizationDB
minDifficulty uint16
}
// AuthorizationDB stores authorizations which may be claimed in exchange for a
// certificate signature.
type AuthorizationDB struct {
DB storage.KeyValueStore
}
// Authorizations is a slice of authorizations for convenient de/serialization
// and grouping.
type Authorizations []*Authorization
// Authorization represents a single-use authorization token and its status
type Authorization struct {
Token Token
Claim *Claim
}
// Token is a userID and a random byte array, when serialized, can be used like
// a pre-shared key for claiming certificate signatures.
type Token struct {
// NB: currently email address for convenience
UserID string
Data [tokenDataLength]byte
}
// ClaimOpts hold parameters for claiming an authorization
type ClaimOpts struct {
Req *pb.SigningRequest
Peer *peer.Peer
ChainBytes [][]byte
MinDifficulty uint16
}
// Claim holds information about the circumstances under which an authorization
// token was claimed.
type Claim struct {
Addr string
Timestamp int64
Identity *identity.PeerIdentity
SignedChainBytes [][]byte
}
// Client implements pb.CertificateClient
type Client struct {
conn *grpc.ClientConn
client pb.CertificatesClient
}
func init() {
gob.Register(&ecdsa.PublicKey{})
gob.Register(&rsa.PublicKey{})
gob.Register(elliptic.P256())
}
// NewServer creates a new certificate signing grpc server
func NewServer(log *zap.Logger, signer *identity.FullCertificateAuthority, authDB *AuthorizationDB, minDifficulty uint16) *CertificateSigner {
return &CertificateSigner{
log: log,
signer: signer,
authDB: authDB,
minDifficulty: minDifficulty,
}
}
// NewClient creates a new certificate signing grpc client
func NewClient(ctx context.Context, tc transport.Client, address string) (*Client, error) {
conn, err := tc.DialAddress(ctx, address)
if err != nil {
return nil, err
}
return &Client{
conn: conn,
client: pb.NewCertificatesClient(conn),
}, nil
}
// NewClientFrom creates a new certificate signing grpc client from an existing
// grpc cert signing client
func NewClientFrom(client pb.CertificatesClient) (*Client, error) {
return &Client{
client: client,
}, nil
}
// NewAuthorization creates a new, unclaimed authorization with a random token value
func NewAuthorization(userID string) (*Authorization, error) {
token := Token{UserID: userID}
_, err := rand.Read(token.Data[:])
if err != nil {
return nil, ErrAuthorization.Wrap(err)
}
return &Authorization{
Token: token,
}, nil
}
// ParseToken splits the token string on the delimiter to get a userID and data
// for a token and base58 decodes the data.
func ParseToken(tokenString string) (*Token, error) {
splitAt := strings.LastIndex(tokenString, tokenDelimiter)
if splitAt == -1 {
return nil, ErrInvalidToken.New("delimiter missing")
}
userID, b58Data := tokenString[:splitAt], tokenString[splitAt+1:]
if len(userID) == 0 {
return nil, ErrInvalidToken.New("user ID missing")
}
data, _, err := base58.CheckDecode(b58Data)
if err != nil {
return nil, ErrInvalidToken.Wrap(err)
}
if len(data) != tokenDataLength {
return nil, ErrInvalidToken.New("data size mismatch")
}
t := &Token{
UserID: userID,
}
copy(t.Data[:], data)
return t, nil
}
// Close closes the client
func (c *Client) Close() error {
if c.conn != nil {
return c.conn.Close()
}
return nil
}
// Sign claims an authorization using the token string and returns a signed
// copy of the client's CA certificate
func (c *Client) Sign(ctx context.Context, tokenStr string) (_ [][]byte, err error) {
defer mon.Task()(&ctx)(&err)
res, err := c.client.Sign(ctx, &pb.SigningRequest{
AuthToken: tokenStr,
Timestamp: time.Now().Unix(),
})
if err != nil {
return nil, err
}
return res.Chain, nil
}
// Sign signs the CA certificate of the remote peer's identity with the signer's certificate.
// Returns a certificate chain consisting of the remote peer's CA followed by the signer's chain.
func (c CertificateSigner) Sign(ctx context.Context, req *pb.SigningRequest) (_ *pb.SigningResponse, err error) {
defer mon.Task()(&ctx)(&err)
grpcPeer, ok := peer.FromContext(ctx)
if !ok {
// TODO: better error
return nil, errs.New("unable to get peer from context")
}
peerIdent, err := identity.PeerIdentityFromPeer(grpcPeer)
if err != nil {
return nil, err
}
signedPeerCA, err := c.signer.Sign(peerIdent.CA)
if err != nil {
return nil, err
}
signedChainBytes := [][]byte{signedPeerCA.Raw, c.signer.Cert.Raw}
signedChainBytes = append(signedChainBytes, c.signer.RawRestChain()...)
err = c.authDB.Claim(ctx, &ClaimOpts{
Req: req,
Peer: grpcPeer,
ChainBytes: signedChainBytes,
MinDifficulty: c.minDifficulty,
})
if err != nil {
return nil, err
}
difficulty, err := peerIdent.ID.Difficulty()
if err != nil {
c.log.Error("error checking difficulty", zap.Error(err))
}
token, err := ParseToken(req.AuthToken)
if err != nil {
c.log.Error("error parsing auth token", zap.Error(err))
}
tokenFormatter := Authorization{
Token: *token,
}
c.log.Info("certificate successfully signed",
zap.Stringer("node ID", peerIdent.ID),
zap.Uint16("difficulty", difficulty),
zap.Stringer("truncated token", tokenFormatter),
)
return &pb.SigningResponse{
Chain: signedChainBytes,
}, nil
}
// Close closes the authorization database's underlying store.
func (authDB *AuthorizationDB) Close() error {
return ErrAuthorizationDB.Wrap(authDB.DB.Close())
}
// Create creates a new authorization and adds it to the authorization database.
func (authDB *AuthorizationDB) Create(ctx context.Context, userID string, count int) (_ Authorizations, err error) {
defer mon.Task()(&ctx)(&err)
if len(userID) == 0 {
return nil, ErrAuthorizationDB.New("userID cannot be empty")
}
if count < 1 {
return nil, ErrAuthorizationCount
}
var (
newAuths Authorizations
authErrs errs.Group
)
for i := 0; i < count; i++ {
auth, err := NewAuthorization(userID)
if err != nil {
authErrs.Add(err)
continue
}
newAuths = append(newAuths, auth)
}
if err := authErrs.Err(); err != nil {
return nil, ErrAuthorizationDB.Wrap(err)
}
if err := authDB.add(ctx, userID, newAuths); err != nil {
return nil, err
}
return newAuths, nil
}
// Get retrieves authorizations by user ID.
func (authDB *AuthorizationDB) Get(ctx context.Context, userID string) (_ Authorizations, err error) {
defer mon.Task()(&ctx)(&err)
authsBytes, err := authDB.DB.Get(ctx, storage.Key(userID))
if err != nil && !storage.ErrKeyNotFound.Has(err) {
return nil, ErrAuthorizationDB.Wrap(err)
}
if authsBytes == nil {
return nil, nil
}
var auths Authorizations
if err := auths.Unmarshal(authsBytes); err != nil {
return nil, ErrAuthorizationDB.Wrap(err)
}
return auths, nil
}
// UserIDs returns a list of all userIDs present in the authorization database.
func (authDB *AuthorizationDB) UserIDs(ctx context.Context) (userIDs []string, err error) {
defer mon.Task()(&ctx)(&err)
err = authDB.DB.Iterate(ctx, storage.IterateOptions{
Recurse: true,
}, func(ctx context.Context, iterator storage.Iterator) error {
var listItem storage.ListItem
for iterator.Next(ctx, &listItem) {
userIDs = append(userIDs, listItem.Key.String())
}
return nil
})
return userIDs, err
}
// List returns all authorizations in the database.
func (authDB *AuthorizationDB) List(ctx context.Context) (auths Authorizations, err error) {
defer mon.Task()(&ctx)(&err)
err = authDB.DB.Iterate(ctx, storage.IterateOptions{
Recurse: true,
}, func(ctx context.Context, iterator storage.Iterator) error {
var listErrs errs.Group
var listItem storage.ListItem
for iterator.Next(ctx, &listItem) {
var nextAuths Authorizations
if err := nextAuths.Unmarshal(listItem.Value); err != nil {
listErrs.Add(err)
}
auths = append(auths, nextAuths...)
}
return listErrs.Err()
})
return auths, err
}
// Claim marks an authorization as claimed and records claim information.
func (authDB *AuthorizationDB) Claim(ctx context.Context, opts *ClaimOpts) (err error) {
defer mon.Task()(&ctx)(&err)
now := time.Now().Unix()
if !(now-MaxClaimDelaySeconds < opts.Req.Timestamp) ||
!(opts.Req.Timestamp < now+MaxClaimDelaySeconds) {
return ErrAuthorization.New("claim timestamp is outside of max delay window: %d", opts.Req.Timestamp)
}
ident, err := identity.PeerIdentityFromPeer(opts.Peer)
if err != nil {
return err
}
peerDifficulty, err := ident.ID.Difficulty()
if err != nil {
return err
}
if peerDifficulty < opts.MinDifficulty {
return ErrAuthorization.New("difficulty must be greater than: %d", opts.MinDifficulty)
}
token, err := ParseToken(opts.Req.AuthToken)
if err != nil {
return err
}
auths, err := authDB.Get(ctx, token.UserID)
if err != nil {
return err
}
for i, auth := range auths {
if auth.Token.Equal(token) {
if auth.Claim != nil {
return ErrAuthorization.New("authorization has already been claimed: %s", auth.String())
}
auths[i] = &Authorization{
Token: auth.Token,
Claim: &Claim{
Timestamp: now,
Addr: opts.Peer.Addr.String(),
Identity: ident,
SignedChainBytes: opts.ChainBytes,
},
}
if err := authDB.put(ctx, token.UserID, auths); err != nil {
return err
}
break
}
}
return nil
}
// Unclaim removes a claim from an authorization.
func (authDB *AuthorizationDB) Unclaim(ctx context.Context, authToken string) (err error) {
defer mon.Task()(&ctx)(&err)
token, err := ParseToken(authToken)
if err != nil {
return err
}
auths, err := authDB.Get(ctx, token.UserID)
if err != nil {
return err
}
for i, auth := range auths {
if auth.Token.Equal(token) {
auths[i].Claim = nil
return authDB.put(ctx, token.UserID, auths)
}
}
return errs.New("token not found in authorizations DB")
}
func (authDB *AuthorizationDB) add(ctx context.Context, userID string, newAuths Authorizations) (err error) {
defer mon.Task()(&ctx)(&err)
auths, err := authDB.Get(ctx, userID)
if err != nil {
return err
}
auths = append(auths, newAuths...)
return authDB.put(ctx, userID, auths)
}
func (authDB *AuthorizationDB) put(ctx context.Context, userID string, auths Authorizations) (err error) {
defer mon.Task()(&ctx)(&err)
authsBytes, err := auths.Marshal()
if err != nil {
return ErrAuthorizationDB.Wrap(err)
}
if err := authDB.DB.Put(ctx, storage.Key(userID), authsBytes); err != nil {
return ErrAuthorizationDB.Wrap(err)
}
return nil
}
// Unmarshal deserializes a set of authorizations
func (a *Authorizations) Unmarshal(data []byte) error {
decoder := gob.NewDecoder(bytes.NewBuffer(data))
if err := decoder.Decode(a); err != nil {
return ErrAuthorization.Wrap(err)
}
return nil
}
// Marshal serializes a set of authorizations
func (a Authorizations) Marshal() ([]byte, error) {
data := new(bytes.Buffer)
encoder := gob.NewEncoder(data)
err := encoder.Encode(a)
if err != nil {
return nil, ErrAuthorization.Wrap(err)
}
return data.Bytes(), nil
}
// Group separates a set of authorizations into a set of claimed and a set of open authorizations.
func (a Authorizations) Group() (claimed, open Authorizations) {
for _, auth := range a {
if auth.Claim != nil {
// TODO: check if claim is valid? what if not?
claimed = append(claimed, auth)
} else {
open = append(open, auth)
}
}
return claimed, open
}
// String implements the stringer interface and prevents authorization data
// from completely leaking into logs and errors.
func (a Authorization) String() string {
fmtLen := strconv.Itoa(len(a.Token.UserID) + 7)
return fmt.Sprintf("%."+fmtLen+"s..", a.Token.String())
}
// Equal checks if two tokens have equal user IDs and data
func (t *Token) Equal(cmpToken *Token) bool {
return t.UserID == cmpToken.UserID && bytes.Equal(t.Data[:], cmpToken.Data[:])
}
// String implements the stringer interface. Base68 w/ version and checksum bytes
// are used for easy and reliable human transport.
func (t *Token) String() string {
return fmt.Sprintf("%s:%s", t.UserID, base58.CheckEncode(t.Data[:], tokenVersion))
}

140
pkg/certificates/config.go
View File

@ -1,140 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package certificates
import (
"context"
"os"
"github.com/zeebo/errs"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"
"storj.io/storj/internal/dbutil"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/server"
"storj.io/storj/pkg/transport"
"storj.io/storj/storage/boltdb"
"storj.io/storj/storage/redis"
)
// CertClientConfig is a config struct for use with a certificate signing service client
type CertClientConfig struct {
Address string `help:"address of the certificate signing rpc service"`
TLS tlsopts.Config
}
// CertServerConfig is a config struct for use with a certificate signing service server
type CertServerConfig struct {
Overwrite bool `default:"false" help:"if true, overwrites config AND authorization db is truncated" setup:"true"`
AuthorizationDBURL string `default:"bolt://$CONFDIR/authorizations.db" help:"url to the certificate signing authorization database"`
MinDifficulty uint `default:"30" help:"minimum difficulty of the requester's identity required to claim an authorization"`
CA identity.FullCAConfig
}
// Sign submits a certificate signing request given the config
func (c CertClientConfig) Sign(ctx context.Context, ident *identity.FullIdentity, authToken string, revDB extensions.RevocationDB) (_ [][]byte, err error) {
defer mon.Task()(&ctx)(&err)
tlsOpts, err := tlsopts.NewOptions(ident, c.TLS, revDB)
if err != nil {
return nil, err
}
client, err := NewClient(ctx, transport.NewClient(tlsOpts), c.Address)
if err != nil {
return nil, err
}
return client.Sign(ctx, authToken)
}
// NewAuthDB creates or opens the authorization database specified by the config
func (c CertServerConfig) NewAuthDB() (*AuthorizationDB, error) {
// TODO: refactor db selection logic?
driver, source, err := dbutil.SplitConnstr(c.AuthorizationDBURL)
if err != nil {
return nil, extensions.ErrRevocationDB.Wrap(err)
}
authDB := new(AuthorizationDB)
switch driver {
case "bolt":
_, err := os.Stat(source)
if c.Overwrite && err == nil {
if err := os.Remove(source); err != nil {
return nil, err
}
}
authDB.DB, err = boltdb.New(source, AuthorizationsBucket)
if err != nil {
return nil, ErrAuthorizationDB.Wrap(err)
}
case "redis":
redisClient, err := redis.NewClientFrom(c.AuthorizationDBURL)
if err != nil {
return nil, ErrAuthorizationDB.Wrap(err)
}
if c.Overwrite {
if err := redisClient.FlushDB(); err != nil {
return nil, err
}
}
authDB.DB = redisClient
default:
return nil, ErrAuthorizationDB.New("database scheme not supported: %s", driver)
}
return authDB, nil
}
// Run implements the responsibility interface, starting a certificate signing server.
func (c CertServerConfig) Run(ctx context.Context, srv *server.Server) (err error) {
defer mon.Task()(&ctx)(&err)
authDB, err := c.NewAuthDB()
if err != nil {
return err
}
defer func() {
err = errs.Combine(err, authDB.Close())
}()
signer, err := c.CA.Load()
if err != nil {
return err
}
certSrv := NewServer(
zap.L(), // TODO: pass this in from somewhere
signer,
authDB,
uint16(c.MinDifficulty),
)
pb.RegisterCertificatesServer(srv.GRPC(), certSrv)
certSrv.log.Info(
"Certificate signing server running",
zap.Stringer("address", srv.Addr()),
)
ctx, cancel := context.WithCancel(ctx)
var group errgroup.Group
group.Go(func() error {
defer cancel()
<-ctx.Done()
return srv.Close()
})
group.Go(func() error {
defer cancel()
return srv.Run(ctx)
})
return group.Wait()
}

6
scripts/test-certificates.sh
View File

@ -35,8 +35,8 @@ _certificates() {
exec certificates --identity-dir "$ident_dir" \
--config-dir "$CERTS_DIR" \
"$subcommand" \
--signer.ca.cert-path "$ca_cert_path" \
--signer.ca.key-path "$ca_key_path" \
--signer.cert-path "$ca_cert_path" \
--signer.key-path "$ca_key_path" \
--server.address "$CERTS_ADDR" \
--server.private-address "$CERTS_ADDR_PRIV" \
--server.revocation-dburl="$rev_dburl" \
@ -78,7 +78,7 @@ for i in {0..4}; do
done
exported_auths=$(_certificates auth export)
_certificates run --signer.min-difficulty 0 &
_certificates run --min-difficulty 0 &
CERTS_PID=$!
sleep 1