Wait-list gating (#534)

pkg/peertls/peertls.go

@@ -36,8 +36,15 @@ var (
 	ErrTLSTemplate = errs.Class("tls template error")
 	// ErrVerifyPeerCert is used when an error occurs during `VerifyPeerCertificate`
 	ErrVerifyPeerCert = errs.Class("tls peer certificate verification error")
+	// ErrParseCerts is used when an error occurs while parsing a certificate or cert chain
+	ErrParseCerts = errs.Class("unable to parse certificate")
 	// ErrVerifySignature is used when a cert-chain signature verificaion error occurs
 	ErrVerifySignature = errs.Class("tls certificate signature verification error")
+	// ErrVerifyCertificateChain is used when a certificate chain can't be verified from leaf to root
+	// (i.e.: each cert in the chain should be signed by the preceding cert and the root should be self-signed)
+	ErrVerifyCertificateChain = errs.Class("certificate chain signature verification failed")
+	// ErrVerifyCAWhitelist is used when the leaf of a peer certificate isn't signed by any CA in the whitelist
+	ErrVerifyCAWhitelist = errs.Class("certificate isn't signed by any CA in the whitelist")
 )
 
 // PeerCertVerificationFunc is the signature for a `*tls.Config{}`'s
@@ -90,13 +97,13 @@ func VerifyPeerFunc(next ...PeerCertVerificationFunc) PeerCertVerificationFunc {
 	return func(chain [][]byte, _ [][]*x509.Certificate) error {
 		c, err := parseCertificateChains(chain)
 		if err != nil {
-			return err
+			return ErrVerifyPeerCert.Wrap(err)
 		}
 
 		for _, n := range next {
 			if n != nil {
 				if err := n(chain, [][]*x509.Certificate{c}); err != nil {
-					return err
+					return ErrVerifyPeerCert.Wrap(err)
 				}
 			}
 		}
@@ -110,6 +117,25 @@ func VerifyPeerCertChains(_ [][]byte, parsedChains [][]*x509.Certificate) error
 	return verifyChainSignatures(parsedChains[0])
 }
 
+// VerifyCAWhitelist verifies that the peer identity's leaf was signed by any one of the
+// (certificate authority) certificates in the provided whitelist
+func VerifyCAWhitelist(cas []*x509.Certificate) PeerCertVerificationFunc {
+	if cas == nil {
+		return nil
+	}
+
+	return func(_ [][]byte, parsedChains [][]*x509.Certificate) error {
+		var err error
+		for _, ca := range cas {
+			err = verifyCertSignature(ca, parsedChains[0][0])
+			if err == nil {
+				return nil
+			}
+		}
+		return ErrVerifyCAWhitelist.Wrap(err)
+	}
+}
+
 // NewKeyBlock converts an ASN1/DER-encoded byte-slice of a private key into
 // a `pem.Block` pointer
 func NewKeyBlock(b []byte) *pem.Block {

pkg/peertls/peertls_test.go

@@ -124,4 +124,65 @@ func TestVerifyPeerCertChains(t *testing.T) {
 
 	err = VerifyPeerFunc(VerifyPeerCertChains)([][]byte{l.Raw, c.Raw}, nil)
 	assert.NoError(t, err)
+
+	c, err = NewCert(ct, nil, &cp.PublicKey, k)
+	assert.NoError(t, err)
+
+	k2, err := NewKey()
+	assert.NoError(t, err)
+
+	l, err = NewCert(lt, nil, &lp.PublicKey, k2)
+	assert.NoError(t, err)
+
+	err = VerifyPeerFunc(VerifyPeerCertChains)([][]byte{l.Raw, c.Raw}, nil)
+	assert.True(t, ErrVerifyPeerCert.Has(err))
+	assert.True(t, ErrVerifyCertificateChain.Has(err))
+}
+
+func TestVerifyCAWhitelist(t *testing.T) {
+	k, err := NewKey()
+	assert.NoError(t, err)
+
+	ct, err := CATemplate()
+	assert.NoError(t, err)
+
+	cp, ok := k.(*ecdsa.PrivateKey)
+	assert.True(t, ok)
+	c, err := NewCert(ct, nil, &cp.PublicKey, k)
+	assert.NoError(t, err)
+
+	lt, err := LeafTemplate()
+	assert.NoError(t, err)
+
+	lp, ok := k.(*ecdsa.PrivateKey)
+	assert.True(t, ok)
+	l, err := NewCert(lt, ct, &lp.PublicKey, k)
+	assert.NoError(t, err)
+
+	err = VerifyPeerFunc(VerifyCAWhitelist(nil))([][]byte{l.Raw, c.Raw}, nil)
+	assert.NoError(t, err)
+
+	err = VerifyPeerFunc(VerifyCAWhitelist([]*x509.Certificate{c}))([][]byte{l.Raw, c.Raw}, nil)
+	assert.NoError(t, err)
+
+	zk, err := NewKey()
+	assert.NoError(t, err)
+
+	zt, err := CATemplate()
+	assert.NoError(t, err)
+
+	zp, ok := zk.(*ecdsa.PrivateKey)
+	assert.True(t, ok)
+	z, err := NewCert(zt, nil, &zp.PublicKey, zk)
+	assert.NoError(t, err)
+
+	err = VerifyPeerFunc(VerifyCAWhitelist([]*x509.Certificate{z}))([][]byte{l.Raw, c.Raw}, nil)
+	assert.True(t, ErrVerifyCAWhitelist.Has(err))
+	assert.True(t, ErrVerifySignature.Has(err))
+
+	err = VerifyPeerFunc(VerifyCAWhitelist([]*x509.Certificate{z, c}))([][]byte{l.Raw, c.Raw}, nil)
+	assert.NoError(t, err)
+
+	err = VerifyPeerFunc(VerifyCAWhitelist([]*x509.Certificate{c, z}))([][]byte{l.Raw, c.Raw}, nil)
+	assert.NoError(t, err)
 }

pkg/peertls/utils.go

@@ -44,7 +44,7 @@ func parseCerts(rawCerts [][]byte) ([]*x509.Certificate, error) {
 		var err error
 		certs[i], err = x509.ParseCertificate(c)
 		if err != nil {
-			return nil, ErrVerifyPeerCert.New("unable to parse certificate: %v", err)
+			return nil, ErrParseCerts.New("unable to parse certificate at index %d", i)
 		}
 	}
 	return certs, nil
@@ -54,53 +54,48 @@ func verifyChainSignatures(certs []*x509.Certificate) error {
 	for i, cert := range certs {
 		j := len(certs)
 		if i+1 < j {
-			isValid, err := verifyCertSignature(certs[i+1], cert)
+			err := verifyCertSignature(certs[i+1], cert)
 			if err != nil {
-				return ErrVerifyPeerCert.Wrap(err)
+				return ErrVerifyCertificateChain.Wrap(err)
-			}
-
-			if !isValid {
-				return ErrVerifyPeerCert.New("certificate chain signature verification failed")
 			}
 
 			continue
 		}
 
-		rootIsValid, err := verifyCertSignature(cert, cert)
+		err := verifyCertSignature(cert, cert)
 		if err != nil {
-			return ErrVerifyPeerCert.Wrap(err)
+			return ErrVerifyCertificateChain.Wrap(err)
 		}
 
-		if !rootIsValid {
-			return ErrVerifyPeerCert.New("certificate chain signature verification failed")
-		}
 	}
 
 	return nil
 }
 
-func verifyCertSignature(parentCert, childCert *x509.Certificate) (bool, error) {
+func verifyCertSignature(parentCert, childCert *x509.Certificate) error {
 	pubKey, ok := parentCert.PublicKey.(*ecdsa.PublicKey)
 	if !ok {
-		return false, ErrUnsupportedKey.New("%T", parentCert.PublicKey)
+		return ErrUnsupportedKey.New("%T", parentCert.PublicKey)
 	}
 
 	signature := new(ecdsaSignature)
 
 	if _, err := asn1.Unmarshal(childCert.Signature, signature); err != nil {
-		return false, ErrVerifySignature.New("unable to unmarshal ecdsa signature: %v", err)
+		return ErrVerifySignature.New("unable to unmarshal ecdsa signature: %v", err)
 	}
 
 	h := crypto.SHA256.New()
 	_, err := h.Write(childCert.RawTBSCertificate)
 	if err != nil {
-		return false, err
+		return ErrVerifySignature.Wrap(err)
 	}
 	digest := h.Sum(nil)
 
-	isValid := ecdsa.Verify(pubKey, digest, signature.R, signature.S)
+	if !ecdsa.Verify(pubKey, digest, signature.R, signature.S) {
+		return ErrVerifySignature.New("signature is not valid")
+	}
 
-	return isValid, nil
+	return nil
 }
 
 func newSerialNumber() (*big.Int, error) {

pkg/provider/identity.go

@@ -53,6 +53,9 @@ type FullIdentity struct {
 	ID nodeID
 	// Key is the key this identity uses with the leaf for communication.
 	Key crypto.PrivateKey
+	// PeerCAWhitelist is a whitelist of CA certs which, if present, restricts which peers this identity will verify as valid;
+	// peer certs must be signed by a CA in this list to pass peer certificate verification.
+	PeerCAWhitelist []*x509.Certificate
 }
 
 // IdentitySetupConfig allows you to run a set of Responsibilities with the given
@@ -67,14 +70,15 @@ type IdentitySetupConfig struct {
 // IdentityConfig allows you to run a set of Responsibilities with the given
 // identity. You can also just load an Identity from disk.
 type IdentityConfig struct {
-	CertPath string `help:"path to the certificate chain for this identity" default:"$CONFDIR/identity.cert"`
+	CertPath            string `help:"path to the certificate chain for this identity" default:"$CONFDIR/identity.cert"`
-	KeyPath  string `help:"path to the private key for this identity" default:"$CONFDIR/identity.key"`
+	KeyPath             string `help:"path to the private key for this identity" default:"$CONFDIR/identity.key"`
-	Address  string `help:"address to listen on" default:":7777"`
+	PeerCAWhitelistPath string `help:"path to the CA cert whitelist (peer identities must be signed by one these to be verified)"`
+	Address             string `help:"address to listen on" default:":7777"`
 }
 
 // FullIdentityFromPEM loads a FullIdentity from a certificate chain and
 // private key file
-func FullIdentityFromPEM(chainPEM, keyPEM []byte) (*FullIdentity, error) {
+func FullIdentityFromPEM(chainPEM, keyPEM, CAWhitelistPEM []byte) (*FullIdentity, error) {
 	cb, err := decodePEM(chainPEM)
 	if err != nil {
 		return nil, errs.Wrap(err)
@@ -101,11 +105,27 @@ func FullIdentityFromPEM(chainPEM, keyPEM []byte) (*FullIdentity, error) {
 		return nil, err
 	}
 
+	var (
+		wb        [][]byte
+		whitelist []*x509.Certificate
+	)
+	if CAWhitelistPEM != nil {
+		wb, err = decodePEM(CAWhitelistPEM)
+		if err != nil {
+			return nil, errs.Wrap(err)
+		}
+		whitelist, err = ParseCertChain(wb)
+		if err != nil {
+			return nil, errs.Wrap(err)
+		}
+	}
+
 	return &FullIdentity{
-		CA:   ch[1],
+		CA:              ch[1],
-		Leaf: ch[0],
+		Leaf:            ch[0],
-		Key:  k,
+		Key:             k,
-		ID:   i,
+		ID:              i,
+		PeerCAWhitelist: whitelist,
 	}, nil
 }
 
@@ -190,8 +210,12 @@ func (ic IdentityConfig) Load() (*FullIdentity, error) {
 	if err != nil {
 		return nil, peertls.ErrNotExist.Wrap(err)
 	}
+	w, err := ioutil.ReadFile(ic.PeerCAWhitelistPath)
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
 
-	fi, err := FullIdentityFromPEM(c, k)
+	fi, err := FullIdentityFromPEM(c, k, w)
 	if err != nil {
 		return nil, errs.New("failed to load identity %#v, %#v: %v",
 			ic.CertPath, ic.KeyPath, err)
@@ -251,19 +275,23 @@ func (ic IdentityConfig) Run(ctx context.Context, interceptor grpc.UnaryServerIn
 
 // ServerOption returns a grpc `ServerOption` for incoming connections
 // to the node with this full identity
-func (fi *FullIdentity) ServerOption() (grpc.ServerOption, error) {
+func (fi *FullIdentity) ServerOption(pcvFuncs ...peertls.PeerCertVerificationFunc) (grpc.ServerOption, error) {
 	ch := [][]byte{fi.Leaf.Raw, fi.CA.Raw}
 	c, err := peertls.TLSCert(ch, fi.Leaf, fi.Key)
 	if err != nil {
 		return nil, err
 	}
 
+	pcvFuncs = append(
+		[]peertls.PeerCertVerificationFunc{peertls.VerifyPeerCertChains},
+		pcvFuncs...,
+	)
 	tlsConfig := &tls.Config{
 		Certificates:       []tls.Certificate{*c},
 		InsecureSkipVerify: true,
 		ClientAuth:         tls.RequireAnyClientCert,
 		VerifyPeerCertificate: peertls.VerifyPeerFunc(
-			peertls.VerifyPeerCertChains,
+			pcvFuncs...,
 		),
 	}
 

pkg/provider/identity_test.go

@@ -9,8 +9,10 @@ import (
 	"crypto/ecdsa"
 	"crypto/x509"
 	"encoding/pem"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
 	"testing"
@@ -87,7 +89,7 @@ func TestFullIdentityFromPEM(t *testing.T) {
 	keyPEM := bytes.NewBuffer([]byte{})
 	assert.NoError(t, pem.Encode(keyPEM, peertls.NewKeyBlock(lkB)))
 
-	fi, err := FullIdentityFromPEM(chainPEM.Bytes(), keyPEM.Bytes())
+	fi, err := FullIdentityFromPEM(chainPEM.Bytes(), keyPEM.Bytes(), nil)
 	assert.NoError(t, err)
 	assert.Equal(t, l.Raw, fi.Leaf.Raw)
 	assert.Equal(t, c.Raw, fi.CA.Raw)
@@ -183,7 +185,7 @@ AwEHoUQDQgAEoLy/0hs5deTXZunRumsMkiHpF0g8wAc58aXANmr7Mxx9tzoIYFnx
 	ic, cleanup, err := tempIdentityConfig()
 	assert.NoError(t, err)
 
-	fi, err := FullIdentityFromPEM([]byte(chain), []byte(key))
+	fi, err := FullIdentityFromPEM([]byte(chain), []byte(key), nil)
 	assert.NoError(t, err)
 
 	return cleanup, ic, fi, difficulty
@@ -198,6 +200,7 @@ func TestIdentityConfig_LoadIdentity(t *testing.T) {
 
 	fi, err := ic.Load()
 	assert.NoError(t, err)
+	assert.Nil(t, fi.PeerCAWhitelist)
 	assert.NotEmpty(t, fi)
 	assert.NotEmpty(t, fi.Key)
 	assert.NotEmpty(t, fi.Leaf)
@@ -208,6 +211,24 @@ func TestIdentityConfig_LoadIdentity(t *testing.T) {
 	assert.Equal(t, expectedFI.Leaf, fi.Leaf)
 	assert.Equal(t, expectedFI.CA, fi.CA)
 	assert.Equal(t, expectedFI.ID.Bytes(), fi.ID.Bytes())
+
+	tmp := path.Join(os.TempDir(), "temp-ca-whitelist.pem")
+	w, err := os.Create(tmp)
+	assert.NoError(t, err)
+	defer func() {
+		err := os.RemoveAll(tmp)
+		if err != nil {
+			fmt.Printf("unable to cleanup temp ca whitelist at \"%s\": %s", tmp, err.Error())
+		}
+	}()
+
+	err = peertls.WriteChain(w, fi.CA)
+	assert.NoError(t, err)
+
+	ic.PeerCAWhitelistPath = tmp
+	fi, err = ic.Load()
+	assert.NoError(t, err)
+	assert.NotEmpty(t, fi.PeerCAWhitelist)
 }
 
 func TestNodeID_Difficulty(t *testing.T) {

pkg/provider/provider.go

@@ -15,6 +15,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
+	"storj.io/storj/pkg/peertls"
 	"storj.io/storj/storage"
 )
 
@@ -44,7 +45,7 @@ type Provider struct {
 func NewProvider(identity *FullIdentity, lis net.Listener, interceptor grpc.UnaryServerInterceptor,
 	responsibilities ...Responsibility) (*Provider, error) {
 	// NB: talk to anyone with an identity
-	ident, err := identity.ServerOption()
+	ident, err := identity.ServerOption(peertls.VerifyCAWhitelist(identity.PeerCAWhitelist))
 	if err != nil {
 		return nil, err
 	}