JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Node Identity (#193)

Browse Source 
* peertls: don't log errors for double close

understood that this part of the code is undergoing heavy change
right now, but just want to make sure this fix gets incorporated
somewhere

* git cleanup: node-id stuff

* cleanup

* rename identity_util.go

* wip `CertificateAuthority` refactor

* refactoring

* gitignore update

* wip

* Merge remote-tracking branch 'storj/doubleclose' into node-id3

* storj/doubleclose:
  peertls: don't log errors for double close

* add peertls tests & gomports

* wip:

+ refactor
+ style changes
+ cleanup
+ [wip] add version to CA and identity configs
+ [wip] heavy client setup

* refactor

* wip:

+ refactor
+ style changes
+ add `CAConfig.Load`
+ add `CAConfig.Save`

* wip:

+ add `LoadOrCreate` and `Create` to CA and Identity configs
+ add overwrite to CA and identity configs
+ heavy client setup
+ refactor
+ style changes
+ cleanup

* wip

* fixing things

* fixing things

* wip hc setup

* hc setup:

+ refactor
+ bugfixing

* improvements based on reveiw feedback

* goimports

* improvements:

+ responding to review feedback
+ refactor

* feedback-based improvements

* feedback-based improvements

* feedback-based improvements

* feedback-based improvements

* feedback-based improvements

* feedback-based improvements

* cleanup

* refactoring CA and Identity structs

* Merge branch 'master' into node-id3

* move version field to setup config structs for CA and identity

* fix typo

* responding to revieiw feedback

* responding to revieiw feedback

* responding to revieiw feedback

* responding to revieiw feedback

* responding to revieiw feedback

* responding to revieiw feedback

* Merge branch 'master' into node-id3

* fix gateway setup finally

* go imports

* fix `FullCertificateAuthority.GenerateIdentity`

* cleanup overlay tests

* bugfixing

* update ca/identity setup

* go imports

* fix peertls test copy/paste fail

* responding to review feedback

* setup tweaking

* update farmer setup
This commit is contained in:
Bryan White 2018-08-13 10:39:45 +02:00 committed by GitHub
parent 198f7fd506
commit 5d20cf8829
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
41 changed files with 1310 additions and 1441 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -23,7 +23,7 @@ debug
./storj
# Jetbrains
.idea/*
.idea/
# vendor
vendor

2
cmd/captplanet/run.go
View File

@ -81,7 +81,7 @@ func cmdRun(cmd *cobra.Command, args []string) (err error) {
runCfg.Farmers[i].Kademlia,
runCfg.Farmers[i].Storage)
}(i)
identity, err := runCfg.Farmers[i].Identity.LoadIdentity()
identity, err := runCfg.Farmers[i].Identity.Load()
if err != nil {
return err
}

59
cmd/captplanet/setup.go
View File

@ -14,16 +14,22 @@ import (
"github.com/spf13/cobra"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/peertls"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/provider"
)
// Config defines broad Captain Planet configuration
type Config struct {
BasePath string `help:"base path for captain planet storage" default:"$CONFDIR"`
ListenHost string `help:"the host for providers to listen on" default:"127.0.0.1"`
StartingPort int `help:"all providers will listen on ports consecutively starting with this one" default:"7777"`
Overwrite bool `help:"whether to overwrite pre-existing configuration files" default:"false"`
HCCA provider.CASetupConfig
HCIdentity provider.IdentitySetupConfig
GWCA provider.CASetupConfig
GWIdentity provider.IdentitySetupConfig
FarmerCA provider.CASetupConfig
FarmerIdentity provider.IdentitySetupConfig
BasePath string `help:"base path for captain planet storage" default:"$CONFDIR"`
ListenHost string `help:"the host for providers to listen on" default:"127.0.0.1"`
StartingPort int `help:"all providers will listen on ports consecutively starting with this one" default:"7777"`
Overwrite bool `help:"whether to overwrite pre-existing configuration files" default:"false"`
}
var (
@ -54,8 +60,11 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
if err != nil {
return err
}
identPath := filepath.Join(hcPath, "ident")
_, err = peertls.NewTLSFileOptions(identPath, identPath, true, true)
setupCfg.HCCA.CertPath = filepath.Join(hcPath, "ca.cert")
setupCfg.HCCA.KeyPath = filepath.Join(hcPath, "ca.key")
setupCfg.HCIdentity.CertPath = filepath.Join(hcPath, "identity.cert")
setupCfg.HCIdentity.KeyPath = filepath.Join(hcPath, "identity.key")
err = provider.SetupIdentity(process.Ctx(cmd), setupCfg.HCCA, setupCfg.HCIdentity)
if err != nil {
return err
}
@ -66,8 +75,13 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
if err != nil {
return err
}
identPath = filepath.Join(farmerPath, "ident")
_, err = peertls.NewTLSFileOptions(identPath, identPath, true, true)
farmerCA := setupCfg.FarmerCA
farmerCA.CertPath = filepath.Join(farmerPath, "ca.cert")
farmerCA.KeyPath = filepath.Join(farmerPath, "ca.key")
farmerIdentity := setupCfg.FarmerIdentity
farmerIdentity.CertPath = filepath.Join(farmerPath, "identity.cert")
farmerIdentity.KeyPath = filepath.Join(farmerPath, "identity.key")
err := provider.SetupIdentity(process.Ctx(cmd), farmerCA, farmerIdentity)
if err != nil {
return err
}
@ -78,8 +92,11 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
if err != nil {
return err
}
identPath = filepath.Join(gwPath, "ident")
_, err = peertls.NewTLSFileOptions(identPath, identPath, true, true)
setupCfg.GWCA.CertPath = filepath.Join(gwPath, "ca.cert")
setupCfg.GWCA.KeyPath = filepath.Join(gwPath, "ca.key")
setupCfg.GWIdentity.CertPath = filepath.Join(gwPath, "identity.cert")
setupCfg.GWIdentity.KeyPath = filepath.Join(gwPath, "identity.key")
err = provider.SetupIdentity(process.Ctx(cmd), setupCfg.GWCA, setupCfg.GWIdentity)
if err != nil {
return err
}
@ -92,10 +109,8 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
}
overrides := map[string]interface{}{
"heavy-client.identity.cert-path": filepath.Join(
setupCfg.BasePath, "hc", "ident.leaf.cert"),
"heavy-client.identity.key-path": filepath.Join(
setupCfg.BasePath, "hc", "ident.leaf.key"),
"heavy-client.identity.cert-path": setupCfg.HCIdentity.CertPath,
"heavy-client.identity.key-path": setupCfg.HCIdentity.KeyPath,
"heavy-client.identity.address": joinHostPort(
setupCfg.ListenHost, startingPort+1),
"heavy-client.kademlia.todo-listen-addr": joinHostPort(
@ -106,10 +121,8 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
setupCfg.BasePath, "hc", "pointerdb.db"),
"heavy-client.overlay.database-url": "bolt://" + filepath.Join(
setupCfg.BasePath, "hc", "overlay.db"),
"gateway.cert-path": filepath.Join(
setupCfg.BasePath, "gw", "ident.leaf.cert"),
"gateway.key-path": filepath.Join(
setupCfg.BasePath, "gw", "ident.leaf.key"),
"gateway.cert-path": setupCfg.GWIdentity.CertPath,
"gateway.key-path": setupCfg.GWIdentity.KeyPath,
"gateway.address": joinHostPort(
setupCfg.ListenHost, startingPort),
"gateway.overlay-addr": joinHostPort(
@ -123,19 +136,19 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
}
for i := 0; i < len(runCfg.Farmers); i++ {
basepath := filepath.Join(setupCfg.BasePath, fmt.Sprintf("f%d", i))
farmerPath := filepath.Join(setupCfg.BasePath, fmt.Sprintf("f%d", i))
farmer := fmt.Sprintf("farmers.%02d.", i)
overrides[farmer+"identity.cert-path"] = filepath.Join(
basepath, "ident.leaf.cert")
farmerPath, "identity.cert")
overrides[farmer+"identity.key-path"] = filepath.Join(
basepath, "ident.leaf.key")
farmerPath, "identity.key")
overrides[farmer+"identity.address"] = joinHostPort(
setupCfg.ListenHost, startingPort+i*2+3)
overrides[farmer+"kademlia.todo-listen-addr"] = joinHostPort(
setupCfg.ListenHost, startingPort+i*2+4)
overrides[farmer+"kademlia.bootstrap-addr"] = joinHostPort(
setupCfg.ListenHost, startingPort+1)
overrides[farmer+"storage.path"] = filepath.Join(basepath, "data")
overrides[farmer+"storage.path"] = filepath.Join(farmerPath, "data")
}
return process.SaveConfig(runCmd.Flags(),

27
cmd/gw/main.go
View File

@ -12,8 +12,8 @@ import (
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/miniogw"
"storj.io/storj/pkg/peertls"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/provider"
)
var (
@ -34,8 +34,11 @@ var (
runCfg miniogw.Config
setupCfg struct {
BasePath string `default:"$CONFDIR" help:"base path for setup"`
Overwrite bool `default:"false" help:"whether to overwrite pre-existing configuration files"`
CA provider.CASetupConfig
Identity provider.IdentitySetupConfig
BasePath string `default:"$CONFDIR" help:"base path for setup"`
Concurrency uint `default:"4" help:"number of concurrent workers for certificate authority generation"`
Overwrite bool `default:"false" help:"whether to overwrite pre-existing configuration files"`
}
defaultConfDir = "$HOME/.storj/gw"
@ -64,14 +67,26 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
return err
}
identityPath := filepath.Join(setupCfg.BasePath, "identity")
_, err = peertls.NewTLSFileOptions(identityPath, identityPath, true, true)
// TODO: handle setting base path *and* identity file paths via args
// NB: if base path is set this overrides identity and CA path options
if setupCfg.BasePath != defaultConfDir {
setupCfg.CA.CertPath = filepath.Join(setupCfg.BasePath, "ca.cert")
setupCfg.CA.KeyPath = filepath.Join(setupCfg.BasePath, "ca.key")
setupCfg.Identity.CertPath = filepath.Join(setupCfg.BasePath, "identity.cert")
setupCfg.Identity.KeyPath = filepath.Join(setupCfg.BasePath, "identity.key")
}
err = provider.SetupIdentity(process.Ctx(cmd), setupCfg.CA, setupCfg.Identity)
if err != nil {
return err
}
o := map[string]interface{}{
"identity.cert-path": setupCfg.CA.CertPath,
"identity.key-path": setupCfg.CA.CertPath,
}
return process.SaveConfig(runCmd.Flags(),
filepath.Join(setupCfg.BasePath, "config.yaml"), nil)
filepath.Join(setupCfg.BasePath, "config.yaml"), o)
}
func main() {

24
cmd/hc/main.go
View File

@ -9,11 +9,9 @@ import (
"path/filepath"
"github.com/spf13/cobra"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/overlay"
"storj.io/storj/pkg/peertls"
"storj.io/storj/pkg/pointerdb"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/provider"
@ -43,7 +41,9 @@ var (
}
setupCfg struct {
BasePath string `default:"$CONFDIR" help:"base path for setup"`
Overwrite bool `default:"false" help:"whether to overwrite pre-existing configuration files"`
CA provider.CASetupConfig
Identity provider.IdentitySetupConfig
Overwrite bool `default:"false" help:"whether to overwrite pre-existing configuration files"`
}
defaultConfDir = "$HOME/.storj/hc"
@ -73,14 +73,26 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
return err
}
identityPath := filepath.Join(setupCfg.BasePath, "identity")
_, err = peertls.NewTLSFileOptions(identityPath, identityPath, true, true)
// TODO: handle setting base path *and* identity file paths via args
// NB: if base path is set this overrides identity and CA path options
if setupCfg.BasePath != defaultConfDir {
setupCfg.CA.CertPath = filepath.Join(setupCfg.BasePath, "ca.cert")
setupCfg.CA.KeyPath = filepath.Join(setupCfg.BasePath, "ca.key")
setupCfg.Identity.CertPath = filepath.Join(setupCfg.BasePath, "identity.cert")
setupCfg.Identity.KeyPath = filepath.Join(setupCfg.BasePath, "identity.key")
}
err = provider.SetupIdentity(process.Ctx(cmd), setupCfg.CA, setupCfg.Identity)
if err != nil {
return err
}
o := map[string]interface{}{
"identity.cert-path": setupCfg.CA.CertPath,
"identity.key-path": setupCfg.CA.CertPath,
}
return process.SaveConfig(runCmd.Flags(),
filepath.Join(setupCfg.BasePath, "config.yaml"), nil)
filepath.Join(setupCfg.BasePath, "config.yaml"), o)
}
func main() {

1
cmd/piecestore-farmer/cmd/create.go
View File

@ -26,6 +26,7 @@ var createCmd = &cobra.Command{
func init() {
RootCmd.AddCommand(createCmd)
// TODO@ASK: this does not create an identity
nodeID, err := kademlia.NewID()
if err != nil {
zap.S().Fatal(err)

2
pkg/dht/dht.go
View File

@ -32,7 +32,7 @@ type RoutingTable interface {
Local() proto.Node
K() int
CacheSize() int
GetBucket(id string) (bucket Bucket, ok bool)
GetBuckets() ([]Bucket, error)

3
pkg/eestream/mocks/mock_client.go
View File

@ -5,8 +5,9 @@
package mock_eestream
import (
gomock "github.com/golang/mock/gomock"
reflect "reflect"
gomock "github.com/golang/mock/gomock"
)
// MockErasureScheme is a mock of ErasureScheme interface

2
pkg/kademlia/config.go
View File

@ -89,4 +89,4 @@ func LoadFromContext(ctx context.Context) *Kademlia {
return v
}
return nil
}
}

1
pkg/kademlia/node_id.go
View File

@ -24,6 +24,7 @@ func StringToNodeID(s string) *NodeID {
return &n
}
// TODO@ASK: this should be removed; superseded by `CASetupConfig.Create` / `IdentitySetupConfig.Create`
// NewID returns a pointer to a newly intialized NodeID
func NewID() (*NodeID, error) {
b, err := newID()

7
pkg/kademlia/routing.go
View File

@ -4,10 +4,10 @@
package kademlia
import (
"encoding/binary"
"context"
"encoding/binary"
"encoding/hex"
"sync"
"time"
@ -69,7 +69,6 @@ func NewRoutingTable(localNode *proto.Node, options *RoutingOptions) (*RoutingTa
return rt, nil
}
// Local returns the local nodes ID
func (rt *RoutingTable) Local() proto.Node {
return *rt.self
@ -137,7 +136,7 @@ func (rt *RoutingTable) FindNear(id dht.NodeID, limit int) ([]*proto.Node, error
} else {
nearIDs = sortedIDs[1 : limit+1]
}
ids,serializedNodes, err := rt.getNodesFromIDs(nearIDs)
ids, serializedNodes, err := rt.getNodesFromIDs(nearIDs)
if err != nil {
return []*proto.Node{}, RoutingErr.New("could not get nodes %s", err)
}

7
pkg/kademlia/routing_helpers.go
View File

@ -8,8 +8,9 @@ import (
"math/rand"
// "strconv"
// "fmt"
"time"
"encoding/binary"
"time"
pb "github.com/golang/protobuf/proto"
proto "storj.io/storj/protos/overlay"
@ -40,7 +41,7 @@ func (rt *RoutingTable) addNode(node *proto.Node) error {
kadBucketID, err := rt.getKBucketID(nodeKey)
if err != nil {
return RoutingErr.New("could not getKBucketID: %s", err)
}
}
hasRoom, err := rt.kadBucketHasRoom(kadBucketID)
if err != nil {
return err
@ -294,7 +295,7 @@ func (rt *RoutingTable) getNodeIDsWithinKBucket(bucketID storage.Key) (storage.K
return nil, nil
}
// getNodesFromIDs: helper, returns
// getNodesFromIDs: helper, returns
func (rt *RoutingTable) getNodesFromIDs(nodeIDs storage.Keys) (storage.Keys, []storage.Value, error) {
var nodes []storage.Value
for _, v := range nodeIDs {

2
pkg/kademlia/routing_test.go
View File

@ -109,7 +109,7 @@ func TestSetBucketTimestamp(t *testing.T) {
idStr := string(id)
rt := createRT(id)
now := time.Now().UTC()
err := rt.createOrUpdateKBucket(id, now)
assert.NoError(t, err)
ti, err := rt.GetBucketTimestamp(idStr, nil)

2
pkg/miniogw/config.go
View File

@ -66,7 +66,7 @@ type Config struct {
func (c Config) Run(ctx context.Context) (err error) {
defer mon.Task()(&ctx)(&err)
identity, err := c.LoadIdentity()
identity, err := c.Load()
if err != nil {
return err
}

2
pkg/node/server.go
View File

@ -21,6 +21,6 @@ func (s *Server) Query(ctx context.Context, req proto.QueryRequest) (proto.Query
// TODO(coyle): this will need to be added to the overlay service
//look for node in routing table?
//If not in there, add node to routing table?
return proto.QueryResponse{}, nil
}

6
pkg/overlay/cache.go
View File

@ -5,8 +5,8 @@ package overlay
import (
"context"
"log"
"crypto/rand"
"log"
"github.com/gogo/protobuf/proto"
"github.com/zeebo/errs"
@ -139,7 +139,7 @@ func (o *Cache) Refresh(ctx context.Context) error {
return err
}
}
// TODO: Kademlia hooks to do this automatically rather than at interval
nodes, err := o.DHT.GetNodes(ctx, "", 128)
if err != nil {
@ -156,7 +156,7 @@ func (o *Cache) Refresh(ctx context.Context) error {
if err != nil {
return err
}
}
return err

2
pkg/overlay/config.go
View File

@ -88,7 +88,7 @@ func (c Config) Run(ctx context.Context, server *provider.Provider) (
if err != nil {
zap.S().Error("Error with cache refresh: ", err)
}
case <- ctx.Done():
case <-ctx.Done():
return
}
}

3
pkg/overlay/mocks/mock_client.go
View File

@ -6,8 +6,9 @@ package mock_overlay
import (
context "context"
gomock "github.com/golang/mock/gomock"
reflect "reflect"
gomock "github.com/golang/mock/gomock"
dht "storj.io/storj/pkg/dht"
overlay "storj.io/storj/protos/overlay"
)

2
pkg/overlay/server.go
View File

@ -70,7 +70,7 @@ func (o *Server) FindStorageNodes(ctx context.Context, req *proto.FindStorageNod
}
if len(result) < int(maxNodes) {
fmt.Printf("result %v",result)
fmt.Printf("result %v", result)
return nil, status.Errorf(codes.ResourceExhausted, fmt.Sprintf("requested %d nodes, only %d nodes matched the criteria requested", maxNodes, len(result)))
}

46
pkg/overlay/service.go
View File

@ -9,7 +9,6 @@ import (
"gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/peertls"
proto "storj.io/storj/protos/overlay"
)
@ -36,48 +35,3 @@ func NewClient(serverAddr string, opts ...grpc.DialOption) (proto.OverlayClient,
return proto.NewOverlayClient(conn), nil
}
// NewTLSServer returns a newly initialized gRPC overlay server, configured with TLS
func NewTLSServer(k *kademlia.Kademlia, cache *Cache, l *zap.Logger, m *monkit.Registry, fopts peertls.TLSFileOptions) (_ *grpc.Server, _ error) {
t, err := peertls.NewTLSFileOptions(
fopts.RootCertRelPath,
fopts.RootKeyRelPath,
fopts.Create,
fopts.Overwrite,
)
if err != nil {
return nil, err
}
grpcServer := grpc.NewServer(t.ServerOption())
proto.RegisterOverlayServer(grpcServer, &Server{
dht: k,
cache: cache,
logger: l,
metrics: m,
})
return grpcServer, nil
}
// NewTLSClient connects to grpc server at the provided address with the provided options plus TLS option(s)
// returns a new instance of an overlay Client
func NewTLSClient(serverAddr *string, fopts peertls.TLSFileOptions, opts ...grpc.DialOption) (proto.OverlayClient, error) {
t, err := peertls.NewTLSFileOptions(
fopts.RootCertRelPath,
fopts.RootCertRelPath,
fopts.Create,
fopts.Overwrite,
)
if err != nil {
return nil, err
}
opts = append(opts, t.DialOption())
conn, err := grpc.Dial(*serverAddr, opts...)
if err != nil {
return nil, err
}
return proto.NewOverlayClient(conn), nil
}

158
pkg/overlay/service_test.go
View File

@ -6,16 +6,12 @@ package overlay
import (
"context"
"fmt"
"io/ioutil"
"net"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"google.golang.org/grpc"
"storj.io/storj/pkg/peertls"
proto "storj.io/storj/protos/overlay" // naming proto to avoid confusion with this package
)
@ -30,99 +26,6 @@ func TestNewServer(t *testing.T) {
srv.Stop()
}
func TestNewClient_CreateTLS(t *testing.T) {
var err error
tmpPath, err := ioutil.TempDir("", "TestNewClient")
assert.NoError(t, err)
defer os.RemoveAll(tmpPath)
lis, err := net.Listen("tcp", fmt.Sprintf(":%d", 0))
assert.NoError(t, err)
basePath := filepath.Join(tmpPath, "TestNewClient_CreateTLS")
srv, tlsOpts := newMockTLSServer(t, basePath, true)
go srv.Serve(lis)
defer srv.Stop()
address := lis.Addr().String()
c, err := NewClient(address, tlsOpts.DialOption())
assert.NoError(t, err)
r, err := c.Lookup(context.Background(), &proto.LookupRequest{})
assert.NoError(t, err)
assert.NotNil(t, r)
}
func TestNewClient_LoadTLS(t *testing.T) {
var err error
tmpPath, err := ioutil.TempDir("", "TestNewClient")
assert.NoError(t, err)
defer os.RemoveAll(tmpPath)
basePath := filepath.Join(tmpPath, "TestNewClient_LoadTLS")
_, err = peertls.NewTLSFileOptions(
basePath,
basePath,
true,
false,
)
assert.NoError(t, err)
lis, err := net.Listen("tcp", fmt.Sprintf(":%d", 0))
assert.NoError(t, err)
// NB: do NOT create a cert, it should be loaded from disk
srv, tlsOpts := newMockTLSServer(t, basePath, false)
go srv.Serve(lis)
defer srv.Stop()
address := lis.Addr().String()
c, err := NewClient(address, tlsOpts.DialOption())
assert.NoError(t, err)
r, err := c.Lookup(context.Background(), &proto.LookupRequest{})
assert.NoError(t, err)
assert.NotNil(t, r)
}
func TestNewClient_IndependentTLS(t *testing.T) {
var err error
tmpPath, err := ioutil.TempDir("", "TestNewClient_IndependentTLS")
assert.NoError(t, err)
defer os.RemoveAll(tmpPath)
clientBasePath := filepath.Join(tmpPath, "client")
serverBasePath := filepath.Join(tmpPath, "server")
lis, err := net.Listen("tcp", fmt.Sprintf(":%d", 0))
assert.NoError(t, err)
srv, _ := newMockTLSServer(t, serverBasePath, true)
go srv.Serve(lis)
defer srv.Stop()
clientTLSOps, err := peertls.NewTLSFileOptions(
clientBasePath,
clientBasePath,
true,
false,
)
assert.NoError(t, err)
address := lis.Addr().String()
c, err := NewClient(address, clientTLSOps.DialOption())
assert.NoError(t, err)
r, err := c.Lookup(context.Background(), &proto.LookupRequest{})
assert.NoError(t, err)
assert.NotNil(t, r)
}
func newMockServer(opts ...grpc.ServerOption) *grpc.Server {
grpcServer := grpc.NewServer(opts...)
proto.RegisterOverlayServer(grpcServer, &MockOverlay{})
@ -130,20 +33,6 @@ func newMockServer(opts ...grpc.ServerOption) *grpc.Server {
return grpcServer
}
func newMockTLSServer(t *testing.T, tlsBasePath string, create bool) (*grpc.Server, *peertls.TLSFileOptions) {
tlsOpts, err := peertls.NewTLSFileOptions(
tlsBasePath,
tlsBasePath,
create,
false,
)
assert.NoError(t, err)
assert.NotNil(t, tlsOpts)
grpcServer := newMockServer(tlsOpts.ServerOption())
return grpcServer, tlsOpts
}
type MockOverlay struct{}
func (o *MockOverlay) FindStorageNodes(ctx context.Context, req *proto.FindStorageNodesRequest) (*proto.FindStorageNodesResponse, error) {
@ -154,53 +43,6 @@ func (o *MockOverlay) Lookup(ctx context.Context, req *proto.LookupRequest) (*pr
return &proto.LookupResponse{}, nil
}
func TestNewTLSServer_Fails(t *testing.T) {
server, err := NewTLSServer(nil, nil, nil, nil, peertls.TLSFileOptions{})
assert.Error(t, err)
assert.NotNil(t, err)
assert.Nil(t, server)
}
func TestNewTLSServer(t *testing.T) {
opts, tempPath := newTLSFileOptions(t)
defer os.RemoveAll(tempPath)
server, err := NewTLSServer(nil, nil, nil, nil, *opts)
assert.NotNil(t, server)
assert.NoError(t, err)
}
func TestNewTLSClient_Fails(t *testing.T) {
address := "127.0.0.1:15550"
client, err := NewTLSClient(&address, peertls.TLSFileOptions{}, nil)
assert.Error(t, err)
assert.NotNil(t, err)
assert.Nil(t, client)
}
func newTLSFileOptions(t *testing.T) (*peertls.TLSFileOptions, string) {
tempPath, err := ioutil.TempDir("", "TestNewPeerTLS")
assert.NoError(t, err)
basePath := filepath.Join(tempPath, "TestNewPeerTLS")
opts, err := peertls.NewTLSFileOptions(
basePath,
basePath,
true,
true,
)
assert.NoError(t, err)
return opts, tempPath
}
func TestNewServerNilArgs(t *testing.T) {
server := NewServer(nil, nil, nil, nil)

83
pkg/peertls/encoding_uitl.go
View File

@ -1,83 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package peertls
import (
"crypto/ecdsa"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"github.com/zeebo/errs"
)
const (
// BlockTypeEcPrivateKey is the value to define a block type of private key
BlockTypeEcPrivateKey = "EC PRIVATE KEY"
// BlockTypeCertificate is the value to define a block type of certificate
BlockTypeCertificate = "CERTIFICATE"
)
func newKeyBlock(b []byte) *pem.Block {
return &pem.Block{Type: BlockTypeEcPrivateKey, Bytes: b}
}
func newCertBlock(b []byte) *pem.Block {
return &pem.Block{Type: BlockTypeCertificate, Bytes: b}
}
func keyToDERBytes(key *ecdsa.PrivateKey) ([]byte, error) {
b, err := x509.MarshalECPrivateKey(key)
if err != nil {
return nil, errs.New("unable to marshal ECDSA private key", err)
}
return b, nil
}
func keyToBlock(key *ecdsa.PrivateKey) (*pem.Block, error) {
b, err := keyToDERBytes(key)
if err != nil {
return nil, err
}
return newKeyBlock(b), nil
}
func certFromPEMs(certPEMBytes, keyPEMBytes []byte) (*tls.Certificate, error) {
certDERs := [][]byte{}
for {
var certDERBlock *pem.Block
certDERBlock, certPEMBytes = pem.Decode(certPEMBytes)
if certDERBlock == nil {
break
}
certDERs = append(certDERs, certDERBlock.Bytes)
}
keyPEMBlock, _ := pem.Decode(keyPEMBytes)
if keyPEMBlock == nil {
return nil, errs.New("unable to decode key PEM data")
}
return certFromDERs(certDERs, keyPEMBlock.Bytes)
}
func certFromDERs(certDERBytes [][]byte, keyDERBytes []byte) (*tls.Certificate, error) {
var (
err error
cert = new(tls.Certificate)
)
cert.Certificate = certDERBytes
cert.PrivateKey, err = x509.ParseECPrivateKey(keyDERBytes)
if err != nil {
return nil, errs.New("unable to parse EC private key", err)
}
return cert, nil
}

174
pkg/peertls/generate.go
View File

@ -1,174 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package peertls
// Many cryptography standards use ASN.1 to define their data structures,
// and Distinguished Encoding Rules (DER) to serialize those structures.
// Because DER produces binary output, it can be challenging to transmit
// the resulting files through systems, like electronic mail, that only
// support ASCII. The PEM format solves this problem by encoding the
// binary data using base64.
// (see https://en.wikipedia.org/wiki/Privacy-enhanced_Electronic_Mail)
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"io/ioutil"
"math/big"
"time"
"github.com/zeebo/errs"
)
const (
// OneYear is the integer represtentation of a calendar year
OneYear = 365 * 24 * time.Hour
)
func (t *TLSFileOptions) generateTLS() error {
if err := t.EnsureAbsPaths(); err != nil {
return ErrGenerate.Wrap(err)
}
rootKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return ErrGenerate.New("failed to generateServerTLS root private key", err)
}
rootT, err := rootTemplate(t)
if err != nil {
return ErrGenerate.Wrap(err)
}
rootC, err := createAndWrite(
t.RootCertAbsPath,
t.RootKeyAbsPath,
rootT,
rootT,
nil,
&rootKey.PublicKey,
rootKey,
rootKey,
)
if err != nil {
return ErrGenerate.Wrap(err)
}
newKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return ErrGenerate.New("failed to generateTLS client private key", err)
}
leafT, err := leafTemplate(t)
if err != nil {
return ErrGenerate.Wrap(err)
}
leafC, err := createAndWrite(
t.LeafCertAbsPath,
t.LeafKeyAbsPath,
leafT,
rootT,
rootC.Certificate,
&newKey.PublicKey,
rootKey,
newKey,
)
if err != nil {
return ErrGenerate.Wrap(err)
}
t.LeafCertificate = leafC
return nil
}
// LoadCert reads and parses a cert/privkey pair from a pair
// of files. The files must contain PEM encoded data. The certificate file
// may contain intermediate certificates following the leaf certificate to
// form a certificate chain. On successful return, Certificate.Leaf will
// be nil because the parsed form of the certificate is not retained.
func LoadCert(certFile, keyFile string) (*tls.Certificate, error) {
certPEMBytes, err := ioutil.ReadFile(certFile)
if err != nil {
return &tls.Certificate{}, err
}
keyPEMBytes, err := ioutil.ReadFile(keyFile)
if err != nil {
return &tls.Certificate{}, err
}
return certFromPEMs(certPEMBytes, keyPEMBytes)
}
func createAndWrite(
certPath,
keyPath string,
template,
parentTemplate *x509.Certificate,
parentDERCerts [][]byte,
pubKey *ecdsa.PublicKey,
rootKey,
privKey *ecdsa.PrivateKey) (*tls.Certificate, error) {
DERCerts, keyDERBytes, err := createDERs(
template,
parentTemplate,
parentDERCerts,
pubKey,
rootKey,
privKey,
)
if err != nil {
return nil, err
}
if err := writeCerts(DERCerts, certPath); err != nil {
return nil, err
}
if err := writeKey(privKey, keyPath); err != nil {
return nil, err
}
return certFromDERs(DERCerts, keyDERBytes)
}
func createDERs(
template,
parentTemplate *x509.Certificate,
parentDERCerts [][]byte,
pubKey *ecdsa.PublicKey,
rootKey,
privKey *ecdsa.PrivateKey) (_ [][]byte, _ []byte, _ error) {
certDERBytes, err := x509.CreateCertificate(rand.Reader, template, parentTemplate, pubKey, rootKey)
if err != nil {
return nil, nil, err
}
DERCerts := [][]byte{}
DERCerts = append(DERCerts, certDERBytes)
DERCerts = append(DERCerts, parentDERCerts...)
keyDERBytes, err := keyToDERBytes(privKey)
if err != nil {
return nil, nil, err
}
return DERCerts, keyDERBytes, nil
}
func newSerialNumber() (*big.Int, error) {
serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
if err != nil {
return nil, errs.New("failed to generateServerTls serial number: %s", err.Error())
}
return serialNumber, nil
}

77
pkg/peertls/io_util.go
View File

@ -1,77 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package peertls
import (
"crypto/ecdsa"
"encoding/pem"
"io"
"log"
"os"
"github.com/zeebo/errs"
)
func writePem(block *pem.Block, file io.WriteCloser) error {
if err := pem.Encode(file, block); err != nil {
return errs.New("unable to PEM-encode/write bytes to file", err)
}
return nil
}
func writeCerts(certs [][]byte, path string) error {
file, err := os.Create(path)
if err != nil {
return errs.New("unable to open file \"%s\" for writing", path, err)
}
defer func() {
if err := file.Close(); err != nil && err != os.ErrClosed {
log.Printf("Failed to close file: %s\n", err)
}
}()
for _, cert := range certs {
if err := writePem(newCertBlock(cert), file); err != nil {
return err
}
}
if err := file.Close(); err != nil {
return errs.New("unable to close cert file \"%s\"", path, err)
}
return nil
}
func writeKey(key *ecdsa.PrivateKey, path string) error {
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return errs.New("unable to open \"%s\" for writing", path, err)
}
defer func() {
if err := file.Close(); err != nil && err != os.ErrClosed {
log.Printf("Failed to close file: %s\n", err)
}
}()
block, err := keyToBlock(key)
if err != nil {
return err
}
if err := writePem(block, file); err != nil {
return err
}
if err := file.Close(); err != nil {
return errs.New("unable to close key filei \"%s\"", path, err)
}
return nil
}

249
pkg/peertls/peertls.go
View File

@ -6,26 +6,32 @@ package peertls
import (
"crypto"
"crypto/ecdsa"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"encoding/asn1"
"fmt"
"math/big"
"os"
"encoding/pem"
"io"
"github.com/zeebo/errs"
)
const (
// BlockTypeEcPrivateKey is the value to define a block type of private key
BlockTypeEcPrivateKey = "EC PRIVATE KEY"
// BlockTypeCertificate is the value to define a block type of certificate
BlockTypeCertificate = "CERTIFICATE"
// BlockTypeIDOptions is the value to define a block type of id options
// (e.g. `version`)
BlockTypeIDOptions = "ID OPTIONS"
)
var (
// ErrNotExist is used when a file or directory doesn't exist
ErrNotExist = errs.Class("file or directory not found error")
// ErrNoOverwrite is used when `create == true && overwrite == false`
// and tls certs/keys already exist at the specified paths
ErrNoOverwrite = errs.Class("tls overwrite disabled error")
// ErrGenerate is used when an error occured during cert/key generation
ErrGenerate = errs.Class("tls generation error")
// ErrTLSOptions is used inconsistently and should probably just be removed
ErrTLSOptions = errs.Class("tls options error")
ErrUnsupportedKey = errs.Class("unsupported key type")
// ErrTLSTemplate is used when an error occurs during tls template generation
ErrTLSTemplate = errs.Class("tls template error")
// ErrVerifyPeerCert is used when an error occurs during `VerifyPeerCertificate`
@ -34,131 +40,134 @@ var (
ErrVerifySignature = errs.Class("tls certificate signature verification error")
)
// IsNotExist checks that a file or directory does not exist
func IsNotExist(err error) bool {
return os.IsNotExist(err) || ErrNotExist.Has(err)
// PeerCertVerificationFunc is the signature for a `*tls.Config{}`'s
// `VerifyPeerCertificate` function.
type PeerCertVerificationFunc func([][]byte, [][]*x509.Certificate) error
func NewKey() (crypto.PrivateKey, error) {
k, err := ecdsa.GenerateKey(authECCurve, rand.Reader)
if err != nil {
return nil, ErrGenerate.New("failed to generate private key", err)
}
return k, nil
}
// TLSFileOptions stores information about a tls certificate and key, and options for use with tls helper functions/methods
type TLSFileOptions struct {
RootCertRelPath string
RootCertAbsPath string
LeafCertRelPath string
LeafCertAbsPath string
// NB: Populate absolute paths from relative paths,
// with respect to pwd via `.EnsureAbsPaths`
RootKeyRelPath string
RootKeyAbsPath string
LeafKeyRelPath string
LeafKeyAbsPath string
LeafCertificate *tls.Certificate
// Create if cert or key nonexistent
Create bool
// Overwrite if `create` is true and cert and/or key exist
Overwrite bool
// NewCert returns a new x509 certificate using the provided templates and
// signed by the `signer` key
func NewCert(template, parentTemplate *x509.Certificate, signer crypto.PrivateKey) (*x509.Certificate, error) {
k, ok := signer.(*ecdsa.PrivateKey)
if !ok {
return nil, ErrUnsupportedKey.New("%T", k)
}
if parentTemplate == nil {
parentTemplate = template
}
cb, err := x509.CreateCertificate(
rand.Reader,
template,
parentTemplate,
&k.PublicKey,
k,
)
if err != nil {
return nil, errs.Wrap(err)
}
c, err := x509.ParseCertificate(cb)
if err != nil {
return nil, errs.Wrap(err)
}
return c, nil
}
type ecdsaSignature struct {
R, S *big.Int
}
// VerifyPeerCertificate verifies that the provided raw certificates are valid
func VerifyPeerCertificate(rawCerts [][]byte, _ [][]*x509.Certificate) error {
// Verify parent ID/sig
// Verify leaf ID/sig
// Verify leaf signed by parent
// TODO(bryanchriswhite): see "S/Kademlia extensions - Secure nodeId generation"
// (https://www.pivotaltracker.com/story/show/158238535)
for i, cert := range rawCerts {
isValid := false
if i < len(rawCerts)-1 {
parentCert, err := x509.ParseCertificate(rawCerts[i+1])
if err != nil {
return ErrVerifyPeerCert.New("unable to parse certificate", err)
}
childCert, err := x509.ParseCertificate(cert)
if err != nil {
return ErrVerifyPeerCert.New("unable to parse certificate", err)
}
isValid, err = verifyCertSignature(parentCert, childCert)
if err != nil {
return ErrVerifyPeerCert.Wrap(err)
}
} else {
rootCert, err := x509.ParseCertificate(cert)
if err != nil {
return ErrVerifyPeerCert.New("unable to parse certificate", err)
}
isValid, err = verifyCertSignature(rootCert, rootCert)
if err != nil {
return ErrVerifyPeerCert.Wrap(err)
}
// VerifyPeerFunc combines multiple `*tls.Config#VerifyPeerCertificate`
// functions and adds certificate parsing.
func VerifyPeerFunc(next ...PeerCertVerificationFunc) PeerCertVerificationFunc {
return func(chain [][]byte, _ [][]*x509.Certificate) error {
c, err := parseCertificateChains(chain)
if err != nil {
return err
}
if !isValid {
return ErrVerifyPeerCert.New("certificate chain signature verification failed")
for _, n := range next {
if n != nil {
if err := n(chain, [][]*x509.Certificate{c}); err != nil {
return err
}
}
}
return nil
}
}
func VerifyPeerCertChains(_ [][]byte, parsedChains [][]*x509.Certificate) error {
return verifyChainSignatures(parsedChains[0])
}
// NewKeyBlock converts an ASN1/DER-encoded byte-slice of a private key into
// a `pem.Block` pointer
func NewKeyBlock(b []byte) *pem.Block {
return &pem.Block{Type: BlockTypeEcPrivateKey, Bytes: b}
}
// NewCertBlock converts an ASN1/DER-encoded byte-slice of a tls certificate
// into a `pem.Block` pointer
func NewCertBlock(b []byte) *pem.Block {
return &pem.Block{Type: BlockTypeCertificate, Bytes: b}
}
func TLSCert(chain [][]byte, leaf *x509.Certificate, key crypto.PrivateKey) (*tls.Certificate, error) {
var err error
if leaf == nil {
leaf, err = x509.ParseCertificate(chain[0])
if err != nil {
return nil, err
}
}
return &tls.Certificate{
Leaf: leaf,
Certificate: chain,
PrivateKey: key,
}, nil
}
// WriteChain writes the certificate chain (leaf-first) to the writer, PEM-encoded.
func WriteChain(w io.Writer, chain ...*x509.Certificate) error {
if len(chain) < 1 {
return errs.New("expected at least one certificate for writing")
}
for _, c := range chain {
if err := pem.Encode(w, NewCertBlock(c.Raw)); err != nil {
return errs.Wrap(err)
}
}
return nil
}
// NewTLSFileOptions initializes a new `TLSFileOption` struct given the arguments
func NewTLSFileOptions(baseCertPath, baseKeyPath string, create, overwrite bool) (_ *TLSFileOptions, _ error) {
t := &TLSFileOptions{
RootCertRelPath: fmt.Sprintf("%s.root.cert", baseCertPath),
RootKeyRelPath: fmt.Sprintf("%s.root.key", baseKeyPath),
LeafCertRelPath: fmt.Sprintf("%s.leaf.cert", baseCertPath),
LeafKeyRelPath: fmt.Sprintf("%s.leaf.key", baseKeyPath),
Overwrite: overwrite,
Create: create,
// WriteChain writes the private key to the writer, PEM-encoded.
func WriteKey(w io.Writer, key crypto.PrivateKey) error {
var (
kb []byte
err error
)
switch k := key.(type) {
case *ecdsa.PrivateKey:
kb, err = x509.MarshalECPrivateKey(k)
if err != nil {
return errs.Wrap(err)
}
default:
return ErrUnsupportedKey.New("%T", k)
}
if err := t.EnsureExists(); err != nil {
return nil, err
if err := pem.Encode(w, NewKeyBlock(kb)); err != nil {
return errs.Wrap(err)
}
return t, nil
}
func verifyCertSignature(parentCert, childCert *x509.Certificate) (bool, error) {
pubkey := parentCert.PublicKey.(*ecdsa.PublicKey)
signature := new(ecdsaSignature)
if _, err := asn1.Unmarshal(childCert.Signature, signature); err != nil {
return false, ErrVerifySignature.New("unable to unmarshal ecdsa signature", err)
}
h := crypto.SHA256.New()
_, err := h.Write(childCert.RawTBSCertificate)
if err != nil {
return false, err
}
digest := h.Sum(nil)
isValid := ecdsa.Verify(pubkey, digest, signature.R, signature.S)
return isValid, nil
}
// * Copyright 2017 gRPC authors.
// * Licensed under the Apache License, Version 2.0 (the "License");
// * (see https://github.com/grpc/grpc-go/blob/v1.13.0/credentials/credentials_util_go18.go)
// cloneTLSConfig returns a shallow clone of the exported
// fields of cfg, ignoring the unexported sync.Once, which
// contains a mutex and must not be copied.
//
// If cfg is nil, a new zero tls.Config is returned.
func cloneTLSConfig(cfg *tls.Config) *tls.Config {
if cfg == nil {
return &tls.Config{}
}
return cfg.Clone()
return nil
}

531
pkg/peertls/peertls_test.go
View File

@ -5,475 +5,92 @@ package peertls
import (
"bytes"
"crypto"
"crypto/ecdsa"
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
"reflect"
"testing"
"testing/quick"
"github.com/stretchr/testify/assert"
"github.com/zeebo/errs"
)
var quickConfig = &quick.Config{
Values: func(values []reflect.Value, r *rand.Rand) {
randHex := fmt.Sprintf("%x", r.Uint32())
values[0] = reflect.ValueOf(randHex)
},
func TestGenerate_CA(t *testing.T) {
k, err := NewKey()
assert.NoError(t, err)
ct, err := CATemplate()
assert.NoError(t, err)
c, err := NewCert(ct, nil, k)
assert.NoError(t, err)
assert.NotEmpty(t, k.(*ecdsa.PrivateKey))
assert.NotEmpty(t, c)
assert.NotEmpty(t, c.PublicKey.(*ecdsa.PublicKey))
err = c.CheckSignatureFrom(c)
assert.NoError(t, err)
}
var quickTLSOptionsConfig = &quick.Config{
Values: func(values []reflect.Value, r *rand.Rand) {
for i := range [3]bool{} {
randHex := fmt.Sprintf("%x", r.Uint32())
values[i] = reflect.ValueOf(randHex)
func TestGenerate_Leaf(t *testing.T) {
k, err := NewKey()
assert.NoError(t, err)
ct, err := CATemplate()
assert.NoError(t, err)
c, err := NewCert(ct, nil, k)
assert.NoError(t, err)
lt, err := LeafTemplate()
assert.NoError(t, err)
l, err := NewCert(lt, ct, k)
assert.NoError(t, err)
assert.NotEmpty(t, k.(*ecdsa.PrivateKey))
assert.NotEmpty(t, l)
assert.NotEmpty(t, l.PublicKey.(*ecdsa.PublicKey))
err = l.CheckSignatureFrom(c)
assert.NoError(t, err)
}
func TestVerifyPeerFunc(t *testing.T) {
k, err := NewKey()
assert.NoError(t, err)
ct, err := CATemplate()
assert.NoError(t, err)
c, err := NewCert(ct, nil, k)
assert.NoError(t, err)
lt, err := LeafTemplate()
assert.NoError(t, err)
l, err := NewCert(lt, ct, k)
assert.NoError(t, err)
testFunc := func(chain [][]byte, parsedChains [][]*x509.Certificate) error {
switch {
case bytes.Compare(chain[1], c.Raw) != 0:
return errs.New("CA cert doesn't match")
case bytes.Compare(chain[0], l.Raw) != 0:
return errs.New("leaf's CA cert doesn't match")
case l.PublicKey.(*ecdsa.PublicKey).Curve != parsedChains[0][0].PublicKey.(*ecdsa.PublicKey).Curve:
return errs.New("leaf public key doesn't match")
case l.PublicKey.(*ecdsa.PublicKey).X.Cmp(parsedChains[0][0].PublicKey.(*ecdsa.PublicKey).X) != 0:
return errs.New("leaf public key doesn't match")
case l.PublicKey.(*ecdsa.PublicKey).Y.Cmp(parsedChains[0][0].PublicKey.(*ecdsa.PublicKey).Y) != 0:
return errs.New("leaf public key doesn't match")
case bytes.Compare(parsedChains[0][1].Raw, c.Raw) != 0:
return errs.New("parsed CA cert doesn't match")
case bytes.Compare(parsedChains[0][0].Raw, l.Raw) != 0:
return errs.New("parsed leaf cert doesn't match")
}
randBool := r.Uint32()&0x01 != 0
values[3] = reflect.ValueOf(randBool)
},
}
var quickLog = func(msg string, obj interface{}, err error) {
if msg != "" {
fmt.Printf("%s:\n", msg)
return nil
}
if obj != nil {
fmt.Printf("obj: %v\n", obj)
}
if err != nil {
fmt.Printf("%+v\n", err)
}
}
type tlsFileOptionsTestCase struct {
tlsFileOptions *TLSFileOptions
before func(*tlsFileOptionsTestCase) error
after func(*tlsFileOptionsTestCase) error
}
func TestNewTLSFileOptions(t *testing.T) {
f := func(cert, key, hosts string, overwrite bool) bool {
tempPath, err := ioutil.TempDir("", "TestNewTLSFileOptions")
assert.NoError(t, err)
defer os.RemoveAll(tempPath)
certBasePath := filepath.Join(tempPath, cert)
keyBasePath := filepath.Join(tempPath, key)
certPath := fmt.Sprintf("%s.leaf.cert", certBasePath)
keyPath := fmt.Sprintf("%s.leaf.key", keyBasePath)
opts, err := NewTLSFileOptions(certBasePath, keyBasePath, true, overwrite)
if !assert.NoError(t, err) {
quickLog("", nil, err)
return false
}
if !assert.Equal(t, opts.RootCertRelPath, fmt.Sprintf("%s.%s.cert", certBasePath, "root")) {
return false
}
if !assert.Equal(t, opts.RootKeyRelPath, fmt.Sprintf("%s.%s.key", keyBasePath, "root")) {
return false
}
if !assert.NotEmpty(t, opts.LeafCertificate) {
return false
}
if !assert.NotEmpty(t, opts.LeafCertificate.PrivateKey) {
return false
}
if !assert.Equal(t, opts.LeafCertRelPath, certPath) {
return false
}
if !assert.Equal(t, opts.LeafKeyRelPath, keyPath) {
return false
}
if !assert.Equal(t, opts.Overwrite, overwrite) {
return false
}
// TODO(bryanchriswhite): check cert/key bytes in memory vs disk
return true
}
err := quick.Check(f, quickTLSOptionsConfig)
err = VerifyPeerFunc(testFunc)([][]byte{l.Raw, c.Raw}, nil)
assert.NoError(t, err)
}
func TestEnsureAbsPath(t *testing.T) {
f := func(val string) (_ bool) {
opts := &TLSFileOptions{
RootCertRelPath: fmt.Sprintf("%s.root.cert", val),
RootKeyRelPath: fmt.Sprintf("%s.root.key", val),
LeafCertRelPath: fmt.Sprintf("%s.leaf.cert", val),
LeafKeyRelPath: fmt.Sprintf("%s.leaf.key", val),
}
opts.EnsureAbsPaths()
// TODO(bryanchriswhite) cleanup/refactor
for _, requiredRole := range opts.requiredFiles() {
for absPtr, role := range opts.pathRoleMap() {
if role == requiredRole {
if *absPtr == "" {
msg := fmt.Sprintf("absolute path for %s is empty string", fileLabels[role])
quickLog(msg, opts, nil)
return false
}
}
}
}
for _, requiredRole := range opts.requiredFiles() {
for absPtr, role := range opts.pathRoleMap() {
base := filepath.Base
if role == requiredRole {
relPath := opts.pathMap()[absPtr]
if base(*absPtr) != base(relPath) {
quickLog("basenames don't match", opts, nil)
return false
}
}
}
}
return true
}
err := quick.Check(f, quickConfig)
assert.NoError(t, err)
}
func TestGenerate(t *testing.T) {
tempPath, err := ioutil.TempDir("", "TestGenerate")
assert.NoError(t, err)
defer os.RemoveAll(tempPath)
f := func(val string) (_ bool) {
basePath := filepath.Join(tempPath, val)
RootCertPath := fmt.Sprintf("%s.root.cert", basePath)
RootKeyPath := fmt.Sprintf("%s.root.key", basePath)
LeafCertPath := fmt.Sprintf("%s.leaf.cert", basePath)
LeafKeyPath := fmt.Sprintf("%s.leaf.key", basePath)
opts := &TLSFileOptions{
RootCertAbsPath: RootCertPath,
RootKeyAbsPath: RootKeyPath,
LeafCertAbsPath: LeafCertPath,
LeafKeyAbsPath: LeafKeyPath,
Create: true,
Overwrite: false,
}
if err := opts.generateTLS(); err != nil {
quickLog("generateTLS error", opts, err)
return false
}
leafCert, err := LoadCert(LeafCertPath, LeafKeyPath)
if err != nil {
quickLog("error leaf loading cert", opts, err)
return false
}
if !certsMatch(leafCert, opts.LeafCertificate) {
quickLog("certs don't match", opts, nil)
return false
}
if !keysMatch(
privKeyBytes(t, opts.LeafCertificate.PrivateKey),
privKeyBytes(t, leafCert.PrivateKey),
) {
quickLog("generated and loaded leaf keys don't match", opts, nil)
return false
}
return true
}
err = quick.Check(f, quickConfig)
assert.NoError(t, err)
}
func TestLoadTLS(t *testing.T) {
tempPath, err := ioutil.TempDir("", "TestLoadTLS")
assert.NoError(t, err)
defer os.RemoveAll(tempPath)
f := func(val string) bool {
var err error
basePath := filepath.Join(tempPath, val)
assert.NoError(t, err)
defer os.RemoveAll(basePath)
// Generate/write certs/keys to files
generatedTLS, err := NewTLSFileOptions(
basePath,
basePath,
true,
true,
)
if err != nil {
quickLog("NewTLSFileOptions error", nil, err)
return false
}
loadedTLS, err := NewTLSFileOptions(
basePath,
basePath,
false,
false,
)
if err != nil {
quickLog("NewTLSFileOptions error", nil, err)
return false
}
if !certsMatch(
generatedTLS.LeafCertificate,
loadedTLS.LeafCertificate,
) {
return false
}
if !keysMatch(
privKeyBytes(t, generatedTLS.LeafCertificate.PrivateKey),
privKeyBytes(t, loadedTLS.LeafCertificate.PrivateKey),
) {
quickLog("keys don't match", nil, nil)
return false
}
return true
}
err = quick.Check(f, quickConfig)
assert.NoError(t, err)
}
func TestEnsureExists_Create(t *testing.T) {
tempPath, err := ioutil.TempDir("", "TestEnsureExists_Create")
assert.NoError(t, err)
defer os.RemoveAll(tempPath)
f := func(val string) bool {
basePath := filepath.Join(tempPath, val)
RootCertPath := fmt.Sprintf("%s.root.cert", basePath)
RootKeyPath := fmt.Sprintf("%s.root.key", basePath)
LeafCertPath := fmt.Sprintf("%s.leaf.cert", basePath)
LeafKeyPath := fmt.Sprintf("%s.leaf.key", basePath)
opts := &TLSFileOptions{
RootCertAbsPath: RootCertPath,
RootKeyAbsPath: RootKeyPath,
LeafCertAbsPath: LeafCertPath,
LeafKeyAbsPath: LeafKeyPath,
Create: true,
Overwrite: false,
}
err := opts.EnsureExists()
if err != nil {
quickLog("ensureExists err", opts, err)
return false
}
for _, requiredRole := range opts.requiredFiles() {
for absPtr, role := range opts.pathRoleMap() {
if role == requiredRole {
if _, err = os.Stat(*absPtr); err != nil {
quickLog("path doesn't exist", opts, nil)
return false
}
}
}
}
// TODO: check for *tls.Certificate and pubkey
return true
}
err = quick.Check(f, quickConfig)
assert.NoError(t, err)
}
func TestEnsureExists_Overwrite(t *testing.T) {
tempPath, err := ioutil.TempDir("", "TestEnsureExists_Overwrite")
assert.NoError(t, err)
defer os.RemoveAll(tempPath)
f := func(val string) (_ bool) {
basePath := filepath.Join(tempPath, val)
RootCertPath := fmt.Sprintf("%s.root.cert", basePath)
RootKeyPath := fmt.Sprintf("%s.root.key", basePath)
LeafCertPath := fmt.Sprintf("%s.leaf.cert", basePath)
LeafKeyPath := fmt.Sprintf("%s.leaf.key", basePath)
checkFiles := func(opts *TLSFileOptions, checkSize bool) bool {
for _, requiredRole := range opts.requiredFiles() {
for absPtr, role := range opts.pathRoleMap() {
if role == requiredRole {
f, err := os.Stat(*absPtr)
if err != nil {
quickLog(fmt.Sprintf("%s path doesn't exist", *absPtr), opts, nil)
return false
}
if checkSize && !(f.Size() > 0) {
quickLog(fmt.Sprintf("%s has size 0", *absPtr), opts, nil)
return false
}
}
}
}
return true
}
requiredFiles := []string{
RootCertPath,
RootKeyPath,
LeafCertPath,
LeafKeyPath,
}
for _, path := range requiredFiles {
if c, err := os.Create(path); err != nil {
quickLog("", nil, errs.Wrap(err))
return false
} else {
c.Close()
}
}
opts := &TLSFileOptions{
RootCertAbsPath: RootCertPath,
RootKeyAbsPath: RootKeyPath,
LeafCertAbsPath: LeafCertPath,
LeafKeyAbsPath: LeafKeyPath,
Create: true,
Overwrite: true,
}
// Ensure files exist to be overwritten
checkFiles(opts, false)
if err := opts.EnsureExists(); err != nil {
quickLog("ensureExists err", opts, err)
return false
}
checkFiles(opts, true)
return true
}
err = quick.Check(f, quickConfig)
assert.NoError(t, err)
}
func TestEnsureExists_NotExistError(t *testing.T) {
tempPath, err := ioutil.TempDir("", "TestEnsureExists_NotExistError")
assert.NoError(t, err)
defer os.RemoveAll(tempPath)
f := func(val string) (_ bool) {
basePath := filepath.Join(tempPath, val)
RootCertPath := fmt.Sprintf("%s.root.cert", basePath)
RootKeyPath := fmt.Sprintf("%s.root.key", basePath)
LeafCertPath := fmt.Sprintf("%s.leaf.cert", basePath)
LeafKeyPath := fmt.Sprintf("%s.leaf.key", basePath)
opts := &TLSFileOptions{
RootCertAbsPath: RootCertPath,
RootKeyAbsPath: RootKeyPath,
LeafCertAbsPath: LeafCertPath,
LeafKeyAbsPath: LeafKeyPath,
Create: false,
Overwrite: false,
}
if err := opts.EnsureExists(); err != nil {
if IsNotExist(err) {
return true
}
quickLog("unexpected err", opts, err)
return false
}
quickLog("didn't error but should've", opts, nil)
return false
}
err = quick.Check(f, quickConfig)
assert.NoError(t, err)
}
func TestNewTLSConfig(t *testing.T) {
tempPath, err := ioutil.TempDir("", "TestNewPeerTLS")
assert.NoError(t, err)
defer os.RemoveAll(tempPath)
basePath := filepath.Join(tempPath, "TestNewPeerTLS")
opts, err := NewTLSFileOptions(
basePath,
basePath,
true,
true,
)
assert.NoError(t, err)
config := opts.NewTLSConfig(nil)
assert.Equal(t, *opts.LeafCertificate, config.Certificates[0])
}
func privKeyBytes(t *testing.T, key crypto.PrivateKey) []byte {
switch key.(type) {
case *ecdsa.PrivateKey:
default:
quickLog("non-ecdsa private key", key, nil)
panic("non-ecdsa private key")
}
ecKey := key.(*ecdsa.PrivateKey)
b, err := x509.MarshalECPrivateKey(ecKey)
assert.NoError(t, err)
return b
}
func certsMatch(c1, c2 *tls.Certificate) bool {
for i, cert := range c1.Certificate {
if bytes.Compare(cert, c2.Certificate[i]) != 0 {
return false
}
}
return true
}
func keysMatch(k1, k2 []byte) bool {
return bytes.Compare(k1, k2) == 0
}

4
pkg/peertls/templates.go
View File

@ -7,7 +7,7 @@ import (
"crypto/x509"
)
func rootTemplate(t *TLSFileOptions) (*x509.Certificate, error) {
func CATemplate() (*x509.Certificate, error) {
serialNumber, err := newSerialNumber()
if err != nil {
return nil, ErrTLSTemplate.Wrap(err)
@ -24,7 +24,7 @@ func rootTemplate(t *TLSFileOptions) (*x509.Certificate, error) {
return template, nil
}
func leafTemplate(t *TLSFileOptions) (*x509.Certificate, error) {
func LeafTemplate() (*x509.Certificate, error) {
serialNumber, err := newSerialNumber()
if err != nil {
return nil, ErrTLSTemplate.Wrap(err)

206
pkg/peertls/tlsfileoptions.go
View File

@ -1,206 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package peertls
import (
"crypto/tls"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/zeebo/errs"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
)
type fileRole int
const (
rootCert fileRole = iota
rootKey
leafCert
leafKey
)
var (
fileLabels = map[fileRole]string{
rootCert: "root certificate",
rootKey: "root key",
leafCert: "leaf certificate",
leafKey: "leaf key",
}
)
// EnsureAbsPaths ensures that the absolute path fields are not empty, deriving them from the relative paths if not
func (t *TLSFileOptions) EnsureAbsPaths() error {
for _, role := range t.requiredFiles() {
for absPtr, relPath := range t.pathMap() {
if t.pathRoleMap()[absPtr] == role {
if *absPtr == "" {
if relPath == "" {
return ErrTLSOptions.New("no relative %s path provided", fileLabels[t.pathRoleMap()[absPtr]])
}
absPath, err := filepath.Abs(relPath)
if err != nil {
return ErrTLSOptions.Wrap(err)
}
*absPtr = absPath
}
}
}
}
return nil
}
// EnsureExists checks whether the cert/key exists and whether to create/overwrite them given `t`s field values.
func (t *TLSFileOptions) EnsureExists() error {
if err := t.EnsureAbsPaths(); err != nil {
return err
}
hasRequiredFiles, err := t.hasRequiredFiles()
if err != nil {
return err
}
if t.Create && !t.Overwrite && hasRequiredFiles {
return ErrNoOverwrite.New("certificates and keys exist; refusing to create without overwrite")
}
// NB: even when `overwrite` is false, this WILL overwrite
// a key if the cert is missing (vice versa)
if t.Create && (t.Overwrite || !hasRequiredFiles) {
if err := t.generateTLS(); err != nil {
return err
}
return nil
}
if !hasRequiredFiles {
missing, _ := t.missingFiles()
return ErrNotExist.New(fmt.Sprintf(strings.Join(missing, ", ")))
}
// NB: ensure `t.Certificate` is not nil when create is false
if !t.Create {
return t.loadTLS()
}
return nil
}
// NewTLSConfig returns a new tls config with defaults set
func (t *TLSFileOptions) NewTLSConfig(c *tls.Config) *tls.Config {
config := cloneTLSConfig(c)
config.Certificates = []tls.Certificate{*t.LeafCertificate}
// Skip normal verification
config.InsecureSkipVerify = true
// Required client certificate
config.ClientAuth = tls.RequireAnyClientCert
// Custom verification logic for *both* client and server
config.VerifyPeerCertificate = VerifyPeerCertificate
return config
}
// NewPeerTLS returns configured TLS transport credentials with the provided config
func (t *TLSFileOptions) NewPeerTLS(config *tls.Config) credentials.TransportCredentials {
return credentials.NewTLS(t.NewTLSConfig(config))
}
// DialOption returns a new grpc ServerOption with a PeerTLS transport credentials
func (t *TLSFileOptions) DialOption() grpc.DialOption {
return grpc.WithTransportCredentials(t.NewPeerTLS(nil))
}
// ServerOption returns a new grpc ServerOption with a PeerTLS initalized
func (t *TLSFileOptions) ServerOption() grpc.ServerOption {
return grpc.Creds(t.NewPeerTLS(nil))
}
func (t *TLSFileOptions) loadTLS() (_ error) {
leafC, err := LoadCert(t.LeafCertAbsPath, t.LeafKeyAbsPath)
if err != nil {
return err
}
t.LeafCertificate = leafC
return nil
}
func (t *TLSFileOptions) missingFiles() ([]string, error) {
missingFiles := []string{}
paths := map[fileRole]string{
rootCert: t.RootCertAbsPath,
rootKey: t.RootKeyAbsPath,
leafCert: t.LeafCertAbsPath,
leafKey: t.LeafKeyAbsPath,
}
requiredFiles := t.requiredFiles()
for _, requiredRole := range requiredFiles {
for role, path := range paths {
if role == requiredRole {
if _, err := os.Stat(path); err != nil {
if !IsNotExist(err) {
return nil, errs.Wrap(err)
}
missingFiles = append(missingFiles, fileLabels[role])
}
}
}
}
return missingFiles, nil
}
func (t *TLSFileOptions) requiredFiles() []fileRole {
var roles = []fileRole{}
// rootCert is always required
roles = append(roles, rootCert, leafCert, leafKey)
if t.Create {
// required for writing rootKey when create is true
roles = append(roles, rootKey)
}
return roles
}
func (t *TLSFileOptions) hasRequiredFiles() (bool, error) {
missingFiles, err := t.missingFiles()
if err != nil {
return false, err
}
return len(missingFiles) == 0, nil
}
func (t *TLSFileOptions) pathMap() map[*string]string {
return map[*string]string{
&t.RootCertAbsPath: t.RootCertRelPath,
&t.RootKeyAbsPath: t.RootKeyRelPath,
&t.LeafCertAbsPath: t.LeafCertRelPath,
&t.LeafKeyAbsPath: t.LeafKeyRelPath,
}
}
func (t *TLSFileOptions) pathRoleMap() map[*string]fileRole {
return map[*string]fileRole{
&t.RootCertAbsPath: rootCert,
&t.RootKeyAbsPath: rootKey,
&t.LeafCertAbsPath: leafCert,
&t.LeafKeyAbsPath: leafKey,
}
}

110
pkg/peertls/utils.go Normal file
View File

@ -0,0 +1,110 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package peertls
// Many cryptography standards use ASN.1 to define their data structures,
// and Distinguished Encoding Rules (DER) to serialize those structures.
// Because DER produces binary output, it can be challenging to transmit
// the resulting files through systems, like electronic mail, that only
// support ASCII. The PEM format solves this problem by encoding the
// binary data using base64.
// (see https://en.wikipedia.org/wiki/Privacy-enhanced_Electronic_Mail)
import (
"crypto"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"encoding/asn1"
"math/big"
"github.com/zeebo/errs"
)
type ecdsaSignature struct {
R, S *big.Int
}
var authECCurve = elliptic.P256()
func parseCertificateChains(rawCerts [][]byte) ([]*x509.Certificate, error) {
parsedCerts, err := parseCerts(rawCerts)
if err != nil {
return nil, err
}
return parsedCerts, nil
}
func parseCerts(rawCerts [][]byte) ([]*x509.Certificate, error) {
certs := make([]*x509.Certificate, len(rawCerts))
for i, c := range rawCerts {
var err error
certs[i], err = x509.ParseCertificate(c)
if err != nil {
return nil, ErrVerifyPeerCert.New("unable to parse certificate", err)
}
}
return certs, nil
}
func verifyChainSignatures(certs []*x509.Certificate) error {
for i, cert := range certs {
j := len(certs)
if i+1 < j {
isValid, err := verifyCertSignature(certs[i], cert)
if err != nil {
return ErrVerifyPeerCert.Wrap(err)
}
if !isValid {
return ErrVerifyPeerCert.New("certificate chain signature verification failed")
}
continue
}
rootIsValid, err := verifyCertSignature(cert, cert)
if err != nil {
return ErrVerifyPeerCert.Wrap(err)
}
if !rootIsValid {
return ErrVerifyPeerCert.New("certificate chain signature verification failed")
}
}
return nil
}
func verifyCertSignature(parentCert, childCert *x509.Certificate) (bool, error) {
pubKey := parentCert.PublicKey.(*ecdsa.PublicKey)
signature := new(ecdsaSignature)
if _, err := asn1.Unmarshal(childCert.Signature, signature); err != nil {
return false, ErrVerifySignature.New("unable to unmarshal ecdsa signature", err)
}
h := crypto.SHA256.New()
_, err := h.Write(childCert.RawTBSCertificate)
if err != nil {
return false, err
}
digest := h.Sum(nil)
isValid := ecdsa.Verify(pubKey, digest, signature.R, signature.S)
return isValid, nil
}
func newSerialNumber() (*big.Int, error) {
serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
if err != nil {
return nil, errs.New("failed to generateServerTls serial number: %s", err.Error())
}
return serialNumber, nil
}

4
pkg/pointerdb/client.go
View File

@ -21,7 +21,7 @@ var (
// PointerDB creates a grpcClient
type PointerDB struct {
grpcClient pb.PointerDBClient
APIKey []byte
APIKey []byte
}
// a compiler trick to make sure *Overlay implements Client
@ -52,7 +52,7 @@ func NewClient(address string, APIKey []byte) (*PointerDB, error) {
}
return &PointerDB{
grpcClient: c,
APIKey: APIKey,
APIKey: APIKey,
}, nil
}

3
pkg/pointerdb/mocks/mock_client.go
View File

@ -6,8 +6,9 @@ package mock_pointerdb
import (
context "context"
gomock "github.com/golang/mock/gomock"
reflect "reflect"
gomock "github.com/golang/mock/gomock"
paths "storj.io/storj/pkg/paths"
pointerdb "storj.io/storj/pkg/pointerdb"
pointerdb0 "storj.io/storj/protos/pointerdb"

2
pkg/pointerdb/pointerdb.go
View File

@ -293,4 +293,4 @@ func (s *Server) Delete(ctx context.Context, req *pb.DeleteRequest) (resp *pb.De
}
s.logger.Debug("deleted pointer at path: " + string(req.GetPath()))
return &pb.DeleteResponse{}, nil
}
}

50
pkg/provider/cert_authority_test.go Normal file
View File

@ -0,0 +1,50 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package provider
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
)
func TestGenerateCA(t *testing.T) {
expectedDifficulty := uint16(4)
ca, err := GenerateCA(context.Background(), expectedDifficulty, 5)
assert.NoError(t, err)
assert.NotEmpty(t, ca)
actualDifficulty := ca.ID.Difficulty()
assert.True(t, actualDifficulty >= expectedDifficulty)
}
func BenchmarkGenerateCA_Difficulty8_Concurrency1(b *testing.B) {
for i := 0; i < b.N; i++ {
expectedDifficulty := uint16(8)
GenerateCA(nil, expectedDifficulty, 1)
}
}
func BenchmarkGenerateCA_Difficulty8_Concurrency2(b *testing.B) {
for i := 0; i < b.N; i++ {
expectedDifficulty := uint16(8)
GenerateCA(nil, expectedDifficulty, 2)
}
}
func BenchmarkGenerateCA_Difficulty8_Concurrency5(b *testing.B) {
for i := 0; i < b.N; i++ {
expectedDifficulty := uint16(8)
GenerateCA(nil, expectedDifficulty, 5)
}
}
func BenchmarkGenerateCA_Difficulty8_Concurrency10(b *testing.B) {
for i := 0; i < b.N; i++ {
expectedDifficulty := uint16(8)
GenerateCA(nil, expectedDifficulty, 10)
}
}

181
pkg/provider/certificate_authority.go Normal file
View File

@ -0,0 +1,181 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package provider
import (
"context"
"crypto"
"crypto/x509"
"encoding/pem"
"io/ioutil"
"os"
"github.com/zeebo/errs"
"storj.io/storj/pkg/peertls"
"storj.io/storj/pkg/utils"
)
// PeerCertificateAuthority represents the CA which is used to validate peer identities
type PeerCertificateAuthority struct {
// Cert is the x509 certificate of the CA
Cert *x509.Certificate
// The ID is calculated from the CA public key.
ID nodeID
}
// FullCertificateAuthority represents the CA which is used to author and validate full identities
type FullCertificateAuthority struct {
// Cert is the x509 certificate of the CA
Cert *x509.Certificate
// The ID is calculated from the CA public key.
ID nodeID
// Key is the private key of the CA
Key crypto.PrivateKey
}
type CASetupConfig struct {
CertPath string `help:"path to the certificate chain for this identity" default:"$CONFDIR/ca.cert"`
KeyPath string `help:"path to the private key for this identity" default:"$CONFDIR/ca.key"`
Difficulty uint64 `help:"minimum difficulty for identity generation" default:"24"`
Timeout string `help:"timeout for CA generation; golang duration string (0 no timeout)" default:"5m"`
Overwrite bool `help:"if true, existing CA certs AND keys will overwritten" default:"false"`
Concurrency uint `help:"number of concurrent workers for certificate authority generation" default:"4"`
}
type CAConfig struct {
CertPath string `help:"path to the certificate chain for this identity" default:"$CONFDIR/ca.cert"`
KeyPath string `help:"path to the private key for this identity" default:"$CONFDIR/ca.key"`
}
// Stat returns the status of the CA cert/key files for the config
func (caS CASetupConfig) Stat() TlsFilesStat {
return statTLSFiles(caS.CertPath, caS.KeyPath)
}
// Create generates and saves a CA using the config
func (caS CASetupConfig) Create(ctx context.Context, concurrency uint) (*FullCertificateAuthority, error) {
ca, err := GenerateCA(ctx, uint16(caS.Difficulty), concurrency)
if err != nil {
return nil, err
}
caC := CAConfig{
CertPath: caS.CertPath,
KeyPath: caS.KeyPath,
}
return ca, caC.Save(ca)
}
// Load loads a CA from the given configuration
func (caC CAConfig) Load() (*FullCertificateAuthority, error) {
cd, err := ioutil.ReadFile(caC.CertPath)
if err != nil {
return nil, peertls.ErrNotExist.Wrap(err)
}
kb, err := ioutil.ReadFile(caC.KeyPath)
if err != nil {
return nil, peertls.ErrNotExist.Wrap(err)
}
var cb [][]byte
for {
var cp *pem.Block
cp, cd = pem.Decode(cd)
if cp == nil {
break
}
cb = append(cb, cp.Bytes)
}
c, err := ParseCertChain(cb)
if err != nil {
return nil, errs.New("failed to load identity %#v, %#v: %v",
caC.CertPath, caC.KeyPath, err)
}
kp, _ := pem.Decode(kb)
k, err := x509.ParseECPrivateKey(kp.Bytes)
if err != nil {
return nil, errs.New("unable to parse EC private key", err)
}
i, err := idFromKey(k)
if err != nil {
return nil, err
}
return &FullCertificateAuthority{
Cert: c[0],
Key: k,
ID: i,
}, nil
}
// GenerateCA creates a new full identity with the given difficulty
func GenerateCA(ctx context.Context, difficulty uint16, concurrency uint) (*FullCertificateAuthority, error) {
if concurrency < 1 {
concurrency = 1
}
ctx, cancel := context.WithCancel(ctx)
eC := make(chan error)
caC := make(chan FullCertificateAuthority, 1)
for i := 0; i < int(concurrency); i++ {
go generateCAWorker(ctx, difficulty, caC, eC)
}
select {
case ca := <-caC:
cancel()
return &ca, nil
case err := <-eC:
cancel()
return nil, err
}
}
// Save saves a CA with the given configuration
func (caC CAConfig) Save(ca *FullCertificateAuthority) error {
f := os.O_WRONLY | os.O_CREATE
c, err := openCert(caC.CertPath, f)
if err != nil {
return err
}
defer utils.LogClose(c)
k, err := openKey(caC.KeyPath, f)
if err != nil {
return err
}
defer utils.LogClose(k)
if err = peertls.WriteChain(c, ca.Cert); err != nil {
return err
}
if err = peertls.WriteKey(k, ca.Key); err != nil {
return err
}
return nil
}
// Generate Identity generates a new `FullIdentity` based on the CA. The CA
// cert is included in the identity's cert chain and the identity's leaf cert
// is signed by the CA.
func (ca FullCertificateAuthority) GenerateIdentity() (*FullIdentity, error) {
lT, err := peertls.LeafTemplate()
if err != nil {
return nil, err
}
l, err := peertls.NewCert(lT, ca.Cert, ca.Key)
if err != nil {
return nil, err
}
k, err := peertls.NewKey()
if err != nil {
return nil, err
}
return &FullIdentity{
CA: ca.Cert,
Leaf: l,
Key: k,
ID: ca.ID,
}, nil
}

264
pkg/provider/identity.go
View File

@ -6,55 +6,198 @@ package provider
import (
"context"
"crypto"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"io/ioutil"
"net"
"os"
base58 "github.com/jbenet/go-base58"
"github.com/zeebo/errs"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"encoding/base64"
"fmt"
"math/bits"
"storj.io/storj/pkg/dht"
"storj.io/storj/pkg/peertls"
"storj.io/storj/pkg/utils"
)
const (
IdentityLength = uint16(256)
)
var (
ErrDifficulty = errs.Class("difficulty error")
)
// PeerIdentity represents another peer on the network.
type PeerIdentity struct {
// CA represents the peer's self-signed CA
CA *x509.Certificate
// Leaf represents the leaf they're currently using. The leaf should be
// signed by the CA. The leaf is what is used for communication.
Leaf *x509.Certificate
// The ID taken from the CA public key
ID nodeID
}
// FullIdentity represents you on the network. In addition to a PeerIdentity,
// a FullIdentity also has a Key, which a PeerIdentity doesn't have.
type FullIdentity struct {
// CA represents the peer's self-signed CA. The ID is taken from this cert.
CA *x509.Certificate
// Leaf represents the leaf they're currently using. The leaf should be
// signed by the CA. The leaf is what is used for communication.
Leaf *x509.Certificate
// The ID is calculated from the CA cert.
ID dht.NodeID
// The ID taken from the CA public key
ID nodeID
// Key is the key this identity uses with the leaf for communication.
Key crypto.PrivateKey
}
// FullIdentity represents you on the network. In addition to a PeerIdentity,
// a FullIdentity also has a PrivateKey, which a PeerIdentity doesn't have.
// The PrivateKey should be for the PeerIdentity's Leaf certificate.
type FullIdentity struct {
PeerIdentity
PrivateKey crypto.PrivateKey
todoCert *tls.Certificate // TODO(jt): get rid of this and only use the above
// IdentityConfig allows you to run a set of Responsibilities with the given
// identity. You can also just load an Identity from disk.
type IdentitySetupConfig struct {
CertPath string `help:"path to the certificate chain for this identity" default:"$CONFDIR/identity.cert"`
KeyPath string `help:"path to the private key for this identity" default:"$CONFDIR/identity.key"`
Overwrite bool `help:"if true, existing identity certs AND keys will overwritten for" default:"false"`
Version string `help:"semantic version of identity storage format" default:"0"`
}
// IdentityConfig allows you to run a set of Responsibilities with the given
// identity. You can also just load an Identity from disk.
type IdentityConfig struct {
CertPath string `help:"path to the certificate chain for this identity" default:"$CONFDIR/identity.leaf.cert"`
KeyPath string `help:"path to the private key for this identity" default:"$CONFDIR/identity.leaf.key"`
CertPath string `help:"path to the certificate chain for this identity" default:"$CONFDIR/identity.cert"`
KeyPath string `help:"path to the private key for this identity" default:"$CONFDIR/identity.key"`
Address string `help:"address to listen on" default:":7777"`
}
// LoadIdentity loads a FullIdentity from the given configuration
func (ic IdentityConfig) LoadIdentity() (*FullIdentity, error) {
pi, err := FullIdentityFromFiles(ic.CertPath, ic.KeyPath)
// FullIdentityFromPEM loads a FullIdentity from a certificate chain and
// private key file
func FullIdentityFromPEM(chainPEM, keyPEM []byte) (*FullIdentity, error) {
cb, err := decodePEM(chainPEM)
if err != nil {
return nil, Error.New("failed to load identity %#v, %#v: %v",
return nil, errs.Wrap(err)
}
if len(cb) < 2 {
return nil, errs.New("too few certificates in chain")
}
kb, err := decodePEM(keyPEM)
if err != nil {
return nil, errs.Wrap(err)
}
// NB: there shouldn't be multiple keys in the key file but if there
// are, this uses the first one
k, err := x509.ParseECPrivateKey(kb[0])
if err != nil {
return nil, errs.New("unable to parse EC private key", err)
}
ch, err := ParseCertChain(cb)
if err != nil {
return nil, errs.Wrap(err)
}
i, err := idFromKey(ch[1].PublicKey)
if err != nil {
return nil, err
}
return &FullIdentity{
CA: ch[1],
Leaf: ch[0],
Key: k,
ID: i,
}, nil
}
// ParseCertChain converts a chain of certificate bytes into x509 certs
func ParseCertChain(chain [][]byte) ([]*x509.Certificate, error) {
c := make([]*x509.Certificate, len(chain))
for i, ct := range chain {
cp, err := x509.ParseCertificate(ct)
if err != nil {
return nil, errs.Wrap(err)
}
c[i] = cp
}
return c, nil
}
// PeerIdentityFromCerts loads a PeerIdentity from a pair of leaf and ca x509 certificates
func PeerIdentityFromCerts(leaf, ca *x509.Certificate) (*PeerIdentity, error) {
i, err := idFromKey(ca.PublicKey.(crypto.PublicKey))
if err != nil {
return nil, err
}
return &PeerIdentity{
CA: ca,
ID: i,
Leaf: leaf,
}, nil
}
// Stat returns the status of the identity cert/key files for the config
func (is IdentitySetupConfig) Stat() TlsFilesStat {
return statTLSFiles(is.CertPath, is.KeyPath)
}
// Create generates and saves a CA using the config
func (is IdentitySetupConfig) Create(ca *FullCertificateAuthority) (*FullIdentity, error) {
fi, err := ca.GenerateIdentity()
if err != nil {
return nil, err
}
fi.CA = ca.Cert
ic := IdentityConfig{
CertPath: is.CertPath,
KeyPath: is.KeyPath,
}
return fi, ic.Save(fi)
}
// Load loads a FullIdentity from the config
func (ic IdentityConfig) Load() (*FullIdentity, error) {
c, err := ioutil.ReadFile(ic.CertPath)
if err != nil {
return nil, peertls.ErrNotExist.Wrap(err)
}
k, err := ioutil.ReadFile(ic.KeyPath)
if err != nil {
return nil, peertls.ErrNotExist.Wrap(err)
}
fi, err := FullIdentityFromPEM(c, k)
if err != nil {
return nil, errs.New("failed to load identity %#v, %#v: %v",
ic.CertPath, ic.KeyPath, err)
}
return pi, nil
return fi, nil
}
// Save saves a FullIdentity according to the config
func (ic IdentityConfig) Save(fi *FullIdentity) error {
f := os.O_WRONLY | os.O_CREATE
c, err := openCert(ic.CertPath, f)
if err != nil {
return err
}
defer utils.LogClose(c)
k, err := openKey(ic.KeyPath, f)
if err != nil {
return err
}
defer utils.LogClose(k)
if err = peertls.WriteChain(c, fi.Leaf, fi.CA); err != nil {
return err
}
if err = peertls.WriteKey(k, fi.Key); err != nil {
return err
}
return nil
}
// Run will run the given responsibilities with the configured identity.
@ -63,7 +206,7 @@ func (ic IdentityConfig) Run(ctx context.Context,
err error) {
defer mon.Task()(&ctx)(&err)
pi, err := ic.LoadIdentity()
pi, err := ic.Load()
if err != nil {
return err
}
@ -85,39 +228,72 @@ func (ic IdentityConfig) Run(ctx context.Context,
return s.Run(ctx)
}
// PeerIdentityFromCertChain loads a PeerIdentity from a chain of certificates
func PeerIdentityFromCertChain(chain [][]byte) (*PeerIdentity, error) {
// TODO(jt): yeah, this totally does not do the right thing yet
// TODO(jt): fill this in correctly.
hash := sha256.Sum256(chain[0]) // TODO(jt): this is wrong
return &PeerIdentity{
CA: nil, // TODO(jt)
Leaf: nil, // TODO(jt)
ID: nodeID(base58.Encode(hash[:])), // TODO(jt): this is wrong
}, nil
// ServerOption returns a grpc `ServerOption` for incoming connections
// to the node with this full identity
func (fi *FullIdentity) ServerOption() (grpc.ServerOption, error) {
ch := [][]byte{fi.Leaf.Raw, fi.CA.Raw}
c, err := peertls.TLSCert(ch, fi.Leaf, fi.Key)
if err != nil {
return nil, err
}
tlsConfig := &tls.Config{
Certificates: []tls.Certificate{*c},
InsecureSkipVerify: true,
ClientAuth: tls.RequireAnyClientCert,
VerifyPeerCertificate: peertls.VerifyPeerFunc(
peertls.VerifyPeerCertChains,
),
}
return grpc.Creds(credentials.NewTLS(tlsConfig)), nil
}
// FullIdentityFromFiles loads a FullIdentity from a certificate chain and
// private key file
func FullIdentityFromFiles(certPath, keyPath string) (*FullIdentity, error) {
cert, err := peertls.LoadCert(certPath, keyPath)
// DialOption returns a grpc `DialOption` for making outgoing connections
// to the node with this peer identity
func (pi *PeerIdentity) DialOption(difficulty uint16) (grpc.DialOption, error) {
ch := [][]byte{pi.Leaf.Raw, pi.CA.Raw}
c, err := peertls.TLSCert(ch, pi.Leaf, nil)
if err != nil {
return nil, Error.Wrap(err)
return nil, err
}
peer, err := PeerIdentityFromCertChain(cert.Certificate)
if err != nil {
return nil, Error.Wrap(err)
tlsConfig := &tls.Config{
Certificates: []tls.Certificate{*c},
InsecureSkipVerify: true,
VerifyPeerCertificate: peertls.VerifyPeerFunc(
peertls.VerifyPeerCertChains,
),
}
return &FullIdentity{
PeerIdentity: *peer,
PrivateKey: cert.PrivateKey,
todoCert: cert,
}, nil
return grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), nil
}
type nodeID string
func (n nodeID) String() string { return string(n) }
func (n nodeID) Bytes() []byte { return []byte(n) }
func (n nodeID) Difficulty() uint16 {
hash, err := base64.URLEncoding.DecodeString(n.String())
if err != nil {
zap.S().Error(errs.Wrap(err))
}
for i := 1; i < len(hash); i++ {
b := hash[len(hash)-i]
if b != 0 {
zeroBits := bits.TrailingZeros16(uint16(b))
if zeroBits == 16 {
zeroBits = 0
}
return uint16((i-1)*8 + zeroBits)
}
}
// NB: this should never happen
reason := fmt.Sprintf("difficulty matches hash length! hash: %s", hash)
zap.S().Error(reason)
panic(reason)
}

206
pkg/provider/identity_test.go Normal file
View File

@ -0,0 +1,206 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package provider
import (
"bytes"
"crypto/ecdsa"
"crypto/x509"
"encoding/pem"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"storj.io/storj/pkg/peertls"
)
func TestPeerIdentityFromCertChain(t *testing.T) {
k, err := peertls.NewKey()
assert.NoError(t, err)
caT, err := peertls.CATemplate()
assert.NoError(t, err)
c, err := peertls.NewCert(caT, nil, k)
assert.NoError(t, err)
lT, err := peertls.LeafTemplate()
assert.NoError(t, err)
l, err := peertls.NewCert(lT, caT, k)
assert.NoError(t, err)
pi, err := PeerIdentityFromCerts(l, c)
assert.NoError(t, err)
assert.Equal(t, c, pi.CA)
assert.Equal(t, l, pi.Leaf)
assert.NotEmpty(t, pi.ID)
}
func TestFullIdentityFromPEM(t *testing.T) {
ck, err := peertls.NewKey()
assert.NoError(t, err)
caT, err := peertls.CATemplate()
assert.NoError(t, err)
c, err := peertls.NewCert(caT, nil, ck)
assert.NoError(t, err)
assert.NoError(t, err)
assert.NotEmpty(t, c)
lT, err := peertls.LeafTemplate()
assert.NoError(t, err)
l, err := peertls.NewCert(lT, caT, ck)
assert.NoError(t, err)
assert.NotEmpty(t, l)
chainPEM := bytes.NewBuffer([]byte{})
pem.Encode(chainPEM, peertls.NewCertBlock(l.Raw))
pem.Encode(chainPEM, peertls.NewCertBlock(c.Raw))
lk, err := peertls.NewKey()
assert.NoError(t, err)
lkE, ok := lk.(*ecdsa.PrivateKey)
assert.True(t, ok)
assert.NotEmpty(t, lkE)
lkB, err := x509.MarshalECPrivateKey(lkE)
assert.NoError(t, err)
assert.NotEmpty(t, lkB)
keyPEM := bytes.NewBuffer([]byte{})
pem.Encode(keyPEM, peertls.NewKeyBlock(lkB))
fi, err := FullIdentityFromPEM(chainPEM.Bytes(), keyPEM.Bytes())
assert.NoError(t, err)
assert.Equal(t, l.Raw, fi.Leaf.Raw)
assert.Equal(t, c.Raw, fi.CA.Raw)
assert.Equal(t, lk, fi.Key)
}
func TestIdentityConfig_SaveIdentity(t *testing.T) {
done, ic, fi, _ := tempIdentity(t)
defer done()
chainPEM := bytes.NewBuffer([]byte{})
pem.Encode(chainPEM, peertls.NewCertBlock(fi.Leaf.Raw))
pem.Encode(chainPEM, peertls.NewCertBlock(fi.CA.Raw))
privateKey, ok := fi.Key.(*ecdsa.PrivateKey)
assert.True(t, ok)
assert.NotEmpty(t, privateKey)
keyBytes, err := x509.MarshalECPrivateKey(privateKey)
assert.NoError(t, err)
assert.NotEmpty(t, keyBytes)
keyPEM := bytes.NewBuffer([]byte{})
pem.Encode(keyPEM, peertls.NewKeyBlock(keyBytes))
err = ic.Save(fi)
assert.NoError(t, err)
certInfo, err := os.Stat(ic.CertPath)
assert.NoError(t, err)
assert.Equal(t, os.FileMode(0644), certInfo.Mode())
keyInfo, err := os.Stat(ic.KeyPath)
assert.NoError(t, err)
assert.Equal(t, os.FileMode(0600), keyInfo.Mode())
savedChainPEM, err := ioutil.ReadFile(ic.CertPath)
assert.NoError(t, err)
savedKeyPEM, err := ioutil.ReadFile(ic.KeyPath)
assert.NoError(t, err)
assert.Equal(t, chainPEM.Bytes(), savedChainPEM)
assert.Equal(t, keyPEM.Bytes(), savedKeyPEM)
}
func tempIdentityConfig() (*IdentityConfig, func(), error) {
tmpDir, err := ioutil.TempDir("", "tempIdentity")
if err != nil {
return nil, nil, err
}
cleanup := func() { os.RemoveAll(tmpDir) }
return &IdentityConfig{
CertPath: filepath.Join(tmpDir, "chain.pem"),
KeyPath: filepath.Join(tmpDir, "key.pem"),
}, cleanup, nil
}
func tempIdentity(t *testing.T) (func(), *IdentityConfig, *FullIdentity, uint16) {
// NB: known difficulty
difficulty := uint16(12)
chain := `-----BEGIN CERTIFICATE-----
MIIBQTCB6KADAgECAhEA7iLmNy8uop2bC4Yv1uXvwjAKBggqhkjOPQQDAjAAMCIY
DzAwMDEwMTAxMDAwMDAwWhgPMDAwMTAxMDEwMDAwMDBaMAAwWTATBgcqhkjOPQIB
BggqhkjOPQMBBwNCAATD84AzWKMs7rSuQ0pGbtQE5X6EvKe74ORUgayxLimvs0dX
1KOLg5XmbUF4bwHPvkbDLUlSCWx5qgFmL+XhuR5doz8wPTAOBgNVHQ8BAf8EBAMC
BaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAw
CgYIKoZIzj0EAwIDSAAwRQIgQkJgjRar0nIOQbEAin5bQe4+9BUjSIQzrlkJgXsC
liICIQDz6LeN9nRKCuRcqiK8tnaKbOJ+/Q3PQNHuK7coFFuB1g==
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIBOjCB4aADAgECAhEA4A+Fdf1cyylCp0GCWMtpJDAKBggqhkjOPQQDAjAAMCIY
DzAwMDEwMTAxMDAwMDAwWhgPMDAwMTAxMDEwMDAwMDBaMAAwWTATBgcqhkjOPQIB
BggqhkjOPQMBBwNCAAQz10hua+xRFmIRKJLMZh9os3PM3mWtElD3WyoR2U6m6U1B
zRJ7cXS0CaPsbilglXjnWHOSV6QKmgcHYTroWkgvozgwNjAOBgNVHQ8BAf8EBAMC
AgQwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjO
PQQDAgNIADBFAiEAnvRK+MtT7hWt9CeQvKID40CcPJDhYIEQjN91W1sseNICICgL
y9HDctQtMjRMG3UHifkDl7kPINkiP7w068I5RWvx
-----END CERTIFICATE-----`
key := `-----BEGIN EC PRIVATE KEY-----
MHcCAQEEILQar8Z01NkX/czx8yGevdBATINSW1+U6AQS0Sl5WbdVoAoGCCqGSM49
AwEHoUQDQgAEw/OAM1ijLO60rkNKRm7UBOV+hLynu+DkVIGssS4pr7NHV9Sji4OV
5m1BeG8Bz75Gwy1JUglseaoBZi/l4bkeXQ==
-----END EC PRIVATE KEY-----`
ic, cleanup, err := tempIdentityConfig()
fi, err := FullIdentityFromPEM([]byte(chain), []byte(key))
assert.NoError(t, err)
return cleanup, ic, fi, difficulty
}
func TestIdentityConfig_LoadIdentity(t *testing.T) {
done, ic, expectedFI, _ := tempIdentity(t)
defer done()
err := ic.Save(expectedFI)
assert.NoError(t, err)
fi, err := ic.Load()
assert.NoError(t, err)
assert.NotEmpty(t, fi)
assert.NotEmpty(t, fi.Key)
assert.NotEmpty(t, fi.Leaf)
assert.NotEmpty(t, fi.CA)
assert.NotEmpty(t, fi.ID.Bytes())
assert.Equal(t, expectedFI.Key, fi.Key)
assert.Equal(t, expectedFI.Leaf, fi.Leaf)
assert.Equal(t, expectedFI.CA, fi.CA)
assert.Equal(t, expectedFI.ID.Bytes(), fi.ID.Bytes())
}
func TestNodeID_Difficulty(t *testing.T) {
done, _, fi, knownDifficulty := tempIdentity(t)
defer done()
difficulty := fi.ID.Difficulty()
assert.True(t, difficulty >= knownDifficulty)
}

53
pkg/provider/provider.go
View File

@ -5,13 +5,16 @@ package provider
import (
"context"
"crypto/tls"
"net"
"time"
"github.com/zeebo/errs"
"go.uber.org/zap"
"google.golang.org/grpc"
)
"storj.io/storj/pkg/peertls"
var (
ErrSetup = errs.Class("setup error")
)
// Responsibility represents a specific gRPC method collection to be registered
@ -34,18 +37,56 @@ type Provider struct {
// of responsibilities.
func NewProvider(identity *FullIdentity, lis net.Listener,
responsibilities ...Responsibility) (*Provider, error) {
// NB: talk to anyone with an identity
s, err := identity.ServerOption()
if err != nil {
return nil, err
}
return &Provider{
lis: lis,
g: grpc.NewServer(
grpc.StreamInterceptor(streamInterceptor),
grpc.UnaryInterceptor(unaryInterceptor),
s,
),
next: responsibilities,
identity: identity,
}, nil
}
// SetupIdentity ensures a CA and identity exist and returns a config overrides map
func SetupIdentity(ctx context.Context, c CASetupConfig, i IdentitySetupConfig) error {
if s := c.Stat(); s == NoCertNoKey || c.Overwrite {
t, err := time.ParseDuration(c.Timeout)
if err != nil {
return errs.Wrap(err)
}
ctx, _ = context.WithTimeout(ctx, t)
// Load or create a certificate authority
ca, err := c.Create(ctx, 4)
if err != nil {
return err
}
if s := i.Stat(); s == NoCertNoKey || i.Overwrite {
// Create identity from new CA
_, err = i.Create(ca)
if err != nil {
return err
}
return nil
} else {
return ErrSetup.New("identity file(s) exist: %s", s)
}
} else {
return ErrSetup.New("certificate authority file(s) exist: %s", s)
}
}
// Identity returns the provider's identity
func (p *Provider) Identity() *FullIdentity { return p.identity }
@ -73,14 +114,6 @@ func (p *Provider) Run(ctx context.Context) (err error) {
return p.g.Serve(p.lis)
}
// TLSConfig returns the provider's identity as a TLS Config
func (p *Provider) TLSConfig() *tls.Config {
// TODO(jt): get rid of tls.Certificate
return (&peertls.TLSFileOptions{
LeafCertificate: p.identity.todoCert,
}).NewTLSConfig(nil)
}
func streamInterceptor(srv interface{}, ss grpc.ServerStream,
info *grpc.StreamServerInfo, handler grpc.StreamHandler) (err error) {
err = handler(srv, ss)

165
pkg/provider/utils.go Normal file
View File

@ -0,0 +1,165 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package provider
import (
"context"
"crypto"
"crypto/ecdsa"
"crypto/x509"
"encoding/base64"
"encoding/pem"
"os"
"path/filepath"
"github.com/zeebo/errs"
"golang.org/x/crypto/sha3"
"storj.io/storj/pkg/peertls"
)
type TlsFilesStat int
const (
NoCertNoKey = iota
CertNoKey
NoCertKey
CertKey
)
var (
ErrZeroBytes = errs.New("byte slice was unexpectedly empty")
)
func decodePEM(PEMBytes []byte) ([][]byte, error) {
DERBytes := [][]byte{}
for {
var DERBlock *pem.Block
DERBlock, PEMBytes = pem.Decode(PEMBytes)
if DERBlock == nil {
break
}
DERBytes = append(DERBytes, DERBlock.Bytes)
}
if len(DERBytes) == 0 || len(DERBytes[0]) == 0 {
return nil, ErrZeroBytes
}
return DERBytes, nil
}
func generateCAWorker(ctx context.Context, difficulty uint16, caC chan FullCertificateAuthority, eC chan error) {
var (
k crypto.PrivateKey
i nodeID
err error
)
for {
select {
case <-ctx.Done():
return
default:
k, err = peertls.NewKey()
switch kE := k.(type) {
case *ecdsa.PrivateKey:
i, err = idFromKey(&kE.PublicKey)
if err != nil {
eC <- err
return
}
default:
eC <- peertls.ErrUnsupportedKey.New("%T", k)
return
}
}
if i.Difficulty() >= difficulty {
break
}
}
ct, err := peertls.CATemplate()
if err != nil {
eC <- err
return
}
c, err := peertls.NewCert(ct, nil, k)
if err != nil {
eC <- err
return
}
ca := FullCertificateAuthority{
Cert: c,
Key: k,
ID: i,
}
caC <- ca
return
}
func idFromKey(k crypto.PublicKey) (nodeID, error) {
kb, err := x509.MarshalPKIXPublicKey(k)
if err != nil {
return "", errs.Wrap(err)
}
hash := make([]byte, IdentityLength)
sha3.ShakeSum256(hash, kb)
return nodeID(base64.URLEncoding.EncodeToString(hash)), nil
}
func openCert(path string, flag int) (*os.File, error) {
if err := os.MkdirAll(filepath.Dir(path), 744); err != nil {
return nil, errs.Wrap(err)
}
c, err := os.OpenFile(path, flag, 0644)
if err != nil {
return nil, errs.New("unable to open cert file for writing \"%s\"", path, err)
}
return c, nil
}
func openKey(path string, flag int) (*os.File, error) {
if err := os.MkdirAll(filepath.Dir(path), 700); err != nil {
return nil, errs.Wrap(err)
}
k, err := os.OpenFile(path, flag, 0600)
if err != nil {
return nil, errs.New("unable to open key file for writing \"%s\"", path, err)
}
return k, nil
}
func statTLSFiles(certPath, keyPath string) TlsFilesStat {
s := 0
_, err := os.Stat(certPath)
if err == nil {
s += 1
}
_, err = os.Stat(keyPath)
if err == nil {
s += 2
}
return TlsFilesStat(s)
}
func (t TlsFilesStat) String() string {
switch t {
case CertKey:
return "certificate and key"
case CertNoKey:
return "certificate"
case NoCertKey:
return "key"
default:
return ""
}
}

4
pkg/ranger/content.go
View File

@ -126,7 +126,7 @@ func ServeContent(ctx context.Context, w http.ResponseWriter, r *http.Request,
pr, pw := io.Pipe()
mw := multipart.NewWriter(pw)
w.Header().Set("Content-Type",
"multipart/byteranges; boundary=" + mw.Boundary())
"multipart/byteranges; boundary="+mw.Boundary())
sendContent = func() (io.ReadCloser, error) { return ioutil.NopCloser(pr), nil }
// cause writing goroutine to fail and exit if CopyN doesn't finish.
defer func() {
@ -259,7 +259,7 @@ func checkPreconditions(w http.ResponseWriter, r *http.Request,
type condResult int
const (
condNone condResult = iota
condNone condResult = iota
condTrue
condFalse
)

5
pkg/storage/ec/mocks/mock_client.go
View File

@ -6,14 +6,15 @@ package mock_ecclient
import (
context "context"
gomock "github.com/golang/mock/gomock"
io "io"
reflect "reflect"
time "time"
gomock "github.com/golang/mock/gomock"
eestream "storj.io/storj/pkg/eestream"
client "storj.io/storj/pkg/piecestore/rpc/client"
ranger "storj.io/storj/pkg/ranger"
overlay "storj.io/storj/protos/overlay"
time "time"
)
// MockClient is a mock of Client interface

20
pkg/utils/io.go
View File

@ -3,6 +3,13 @@
package utils
import (
"io"
"os"
"go.uber.org/zap"
)
// ReaderSource takes a src func and turns it into an io.Reader
type ReaderSource struct {
src func() ([]byte, error)
@ -28,3 +35,16 @@ func (rs *ReaderSource) Read(p []byte) (n int, err error) {
rs.buf = rs.buf[n:]
return n, rs.err
}
// LogClose closes an io.Closer, logging the error if there is one that isn't
// os.ErrClosed
func LogClose(fh io.Closer) {
err := fh.Close()
if err == nil || err == os.ErrClosed {
return
}
if perr, ok := err.(*os.PathError); ok && perr.Err == os.ErrClosed {
return
}
zap.S().Errorf("Failed to close file: %s", err)
}