JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

lib/uplink: remove redis and bolt dependencies (#2812)

Browse Source 
* identity: remove redis and bolt dependencies

* identity: move revDB creation to main files
This commit is contained in:
Isaac Hess 2019-08-19 16:10:38 -06:00 committed by GitHub
parent 8832a393e1
commit 25154720bd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 205 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bootstrap/peer.go
View File

@ -18,6 +18,7 @@ import (
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/server"
"storj.io/storj/pkg/storj"
@ -84,7 +85,7 @@ type Peer struct {
}
// New creates a new Bootstrap Node.
func New(log *zap.Logger, full *identity.FullIdentity, db DB, config Config, versionInfo version.Info) (*Peer, error) {
func New(log *zap.Logger, full *identity.FullIdentity, db DB, revDB extensions.RevocationDB, config Config, versionInfo version.Info) (*Peer, error) {
peer := &Peer{
Log: log,
Identity: full,
@ -104,7 +105,8 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB, config Config, ver
{ // setup listener and server
sc := config.Server
options, err := tlsopts.NewOptions(peer.Identity, sc.Config)
options, err := tlsopts.NewOptions(peer.Identity, sc.Config, revDB)
if err != nil {
return nil, errs.Combine(err, peer.Close())
}

8
cmd/bootstrap/main.go
View File

@ -18,6 +18,7 @@ import (
"storj.io/storj/internal/version"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/revocation"
)
var (
@ -87,7 +88,12 @@ func cmdRun(cmd *cobra.Command, args []string) (err error) {
err = errs.Combine(err, db.Close())
}()
peer, err := bootstrap.New(log, identity, db, runCfg, version.Build)
revDB, err := revocation.NewDBFromCfg(runCfg.Server.Config)
if err != nil {
return errs.New("Error creating revocation database: %+v", err)
}
peer, err := bootstrap.New(log, identity, db, revDB, runCfg, version.Build)
if err != nil {
return err
}

9
cmd/certificates/main.go
View File

@ -5,6 +5,7 @@ package main
import (
"github.com/spf13/cobra"
"github.com/zeebo/errs"
"go.uber.org/zap"
"storj.io/storj/internal/fpath"
@ -12,6 +13,7 @@ import (
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/revocation"
"storj.io/storj/pkg/server"
)
@ -60,7 +62,12 @@ func cmdRun(cmd *cobra.Command, args []string) error {
zap.S().Fatal(err)
}
return config.Server.Run(ctx, zap.L(), identity, nil, config.Signer)
revDB, err := revocation.NewDBFromCfg(config.Server.Config.Config)
if err != nil {
return errs.New("Error creating revocation database: %+v", err)
}
return config.Server.Run(ctx, zap.L(), identity, revDB, nil, config.Signer)
}
func main() {

3
cmd/identity/certificate_authority.go
View File

@ -15,6 +15,7 @@ import (
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/revocation"
)
var (
@ -196,7 +197,7 @@ func cmdRevokePeerCA(cmd *cobra.Command, args []string) (err error) {
return err
}
revDB, err := identity.NewRevocationDB(revokePeerCACfg.RevocationDBURL)
revDB, err := revocation.NewDB(revokePeerCACfg.RevocationDBURL)
if err != nil {
return err
}

8
cmd/identity/main.go
View File

@ -22,6 +22,7 @@ import (
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/pkcrypto"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/revocation"
)
const (
@ -185,7 +186,12 @@ func cmdAuthorize(cmd *cobra.Command, args []string) error {
// Ensure we dont enforce a signed Peer Identity
config.Signer.TLS.UsePeerCAWhitelist = false
signedChainBytes, err := config.Signer.Sign(ctx, ident, authToken)
revDB, err := revocation.NewDBFromCfg(config.Signer.TLS)
if err != nil {
return errs.New("Error creating revocation database: %+v", err)
}
signedChainBytes, err := config.Signer.Sign(ctx, ident, authToken, revDB)
if err != nil {
return errs.New("error occurred while signing certificate: %s\n(identity files were still generated and saved, if you try again existing files will be loaded)", err)
}

4
cmd/identity/revocations.go
View File

@ -13,8 +13,8 @@ import (
"github.com/zeebo/errs"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/revocation"
)
var (
@ -43,7 +43,7 @@ func cmdRevocations(cmd *cobra.Command, args []string) error {
revCfg.RevocationDBURL = "bolt://" + filepath.Join(configDir, args[0], "revocations.db")
}
revDB, err := identity.NewRevocationDB(revCfg.RevocationDBURL)
revDB, err := revocation.NewDB(revCfg.RevocationDBURL)
if err != nil {
return err
}

8
cmd/satellite/main.go
View File

@ -21,6 +21,7 @@ import (
"storj.io/storj/internal/version"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/revocation"
"storj.io/storj/satellite"
"storj.io/storj/satellite/satellitedb"
)
@ -130,7 +131,12 @@ func cmdRun(cmd *cobra.Command, args []string) (err error) {
err = errs.Combine(err, db.Close())
}()
peer, err := satellite.New(log, identity, db, &runCfg.Config, version.Build)
revDB, err := revocation.NewDBFromCfg(runCfg.Config.Server.Config)
if err != nil {
return errs.New("Error creating revocation database: %+v", err)
}
peer, err := satellite.New(log, identity, db, revDB, &runCfg.Config, version.Build)
if err != nil {
return err
}

8
cmd/storagenode/main.go
View File

@ -20,6 +20,7 @@ import (
"storj.io/storj/internal/version"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/revocation"
"storj.io/storj/pkg/storj"
"storj.io/storj/storagenode"
"storj.io/storj/storagenode/storagenodedb"
@ -139,7 +140,12 @@ func cmdRun(cmd *cobra.Command, args []string) (err error) {
err = errs.Combine(err, db.Close())
}()
peer, err := storagenode.New(log, identity, db, runCfg.Config, version.Build)
revDB, err := revocation.NewDBFromCfg(runCfg.Server.Config)
if err != nil {
return errs.New("Error creating revocation database: %+v", err)
}
peer, err := storagenode.New(log, identity, db, revDB, runCfg.Config, version.Build)
if err != nil {
return err
}

2
examples/grpc-debug/main.go
View File

@ -32,7 +32,7 @@ func main() {
if err != nil {
panic(err)
}
clientOptions, err := tlsopts.NewOptions(identity, tlsopts.Config{})
clientOptions, err := tlsopts.NewOptions(identity, tlsopts.Config{}, nil)
if err != nil {
panic(err)
}

10
internal/testplanet/bootstrap.go
View File

@ -10,6 +10,8 @@ import (
"strings"
"time"
"github.com/zeebo/errs"
"storj.io/storj/bootstrap"
"storj.io/storj/bootstrap/bootstrapdb"
"storj.io/storj/bootstrap/bootstrapweb/bootstrapserver"
@ -17,6 +19,7 @@ import (
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/revocation"
"storj.io/storj/pkg/server"
"storj.io/storj/versioncontrol"
)
@ -97,7 +100,12 @@ func (planet *Planet) newBootstrap() (peer *bootstrap.Peer, err error) {
var verInfo version.Info
verInfo = planet.NewVersionInfo()
peer, err = bootstrap.New(log, identity, db, config, verInfo)
revDB, err := revocation.NewDBFromCfg(config.Server.Config)
if err != nil {
return nil, errs.New("Error creating revocation database: %+v", err)
}
peer, err = bootstrap.New(log, identity, db, revDB, config, verInfo)
if err != nil {
return nil, err
}

10
internal/testplanet/satellite.go
View File

@ -10,10 +10,13 @@ import (
"strings"
"time"
"github.com/zeebo/errs"
"storj.io/storj/internal/memory"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/revocation"
"storj.io/storj/pkg/server"
"storj.io/storj/satellite"
"storj.io/storj/satellite/accounting/rollup"
@ -213,7 +216,12 @@ func (planet *Planet) newSatellites(count int) ([]*satellite.Peer, error) {
verInfo := planet.NewVersionInfo()
peer, err := satellite.New(log, identity, db, &config, verInfo)
revDB, err := revocation.NewDBFromCfg(config.Server.Config)
if err != nil {
return xs, errs.New("Error creating revocation database: %+v", err)
}
peer, err := satellite.New(log, identity, db, revDB, &config, verInfo)
if err != nil {
return xs, err
}

10
internal/testplanet/storagenode.go
View File

@ -11,10 +11,13 @@ import (
"strings"
"time"
"github.com/zeebo/errs"
"storj.io/storj/internal/memory"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/revocation"
"storj.io/storj/pkg/server"
"storj.io/storj/pkg/storj"
"storj.io/storj/storagenode"
@ -159,7 +162,12 @@ func (planet *Planet) newStorageNodes(count int, whitelistedSatellites storj.Nod
}
}
peer, err := storagenode.New(log, identity, db, config, verInfo)
revDB, err := revocation.NewDBFromCfg(config.Server.Config)
if err != nil {
return nil, errs.New("Error creating revocation database: %+v", err)
}
peer, err := storagenode.New(log, identity, db, revDB, config, verInfo)
if err != nil {
return xs, err
}

2
internal/testplanet/uplink.go
View File

@ -64,7 +64,7 @@ func (planet *Planet) newUplink(name string, storageNodeCount int) (*Uplink, err
tlsOpts, err := tlsopts.NewOptions(identity, tlsopts.Config{
PeerIDVersions: strconv.Itoa(int(planet.config.IdentityVersion.Number)),
})
}, nil)
if err != nil {
return nil, err
}

8
internal/testplanet/uplink_test.go
View File

@ -22,6 +22,7 @@ import (
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/revocation"
"storj.io/storj/pkg/server"
"storj.io/storj/pkg/storj"
"storj.io/storj/uplink"
@ -204,7 +205,7 @@ func TestDownloadFromUnresponsiveNode(t *testing.T) {
wl, err := planet.WriteWhitelist(storj.LatestIDVersion())
require.NoError(t, err)
options, err := tlsopts.NewOptions(storageNode.Identity, tlsopts.Config{
tlscfg := tlsopts.Config{
RevocationDBURL: "bolt://" + filepath.Join(ctx.Dir("fakestoragenode"), "revocation.db"),
UsePeerCAWhitelist: true,
PeerCAWhitelistPath: wl,
@ -213,7 +214,10 @@ func TestDownloadFromUnresponsiveNode(t *testing.T) {
Revocation: false,
WhitelistSignedLeaf: false,
},
})
}
revDB, err := revocation.NewDBFromCfg(tlscfg)
require.NoError(t, err)
options, err := tlsopts.NewOptions(storageNode.Identity, tlscfg, revDB)
require.NoError(t, err)
server, err := server.New(storageNode.Log.Named("mock-server"), options, storageNode.Addr(), storageNode.PrivateAddr(), nil)

12
internal/testidentity/revocation_db.go → internal/testrevocation/revocation_db.go
View File

@ -1,7 +1,7 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package testidentity
package testrevocation
import (
"testing"
@ -10,8 +10,8 @@ import (
"github.com/stretchr/testify/require"
"storj.io/storj/internal/testcontext"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/revocation"
"storj.io/storj/storage"
)
@ -29,11 +29,11 @@ func RevocationDBsTest(t *testing.T, test func(*testing.T, extensions.Revocation
{
// Test using redis-backed revocation DB
dbURL := "redis://" + redisServer.Addr() + "?db=0"
redisRevDB, err := identity.NewRevocationDB(dbURL)
redisRevDB, err := revocation.NewDB(dbURL)
require.NoError(t, err)
defer ctx.Check(redisRevDB.Close)
test(t, redisRevDB, redisRevDB.DB)
test(t, redisRevDB, redisRevDB.KVStore)
}
})
@ -47,11 +47,11 @@ func RevocationDBsTest(t *testing.T, test func(*testing.T, extensions.Revocation
revocationDBPath := ctx.File("revocations.db")
dbURL := "bolt://" + revocationDBPath
boltRevDB, err := identity.NewRevocationDB(dbURL)
boltRevDB, err := revocation.NewDB(dbURL)
require.NoError(t, err)
defer ctx.Check(boltRevDB.Close)
test(t, boltRevDB, boltRevDB.DB)
test(t, boltRevDB, boltRevDB.KVStore)
}
})
}

2
lib/uplink/uplink.go
View File

@ -137,7 +137,7 @@ func NewUplink(ctx context.Context, cfg *Config) (_ *Uplink, err error) {
PeerCAWhitelistPath: cfg.Volatile.TLS.PeerCAWhitelistPath,
PeerIDVersions: "0",
}
tlsOpts, err := tlsopts.NewOptions(ident, tlsConfig)
tlsOpts, err := tlsopts.NewOptions(ident, tlsConfig, nil)
if err != nil {
return nil, err
}

11
pkg/certificates/certificates_test.go
View File

@ -27,6 +27,7 @@ import (
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/pkcrypto"
"storj.io/storj/pkg/revocation"
"storj.io/storj/pkg/server"
"storj.io/storj/pkg/storj"
"storj.io/storj/pkg/transport"
@ -605,7 +606,11 @@ func TestCertificateSigner_Sign_E2E(t *testing.T) {
Address: "127.0.0.1:0",
PrivateAddress: "127.0.0.1:0",
}
serverOpts, err := tlsopts.NewOptions(serverIdent, sc.Config)
revDB, err := revocation.NewDBFromCfg(sc.Config)
require.NoError(t, err)
serverOpts, err := tlsopts.NewOptions(serverIdent, sc.Config, revDB)
require.NoError(t, err)
require.NotNil(t, serverOpts)
@ -620,7 +625,7 @@ func TestCertificateSigner_Sign_E2E(t *testing.T) {
})
defer ctx.Check(service.Close)
clientOpts, err := tlsopts.NewOptions(clientIdent, tlsopts.Config{PeerIDVersions: "*"})
clientOpts, err := tlsopts.NewOptions(clientIdent, tlsopts.Config{PeerIDVersions: "*"}, nil)
require.NoError(t, err)
clientTransport := transport.NewClient(clientOpts)
@ -704,7 +709,7 @@ func TestNewClient(t *testing.T) {
}
})
tlsOptions, err := tlsopts.NewOptions(ident, tlsopts.Config{})
tlsOptions, err := tlsopts.NewOptions(ident, tlsopts.Config{}, nil)
require.NoError(t, err)
clientTransport := transport.NewClient(tlsOptions)

5
pkg/certificates/config.go
View File

@ -37,9 +37,10 @@ type CertServerConfig struct {
}
// Sign submits a certificate signing request given the config
func (c CertClientConfig) Sign(ctx context.Context, ident *identity.FullIdentity, authToken string) (_ [][]byte, err error) {
func (c CertClientConfig) Sign(ctx context.Context, ident *identity.FullIdentity, authToken string, revDB extensions.RevocationDB) (_ [][]byte, err error) {
defer mon.Task()(&ctx)(&err)
tlsOpts, err := tlsopts.NewOptions(ident, c.TLS)
tlsOpts, err := tlsopts.NewOptions(ident, c.TLS, revDB)
if err != nil {
return nil, err
}

2
pkg/kademlia/kademlia_planet_test.go
View File

@ -81,7 +81,7 @@ func TestPingTimeout(t *testing.T) {
self := planet.StorageNodes[0]
routingTable := self.Kademlia.RoutingTable
tlsOpts, err := tlsopts.NewOptions(self.Identity, tlsopts.Config{})
tlsOpts, err := tlsopts.NewOptions(self.Identity, tlsopts.Config{}, nil)
require.NoError(t, err)
self.Transport = transport.NewClientWithTimeouts(tlsOpts, transport.Timeouts{

8
pkg/kademlia/kademlia_test.go
View File

@ -168,7 +168,7 @@ func testNode(ctx *testcontext.Context, name string, t *testing.T, bn []pb.Node)
serverOptions, err := tlsopts.NewOptions(fid, tlsopts.Config{
PeerIDVersions: "*",
})
}, nil)
require.NoError(t, err)
identOpt := serverOptions.ServerOption()
@ -307,7 +307,7 @@ func startTestNodeServer(ctx *testcontext.Context) (*grpc.Server, *mockNodesServ
return nil, nil, nil, ""
}
serverOptions, err := tlsopts.NewOptions(fullIdentity, tlsopts.Config{})
serverOptions, err := tlsopts.NewOptions(fullIdentity, tlsopts.Config{}, nil)
if err != nil {
return nil, nil, nil, ""
}
@ -337,7 +337,7 @@ func newTestServer(ctx *testcontext.Context) (*grpc.Server, *mockNodesServer) {
if err != nil {
return nil, nil
}
serverOptions, err := tlsopts.NewOptions(fullIdentity, tlsopts.Config{})
serverOptions, err := tlsopts.NewOptions(fullIdentity, tlsopts.Config{}, nil)
if err != nil {
return nil, nil
}
@ -424,7 +424,7 @@ func newKademlia(log *zap.Logger, nodeType pb.NodeType, bootstrapNodes []pb.Node
tlsOptions, err := tlsopts.NewOptions(identity, tlsopts.Config{
PeerIDVersions: "*",
})
}, nil)
if err != nil {
return nil, err
}

6
pkg/kademlia/kademliaclient/kademliaclient_test.go
View File

@ -179,7 +179,7 @@ func TestSlowDialerHasTimeout(t *testing.T) {
func() { // PingNode
self := planet.StorageNodes[0]
tlsOpts, err := tlsopts.NewOptions(self.Identity, tlsopts.Config{})
tlsOpts, err := tlsopts.NewOptions(self.Identity, tlsopts.Config{}, nil)
require.NoError(t, err)
self.Transport = transport.NewClientWithTimeouts(tlsOpts, transport.Timeouts{
@ -215,7 +215,7 @@ func TestSlowDialerHasTimeout(t *testing.T) {
func() { // FetchPeerIdentity
self := planet.StorageNodes[1]
tlsOpts, err := tlsopts.NewOptions(self.Identity, tlsopts.Config{})
tlsOpts, err := tlsopts.NewOptions(self.Identity, tlsopts.Config{}, nil)
require.NoError(t, err)
self.Transport = transport.NewClientWithTimeouts(tlsOpts, transport.Timeouts{
@ -252,7 +252,7 @@ func TestSlowDialerHasTimeout(t *testing.T) {
func() { // Lookup
self := planet.StorageNodes[2]
tlsOpts, err := tlsopts.NewOptions(self.Identity, tlsopts.Config{})
tlsOpts, err := tlsopts.NewOptions(self.Identity, tlsopts.Config{}, nil)
require.NoError(t, err)
self.Transport = transport.NewClientWithTimeouts(tlsOpts, transport.Timeouts{

8
pkg/peertls/extensions/revocations_test.go
View File

@ -12,8 +12,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"storj.io/storj/internal/testidentity"
"storj.io/storj/internal/testpeertls"
"storj.io/storj/internal/testrevocation"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/peertls"
"storj.io/storj/pkg/peertls/extensions"
@ -24,7 +24,7 @@ import (
var ctx = context.Background() // test context
func TestRevocationCheckHandler(t *testing.T) {
testidentity.RevocationDBsTest(t, func(t *testing.T, revDB extensions.RevocationDB, _ storage.KeyValueStore) {
testrevocation.RevocationDBsTest(t, func(t *testing.T, revDB extensions.RevocationDB, _ storage.KeyValueStore) {
keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number)
assert.NoError(t, err)
@ -66,7 +66,7 @@ func TestRevocationCheckHandler(t *testing.T) {
}
})
testidentity.RevocationDBsTest(t, func(t *testing.T, revDB extensions.RevocationDB, _ storage.KeyValueStore) {
testrevocation.RevocationDBsTest(t, func(t *testing.T, revDB extensions.RevocationDB, _ storage.KeyValueStore) {
t.Log("new revocation DB")
keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number)
assert.NoError(t, err)
@ -118,7 +118,7 @@ func TestRevocationCheckHandler(t *testing.T) {
}
func TestRevocationUpdateHandler(t *testing.T) {
testidentity.RevocationDBsTest(t, func(t *testing.T, revDB extensions.RevocationDB, _ storage.KeyValueStore) {
testrevocation.RevocationDBsTest(t, func(t *testing.T, revDB extensions.RevocationDB, _ storage.KeyValueStore) {
keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number)
assert.NoError(t, err)

16
pkg/peertls/tlsopts/options.go
View File

@ -28,7 +28,7 @@ var (
type Options struct {
Config Config
Ident *identity.FullIdentity
RevDB *identity.RevocationDB
RevDB extensions.RevocationDB
PeerCAWhitelist []*x509.Certificate
VerificationFuncs *VerificationFuncs
Cert *tls.Certificate
@ -44,10 +44,13 @@ type VerificationFuncs struct {
// ExtensionMap maps `pkix.Extension`s to their respective asn1 object ID string.
type ExtensionMap map[string]pkix.Extension
// NewOptions is a constructor for `tls options` given an identity and config.
func NewOptions(i *identity.FullIdentity, c Config) (*Options, error) {
// NewOptions is a constructor for `tls options` given an identity, config, and
// revocation DB. A caller may pass a nil revocation DB if the revocation
// extension is disabled.
func NewOptions(i *identity.FullIdentity, c Config, revDB extensions.RevocationDB) (*Options, error) {
opts := &Options{
Config: c,
RevDB: revDB,
Ident: i,
VerificationFuncs: new(VerificationFuncs),
}
@ -98,13 +101,6 @@ func (opts *Options) configure() (err error) {
opts.VerificationFuncs.ClientAdd(peertls.VerifyCAWhitelist(opts.PeerCAWhitelist))
}
if opts.Config.Extensions.Revocation {
opts.RevDB, err = identity.NewRevocationDB(opts.Config.RevocationDBURL)
if err != nil {
return err
}
}
opts.handleExtensions(extensions.AllHandlers)
opts.Cert, err = peertls.TLSCert(opts.Ident.RawChain(), opts.Ident.Leaf, opts.Ident.Key)

11
pkg/peertls/tlsopts/options_test.go
View File

@ -18,6 +18,7 @@ import (
"storj.io/storj/pkg/peertls"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/revocation"
"storj.io/storj/pkg/storj"
"storj.io/storj/pkg/transport"
)
@ -105,7 +106,9 @@ func TestNewOptions(t *testing.T) {
for _, c := range cases {
t.Log(c.testID)
opts, err := tlsopts.NewOptions(fi, c.config)
revDB, err := revocation.NewDBFromCfg(c.config)
require.NoError(t, err)
opts, err := tlsopts.NewOptions(fi, c.config, revDB)
assert.NoError(t, err)
assert.True(t, reflect.DeepEqual(fi, opts.Ident))
assert.Equal(t, c.config, opts.Config)
@ -128,7 +131,7 @@ func TestOptions_ServerOption_Peer_CA_Whitelist(t *testing.T) {
testidentity.CompleteIdentityVersionsTest(t, func(t *testing.T, version storj.IDVersion, ident *identity.FullIdentity) {
opts, err := tlsopts.NewOptions(ident, tlsopts.Config{
PeerIDVersions: "*",
})
}, nil)
require.NoError(t, err)
dialOption, err := opts.DialOption(target.Id)
@ -148,7 +151,7 @@ func TestOptions_DialOption_error_on_empty_ID(t *testing.T) {
testidentity.CompleteIdentityVersionsTest(t, func(t *testing.T, version storj.IDVersion, ident *identity.FullIdentity) {
opts, err := tlsopts.NewOptions(ident, tlsopts.Config{
PeerIDVersions: "*",
})
}, nil)
require.NoError(t, err)
dialOption, err := opts.DialOption(storj.NodeID{})
@ -161,7 +164,7 @@ func TestOptions_DialUnverifiedIDOption(t *testing.T) {
testidentity.CompleteIdentityVersionsTest(t, func(t *testing.T, version storj.IDVersion, ident *identity.FullIdentity) {
opts, err := tlsopts.NewOptions(ident, tlsopts.Config{
PeerIDVersions: "*",
})
}, nil)
require.NoError(t, err)
dialOption := opts.DialUnverifiedIDOption()

5
pkg/peertls/tlsopts/tls_test.go
View File

@ -14,6 +14,7 @@ import (
"storj.io/storj/internal/testcontext"
"storj.io/storj/internal/testidentity"
"storj.io/storj/internal/testpeertls"
"storj.io/storj/internal/testrevocation"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/peertls"
"storj.io/storj/pkg/peertls/extensions"
@ -94,7 +95,7 @@ func TestExtensionMap_HandleExtensions(t *testing.T) {
err = rev.Verify(newRevokedLeafChain[peertls.CAIndex])
require.NoError(t, err)
testidentity.RevocationDBsTest(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) {
testrevocation.RevocationDBsTest(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) {
opts := &extensions.Options{
RevDB: revDB,
PeerIDVersions: "*",
@ -127,7 +128,7 @@ func TestExtensionMap_HandleExtensions_error(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
testidentity.RevocationDBsTest(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) {
testrevocation.RevocationDBsTest(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) {
keys, chain, oldRevocation, err := testpeertls.NewRevokedLeafChain()
assert.NoError(t, err)

82
pkg/identity/revocation_db.go → pkg/revocation/revocation_db.go
View File

@ -1,44 +1,66 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package identity
package revocation
import (
"context"
"crypto/x509"
"crypto/x509/pkix"
"github.com/zeebo/errs"
"gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/internal/dbutil"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/peertls"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/storage"
"storj.io/storj/storage/boltdb"
"storj.io/storj/storage/redis"
)
// RevocationDB stores the most recently seen revocation for each nodeID
var (
mon = monkit.Package()
// Error is a pkg/revocation error
Error = errs.Class("revocation error")
)
// DB stores the most recently seen revocation for each nodeID
// (i.e. nodeID [CA certificate's public key hash] is the key, values is
// the most recently seen revocation).
type RevocationDB struct {
DB storage.KeyValueStore
type DB struct {
KVStore storage.KeyValueStore
}
// NewRevocationDB returns a new revocation database given the URL
func NewRevocationDB(revocationDBURL string) (*RevocationDB, error) {
driver, source, err := dbutil.SplitConnstr(revocationDBURL)
// NewDBFromCfg is a convenience method to create a revocation DB
// directly from a config. If the revocation extension option is not set, it
// returns a nil db with no error.
func NewDBFromCfg(cfg tlsopts.Config) (*DB, error) {
if !cfg.Extensions.Revocation {
return nil, nil
}
return NewDB(cfg.RevocationDBURL)
}
// NewDB returns a new revocation database given the URL
func NewDB(dbURL string) (*DB, error) {
driver, source, err := dbutil.SplitConnstr(dbURL)
if err != nil {
return nil, extensions.ErrRevocationDB.Wrap(err)
}
var db *RevocationDB
var db *DB
switch driver {
case "bolt":
db, err = newRevocationDBBolt(source)
db, err = newDBBolt(source)
if err != nil {
return nil, extensions.ErrRevocationDB.Wrap(err)
}
case "redis":
db, err = newRevocationDBRedis(revocationDBURL)
db, err = newDBRedis(dbURL)
if err != nil {
return nil, extensions.ErrRevocationDB.Wrap(err)
}
@ -49,38 +71,38 @@ func NewRevocationDB(revocationDBURL string) (*RevocationDB, error) {
return db, nil
}
// newRevocationDBBolt creates a bolt-backed RevocationDB
func newRevocationDBBolt(path string) (*RevocationDB, error) {
// newDBBolt creates a bolt-backed DB
func newDBBolt(path string) (*DB, error) {
client, err := boltdb.New(path, extensions.RevocationBucket)
if err != nil {
return nil, err
}
return &RevocationDB{
DB: client,
return &DB{
KVStore: client,
}, nil
}
// newRevocationDBRedis creates a redis-backed RevocationDB.
func newRevocationDBRedis(address string) (*RevocationDB, error) {
// newDBRedis creates a redis-backed DB.
func newDBRedis(address string) (*DB, error) {
client, err := redis.NewClientFrom(address)
if err != nil {
return nil, err
}
return &RevocationDB{
DB: client,
return &DB{
KVStore: client,
}, nil
}
// Get attempts to retrieve the most recent revocation for the given cert chain
// (the key used in the underlying database is the nodeID of the certificate chain).
func (r RevocationDB) Get(ctx context.Context, chain []*x509.Certificate) (_ *extensions.Revocation, err error) {
func (db DB) Get(ctx context.Context, chain []*x509.Certificate) (_ *extensions.Revocation, err error) {
defer mon.Task()(&ctx)(&err)
nodeID, err := NodeIDFromCert(chain[peertls.CAIndex])
nodeID, err := identity.NodeIDFromCert(chain[peertls.CAIndex])
if err != nil {
return nil, extensions.ErrRevocation.Wrap(err)
}
revBytes, err := r.DB.Get(ctx, nodeID.Bytes())
revBytes, err := db.KVStore.Get(ctx, nodeID.Bytes())
if err != nil && !storage.ErrKeyNotFound.Has(err) {
return nil, extensions.ErrRevocationDB.Wrap(err)
}
@ -98,7 +120,7 @@ func (r RevocationDB) Get(ctx context.Context, chain []*x509.Certificate) (_ *ex
// Put stores the most recent revocation for the given cert chain IF the timestamp
// is newer than the current value (the key used in the underlying database is
// the nodeID of the certificate chain).
func (r RevocationDB) Put(ctx context.Context, chain []*x509.Certificate, revExt pkix.Extension) (err error) {
func (db DB) Put(ctx context.Context, chain []*x509.Certificate, revExt pkix.Extension) (err error) {
defer mon.Task()(&ctx)(&err)
ca := chain[peertls.CAIndex]
var rev extensions.Revocation
@ -113,32 +135,32 @@ func (r RevocationDB) Put(ctx context.Context, chain []*x509.Certificate, revExt
return err
}
lastRev, err := r.Get(ctx, chain)
lastRev, err := db.Get(ctx, chain)
if err != nil {
return err
} else if lastRev != nil && lastRev.Timestamp >= rev.Timestamp {
return extensions.ErrRevocationTimestamp
}
nodeID, err := NodeIDFromCert(ca)
nodeID, err := identity.NodeIDFromCert(ca)
if err != nil {
return extensions.ErrRevocationDB.Wrap(err)
}
if err := r.DB.Put(ctx, nodeID.Bytes(), revExt.Value); err != nil {
if err := db.KVStore.Put(ctx, nodeID.Bytes(), revExt.Value); err != nil {
return extensions.ErrRevocationDB.Wrap(err)
}
return nil
}
// List lists all revocations in the store
func (r RevocationDB) List(ctx context.Context) (revs []*extensions.Revocation, err error) {
func (db DB) List(ctx context.Context) (revs []*extensions.Revocation, err error) {
defer mon.Task()(&ctx)(&err)
keys, err := r.DB.List(ctx, []byte{}, 0)
keys, err := db.KVStore.List(ctx, []byte{}, 0)
if err != nil {
return nil, extensions.ErrRevocationDB.Wrap(err)
}
marshaledRevs, err := r.DB.GetAll(ctx, keys)
marshaledRevs, err := db.KVStore.GetAll(ctx, keys)
if err != nil {
return nil, extensions.ErrRevocationDB.Wrap(err)
}
@ -155,6 +177,6 @@ func (r RevocationDB) List(ctx context.Context) (revs []*extensions.Revocation,
}
// Close closes the underlying store
func (r RevocationDB) Close() error {
return r.DB.Close()
func (db DB) Close() error {
return db.KVStore.Close()
}

10
pkg/identity/revocation_db_test.go → pkg/revocation/revocation_db_test.go
View File

@ -1,7 +1,7 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package identity_test
package revocation_test
import (
"bytes"
@ -13,8 +13,8 @@ import (
"github.com/stretchr/testify/require"
"storj.io/storj/internal/testcontext"
"storj.io/storj/internal/testidentity"
"storj.io/storj/internal/testpeertls"
"storj.io/storj/internal/testrevocation"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/peertls"
"storj.io/storj/pkg/peertls/extensions"
@ -26,7 +26,7 @@ func TestRevocationDB_Get(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
testidentity.RevocationDBsTest(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) {
testrevocation.RevocationDBsTest(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) {
keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number)
require.NoError(t, err)
@ -64,7 +64,7 @@ func TestRevocationDB_Put_success(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
testidentity.RevocationDBsTest(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) {
testrevocation.RevocationDBsTest(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) {
keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number)
require.NoError(t, err)
@ -114,7 +114,7 @@ func TestRevocationDB_Put_error(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
testidentity.RevocationDBsTest(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) {
testrevocation.RevocationDBsTest(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) {
keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number)
require.NoError(t, err)

11
pkg/server/config.go
View File

@ -11,6 +11,7 @@ import (
"google.golang.org/grpc"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/peertls/tlsopts"
)
@ -23,10 +24,16 @@ type Config struct {
}
// Run will run the given responsibilities with the configured identity.
func (sc Config) Run(ctx context.Context, log *zap.Logger, identity *identity.FullIdentity, interceptor grpc.UnaryServerInterceptor, services ...Service) (err error) {
func (sc Config) Run(ctx context.Context, log *zap.Logger, identity *identity.FullIdentity, revDB extensions.RevocationDB, interceptor grpc.UnaryServerInterceptor, services ...Service) (err error) {
defer mon.Task()(&ctx)(&err)
opts, err := tlsopts.NewOptions(identity, sc.Config)
// Ensure revDB is not nil, since we call Close() below we do not want a
// panic
if revDB == nil {
return Error.New("revDB cannot be nil in call to Run")
}
opts, err := tlsopts.NewOptions(identity, sc.Config, revDB)
if err != nil {
return err
}

6
pkg/transport/transport_test.go
View File

@ -48,12 +48,12 @@ func TestDialNode(t *testing.T) {
UsePeerCAWhitelist: true,
PeerCAWhitelistPath: whitelistPath,
PeerIDVersions: "*",
})
}, nil)
require.NoError(t, err)
unsignedClientOpts, err := tlsopts.NewOptions(unsignedIdent, tlsopts.Config{
PeerIDVersions: "*",
})
}, nil)
require.NoError(t, err)
t.Run("DialNode with invalid targets", func(t *testing.T) {
@ -214,7 +214,7 @@ func TestDialNode_BadServerCertificate(t *testing.T) {
opts, err := tlsopts.NewOptions(ident, tlsopts.Config{
UsePeerCAWhitelist: true,
PeerCAWhitelistPath: whitelistPath,
})
}, nil)
require.NoError(t, err)
t.Run("DialNode with bad server certificate", func(t *testing.T) {

2
satellite/audit/reverify_test.go
View File

@ -307,7 +307,7 @@ func TestReverifyOfflineDialTimeout(t *testing.T) {
BytesPerSecond: 1 * memory.KiB,
}
tlsOpts, err := tlsopts.NewOptions(planet.Satellites[0].Identity, tlsopts.Config{})
tlsOpts, err := tlsopts.NewOptions(planet.Satellites[0].Identity, tlsopts.Config{}, nil)
require.NoError(t, err)
newTransport := transport.NewClientWithTimeouts(tlsOpts, transport.Timeouts{

4
satellite/audit/verifier_test.go
View File

@ -201,7 +201,7 @@ func TestDownloadSharesDialTimeout(t *testing.T) {
BytesPerSecond: 1 * memory.KiB,
}
tlsOpts, err := tlsopts.NewOptions(planet.Satellites[0].Identity, tlsopts.Config{})
tlsOpts, err := tlsopts.NewOptions(planet.Satellites[0].Identity, tlsopts.Config{}, nil)
require.NoError(t, err)
newTransport := transport.NewClientWithTimeouts(tlsOpts, transport.Timeouts{
@ -427,7 +427,7 @@ func TestVerifierDialTimeout(t *testing.T) {
BytesPerSecond: 1 * memory.KiB,
}
tlsOpts, err := tlsopts.NewOptions(planet.Satellites[0].Identity, tlsopts.Config{})
tlsOpts, err := tlsopts.NewOptions(planet.Satellites[0].Identity, tlsopts.Config{}, nil)
require.NoError(t, err)
newTransport := transport.NewClientWithTimeouts(tlsOpts, transport.Timeouts{

6
satellite/peer.go
View File

@ -25,6 +25,7 @@ import (
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/server"
"storj.io/storj/pkg/signing"
@ -228,7 +229,7 @@ type Peer struct {
}
// New creates a new satellite
func New(log *zap.Logger, full *identity.FullIdentity, db DB, config *Config, versionInfo version.Info) (*Peer, error) {
func New(log *zap.Logger, full *identity.FullIdentity, db DB, revDB extensions.RevocationDB, config *Config, versionInfo version.Info) (*Peer, error) {
peer := &Peer{
Log: log,
Identity: full,
@ -249,7 +250,8 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB, config *Config, ve
{ // setup listener and server
log.Debug("Starting listener and server")
sc := config.Server
options, err := tlsopts.NewOptions(peer.Identity, sc.Config)
options, err := tlsopts.NewOptions(peer.Identity, sc.Config, revDB)
if err != nil {
return nil, errs.Combine(err, peer.Close())
}

6
storagenode/peer.go
View File

@ -18,6 +18,7 @@ import (
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/server"
"storj.io/storj/pkg/signing"
@ -155,7 +156,7 @@ type Peer struct {
}
// New creates a new Storage Node.
func New(log *zap.Logger, full *identity.FullIdentity, db DB, config Config, versionInfo version.Info) (*Peer, error) {
func New(log *zap.Logger, full *identity.FullIdentity, db DB, revDB extensions.RevocationDB, config Config, versionInfo version.Info) (*Peer, error) {
peer := &Peer{
Log: log,
Identity: full,
@ -175,7 +176,8 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB, config Config, ver
{ // setup listener and server
sc := config.Server
options, err := tlsopts.NewOptions(peer.Identity, sc.Config)
options, err := tlsopts.NewOptions(peer.Identity, sc.Config, revDB)
if err != nil {
return nil, errs.Combine(err, peer.Close())
}