JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove Kademlia dependencies from Satellite and Storagenode (#2966)

Browse Source 
What:

cmd/inspector/main.go: removes kad commands
internal/testplanet/planet.go: Waits for contact chore to finish
satellite/contact/nodesservice.go: creates an empty nodes service implementation
satellite/contact/service.go: implements Local and FetchInfo methods & adds external address config value
satellite/discovery/service.go: replaces kad.FetchInfo with contact.FetchInfo in Refresh() & removes Discover()
satellite/peer.go: sets up contact service and endpoints
storagenode/console/service.go: replaces nodeID with contact.Local()
storagenode/contact/chore.go: replaces routing table with contact service
storagenode/contact/nodesservice.go: creates empty implementation for ping and request info nodes service & implements RequestInfo method
storagenode/contact/service.go: creates a service to return the local node and update its own capacity
storagenode/monitor/monitor.go: uses contact service in place of routing table
storagenode/operator.go: moves operatorconfig from kad into its own setup
storagenode/peer.go: sets up contact service, chore, pingstats and endpoints
satellite/overlay/config.go: changes NodeSelectionConfig.OnlineWindow default to 4hr to allow for accurate repair selection
Removes kademlia setups in:

cmd/storagenode/main.go
cmd/storj-sim/network.go
internal/testplane/planet.go
internal/testplanet/satellite.go
internal/testplanet/storagenode.go
satellite/peer.go
scripts/test-sim-backwards.sh
scripts/testdata/satellite-config.yaml.lock
storagenode/inspector/inspector.go
storagenode/peer.go
storagenode/storagenodedb/database.go
Why: Replacing Kademlia

Please describe the tests:
• internal/testplanet/planet_test.go:

TestBasic: assert that the storagenode can check in with the satellite without any errors
TestContact: test that all nodes get inserted into both satellites' overlay cache during testplanet setup
• satellite/contact/contact_test.go:

TestFetchInfo: Tests that the FetchInfo method returns the correct info
• storagenode/contact/contact_test.go:

TestNodeInfoUpdated: tests that the contact chore updates the node information
TestRequestInfoEndpoint: tests that the Request info endpoint returns the correct info
Please describe the performance impact: Node discovery should be at least slightly more performant since each node connects directly to each satellite and no longer needs to wait for bootstrapping. It probably won't be faster in real time on start up since each node waits a random amount of time (less than 1 hr) to initialize its first connection (jitter).
This commit is contained in:
Jennifer Li Johnson 2019-09-19 15:56:34 -04:00 committed by GitHub
parent 93788e5218
commit 724bb44723
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 781 additions and 1419 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -81,7 +81,7 @@ go test -v ./...
```
You can also execute only a single test package if you like. For example:
`go test ./pkg/kademlia`. Add `-v` for more informations about the executed unit
`go test ./pkg/identity`. Add `-v` for more informations about the executed unit
tests.
### Push up a pull request

203
cmd/inspector/main.go
View File

@ -13,14 +13,11 @@ import (
"strconv"
"strings"
"github.com/gogo/protobuf/jsonpb"
"github.com/gogo/protobuf/proto"
prompt "github.com/segmentio/go-prompt"
"github.com/spf13/cobra"
"github.com/zeebo/errs"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/kademlia/routinggraph"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/storj"
@ -55,11 +52,7 @@ var (
// Commander CLI
rootCmd = &cobra.Command{
Use: "inspector",
Short: "CLI for interacting with Storj Kademlia network",
}
kadCmd = &cobra.Command{
Use: "kad",
Short: "commands for kademlia/overlay",
Short: "CLI for interacting with Storj network",
}
statsCmd = &cobra.Command{
Use: "statdb",
@ -74,39 +67,6 @@ var (
Short: "list segments in irreparable database",
RunE: getSegments,
}
countNodeCmd = &cobra.Command{
Use: "count",
Short: "count nodes in kademlia and overlay",
RunE: CountNodes,
}
pingNodeCmd = &cobra.Command{
Use: "ping <node_id> <ip:port>",
Short: "ping node at provided ID",
Args: cobra.MinimumNArgs(2),
RunE: PingNode,
}
lookupNodeCmd = &cobra.Command{
Use: "lookup <node_id>",
Short: "lookup a node by ID only",
Args: cobra.MinimumNArgs(1),
RunE: LookupNode,
}
nodeInfoCmd = &cobra.Command{
Use: "node-info <node_id>",
Short: "get node info directly from node",
Args: cobra.MinimumNArgs(1),
RunE: NodeInfo,
}
dumpNodesCmd = &cobra.Command{
Use: "dump-nodes",
Short: "dump all nodes in the routing table",
RunE: DumpNodes,
}
drawTableCmd = &cobra.Command{
Use: "routing-graph",
Short: "Dumps a graph of the routing table in the dot format",
RunE: DrawTableAsGraph,
}
objectHealthCmd = &cobra.Command{
Use: "object <project-id> <bucket> <encrypted-path>",
Short: "Get stats about an object's health",
@ -121,16 +81,15 @@ var (
}
)
// Inspector gives access to kademlia and overlay.
// Inspector gives access to overlay.
type Inspector struct {
identity *identity.FullIdentity
kadclient pb.KadInspectorClient
overlayclient pb.OverlayInspectorClient
irrdbclient pb.IrreparableInspectorClient
healthclient pb.HealthInspectorClient
}
// NewInspector creates a new gRPC inspector client for access to kademlia and overlay.
// NewInspector creates a new gRPC inspector client for access to overlay.
func NewInspector(address, path string) (*Inspector, error) {
ctx := context.Background()
@ -149,160 +108,12 @@ func NewInspector(address, path string) (*Inspector, error) {
return &Inspector{
identity: id,
kadclient: pb.NewKadInspectorClient(conn),
overlayclient: pb.NewOverlayInspectorClient(conn),
irrdbclient: pb.NewIrreparableInspectorClient(conn),
healthclient: pb.NewHealthInspectorClient(conn),
}, nil
}
// CountNodes returns the number of nodes in kademlia
func CountNodes(cmd *cobra.Command, args []string) (err error) {
i, err := NewInspector(*Addr, *IdentityPath)
if err != nil {
return ErrInspectorDial.Wrap(err)
}
kadcount, err := i.kadclient.CountNodes(context.Background(), &pb.CountNodesRequest{})
if err != nil {
return ErrRequest.Wrap(err)
}
fmt.Printf("Kademlia node count: %+v\n", kadcount.Count)
return nil
}
// LookupNode starts a Kademlia lookup for the provided Node ID
func LookupNode(cmd *cobra.Command, args []string) (err error) {
i, err := NewInspector(*Addr, *IdentityPath)
if err != nil {
return ErrInspectorDial.Wrap(err)
}
n, err := i.kadclient.LookupNode(context.Background(), &pb.LookupNodeRequest{
Id: args[0],
})
if err != nil {
return ErrRequest.Wrap(err)
}
fmt.Println(prettyPrint(n))
return nil
}
// NodeInfo get node info directly from the node with provided Node ID
func NodeInfo(cmd *cobra.Command, args []string) (err error) {
i, err := NewInspector(*Addr, *IdentityPath)
if err != nil {
return ErrInspectorDial.Wrap(err)
}
// first lookup the node to get its address
n, err := i.kadclient.LookupNode(context.Background(), &pb.LookupNodeRequest{
Id: args[0],
})
if err != nil {
return ErrRequest.Wrap(err)
}
// now ask the node directly for its node info
info, err := i.kadclient.NodeInfo(context.Background(), &pb.NodeInfoRequest{
Id: n.GetNode().Id,
Address: n.GetNode().GetAddress(),
})
if err != nil {
return ErrRequest.Wrap(err)
}
fmt.Println(prettyPrint(info))
return nil
}
// DrawTableAsGraph outputs the table routing as a graph
func DrawTableAsGraph(cmd *cobra.Command, args []string) (err error) {
i, err := NewInspector(*Addr, *IdentityPath)
if err != nil {
return ErrInspectorDial.Wrap(err)
}
// retrieve buckets
info, err := i.kadclient.GetBucketList(context.Background(), &pb.GetBucketListRequest{})
if err != nil {
return ErrRequest.Wrap(err)
}
err = routinggraph.Draw(os.Stdout, info)
if err != nil {
return ErrRequest.Wrap(err)
}
return nil
}
// DumpNodes outputs a json list of every node in every bucket in the satellite
func DumpNodes(cmd *cobra.Command, args []string) (err error) {
i, err := NewInspector(*Addr, *IdentityPath)
if err != nil {
return ErrInspectorDial.Wrap(err)
}
nodes, err := i.kadclient.FindNear(context.Background(), &pb.FindNearRequest{
Start: storj.NodeID{},
Limit: 100000,
})
if err != nil {
return err
}
fmt.Println(prettyPrint(nodes))
return nil
}
func prettyPrint(unformatted proto.Message) string {
m := jsonpb.Marshaler{Indent: " ", EmitDefaults: true}
formatted, err := m.MarshalToString(unformatted)
if err != nil {
fmt.Println("Error", err)
os.Exit(1)
}
return formatted
}
// PingNode sends a PING RPC across the Kad network to check node availability
func PingNode(cmd *cobra.Command, args []string) (err error) {
nodeID, err := storj.NodeIDFromString(args[0])
if err != nil {
return err
}
i, err := NewInspector(*Addr, *IdentityPath)
if err != nil {
return ErrInspectorDial.Wrap(err)
}
fmt.Printf("Pinging node %s at %s", args[0], args[1])
p, err := i.kadclient.PingNode(context.Background(), &pb.PingNodeRequest{
Id: nodeID,
Address: args[1],
})
var okayString string
if p != nil && p.Ok {
okayString = "OK"
} else {
okayString = "Error"
}
fmt.Printf("\n -- Ping response: %s\n", okayString)
if err != nil {
fmt.Printf(" -- Error: %v\n", err)
}
return nil
}
// ObjectHealth gets information about the health of an object on the network
func ObjectHealth(cmd *cobra.Command, args []string) (err error) {
ctx := context.Background()
@ -604,18 +415,10 @@ func sortSegments(segments []*pb.IrreparableSegment) map[string][]*pb.Irreparabl
}
func init() {
rootCmd.AddCommand(kadCmd)
rootCmd.AddCommand(statsCmd)
rootCmd.AddCommand(irreparableCmd)
rootCmd.AddCommand(healthCmd)
kadCmd.AddCommand(countNodeCmd)
kadCmd.AddCommand(pingNodeCmd)
kadCmd.AddCommand(lookupNodeCmd)
kadCmd.AddCommand(nodeInfoCmd)
kadCmd.AddCommand(dumpNodesCmd)
kadCmd.AddCommand(drawTableCmd)
healthCmd.AddCommand(objectHealthCmd)
healthCmd.AddCommand(segmentHealthCmd)

6
cmd/storagenode/entrypoint
View File

@ -10,9 +10,9 @@ RUN_PARAMS="${RUN_PARAMS:-} --identity-dir identity"
RUN_PARAMS="${RUN_PARAMS:-} --metrics.app-suffix=-alpha"
RUN_PARAMS="${RUN_PARAMS:-} --metrics.interval=30m"
RUN_PARAMS="${RUN_PARAMS:-} --kademlia.external-address=${ADDRESS}"
RUN_PARAMS="${RUN_PARAMS:-} --kademlia.operator.email=${EMAIL}"
RUN_PARAMS="${RUN_PARAMS:-} --kademlia.operator.wallet=${WALLET}"
RUN_PARAMS="${RUN_PARAMS:-} --contact.external-address=${ADDRESS}"
RUN_PARAMS="${RUN_PARAMS:-} --operator.email=${EMAIL}"
RUN_PARAMS="${RUN_PARAMS:-} --operator.wallet=${WALLET}"
RUN_PARAMS="${RUN_PARAMS:-} --console.address=:14002"
RUN_PARAMS="${RUN_PARAMS:-} --console.static-dir=/app"
RUN_PARAMS="${RUN_PARAMS:-} --storage.allocated-bandwidth=${BANDWIDTH}"

9
cmd/storagenode/main.go
View File

@ -107,11 +107,10 @@ func init() {
func databaseConfig(config storagenode.Config) storagenodedb.Config {
return storagenodedb.Config{
Storage: config.Storage.Path,
Info: filepath.Join(config.Storage.Path, "piecestore.db"),
Info2: filepath.Join(config.Storage.Path, "info.db"),
Pieces: config.Storage.Path,
Kademlia: config.Kademlia.DBPath,
Storage: config.Storage.Path,
Info: filepath.Join(config.Storage.Path, "piecestore.db"),
Info2: filepath.Join(config.Storage.Path, "info.db"),
Pieces: config.Storage.Path,
}
}

7
cmd/storj-sim/network.go
View File

@ -282,8 +282,6 @@ func newNetwork(flags *Flags) (*Processes, error) {
"--server.address", process.Address,
"--server.private-address", net.JoinHostPort(host, port(satellitePeer, i, privateGRPC)),
"--kademlia.bootstrap-addr", bootstrap.Address,
"--server.extensions.revocation=false",
"--server.use-peer-ca-whitelist=false",
@ -450,9 +448,8 @@ func newNetwork(flags *Flags) (*Processes, error) {
"--server.address", process.Address,
"--server.private-address", net.JoinHostPort(host, port(storagenodePeer, i, privateGRPC)),
"--kademlia.bootstrap-addr", bootstrap.Address,
"--kademlia.operator.email", fmt.Sprintf("storage%d@mail.test", i),
"--kademlia.operator.wallet", "0x0123456789012345678901234567890123456789",
"--operator.email", fmt.Sprintf("storage%d@mail.test", i),
"--operator.wallet", "0x0123456789012345678901234567890123456789",
"--storage2.monitor.minimum-disk-space", "0",
"--storage2.monitor.minimum-bandwidth", "0",

65
internal/testplanet/planet.go
View File

@ -23,7 +23,6 @@ import (
"storj.io/storj/bootstrap"
"storj.io/storj/internal/testidentity"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/storj"
"storj.io/storj/satellite"
"storj.io/storj/satellite/overlay"
@ -199,19 +198,6 @@ func NewCustom(log *zap.Logger, config Config) (*Planet, error) {
return nil, errs.Combine(err, planet.Shutdown())
}
// init Satellites
for _, satellite := range planet.Satellites {
if len(satellite.Kademlia.Service.GetBootstrapNodes()) == 0 {
satellite.Kademlia.Service.SetBootstrapNodes([]pb.Node{planet.Bootstrap.Local().Node})
}
}
// init storage nodes
for _, storageNode := range planet.StorageNodes {
if len(storageNode.Kademlia.Service.GetBootstrapNodes()) == 0 {
storageNode.Kademlia.Service.SetBootstrapNodes([]pb.Node{planet.Bootstrap.Local().Node})
}
}
return planet, nil
}
@ -231,56 +217,11 @@ func (planet *Planet) Start(ctx context.Context) {
return peer.peer.Run(peer.ctx)
})
}
for _, peer := range planet.StorageNodes {
peer.Contact.Chore.Loop.TriggerWait()
}
planet.started = true
planet.Bootstrap.Kademlia.Service.WaitForBootstrap()
for _, peer := range planet.StorageNodes {
peer.Kademlia.Service.WaitForBootstrap()
}
for _, peer := range planet.Satellites {
peer.Kademlia.Service.WaitForBootstrap()
}
planet.Reconnect(ctx)
}
// Reconnect reconnects all nodes with each other.
func (planet *Planet) Reconnect(ctx context.Context) {
log := planet.log.Named("reconnect")
var group errgroup.Group
// TODO: instead of pinging try to use Lookups or natural discovery to ensure
// everyone finds everyone else
for _, storageNode := range planet.StorageNodes {
storageNode := storageNode
group.Go(func() error {
_, err := storageNode.Kademlia.Service.Ping(ctx, planet.Bootstrap.Local().Node)
if err != nil {
log.Error("storage node did not find bootstrap", zap.Error(err))
}
return nil
})
}
for _, satellite := range planet.Satellites {
satellite := satellite
group.Go(func() error {
for _, storageNode := range planet.StorageNodes {
_, err := satellite.Kademlia.Service.Ping(ctx, storageNode.Local().Node)
if err != nil {
log.Error("satellite did not find storage node", zap.Error(err))
}
}
return nil
})
}
_ = group.Wait() // none of the goroutines return an error
}
// StopPeer stops a single peer in the planet

39
internal/testplanet/planet_test.go
View File

@ -13,6 +13,7 @@ import (
"storj.io/storj/internal/testcontext"
"storj.io/storj/internal/testplanet"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/storj"
)
@ -37,21 +38,18 @@ func TestBasic(t *testing.T) {
t.Log("UPLINK", uplink.ID(), uplink.Addr())
}
// ping a satellite
_, err = planet.StorageNodes[0].Kademlia.Service.Ping(ctx, planet.Satellites[0].Local().Node)
sat := planet.Satellites[0].Local().Node
node := planet.StorageNodes[0].Local()
conn, err := planet.StorageNodes[0].Transport.DialNode(ctx, &sat)
require.NoError(t, err)
// ping a storage node
_, err = planet.StorageNodes[0].Kademlia.Service.Ping(ctx, planet.StorageNodes[1].Local().Node)
_, err = pb.NewNodeClient(conn).CheckIn(ctx, &pb.CheckInRequest{
Address: node.GetAddress().GetAddress(),
Version: &node.Version,
Capacity: &node.Capacity,
Operator: &node.Operator,
})
require.NoError(t, err)
err = planet.StopPeer(planet.StorageNodes[1])
require.NoError(t, err)
// ping a stopped storage node
_, err = planet.StorageNodes[0].Kademlia.Service.Ping(ctx, planet.StorageNodes[1].Local().Node)
require.Error(t, err)
// wait a bit to see whether some failures occur
time.Sleep(time.Second)
}
@ -61,6 +59,23 @@ func TestBasic(t *testing.T) {
}
}
// test that nodes get put into each satellite's overlay cache
func TestContact(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 2, StorageNodeCount: 5, UplinkCount: 0,
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
satellite0 := planet.Satellites[0]
satellite1 := planet.Satellites[1]
for _, n := range planet.StorageNodes {
_, err := satellite0.Overlay.Service.Get(ctx, n.ID())
require.NoError(t, err)
_, err = satellite1.Overlay.Service.Get(ctx, n.ID())
require.NoError(t, err)
}
})
}
func BenchmarkCreate(b *testing.B) {
storageNodes := []int{4, 10, 100}
for _, count := range storageNodes {

13
internal/testplanet/satellite.go
View File

@ -7,13 +7,11 @@ import (
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/zeebo/errs"
"storj.io/storj/internal/memory"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/revocation"
@ -86,16 +84,6 @@ func (planet *Planet) newSatellites(count int) ([]*SatelliteSystem, error) {
},
},
},
Kademlia: kademlia.Config{
Alpha: 5,
BootstrapBackoffBase: 500 * time.Millisecond,
BootstrapBackoffMax: 2 * time.Second,
DBPath: storageDir, // TODO: replace with master db
Operator: kademlia.OperatorConfig{
Email: prefix + "@mail.test",
Wallet: "0x" + strings.Repeat("00", 20),
},
},
Overlay: overlay.Config{
Node: overlay.NodeSelectionConfig{
UptimeCount: 0,
@ -122,7 +110,6 @@ func (planet *Planet) newSatellites(count int) ([]*SatelliteSystem, error) {
UpdateStatsBatchSize: 100,
},
Discovery: discovery.Config{
DiscoveryInterval: 1 * time.Second,
RefreshInterval: 1 * time.Second,
RefreshLimit: 100,
RefreshConcurrency: 2,

22
internal/testplanet/storagenode.go
View File

@ -14,7 +14,6 @@ import (
"github.com/zeebo/errs"
"storj.io/storj/internal/memory"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/revocation"
@ -72,15 +71,9 @@ func (planet *Planet) newStorageNodes(count int, whitelistedSatellites storj.Nod
},
},
},
Kademlia: kademlia.Config{
BootstrapBackoffBase: 500 * time.Millisecond,
BootstrapBackoffMax: 2 * time.Second,
Alpha: 5,
DBPath: filepath.Join(storageDir, "kademlia/"),
Operator: kademlia.OperatorConfig{
Email: prefix + "@mail.test",
Wallet: "0x" + strings.Repeat("00", 20),
},
Operator: storagenode.OperatorConfig{
Email: prefix + "@mail.test",
Wallet: "0x" + strings.Repeat("00", 20),
},
Storage: piecestore.OldConfig{
Path: filepath.Join(storageDir, "pieces/"),
@ -145,11 +138,10 @@ func (planet *Planet) newStorageNodes(count int, whitelistedSatellites storj.Nod
verisonInfo := planet.NewVersionInfo()
storageConfig := storagenodedb.Config{
Storage: config.Storage.Path,
Info: filepath.Join(config.Storage.Path, "piecestore.db"),
Info2: filepath.Join(config.Storage.Path, "info.db"),
Pieces: config.Storage.Path,
Kademlia: config.Kademlia.DBPath,
Storage: config.Storage.Path,
Info: filepath.Join(config.Storage.Path, "piecestore.db"),
Info2: filepath.Join(config.Storage.Path, "info.db"),
Pieces: config.Storage.Path,
}
var db storagenode.DB

3
internal/testplanet/uplink_test.go
View File

@ -81,9 +81,6 @@ func TestDownloadWithSomeNodesOffline(t *testing.T) {
ul := planet.Uplinks[0]
satellite := planet.Satellites[0]
// stop discovery service so that we do not get a race condition when we delete nodes from overlay
satellite.Discovery.Service.Discovery.Stop()
testData := testrand.Bytes(memory.MiB)
err := ul.UploadWithConfig(ctx, satellite, &uplink.RSConfig{

258
pkg/kademlia/kademlia_planet_test.go
View File

@ -1,258 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package kademlia_test
import (
"context"
"io"
"net"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zeebo/errs"
"go.uber.org/zap"
"go.uber.org/zap/zaptest"
"golang.org/x/sync/errgroup"
"google.golang.org/grpc/codes"
"storj.io/storj/internal/errs2"
"storj.io/storj/internal/memory"
"storj.io/storj/internal/testcontext"
"storj.io/storj/internal/testplanet"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/transport"
"storj.io/storj/satellite"
"storj.io/storj/storagenode"
)
func TestFetchPeerIdentity(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1, StorageNodeCount: 1, UplinkCount: 0,
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
sat := planet.Satellites[0]
peerID, err := planet.StorageNodes[0].Kademlia.Service.FetchPeerIdentity(ctx, sat.ID())
require.NoError(t, err)
require.Equal(t, sat.ID(), peerID.ID)
require.True(t, sat.Identity.Leaf.Equal(peerID.Leaf))
require.True(t, sat.Identity.CA.Equal(peerID.CA))
})
}
func TestRequestInfo(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1, StorageNodeCount: 1, UplinkCount: 0,
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
node := planet.StorageNodes[0]
info, err := planet.Satellites[0].Kademlia.Service.FetchInfo(ctx, node.Local().Node)
require.NoError(t, err)
require.Equal(t, node.Local().Type, info.GetType())
require.Empty(t, cmp.Diff(node.Local().Operator, *info.GetOperator(), cmp.Comparer(pb.Equal)))
require.Empty(t, cmp.Diff(node.Local().Capacity, *info.GetCapacity(), cmp.Comparer(pb.Equal)))
require.Empty(t, cmp.Diff(node.Local().Version, *info.GetVersion(), cmp.Comparer(pb.Equal)))
})
}
func TestRequestInfoUntrusted(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1, StorageNodeCount: 1, UplinkCount: 0,
Reconfigure: testplanet.Reconfigure{
StorageNode: func(index int, config *storagenode.Config) {
config.Storage.WhitelistedSatellites = nil
},
},
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
_, err := planet.Satellites[0].Kademlia.Service.FetchInfo(ctx, planet.StorageNodes[0].Local().Node)
require.Error(t, err)
assert.True(t, errs2.IsRPC(err, codes.PermissionDenied), "unexpected error: %+v", err)
})
}
func TestPingTimeout(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1, StorageNodeCount: 4, UplinkCount: 0,
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
self := planet.StorageNodes[0]
routingTable := self.Kademlia.RoutingTable
tlsOpts, err := tlsopts.NewOptions(self.Identity, tlsopts.Config{}, nil)
require.NoError(t, err)
self.Transport = transport.NewClientWithTimeouts(tlsOpts, transport.Timeouts{
Request: 1 * time.Millisecond,
})
network := &transport.SimulatedNetwork{
DialLatency: 300 * time.Second,
BytesPerSecond: 1 * memory.KB,
}
slowClient := network.NewClient(self.Transport)
require.NotNil(t, slowClient)
newService, err := kademlia.NewService(zaptest.NewLogger(t), slowClient, routingTable, kademlia.Config{})
require.NoError(t, err)
target := pb.Node{
Id: planet.StorageNodes[2].ID(),
Address: &pb.NodeAddress{
Transport: pb.NodeTransport_TCP_TLS_GRPC,
Address: planet.StorageNodes[2].Addr(),
},
}
_, err = newService.Ping(ctx, target)
require.Error(t, err, context.DeadlineExceeded)
require.True(t, kademlia.NodeErr.Has(err) && transport.Error.Has(err))
})
}
func TestBootstrapBackoffReconnect(t *testing.T) {
// TODO(nat): skipping because flakily erroring with "panic: planet took too long to shutdown"
// or kademlia_planet_test.go:139: dial tcp 127.0.0.1:40409: connect: connection refused
t.Skip("flaky")
ctx := testcontext.New(t)
defer ctx.Cleanup()
log := zaptest.NewLogger(t)
// This sets up an unreliable proxy server which will receive conns from
// storage nodes and the satellite, but drops the connections of the first
// `dropCount` number of connections to the bootstrap node (proxy.target).
// This should test that the Bootstrap function will retry a connection
// if it initially fails.
proxy, err := newBadProxy(log.Named("proxy"), "127.0.0.1:0", 4)
require.NoError(t, err)
planet, err := testplanet.NewCustom(log, testplanet.Config{
SatelliteCount: 1, StorageNodeCount: 4, UplinkCount: 0,
Reconfigure: testplanet.Reconfigure{
Satellite: func(log *zap.Logger, index int, config *satellite.Config) {
config.Kademlia.BootstrapAddr = proxy.listener.Addr().String()
},
StorageNode: func(index int, config *storagenode.Config) {
config.Kademlia.BootstrapAddr = proxy.listener.Addr().String()
config.Kademlia.BootstrapBackoffBase = 100 * time.Millisecond
config.Kademlia.BootstrapBackoffMax = 3 * time.Second
},
},
})
require.NoError(t, err)
// We set the bad proxy's "target" to the bootstrap node's addr
// (which was selected when the new custom planet was set up).
proxy.target = planet.Bootstrap.Addr()
var group errgroup.Group
group.Go(func() error { return proxy.run(ctx) })
defer ctx.Check(group.Wait)
defer ctx.Check(proxy.close)
planet.Start(ctx)
ctx.Check(planet.Shutdown)
}
type badProxy struct {
log *zap.Logger
target string
dropCount int
listener net.Listener
done chan struct{}
}
func newBadProxy(log *zap.Logger, addr string, dropCount int) (*badProxy, error) {
listener, err := net.Listen("tcp", addr)
if err != nil {
return nil, errs.Wrap(err)
}
return &badProxy{
log: log,
target: "",
dropCount: dropCount,
listener: listener,
done: make(chan struct{}),
}, nil
}
func (proxy *badProxy) close() error {
close(proxy.done)
return proxy.listener.Close()
}
func (proxy *badProxy) run(ctx context.Context) error {
var group errgroup.Group
group.Go(func() (err error) {
var connections errs2.Group
defer func() {
var errlist errs.Group
errlist.Add(err)
errlist.Add(connections.Wait()...)
err = errlist.Err()
}()
var conns int
for {
conn, err := proxy.listener.Accept()
if err != nil {
select {
case <-proxy.done:
return nil
default:
}
return errs.Wrap(err)
}
conns++
if conns < proxy.dropCount {
if err := conn.Close(); err != nil {
return errs.Wrap(err)
}
continue
}
connections.Go(func() error {
defer func() {
err = errs.Combine(err, conn.Close())
}()
targetConn, err := net.Dial("tcp", proxy.target)
if err != nil {
return err
}
defer func() { err = errs.Combine(err, targetConn.Close()) }()
var pipe errs2.Group
pipe.Go(func() error {
_, err := io.Copy(targetConn, conn)
// since planet is shutting down a forced close is to be expected
if err != nil {
proxy.log.Debug("copy error", zap.Error(err))
}
return nil
})
pipe.Go(func() error {
_, err := io.Copy(conn, targetConn)
// since planet is shutting down a forced close is to be expected
if err != nil {
proxy.log.Debug("copy error", zap.Error(err))
}
return nil
})
return errs.Combine(pipe.Wait()...)
})
}
})
return errs.Wrap(group.Wait())
}

567
pkg/kademlia/kademliaclient/kademliaclient_test.go
View File

@ -3,297 +3,276 @@
package kademliaclient_test
import (
"context"
"fmt"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/zeebo/errs"
"go.uber.org/zap/zaptest"
"golang.org/x/sync/errgroup"
"storj.io/storj/internal/memory"
"storj.io/storj/internal/testcontext"
"storj.io/storj/internal/testplanet"
"storj.io/storj/pkg/kademlia/kademliaclient"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/peertls/tlsopts"
"storj.io/storj/pkg/storj"
"storj.io/storj/pkg/transport"
)
func TestDialer(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
planet, err := testplanet.New(t, 1, 4, 3)
require.NoError(t, err)
defer ctx.Check(planet.Shutdown)
planet.Start(ctx)
expectedKademliaEntries := len(planet.Satellites) + len(planet.StorageNodes)
// TODO: also use satellites
peers := planet.StorageNodes
{ // PingNode: storage node pings all other storage nodes
self := planet.StorageNodes[0]
dialer := kademliaclient.NewDialer(zaptest.NewLogger(t), self.Transport)
defer ctx.Check(dialer.Close)
var group errgroup.Group
defer ctx.Check(group.Wait)
for _, peer := range peers {
peer := peer
group.Go(func() error {
pinged, err := dialer.PingNode(ctx, peer.Local().Node)
var pingErr error
if !pinged {
pingErr = fmt.Errorf("ping to %s should have succeeded", peer.ID())
}
return errs.Combine(pingErr, err)
})
}
}
{ // FetchPeerIdentity: storage node fetches identity of the satellite
self := planet.StorageNodes[0]
dialer := kademliaclient.NewDialer(zaptest.NewLogger(t), self.Transport)
defer ctx.Check(dialer.Close)
var group errgroup.Group
defer ctx.Check(group.Wait)
group.Go(func() error {
ident, err := dialer.FetchPeerIdentity(ctx, planet.Satellites[0].Local().Node)
if err != nil {
return fmt.Errorf("failed to fetch peer identity")
}
if ident.ID != planet.Satellites[0].Local().Id {
return fmt.Errorf("fetched wrong identity")
}
ident, err = dialer.FetchPeerIdentityUnverified(ctx, planet.Satellites[0].Addr())
if err != nil {
return fmt.Errorf("failed to fetch peer identity from address")
}
if ident.ID != planet.Satellites[0].Local().Id {
return fmt.Errorf("fetched wrong identity from address")
}
return nil
})
}
{ // Lookup: storage node query every node for everyone elese
self := planet.StorageNodes[1]
dialer := kademliaclient.NewDialer(zaptest.NewLogger(t), self.Transport)
defer ctx.Check(dialer.Close)
var group errgroup.Group
defer ctx.Check(group.Wait)
for _, peer := range peers {
peer := peer
group.Go(func() error {
for _, target := range peers {
errTag := fmt.Errorf("lookup peer:%s target:%s", peer.ID(), target.ID())
selfnode := self.Local().Node
results, err := dialer.Lookup(ctx, &selfnode, peer.Local().Node, target.Local().Node.Id, self.Kademlia.RoutingTable.K())
if err != nil {
return errs.Combine(errTag, err)
}
if containsResult(results, target.ID()) {
continue
}
// with small network we expect to return everything
if len(results) != expectedKademliaEntries {
return errs.Combine(errTag, fmt.Errorf("expected %d got %d: %s", expectedKademliaEntries, len(results), pb.NodesToIDs(results)))
}
return nil
}
return nil
})
}
}
{ // Lookup: storage node queries every node for missing storj.NodeID{} and storj.NodeID{255}
self := planet.StorageNodes[2]
dialer := kademliaclient.NewDialer(zaptest.NewLogger(t), self.Transport)
defer ctx.Check(dialer.Close)
targets := []storj.NodeID{
{}, // empty target
{255}, // non-empty
}
var group errgroup.Group
defer ctx.Check(group.Wait)
for _, target := range targets {
target := target
for _, peer := range peers {
peer := peer
group.Go(func() error {
errTag := fmt.Errorf("invalid lookup peer:%s target:%s", peer.ID(), target)
selfnode := self.Local().Node
results, err := dialer.Lookup(ctx, &selfnode, peer.Local().Node, target, self.Kademlia.RoutingTable.K())
if err != nil {
return errs.Combine(errTag, err)
}
// with small network we expect to return everything
if len(results) != expectedKademliaEntries {
return errs.Combine(errTag, fmt.Errorf("expected %d got %d: %s", expectedKademliaEntries, len(results), pb.NodesToIDs(results)))
}
return nil
})
}
}
}
}
func TestSlowDialerHasTimeout(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
planet, err := testplanet.New(t, 1, 4, 0)
require.NoError(t, err)
defer ctx.Check(planet.Shutdown)
planet.Start(ctx)
// TODO: also use satellites
peers := planet.StorageNodes
func() { // PingNode
self := planet.StorageNodes[0]
tlsOpts, err := tlsopts.NewOptions(self.Identity, tlsopts.Config{}, nil)
require.NoError(t, err)
self.Transport = transport.NewClientWithTimeouts(tlsOpts, transport.Timeouts{
Dial: 20 * time.Millisecond,
})
network := &transport.SimulatedNetwork{
DialLatency: 200 * time.Second,
BytesPerSecond: 1 * memory.KB,
}
slowClient := network.NewClient(self.Transport)
require.NotNil(t, slowClient)
dialer := kademliaclient.NewDialer(zaptest.NewLogger(t), slowClient)
defer ctx.Check(dialer.Close)
var group errgroup.Group
defer ctx.Check(group.Wait)
for _, peer := range peers {
peer := peer
group.Go(func() error {
_, err := dialer.PingNode(ctx, peer.Local().Node)
if !transport.Error.Has(err) || errs.Unwrap(err) != context.DeadlineExceeded {
return errs.New("invalid error: %v", err)
}
return nil
})
}
}()
func() { // FetchPeerIdentity
self := planet.StorageNodes[1]
tlsOpts, err := tlsopts.NewOptions(self.Identity, tlsopts.Config{}, nil)
require.NoError(t, err)
self.Transport = transport.NewClientWithTimeouts(tlsOpts, transport.Timeouts{
Dial: 20 * time.Millisecond,
})
network := &transport.SimulatedNetwork{
DialLatency: 200 * time.Second,
BytesPerSecond: 1 * memory.KB,
}
slowClient := network.NewClient(self.Transport)
require.NotNil(t, slowClient)
dialer := kademliaclient.NewDialer(zaptest.NewLogger(t), slowClient)
defer ctx.Check(dialer.Close)
var group errgroup.Group
defer ctx.Check(group.Wait)
group.Go(func() error {
_, err := dialer.FetchPeerIdentity(ctx, planet.Satellites[0].Local().Node)
if !transport.Error.Has(err) || errs.Unwrap(err) != context.DeadlineExceeded {
return errs.New("invalid error: %v", err)
}
_, err = dialer.FetchPeerIdentityUnverified(ctx, planet.Satellites[0].Addr())
if !transport.Error.Has(err) || errs.Unwrap(err) != context.DeadlineExceeded {
return errs.New("invalid error: %v", err)
}
return nil
})
}()
func() { // Lookup
self := planet.StorageNodes[2]
tlsOpts, err := tlsopts.NewOptions(self.Identity, tlsopts.Config{}, nil)
require.NoError(t, err)
self.Transport = transport.NewClientWithTimeouts(tlsOpts, transport.Timeouts{
Dial: 20 * time.Millisecond,
})
network := &transport.SimulatedNetwork{
DialLatency: 200 * time.Second,
BytesPerSecond: 1 * memory.KB,
}
slowClient := network.NewClient(self.Transport)
require.NotNil(t, slowClient)
dialer := kademliaclient.NewDialer(zaptest.NewLogger(t), slowClient)
defer ctx.Check(dialer.Close)
var group errgroup.Group
defer ctx.Check(group.Wait)
for _, peer := range peers {
peer := peer
group.Go(func() error {
for _, target := range peers {
selfnode := self.Local().Node
_, err := dialer.Lookup(ctx, &selfnode, peer.Local().Node, target.Local().Node.Id, self.Kademlia.RoutingTable.K())
if !transport.Error.Has(err) || errs.Unwrap(err) != context.DeadlineExceeded {
return errs.New("invalid error: %v (peer:%s target:%s)", err, peer.ID(), target.ID())
}
}
return nil
})
}
}()
}
func containsResult(nodes []*pb.Node, target storj.NodeID) bool {
for _, node := range nodes {
if node.Id == target {
return true
}
}
return false
}
//func TestDialer(t *testing.T) {
// ctx := testcontext.New(t)
// defer ctx.Cleanup()
//
// planet, err := testplanet.New(t, 1, 4, 3)
// require.NoError(t, err)
// defer ctx.Check(planet.Shutdown)
//
// planet.Start(ctx)
//
// expectedKademliaEntries := len(planet.Satellites) + len(planet.StorageNodes)
//
// // TODO: also use satellites
// peers := planet.StorageNodes
//
// { // PingNode: storage node pings all other storage nodes
// self := planet.StorageNodes[0]
//
// dialer := kademliaclient.NewDialer(zaptest.NewLogger(t), self.Transport)
// defer ctx.Check(dialer.Close)
//
// var group errgroup.Group
// defer ctx.Check(group.Wait)
//
// for _, peer := range peers {
// peer := peer
// group.Go(func() error {
// pinged, err := dialer.PingNode(ctx, peer.Local().Node)
// var pingErr error
// if !pinged {
// pingErr = fmt.Errorf("ping to %s should have succeeded", peer.ID())
// }
// return errs.Combine(pingErr, err)
// })
// }
// }
//
// { // FetchPeerIdentity: storage node fetches identity of the satellite
// self := planet.StorageNodes[0]
//
// dialer := kademliaclient.NewDialer(zaptest.NewLogger(t), self.Transport)
// defer ctx.Check(dialer.Close)
//
// var group errgroup.Group
// defer ctx.Check(group.Wait)
//
// group.Go(func() error {
// ident, err := dialer.FetchPeerIdentity(ctx, planet.Satellites[0].Local().Node)
// if err != nil {
// return fmt.Errorf("failed to fetch peer identity")
// }
// if ident.ID != planet.Satellites[0].Local().Id {
// return fmt.Errorf("fetched wrong identity")
// }
//
// ident, err = dialer.FetchPeerIdentityUnverified(ctx, planet.Satellites[0].Addr())
// if err != nil {
// return fmt.Errorf("failed to fetch peer identity from address")
// }
// if ident.ID != planet.Satellites[0].Local().Id {
// return fmt.Errorf("fetched wrong identity from address")
// }
//
// return nil
// })
// }
//
// { // Lookup: storage node query every node for everyone elese
// self := planet.StorageNodes[1]
// dialer := kademliaclient.NewDialer(zaptest.NewLogger(t), self.Transport)
// defer ctx.Check(dialer.Close)
//
// var group errgroup.Group
// defer ctx.Check(group.Wait)
//
// for _, peer := range peers {
// peer := peer
// group.Go(func() error {
// for _, target := range peers {
// errTag := fmt.Errorf("lookup peer:%s target:%s", peer.ID(), target.ID())
//
// selfnode := self.Local().Node
// results, err := dialer.Lookup(ctx, &selfnode, peer.Local().Node, target.Local().Node.Id, self.Kademlia.RoutingTable.K())
// if err != nil {
// return errs.Combine(errTag, err)
// }
//
// if containsResult(results, target.ID()) {
// continue
// }
//
// // with small network we expect to return everything
// if len(results) != expectedKademliaEntries {
// return errs.Combine(errTag, fmt.Errorf("expected %d got %d: %s", expectedKademliaEntries, len(results), pb.NodesToIDs(results)))
// }
// return nil
// }
// return nil
// })
// }
// }
//
// { // Lookup: storage node queries every node for missing storj.NodeID{} and storj.NodeID{255}
// self := planet.StorageNodes[2]
// dialer := kademliaclient.NewDialer(zaptest.NewLogger(t), self.Transport)
// defer ctx.Check(dialer.Close)
//
// targets := []storj.NodeID{
// {}, // empty target
// {255}, // non-empty
// }
//
// var group errgroup.Group
// defer ctx.Check(group.Wait)
//
// for _, target := range targets {
// target := target
// for _, peer := range peers {
// peer := peer
// group.Go(func() error {
// errTag := fmt.Errorf("invalid lookup peer:%s target:%s", peer.ID(), target)
//
// selfnode := self.Local().Node
// results, err := dialer.Lookup(ctx, &selfnode, peer.Local().Node, target, self.Kademlia.RoutingTable.K())
// if err != nil {
// return errs.Combine(errTag, err)
// }
//
// // with small network we expect to return everything
// if len(results) != expectedKademliaEntries {
// return errs.Combine(errTag, fmt.Errorf("expected %d got %d: %s", expectedKademliaEntries, len(results), pb.NodesToIDs(results)))
// }
// return nil
// })
// }
// }
// }
//}
//
//func TestSlowDialerHasTimeout(t *testing.T) {
// ctx := testcontext.New(t)
// defer ctx.Cleanup()
//
// planet, err := testplanet.New(t, 1, 4, 0)
// require.NoError(t, err)
// defer ctx.Check(planet.Shutdown)
//
// planet.Start(ctx)
//
// // TODO: also use satellites
// peers := planet.StorageNodes
//
// func() { // PingNode
// self := planet.StorageNodes[0]
//
// tlsOpts, err := tlsopts.NewOptions(self.Identity, tlsopts.Config{}, nil)
// require.NoError(t, err)
//
// self.Transport = transport.NewClientWithTimeouts(tlsOpts, transport.Timeouts{
// Dial: 20 * time.Millisecond,
// })
//
// network := &transport.SimulatedNetwork{
// DialLatency: 200 * time.Second,
// BytesPerSecond: 1 * memory.KB,
// }
//
// slowClient := network.NewClient(self.Transport)
// require.NotNil(t, slowClient)
//
// dialer := kademliaclient.NewDialer(zaptest.NewLogger(t), slowClient)
// defer ctx.Check(dialer.Close)
//
// var group errgroup.Group
// defer ctx.Check(group.Wait)
//
// for _, peer := range peers {
// peer := peer
// group.Go(func() error {
// _, err := dialer.PingNode(ctx, peer.Local().Node)
// if !transport.Error.Has(err) || errs.Unwrap(err) != context.DeadlineExceeded {
// return errs.New("invalid error: %v", err)
// }
// return nil
// })
// }
// }()
//
// func() { // FetchPeerIdentity
// self := planet.StorageNodes[1]
//
// tlsOpts, err := tlsopts.NewOptions(self.Identity, tlsopts.Config{}, nil)
// require.NoError(t, err)
//
// self.Transport = transport.NewClientWithTimeouts(tlsOpts, transport.Timeouts{
// Dial: 20 * time.Millisecond,
// })
//
// network := &transport.SimulatedNetwork{
// DialLatency: 200 * time.Second,
// BytesPerSecond: 1 * memory.KB,
// }
//
// slowClient := network.NewClient(self.Transport)
// require.NotNil(t, slowClient)
//
// dialer := kademliaclient.NewDialer(zaptest.NewLogger(t), slowClient)
// defer ctx.Check(dialer.Close)
//
// var group errgroup.Group
// defer ctx.Check(group.Wait)
//
// group.Go(func() error {
// _, err := dialer.FetchPeerIdentity(ctx, planet.Satellites[0].Local().Node)
// if !transport.Error.Has(err) || errs.Unwrap(err) != context.DeadlineExceeded {
// return errs.New("invalid error: %v", err)
// }
// _, err = dialer.FetchPeerIdentityUnverified(ctx, planet.Satellites[0].Addr())
// if !transport.Error.Has(err) || errs.Unwrap(err) != context.DeadlineExceeded {
// return errs.New("invalid error: %v", err)
// }
// return nil
// })
// }()
//
// func() { // Lookup
// self := planet.StorageNodes[2]
//
// tlsOpts, err := tlsopts.NewOptions(self.Identity, tlsopts.Config{}, nil)
// require.NoError(t, err)
//
// self.Transport = transport.NewClientWithTimeouts(tlsOpts, transport.Timeouts{
// Dial: 20 * time.Millisecond,
// })
//
// network := &transport.SimulatedNetwork{
// DialLatency: 200 * time.Second,
// BytesPerSecond: 1 * memory.KB,
// }
//
// slowClient := network.NewClient(self.Transport)
// require.NotNil(t, slowClient)
//
// dialer := kademliaclient.NewDialer(zaptest.NewLogger(t), slowClient)
// defer ctx.Check(dialer.Close)
//
// var group errgroup.Group
// defer ctx.Check(group.Wait)
//
// for _, peer := range peers {
// peer := peer
// group.Go(func() error {
// for _, target := range peers {
// selfnode := self.Local().Node
// _, err := dialer.Lookup(ctx, &selfnode, peer.Local().Node, target.Local().Node.Id, self.Kademlia.RoutingTable.K())
// if !transport.Error.Has(err) || errs.Unwrap(err) != context.DeadlineExceeded {
// return errs.New("invalid error: %v (peer:%s target:%s)", err, peer.ID(), target.ID())
// }
// }
// return nil
// })
// }
// }()
//}
//
//func containsResult(nodes []*pb.Node, target storj.NodeID) bool {
// for _, node := range nodes {
// if node.Id == target {
// return true
// }
// }
// return false
//}

41
pkg/kademlia/lookup_test.go
View File

@ -1,41 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package kademlia_test
import (
"testing"
"github.com/stretchr/testify/assert"
"storj.io/storj/internal/testcontext"
"storj.io/storj/internal/testplanet"
)
func TestLookupNodes(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
planet, err := testplanet.New(t, 1, 8, 0)
if err != nil {
t.Fatal(err)
}
defer ctx.Check(planet.Shutdown)
planet.Start(ctx)
k := planet.Satellites[0].Kademlia.Service
k.WaitForBootstrap() // redundant, but leaving here to be clear
seen, err := k.DumpNodes(ctx)
if err != nil {
t.Fatal(err)
}
assert.NotEqual(t, len(seen), 0)
assert.NotNil(t, seen)
target := seen[0]
found, err := k.FindNode(ctx, target.Id)
assert.NoError(t, err)
assert.Equal(t, target.Id, found.Id)
}

98
pkg/kademlia/merge_test.go
View File

@ -1,98 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package kademlia_test
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
"golang.org/x/sync/errgroup"
"storj.io/storj/bootstrap"
"storj.io/storj/internal/testcontext"
"storj.io/storj/internal/testplanet"
"storj.io/storj/storagenode"
)
func TestMergePlanets(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
log := zaptest.NewLogger(t)
alpha, err := testplanet.NewCustom(log.Named("A"), testplanet.Config{
SatelliteCount: 2,
StorageNodeCount: 5,
})
require.NoError(t, err)
beta, err := testplanet.NewCustom(log.Named("B"), testplanet.Config{
SatelliteCount: 2,
StorageNodeCount: 5,
Identities: alpha.Identities(), // avoid using the same pregenerated identities
Reconfigure: testplanet.Reconfigure{
Bootstrap: func(index int, config *bootstrap.Config) {
config.Kademlia.BootstrapAddr = alpha.Bootstrap.Addr()
},
},
})
require.NoError(t, err)
defer ctx.Check(alpha.Shutdown)
defer ctx.Check(beta.Shutdown)
// during planet.Start
// every satellite & storage node looks itself up from bootstrap
// every storage node pings bootstrap
// every satellite pings every storage node
alpha.Start(ctx)
beta.Start(ctx)
allSatellites := []*testplanet.SatelliteSystem{}
allSatellites = append(allSatellites, alpha.Satellites...)
allSatellites = append(allSatellites, beta.Satellites...)
// make satellites refresh buckets 10 times
var group errgroup.Group
for _, satellite := range allSatellites {
satellite := satellite
group.Go(func() error {
satellite.Kademlia.Service.SetBucketRefreshThreshold(0)
for i := 0; i < 2; i++ {
satellite.Kademlia.Service.RefreshBuckets.TriggerWait()
}
return nil
})
}
_ = group.Wait()
test := func(tag string, satellites []*testplanet.SatelliteSystem, storageNodes []*storagenode.Peer) string {
found, missing := 0, 0
for _, satellite := range satellites {
for _, storageNode := range storageNodes {
node, err := satellite.Overlay.Service.Get(ctx, storageNode.ID())
if assert.NoError(t, err, tag) {
found++
assert.Equal(t, storageNode.Addr(), node.Address.Address, tag)
} else {
missing++
}
}
}
return fmt.Sprintf("%s: Found %v out of %v (missing %v)", tag, found, found+missing, missing)
}
sumAA := test("A-A", alpha.Satellites, alpha.StorageNodes)
sumAB := test("A-B", alpha.Satellites, beta.StorageNodes)
sumBB := test("B-B", beta.Satellites, beta.StorageNodes)
sumBA := test("B-A", beta.Satellites, alpha.StorageNodes)
t.Log(sumAA)
t.Log(sumAB)
t.Log(sumBB)
t.Log(sumBA)
}

6
pkg/pb/contact.proto
View File

@ -28,8 +28,6 @@ message CheckInResponse {
string ping_error_message = 2;
}
message ContactPingRequest {
}
message ContactPingRequest {}
message ContactPingResponse {
}
message ContactPingResponse {}

1
satellite/audit/verifier_test.go
View File

@ -398,7 +398,6 @@ func TestVerifierOfflineNode(t *testing.T) {
queue := audits.Queue
audits.Worker.Loop.Pause()
satellite.Discovery.Service.Discovery.Pause()
ul := planet.Uplinks[0]
testData := testrand.Bytes(8 * memory.KiB)

17
satellite/contact/contact_test.go
View File

@ -51,3 +51,20 @@ func TestSatelliteContactEndpoint(t *testing.T) {
require.Equal(t, ident.PeerIdentity(), peerID)
})
}
func TestFetchInfo(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1, StorageNodeCount: 1, UplinkCount: 0,
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
nodeDossier := planet.StorageNodes[0].Local()
node := pb.Node{Id: nodeDossier.Id, Address: nodeDossier.Address}
resp, err := planet.Satellites[0].Contact.Service.FetchInfo(ctx, node)
require.NotNil(t, resp)
require.NoError(t, err)
require.Equal(t, nodeDossier.Type, resp.Type)
require.Equal(t, &nodeDossier.Operator, resp.Operator)
require.Equal(t, &nodeDossier.Capacity, resp.Capacity)
require.Equal(t, nodeDossier.Version.GetVersion(), resp.Version.GetVersion())
})
}

42
satellite/contact/kademlia.go Normal file
View File

@ -0,0 +1,42 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package contact
import (
"context"
"go.uber.org/zap"
"storj.io/storj/pkg/pb"
)
// KademliaEndpoint implements the NodesServer Interface for backwards compatibility
type KademliaEndpoint struct {
log *zap.Logger
}
// NewKademliaEndpoint returns a new endpoint
func NewKademliaEndpoint(log *zap.Logger) *KademliaEndpoint {
return &KademliaEndpoint{
log: log,
}
}
// Query is a node to node communication query
func (endpoint *KademliaEndpoint) Query(ctx context.Context, req *pb.QueryRequest) (_ *pb.QueryResponse, err error) {
defer mon.Task()(&ctx)(&err)
return &pb.QueryResponse{}, nil
}
// Ping provides an easy way to verify a node is online and accepting requests
func (endpoint *KademliaEndpoint) Ping(ctx context.Context, req *pb.PingRequest) (_ *pb.PingResponse, err error) {
defer mon.Task()(&ctx)(&err)
return &pb.PingResponse{}, nil
}
// RequestInfo returns the node info
func (endpoint *KademliaEndpoint) RequestInfo(ctx context.Context, req *pb.InfoRequest) (_ *pb.InfoResponse, err error) {
defer mon.Task()(&ctx)(&err)
return &pb.InfoResponse{}, nil
}

62
satellite/contact/service.go
View File

@ -4,10 +4,15 @@
package contact
import (
"context"
"sync"
"github.com/zeebo/errs"
"go.uber.org/zap"
"google.golang.org/grpc"
"gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/transport"
"storj.io/storj/satellite/overlay"
)
@ -17,24 +22,77 @@ var Error = errs.Class("contact")
var mon = monkit.Package()
// Config contains configurable values for contact service
type Config struct {
ExternalAddress string `user:"true" help:"the public address of the node, useful for nodes behind NAT" default:""`
}
// Conn represents a connection
type Conn struct {
conn *grpc.ClientConn
client pb.NodesClient
}
// Service is the contact service between storage nodes and satellites.
// It is responsible for updating general node information like address, capacity, and uptime.
// It is also responsible for updating peer identity information for verifying signatures from that node.
//
// architecture: Service
type Service struct {
log *zap.Logger
log *zap.Logger
mutex sync.Mutex
self *overlay.NodeDossier
overlay *overlay.Service
peerIDs overlay.PeerIdentities
transport transport.Client
}
// NewService creates a new contact service.
func NewService(log *zap.Logger, overlay *overlay.Service, peerIDs overlay.PeerIdentities, transport transport.Client) *Service {
func NewService(log *zap.Logger, self *overlay.NodeDossier, overlay *overlay.Service, peerIDs overlay.PeerIdentities, transport transport.Client) *Service {
return &Service{
log: log,
self: self,
overlay: overlay,
peerIDs: peerIDs,
transport: transport,
}
}
// Local returns the satellite node dossier
func (service *Service) Local() overlay.NodeDossier {
service.mutex.Lock()
defer service.mutex.Unlock()
return *service.self
}
// FetchInfo connects to a node and returns its node info.
func (service *Service) FetchInfo(ctx context.Context, target pb.Node) (_ *pb.InfoResponse, err error) {
conn, err := service.dialNode(ctx, target)
if err != nil {
return nil, err
}
resp, err := conn.client.RequestInfo(ctx, &pb.InfoRequest{})
return resp, errs.Combine(err, conn.close())
}
// dialNode dials the specified node.
func (service *Service) dialNode(ctx context.Context, target pb.Node) (_ *Conn, err error) {
defer mon.Task()(&ctx)(&err)
grpcconn, err := service.transport.DialNode(ctx, &target)
return &Conn{
conn: grpcconn,
client: pb.NewNodesClient(grpcconn),
}, err
}
// close disconnects this connection.
func (conn *Conn) close() error {
return conn.conn.Close()
}
// Close closes resources
func (service *Service) Close() error { return nil }

60
satellite/discovery/service.go
View File

@ -5,7 +5,6 @@ package discovery
import (
"context"
"crypto/rand"
"time"
"github.com/zeebo/errs"
@ -14,8 +13,7 @@ import (
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/internal/sync2"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/storj"
"storj.io/storj/satellite/contact"
"storj.io/storj/satellite/overlay"
)
@ -29,7 +27,6 @@ var (
// Config loads on the configuration values for the cache
type Config struct {
RefreshInterval time.Duration `help:"the interval at which the cache refreshes itself in seconds" default:"1s"`
DiscoveryInterval time.Duration `help:"the interval at which the satellite attempts to find new nodes via random node ID lookups" default:"1s"`
RefreshLimit int `help:"the amount of nodes read from the overlay in a single pagination call" default:"100"`
RefreshConcurrency int `help:"the amount of nodes refreshed in parallel" default:"8"`
}
@ -38,38 +35,33 @@ type Config struct {
//
// architecture: Chore
type Discovery struct {
log *zap.Logger
cache *overlay.Service
kad *kademlia.Kademlia
log *zap.Logger
cache *overlay.Service
contact *contact.Service
refreshLimit int
refreshConcurrency int
Refresh sync2.Cycle
Discovery sync2.Cycle
Refresh sync2.Cycle
}
// New returns a new discovery service.
func New(logger *zap.Logger, ol *overlay.Service, kad *kademlia.Kademlia, config Config) *Discovery {
func New(logger *zap.Logger, ol *overlay.Service, contact *contact.Service, config Config) *Discovery {
discovery := &Discovery{
log: logger,
cache: ol,
kad: kad,
log: logger,
cache: ol,
contact: contact,
refreshLimit: config.RefreshLimit,
refreshConcurrency: config.RefreshConcurrency,
}
discovery.Refresh.SetInterval(config.RefreshInterval)
discovery.Discovery.SetInterval(config.DiscoveryInterval)
return discovery
}
// Close closes resources
func (discovery *Discovery) Close() error {
discovery.Refresh.Close()
discovery.Discovery.Close()
return nil
}
@ -85,13 +77,6 @@ func (discovery *Discovery) Run(ctx context.Context) (err error) {
}
return nil
})
discovery.Discovery.Start(ctx, &group, func(ctx context.Context) error {
err := discovery.discover(ctx)
if err != nil {
discovery.log.Error("error with cache discovery: ", zap.Error(err))
}
return nil
})
return group.Wait()
}
@ -119,8 +104,7 @@ func (discovery *Discovery) refresh(ctx context.Context) (err error) {
node := node
limiter.Go(ctx, func() {
// NB: FetchInfo updates node uptime already
info, err := discovery.kad.FetchInfo(ctx, *node)
info, err := discovery.contact.FetchInfo(ctx, *node)
if ctx.Err() != nil {
return
}
@ -144,27 +128,3 @@ func (discovery *Discovery) refresh(ctx context.Context) (err error) {
limiter.Wait()
return nil
}
// Discovery runs lookups for random node ID's to find new nodes in the network
func (discovery *Discovery) discover(ctx context.Context) (err error) {
defer mon.Task()(&ctx)(&err)
r, err := randomID()
if err != nil {
return Error.Wrap(err)
}
_, err = discovery.kad.FindNode(ctx, r)
if err != nil && !kademlia.NodeNotFound.Has(err) {
return Error.Wrap(err)
}
return nil
}
func randomID() (storj.NodeID, error) {
b := make([]byte, 32)
_, err := rand.Read(b)
if err != nil {
return storj.NodeID{}, Error.Wrap(err)
}
return storj.NodeIDFromBytes(b)
}

33
satellite/discovery/service_test.go
View File

@ -7,7 +7,6 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"storj.io/storj/internal/testcontext"
"storj.io/storj/internal/testplanet"
@ -26,35 +25,3 @@ func TestCache_Refresh(t *testing.T) {
}
})
}
func TestCache_Discovery(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1, StorageNodeCount: 1, UplinkCount: 0,
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
satellite := planet.Satellites[0]
testnode := planet.StorageNodes[0]
offlineID := testnode.ID()
satellite.Kademlia.Service.RefreshBuckets.Pause()
satellite.Discovery.Service.Refresh.Pause()
satellite.Discovery.Service.Discovery.Pause()
overlay := satellite.Overlay.Service
// mark node as offline in overlay
_, err := overlay.UpdateUptime(ctx, offlineID, false)
require.NoError(t, err)
node, err := overlay.Get(ctx, offlineID)
assert.NoError(t, err)
assert.False(t, overlay.IsOnline(node))
satellite.Discovery.Service.Discovery.TriggerWait()
found, err := overlay.Get(ctx, offlineID)
assert.NoError(t, err)
assert.Equal(t, offlineID, found.Id)
assert.True(t, overlay.IsOnline(found))
})
}

115
satellite/peer.go
View File

@ -5,12 +5,9 @@ package satellite
import (
"context"
"fmt"
"net"
"net/mail"
"net/smtp"
"os"
"path/filepath"
"github.com/zeebo/errs"
"go.uber.org/zap"
@ -23,7 +20,6 @@ import (
"storj.io/storj/internal/version"
"storj.io/storj/pkg/auth/grpcauth"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/peertls/tlsopts"
@ -61,8 +57,6 @@ import (
"storj.io/storj/satellite/repair/repairer"
"storj.io/storj/satellite/rewards"
"storj.io/storj/satellite/vouchers"
"storj.io/storj/storage"
"storj.io/storj/storage/boltdb"
)
var mon = monkit.Package()
@ -112,7 +106,7 @@ type Config struct {
Identity identity.Config
Server server.Config
Kademlia kademlia.Config
Contact contact.Config
Overlay overlay.Config
Discovery discovery.Config
@ -155,18 +149,10 @@ type Peer struct {
Version *version.Service
// services and endpoints
Kademlia struct {
kdb, ndb, adb storage.KeyValueStore // TODO: move these into DB
RoutingTable *kademlia.RoutingTable
Service *kademlia.Kademlia
Endpoint *kademlia.Endpoint
Inspector *kademlia.Inspector
}
Contact struct {
Service *contact.Service
Endpoint *contact.Endpoint
Service *contact.Service
Endpoint *contact.Endpoint
KEndpoint *contact.KademliaEndpoint
}
Overlay struct {
@ -295,19 +281,17 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB, revocationDB exten
peer.Overlay.DB = overlay.NewCombinedCache(peer.DB.OverlayCache())
peer.Overlay.Service = overlay.NewService(peer.Log.Named("overlay"), peer.Overlay.DB, config.Overlay)
peer.Transport = peer.Transport.WithObservers(peer.Overlay.Service)
peer.Overlay.Inspector = overlay.NewInspector(peer.Overlay.Service)
pb.RegisterOverlayInspectorServer(peer.Server.PrivateGRPC(), peer.Overlay.Inspector)
pb.DRPCRegisterOverlayInspector(peer.Server.PrivateDRPC(), peer.Overlay.Inspector)
}
{ // setup kademlia
log.Debug("Setting up Kademlia")
config := config.Kademlia
// TODO: move this setup logic into kademlia package
if config.ExternalAddress == "" {
config.ExternalAddress = peer.Addr()
{ // setup contact service
log.Debug("Setting up contact service")
c := config.Contact
if c.ExternalAddress == "" {
c.ExternalAddress = peer.Addr()
}
pbVersion, err := versionInfo.Proto()
@ -319,67 +303,25 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB, revocationDB exten
Node: pb.Node{
Id: peer.ID(),
Address: &pb.NodeAddress{
Address: config.ExternalAddress,
Address: c.ExternalAddress,
},
},
Type: pb.NodeType_SATELLITE,
Operator: pb.NodeOperator{
Email: config.Operator.Email,
Wallet: config.Operator.Wallet,
},
Type: pb.NodeType_SATELLITE,
Version: *pbVersion,
}
{ // setup routing table
// TODO: clean this up, should be part of database
log.Debug("Setting up routing table")
bucketIdentifier := peer.ID().String()[:5] // need a way to differentiate between nodes if running more than one simultaneously
dbpath := filepath.Join(config.DBPath, fmt.Sprintf("kademlia_%s.db", bucketIdentifier))
if err := os.MkdirAll(config.DBPath, 0777); err != nil && !os.IsExist(err) {
return nil, err
}
dbs, err := boltdb.NewShared(dbpath, kademlia.KademliaBucket, kademlia.NodeBucket, kademlia.AntechamberBucket)
if err != nil {
return nil, errs.Combine(err, peer.Close())
}
peer.Kademlia.kdb, peer.Kademlia.ndb, peer.Kademlia.adb = dbs[0], dbs[1], dbs[2]
peer.Kademlia.RoutingTable, err = kademlia.NewRoutingTable(peer.Log.Named("routing"), self, peer.Kademlia.kdb, peer.Kademlia.ndb, peer.Kademlia.adb, &config.RoutingTableConfig)
if err != nil {
return nil, errs.Combine(err, peer.Close())
}
peer.Transport = peer.Transport.WithObservers(peer.Kademlia.RoutingTable)
}
peer.Kademlia.Service, err = kademlia.NewService(peer.Log.Named("kademlia"), peer.Transport, peer.Kademlia.RoutingTable, config)
if err != nil {
return nil, errs.Combine(err, peer.Close())
}
peer.Kademlia.Endpoint = kademlia.NewEndpoint(peer.Log.Named("kademlia:endpoint"), peer.Kademlia.Service, nil, peer.Kademlia.RoutingTable, nil)
pb.RegisterNodesServer(peer.Server.GRPC(), peer.Kademlia.Endpoint)
pb.DRPCRegisterNodes(peer.Server.DRPC(), peer.Kademlia.Endpoint)
peer.Kademlia.Inspector = kademlia.NewInspector(peer.Kademlia.Service, peer.Identity)
pb.RegisterKadInspectorServer(peer.Server.PrivateGRPC(), peer.Kademlia.Inspector)
pb.DRPCRegisterKadInspector(peer.Server.PrivateDRPC(), peer.Kademlia.Inspector)
}
{ // setup contact service
log.Debug("Setting up contact service")
peer.Contact.Service = contact.NewService(peer.Log.Named("contact"), peer.Overlay.Service, peer.DB.PeerIdentities(), peer.Transport)
peer.Contact.Service = contact.NewService(peer.Log.Named("contact:service"), self, peer.Overlay.Service, peer.DB.PeerIdentities(), peer.Transport)
peer.Contact.Endpoint = contact.NewEndpoint(peer.Log.Named("contact:endpoint"), peer.Contact.Service)
peer.Contact.KEndpoint = contact.NewKademliaEndpoint(peer.Log.Named("contact:nodes_service_endpoint"))
pb.RegisterNodeServer(peer.Server.GRPC(), peer.Contact.Endpoint)
pb.RegisterNodesServer(peer.Server.GRPC(), peer.Contact.KEndpoint)
pb.DRPCRegisterNode(peer.Server.DRPC(), peer.Contact.Endpoint)
pb.DRPCRegisterNodes(peer.Server.DRPC(), peer.Contact.KEndpoint)
}
{ // setup discovery
log.Debug("Setting up discovery")
config := config.Discovery
peer.Discovery.Service = discovery.New(peer.Log.Named("discovery"), peer.Overlay.Service, peer.Kademlia.Service, config)
peer.Discovery.Service = discovery.New(peer.Log.Named("discovery"), peer.Overlay.Service, peer.Contact.Service, config)
}
{ // setup vouchers
@ -424,7 +366,7 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB, revocationDB exten
config.Orders.Expiration,
&pb.NodeAddress{
Transport: pb.NodeTransport_TCP_TLS_GRPC,
Address: config.Kademlia.ExternalAddress,
Address: config.Contact.ExternalAddress,
},
config.Repairer.MaxExcessRateOptimalThreshold,
)
@ -731,12 +673,6 @@ func (peer *Peer) Run(ctx context.Context) (err error) {
group.Go(func() error {
return errs2.IgnoreCanceled(peer.Version.Run(ctx))
})
group.Go(func() error {
return errs2.IgnoreCanceled(peer.Kademlia.Service.Bootstrap(ctx))
})
group.Go(func() error {
return errs2.IgnoreCanceled(peer.Kademlia.Service.Run(ctx))
})
group.Go(func() error {
return errs2.IgnoreCanceled(peer.Discovery.Service.Run(ctx))
})
@ -846,24 +782,13 @@ func (peer *Peer) Close() error {
errlist.Add(peer.Discovery.Service.Close())
}
// TODO: add kademlia.Endpoint for consistency
if peer.Kademlia.Service != nil {
errlist.Add(peer.Kademlia.Service.Close())
if peer.Contact.Service != nil {
errlist.Add(peer.Contact.Service.Close())
}
if peer.Kademlia.RoutingTable != nil {
errlist.Add(peer.Kademlia.RoutingTable.Close())
}
if peer.Overlay.Service != nil {
errlist.Add(peer.Overlay.Service.Close())
}
if peer.Kademlia.ndb != nil || peer.Kademlia.kdb != nil || peer.Kademlia.adb != nil {
errlist.Add(peer.Kademlia.kdb.Close())
errlist.Add(peer.Kademlia.ndb.Close())
errlist.Add(peer.Kademlia.adb.Close())
}
return errlist.Err()
}
@ -871,7 +796,7 @@ func (peer *Peer) Close() error {
func (peer *Peer) ID() storj.NodeID { return peer.Identity.ID }
// Local returns the peer local node info.
func (peer *Peer) Local() overlay.NodeDossier { return peer.Kademlia.RoutingTable.Local() }
func (peer *Peer) Local() overlay.NodeDossier { return peer.Contact.Service.Local() }
// Addr returns the public address.
func (peer *Peer) Addr() string { return peer.Server.Addr().String() }

6
satellite/repair/repair_test.go
View File

@ -52,7 +52,6 @@ func TestDataRepair(t *testing.T) {
uplinkPeer := planet.Uplinks[0]
satellite := planet.Satellites[0]
// stop discovery service so that we do not get a race condition when we delete nodes from overlay
satellite.Discovery.Service.Discovery.Stop()
satellite.Discovery.Service.Refresh.Stop()
// stop audit to prevent possible interactions i.e. repair timeout problems
satellite.Audit.Worker.Loop.Pause()
@ -188,7 +187,6 @@ func TestCorruptDataRepair_Failed(t *testing.T) {
uplinkPeer := planet.Uplinks[0]
satellite := planet.Satellites[0]
// stop discovery service so that we do not get a race condition when we delete nodes from overlay
satellite.Discovery.Service.Discovery.Stop()
satellite.Discovery.Service.Refresh.Stop()
// stop audit to prevent possible interactions i.e. repair timeout problems
satellite.Audit.Worker.Loop.Pause()
@ -311,7 +309,6 @@ func TestCorruptDataRepair_Succeed(t *testing.T) {
uplinkPeer := planet.Uplinks[0]
satellite := planet.Satellites[0]
// stop discovery service so that we do not get a race condition when we delete nodes from overlay
satellite.Discovery.Service.Discovery.Stop()
satellite.Discovery.Service.Refresh.Stop()
// stop audit to prevent possible interactions i.e. repair timeout problems
satellite.Audit.Worker.Loop.Pause()
@ -428,7 +425,6 @@ func TestRemoveIrreparableSegmentFromQueue(t *testing.T) {
uplinkPeer := planet.Uplinks[0]
satellite := planet.Satellites[0]
// stop discovery service so that we do not get a race condition when we delete nodes from overlay
satellite.Discovery.Service.Discovery.Stop()
satellite.Discovery.Service.Refresh.Stop()
// stop audit to prevent possible interactions i.e. repair timeout problems
satellite.Audit.Worker.Loop.Stop()
@ -513,7 +509,6 @@ func TestRepairMultipleDisqualified(t *testing.T) {
uplinkPeer := planet.Uplinks[0]
satellite := planet.Satellites[0]
// stop discovery service so that we do not get a race condition when we delete nodes from overlay
satellite.Discovery.Service.Discovery.Stop()
satellite.Discovery.Service.Refresh.Stop()
satellite.Repair.Checker.Loop.Pause()
@ -629,7 +624,6 @@ func TestDataRepairUploadLimit(t *testing.T) {
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
satellite := planet.Satellites[0]
// stop discovery service so that we do not get a race condition when we delete nodes from overlay
satellite.Discovery.Service.Discovery.Stop()
satellite.Discovery.Service.Refresh.Stop()
// stop audit to prevent possible interactions i.e. repair timeout problems
satellite.Audit.Worker.Loop.Pause()

7
scripts/test-sim-backwards.sh
View File

@ -50,6 +50,13 @@ PATH=$RELEASE_DIR/bin:$PATH storj-sim -x --host $STORJ_NETWORK_HOST4 network tes
# this replaces anywhere that has "/release/" in the config file, which currently just renames the static dir paths
sed -i -e 's#/release/#/branch/#g' `storj-sim network env SATELLITE_0_DIR`/config.yaml
# remove kademlia from config
sed -i -e 's/kademlia.//g' `storj-sim network env STORAGENODE_5_DIR`/config.yaml
sed -i -e 's/kademlia.//g' `storj-sim network env STORAGENODE_6_DIR`/config.yaml
sed -i -e 's/kademlia.//g' `storj-sim network env STORAGENODE_7_DIR`/config.yaml
sed -i -e 's/kademlia.//g' `storj-sim network env STORAGENODE_8_DIR`/config.yaml
sed -i -e 's/kademlia.//g' `storj-sim network env STORAGENODE_9_DIR`/config.yaml
## Ensure that partially upgraded network works
# keep half of the storage nodes on the old version

36
scripts/testdata/satellite-config.yaml.lock vendored
View File

@ -49,6 +49,9 @@
# stripe api key
# console.stripe-key: ""
# the public address of the node, useful for nodes behind NAT
contact.external-address: ""
# satellite database connection string
# database: postgres://
@ -70,9 +73,6 @@
# If set, a path to write a process trace SVG to
# debug.trace-out: ""
# the interval at which the satellite attempts to find new nodes via random node ID lookups
# discovery.discovery-interval: 1s
# the amount of nodes refreshed in parallel
# discovery.refresh-concurrency: 8
@ -103,36 +103,6 @@ identity.cert-path: /root/.local/share/storj/identity/satellite/identity.cert
# path to the private key for this identity
identity.key-path: /root/.local/share/storj/identity/satellite/identity.key
# alpha is a system wide concurrency parameter
# kademlia.alpha: 5
# the Kademlia node to bootstrap against
# kademlia.bootstrap-addr: bootstrap.storj.io:8888
# the base interval to wait when retrying bootstrap
# kademlia.bootstrap-backoff-base: 1s
# the maximum amount of time to wait when retrying bootstrap
# kademlia.bootstrap-backoff-max: 30s
# size of each Kademlia bucket
# kademlia.bucket-size: 20
# the path for storage node db services to be created on
# kademlia.db-path: testdata/kademlia
# the public address of the Kademlia node, useful for nodes behind NAT
kademlia.external-address: ""
# operator email address
kademlia.operator.email: ""
# operator wallet address
kademlia.operator.wallet: ""
# size of Kademlia replacement cache
# kademlia.replacement-cache-size: 5
# what to use for storing real-time accounting data
# live-accounting.storage-backend: plainmemory

26
storagenode/console/service.go
View File

@ -34,28 +34,29 @@ var (
//
// architecture: Service
type Service struct {
log *zap.Logger
log *zap.Logger
trust *trust.Pool
bandwidthDB bandwidth.DB
reputationDB reputation.DB
storageUsageDB storageusage.DB
pieceStore *pieces.Store
version *version.Service
pingStats *contact.PingStats
contact *contact.Service
version *version.Service
pingStats *contact.PingStats
allocatedBandwidth memory.Size
allocatedDiskSpace memory.Size
nodeID storj.NodeID
walletAddress string
startedAt time.Time
versionInfo version.Info
walletAddress string
startedAt time.Time
versionInfo version.Info
}
// NewService returns new instance of Service.
func NewService(log *zap.Logger, bandwidth bandwidth.DB, pieceStore *pieces.Store, version *version.Service,
allocatedBandwidth, allocatedDiskSpace memory.Size, walletAddress string, versionInfo version.Info, trust *trust.Pool,
reputationDB reputation.DB, storageUsageDB storageusage.DB, pingStats *contact.PingStats, myNodeID storj.NodeID) (*Service, error) {
reputationDB reputation.DB, storageUsageDB storageusage.DB, pingStats *contact.PingStats, contact *contact.Service) (*Service, error) {
if log == nil {
return nil, errs.New("log can't be nil")
}
@ -76,6 +77,9 @@ func NewService(log *zap.Logger, bandwidth bandwidth.DB, pieceStore *pieces.Stor
return nil, errs.New("pingStats can't be nil")
}
if contact == nil {
return nil, errs.New("contact service can't be nil")
}
return &Service{
log: log,
trust: trust,
@ -87,7 +91,7 @@ func NewService(log *zap.Logger, bandwidth bandwidth.DB, pieceStore *pieces.Stor
pingStats: pingStats,
allocatedBandwidth: allocatedBandwidth,
allocatedDiskSpace: allocatedDiskSpace,
nodeID: myNodeID,
contact: contact,
walletAddress: walletAddress,
startedAt: time.Now(),
versionInfo: versionInfo,
@ -123,7 +127,7 @@ func (s *Service) GetDashboardData(ctx context.Context) (_ *Dashboard, err error
defer mon.Task()(&ctx)(&err)
data := new(Dashboard)
data.NodeID = s.nodeID
data.NodeID = s.contact.Local().Id
data.Wallet = s.walletAddress
data.Version = s.versionInfo.Version
data.UpToDate = s.version.IsAllowed()

20
storagenode/contact/chore.go
View File

@ -11,30 +11,19 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"
"gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/internal/sync2"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/transport"
"storj.io/storj/storagenode/trust"
)
var mon = monkit.Package()
// Config contains configurable parameters for contact chore
type Config struct {
Interval time.Duration `help:"how frequently the node contact chore should run" releaseDefault:"1h" devDefault:"30s"`
// MaxSleep should remain at default value to decrease traffic congestion to satellite
MaxSleep time.Duration `help:"maximum duration to wait before pinging satellites" releaseDefault:"45m" devDefault:"0s" hidden:"true"`
}
// Chore is the contact chore for nodes announcing themselves to their trusted satellites
//
// architecture: Chore
type Chore struct {
log *zap.Logger
rt *kademlia.RoutingTable
service *Service
transport transport.Client
trust *trust.Pool
@ -44,10 +33,10 @@ type Chore struct {
}
// NewChore creates a new contact chore
func NewChore(log *zap.Logger, interval time.Duration, maxSleep time.Duration, trust *trust.Pool, transport transport.Client, rt *kademlia.RoutingTable) *Chore {
func NewChore(log *zap.Logger, interval time.Duration, maxSleep time.Duration, trust *trust.Pool, transport transport.Client, service *Service) *Chore {
return &Chore{
log: log,
rt: rt,
service: service,
transport: transport,
trust: trust,
@ -75,9 +64,8 @@ func (chore *Chore) Run(ctx context.Context) (err error) {
func (chore *Chore) pingSatellites(ctx context.Context) (err error) {
defer mon.Task()(&ctx)(&err)
var group errgroup.Group
self := chore.rt.Local()
self := chore.service.Local()
satellites := chore.trust.GetSatellites(ctx)
for _, satellite := range satellites {
satellite := satellite

25
storagenode/contact/contact_test.go
View File

@ -40,7 +40,7 @@ func TestStoragenodeContactEndpoint(t *testing.T) {
})
}
func TestContactChore(t *testing.T) {
func TestNodeInfoUpdated(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1, StorageNodeCount: 1, UplinkCount: 0,
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
@ -60,7 +60,7 @@ func TestContactChore(t *testing.T) {
}
require.NotEqual(t, oldCapacity, newCapacity)
node.Kademlia.RoutingTable.UpdateSelf(&newCapacity)
node.Contact.Service.UpdateSelf(&newCapacity)
node.Contact.Chore.Loop.TriggerWait()
@ -74,3 +74,24 @@ func TestContactChore(t *testing.T) {
require.Equal(t, newCapacity, newInfo.Capacity)
})
}
func TestRequestInfoEndpoint(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1, StorageNodeCount: 1, UplinkCount: 0,
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
nodeDossier := planet.StorageNodes[0].Local()
// Satellite Trusted
conn, err := planet.Satellites[0].Transport.DialNode(ctx, &nodeDossier.Node)
require.NoError(t, err)
defer ctx.Check(conn.Close)
resp, err := pb.NewNodesClient(conn).RequestInfo(ctx, &pb.InfoRequest{})
require.NotNil(t, resp)
require.NoError(t, err)
require.Equal(t, nodeDossier.Type, resp.Type)
require.Equal(t, &nodeDossier.Operator, resp.Operator)
require.Equal(t, &nodeDossier.Capacity, resp.Capacity)
require.Equal(t, nodeDossier.Version.Version, resp.Version.Version)
})
}

76
storagenode/contact/kademlia.go Normal file
View File

@ -0,0 +1,76 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package contact
import (
"context"
"go.uber.org/zap"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/storj"
)
// SatelliteIDVerifier checks if the connection is from a trusted satellite
type SatelliteIDVerifier interface {
VerifySatelliteID(ctx context.Context, id storj.NodeID) error
}
// KademliaEndpoint implements the NodesServer Interface for backwards compatibility
type KademliaEndpoint struct {
log *zap.Logger
service *Service
trust SatelliteIDVerifier
}
// NewKademliaEndpoint returns a new endpoint
func NewKademliaEndpoint(log *zap.Logger, service *Service, trust SatelliteIDVerifier) *KademliaEndpoint {
return &KademliaEndpoint{
log: log,
service: service,
trust: trust,
}
}
// Query is a node to node communication query
func (endpoint *KademliaEndpoint) Query(ctx context.Context, req *pb.QueryRequest) (_ *pb.QueryResponse, err error) {
defer mon.Task()(&ctx)(&err)
return &pb.QueryResponse{}, nil
}
// Ping provides an easy way to verify a node is online and accepting requests
func (endpoint *KademliaEndpoint) Ping(ctx context.Context, req *pb.PingRequest) (_ *pb.PingResponse, err error) {
defer mon.Task()(&ctx)(&err)
return &pb.PingResponse{}, nil
}
// RequestInfo returns the node info
func (endpoint *KademliaEndpoint) RequestInfo(ctx context.Context, req *pb.InfoRequest) (_ *pb.InfoResponse, err error) {
defer mon.Task()(&ctx)(&err)
self := endpoint.service.Local()
if endpoint.trust == nil {
return nil, status.Error(codes.Internal, "missing trust")
}
peer, err := identity.PeerIdentityFromContext(ctx)
if err != nil {
return nil, status.Error(codes.Unauthenticated, err.Error())
}
err = endpoint.trust.VerifySatelliteID(ctx, peer.ID)
if err != nil {
return nil, status.Errorf(codes.PermissionDenied, "untrusted peer %v", peer.ID)
}
return &pb.InfoResponse{
Type: self.Type,
Operator: &self.Operator,
Capacity: &self.Capacity,
Version: &self.Version,
}, nil
}

64
storagenode/contact/service.go Normal file
View File

@ -0,0 +1,64 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package contact
import (
"sync"
"time"
"github.com/zeebo/errs"
"go.uber.org/zap"
"gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/pb"
"storj.io/storj/satellite/overlay"
)
// Error is the default error class for contact package
var Error = errs.Class("contact")
var mon = monkit.Package()
// Config contains configurable values for contact service
type Config struct {
ExternalAddress string `user:"true" help:"the public address of the node, useful for nodes behind NAT" default:""`
// Chore config values
Interval time.Duration `help:"how frequently the node contact chore should run" releaseDefault:"1h" devDefault:"30s"`
// MaxSleep should remain at default value to decrease traffic congestion to satellite
MaxSleep time.Duration `help:"maximum duration to wait before pinging satellites" releaseDefault:"45m" devDefault:"0s" hidden:"true"`
}
// Service is the contact service between storage nodes and satellites
type Service struct {
log *zap.Logger
mutex *sync.Mutex
self *overlay.NodeDossier
}
// NewService creates a new contact service
func NewService(log *zap.Logger, self *overlay.NodeDossier) *Service {
return &Service{
log: log,
mutex: &sync.Mutex{},
self: self,
}
}
// Local returns the storagenode node-dossier
func (service *Service) Local() overlay.NodeDossier {
service.mutex.Lock()
defer service.mutex.Unlock()
return *service.self
}
// UpdateSelf updates the local node with the capacity
func (service *Service) UpdateSelf(capacity *pb.NodeCapacity) {
service.mutex.Lock()
defer service.mutex.Unlock()
if capacity != nil {
service.self.Capacity = *capacity
}
}

34
storagenode/inspector/inspector.go
View File

@ -6,17 +6,14 @@ package inspector
import (
"context"
"net"
"strings"
"time"
"github.com/golang/protobuf/ptypes"
"github.com/zeebo/errs"
"go.uber.org/zap"
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/storj"
"storj.io/storj/storagenode/bandwidth"
"storj.io/storj/storagenode/contact"
"storj.io/storj/storagenode/pieces"
@ -36,34 +33,37 @@ var (
type Endpoint struct {
log *zap.Logger
pieceStore *pieces.Store
kademlia *kademlia.Kademlia
contact *contact.Service
pingStats *contact.PingStats
usageDB bandwidth.DB
startTime time.Time
pieceStoreConfig piecestore.OldConfig
dashboardAddress net.Addr
externalAddress string
}
// NewEndpoint creates piecestore inspector instance
func NewEndpoint(
log *zap.Logger,
pieceStore *pieces.Store,
kademlia *kademlia.Kademlia,
contact *contact.Service,
pingStats *contact.PingStats,
usageDB bandwidth.DB,
pieceStoreConfig piecestore.OldConfig,
dashbaordAddress net.Addr) *Endpoint {
dashbaordAddress net.Addr,
externalAddress string) *Endpoint {
return &Endpoint{
log: log,
pieceStore: pieceStore,
kademlia: kademlia,
contact: contact,
pingStats: pingStats,
usageDB: usageDB,
pieceStoreConfig: pieceStoreConfig,
dashboardAddress: dashbaordAddress,
startTime: time.Now(),
externalAddress: externalAddress,
}
}
@ -114,26 +114,12 @@ func (inspector *Endpoint) getDashboardData(ctx context.Context) (_ *pb.Dashboar
return &pb.DashboardResponse{}, Error.Wrap(err)
}
// TODO: querying all nodes is slow, find a more performant way to do this.
nodes, err := inspector.kademlia.FindNear(ctx, storj.NodeID{}, 10000000)
if err != nil {
return &pb.DashboardResponse{}, Error.Wrap(err)
}
bootstrapNodes := inspector.kademlia.GetBootstrapNodes()
bsNodes := make([]string, len(bootstrapNodes))
for i, node := range bootstrapNodes {
bsNodes[i] = node.Address.Address
}
lastPingedAt, lastPingFromID, lastPingFromAddress := inspector.pingStats.WhenLastPinged()
return &pb.DashboardResponse{
NodeId: inspector.kademlia.Local().Id,
NodeConnections: int64(len(nodes)),
BootstrapAddress: strings.Join(bsNodes, ", "),
NodeId: inspector.contact.Local().Id,
InternalAddress: "",
ExternalAddress: inspector.kademlia.Local().Address.Address,
ExternalAddress: inspector.contact.Local().Address.Address,
LastPinged: lastPingedAt,
LastPingFromId: &lastPingFromID,
LastPingFromAddress: lastPingFromAddress,

1
storagenode/inspector/inspector_test.go
View File

@ -135,7 +135,6 @@ func TestInspectorDashboard(t *testing.T) {
assert.True(t, response.Uptime.Nanos > 0)
assert.Equal(t, storageNode.ID(), response.NodeId)
assert.Equal(t, storageNode.Addr(), response.ExternalAddress)
assert.Equal(t, int64(len(planet.StorageNodes)+len(planet.Satellites)), response.NodeConnections)
assert.NotNil(t, response.Stats)
}
})

10
storagenode/monitor/monitor.go
View File

@ -13,9 +13,9 @@ import (
"storj.io/storj/internal/memory"
"storj.io/storj/internal/sync2"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/pb"
"storj.io/storj/storagenode/bandwidth"
"storj.io/storj/storagenode/contact"
"storj.io/storj/storagenode/pieces"
)
@ -38,8 +38,8 @@ type Config struct {
// architecture: Service
type Service struct {
log *zap.Logger
routingTable *kademlia.RoutingTable
store *pieces.Store
contact *contact.Service
usageDB bandwidth.DB
allocatedDiskSpace int64
allocatedBandwidth int64
@ -50,11 +50,11 @@ type Service struct {
// TODO: should it be responsible for monitoring actual bandwidth as well?
// NewService creates a new storage node monitoring service.
func NewService(log *zap.Logger, routingTable *kademlia.RoutingTable, store *pieces.Store, usageDB bandwidth.DB, allocatedDiskSpace, allocatedBandwidth int64, interval time.Duration, config Config) *Service {
func NewService(log *zap.Logger, store *pieces.Store, contact *contact.Service, usageDB bandwidth.DB, allocatedDiskSpace, allocatedBandwidth int64, interval time.Duration, config Config) *Service {
return &Service{
log: log,
routingTable: routingTable,
store: store,
contact: contact,
usageDB: usageDB,
allocatedDiskSpace: allocatedDiskSpace,
allocatedBandwidth: allocatedBandwidth,
@ -152,7 +152,7 @@ func (service *Service) updateNodeInformation(ctx context.Context) (err error) {
return Error.Wrap(err)
}
service.routingTable.UpdateSelf(&pb.NodeCapacity{
service.contact.UpdateSelf(&pb.NodeCapacity{
FreeBandwidth: service.allocatedBandwidth - usedBandwidth,
FreeDisk: service.allocatedDiskSpace - usedSpace,
})

4
storagenode/monitor/monitor_test.go
View File

@ -26,7 +26,7 @@ func TestMonitor(t *testing.T) {
for _, storageNode := range planet.StorageNodes {
storageNode.Storage2.Monitor.Loop.Pause()
info, err := satellite.Kademlia.Service.FetchInfo(ctx, storageNode.Local().Node)
info, err := satellite.Contact.Service.FetchInfo(ctx, storageNode.Local().Node)
require.NoError(t, err)
// assume that all storage nodes have the same initial values
@ -42,7 +42,7 @@ func TestMonitor(t *testing.T) {
for _, storageNode := range planet.StorageNodes {
storageNode.Storage2.Monitor.Loop.TriggerWait()
info, err := satellite.Kademlia.Service.FetchInfo(ctx, storageNode.Local().Node)
info, err := satellite.Contact.Service.FetchInfo(ctx, storageNode.Local().Node)
require.NoError(t, err)
stats, err := storageNode.Storage2.Inspector.Stats(ctx, &pb.StatsRequest{})

50
storagenode/operator.go Normal file
View File

@ -0,0 +1,50 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package storagenode
import (
"fmt"
"regexp"
"go.uber.org/zap"
)
// OperatorConfig defines properties related to storage node operator metadata
type OperatorConfig struct {
Email string `user:"true" help:"operator email address" default:""`
Wallet string `user:"true" help:"operator wallet address" default:""`
}
// Verify verifies whether operator config is valid.
func (c OperatorConfig) Verify(log *zap.Logger) error {
if err := isOperatorEmailValid(log, c.Email); err != nil {
return err
}
if err := isOperatorWalletValid(log, c.Wallet); err != nil {
return err
}
return nil
}
func isOperatorEmailValid(log *zap.Logger, email string) error {
if email == "" {
log.Sugar().Warn("Operator email address isn't specified.")
} else {
log.Sugar().Info("Operator email: ", email)
}
return nil
}
func isOperatorWalletValid(log *zap.Logger, wallet string) error {
if wallet == "" {
return fmt.Errorf("operator wallet address isn't specified")
}
r := regexp.MustCompile("^0x[a-fA-F0-9]{40}$")
if match := r.MatchString(wallet); !match {
return fmt.Errorf("operator wallet address isn't valid")
}
log.Sugar().Info("operator wallet: ", wallet)
return nil
}

105
storagenode/peer.go
View File

@ -10,12 +10,11 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/internal/errs2"
"storj.io/storj/internal/version"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/peertls/extensions"
"storj.io/storj/pkg/peertls/tlsopts"
@ -65,17 +64,16 @@ type DB interface {
UsedSerials() piecestore.UsedSerials
Reputation() reputation.DB
StorageUsage() storageusage.DB
// TODO: use better interfaces
RoutingTable() (kdb, ndb, adb storage.KeyValueStore)
}
// Config is all the configuration parameters for a Storage Node
type Config struct {
Identity identity.Config
Server server.Config
Kademlia kademlia.Config
Server server.Config
Contact contact.Config
Operator OperatorConfig
// TODO: flatten storage config and only keep the new one
Storage piecestore.OldConfig
@ -91,13 +89,11 @@ type Config struct {
Version version.Config
Bandwidth bandwidth.Config
Contact contact.Config
}
// Verify verifies whether configuration is consistent and acceptable.
func (config *Config) Verify(log *zap.Logger) error {
return config.Kademlia.Verify(log)
return config.Operator.Verify(log)
}
// Peer is the representation of a Storage Node.
@ -117,16 +113,12 @@ type Peer struct {
// services and endpoints
// TODO: similar grouping to satellite.Peer
Kademlia struct {
RoutingTable *kademlia.RoutingTable
Service *kademlia.Kademlia
Endpoint *kademlia.Endpoint
Inspector *kademlia.Inspector
}
Contact struct {
Endpoint *contact.Endpoint
Service *contact.Service
Chore *contact.Chore
Endpoint *contact.Endpoint
KEndpoint *contact.KademliaEndpoint
PingStats *contact.PingStats
}
@ -195,36 +187,29 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB, revocationDB exten
}
}
{ // setup trust pool before kademlia
{ // setup trust pool
peer.Storage2.Trust, err = trust.NewPool(peer.Transport, config.Storage.WhitelistedSatellites)
if err != nil {
return nil, errs.Combine(err, peer.Close())
}
}
// Set up Contact.PingStats before Kademlia (until Kademlia goes away, at which point this can
// be folded back in with the Contact setup block). Both services must share pointers to this
// PingStats instance for now.
peer.Contact.PingStats = &contact.PingStats{}
{ // setup kademlia
config := config.Kademlia
// TODO: move this setup logic into kademlia package
if config.ExternalAddress == "" {
config.ExternalAddress = peer.Addr()
{ // setup contact service
c := config.Contact
if c.ExternalAddress == "" {
c.ExternalAddress = peer.Addr()
}
pbVersion, err := versionInfo.Proto()
if err != nil {
return nil, errs.Combine(err, peer.Close())
}
self := &overlay.NodeDossier{
Node: pb.Node{
Id: peer.ID(),
Address: &pb.NodeAddress{
Transport: pb.NodeTransport_TCP_TLS_GRPC,
Address: config.ExternalAddress,
Address: c.ExternalAddress,
},
},
Type: pb.NodeType_STORAGE,
@ -234,34 +219,15 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB, revocationDB exten
},
Version: *pbVersion,
}
kdb, ndb, adb := peer.DB.RoutingTable()
peer.Kademlia.RoutingTable, err = kademlia.NewRoutingTable(peer.Log.Named("routing"), self, kdb, ndb, adb, &config.RoutingTableConfig)
if err != nil {
return nil, errs.Combine(err, peer.Close())
}
peer.Transport = peer.Transport.WithObservers(peer.Kademlia.RoutingTable)
peer.Kademlia.Service, err = kademlia.NewService(peer.Log.Named("kademlia"), peer.Transport, peer.Kademlia.RoutingTable, config)
if err != nil {
return nil, errs.Combine(err, peer.Close())
}
peer.Kademlia.Endpoint = kademlia.NewEndpoint(peer.Log.Named("kademlia:endpoint"), peer.Kademlia.Service, peer.Contact.PingStats, peer.Kademlia.RoutingTable, peer.Storage2.Trust)
pb.RegisterNodesServer(peer.Server.GRPC(), peer.Kademlia.Endpoint)
pb.DRPCRegisterNodes(peer.Server.DRPC(), peer.Kademlia.Endpoint)
peer.Kademlia.Inspector = kademlia.NewInspector(peer.Kademlia.Service, peer.Identity)
pb.RegisterKadInspectorServer(peer.Server.PrivateGRPC(), peer.Kademlia.Inspector)
pb.DRPCRegisterKadInspector(peer.Server.PrivateDRPC(), peer.Kademlia.Inspector)
}
{ // setup contact service
peer.Contact.Chore = contact.NewChore(peer.Log.Named("contact:chore"), config.Contact.Interval, config.Contact.MaxSleep, peer.Storage2.Trust, peer.Transport, peer.Kademlia.RoutingTable)
peer.Contact.PingStats = new(contact.PingStats)
peer.Contact.Service = contact.NewService(peer.Log.Named("contact:service"), self)
peer.Contact.Chore = contact.NewChore(peer.Log.Named("contact:chore"), config.Contact.Interval, config.Contact.MaxSleep, peer.Storage2.Trust, peer.Transport, peer.Contact.Service)
peer.Contact.Endpoint = contact.NewEndpoint(peer.Log.Named("contact:endpoint"), peer.Contact.PingStats)
peer.Contact.KEndpoint = contact.NewKademliaEndpoint(peer.Log.Named("contact:nodes_service_endpoint"), peer.Contact.Service, peer.Storage2.Trust)
pb.RegisterContactServer(peer.Server.GRPC(), peer.Contact.Endpoint)
pb.RegisterNodesServer(peer.Server.GRPC(), peer.Contact.KEndpoint)
pb.DRPCRegisterContact(peer.Server.DRPC(), peer.Contact.Endpoint)
pb.DRPCRegisterNodes(peer.Server.DRPC(), peer.Contact.KEndpoint)
}
{ // setup storage
@ -283,8 +249,8 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB, revocationDB exten
peer.Storage2.Monitor = monitor.NewService(
log.Named("piecestore:monitor"),
peer.Kademlia.RoutingTable,
peer.Storage2.Store,
peer.Contact.Service,
peer.DB.Bandwidth(),
config.Storage.AllocatedDiskSpace.Int64(),
config.Storage.AllocatedBandwidth.Int64(),
@ -363,13 +329,13 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB, revocationDB exten
peer.Version,
config.Storage.AllocatedBandwidth,
config.Storage.AllocatedDiskSpace,
config.Kademlia.Operator.Wallet,
config.Operator.Wallet,
versionInfo,
peer.Storage2.Trust,
peer.DB.Reputation(),
peer.DB.StorageUsage(),
peer.Contact.PingStats,
peer.Local().Id)
peer.Contact.Service)
if err != nil {
return nil, errs.Combine(err, peer.Close())
@ -392,11 +358,12 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB, revocationDB exten
peer.Storage2.Inspector = inspector.NewEndpoint(
peer.Log.Named("pieces:inspector"),
peer.Storage2.Store,
peer.Kademlia.Service,
peer.Contact.Service,
peer.Contact.PingStats,
peer.DB.Bandwidth(),
config.Storage,
peer.Console.Listener.Addr(),
config.Contact.ExternalAddress,
)
pb.RegisterPieceStoreInspectorServer(peer.Server.PrivateGRPC(), peer.Storage2.Inspector)
pb.DRPCRegisterPieceStoreInspector(peer.Server.PrivateDRPC(), peer.Storage2.Inspector)
@ -418,14 +385,9 @@ func (peer *Peer) Run(ctx context.Context) (err error) {
group.Go(func() error {
return errs2.IgnoreCanceled(peer.Version.Run(ctx))
})
group.Go(func() error {
return errs2.IgnoreCanceled(peer.Kademlia.Service.Bootstrap(ctx))
return errs2.IgnoreCanceled(peer.Contact.Chore.Run(ctx))
})
group.Go(func() error {
return errs2.IgnoreCanceled(peer.Kademlia.Service.Run(ctx))
})
group.Go(func() error {
return errs2.IgnoreCanceled(peer.Collector.Run(ctx))
})
@ -462,10 +424,6 @@ func (peer *Peer) Run(ctx context.Context) (err error) {
return errs2.IgnoreCanceled(peer.Console.Endpoint.Run(ctx))
})
group.Go(func() error {
return errs2.IgnoreCanceled(peer.Contact.Chore.Run(ctx))
})
return group.Wait()
}
@ -481,7 +439,6 @@ func (peer *Peer) Close() error {
}
// close services in reverse initialization order
if peer.Contact.Chore != nil {
errlist.Add(peer.Contact.Chore.Close())
}
@ -504,12 +461,6 @@ func (peer *Peer) Close() error {
errlist.Add(peer.Collector.Close())
}
if peer.Kademlia.Service != nil {
errlist.Add(peer.Kademlia.Service.Close())
}
if peer.Kademlia.RoutingTable != nil {
errlist.Add(peer.Kademlia.RoutingTable.Close())
}
if peer.Console.Endpoint != nil {
errlist.Add(peer.Console.Endpoint.Close())
} else if peer.Console.Listener != nil {
@ -527,7 +478,7 @@ func (peer *Peer) Close() error {
func (peer *Peer) ID() storj.NodeID { return peer.Identity.ID }
// Local returns the peer local node info.
func (peer *Peer) Local() overlay.NodeDossier { return peer.Kademlia.RoutingTable.Local() }
func (peer *Peer) Local() overlay.NodeDossier { return peer.Contact.Service.Local() }
// Addr returns the public address.
func (peer *Peer) Addr() string { return peer.Server.Addr().String() }

35
storagenode/storagenodedb/database.go
View File

@ -15,13 +15,11 @@ import (
_ "github.com/mattn/go-sqlite3" // used indirectly.
"github.com/zeebo/errs"
"go.uber.org/zap"
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/internal/dbutil"
"storj.io/storj/internal/migrate"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/storage"
"storj.io/storj/storage/boltdb"
"storj.io/storj/storage/filestore"
"storj.io/storj/storagenode"
"storj.io/storj/storagenode/bandwidth"
@ -69,10 +67,9 @@ type SQLDB interface {
// Config configures storage node database
type Config struct {
// TODO: figure out better names
Storage string
Info string
Info2 string
Kademlia string
Storage string
Info string
Info2 string
Pieces string
}
@ -99,8 +96,6 @@ type DB struct {
usedSerialsDB *usedSerialsDB
satellitesDB *satellitesDB
kdb, ndb, adb storage.KeyValueStore
sqlDatabases map[string]*sql.DB
}
@ -112,17 +107,9 @@ func New(log *zap.Logger, config Config) (*DB, error) {
}
pieces := filestore.New(log, piecesDir)
dbs, err := boltdb.NewShared(config.Kademlia, kademlia.KademliaBucket, kademlia.NodeBucket, kademlia.AntechamberBucket)
if err != nil {
return nil, err
}
db := &DB{
log: log,
pieces: pieces,
kdb: dbs[0],
ndb: dbs[1],
adb: dbs[2],
dbDirectory: filepath.Dir(config.Info2),
@ -156,6 +143,7 @@ func (db *DB) openDatabases() error {
if err != nil {
return errs.Combine(err, db.closeDatabases())
}
db.deprecatedInfoDB.Configure(deprecatedInfoDB)
db.bandwidthDB.Configure(deprecatedInfoDB)
db.ordersDB.Configure(deprecatedInfoDB)
@ -204,13 +192,7 @@ func (db *DB) CreateTables() error {
// Close closes any resources.
func (db *DB) Close() error {
return errs.Combine(
db.kdb.Close(),
db.ndb.Close(),
db.adb.Close(),
db.closeDatabases(),
)
return db.closeDatabases()
}
// closeDatabases closes all the SQLite database connections and removes them from the associated maps.
@ -283,11 +265,6 @@ func (db *DB) UsedSerials() piecestore.UsedSerials {
return db.usedSerialsDB
}
// RoutingTable returns kademlia routing table
func (db *DB) RoutingTable() (kdb, ndb, adb storage.KeyValueStore) {
return db.kdb, db.ndb, db.adb
}
// RawDatabases are required for testing purposes
func (db *DB) RawDatabases() map[string]SQLDB {
return map[string]SQLDB{

9
storagenode/storagenodedb/migrations_test.go
View File

@ -80,11 +80,10 @@ func TestMigrate(t *testing.T) {
storageDir := ctx.Dir("storage")
cfg := storagenodedb.Config{
Pieces: storageDir,
Storage: storageDir,
Info: filepath.Join(storageDir, "piecestore.db"),
Info2: filepath.Join(storageDir, "info.db"),
Kademlia: filepath.Join(storageDir, "kademlia"),
Pieces: storageDir,
Storage: storageDir,
Info: filepath.Join(storageDir, "piecestore.db"),
Info2: filepath.Join(storageDir, "info.db"),
}
// create a new satellitedb connection

9
storagenode/storagenodedb/storagenodedbtest/run.go
View File

@ -28,11 +28,10 @@ func Run(t *testing.T, test func(t *testing.T, db storagenode.DB)) {
storageDir := ctx.Dir("storage")
cfg := storagenodedb.Config{
Storage: storageDir,
Info: filepath.Join(storageDir, "piecestore.db"),
Info2: filepath.Join(storageDir, "info.db"),
Pieces: storageDir,
Kademlia: filepath.Join(storageDir, "kad.db"),
Storage: storageDir,
Info: filepath.Join(storageDir, "piecestore.db"),
Info2: filepath.Join(storageDir, "info.db"),
Pieces: storageDir,
}
db, err := storagenodedb.New(log, cfg)

14
storagenode/storagenodedb/storagenodedbtest/run_test.go
View File

@ -38,9 +38,8 @@ func TestFileConcurrency(t *testing.T) {
log := zaptest.NewLogger(t)
db, err := storagenodedb.New(log, storagenodedb.Config{
Pieces: ctx.Dir("storage"),
Info2: ctx.Dir("storage") + "/info.db",
Kademlia: ctx.Dir("storage") + "/kademlia",
Pieces: ctx.Dir("storage"),
Info2: ctx.Dir("storage") + "/info.db",
})
if err != nil {
t.Fatal(err)
@ -58,11 +57,10 @@ func TestInMemoryConcurrency(t *testing.T) {
storageDir := ctx.Dir("storage")
cfg := storagenodedb.Config{
Pieces: storageDir,
Storage: storageDir,
Info: filepath.Join(storageDir, "piecestore.db"),
Info2: filepath.Join(storageDir, "info.db"),
Kademlia: filepath.Join(storageDir, "kademlia"),
Pieces: storageDir,
Storage: storageDir,
Info: filepath.Join(storageDir, "piecestore.db"),
Info2: filepath.Join(storageDir, "info.db"),
}
db, err := storagenodedb.New(log, cfg)