2019-02-04 16:56:10 +00:00
|
|
|
// Copyright (C) 2019 Storj Labs, Inc.
|
|
|
|
// See LICENSE for copying information
|
|
|
|
|
|
|
|
package testplanet_test
|
|
|
|
|
|
|
|
import (
|
2019-06-21 17:41:39 +01:00
|
|
|
"bytes"
|
2019-04-25 09:17:26 +01:00
|
|
|
"context"
|
2019-06-21 17:41:39 +01:00
|
|
|
"fmt"
|
|
|
|
"math/rand"
|
2019-04-25 09:17:26 +01:00
|
|
|
"path/filepath"
|
2019-02-04 16:56:10 +00:00
|
|
|
"testing"
|
2019-04-25 09:17:26 +01:00
|
|
|
"time"
|
2019-02-04 16:56:10 +00:00
|
|
|
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
"github.com/stretchr/testify/require"
|
2019-04-10 18:35:43 +01:00
|
|
|
"golang.org/x/sync/errgroup"
|
2019-02-04 16:56:10 +00:00
|
|
|
|
|
|
|
"storj.io/storj/internal/memory"
|
|
|
|
"storj.io/storj/internal/testcontext"
|
|
|
|
"storj.io/storj/internal/testplanet"
|
2019-03-26 18:09:44 +00:00
|
|
|
"storj.io/storj/pkg/pb"
|
2019-04-25 09:17:26 +01:00
|
|
|
"storj.io/storj/pkg/peertls/extensions"
|
|
|
|
"storj.io/storj/pkg/peertls/tlsopts"
|
|
|
|
"storj.io/storj/pkg/server"
|
2019-03-26 18:09:44 +00:00
|
|
|
"storj.io/storj/pkg/storj"
|
|
|
|
"storj.io/storj/uplink"
|
2019-02-04 16:56:10 +00:00
|
|
|
)
|
|
|
|
|
2019-06-21 17:41:39 +01:00
|
|
|
func TestUplinksParallel(t *testing.T) {
|
2019-06-25 17:30:39 +01:00
|
|
|
t.Skip("flaky")
|
|
|
|
|
2019-06-21 17:41:39 +01:00
|
|
|
const uplinkCount = 3
|
|
|
|
const parallelCount = 2
|
|
|
|
|
2019-04-22 10:07:50 +01:00
|
|
|
testplanet.Run(t, testplanet.Config{
|
2019-06-21 17:41:39 +01:00
|
|
|
SatelliteCount: 1, StorageNodeCount: 6, UplinkCount: uplinkCount,
|
2019-04-22 10:07:50 +01:00
|
|
|
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
2019-06-21 17:41:39 +01:00
|
|
|
satellite := planet.Satellites[0]
|
2019-02-04 16:56:10 +00:00
|
|
|
|
2019-06-21 17:41:39 +01:00
|
|
|
var group errgroup.Group
|
|
|
|
for i := range planet.Uplinks {
|
|
|
|
uplink := planet.Uplinks[i]
|
2019-02-04 16:56:10 +00:00
|
|
|
|
2019-06-21 17:41:39 +01:00
|
|
|
for p := 0; p < parallelCount; p++ {
|
|
|
|
suffix := fmt.Sprintf("-%d-%d", i, p)
|
|
|
|
group.Go(func() error {
|
|
|
|
random := rand.New(rand.NewSource(rand.Int63()))
|
|
|
|
|
|
|
|
data := make([]byte, 100*memory.KiB.Int()+random.Intn(500)*memory.KiB.Int())
|
|
|
|
_, err := random.Read(data)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
err = uplink.Upload(ctx, satellite, "testbucket"+suffix, "test/path"+suffix, data)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
downloaded, err := uplink.Download(ctx, satellite, "testbucket"+suffix, "test/path"+suffix)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if !bytes.Equal(data, downloaded) {
|
|
|
|
return fmt.Errorf("upload != download data: %s", suffix)
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
err := group.Wait()
|
|
|
|
require.NoError(t, err)
|
2019-04-22 10:07:50 +01:00
|
|
|
})
|
|
|
|
}
|
2019-02-04 16:56:10 +00:00
|
|
|
|
2019-04-22 10:07:50 +01:00
|
|
|
func TestDownloadWithSomeNodesOffline(t *testing.T) {
|
|
|
|
testplanet.Run(t, testplanet.Config{
|
|
|
|
SatelliteCount: 1, StorageNodeCount: 5, UplinkCount: 1,
|
|
|
|
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
|
|
|
// first, upload some remote data
|
|
|
|
ul := planet.Uplinks[0]
|
|
|
|
satellite := planet.Satellites[0]
|
2019-02-04 16:56:10 +00:00
|
|
|
|
2019-04-22 10:07:50 +01:00
|
|
|
// stop discovery service so that we do not get a race condition when we delete nodes from overlay cache
|
|
|
|
satellite.Discovery.Service.Discovery.Stop()
|
2019-02-04 16:56:10 +00:00
|
|
|
|
2019-04-22 10:07:50 +01:00
|
|
|
testData := make([]byte, 1*memory.MiB)
|
|
|
|
_, err := rand.Read(testData)
|
|
|
|
require.NoError(t, err)
|
2019-03-26 18:09:44 +00:00
|
|
|
|
2019-04-22 10:07:50 +01:00
|
|
|
err = ul.UploadWithConfig(ctx, satellite, &uplink.RSConfig{
|
|
|
|
MinThreshold: 2,
|
|
|
|
RepairThreshold: 3,
|
|
|
|
SuccessThreshold: 4,
|
|
|
|
MaxThreshold: 5,
|
|
|
|
}, "testbucket", "test/path", testData)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
// get a remote segment from pointerdb
|
|
|
|
pdb := satellite.Metainfo.Service
|
2019-06-05 15:23:10 +01:00
|
|
|
listResponse, _, err := pdb.List(ctx, "", "", "", true, 0, 0)
|
2019-04-10 13:10:08 +01:00
|
|
|
require.NoError(t, err)
|
2019-04-22 10:07:50 +01:00
|
|
|
|
|
|
|
var path string
|
|
|
|
var pointer *pb.Pointer
|
|
|
|
for _, v := range listResponse {
|
|
|
|
path = v.GetPath()
|
2019-06-05 15:23:10 +01:00
|
|
|
pointer, err = pdb.Get(ctx, path)
|
2019-04-22 10:07:50 +01:00
|
|
|
require.NoError(t, err)
|
|
|
|
if pointer.GetType() == pb.Pointer_REMOTE {
|
|
|
|
break
|
|
|
|
}
|
2019-03-26 18:09:44 +00:00
|
|
|
}
|
2019-04-22 10:07:50 +01:00
|
|
|
|
|
|
|
// calculate how many storagenodes to kill
|
|
|
|
redundancy := pointer.GetRemote().GetRedundancy()
|
|
|
|
remotePieces := pointer.GetRemote().GetRemotePieces()
|
|
|
|
minReq := redundancy.GetMinReq()
|
|
|
|
numPieces := len(remotePieces)
|
|
|
|
toKill := numPieces - int(minReq)
|
|
|
|
|
|
|
|
nodesToKill := make(map[storj.NodeID]bool)
|
|
|
|
for i, piece := range remotePieces {
|
|
|
|
if i >= toKill {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
nodesToKill[piece.NodeId] = true
|
2019-04-10 13:10:08 +01:00
|
|
|
}
|
2019-03-26 18:09:44 +00:00
|
|
|
|
2019-04-22 10:07:50 +01:00
|
|
|
for _, node := range planet.StorageNodes {
|
|
|
|
if nodesToKill[node.ID()] {
|
|
|
|
err = planet.StopPeer(node)
|
|
|
|
require.NoError(t, err)
|
2019-03-26 18:09:44 +00:00
|
|
|
|
2019-04-22 10:07:50 +01:00
|
|
|
// mark node as offline in overlay cache
|
|
|
|
_, err = satellite.Overlay.Service.UpdateUptime(ctx, node.ID(), false)
|
|
|
|
require.NoError(t, err)
|
|
|
|
}
|
2019-03-26 18:09:44 +00:00
|
|
|
}
|
|
|
|
|
2019-04-22 10:07:50 +01:00
|
|
|
// we should be able to download data without any of the original nodes
|
|
|
|
newData, err := ul.Download(ctx, satellite, "testbucket", "test/path")
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, testData, newData)
|
|
|
|
})
|
2019-03-26 18:09:44 +00:00
|
|
|
}
|
2019-04-10 18:35:43 +01:00
|
|
|
|
2019-04-25 09:17:26 +01:00
|
|
|
type piecestoreMock struct {
|
|
|
|
}
|
|
|
|
|
|
|
|
func (mock *piecestoreMock) Upload(server pb.Piecestore_UploadServer) error {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
func (mock *piecestoreMock) Download(server pb.Piecestore_DownloadServer) error {
|
|
|
|
timoutTicker := time.NewTicker(30 * time.Second)
|
|
|
|
defer timoutTicker.Stop()
|
|
|
|
|
|
|
|
select {
|
|
|
|
case <-timoutTicker.C:
|
|
|
|
return nil
|
|
|
|
case <-server.Context().Done():
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
func (mock *piecestoreMock) Delete(ctx context.Context, delete *pb.PieceDeleteRequest) (_ *pb.PieceDeleteResponse, err error) {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestDownloadFromUnresponsiveNode(t *testing.T) {
|
|
|
|
testplanet.Run(t, testplanet.Config{
|
|
|
|
SatelliteCount: 1, StorageNodeCount: 5, UplinkCount: 1,
|
|
|
|
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
|
|
|
|
|
|
|
expectedData := make([]byte, 1*memory.MiB)
|
|
|
|
_, err := rand.Read(expectedData)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
|
|
|
|
err = planet.Uplinks[0].UploadWithConfig(ctx, planet.Satellites[0], &uplink.RSConfig{
|
|
|
|
MinThreshold: 2,
|
|
|
|
RepairThreshold: 3,
|
|
|
|
SuccessThreshold: 4,
|
|
|
|
MaxThreshold: 5,
|
|
|
|
}, "testbucket", "test/path", expectedData)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
// get a remote segment from pointerdb
|
|
|
|
pdb := planet.Satellites[0].Metainfo.Service
|
2019-06-05 15:23:10 +01:00
|
|
|
listResponse, _, err := pdb.List(ctx, "", "", "", true, 0, 0)
|
2019-04-25 09:17:26 +01:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
var path string
|
|
|
|
var pointer *pb.Pointer
|
|
|
|
for _, v := range listResponse {
|
|
|
|
path = v.GetPath()
|
2019-06-05 15:23:10 +01:00
|
|
|
pointer, err = pdb.Get(ctx, path)
|
2019-04-25 09:17:26 +01:00
|
|
|
require.NoError(t, err)
|
|
|
|
if pointer.GetType() == pb.Pointer_REMOTE {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
stopped := false
|
|
|
|
// choose used storage node and replace it with fake listener
|
|
|
|
unresponsiveNode := pointer.Remote.RemotePieces[0].NodeId
|
|
|
|
for _, storageNode := range planet.StorageNodes {
|
|
|
|
if storageNode.ID() == unresponsiveNode {
|
|
|
|
err = planet.StopPeer(storageNode)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
wl, err := planet.WriteWhitelist(storj.LatestIDVersion())
|
|
|
|
require.NoError(t, err)
|
|
|
|
options, err := tlsopts.NewOptions(storageNode.Identity, tlsopts.Config{
|
|
|
|
RevocationDBURL: "bolt://" + filepath.Join(ctx.Dir("fakestoragenode"), "revocation.db"),
|
|
|
|
UsePeerCAWhitelist: true,
|
|
|
|
PeerCAWhitelistPath: wl,
|
|
|
|
PeerIDVersions: "*",
|
|
|
|
Extensions: extensions.Config{
|
|
|
|
Revocation: false,
|
|
|
|
WhitelistSignedLeaf: false,
|
|
|
|
},
|
|
|
|
})
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
server, err := server.New(options, storageNode.Addr(), storageNode.PrivateAddr(), nil)
|
|
|
|
require.NoError(t, err)
|
|
|
|
pb.RegisterPiecestoreServer(server.GRPC(), &piecestoreMock{})
|
|
|
|
go func() {
|
|
|
|
err := server.Run(ctx)
|
|
|
|
require.NoError(t, err)
|
|
|
|
}()
|
|
|
|
stopped = true
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
assert.True(t, stopped, "no storage node was altered")
|
|
|
|
|
|
|
|
data, err := planet.Uplinks[0].Download(ctx, planet.Satellites[0], "testbucket", "test/path")
|
|
|
|
assert.NoError(t, err)
|
|
|
|
|
|
|
|
assert.Equal(t, expectedData, data)
|
|
|
|
})
|
|
|
|
}
|