storj/satellite/repair/irreparable/irreparable_test.go

// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.

package irreparable_test

import (
	"strconv"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/stretchr/testify/require"

	"storj.io/common/pb"
	"storj.io/common/testcontext"
	"storj.io/storj/private/testplanet"
	"storj.io/storj/satellite"
	"storj.io/storj/satellite/internalpb"
	"storj.io/storj/satellite/metainfo/metabase"
	"storj.io/storj/satellite/satellitedb/satellitedbtest"
)

func TestIrreparable(t *testing.T) {
	satellitedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db satellite.DB) {
		irrdb := db.Irreparable()

		// Create and insert test segment infos into DB
		var segments []*internalpb.IrreparableSegment
		for i := 0; i < 3; i++ {
			segments = append(segments, &internalpb.IrreparableSegment{
				Path: []byte(strconv.Itoa(i)),
				SegmentDetail: &pb.Pointer{
					CreationDate: time.Now(),
				},
				LostPieces:         int32(i),
				LastRepairAttempt:  time.Now().Unix(),
				RepairAttemptCount: int64(10),
			})

			err := irrdb.IncrementRepairAttempts(ctx, segments[i])
			require.NoError(t, err)
		}

		{ // GetLimited limit 1, starting from the beginning
			lastSeenSegmentPath := []byte{}
			segs, err := irrdb.GetLimited(ctx, 1, lastSeenSegmentPath)
			require.NoError(t, err)
			require.Equal(t, 1, len(segs))
			require.Empty(t, cmp.Diff(segments[0], segs[0], cmp.Comparer(pb.Equal)))
		}

		{ // GetLimited limit 1, starting after the first item
			lastSeenSegmentPath := []byte(strconv.Itoa(0))
			segs, err := irrdb.GetLimited(ctx, 1, lastSeenSegmentPath)
			require.NoError(t, err)
			require.Equal(t, 1, len(segs))
			require.Empty(t, cmp.Diff(segments[1], segs[0], cmp.Comparer(pb.Equal)))

		}

		{ // GetLimited limit 2, starting from the beginning
			lastSeenSegmentPath := []byte{}
			segs, err := irrdb.GetLimited(ctx, 2, lastSeenSegmentPath)
			require.NoError(t, err)
			require.Equal(t, 2, len(segs))
			require.Empty(t, cmp.Diff(segments[0], segs[0], cmp.Comparer(pb.Equal)))
			require.Empty(t, cmp.Diff(segments[1], segs[1], cmp.Comparer(pb.Equal)))
		}

		{ // GetLimited limit 2, starting after the first item
			lastSeenSegmentPath := []byte(strconv.Itoa(0))
			segs, err := irrdb.GetLimited(ctx, 2, lastSeenSegmentPath)
			require.NoError(t, err)
			require.Equal(t, 2, len(segs))
			require.Empty(t, cmp.Diff(segments[1], segs[0], cmp.Comparer(pb.Equal)))
			require.Empty(t, cmp.Diff(segments[2], segs[1], cmp.Comparer(pb.Equal)))

		}

		{ // GetLimited limit 3, starting after the first item
			lastSeenSegmentPath := []byte(strconv.Itoa(0))
			segs, err := irrdb.GetLimited(ctx, 3, lastSeenSegmentPath)
			require.NoError(t, err)
			require.Equal(t, 2, len(segs))
			require.Empty(t, cmp.Diff(segments[1], segs[0], cmp.Comparer(pb.Equal)))
			require.Empty(t, cmp.Diff(segments[2], segs[1], cmp.Comparer(pb.Equal)))
		}

		{ // Test repair count incrementation
			err := irrdb.IncrementRepairAttempts(ctx, segments[0])
			require.NoError(t, err)
			segments[0].RepairAttemptCount++

			dbxInfo, err := irrdb.Get(ctx, segments[0].Path)
			require.NoError(t, err)
			require.Empty(t, cmp.Diff(segments[0], dbxInfo, cmp.Comparer(pb.Equal)))
		}

		{ // Delete existing entry
			err := irrdb.Delete(ctx, segments[0].Path)
			require.NoError(t, err)

			_, err = irrdb.Get(ctx, segments[0].Path)
			require.Error(t, err)
		}
	})
}

func TestIrreparableProcess(t *testing.T) {
	testplanet.Run(t, testplanet.Config{
		SatelliteCount: 1, StorageNodeCount: 3, UplinkCount: 0,
	}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
		checker := planet.Satellites[0].Repair.Checker
		checker.Loop.Stop()
		checker.IrreparableLoop.Stop()
		irreparabledb := planet.Satellites[0].DB.Irreparable()
		queue := planet.Satellites[0].DB.RepairQueue()

		seg := &internalpb.IrreparableSegment{
			Path: []byte{1},
			SegmentDetail: &pb.Pointer{
				Type:         pb.Pointer_REMOTE,
				CreationDate: time.Now(),
				Remote: &pb.RemoteSegment{
					Redundancy: &pb.RedundancyScheme{
						MinReq:           1,
						RepairThreshold:  2,
						SuccessThreshold: 3,
						Total:            4,
					},
					RemotePieces: []*pb.RemotePiece{
						{
							NodeId: planet.StorageNodes[0].ID(),
						},
						{
							NodeId: planet.StorageNodes[1].ID(),
						},
						{
							NodeId: planet.StorageNodes[2].ID(),
						},
					},
				},
			},
			LostPieces:         int32(4),
			LastRepairAttempt:  time.Now().Unix(),
			RepairAttemptCount: int64(10),
		}

		require.NoError(t, irreparabledb.IncrementRepairAttempts(ctx, seg))

		result, err := irreparabledb.Get(ctx, metabase.SegmentKey(seg.GetPath()))
		require.NoError(t, err)
		require.NotNil(t, result)

		// test healthy segment is removed from irreparable DB
		require.NoError(t, checker.IrreparableProcess(ctx))

		result, err = irreparabledb.Get(ctx, metabase.SegmentKey(seg.GetPath()))
		require.Error(t, err)
		require.Nil(t, result)

		// test unhealthy repairable segment is removed from irreparable DB and inserted into repair queue
		seg.SegmentDetail.Remote.RemotePieces[0] = &pb.RemotePiece{}
		seg.SegmentDetail.Remote.RemotePieces[1] = &pb.RemotePiece{}

		require.NoError(t, irreparabledb.IncrementRepairAttempts(ctx, seg))
		require.NoError(t, checker.IrreparableProcess(ctx))

		result, err = irreparabledb.Get(ctx, metabase.SegmentKey(seg.GetPath()))
		require.Error(t, err)
		require.Nil(t, result)

		injured, err := queue.Select(ctx)
		require.NoError(t, err)
		require.Equal(t, seg.GetPath(), injured.GetPath())

		n, err := queue.Clean(ctx, time.Now())
		require.NoError(t, err)
		require.EqualValues(t, 1, n)

		// test irreparable segment remains in irreparable DB and repair_attempt_count is incremented
		seg.SegmentDetail.Remote.RemotePieces[2] = &pb.RemotePiece{}

		require.NoError(t, irreparabledb.IncrementRepairAttempts(ctx, seg))
		require.NoError(t, checker.IrreparableProcess(ctx))

		result, err = irreparabledb.Get(ctx, metabase.SegmentKey(seg.GetPath()))
		require.NoError(t, err)
		require.Equal(t, seg.GetPath(), result.Path)
		require.Equal(t, seg.RepairAttemptCount+1, result.RepairAttemptCount)
	})
}