storj/satellite/metabase/list_verify_segments_test.go

// Copyright (C) 2022 Storj Labs, Inc.
// See LICENSE for copying information.

package metabase_test

import (
	"testing"
	"time"

	"github.com/stretchr/testify/require"

	"storj.io/common/storj"
	"storj.io/common/testcontext"
	"storj.io/common/testrand"
	"storj.io/common/uuid"
	"storj.io/storj/satellite/metabase"
	"storj.io/storj/satellite/metabase/metabasetest"
)

func TestListVerifySegments(t *testing.T) {
	metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) {
		obj := metabasetest.RandObjectStream()

		t.Run("Invalid limit", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					Limit: -1,
				},
				ErrClass: &metabase.ErrInvalidRequest,
				ErrText:  "invalid limit: -1",
			}.Check(ctx, t, db)

			metabasetest.Verify{}.Check(ctx, t, db)
		})

		t.Run("no segments", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					Limit: 1,
				},
				Result: metabase.ListVerifySegmentsResult{},
			}.Check(ctx, t, db)

			metabasetest.Verify{}.Check(ctx, t, db)
		})

		t.Run("aost", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					Limit:              1,
					AsOfSystemTime:     time.Now(),
					AsOfSystemInterval: time.Nanosecond,
				},
				Result: metabase.ListVerifySegmentsResult{},
			}.Check(ctx, t, db)

			metabasetest.Verify{}.Check(ctx, t, db)
		})

		t.Run("single object segments", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			_ = metabasetest.CreateObject(ctx, t, db, obj, 10)

			expectedSegments := make([]metabase.VerifySegment, 10)
			for i := range expectedSegments {
				expectedSegments[i] = defaultVerifySegment(obj.StreamID, uint32(i))
			}

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					Limit: 10,
				},
				Result: metabase.ListVerifySegmentsResult{
					Segments: expectedSegments,
				},
			}.Check(ctx, t, db)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					CursorStreamID: uuidBefore(obj.StreamID),
					CursorPosition: metabase.SegmentPosition{},
					Limit:          1,
				},
				Result: metabase.ListVerifySegmentsResult{
					Segments: expectedSegments[:1],
				},
			}.Check(ctx, t, db)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					Limit:          2,
					CursorStreamID: obj.StreamID,
					CursorPosition: metabase.SegmentPosition{
						Index: 1,
					},
				},
				Result: metabase.ListVerifySegmentsResult{
					Segments: expectedSegments[2:4],
				},
			}.Check(ctx, t, db)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					Limit:          2,
					CursorStreamID: obj.StreamID,
					CursorPosition: metabase.SegmentPosition{
						Index: 10,
					},
				},
				Result: metabase.ListVerifySegmentsResult{},
			}.Check(ctx, t, db)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					CursorStreamID: obj.StreamID,
					CursorPosition: metabase.SegmentPosition{
						Part:  1,
						Index: 10,
					},
					Limit: 2,
				},
				Result: metabase.ListVerifySegmentsResult{},
			}.Check(ctx, t, db)
		})

		t.Run("many objects segments", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			expectedVerifySegments := []metabase.VerifySegment{}

			for i := 0; i < 5; i++ {
				obj = metabasetest.RandObjectStream()
				obj.StreamID[0] = byte(i) // make StreamIDs ordered
				_ = metabasetest.CreateObject(ctx, t, db, obj, 1)

				expectedVerifySegments = append(expectedVerifySegments, defaultVerifySegment(obj.StreamID, 0))
			}

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					Limit: 5,
				},
				Result: metabase.ListVerifySegmentsResult{
					Segments: expectedVerifySegments,
				},
			}.Check(ctx, t, db)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					Limit: 2,
				},
				Result: metabase.ListVerifySegmentsResult{
					Segments: expectedVerifySegments[:2],
				},
			}.Check(ctx, t, db)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					CursorStreamID: uuidBefore(expectedVerifySegments[2].StreamID),
					Limit:          2,
				},
				Result: metabase.ListVerifySegmentsResult{
					Segments: expectedVerifySegments[2:4],
				},
			}.Check(ctx, t, db)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					CursorStreamID: expectedVerifySegments[4].StreamID,
					Limit:          1,
				},
				Result: metabase.ListVerifySegmentsResult{},
			}.Check(ctx, t, db)
		})

		t.Run("mixed with inline segments", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			expectedVerifySegments := []metabase.VerifySegment{}

			obj.StreamID = uuid.UUID{0}
			for i := 0; i < 5; i++ {
				// object with inline segment
				obj.ObjectKey = metabasetest.RandObjectKey()
				obj.StreamID[obj.StreamID.Size()-1]++
				createInlineSegment := func(object metabase.Object, index int) metabase.Segment {
					err := db.CommitInlineSegment(ctx, metabase.CommitInlineSegment{
						ObjectStream: obj,
						Position: metabase.SegmentPosition{
							Index: uint32(index),
						},
						EncryptedKey:      testrand.Bytes(32),
						EncryptedKeyNonce: testrand.Bytes(32),
					})
					require.NoError(t, err)
					return metabase.Segment{}
				}
				metabasetest.CreateTestObject{
					CreateSegment: createInlineSegment,
				}.Run(ctx, t, db, obj, 1)

				// object with remote segment
				obj.ObjectKey = metabasetest.RandObjectKey()
				obj.StreamID[obj.StreamID.Size()-1]++
				metabasetest.CreateObject(ctx, t, db, obj, 1)

				expectedVerifySegments = append(expectedVerifySegments, defaultVerifySegment(obj.StreamID, 0))
			}

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					Limit: 5,
				},
				Result: metabase.ListVerifySegmentsResult{
					Segments: expectedVerifySegments,
				},
			}.Check(ctx, t, db)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					Limit: 2,
				},
				Result: metabase.ListVerifySegmentsResult{
					Segments: expectedVerifySegments[:2],
				},
			}.Check(ctx, t, db)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					CursorStreamID: uuidBefore(expectedVerifySegments[2].StreamID),
					Limit:          2,
				},
				Result: metabase.ListVerifySegmentsResult{
					Segments: expectedVerifySegments[2:4],
				},
			}.Check(ctx, t, db)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					CursorStreamID: expectedVerifySegments[4].StreamID,
					Limit:          1,
				},
				Result: metabase.ListVerifySegmentsResult{},
			}.Check(ctx, t, db)
		})

		t.Run("streamID list", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			expectedVerifySegments := []metabase.VerifySegment{}

			nbBuckets := 3
			bucketList := metabase.ListVerifyBucketList{}
			for i := 0; i < nbBuckets; i++ {
				projectID := testrand.UUID()
				bucketName := testrand.BucketName()
				bucketList.Add(projectID, bucketName)

				obj := metabasetest.RandObjectStream()
				obj.ProjectID = projectID
				obj.BucketName = bucketName
				obj.StreamID[0] = byte(i) // make StreamIDs ordered
				_ = metabasetest.CreateObject(ctx, t, db, obj, 1)
				expectedVerifySegments = append(expectedVerifySegments, defaultVerifySegment(obj.StreamID, 0))
				// create a un-related object
				_ = metabasetest.CreateObject(ctx, t, db, metabasetest.RandObjectStream(), 2)

			}

			opts := metabase.ListBucketsStreamIDs{
				BucketList: bucketList,
				Limit:      10,
			}
			streamIDList, err := db.ListBucketsStreamIDs(ctx, opts)
			require.NoError(t, err)
			require.Len(t, streamIDList.StreamIDs, nbBuckets)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					StreamIDs: streamIDList.StreamIDs,
					Limit:     5,
				},
				Result: metabase.ListVerifySegmentsResult{
					Segments: expectedVerifySegments,
				},
			}.Check(ctx, t, db)
		})

		t.Run("creation time filtering", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			now := time.Now()

			expectedVerifySegments := []metabase.VerifySegment{}

			for i := 0; i < 5; i++ {
				obj = metabasetest.RandObjectStream()
				obj.StreamID[0] = byte(i) // make StreamIDs ordered
				_ = metabasetest.CreateObject(ctx, t, db, obj, 1)

				expectedVerifySegments = append(expectedVerifySegments, defaultVerifySegment(obj.StreamID, 0))
			}

			date := func(t time.Time) *time.Time {
				return &t
			}

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					CreatedAfter: date(now.Add(-time.Hour)),
					Limit:        10,
				},
				Result: metabase.ListVerifySegmentsResult{Segments: expectedVerifySegments},
			}.Check(ctx, t, db)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					CreatedAfter: date(now.Add(time.Hour)),
					Limit:        10,
				},
				Result: metabase.ListVerifySegmentsResult{},
			}.Check(ctx, t, db)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					CreatedBefore: date(now.Add(time.Hour)),
					Limit:         10,
				},
				Result: metabase.ListVerifySegmentsResult{Segments: expectedVerifySegments},
			}.Check(ctx, t, db)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					CreatedBefore: date(now.Add(-time.Hour)),
					Limit:         10,
				},
				Result: metabase.ListVerifySegmentsResult{},
			}.Check(ctx, t, db)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					CreatedAfter:  date(now.Add(-time.Hour)),
					CreatedBefore: date(now.Add(time.Hour)),
					Limit:         10,
				},
				Result: metabase.ListVerifySegmentsResult{Segments: expectedVerifySegments},
			}.Check(ctx, t, db)

			metabasetest.ListVerifySegments{
				Opts: metabase.ListVerifySegments{
					CreatedAfter:  date(now.Add(time.Hour)),
					CreatedBefore: date(now.Add(2 * time.Hour)),
					Limit:         10,
				},
				Result: metabase.ListVerifySegmentsResult{},
			}.Check(ctx, t, db)
		})
	})
}

func TestListBucketsStreamIDs(t *testing.T) {
	metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) {
		t.Run("many objects segments", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			nbBuckets := 3
			bucketList := metabase.ListVerifyBucketList{}
			obj := metabasetest.RandObjectStream()
			for i := 0; i < nbBuckets; i++ {
				projectID := testrand.UUID()
				projectID[0] = byte(i) // make projectID ordered
				bucketName := testrand.BucketName()
				bucketList.Add(projectID, bucketName)

				obj.ProjectID = projectID
				obj.BucketName = bucketName
				obj.StreamID[0] = byte(i) // make StreamIDs ordered
				_ = metabasetest.CreateObject(ctx, t, db, obj, 3)
				// create a un-related object
				_ = metabasetest.CreateObject(ctx, t, db, metabasetest.RandObjectStream(), 2)

			}

			opts := metabase.ListBucketsStreamIDs{
				BucketList: bucketList,

				Limit: 10,
			}
			listStreamIDsResult, err := db.ListBucketsStreamIDs(ctx, opts)
			require.NoError(t, err)
			require.Len(t, listStreamIDsResult.StreamIDs, nbBuckets)
			require.Equal(t, obj.ProjectID, listStreamIDsResult.LastBucket.ProjectID)
			require.Equal(t, obj.BucketName, listStreamIDsResult.LastBucket.BucketName)
			require.Equal(t, obj.StreamID,
				listStreamIDsResult.StreamIDs[len(listStreamIDsResult.StreamIDs)-1])
			// TODO more test cases
		})
	})
}

func uuidBefore(v uuid.UUID) uuid.UUID {
	for i := len(v) - 1; i >= 0; i-- {
		v[i]--
		if v[i] != 0xFF { // we didn't wrap around
			break
		}
	}
	return v
}

func defaultVerifySegment(streamID uuid.UUID, index uint32) metabase.VerifySegment {
	return metabase.VerifySegment{
		StreamID: streamID,
		Position: metabase.SegmentPosition{
			Index: index,
		},
		CreatedAt:   time.Now(),
		RootPieceID: storj.PieceID{1},
		AliasPieces: metabase.AliasPieces{{Number: 0, Alias: 1}},
		Redundancy:  metabasetest.DefaultRedundancy,
	}
}