storj/satellite/metabase/bench_expired_test.go

// Copyright (C) 2021 Storj Labs, Inc.
// See LICENSE for copying information.
package metabase_test

import (
	"fmt"
	"testing"
	"time"

	"github.com/stretchr/testify/require"

	"storj.io/common/memory"
	"storj.io/common/storj"
	"storj.io/common/testcontext"
	"storj.io/common/testrand"
	"storj.io/storj/satellite/metabase"
)

var letters = []rune("abcdefghijklmnopqrstuvwxyz")

func randBucketname(n int) string {
	b := make([]rune, n)
	for i := range b {
		b[i] = letters[testrand.Intn(len(letters))]
	}
	return string(b)
}

func BenchmarkExpiredDeletion(b *testing.B) {
	if testing.Short() {
		expiredScenario{
			objects:           10,
			segmentsPerObject: 2,
			expiredRatio:      1,
		}.Run(b)
		return
	}
	expiredScenario{
		objects:           1000,
		segmentsPerObject: 10,
		expiredRatio:      0.001,
	}.Run(b)
}

type expiredScenario struct {
	objects           int
	segmentsPerObject int
	expiredRatio      float32
	// info filled in during execution.
	redundancy   storj.RedundancyScheme
	objectStream []metabase.ObjectStream
}

type expirationDateGenerator struct {
	expired    int
	nonExpired int
	now        time.Time
}

func (g *expirationDateGenerator) init(total int, ratio float32, now time.Time) {
	g.expired = int(ratio * float32(total))
	g.nonExpired = total - g.expired
	g.now = now
}

func (g *expirationDateGenerator) getDeadline() time.Time {
	timeInterval := time.Duration(testrand.Intn(36))*time.Hour + time.Hour
	if g.expired == 0 && g.nonExpired != 0 {
		g.nonExpired--
		return g.now.Add(timeInterval)
	}
	if g.nonExpired == 0 && g.expired != 0 {
		g.expired--
		return g.now.Add(-timeInterval)
	}
	expired := (testrand.Intn(2) == 0)
	if expired {
		g.expired--
		return g.now.Add(-timeInterval)
	}
	g.nonExpired--
	return g.now.Add(timeInterval)
}

// Run runs the scenario as a subtest.
func (s expiredScenario) Run(b *testing.B) {
	b.Run(s.name(), func(b *testing.B) { Bench(b, s.run) })
}

// name returns the scenario arguments as a string.
func (s *expiredScenario) name() string {
	return fmt.Sprintf("objects=%d,expiredRatio:%f", s.objects, s.expiredRatio)
}

// run runs the specified scenario.
//
// nolint: scopelint // This heavily uses loop variables without goroutines, avoiding these would add lots of boilerplate.
func (s *expiredScenario) run(ctx *testcontext.Context, b *testing.B, db *metabase.DB) {
	if s.redundancy.IsZero() {
		s.redundancy = storj.RedundancyScheme{
			Algorithm:      storj.ReedSolomon,
			RequiredShares: 29,
			RepairShares:   50,
			OptimalShares:  85,
			TotalShares:    90,
			ShareSize:      256,
		}
	}

	nodes := make([]storj.NodeID, 10000)
	for i := range nodes {
		nodes[i] = testrand.NodeID()
	}

	now := time.Now()

	m := make(Metrics, 0, b.N)
	defer m.Report(b, "ns/loop")
	b.Run("Delete expired objects", func(b *testing.B) {
		for i := 0; i < b.N; i++ {
			// wipe data so we can do the exact same test
			DeleteAll{}.Check(ctx, b, db)
			s.objectStream = nil
			var expiredGenerator expirationDateGenerator

			expiredGenerator.init(s.objects, s.expiredRatio, now)
			for objectIndex := 0; objectIndex < s.objects; objectIndex++ {

				expiresAt := expiredGenerator.getDeadline()
				objectStream := metabase.ObjectStream{
					ProjectID:  testrand.UUID(),
					BucketName: randBucketname(10),
					ObjectKey:  metabase.ObjectKey(testrand.UUID().String()),
					Version:    1,
					StreamID:   testrand.UUID(),
				}
				s.objectStream = append(s.objectStream, objectStream)
				_, err := db.BeginObjectExactVersion(ctx, metabase.BeginObjectExactVersion{
					ObjectStream: objectStream,
					Encryption: storj.EncryptionParameters{
						CipherSuite: storj.EncAESGCM,
						BlockSize:   256,
					},
					ExpiresAt: &expiresAt,
				})
				require.NoError(b, err)

				for segment := 0; segment < s.segmentsPerObject-1; segment++ {
					rootPieceID := testrand.PieceID()
					pieces := randPieces(int(s.redundancy.OptimalShares), nodes)

					err := db.BeginSegment(ctx, metabase.BeginSegment{
						ObjectStream: objectStream,
						Position: metabase.SegmentPosition{
							Part:  uint32(0),
							Index: uint32(segment),
						},
						RootPieceID: rootPieceID,
						Pieces:      pieces,
					})
					require.NoError(b, err)

					segmentSize := testrand.Intn(64*memory.MiB.Int()) + 1
					encryptedKey := testrand.BytesInt(storj.KeySize)
					encryptedKeyNonce := testrand.BytesInt(storj.NonceSize)

					err = db.CommitSegment(ctx, metabase.CommitSegment{
						ObjectStream: objectStream,
						Position: metabase.SegmentPosition{
							Part:  uint32(0),
							Index: uint32(segment),
						},
						EncryptedKey:      encryptedKey,
						EncryptedKeyNonce: encryptedKeyNonce,
						PlainSize:         int32(segmentSize),
						EncryptedSize:     int32(segmentSize),
						RootPieceID:       rootPieceID,
						Pieces:            pieces,
						Redundancy:        s.redundancy,
					})
					require.NoError(b, err)
				}

				_, err = db.CommitObject(ctx, metabase.CommitObject{
					ObjectStream: objectStream,
				})
				require.NoError(b, err)
			}

			m.Record(func() {
				err := db.DeleteExpiredObjects(ctx, metabase.DeleteExpiredObjects{
					ExpiredBefore: now,
				})
				require.NoError(b, err)
			})
		}
	})
	if len(s.objectStream) == 0 {
		b.Fatal("no objects uploaded")
	}
}