301 lines
8.3 KiB
Go
301 lines
8.3 KiB
Go
// Copyright (C) 2021 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package metabase_test
|
|
|
|
import (
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"storj.io/common/testcontext"
|
|
"storj.io/common/testrand"
|
|
"storj.io/common/uuid"
|
|
"storj.io/storj/satellite/metabase"
|
|
"storj.io/storj/satellite/metabase/metabasetest"
|
|
)
|
|
|
|
func TestCollectBucketTallies(t *testing.T) {
|
|
metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) {
|
|
t.Run("empty from", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
metabasetest.CollectBucketTallies{
|
|
Opts: metabase.CollectBucketTallies{
|
|
To: metabase.BucketLocation{
|
|
ProjectID: testrand.UUID(),
|
|
BucketName: "name does not exist 2",
|
|
},
|
|
},
|
|
Result: nil,
|
|
}.Check(ctx, t, db)
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("empty to", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
metabasetest.CollectBucketTallies{
|
|
Opts: metabase.CollectBucketTallies{
|
|
From: metabase.BucketLocation{
|
|
ProjectID: testrand.UUID(),
|
|
BucketName: "name does not exist",
|
|
},
|
|
},
|
|
ErrClass: &metabase.ErrInvalidRequest,
|
|
ErrText: "project ID To is before project ID From",
|
|
}.Check(ctx, t, db)
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("empty bucket", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
randStream := metabasetest.RandObjectStream()
|
|
|
|
obj := metabasetest.CreateObject(ctx, t, db, metabase.ObjectStream{
|
|
ProjectID: randStream.ProjectID,
|
|
BucketName: randStream.BucketName,
|
|
ObjectKey: randStream.ObjectKey,
|
|
Version: randStream.Version,
|
|
StreamID: randStream.StreamID,
|
|
}, 0)
|
|
|
|
metabasetest.DeleteObjectExactVersion{
|
|
Opts: metabase.DeleteObjectExactVersion{
|
|
Version: randStream.Version,
|
|
ObjectLocation: metabase.ObjectLocation{
|
|
ProjectID: randStream.ProjectID,
|
|
BucketName: randStream.BucketName,
|
|
ObjectKey: randStream.ObjectKey,
|
|
},
|
|
},
|
|
Result: metabase.DeleteObjectResult{
|
|
Removed: []metabase.Object{obj},
|
|
},
|
|
}.Check(ctx, t, db)
|
|
|
|
metabasetest.CollectBucketTallies{
|
|
Opts: metabase.CollectBucketTallies{
|
|
To: metabase.BucketLocation{
|
|
ProjectID: randStream.ProjectID,
|
|
BucketName: randStream.BucketName,
|
|
},
|
|
},
|
|
Result: nil,
|
|
}.Check(ctx, t, db)
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("empty request", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
metabasetest.CollectBucketTallies{
|
|
Opts: metabase.CollectBucketTallies{
|
|
From: metabase.BucketLocation{},
|
|
To: metabase.BucketLocation{},
|
|
},
|
|
Result: nil,
|
|
}.Check(ctx, t, db)
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("invalid bucket name", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
projectA := uuid.UUID{1}
|
|
projectB := uuid.UUID{2}
|
|
|
|
metabasetest.CollectBucketTallies{
|
|
Opts: metabase.CollectBucketTallies{
|
|
From: metabase.BucketLocation{
|
|
ProjectID: projectA,
|
|
BucketName: "a\\",
|
|
},
|
|
To: metabase.BucketLocation{
|
|
ProjectID: projectB,
|
|
BucketName: "b\\",
|
|
},
|
|
},
|
|
Result: nil,
|
|
}.Check(ctx, t, db)
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("pending and committed", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
pending := metabasetest.RandObjectStream()
|
|
committed := metabasetest.RandObjectStream()
|
|
committed.ProjectID = pending.ProjectID
|
|
committed.BucketName = pending.BucketName + "q"
|
|
|
|
encryptedMetadata := testrand.Bytes(1024)
|
|
encryptedMetadataNonce := testrand.Nonce()
|
|
encryptedMetadataKey := testrand.Bytes(265)
|
|
|
|
metabasetest.BeginObjectExactVersion{
|
|
Opts: metabase.BeginObjectExactVersion{
|
|
ObjectStream: pending,
|
|
Encryption: metabasetest.DefaultEncryption,
|
|
EncryptedMetadata: encryptedMetadata,
|
|
EncryptedMetadataNonce: encryptedMetadataNonce[:],
|
|
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
|
|
},
|
|
}.Check(ctx, t, db)
|
|
|
|
metabasetest.CreateObject(ctx, t, db, committed, 1)
|
|
|
|
expected := []metabase.BucketTally{
|
|
{
|
|
BucketLocation: metabase.BucketLocation{
|
|
ProjectID: pending.ProjectID,
|
|
BucketName: pending.BucketName,
|
|
},
|
|
ObjectCount: 1,
|
|
PendingObjectCount: 1,
|
|
TotalSegments: 0,
|
|
TotalBytes: 0,
|
|
MetadataSize: 1024,
|
|
},
|
|
{
|
|
BucketLocation: metabase.BucketLocation{
|
|
ProjectID: committed.ProjectID,
|
|
BucketName: committed.BucketName,
|
|
},
|
|
ObjectCount: 1,
|
|
PendingObjectCount: 0,
|
|
TotalSegments: 1,
|
|
TotalBytes: 1024,
|
|
MetadataSize: 0,
|
|
},
|
|
}
|
|
|
|
metabasetest.CollectBucketTallies{
|
|
Opts: metabase.CollectBucketTallies{
|
|
From: metabase.BucketLocation{
|
|
ProjectID: pending.ProjectID,
|
|
BucketName: pending.BucketName,
|
|
},
|
|
To: metabase.BucketLocation{
|
|
ProjectID: committed.ProjectID,
|
|
BucketName: committed.BucketName,
|
|
},
|
|
},
|
|
Result: expected,
|
|
}.Check(ctx, t, db)
|
|
|
|
metabasetest.CollectBucketTallies{
|
|
Opts: metabase.CollectBucketTallies{
|
|
From: metabase.BucketLocation{
|
|
ProjectID: pending.ProjectID,
|
|
BucketName: pending.BucketName,
|
|
},
|
|
To: metabase.BucketLocation{
|
|
ProjectID: committed.ProjectID,
|
|
BucketName: committed.BucketName,
|
|
},
|
|
AsOfSystemTime: time.Now(),
|
|
},
|
|
Result: expected,
|
|
}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("multiple projects", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
projects := []uuid.UUID{}
|
|
for i := 0; i < 10; i++ {
|
|
p := testrand.UUID()
|
|
p[0] = byte(i)
|
|
projects = append(projects, p)
|
|
}
|
|
bucketNames := strings.Split("abcde", "")
|
|
bucketLocations := make([]metabase.BucketLocation, 0, len(projects)*len(bucketNames))
|
|
|
|
expected := make([]metabase.BucketTally, 0, len(projects)*len(bucketNames))
|
|
for _, projectID := range projects {
|
|
for _, bucketName := range bucketNames {
|
|
bucketLocations = append(bucketLocations, metabase.BucketLocation{
|
|
ProjectID: projectID,
|
|
BucketName: bucketName,
|
|
})
|
|
rawObjects := createObjects(ctx, t, db, 1, projectID, bucketName)
|
|
for _, obj := range rawObjects {
|
|
expected = append(expected, bucketTallyFromRaw(obj))
|
|
}
|
|
}
|
|
}
|
|
sortBucketLocations(bucketLocations)
|
|
|
|
metabasetest.CollectBucketTallies{
|
|
Opts: metabase.CollectBucketTallies{
|
|
From: bucketLocations[0],
|
|
To: bucketLocations[len(bucketLocations)-1],
|
|
},
|
|
Result: expected,
|
|
}.Check(ctx, t, db)
|
|
|
|
metabasetest.CollectBucketTallies{
|
|
Opts: metabase.CollectBucketTallies{
|
|
From: bucketLocations[0],
|
|
To: bucketLocations[len(bucketLocations)-1],
|
|
AsOfSystemTime: time.Now(),
|
|
},
|
|
Result: expected,
|
|
}.Check(ctx, t, db)
|
|
|
|
metabasetest.CollectBucketTallies{
|
|
Opts: metabase.CollectBucketTallies{
|
|
From: bucketLocations[0],
|
|
To: bucketLocations[15],
|
|
AsOfSystemTime: time.Now(),
|
|
},
|
|
Result: expected[0:16],
|
|
}.Check(ctx, t, db)
|
|
|
|
metabasetest.CollectBucketTallies{
|
|
Opts: metabase.CollectBucketTallies{
|
|
From: bucketLocations[16],
|
|
To: bucketLocations[34],
|
|
AsOfSystemTime: time.Now(),
|
|
},
|
|
Result: expected[16:35],
|
|
}.Check(ctx, t, db)
|
|
|
|
metabasetest.CollectBucketTallies{
|
|
Opts: metabase.CollectBucketTallies{
|
|
From: bucketLocations[30],
|
|
To: bucketLocations[10],
|
|
AsOfSystemTime: time.Now(),
|
|
},
|
|
ErrClass: &metabase.ErrInvalidRequest,
|
|
ErrText: "project ID To is before project ID From",
|
|
}.Check(ctx, t, db)
|
|
})
|
|
})
|
|
}
|
|
|
|
func bucketTallyFromRaw(m metabase.RawObject) metabase.BucketTally {
|
|
return metabase.BucketTally{
|
|
BucketLocation: metabase.BucketLocation{
|
|
ProjectID: m.ProjectID,
|
|
BucketName: m.BucketName,
|
|
},
|
|
ObjectCount: 1,
|
|
TotalSegments: int64(m.SegmentCount),
|
|
TotalBytes: m.TotalEncryptedSize,
|
|
MetadataSize: int64(len(m.EncryptedMetadata)),
|
|
}
|
|
}
|
|
|
|
func sortBucketLocations(bc []metabase.BucketLocation) {
|
|
sort.Slice(bc, func(i, j int) bool {
|
|
if bc[i].ProjectID == bc[j].ProjectID {
|
|
return bc[i].BucketName < bc[j].BucketName
|
|
}
|
|
return bc[i].ProjectID.Less(bc[j].ProjectID)
|
|
})
|
|
}
|