JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix bucket_storage_tally data (#2165)

Browse Source
This commit is contained in:
ethanadams 2019-06-13 12:58:40 -04:00 committed by Jennifer Li Johnson
parent df2fad15d8
commit 4e1cc37a4c
2 changed files with 101 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
pkg/accounting/tally/tally.go
View File

@ -91,6 +91,7 @@ func (t *Service) Tally(ctx context.Context) (err error) {
errAtRest = errs.New("Saving storage node data-at-rest failed : %v", err)
}
}
if len(bucketData) > 0 {
_, err = t.projectAccountingDB.SaveTallies(ctx, latestTally, bucketData)
if err != nil {
@ -113,9 +114,8 @@ func (t *Service) CalculateAtRestData(ctx context.Context) (latestTally time.Tim
nodeData = make(map[storj.NodeID]float64)
bucketTallies = make(map[string]*accounting.BucketTally)
var currentBucket string
var bucketCount int64
var totalTallies, currentBucketTally accounting.BucketTally
var totalTallies accounting.BucketTally
err = t.metainfo.Iterate(ctx, "", "", true, false,
func(ctx context.Context, it storage.Iterator) error {
@ -137,27 +137,20 @@ func (t *Service) CalculateAtRestData(ctx context.Context) (latestTally time.Tim
if len(pathElements) == 3 {
bucketCount++
} else if len(pathElements) >= 4 {
project, segment, bucketName := pathElements[0], pathElements[1], pathElements[2]
bucketID := storj.JoinPaths(project, bucketName)
// paths are iterated in order, so everything in a bucket is
// iterated together. When a project or bucket changes,
// the previous bucket is completely finished.
if currentBucket != bucketID {
if currentBucket != "" {
// report the previous bucket and add to the totals
currentBucketTally.Report("bucket")
totalTallies.Combine(&currentBucketTally)
bucketTally := bucketTallies[bucketID]
if bucketTally == nil {
bucketTally = &accounting.BucketTally{}
bucketTally.ProjectID = []byte(project)
bucketTally.BucketName = []byte(bucketName)
// add currentBucketTally to bucketTallies
bucketTallies[currentBucket] = &currentBucketTally
currentBucketTally = accounting.BucketTally{}
}
currentBucket = bucketID
bucketTallies[bucketID] = bucketTally
}
currentBucketTally.AddSegment(pointer, segment == "l")
bucketTally.AddSegment(pointer, segment == "l")
}
remote := pointer.GetRemote()
@ -192,11 +185,11 @@ func (t *Service) CalculateAtRestData(ctx context.Context) (latestTally time.Tim
return latestTally, nodeData, bucketTallies, Error.Wrap(err)
}
if currentBucket != "" {
// wrap up the last bucket
totalTallies.Combine(&currentBucketTally)
bucketTallies[currentBucket] = &currentBucketTally
for _, bucketTally := range bucketTallies {
bucketTally.Report("bucket")
totalTallies.Combine(bucketTally)
}
totalTallies.Report("total")
mon.IntVal("bucket_count").Observe(bucketCount)

104
pkg/accounting/tally/tally_test.go
View File

@ -64,6 +64,13 @@ func TestOnlyInline(t *testing.T) {
tallySvc := planet.Satellites[0].Accounting.Tally
uplink := planet.Uplinks[0]
ps, err1 := planet.Satellites[0].DB.Console().Projects().GetAll(ctx)
if err1 != nil {
assert.NoError(t, err1)
}
project := ps[0]
projectID := []byte(project.ID.String())
// Setup: create data for the uplink to upload
expectedData := make([]byte, 1*memory.KiB)
_, err := rand.Read(expectedData)
@ -75,7 +82,10 @@ func TestOnlyInline(t *testing.T) {
require.NoError(t, err)
// Setup: The data in this tally should match the pointer that the uplink.upload created
expectedBucketName := "testbucket"
expectedTally := accounting.BucketTally{
BucketName: []byte(expectedBucketName),
ProjectID: projectID,
Segments: 1,
InlineSegments: 1,
Files: 1,
@ -86,7 +96,6 @@ func TestOnlyInline(t *testing.T) {
}
// Execute test: upload a file, then calculate at rest data
expectedBucketName := "testbucket"
err = uplink.Upload(ctx, planet.Satellites[0], expectedBucketName, "test/path", expectedData)
assert.NoError(t, err)
@ -109,7 +118,7 @@ func TestOnlyInline(t *testing.T) {
})
}
func TestCalculateAtRestData(t *testing.T) {
func TestCalculateNodeAtRestData(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1, StorageNodeCount: 6, UplinkCount: 1,
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
@ -126,22 +135,12 @@ func TestCalculateAtRestData(t *testing.T) {
expectedTotalBytes, err := encryption.CalcEncryptedSize(int64(len(expectedData)), uplinkConfig.GetEncryptionScheme())
require.NoError(t, err)
// Setup: The data in this tally should match the pointer that the uplink.upload created
expectedTally := accounting.BucketTally{
Segments: 1,
RemoteSegments: 1,
Files: 1,
RemoteFiles: 1,
Bytes: expectedTotalBytes,
RemoteBytes: expectedTotalBytes,
MetadataSize: 112, // brittle, this is hardcoded since its too difficult to get this value progamatically
}
// Execute test: upload a file, then calculate at rest data
expectedBucketName := "testbucket"
err = uplink.Upload(ctx, planet.Satellites[0], expectedBucketName, "test/path", expectedData)
assert.NoError(t, err)
_, actualNodeData, actualBucketData, err := tallySvc.CalculateAtRestData(ctx)
_, actualNodeData, _, err := tallySvc.CalculateAtRestData(ctx)
require.NoError(t, err)
// Confirm the correct number of shares were stored
@ -154,12 +153,83 @@ func TestCalculateAtRestData(t *testing.T) {
for _, actualTotalBytes := range actualNodeData {
assert.Equal(t, int64(actualTotalBytes), expectedTotalBytes)
}
})
}
func TestCalculateBucketAtRestData(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1, StorageNodeCount: 6, UplinkCount: 1,
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
tallySvc := planet.Satellites[0].Accounting.Tally
uplink := planet.Uplinks[0]
ps, err1 := planet.Satellites[0].DB.Console().Projects().GetAll(ctx)
if err1 != nil {
assert.NoError(t, err1)
}
project := ps[0]
projectID := []byte(project.ID.String())
// Setup: create 50KiB of data for the uplink to upload
expectedData := make([]byte, 50*memory.KiB)
_, err := rand.Read(expectedData)
require.NoError(t, err)
// Setup: get the expected size of the data that will be stored in pointer
uplinkConfig := uplink.GetConfig(planet.Satellites[0])
expectedTotalBytes, err := encryption.CalcEncryptedSize(int64(len(expectedData)), uplinkConfig.GetEncryptionScheme())
require.NoError(t, err)
// Setup: The data in this tally should match the pointer that the uplink.upload created
expectedBucketName1 := "testbucket1"
expectedTally1 := accounting.BucketTally{
BucketName: []byte(expectedBucketName1),
ProjectID: projectID,
Segments: 1,
RemoteSegments: 1,
Files: 1,
RemoteFiles: 1,
Bytes: expectedTotalBytes,
RemoteBytes: expectedTotalBytes,
MetadataSize: 112, // brittle, this is hardcoded since its too difficult to get this value progamatically
}
expectedBucketName2 := "testbucket2"
expectedTally2 := accounting.BucketTally{
BucketName: []byte(expectedBucketName2),
ProjectID: projectID,
Segments: 2,
RemoteSegments: 2,
Files: 2,
RemoteFiles: 2,
Bytes: expectedTotalBytes * 2,
RemoteBytes: expectedTotalBytes * 2,
MetadataSize: 112 * 2, // brittle, this is hardcoded since its too difficult to get this value progamatically
}
// Execute test: upload a file, then calculate at rest data
err = uplink.Upload(ctx, planet.Satellites[0], expectedBucketName1, "test/path1", expectedData)
assert.NoError(t, err)
err = uplink.Upload(ctx, planet.Satellites[0], expectedBucketName2, "test/path2", expectedData)
assert.NoError(t, err)
err = uplink.Upload(ctx, planet.Satellites[0], expectedBucketName2, "test/path3", expectedData)
assert.NoError(t, err)
_, _, actualBucketData, err := tallySvc.CalculateAtRestData(ctx)
require.NoError(t, err)
// Confirm the correct bucket storage tally was created
assert.Equal(t, len(actualBucketData), 1)
assert.Equal(t, len(actualBucketData), 2)
for bucketID, actualTally := range actualBucketData {
assert.Contains(t, bucketID, expectedBucketName)
assert.Equal(t, *actualTally, expectedTally)
var bucketName = string(actualTally.BucketName)
assert.True(t, bucketName == expectedBucketName1 || bucketName == expectedBucketName2, "Test bucket names do not exist in results")
if bucketName == expectedBucketName1 {
assert.Contains(t, bucketID, expectedBucketName1)
assert.Equal(t, expectedTally1, *actualTally)
} else if bucketName == expectedBucketName2 {
assert.Contains(t, bucketID, expectedBucketName2)
assert.Equal(t, expectedTally2, *actualTally)
}
}
})
}