diff --git a/pkg/accounting/tally/tally.go b/pkg/accounting/tally/tally.go index e95f67169..1081845bd 100644 --- a/pkg/accounting/tally/tally.go +++ b/pkg/accounting/tally/tally.go @@ -91,6 +91,7 @@ func (t *Service) Tally(ctx context.Context) (err error) { errAtRest = errs.New("Saving storage node data-at-rest failed : %v", err) } } + if len(bucketData) > 0 { _, err = t.projectAccountingDB.SaveTallies(ctx, latestTally, bucketData) if err != nil { @@ -113,9 +114,8 @@ func (t *Service) CalculateAtRestData(ctx context.Context) (latestTally time.Tim nodeData = make(map[storj.NodeID]float64) bucketTallies = make(map[string]*accounting.BucketTally) - var currentBucket string var bucketCount int64 - var totalTallies, currentBucketTally accounting.BucketTally + var totalTallies accounting.BucketTally err = t.metainfo.Iterate(ctx, "", "", true, false, func(ctx context.Context, it storage.Iterator) error { @@ -137,27 +137,20 @@ func (t *Service) CalculateAtRestData(ctx context.Context) (latestTally time.Tim if len(pathElements) == 3 { bucketCount++ } else if len(pathElements) >= 4 { - project, segment, bucketName := pathElements[0], pathElements[1], pathElements[2] + bucketID := storj.JoinPaths(project, bucketName) - // paths are iterated in order, so everything in a bucket is - // iterated together. When a project or bucket changes, - // the previous bucket is completely finished. - if currentBucket != bucketID { - if currentBucket != "" { - // report the previous bucket and add to the totals - currentBucketTally.Report("bucket") - totalTallies.Combine(¤tBucketTally) + bucketTally := bucketTallies[bucketID] + if bucketTally == nil { + bucketTally = &accounting.BucketTally{} + bucketTally.ProjectID = []byte(project) + bucketTally.BucketName = []byte(bucketName) - // add currentBucketTally to bucketTallies - bucketTallies[currentBucket] = ¤tBucketTally - currentBucketTally = accounting.BucketTally{} - } - currentBucket = bucketID + bucketTallies[bucketID] = bucketTally } - currentBucketTally.AddSegment(pointer, segment == "l") + bucketTally.AddSegment(pointer, segment == "l") } remote := pointer.GetRemote() @@ -192,11 +185,11 @@ func (t *Service) CalculateAtRestData(ctx context.Context) (latestTally time.Tim return latestTally, nodeData, bucketTallies, Error.Wrap(err) } - if currentBucket != "" { - // wrap up the last bucket - totalTallies.Combine(¤tBucketTally) - bucketTallies[currentBucket] = ¤tBucketTally + for _, bucketTally := range bucketTallies { + bucketTally.Report("bucket") + totalTallies.Combine(bucketTally) } + totalTallies.Report("total") mon.IntVal("bucket_count").Observe(bucketCount) diff --git a/pkg/accounting/tally/tally_test.go b/pkg/accounting/tally/tally_test.go index b5c213ec0..82d3a75fb 100644 --- a/pkg/accounting/tally/tally_test.go +++ b/pkg/accounting/tally/tally_test.go @@ -64,6 +64,13 @@ func TestOnlyInline(t *testing.T) { tallySvc := planet.Satellites[0].Accounting.Tally uplink := planet.Uplinks[0] + ps, err1 := planet.Satellites[0].DB.Console().Projects().GetAll(ctx) + if err1 != nil { + assert.NoError(t, err1) + } + project := ps[0] + projectID := []byte(project.ID.String()) + // Setup: create data for the uplink to upload expectedData := make([]byte, 1*memory.KiB) _, err := rand.Read(expectedData) @@ -75,7 +82,10 @@ func TestOnlyInline(t *testing.T) { require.NoError(t, err) // Setup: The data in this tally should match the pointer that the uplink.upload created + expectedBucketName := "testbucket" expectedTally := accounting.BucketTally{ + BucketName: []byte(expectedBucketName), + ProjectID: projectID, Segments: 1, InlineSegments: 1, Files: 1, @@ -86,7 +96,6 @@ func TestOnlyInline(t *testing.T) { } // Execute test: upload a file, then calculate at rest data - expectedBucketName := "testbucket" err = uplink.Upload(ctx, planet.Satellites[0], expectedBucketName, "test/path", expectedData) assert.NoError(t, err) @@ -109,7 +118,7 @@ func TestOnlyInline(t *testing.T) { }) } -func TestCalculateAtRestData(t *testing.T) { +func TestCalculateNodeAtRestData(t *testing.T) { testplanet.Run(t, testplanet.Config{ SatelliteCount: 1, StorageNodeCount: 6, UplinkCount: 1, }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { @@ -126,22 +135,12 @@ func TestCalculateAtRestData(t *testing.T) { expectedTotalBytes, err := encryption.CalcEncryptedSize(int64(len(expectedData)), uplinkConfig.GetEncryptionScheme()) require.NoError(t, err) - // Setup: The data in this tally should match the pointer that the uplink.upload created - expectedTally := accounting.BucketTally{ - Segments: 1, - RemoteSegments: 1, - Files: 1, - RemoteFiles: 1, - Bytes: expectedTotalBytes, - RemoteBytes: expectedTotalBytes, - MetadataSize: 112, // brittle, this is hardcoded since its too difficult to get this value progamatically - } - // Execute test: upload a file, then calculate at rest data expectedBucketName := "testbucket" err = uplink.Upload(ctx, planet.Satellites[0], expectedBucketName, "test/path", expectedData) + assert.NoError(t, err) - _, actualNodeData, actualBucketData, err := tallySvc.CalculateAtRestData(ctx) + _, actualNodeData, _, err := tallySvc.CalculateAtRestData(ctx) require.NoError(t, err) // Confirm the correct number of shares were stored @@ -154,12 +153,83 @@ func TestCalculateAtRestData(t *testing.T) { for _, actualTotalBytes := range actualNodeData { assert.Equal(t, int64(actualTotalBytes), expectedTotalBytes) } + }) +} + +func TestCalculateBucketAtRestData(t *testing.T) { + testplanet.Run(t, testplanet.Config{ + SatelliteCount: 1, StorageNodeCount: 6, UplinkCount: 1, + }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { + tallySvc := planet.Satellites[0].Accounting.Tally + uplink := planet.Uplinks[0] + + ps, err1 := planet.Satellites[0].DB.Console().Projects().GetAll(ctx) + if err1 != nil { + assert.NoError(t, err1) + } + project := ps[0] + projectID := []byte(project.ID.String()) + + // Setup: create 50KiB of data for the uplink to upload + expectedData := make([]byte, 50*memory.KiB) + _, err := rand.Read(expectedData) + require.NoError(t, err) + + // Setup: get the expected size of the data that will be stored in pointer + uplinkConfig := uplink.GetConfig(planet.Satellites[0]) + expectedTotalBytes, err := encryption.CalcEncryptedSize(int64(len(expectedData)), uplinkConfig.GetEncryptionScheme()) + require.NoError(t, err) + + // Setup: The data in this tally should match the pointer that the uplink.upload created + expectedBucketName1 := "testbucket1" + expectedTally1 := accounting.BucketTally{ + BucketName: []byte(expectedBucketName1), + ProjectID: projectID, + Segments: 1, + RemoteSegments: 1, + Files: 1, + RemoteFiles: 1, + Bytes: expectedTotalBytes, + RemoteBytes: expectedTotalBytes, + MetadataSize: 112, // brittle, this is hardcoded since its too difficult to get this value progamatically + } + + expectedBucketName2 := "testbucket2" + expectedTally2 := accounting.BucketTally{ + BucketName: []byte(expectedBucketName2), + ProjectID: projectID, + Segments: 2, + RemoteSegments: 2, + Files: 2, + RemoteFiles: 2, + Bytes: expectedTotalBytes * 2, + RemoteBytes: expectedTotalBytes * 2, + MetadataSize: 112 * 2, // brittle, this is hardcoded since its too difficult to get this value progamatically + } + + // Execute test: upload a file, then calculate at rest data + err = uplink.Upload(ctx, planet.Satellites[0], expectedBucketName1, "test/path1", expectedData) + assert.NoError(t, err) + err = uplink.Upload(ctx, planet.Satellites[0], expectedBucketName2, "test/path2", expectedData) + assert.NoError(t, err) + err = uplink.Upload(ctx, planet.Satellites[0], expectedBucketName2, "test/path3", expectedData) + assert.NoError(t, err) + + _, _, actualBucketData, err := tallySvc.CalculateAtRestData(ctx) + require.NoError(t, err) // Confirm the correct bucket storage tally was created - assert.Equal(t, len(actualBucketData), 1) + assert.Equal(t, len(actualBucketData), 2) for bucketID, actualTally := range actualBucketData { - assert.Contains(t, bucketID, expectedBucketName) - assert.Equal(t, *actualTally, expectedTally) + var bucketName = string(actualTally.BucketName) + assert.True(t, bucketName == expectedBucketName1 || bucketName == expectedBucketName2, "Test bucket names do not exist in results") + if bucketName == expectedBucketName1 { + assert.Contains(t, bucketID, expectedBucketName1) + assert.Equal(t, expectedTally1, *actualTally) + } else if bucketName == expectedBucketName2 { + assert.Contains(t, bucketID, expectedBucketName2) + assert.Equal(t, expectedTally2, *actualTally) + } } }) }