From d8f64326f59176b2dc41437f8beacaf9d9da7c7e Mon Sep 17 00:00:00 2001 From: Jeremy Wharton Date: Mon, 22 May 2023 12:30:25 -0500 Subject: [PATCH] satellite/accounting/tally: remove objects loop This change removes the use of the objects loop for calculating bucket tallies. It has been superseded by a custom query. Change-Id: I9ea4633006c9af3ea14d7de40871639e7b687c22 --- satellite/accounting/bucketstats.go | 7 +-- satellite/accounting/tally/tally.go | 45 +----------------- satellite/accounting/tally/tally_test.go | 51 --------------------- scripts/testdata/satellite-config.yaml.lock | 5 +- 4 files changed, 4 insertions(+), 104 deletions(-) diff --git a/satellite/accounting/bucketstats.go b/satellite/accounting/bucketstats.go index 2fd429c3d..2c5740e3e 100644 --- a/satellite/accounting/bucketstats.go +++ b/satellite/accounting/bucketstats.go @@ -11,9 +11,7 @@ import ( type BucketTally struct { metabase.BucketLocation - ObjectCount int64 - PendingObjectCount int64 - + ObjectCount int64 TotalSegments int64 TotalBytes int64 @@ -23,10 +21,7 @@ type BucketTally struct { // Combine aggregates all the tallies. func (s *BucketTally) Combine(o *BucketTally) { s.ObjectCount += o.ObjectCount - s.PendingObjectCount += o.PendingObjectCount - s.TotalSegments += o.TotalSegments - s.TotalBytes += o.TotalBytes } diff --git a/satellite/accounting/tally/tally.go b/satellite/accounting/tally/tally.go index 65e7ed267..a4f504bef 100644 --- a/satellite/accounting/tally/tally.go +++ b/satellite/accounting/tally/tally.go @@ -29,10 +29,9 @@ type Config struct { Interval time.Duration `help:"how frequently the tally service should run" releaseDefault:"1h" devDefault:"30s" testDefault:"$TESTINTERVAL"` SaveRollupBatchSize int `help:"how large of batches SaveRollup should process at a time" default:"1000"` ReadRollupBatchSize int `help:"how large of batches GetBandwidthSince should process at a time" default:"10000"` - UseObjectsLoop bool `help:"flag to switch between calculating bucket tallies using objects loop or custom query" default:"false"` UseRangedLoop bool `help:"whether to enable node tally with ranged loop" default:"true"` - ListLimit int `help:"how many objects to query in a batch" default:"2500"` + ListLimit int `help:"how many buckets to query in a batch" default:"2500"` AsOfSystemInterval time.Duration `help:"as of system interval" releaseDefault:"-5m" devDefault:"-1us" testDefault:"-1us"` } @@ -233,15 +232,12 @@ func (service *Service) Tally(ctx context.Context) (err error) { monAccounting.IntVal("total_objects").Observe(total.ObjectCount) //mon:locked monAccounting.IntVal("total_segments").Observe(total.Segments()) //mon:locked monAccounting.IntVal("total_bytes").Observe(total.Bytes()) //mon:locked - monAccounting.IntVal("total_pending_objects").Observe(total.PendingObjectCount) } // return errors if something went wrong. return errAtRest } -var objectFunc = mon.Task() - // BucketTallyCollector collects and adds up tallies for buckets. type BucketTallyCollector struct { Now time.Time @@ -276,24 +272,7 @@ func (observer *BucketTallyCollector) Run(ctx context.Context) (err error) { return err } - if !observer.config.UseObjectsLoop { - return observer.fillBucketTallies(ctx, startingTime) - } - - return observer.metabase.IterateLoopObjects(ctx, metabase.IterateLoopObjects{ - BatchSize: observer.config.ListLimit, - AsOfSystemTime: startingTime, - AsOfSystemInterval: observer.config.AsOfSystemInterval, - }, func(ctx context.Context, it metabase.LoopObjectsIterator) (err error) { - var entry metabase.LoopObjectEntry - for it.Next(ctx, &entry) { - err = observer.object(ctx, entry) - if err != nil { - return err - } - } - return nil - }) + return observer.fillBucketTallies(ctx, startingTime) } // fillBucketTallies collects all bucket tallies and fills observer's buckets map with results. @@ -352,26 +331,6 @@ func (observer *BucketTallyCollector) ensureBucket(location metabase.ObjectLocat return bucket } -// Object is called for each object once. -func (observer *BucketTallyCollector) object(ctx context.Context, object metabase.LoopObjectEntry) error { - defer objectFunc(&ctx)(nil) - - if object.Expired(observer.Now) { - return nil - } - - bucket := observer.ensureBucket(object.ObjectStream.Location()) - bucket.TotalSegments += int64(object.SegmentCount) - bucket.TotalBytes += object.TotalEncryptedSize - bucket.MetadataSize += int64(object.EncryptedMetadataSize) - bucket.ObjectCount++ - if object.Status == metabase.Pending { - bucket.PendingObjectCount++ - } - - return nil -} - func projectTotalsFromBuckets(buckets map[metabase.BucketLocation]*accounting.BucketTally) map[uuid.UUID]accounting.Usage { projectTallyTotals := make(map[uuid.UUID]accounting.Usage) for _, bucket := range buckets { diff --git a/satellite/accounting/tally/tally_test.go b/satellite/accounting/tally/tally_test.go index 13d14d7d0..81c233b29 100644 --- a/satellite/accounting/tally/tally_test.go +++ b/satellite/accounting/tally/tally_test.go @@ -199,59 +199,9 @@ func TestIgnoresExpiredPointers(t *testing.T) { }) } -func TestLiveAccountingWithObjectsLoop(t *testing.T) { - testplanet.Run(t, testplanet.Config{ - SatelliteCount: 1, StorageNodeCount: 4, UplinkCount: 1, - Reconfigure: testplanet.Reconfigure{ - Satellite: testplanet.MaxSegmentSize(20 * memory.KiB), - }, - }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { - tally := planet.Satellites[0].Accounting.Tally - projectID := planet.Uplinks[0].Projects[0].ID - tally.Loop.Pause() - - expectedData := testrand.Bytes(19 * memory.KiB) - - err := planet.Uplinks[0].Upload(ctx, planet.Satellites[0], "testbucket", "test/path", expectedData) - require.NoError(t, err) - - segments, err := planet.Satellites[0].Metabase.DB.TestingAllSegments(ctx) - require.NoError(t, err) - require.Len(t, segments, 1) - - segmentSize := int64(segments[0].EncryptedSize) - - tally.Loop.TriggerWait() - - expectedSize := segmentSize - - total, err := planet.Satellites[0].Accounting.ProjectUsage.GetProjectStorageTotals(ctx, projectID) - require.NoError(t, err) - require.Equal(t, expectedSize, total) - - for i := 0; i < 3; i++ { - err := planet.Uplinks[0].Upload(ctx, planet.Satellites[0], "testbucket", fmt.Sprintf("test/path/%d", i), expectedData) - require.NoError(t, err) - - tally.Loop.TriggerWait() - - expectedSize += segmentSize - - total, err := planet.Satellites[0].Accounting.ProjectUsage.GetProjectStorageTotals(ctx, projectID) - require.NoError(t, err) - require.Equal(t, expectedSize, total) - } - }) -} - func TestLiveAccountingWithCustomSQLQuery(t *testing.T) { testplanet.Run(t, testplanet.Config{ SatelliteCount: 1, StorageNodeCount: 4, UplinkCount: 1, - Reconfigure: testplanet.Reconfigure{ - Satellite: func(log *zap.Logger, index int, config *satellite.Config) { - config.Tally.UseObjectsLoop = false - }, - }, }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { tally := planet.Satellites[0].Accounting.Tally projectID := planet.Uplinks[0].Projects[0].ID @@ -450,7 +400,6 @@ func TestTallyBatchSize(t *testing.T) { for _, batchSize := range []int{1, 2, 3, numberOfBuckets, 14, planet.Satellites[0].Config.Tally.ListLimit} { collector := tally.NewBucketTallyCollector(zaptest.NewLogger(t), time.Now(), planet.Satellites[0].Metabase.DB, planet.Satellites[0].DB.Buckets(), tally.Config{ Interval: 1 * time.Hour, - UseObjectsLoop: false, ListLimit: batchSize, AsOfSystemInterval: 1 * time.Microsecond, }) diff --git a/scripts/testdata/satellite-config.yaml.lock b/scripts/testdata/satellite-config.yaml.lock index 306d2951b..f614ed6f9 100755 --- a/scripts/testdata/satellite-config.yaml.lock +++ b/scripts/testdata/satellite-config.yaml.lock @@ -1099,7 +1099,7 @@ server.private-address: 127.0.0.1:7778 # how frequently the tally service should run # tally.interval: 1h0m0s -# how many objects to query in a batch +# how many buckets to query in a batch # tally.list-limit: 2500 # how large of batches GetBandwidthSince should process at a time @@ -1108,9 +1108,6 @@ server.private-address: 127.0.0.1:7778 # how large of batches SaveRollup should process at a time # tally.save-rollup-batch-size: 1000 -# flag to switch between calculating bucket tallies using objects loop or custom query -# tally.use-objects-loop: false - # whether to enable node tally with ranged loop # tally.use-ranged-loop: true