diff --git a/satellite/accounting/db.go b/satellite/accounting/db.go index bededc7f1..cdb722d75 100644 --- a/satellite/accounting/db.go +++ b/satellite/accounting/db.go @@ -159,6 +159,12 @@ type BucketUsageRollup struct { Before time.Time `json:"before"` } +// Usage contains project's usage split on segments and storage. +type Usage struct { + Storage int64 + Segments int64 +} + // StoragenodeAccounting stores information about bandwidth and storage usage for storage nodes. // // architecture: Database @@ -283,8 +289,8 @@ type Cache interface { // The projectID is inserted to the spaceUsed when it doesn't exists, hence // this method will never return ErrKeyNotFound. AddProjectStorageUsage(ctx context.Context, projectID uuid.UUID, spaceUsed int64) error - // GetAllProjectTotals return the total projects' storage used space. - GetAllProjectTotals(ctx context.Context) (map[uuid.UUID]int64, error) + // GetAllProjectTotals return the total projects' storage and segments used space. + GetAllProjectTotals(ctx context.Context) (map[uuid.UUID]Usage, error) // Close the client, releasing any open resources. Once it's called any other // method must be called. Close() error diff --git a/satellite/accounting/live/live_test.go b/satellite/accounting/live/live_test.go index 8876baced..b244de5b5 100644 --- a/satellite/accounting/live/live_test.go +++ b/satellite/accounting/live/live_test.go @@ -132,17 +132,23 @@ func TestGetAllProjectTotals(t *testing.T) { projectIDs[i] = testrand.UUID() err := cache.AddProjectStorageUsage(ctx, projectIDs[i], int64(i)) require.NoError(t, err) + err = cache.UpdateProjectSegmentUsage(ctx, projectIDs[i], int64(i), time.Hour) + require.NoError(t, err) } - projectTotals, err := cache.GetAllProjectTotals(ctx) + usage, err := cache.GetAllProjectTotals(ctx) require.NoError(t, err) - require.Len(t, projectTotals, len(projectIDs)) + require.Len(t, usage, len(projectIDs)) // make sure each project ID and total was received for _, projID := range projectIDs { - total, err := cache.GetProjectStorageUsage(ctx, projID) + totalStorage, err := cache.GetProjectStorageUsage(ctx, projID) require.NoError(t, err) - assert.Equal(t, total, projectTotals[projID]) + assert.Equal(t, totalStorage, usage[projID].Storage) + + totalSegments, err := cache.GetProjectSegmentUsage(ctx, projID) + require.NoError(t, err) + assert.Equal(t, totalSegments, usage[projID].Segments) } }) } diff --git a/satellite/accounting/live/redis.go b/satellite/accounting/live/redis.go index b07f78897..e4d862ec6 100644 --- a/satellite/accounting/live/redis.go +++ b/satellite/accounting/live/redis.go @@ -163,42 +163,76 @@ func (cache *redisLiveAccounting) AddProjectStorageUsage(ctx context.Context, pr return nil } -// GetAllProjectTotals iterates through the live accounting DB and returns a map of project IDs and totals. +// GetAllProjectTotals iterates through the live accounting DB and returns a map of project IDs and totals, amount of segments. // // TODO (https://storjlabs.atlassian.net/browse/IN-173): see if it possible to // get key/value pairs with one single call. -func (cache *redisLiveAccounting) GetAllProjectTotals(ctx context.Context) (_ map[uuid.UUID]int64, err error) { +func (cache *redisLiveAccounting) GetAllProjectTotals(ctx context.Context) (_ map[uuid.UUID]accounting.Usage, err error) { defer mon.Task()(&ctx)(&err) - projects := make(map[uuid.UUID]int64) + projects := make(map[uuid.UUID]accounting.Usage) it := cache.client.Scan(ctx, 0, "*", 0).Iterator() for it.Next(ctx) { key := it.Val() - // skip bandwidth and segment keys - if strings.HasSuffix(key, "bandwidth") || strings.HasSuffix(key, "segment") { + // skip bandwidth keys + if strings.HasSuffix(key, "bandwidth") { continue } - projectID, err := uuid.FromBytes([]byte(key)) - if err != nil { - return nil, accounting.ErrUnexpectedValue.New("cannot parse the key as UUID; key=%q", key) - } - - if _, seen := projects[projectID]; seen { - continue - } - - val, err := cache.getInt64(ctx, key) - if err != nil { - if accounting.ErrKeyNotFound.Has(err) { - continue + if strings.HasSuffix(key, "segment") { + projectID, err := uuid.FromBytes([]byte(strings.TrimSuffix(key, ":segment"))) + if err != nil { + return nil, accounting.ErrUnexpectedValue.New("cannot parse the key as UUID; key=%q", key) } - return nil, err - } + usage := accounting.Usage{} + if seenUsage, seen := projects[projectID]; seen { + if seenUsage.Segments != 0 { + continue + } - projects[projectID] = val + usage = seenUsage + } + + segmentUsage, err := cache.GetProjectSegmentUsage(ctx, projectID) + if err != nil { + if accounting.ErrKeyNotFound.Has(err) { + continue + } + + return nil, err + } + + usage.Segments = segmentUsage + projects[projectID] = usage + } else { + projectID, err := uuid.FromBytes([]byte(key)) + if err != nil { + return nil, accounting.ErrUnexpectedValue.New("cannot parse the key as UUID; key=%q", key) + } + + usage := accounting.Usage{} + if seenUsage, seen := projects[projectID]; seen { + if seenUsage.Storage != 0 { + continue + } + + usage = seenUsage + } + + storageUsage, err := cache.getInt64(ctx, key) + if err != nil { + if accounting.ErrKeyNotFound.Has(err) { + continue + } + + return nil, err + } + + usage.Storage = storageUsage + projects[projectID] = usage + } } return projects, nil diff --git a/satellite/accounting/tally/tally.go b/satellite/accounting/tally/tally.go index 017906e41..ac586e1c1 100644 --- a/satellite/accounting/tally/tally.go +++ b/satellite/accounting/tally/tally.go @@ -141,14 +141,14 @@ func (service *Service) Tally(ctx context.Context) (err error) { } for projectID, tallyTotal := range tallyProjectTotals { - delta := latestLiveTotals[projectID] - initialLiveTotals[projectID] + delta := latestLiveTotals[projectID].Storage - initialLiveTotals[projectID].Storage if delta < 0 { delta = 0 } // read the method documentation why the increase passed to this method // is calculated in this way - err = service.liveAccounting.AddProjectStorageUsage(ctx, projectID, -latestLiveTotals[projectID]+tallyTotal+(delta/2)) + err = service.liveAccounting.AddProjectStorageUsage(ctx, projectID, -latestLiveTotals[projectID].Storage+tallyTotal+(delta/2)) if err != nil { if accounting.ErrSystemOrNetError.Has(err) { service.log.Error(