From 2d2301d5ff790dbdad4fd4ac0cf99985e84c8d95 Mon Sep 17 00:00:00 2001 From: Yaroslav Vorobiov Date: Thu, 16 May 2019 13:43:46 +0300 Subject: [PATCH] Console buckets page (#1847) --- .../console/consoleweb/consoleql/project.go | 128 +++++++++- .../consoleweb/consoleql/typecreator.go | 34 ++- satellite/console/service.go | 20 +- satellite/console/usagerollups.go | 35 +++ satellite/console/usagerollups_test.go | 221 +++++++++++++++++- satellite/satellitedb/locked.go | 6 + satellite/satellitedb/usagerollups.go | 157 +++++++++++++ web/satellite/src/api/usage.ts | 54 +++++ .../src/components/buckets/BucketArea.vue | 28 ++- .../src/components/buckets/BucketItem.vue | 26 ++- .../src/components/buckets/PaginationArea.vue | 73 +++++- .../src/components/buckets/SearchArea.vue | 24 +- .../src/components/buckets/SortingHeader.vue | 37 +-- .../src/components/project/UsageReport.vue | 6 +- web/satellite/src/router/index.ts | 7 +- web/satellite/src/store/index.ts | 5 +- web/satellite/src/store/modules/usage.ts | 55 ++++- web/satellite/src/store/mutationConstants.ts | 7 + web/satellite/src/types/projects.d.ts | 9 - web/satellite/src/types/usage.d.ts | 40 ++++ .../src/utils/constants/actionNames.ts | 6 + .../src/utils/constants/navigationLinks.ts | 18 +- web/satellite/src/views/Dashboard.vue | 6 + 23 files changed, 902 insertions(+), 100 deletions(-) create mode 100644 web/satellite/src/types/usage.d.ts diff --git a/satellite/console/consoleweb/consoleql/project.go b/satellite/console/consoleweb/consoleql/project.go index b43113d09..9a0a954d2 100644 --- a/satellite/console/consoleweb/consoleql/project.go +++ b/satellite/console/consoleweb/consoleql/project.go @@ -18,8 +18,17 @@ const ( ProjectInputType = "projectInput" // ProjectUsageType is a graphql type name for project usage ProjectUsageType = "projectUsage" + // BucketUsageCursorInputType is a graphql input + // type name for bucket usage cursor + BucketUsageCursorInputType = "bucketUsageCursor" + // BucketUsageType is a graphql type name for bucket usage + BucketUsageType = "bucketUsage" + // BucketUsagePageType is a field name for bucket usage page + BucketUsagePageType = "bucketUsagePage" // FieldName is a field name for "name" FieldName = "name" + // FieldBucketName is a field name for "bucket name" + FieldBucketName = "bucketName" // FieldDescription is a field name for description FieldDescription = "description" // FieldMembers is field name for members @@ -28,12 +37,24 @@ const ( FieldAPIKeys = "apiKeys" // FieldUsage is a field name for usage rollup FieldUsage = "usage" + // FieldBucketUsages is a field name for bucket usages + FieldBucketUsages = "bucketUsages" // FieldStorage is a field name for storage total FieldStorage = "storage" // FieldEgress is a field name for egress total FieldEgress = "egress" // FieldObjectCount is a field name for objects count FieldObjectCount = "objectCount" + // FieldPageCount is a field name for total page count + FieldPageCount = "pageCount" + // FieldCurrentPage is a field name for current page number + FieldCurrentPage = "currentPage" + // FieldTotalCount is a field name for bucket usage count total + FieldTotalCount = "totalCount" + // CursorArg is an argument name for cursor + CursorArg = "cursor" + // PageArg ia an argument name for page number + PageArg = "page" // LimitArg is argument name for limit LimitArg = "limit" // OffsetArg is argument name for offset @@ -149,6 +170,25 @@ func graphqlProject(service *console.Service, types *TypeCreator) *graphql.Objec return service.GetProjectUsage(p.Context, project.ID, since, before) }, }, + FieldBucketUsages: &graphql.Field{ + Type: types.bucketUsagePage, + Args: graphql.FieldConfigArgument{ + BeforeArg: &graphql.ArgumentConfig{ + Type: graphql.NewNonNull(graphql.DateTime), + }, + CursorArg: &graphql.ArgumentConfig{ + Type: graphql.NewNonNull(types.bucketUsageCursor), + }, + }, + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + project, _ := p.Source.(*console.Project) + + before := p.Args[BeforeArg].(time.Time) + cursor := fromMapBucketUsageCursor(p.Args[CursorArg].(map[string]interface{})) + + return service.GetBucketTotals(p.Context, project.ID, cursor, before) + }, + }, }, }) } @@ -168,6 +208,81 @@ func graphqlProjectInput() *graphql.InputObject { }) } +// graphqlBucketUsageCursor creates bucket usage cursor graphql input type +func graphqlBucketUsageCursor() *graphql.InputObject { + return graphql.NewInputObject(graphql.InputObjectConfig{ + Name: BucketUsageCursorInputType, + Fields: graphql.InputObjectConfigFieldMap{ + SearchArg: &graphql.InputObjectFieldConfig{ + Type: graphql.NewNonNull(graphql.String), + }, + LimitArg: &graphql.InputObjectFieldConfig{ + Type: graphql.NewNonNull(graphql.Int), + }, + PageArg: &graphql.InputObjectFieldConfig{ + Type: graphql.NewNonNull(graphql.Int), + }, + }, + }) +} + +// graphqlBucketUsage creates bucket usage grapqhl type +func graphqlBucketUsage() *graphql.Object { + return graphql.NewObject(graphql.ObjectConfig{ + Name: BucketUsageType, + Fields: graphql.Fields{ + FieldBucketName: &graphql.Field{ + Type: graphql.String, + }, + FieldStorage: &graphql.Field{ + Type: graphql.Float, + }, + FieldEgress: &graphql.Field{ + Type: graphql.Float, + }, + FieldObjectCount: &graphql.Field{ + Type: graphql.Float, + }, + SinceArg: &graphql.Field{ + Type: graphql.DateTime, + }, + BeforeArg: &graphql.Field{ + Type: graphql.DateTime, + }, + }, + }) +} + +// graphqlBucketUsagePage creates bucket usage page graphql object +func graphqlBucketUsagePage(types *TypeCreator) *graphql.Object { + return graphql.NewObject(graphql.ObjectConfig{ + Name: BucketUsagePageType, + Fields: graphql.Fields{ + FieldBucketUsages: &graphql.Field{ + Type: graphql.NewList(types.bucketUsage), + }, + SearchArg: &graphql.Field{ + Type: graphql.String, + }, + LimitArg: &graphql.Field{ + Type: graphql.Int, + }, + OffsetArg: &graphql.Field{ + Type: graphql.Int, + }, + FieldPageCount: &graphql.Field{ + Type: graphql.Int, + }, + FieldCurrentPage: &graphql.Field{ + Type: graphql.Int, + }, + FieldTotalCount: &graphql.Field{ + Type: graphql.Int, + }, + }, + }) +} + // graphqlProjectUsage creates project usage graphql type func graphqlProjectUsage() *graphql.Object { return graphql.NewObject(graphql.ObjectConfig{ @@ -192,10 +307,21 @@ func graphqlProjectUsage() *graphql.Object { }) } -// fromMapProjectInfo creates satellite.ProjectInfo from input args +// fromMapProjectInfo creates console.ProjectInfo from input args func fromMapProjectInfo(args map[string]interface{}) (project console.ProjectInfo) { project.Name, _ = args[FieldName].(string) project.Description, _ = args[FieldDescription].(string) return } + +// fromMapBucketUsageCursor creates console.BucketUsageCursor from input args +func fromMapBucketUsageCursor(args map[string]interface{}) (cursor console.BucketUsageCursor) { + limit, _ := args[LimitArg].(int) + page, _ := args[PageArg].(int) + + cursor.Limit = uint(limit) + cursor.Page = uint(page) + cursor.Search, _ = args[SearchArg].(string) + return +} diff --git a/satellite/console/consoleweb/consoleql/typecreator.go b/satellite/console/consoleweb/consoleql/typecreator.go index 1ef1c7ef4..b36b99681 100644 --- a/satellite/console/consoleweb/consoleql/typecreator.go +++ b/satellite/console/consoleweb/consoleql/typecreator.go @@ -18,15 +18,18 @@ type TypeCreator struct { token *graphql.Object - user *graphql.Object - project *graphql.Object - projectUsage *graphql.Object - projectMember *graphql.Object - apiKeyInfo *graphql.Object - createAPIKey *graphql.Object + user *graphql.Object + project *graphql.Object + projectUsage *graphql.Object + bucketUsage *graphql.Object + bucketUsagePage *graphql.Object + projectMember *graphql.Object + apiKeyInfo *graphql.Object + createAPIKey *graphql.Object - userInput *graphql.InputObject - projectInput *graphql.InputObject + userInput *graphql.InputObject + projectInput *graphql.InputObject + bucketUsageCursor *graphql.InputObject } // Create create types and check for error @@ -42,6 +45,11 @@ func (c *TypeCreator) Create(log *zap.Logger, service *console.Service, mailServ return err } + c.bucketUsageCursor = graphqlBucketUsageCursor() + if err := c.bucketUsageCursor.Error(); err != nil { + return err + } + // entities c.user = graphqlUser() if err := c.user.Error(); err != nil { @@ -53,6 +61,16 @@ func (c *TypeCreator) Create(log *zap.Logger, service *console.Service, mailServ return err } + c.bucketUsage = graphqlBucketUsage() + if err := c.bucketUsage.Error(); err != nil { + return err + } + + c.bucketUsagePage = graphqlBucketUsagePage(c) + if err := c.bucketUsagePage.Error(); err != nil { + return err + } + c.apiKeyInfo = graphqlAPIKeyInfo() if err := c.apiKeyInfo.Error(); err != nil { return err diff --git a/satellite/console/service.go b/satellite/console/service.go index f711bd0a6..7b9fa1485 100644 --- a/satellite/console/service.go +++ b/satellite/console/service.go @@ -12,7 +12,7 @@ import ( "github.com/zeebo/errs" "go.uber.org/zap" "golang.org/x/crypto/bcrypt" - monkit "gopkg.in/spacemonkeygo/monkit.v2" + "gopkg.in/spacemonkeygo/monkit.v2" "storj.io/storj/pkg/auth" "storj.io/storj/satellite/console/consoleauth" @@ -806,6 +806,24 @@ func (s *Service) GetProjectUsage(ctx context.Context, projectID uuid.UUID, sinc return projectUsage, nil } +// GetBucketTotals retrieves paged bucket total usages since project creation +func (s *Service) GetBucketTotals(ctx context.Context, projectID uuid.UUID, cursor BucketUsageCursor, before time.Time) (*BucketUsagePage, error) { + var err error + defer mon.Task()(&ctx)(&err) + + auth, err := GetAuth(ctx) + if err != nil { + return nil, err + } + + isMember, err := s.isProjectMember(ctx, auth.User.ID, projectID) + if err != nil { + return nil, err + } + + return s.store.UsageRollups().GetBucketTotals(ctx, projectID, cursor, isMember.project.CreatedAt, before) +} + // GetBucketUsageRollups retrieves summed usage rollups for every bucket of particular project for a given period func (s *Service) GetBucketUsageRollups(ctx context.Context, projectID uuid.UUID, since, before time.Time) ([]BucketUsageRollup, error) { var err error diff --git a/satellite/console/usagerollups.go b/satellite/console/usagerollups.go index 7e53228fe..881ab3996 100644 --- a/satellite/console/usagerollups.go +++ b/satellite/console/usagerollups.go @@ -14,6 +14,7 @@ import ( type UsageRollups interface { GetProjectTotal(ctx context.Context, projectID uuid.UUID, since, before time.Time) (*ProjectUsage, error) GetBucketUsageRollups(ctx context.Context, projectID uuid.UUID, since, before time.Time) ([]BucketUsageRollup, error) + GetBucketTotals(ctx context.Context, projectID uuid.UUID, cursor BucketUsageCursor, since, before time.Time) (*BucketUsagePage, error) } // ProjectUsage consist of period total storage, egress @@ -27,6 +28,40 @@ type ProjectUsage struct { Before time.Time } +// BucketUsage consist of total bucket usage for period +type BucketUsage struct { + ProjectID uuid.UUID + BucketName string + + Storage float64 + Egress float64 + ObjectCount float64 + + Since time.Time + Before time.Time +} + +// BucketUsageCursor holds info for bucket usage +// cursor pagination +type BucketUsageCursor struct { + Search string + Limit uint + Page uint +} + +// BucketUsagePage represents bucket usage page result +type BucketUsagePage struct { + BucketUsages []BucketUsage + + Search string + Limit uint + Offset uint64 + + PageCount uint + CurrentPage uint + TotalCount uint64 +} + // BucketUsageRollup is total bucket usage info // for certain period type BucketUsageRollup struct { diff --git a/satellite/console/usagerollups_test.go b/satellite/console/usagerollups_test.go index 8aaa5bd91..0bb4ba447 100644 --- a/satellite/console/usagerollups_test.go +++ b/satellite/console/usagerollups_test.go @@ -1,17 +1,228 @@ // Copyright (C) 2019 Storj Labs, Inc. // See LICENSE for copying information. -package console +package console_test import ( + "encoding/binary" "fmt" "testing" "time" + + "github.com/skyrings/skyring-common/tools/uuid" + "github.com/stretchr/testify/assert" + + "storj.io/storj/internal/testcontext" + "storj.io/storj/pkg/accounting" + "storj.io/storj/pkg/pb" + "storj.io/storj/satellite" + "storj.io/storj/satellite/console" + "storj.io/storj/satellite/satellitedb/satellitedbtest" +) + +const ( + numBuckets = 5 + tallyIntervals = 10 + + tallyInterval = time.Hour ) func TestUsageRollups(t *testing.T) { - fmt.Println(time.Now().Format(time.RFC3339)) - // 2018-04-02T18:25:11+03:00 - // 2020-04-05T18:25:11+03:00 - // 2019-04-03 17:16:26.822857431+00:00 + satellitedbtest.Run(t, func(t *testing.T, db satellite.DB) { + ctx := testcontext.New(t) + defer ctx.Cleanup() + + now := time.Now() + start := now.Add(tallyInterval * time.Duration(-tallyIntervals)) + + project1, err := uuid.New() + if err != nil { + t.Fatal(err) + } + + project2, err := uuid.New() + if err != nil { + t.Fatal(err) + } + + p1base := binary.BigEndian.Uint64(project1[:8]) >> 48 + p2base := binary.BigEndian.Uint64(project2[:8]) >> 48 + + getValue := func(i, j int, base uint64) int64 { + a := uint64((i+1)*(j+1)) ^ base + a &^= (1 << 63) + return int64(a) + } + + actions := []pb.PieceAction{ + pb.PieceAction_GET, + pb.PieceAction_GET_AUDIT, + pb.PieceAction_GET_REPAIR, + } + + var buckets []string + for i := 0; i < numBuckets; i++ { + bucketName := fmt.Sprintf("bucket_%d", i) + + // project 1 + bucketID := []byte(project1.String() + "/" + bucketName) + for _, action := range actions { + value := getValue(0, i, p1base) + + err := db.Orders().UpdateBucketBandwidthAllocation(ctx, + bucketID, + action, + value*6, + now, + ) + if err != nil { + t.Fatal(err) + } + + err = db.Orders().UpdateBucketBandwidthSettle(ctx, + bucketID, + action, + value*3, + now, + ) + if err != nil { + t.Fatal(err) + } + + err = db.Orders().UpdateBucketBandwidthInline(ctx, + bucketID, + action, + value, + now, + ) + if err != nil { + t.Fatal(err) + } + } + + // project 2 + bucketID = []byte(project2.String() + "/" + bucketName) + for _, action := range actions { + value := getValue(1, i, p2base) + + err := db.Orders().UpdateBucketBandwidthAllocation(ctx, + bucketID, + action, + value*6, + now, + ) + if err != nil { + t.Fatal(err) + } + + err = db.Orders().UpdateBucketBandwidthSettle(ctx, + bucketID, + action, + value*3, + now, + ) + if err != nil { + t.Fatal(err) + } + + err = db.Orders().UpdateBucketBandwidthInline(ctx, + bucketID, + action, + value, + now, + ) + if err != nil { + t.Fatal(err) + } + } + + buckets = append(buckets, bucketName) + } + + for i := 0; i < tallyIntervals; i++ { + interval := start.Add(tallyInterval * time.Duration(i)) + + bucketTallies := make(map[string]*accounting.BucketTally) + for j, bucket := range buckets { + bucketID1 := project1.String() + "/" + bucket + bucketID2 := project2.String() + "/" + bucket + value1 := getValue(i, j, p1base) * 10 + value2 := getValue(i, j, p2base) * 10 + + tally1 := &accounting.BucketTally{ + Segments: value1, + InlineSegments: value1, + RemoteSegments: value1, + Files: value1, + InlineFiles: value1, + RemoteFiles: value1, + Bytes: value1, + InlineBytes: value1, + RemoteBytes: value1, + MetadataSize: value1, + } + + tally2 := &accounting.BucketTally{ + Segments: value2, + InlineSegments: value2, + RemoteSegments: value2, + Files: value2, + InlineFiles: value2, + RemoteFiles: value2, + Bytes: value2, + InlineBytes: value2, + RemoteBytes: value2, + MetadataSize: value2, + } + + bucketTallies[bucketID1] = tally1 + bucketTallies[bucketID2] = tally2 + } + + tallies, err := db.ProjectAccounting().SaveTallies(ctx, interval, bucketTallies) + if err != nil { + t.Fatal(err) + } + if len(tallies) != len(buckets)*2 { + t.Fatal() + } + } + + usageRollups := db.Console().UsageRollups() + + t.Run("test project total", func(t *testing.T) { + projTotal1, err := usageRollups.GetProjectTotal(ctx, *project1, start, now) + assert.NoError(t, err) + assert.NotNil(t, projTotal1) + + projTotal2, err := usageRollups.GetProjectTotal(ctx, *project2, start, now) + assert.NoError(t, err) + assert.NotNil(t, projTotal2) + }) + + t.Run("test bucket usage rollups", func(t *testing.T) { + rollups1, err := usageRollups.GetBucketUsageRollups(ctx, *project1, start, now) + assert.NoError(t, err) + assert.NotNil(t, rollups1) + + rollups2, err := usageRollups.GetBucketUsageRollups(ctx, *project2, start, now) + assert.NoError(t, err) + assert.NotNil(t, rollups2) + }) + + t.Run("test bucket totals", func(t *testing.T) { + cursor := console.BucketUsageCursor{ + Limit: 20, + Page: 1, + } + + totals1, err := usageRollups.GetBucketTotals(ctx, *project1, cursor, start, now) + assert.NoError(t, err) + assert.NotNil(t, totals1) + + totals2, err := usageRollups.GetBucketTotals(ctx, *project2, cursor, start, now) + assert.NoError(t, err) + assert.NotNil(t, totals2) + }) + }) } diff --git a/satellite/satellitedb/locked.go b/satellite/satellitedb/locked.go index f19f179a8..71256857a 100644 --- a/satellite/satellitedb/locked.go +++ b/satellite/satellitedb/locked.go @@ -415,6 +415,12 @@ type lockedUsageRollups struct { db console.UsageRollups } +func (m *lockedUsageRollups) GetBucketTotals(ctx context.Context, projectID uuid.UUID, cursor console.BucketUsageCursor, since time.Time, before time.Time) (*console.BucketUsagePage, error) { + m.Lock() + defer m.Unlock() + return m.db.GetBucketTotals(ctx, projectID, cursor, since, before) +} + func (m *lockedUsageRollups) GetBucketUsageRollups(ctx context.Context, projectID uuid.UUID, since time.Time, before time.Time) ([]console.BucketUsageRollup, error) { m.Lock() defer m.Unlock() diff --git a/satellite/satellitedb/usagerollups.go b/satellite/satellitedb/usagerollups.go index dc636662c..d343ab6d6 100644 --- a/satellite/satellitedb/usagerollups.go +++ b/satellite/satellitedb/usagerollups.go @@ -23,6 +23,8 @@ type usagerollups struct { // GetProjectTotal retrieves project usage for a given period func (db *usagerollups) GetProjectTotal(ctx context.Context, projectID uuid.UUID, since, before time.Time) (usage *console.ProjectUsage, err error) { + since = timeTruncateDown(since) + storageQuery := db.db.All_BucketStorageTally_By_ProjectId_And_BucketName_And_IntervalStart_GreaterOrEqual_And_IntervalStart_LessOrEqual_OrderBy_Desc_IntervalStart roullupsQuery := db.db.Rebind(`SELECT SUM(settled), SUM(inline), action @@ -95,6 +97,8 @@ func (db *usagerollups) GetProjectTotal(ctx context.Context, projectID uuid.UUID // GetBucketUsageRollups retrieves summed usage rollups for every bucket of particular project for a given period func (db *usagerollups) GetBucketUsageRollups(ctx context.Context, projectID uuid.UUID, since, before time.Time) ([]console.BucketUsageRollup, error) { + since = timeTruncateDown(since) + buckets, err := db.getBuckets(ctx, projectID, since, before) if err != nil { return nil, err @@ -177,6 +181,154 @@ func (db *usagerollups) GetBucketUsageRollups(ctx context.Context, projectID uui return bucketUsageRollups, nil } +// GetBucketTotals retrieves bucket usage totals for period of time +func (db *usagerollups) GetBucketTotals(ctx context.Context, projectID uuid.UUID, cursor console.BucketUsageCursor, since, before time.Time) (*console.BucketUsagePage, error) { + since = timeTruncateDown(since) + search := cursor.Search + "%" + + if cursor.Limit > 50 { + cursor.Limit = 50 + } + if cursor.Page == 0 { + return nil, errs.New("page can not be 0") + } + + page := &console.BucketUsagePage{ + Search: cursor.Search, + Limit: cursor.Limit, + Offset: uint64((cursor.Page - 1) * cursor.Limit), + } + + countQuery := db.db.Rebind(`SELECT COUNT(DISTINCT bucket_name) + FROM bucket_bandwidth_rollups + WHERE project_id = ? AND interval_start >= ? AND interval_start <= ? + AND CAST(bucket_name as TEXT) LIKE ?`) + + countRow := db.db.QueryRowContext(ctx, + countQuery, + []byte(projectID.String()), + since, before, + search) + + err := countRow.Scan(&page.TotalCount) + if err != nil { + return nil, err + } + if page.TotalCount == 0 { + return page, nil + } + if page.Offset > page.TotalCount-1 { + return nil, errs.New("page is out of range") + } + + bucketsQuery := db.db.Rebind(`SELECT DISTINCT bucket_name + FROM bucket_bandwidth_rollups + WHERE project_id = ? AND interval_start >= ? AND interval_start <= ? + AND CAST(bucket_name as TEXT) LIKE ? + ORDER BY bucket_name ASC + LIMIT ? OFFSET ?`) + + bucketRows, err := db.db.QueryContext(ctx, + bucketsQuery, + []byte(projectID.String()), + since, before, + search, + page.Limit, + page.Offset) + + if err != nil { + return nil, err + } + defer func() { err = errs.Combine(err, bucketRows.Close()) }() + + var buckets []string + for bucketRows.Next() { + var bucket string + err = bucketRows.Scan(&bucket) + if err != nil { + return nil, err + } + + buckets = append(buckets, bucket) + } + + roullupsQuery := db.db.Rebind(`SELECT SUM(settled), SUM(inline), action + FROM bucket_bandwidth_rollups + WHERE project_id = ? AND bucket_name = ? AND interval_start >= ? AND interval_start <= ? + GROUP BY action`) + + storageQuery := db.db.All_BucketStorageTally_By_ProjectId_And_BucketName_And_IntervalStart_GreaterOrEqual_And_IntervalStart_LessOrEqual_OrderBy_Desc_IntervalStart + + var bucketUsages []console.BucketUsage + for _, bucket := range buckets { + bucketUsage := console.BucketUsage{ + ProjectID: projectID, + BucketName: bucket, + Since: since, + Before: before, + } + + // get bucket_bandwidth_rollups + rollupsRows, err := db.db.QueryContext(ctx, roullupsQuery, []byte(projectID.String()), []byte(bucket), since, before) + if err != nil { + return nil, err + } + defer func() { err = errs.Combine(err, rollupsRows.Close()) }() + + var totalEgress int64 + for rollupsRows.Next() { + var action pb.PieceAction + var settled, inline int64 + + err = rollupsRows.Scan(&settled, &inline, &action) + if err != nil { + return nil, err + } + + // add values for egress + if action == pb.PieceAction_GET || action == pb.PieceAction_GET_AUDIT || action == pb.PieceAction_GET_REPAIR { + totalEgress += settled + inline + } + } + + bucketUsage.Egress = memory.Size(totalEgress).GB() + + bucketStorageTallies, err := storageQuery(ctx, + dbx.BucketStorageTally_ProjectId([]byte(projectID.String())), + dbx.BucketStorageTally_BucketName([]byte(bucket)), + dbx.BucketStorageTally_IntervalStart(since), + dbx.BucketStorageTally_IntervalStart(before)) + + if err != nil { + return nil, err + } + + // fill metadata, objects and stored data + // hours calculated from previous tallies, + // so we skip the most recent one + for i := len(bucketStorageTallies) - 1; i > 0; i-- { + current := bucketStorageTallies[i] + + hours := bucketStorageTallies[i-1].IntervalStart.Sub(current.IntervalStart).Hours() + + bucketUsage.Storage += memory.Size(current.Remote).GB() * hours + bucketUsage.Storage += memory.Size(current.Inline).GB() * hours + bucketUsage.ObjectCount += float64(current.ObjectCount) * hours + } + + bucketUsages = append(bucketUsages, bucketUsage) + } + + page.PageCount = uint(page.TotalCount / uint64(cursor.Limit)) + if page.TotalCount%uint64(cursor.Limit) != 0 { + page.PageCount++ + } + + page.BucketUsages = bucketUsages + page.CurrentPage = cursor.Page + return page, nil +} + // getBuckets list all bucket of certain project for given period func (db *usagerollups) getBuckets(ctx context.Context, projectID uuid.UUID, since, before time.Time) ([]string, error) { bucketsQuery := db.db.Rebind(`SELECT DISTINCT bucket_name @@ -202,3 +354,8 @@ func (db *usagerollups) getBuckets(ctx context.Context, projectID uuid.UUID, sin return buckets, nil } + +// timeTruncateDown truncates down to the hour before to be in sync with orders endpoint +func timeTruncateDown(t time.Time) time.Time { + return time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), 0, 0, 0, t.Location()) +} diff --git a/web/satellite/src/api/usage.ts b/web/satellite/src/api/usage.ts index 2f69733c4..b37399b10 100644 --- a/web/satellite/src/api/usage.ts +++ b/web/satellite/src/api/usage.ts @@ -41,3 +41,57 @@ export async function fetchProjectUsage(projectID: string, since: Date, before: return result; } + +// fetchBucketUsages retrieves bucket usage totals for a particular project +export async function fetchBucketUsages(projectID: string, before: Date, cursor: BucketUsageCursor): Promise> { + let result: RequestResponse = { + errorMessage: '', + isSuccess: false, + data: {} as BucketUsagePage + }; + + let response: any = null; + try { + response = await apollo.query( + { + query: gql(` + query { + project(id: "${projectID}") { + bucketUsages(before: "${before.toISOString()}", cursor: { + limit: ${cursor.limit}, search: "${cursor.search}", page: ${cursor.page} + }) { + bucketUsages{ + bucketName, + storage, + egress, + objectCount, + since, + before + }, + search, + limit, + offset, + pageCount, + currentPage, + totalCount + } + } + }` + ), + fetchPolicy: 'no-cache', + errorPolicy: 'all' + } + ); + } catch (e) { + console.log(e); + } + + if (response.errors) { + result.errorMessage = response.errors[0].message; + } else { + result.isSuccess = true; + result.data = response.data.project.bucketUsages; + } + + return result; +} diff --git a/web/satellite/src/components/buckets/BucketArea.vue b/web/satellite/src/components/buckets/BucketArea.vue index 94ae3496d..b1aecc1ac 100644 --- a/web/satellite/src/components/buckets/BucketArea.vue +++ b/web/satellite/src/components/buckets/BucketArea.vue @@ -3,21 +3,25 @@