diff --git a/satellite/accounting/db.go b/satellite/accounting/db.go index b9ae36658..5fc9234c8 100644 --- a/satellite/accounting/db.go +++ b/satellite/accounting/db.go @@ -5,6 +5,7 @@ package accounting import ( "context" + "fmt" "time" "storj.io/common/memory" @@ -164,6 +165,23 @@ type BucketUsageRollup struct { Before time.Time `json:"before"` } +// ToStringSlice converts rollup values to a slice of strings. +func (b *BucketUsageRollup) ToStringSlice() []string { + return []string{ + b.ProjectID.String(), + b.BucketName, + fmt.Sprintf("%f", b.TotalStoredData), + fmt.Sprintf("%f", b.TotalSegments), + fmt.Sprintf("%f", b.ObjectCount), + fmt.Sprintf("%f", b.MetadataSize), + fmt.Sprintf("%f", b.RepairEgress), + fmt.Sprintf("%f", b.GetEgress), + fmt.Sprintf("%f", b.AuditEgress), + b.Since.String(), + b.Before.String(), + } +} + // Usage contains project's usage split on segments and storage. type Usage struct { Storage int64 diff --git a/satellite/console/consoleweb/consoleapi/usagelimits.go b/satellite/console/consoleweb/consoleapi/usagelimits.go index 2751b27f6..dc0e2d0b3 100644 --- a/satellite/console/consoleweb/consoleapi/usagelimits.go +++ b/satellite/console/consoleweb/consoleapi/usagelimits.go @@ -5,6 +5,7 @@ package consoleapi import ( "context" + "encoding/csv" "encoding/json" "net/http" "strconv" @@ -105,6 +106,63 @@ func (ul *UsageLimits) TotalUsageLimits(w http.ResponseWriter, r *http.Request) } } +// TotalUsageReport returns total usage report for all the projects that user owns. +func (ul *UsageLimits) TotalUsageReport(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + var err error + defer mon.Task()(&ctx)(&err) + + sinceStamp, err := strconv.ParseInt(r.URL.Query().Get("since"), 10, 64) + if err != nil { + ul.serveJSONError(ctx, w, http.StatusBadRequest, err) + return + } + beforeStamp, err := strconv.ParseInt(r.URL.Query().Get("before"), 10, 64) + if err != nil { + ul.serveJSONError(ctx, w, http.StatusBadRequest, err) + return + } + + since := time.Unix(sinceStamp, 0).UTC() + before := time.Unix(beforeStamp, 0).UTC() + + usage, err := ul.service.GetTotalUsageReport(ctx, since, before) + if err != nil { + if console.ErrUnauthorized.Has(err) { + ul.serveJSONError(ctx, w, http.StatusUnauthorized, err) + return + } + + ul.serveJSONError(ctx, w, http.StatusInternalServerError, err) + return + } + + fileName := "storj-report-" + since.Format("2006-01-02") + "-to-" + before.Format("2006-01-02") + ".csv" + + w.Header().Set("Content-Type", "text/csv") + w.Header().Set("Content-Disposition", "attachment;filename="+fileName) + + wr := csv.NewWriter(w) + + csvHeaders := []string{"ProjectID", "BucketName", "TotalStoredData GB-hour", "TotalSegments GB-hour", "ObjectCount GB-hour", "MetadataSize GB-hour", "RepairEgress GB", "GetEgress GB", "AuditEgress GB", "Since", "Before"} + + err = wr.Write(csvHeaders) + if err != nil { + ul.serveJSONError(ctx, w, http.StatusInternalServerError, errs.New("Error writing CSV data")) + return + } + + for _, u := range usage { + err = wr.Write(u.ToStringSlice()) + if err != nil { + ul.serveJSONError(ctx, w, http.StatusInternalServerError, errs.New("Error writing CSV data")) + return + } + } + + wr.Flush() +} + // DailyUsage returns daily usage by project ID. func (ul *UsageLimits) DailyUsage(w http.ResponseWriter, r *http.Request) { ctx := r.Context() diff --git a/satellite/console/consoleweb/consoleapi/usagelimits_test.go b/satellite/console/consoleweb/consoleapi/usagelimits_test.go index 016f32667..f058acff5 100644 --- a/satellite/console/consoleweb/consoleapi/usagelimits_test.go +++ b/satellite/console/consoleweb/consoleapi/usagelimits_test.go @@ -4,10 +4,12 @@ package consoleapi_test import ( + "encoding/csv" "encoding/json" "fmt" "net/http" "strconv" + "strings" "testing" "time" @@ -21,6 +23,7 @@ import ( "storj.io/storj/satellite" "storj.io/storj/satellite/accounting" "storj.io/storj/satellite/console" + "storj.io/storj/satellite/metabase" ) func Test_TotalUsageLimits(t *testing.T) { @@ -167,3 +170,94 @@ func Test_DailyUsage(t *testing.T) { require.GreaterOrEqual(t, output.SettledBandwidthUsage[0].Value, 5*memory.KiB) }) } + +func Test_TotalUsageReport(t *testing.T) { + testplanet.Run(t, testplanet.Config{ + SatelliteCount: 1, StorageNodeCount: 1, UplinkCount: 1, + Reconfigure: testplanet.Reconfigure{ + Satellite: func(log *zap.Logger, index int, config *satellite.Config) { + config.Console.OpenRegistrationEnabled = true + config.Console.RateLimit.Burst = 10 + }, + }, + }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { + var ( + satelliteSys = planet.Satellites[0] + uplink = planet.Uplinks[0] + now = time.Now() + inFiveMinutes = now.Add(5 * time.Minute) + inAnHour = now.Add(1 * time.Hour) + since = fmt.Sprintf("%d", now.Unix()) + before = fmt.Sprintf("%d", inAnHour.Unix()) + expectedCSVValue = fmt.Sprintf("%f", float64(0)) + ) + + newUser := console.CreateUser{ + FullName: "Total Usage Report Test", + ShortName: "", + Email: "ur@test.test", + } + + user, err := satelliteSys.AddUser(ctx, newUser, 3) + require.NoError(t, err) + + project1, err := satelliteSys.AddProject(ctx, user.ID, "testProject1") + require.NoError(t, err) + + project2, err := satelliteSys.AddProject(ctx, user.ID, "testProject2") + require.NoError(t, err) + + bucketName := "bucket" + err = uplink.CreateBucket(ctx, satelliteSys, bucketName) + require.NoError(t, err) + + bucketLoc1 := metabase.BucketLocation{ + ProjectID: project1.ID, + BucketName: bucketName, + } + tally1 := &accounting.BucketTally{ + BucketLocation: bucketLoc1, + } + + bucketLoc2 := metabase.BucketLocation{ + ProjectID: project2.ID, + BucketName: bucketName, + } + tally2 := &accounting.BucketTally{ + BucketLocation: bucketLoc2, + } + + bucketTallies := map[metabase.BucketLocation]*accounting.BucketTally{ + bucketLoc1: tally1, + bucketLoc2: tally2, + } + + err = satelliteSys.DB.ProjectAccounting().SaveTallies(ctx, inFiveMinutes, bucketTallies) + require.NoError(t, err) + + endpoint := fmt.Sprintf("projects/total-usage-report?since=%s&before=%s", since, before) + body, status, err := doRequestWithAuth(ctx, t, satelliteSys, user, http.MethodGet, endpoint, nil) + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + + content := string(body) + reader := csv.NewReader(strings.NewReader(content)) + records, err := reader.ReadAll() + require.NoError(t, err) + require.Len(t, records, 3) + + expectedHeaders := []string{"ProjectID", "BucketName", "TotalStoredData GB-hour", "TotalSegments GB-hour", "ObjectCount GB-hour", "MetadataSize GB-hour", "RepairEgress GB", "GetEgress GB", "AuditEgress GB", "Since", "Before"} + for i, header := range expectedHeaders { + require.Equal(t, header, records[0][i]) + } + + require.Equal(t, project1.PublicID.String(), records[1][0]) + require.Equal(t, project2.PublicID.String(), records[2][0]) + require.Equal(t, bucketName, records[1][1]) + require.Equal(t, bucketName, records[2][1]) + for i := 2; i < 9; i++ { + require.Equal(t, expectedCSVValue, records[1][i]) + require.Equal(t, expectedCSVValue, records[2][i]) + } + }) +} diff --git a/satellite/console/consoleweb/server.go b/satellite/console/consoleweb/server.go index 8047d9152..4028cb2af 100644 --- a/satellite/console/consoleweb/server.go +++ b/satellite/console/consoleweb/server.go @@ -296,6 +296,7 @@ func NewServer(logger *zap.Logger, config Config, service *console.Service, oidc projectsRouter.Handle("/{id}/usage-limits", http.HandlerFunc(usageLimitsController.ProjectUsageLimits)).Methods(http.MethodGet, http.MethodOptions) projectsRouter.Handle("/usage-limits", http.HandlerFunc(usageLimitsController.TotalUsageLimits)).Methods(http.MethodGet, http.MethodOptions) projectsRouter.Handle("/{id}/daily-usage", http.HandlerFunc(usageLimitsController.DailyUsage)).Methods(http.MethodGet, http.MethodOptions) + projectsRouter.Handle("/total-usage-report", http.HandlerFunc(usageLimitsController.TotalUsageReport)).Methods(http.MethodGet, http.MethodOptions) authController := consoleapi.NewAuth(logger, service, accountFreezeService, mailService, server.cookieAuth, server.analytics, config.SatelliteName, server.config.ExternalAddress, config.LetUsKnowURL, config.TermsAndConditionsURL, config.ContactInfoURL, config.GeneralRequestURL) authRouter := router.PathPrefix("/api/v0/auth").Subrouter() diff --git a/satellite/console/service.go b/satellite/console/service.go index 097b3602a..99b435478 100644 --- a/satellite/console/service.go +++ b/satellite/console/service.go @@ -2684,6 +2684,38 @@ func (s *Service) GetAllBucketNames(ctx context.Context, projectID uuid.UUID) (_ return list, nil } +// GetTotalUsageReport retrieves usage rollups for every bucket of all user owned projects for a given period. +func (s *Service) GetTotalUsageReport(ctx context.Context, since, before time.Time) ([]accounting.BucketUsageRollup, error) { + var err error + defer mon.Task()(&ctx)(&err) + + user, err := s.getUserAndAuditLog(ctx, "get user report") + if err != nil { + return nil, Error.Wrap(err) + } + + projects, err := s.store.Projects().GetOwn(ctx, user.ID) + if err != nil { + return nil, Error.Wrap(err) + } + + usage := make([]accounting.BucketUsageRollup, 0) + + for _, p := range projects { + rollups, err := s.projectAccounting.GetBucketUsageRollups(ctx, p.ID, since, before) + if err != nil { + return nil, Error.Wrap(err) + } + + for i := range rollups { + rollups[i].ProjectID = p.PublicID + usage = append(usage, rollups[i]) + } + } + + return usage, nil +} + // GenGetBucketUsageRollups retrieves summed usage rollups for every bucket of particular project for a given period for generated api. func (s *Service) GenGetBucketUsageRollups(ctx context.Context, reqProjectID uuid.UUID, since, before time.Time) (rollups []accounting.BucketUsageRollup, httpError api.HTTPError) { var err error diff --git a/satellite/satellitedb/projectaccounting.go b/satellite/satellitedb/projectaccounting.go index 331c1e9cb..9858b7215 100644 --- a/satellite/satellitedb/projectaccounting.go +++ b/satellite/satellitedb/projectaccounting.go @@ -727,7 +727,7 @@ func (db *ProjectAccounting) GetSingleBucketUsageRollup(ctx context.Context, pro } func (db *ProjectAccounting) getSingleBucketRollup(ctx context.Context, projectID uuid.UUID, bucket string, since, before time.Time) (*accounting.BucketUsageRollup, error) { - roullupsQuery := db.db.Rebind(`SELECT SUM(settled), SUM(inline), action + rollupsQuery := db.db.Rebind(`SELECT SUM(settled), SUM(inline), action FROM bucket_bandwidth_rollups WHERE project_id = ? AND bucket_name = ? AND interval_start >= ? AND interval_start <= ? GROUP BY action`) @@ -743,7 +743,7 @@ func (db *ProjectAccounting) getSingleBucketRollup(ctx context.Context, projectI } // get bucket_bandwidth_rollup - rollupRows, err := db.db.QueryContext(ctx, roullupsQuery, projectID[:], []byte(bucket), since, before) + rollupRows, err := db.db.QueryContext(ctx, rollupsQuery, projectID[:], []byte(bucket), since, before) if err != nil { return nil, err } diff --git a/web/satellite/src/api/projects.ts b/web/satellite/src/api/projects.ts index c3ce926a4..5c036fb0f 100644 --- a/web/satellite/src/api/projects.ts +++ b/web/satellite/src/api/projects.ts @@ -199,6 +199,17 @@ export class ProjectsHttpApi implements ProjectsApi { ); } + /** + * Get link to download total usage report for all the projects that user owns. + * + * @throws Error + */ + public getTotalUsageReportLink(start: Date, end: Date): string { + const since = Time.toUnixTimestamp(start).toString(); + const before = Time.toUnixTimestamp(end).toString(); + return `${this.ROOT_PATH}/total-usage-report?since=${since}&before=${before}`; + } + /** * Get project daily usage for specific date range. * diff --git a/web/satellite/src/components/account/billing/billingTabs/Overview.vue b/web/satellite/src/components/account/billing/billingTabs/Overview.vue index c27b690d4..234e73ed2 100644 --- a/web/satellite/src/components/account/billing/billingTabs/Overview.vue +++ b/web/satellite/src/components/account/billing/billingTabs/Overview.vue @@ -2,93 +2,103 @@ // See LICENSE for copying information.