satellite/{console, web}: added detailed usage report

Allow user to download detailed usage report from Billing -> Overview screen.
Report is a CSV file containing usage data for all the projects user owns.

Issue:
https://github.com/storj/storj/issues/6154

Change-Id: I3109002bf37b1313652a2be3447aaa7bc6204887

satellite/accounting/db.go

@@ -5,6 +5,7 @@ package accounting
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	"storj.io/common/memory"
@@ -164,6 +165,23 @@ type BucketUsageRollup struct {
 	Before time.Time `json:"before"`
 }
 
+// ToStringSlice converts rollup values to a slice of strings.
+func (b *BucketUsageRollup) ToStringSlice() []string {
+	return []string{
+		b.ProjectID.String(),
+		b.BucketName,
+		fmt.Sprintf("%f", b.TotalStoredData),
+		fmt.Sprintf("%f", b.TotalSegments),
+		fmt.Sprintf("%f", b.ObjectCount),
+		fmt.Sprintf("%f", b.MetadataSize),
+		fmt.Sprintf("%f", b.RepairEgress),
+		fmt.Sprintf("%f", b.GetEgress),
+		fmt.Sprintf("%f", b.AuditEgress),
+		b.Since.String(),
+		b.Before.String(),
+	}
+}
+
 // Usage contains project's usage split on segments and storage.
 type Usage struct {
 	Storage  int64

satellite/console/consoleweb/consoleapi/usagelimits.go

@@ -5,6 +5,7 @@ package consoleapi
 
 import (
 	"context"
+	"encoding/csv"
 	"encoding/json"
 	"net/http"
 	"strconv"
@@ -105,6 +106,63 @@ func (ul *UsageLimits) TotalUsageLimits(w http.ResponseWriter, r *http.Request)
 	}
 }
 
+// TotalUsageReport returns total usage report for all the projects that user owns.
+func (ul *UsageLimits) TotalUsageReport(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	var err error
+	defer mon.Task()(&ctx)(&err)
+
+	sinceStamp, err := strconv.ParseInt(r.URL.Query().Get("since"), 10, 64)
+	if err != nil {
+		ul.serveJSONError(ctx, w, http.StatusBadRequest, err)
+		return
+	}
+	beforeStamp, err := strconv.ParseInt(r.URL.Query().Get("before"), 10, 64)
+	if err != nil {
+		ul.serveJSONError(ctx, w, http.StatusBadRequest, err)
+		return
+	}
+
+	since := time.Unix(sinceStamp, 0).UTC()
+	before := time.Unix(beforeStamp, 0).UTC()
+
+	usage, err := ul.service.GetTotalUsageReport(ctx, since, before)
+	if err != nil {
+		if console.ErrUnauthorized.Has(err) {
+			ul.serveJSONError(ctx, w, http.StatusUnauthorized, err)
+			return
+		}
+
+		ul.serveJSONError(ctx, w, http.StatusInternalServerError, err)
+		return
+	}
+
+	fileName := "storj-report-" + since.Format("2006-01-02") + "-to-" + before.Format("2006-01-02") + ".csv"
+
+	w.Header().Set("Content-Type", "text/csv")
+	w.Header().Set("Content-Disposition", "attachment;filename="+fileName)
+
+	wr := csv.NewWriter(w)
+
+	csvHeaders := []string{"ProjectID", "BucketName", "TotalStoredData GB-hour", "TotalSegments GB-hour", "ObjectCount GB-hour", "MetadataSize GB-hour", "RepairEgress GB", "GetEgress GB", "AuditEgress GB", "Since", "Before"}
+
+	err = wr.Write(csvHeaders)
+	if err != nil {
+		ul.serveJSONError(ctx, w, http.StatusInternalServerError, errs.New("Error writing CSV data"))
+		return
+	}
+
+	for _, u := range usage {
+		err = wr.Write(u.ToStringSlice())
+		if err != nil {
+			ul.serveJSONError(ctx, w, http.StatusInternalServerError, errs.New("Error writing CSV data"))
+			return
+		}
+	}
+
+	wr.Flush()
+}
+
 // DailyUsage returns daily usage by project ID.
 func (ul *UsageLimits) DailyUsage(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

satellite/console/consoleweb/consoleapi/usagelimits_test.go

@@ -4,10 +4,12 @@
 package consoleapi_test
 
 import (
+	"encoding/csv"
 	"encoding/json"
 	"fmt"
 	"net/http"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -21,6 +23,7 @@ import (
 	"storj.io/storj/satellite"
 	"storj.io/storj/satellite/accounting"
 	"storj.io/storj/satellite/console"
+	"storj.io/storj/satellite/metabase"
 )
 
 func Test_TotalUsageLimits(t *testing.T) {
@@ -167,3 +170,94 @@ func Test_DailyUsage(t *testing.T) {
 		require.GreaterOrEqual(t, output.SettledBandwidthUsage[0].Value, 5*memory.KiB)
 	})
 }
+
+func Test_TotalUsageReport(t *testing.T) {
+	testplanet.Run(t, testplanet.Config{
+		SatelliteCount: 1, StorageNodeCount: 1, UplinkCount: 1,
+		Reconfigure: testplanet.Reconfigure{
+			Satellite: func(log *zap.Logger, index int, config *satellite.Config) {
+				config.Console.OpenRegistrationEnabled = true
+				config.Console.RateLimit.Burst = 10
+			},
+		},
+	}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
+		var (
+			satelliteSys     = planet.Satellites[0]
+			uplink           = planet.Uplinks[0]
+			now              = time.Now()
+			inFiveMinutes    = now.Add(5 * time.Minute)
+			inAnHour         = now.Add(1 * time.Hour)
+			since            = fmt.Sprintf("%d", now.Unix())
+			before           = fmt.Sprintf("%d", inAnHour.Unix())
+			expectedCSVValue = fmt.Sprintf("%f", float64(0))
+		)
+
+		newUser := console.CreateUser{
+			FullName:  "Total Usage Report Test",
+			ShortName: "",
+			Email:     "ur@test.test",
+		}
+
+		user, err := satelliteSys.AddUser(ctx, newUser, 3)
+		require.NoError(t, err)
+
+		project1, err := satelliteSys.AddProject(ctx, user.ID, "testProject1")
+		require.NoError(t, err)
+
+		project2, err := satelliteSys.AddProject(ctx, user.ID, "testProject2")
+		require.NoError(t, err)
+
+		bucketName := "bucket"
+		err = uplink.CreateBucket(ctx, satelliteSys, bucketName)
+		require.NoError(t, err)
+
+		bucketLoc1 := metabase.BucketLocation{
+			ProjectID:  project1.ID,
+			BucketName: bucketName,
+		}
+		tally1 := &accounting.BucketTally{
+			BucketLocation: bucketLoc1,
+		}
+
+		bucketLoc2 := metabase.BucketLocation{
+			ProjectID:  project2.ID,
+			BucketName: bucketName,
+		}
+		tally2 := &accounting.BucketTally{
+			BucketLocation: bucketLoc2,
+		}
+
+		bucketTallies := map[metabase.BucketLocation]*accounting.BucketTally{
+			bucketLoc1: tally1,
+			bucketLoc2: tally2,
+		}
+
+		err = satelliteSys.DB.ProjectAccounting().SaveTallies(ctx, inFiveMinutes, bucketTallies)
+		require.NoError(t, err)
+
+		endpoint := fmt.Sprintf("projects/total-usage-report?since=%s&before=%s", since, before)
+		body, status, err := doRequestWithAuth(ctx, t, satelliteSys, user, http.MethodGet, endpoint, nil)
+		require.NoError(t, err)
+		require.Equal(t, http.StatusOK, status)
+
+		content := string(body)
+		reader := csv.NewReader(strings.NewReader(content))
+		records, err := reader.ReadAll()
+		require.NoError(t, err)
+		require.Len(t, records, 3)
+
+		expectedHeaders := []string{"ProjectID", "BucketName", "TotalStoredData GB-hour", "TotalSegments GB-hour", "ObjectCount GB-hour", "MetadataSize GB-hour", "RepairEgress GB", "GetEgress GB", "AuditEgress GB", "Since", "Before"}
+		for i, header := range expectedHeaders {
+			require.Equal(t, header, records[0][i])
+		}
+
+		require.Equal(t, project1.PublicID.String(), records[1][0])
+		require.Equal(t, project2.PublicID.String(), records[2][0])
+		require.Equal(t, bucketName, records[1][1])
+		require.Equal(t, bucketName, records[2][1])
+		for i := 2; i < 9; i++ {
+			require.Equal(t, expectedCSVValue, records[1][i])
+			require.Equal(t, expectedCSVValue, records[2][i])
+		}
+	})
+}

satellite/console/consoleweb/server.go

@@ -296,6 +296,7 @@ func NewServer(logger *zap.Logger, config Config, service *console.Service, oidc
 	projectsRouter.Handle("/{id}/usage-limits", http.HandlerFunc(usageLimitsController.ProjectUsageLimits)).Methods(http.MethodGet, http.MethodOptions)
 	projectsRouter.Handle("/usage-limits", http.HandlerFunc(usageLimitsController.TotalUsageLimits)).Methods(http.MethodGet, http.MethodOptions)
 	projectsRouter.Handle("/{id}/daily-usage", http.HandlerFunc(usageLimitsController.DailyUsage)).Methods(http.MethodGet, http.MethodOptions)
+	projectsRouter.Handle("/total-usage-report", http.HandlerFunc(usageLimitsController.TotalUsageReport)).Methods(http.MethodGet, http.MethodOptions)
 
 	authController := consoleapi.NewAuth(logger, service, accountFreezeService, mailService, server.cookieAuth, server.analytics, config.SatelliteName, server.config.ExternalAddress, config.LetUsKnowURL, config.TermsAndConditionsURL, config.ContactInfoURL, config.GeneralRequestURL)
 	authRouter := router.PathPrefix("/api/v0/auth").Subrouter()

satellite/console/service.go

@@ -2684,6 +2684,38 @@ func (s *Service) GetAllBucketNames(ctx context.Context, projectID uuid.UUID) (_
 	return list, nil
 }
 
+// GetTotalUsageReport retrieves usage rollups for every bucket of all user owned projects for a given period.
+func (s *Service) GetTotalUsageReport(ctx context.Context, since, before time.Time) ([]accounting.BucketUsageRollup, error) {
+	var err error
+	defer mon.Task()(&ctx)(&err)
+
+	user, err := s.getUserAndAuditLog(ctx, "get user report")
+	if err != nil {
+		return nil, Error.Wrap(err)
+	}
+
+	projects, err := s.store.Projects().GetOwn(ctx, user.ID)
+	if err != nil {
+		return nil, Error.Wrap(err)
+	}
+
+	usage := make([]accounting.BucketUsageRollup, 0)
+
+	for _, p := range projects {
+		rollups, err := s.projectAccounting.GetBucketUsageRollups(ctx, p.ID, since, before)
+		if err != nil {
+			return nil, Error.Wrap(err)
+		}
+
+		for i := range rollups {
+			rollups[i].ProjectID = p.PublicID
+			usage = append(usage, rollups[i])
+		}
+	}
+
+	return usage, nil
+}
+
 // GenGetBucketUsageRollups retrieves summed usage rollups for every bucket of particular project for a given period for generated api.
 func (s *Service) GenGetBucketUsageRollups(ctx context.Context, reqProjectID uuid.UUID, since, before time.Time) (rollups []accounting.BucketUsageRollup, httpError api.HTTPError) {
 	var err error

satellite/satellitedb/projectaccounting.go

@@ -727,7 +727,7 @@ func (db *ProjectAccounting) GetSingleBucketUsageRollup(ctx context.Context, pro
 }
 
 func (db *ProjectAccounting) getSingleBucketRollup(ctx context.Context, projectID uuid.UUID, bucket string, since, before time.Time) (*accounting.BucketUsageRollup, error) {
-	roullupsQuery := db.db.Rebind(`SELECT SUM(settled), SUM(inline), action
+	rollupsQuery := db.db.Rebind(`SELECT SUM(settled), SUM(inline), action
 		FROM bucket_bandwidth_rollups
 		WHERE project_id = ? AND bucket_name = ? AND interval_start >= ? AND interval_start <= ?
 		GROUP BY action`)
@@ -743,7 +743,7 @@ func (db *ProjectAccounting) getSingleBucketRollup(ctx context.Context, projectI
 	}
 
 	// get bucket_bandwidth_rollup
-	rollupRows, err := db.db.QueryContext(ctx, roullupsQuery, projectID[:], []byte(bucket), since, before)
+	rollupRows, err := db.db.QueryContext(ctx, rollupsQuery, projectID[:], []byte(bucket), since, before)
 	if err != nil {
 		return nil, err
 	}

web/satellite/src/api/projects.ts

@@ -199,6 +199,17 @@ export class ProjectsHttpApi implements ProjectsApi {
         );
     }
 
+    /**
+     * Get link to download total usage report for all the projects that user owns.
+     *
+     * @throws Error
+     */
+    public getTotalUsageReportLink(start: Date, end: Date): string {
+        const since = Time.toUnixTimestamp(start).toString();
+        const before = Time.toUnixTimestamp(end).toString();
+        return `${this.ROOT_PATH}/total-usage-report?since=${since}&before=${before}`;
+    }
+
     /**
      * Get project daily usage for specific date range.
      *

web/satellite/src/components/account/billing/billingTabs/Overview.vue

@@ -2,93 +2,103 @@
 // See LICENSE for copying information.
 
 <template>
-    <div>
+    <div class="total-cost">
-        <div class="total-cost">
+        <div class="total-cost__header-container">
-            <div class="total-cost__header-container">
+            <h3 class="total-cost__header-container__title">Total Cost</h3>
-                <h3 class="total-cost__header-container__title">Total Cost</h3>
+            <div class="total-cost__header-container__date"><CalendarIcon />&nbsp;&nbsp;{{ currentDate }}</div>
-                <div class="total-cost__header-container__date"><CalendarIcon />&nbsp;&nbsp;{{ currentDate }}</div>
+        </div>
+        <div class="total-cost__card-container">
+            <div class="total-cost__card">
+                <EstimatedChargesIcon class="total-cost__card__main-icon" />
+                <p class="total-cost__card__money-text">{{ centsToDollars(priceSummary) }}</p>
+                <p class="total-cost__card__label-text">
+                    Total Estimated Usage
+                    <VInfo class="total-cost__card__label-text__info">
+                        <template #icon>
+                            <InfoIcon />
+                        </template>
+                        <template #message>
+                            <span class="total-cost__card__label-text__info__inner">
+                                This estimate includes all use before subtracting any discounts.
+                                Pro accounts will only be charged for usage above the free tier limits,
+                                and free accounts will not be charged.
+                            </span>
+                        </template>
+                    </VInfo>
+                </p>
+                <p
+                    class="total-cost__card__link-text"
+                    @click="routeToBillingHistory"
+                >
+                    View Billing History →
+                </p>
+            </div>
+            <div class="total-cost__card">
+                <AvailableBalanceIcon class="total-cost__card__main-icon" />
+                <p class="total-cost__card__money-text">{{ balance.formattedCoins }}</p>
+                <p class="total-cost__card__label-text">STORJ Token Balance</p>
+                <p
+                    class="total-cost__card__link-text"
+                    @click="balanceClicked"
+                >
+                    {{ hasZeroCoins ? "Add Funds" : "See Balance" }} →
+                </p>
             </div>
-            <div class="total-cost__card-container">
-                <div class="total-cost__card">
-                    <EstimatedChargesIcon class="total-cost__card__main-icon" />
-                    <p class="total-cost__card__money-text">{{ centsToDollars(priceSummary) }}</p>
-                    <p class="total-cost__card__label-text">
-                        Total Estimated Usage
-                        <VInfo class="total-cost__card__label-text__info">
-                            <template #icon>
-                                <InfoIcon />
-                            </template>
-                            <template #message>
-                                <span class="total-cost__card__label-text__info__inner">
-                                    This estimate includes all use before subtracting any discounts.
-                                    Pro accounts will only be charged for usage above the free tier limits,
-                                    and free accounts will not be charged.
-                                </span>
-                            </template>
-                        </VInfo>
-                    </p>
-                    <p
-                        class="total-cost__card__link-text"
-                        @click="routeToBillingHistory"
-                    >
-                        View Billing History →
-                    </p>
-                </div>
-                <div class="total-cost__card">
-                    <AvailableBalanceIcon class="total-cost__card__main-icon" />
-                    <p class="total-cost__card__money-text">{{ balance.formattedCoins }}</p>
-                    <p class="total-cost__card__label-text">STORJ Token Balance</p>
-                    <p
-                        class="total-cost__card__link-text"
-                        @click="balanceClicked"
-                    >
-                        {{ hasZeroCoins ? "Add Funds" : "See Balance" }} →
-                    </p>
-                </div>
 
-                <div v-if="balance.hasCredits()" class="total-cost__card">
+            <div v-if="balance.hasCredits()" class="total-cost__card">
-                    <AvailableBalanceIcon class="total-cost__card__main-icon" />
+                <AvailableBalanceIcon class="total-cost__card__main-icon" />
-                    <p class="total-cost__card__money-text">{{ balance.formattedCredits }}</p>
+                <p class="total-cost__card__money-text">{{ balance.formattedCredits }}</p>
-                    <p class="total-cost__card__label-text">Legacy STORJ Payments and Bonuses</p>
+                <p class="total-cost__card__label-text">Legacy STORJ Payments and Bonuses</p>
-                </div>
             </div>
         </div>
-        <div v-if="isDataFetching">
+        <div class="total-cost__report">
-            <v-loader />
+            <h3 class="total-cost__report__title">Detailed Usage Report</h3>
-        </div>
+            <p class="total-cost__report__info">Get a complete usage report for all your projects.</p>
-        <div v-else class="cost-by-project">
+            <v-button
-            <h3 class="cost-by-project__title">Cost by Project</h3>
+                class="total-cost__report__button"
-            <div class="cost-by-project__buttons">
+                label="Download Report"
-                <v-button
+                width="fit-content"
-                    label="Edit Payment Method"
+                height="30px"
-                    font-size="13px"
+                is-transparent
-                    width="auto"
+                :on-press="downloadUsageReport"
-                    height="30px"
-                    icon="lock"
-                    :is-transparent="true"
-                    class="cost-by-project__buttons__none-assigned"
-                    :on-press="routeToPaymentMethods"
-                />
-                <v-button
-                    label="See Payments"
-                    font-size="13px"
-                    width="auto"
-                    height="30px"
-                    icon="document"
-                    :is-transparent="true"
-                    class="cost-by-project__buttons__none-assigned"
-                    :on-press="routeToBillingHistory"
-                />
-            </div>
-            <UsageAndChargesItem
-                v-for="id in projectIDs"
-                :key="id"
-                :project-id="id"
-                class="cost-by-project__item"
             />
-            <router-view />
         </div>
     </div>
+    <div v-if="isDataFetching">
+        <v-loader />
+    </div>
+    <div v-else class="cost-by-project">
+        <h3 class="cost-by-project__title">Cost by Project</h3>
+        <div class="cost-by-project__buttons">
+            <v-button
+                label="Edit Payment Method"
+                font-size="13px"
+                width="auto"
+                height="30px"
+                icon="lock"
+                :is-transparent="true"
+                class="cost-by-project__buttons__none-assigned"
+                :on-press="routeToPaymentMethods"
+            />
+            <v-button
+                label="See Payments"
+                font-size="13px"
+                width="auto"
+                height="30px"
+                icon="document"
+                :is-transparent="true"
+                class="cost-by-project__buttons__none-assigned"
+                :on-press="routeToBillingHistory"
+            />
+        </div>
+        <UsageAndChargesItem
+            v-for="id in projectIDs"
+            :key="id"
+            :project-id="id"
+            class="cost-by-project__item"
+        />
+        <router-view />
+    </div>
 </template>
 
 <script setup lang="ts">
@@ -104,6 +114,7 @@ import { useNotify } from '@/utils/hooks';
 import { useBillingStore } from '@/store/modules/billingStore';
 import { useProjectsStore } from '@/store/modules/projectsStore';
 import { useAnalyticsStore } from '@/store/modules/analyticsStore';
+import { Download } from '@/utils/download';
 
 import UsageAndChargesItem from '@/components/account/billing/billingTabs/UsageAndChargesItem.vue';
 import VButton from '@/components/common/VButton.vue';
@@ -172,6 +183,14 @@ function balanceClicked(): void {
     });
 }
 
+/**
+ * Handles download usage report click logic.
+ */
+function downloadUsageReport(): void {
+    const link = projectsStore.getTotalUsageReportLink();
+    Download.fileByLink(link);
+}
+
 /**
  * Lifecycle hook after initial render.
  * Fetches projects and usage rollup.
@@ -208,11 +227,32 @@ onMounted(async () => {
         font-family: 'font_regular', sans-serif;
         margin: 20px 0;
 
+        &__report {
+            box-shadow: 0 0 20px rgb(0 0 0 / 4%);
+            border-radius: 10px;
+            background-color: #fff;
+            padding: 20px;
+            margin-top: 20px;
+
+            &__title,
+            &__info {
+                margin-bottom: 10px;
+            }
+
+            &__button {
+                padding: 0 16px;
+            }
+        }
+
         &__header-container {
             display: flex;
             justify-content: space-between;
             align-items: center;
 
+            &__title {
+                padding-bottom: 10px;
+            }
+
             &__date {
                 display: flex;
                 justify-content: space-between;
@@ -230,7 +270,6 @@ onMounted(async () => {
             display: grid;
             grid-template-columns: 1fr 1fr 1fr;
             gap: 10px;
-            margin-top: 20px;
 
             @media screen and (width <= 786px) {
                 grid-template-columns: 1fr 1fr;

web/satellite/src/store/modules/projectsStore.ts

@@ -49,6 +49,14 @@ export const useProjectsStore = defineStore('projects', () => {
 
     const api: ProjectsApi = new ProjectsHttpApi();
 
+    function getTotalUsageReportLink(): string {
+        const now = new Date();
+        const endUTC = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), now.getUTCHours(), now.getUTCMinutes()));
+        const startUTC = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1, 0, 0));
+
+        return api.getTotalUsageReportLink(startUTC, endUTC);
+    }
+
     async function getProjects(): Promise<Project[]> {
         const projects = await api.get();
 
@@ -341,6 +349,7 @@ export const useProjectsStore = defineStore('projects', () => {
         requestLimitIncrease,
         getProjectLimits,
         getTotalLimits,
+        getTotalUsageReportLink,
         getProjectSalt,
         getUserInvitations,
         respondToInvitation,

web/satellite/src/types/projects.ts

@@ -64,6 +64,13 @@ export interface ProjectsApi {
      */
     getTotalLimits(): Promise<ProjectLimits>;
 
+    /**
+     * Get link to download total usage report for all the projects that user owns.
+     *
+     * @throws Error
+     */
+    getTotalUsageReportLink(start: Date, end: Date): string
+
     /**
      * Get project daily usage by specific date range.
      *

web/satellite/src/utils/download.ts

@@ -14,4 +14,10 @@ export class Download {
         elem.click();
         document.body.removeChild(elem);
     }
+
+    public static fileByLink(link: string): void {
+        const elem = window.document.createElement('a');
+        elem.href = link;
+        elem.click();
+    }
 }