JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/{console, web}: added detailed usage report

Browse Source 
Allow user to download detailed usage report from Billing -> Overview screen.
Report is a CSV file containing usage data for all the projects user owns.

Issue:
https://github.com/storj/storj/issues/6154

Change-Id: I3109002bf37b1313652a2be3447aaa7bc6204887
This commit is contained in:
Vitalii 2023-10-05 17:24:58 +03:00 committed by Storj Robot
parent 41e16bc398
commit e3713fddb8
11 changed files with 358 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
satellite/accounting/db.go
View File

@ -5,6 +5,7 @@ package accounting
import (
"context"
"fmt"
"time"
"storj.io/common/memory"
@ -164,6 +165,23 @@ type BucketUsageRollup struct {
Before time.Time `json:"before"`
}
// ToStringSlice converts rollup values to a slice of strings.
func (b *BucketUsageRollup) ToStringSlice() []string {
return []string{
b.ProjectID.String(),
b.BucketName,
fmt.Sprintf("%f", b.TotalStoredData),
fmt.Sprintf("%f", b.TotalSegments),
fmt.Sprintf("%f", b.ObjectCount),
fmt.Sprintf("%f", b.MetadataSize),
fmt.Sprintf("%f", b.RepairEgress),
fmt.Sprintf("%f", b.GetEgress),
fmt.Sprintf("%f", b.AuditEgress),
b.Since.String(),
b.Before.String(),
}
}
// Usage contains project's usage split on segments and storage.
type Usage struct {
Storage int64

58
satellite/console/consoleweb/consoleapi/usagelimits.go
View File

@ -5,6 +5,7 @@ package consoleapi
import (
"context"
"encoding/csv"
"encoding/json"
"net/http"
"strconv"
@ -105,6 +106,63 @@ func (ul *UsageLimits) TotalUsageLimits(w http.ResponseWriter, r *http.Request)
}
}
// TotalUsageReport returns total usage report for all the projects that user owns.
func (ul *UsageLimits) TotalUsageReport(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
defer mon.Task()(&ctx)(&err)
sinceStamp, err := strconv.ParseInt(r.URL.Query().Get("since"), 10, 64)
if err != nil {
ul.serveJSONError(ctx, w, http.StatusBadRequest, err)
return
}
beforeStamp, err := strconv.ParseInt(r.URL.Query().Get("before"), 10, 64)
if err != nil {
ul.serveJSONError(ctx, w, http.StatusBadRequest, err)
return
}
since := time.Unix(sinceStamp, 0).UTC()
before := time.Unix(beforeStamp, 0).UTC()
usage, err := ul.service.GetTotalUsageReport(ctx, since, before)
if err != nil {
if console.ErrUnauthorized.Has(err) {
ul.serveJSONError(ctx, w, http.StatusUnauthorized, err)
return
}
ul.serveJSONError(ctx, w, http.StatusInternalServerError, err)
return
}
fileName := "storj-report-" + since.Format("2006-01-02") + "-to-" + before.Format("2006-01-02") + ".csv"
w.Header().Set("Content-Type", "text/csv")
w.Header().Set("Content-Disposition", "attachment;filename="+fileName)
wr := csv.NewWriter(w)
csvHeaders := []string{"ProjectID", "BucketName", "TotalStoredData GB-hour", "TotalSegments GB-hour", "ObjectCount GB-hour", "MetadataSize GB-hour", "RepairEgress GB", "GetEgress GB", "AuditEgress GB", "Since", "Before"}
err = wr.Write(csvHeaders)
if err != nil {
ul.serveJSONError(ctx, w, http.StatusInternalServerError, errs.New("Error writing CSV data"))
return
}
for _, u := range usage {
err = wr.Write(u.ToStringSlice())
if err != nil {
ul.serveJSONError(ctx, w, http.StatusInternalServerError, errs.New("Error writing CSV data"))
return
}
}
wr.Flush()
}
// DailyUsage returns daily usage by project ID.
func (ul *UsageLimits) DailyUsage(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()

94
satellite/console/consoleweb/consoleapi/usagelimits_test.go
View File

@ -4,10 +4,12 @@
package consoleapi_test
import (
"encoding/csv"
"encoding/json"
"fmt"
"net/http"
"strconv"
"strings"
"testing"
"time"
@ -21,6 +23,7 @@ import (
"storj.io/storj/satellite"
"storj.io/storj/satellite/accounting"
"storj.io/storj/satellite/console"
"storj.io/storj/satellite/metabase"
)
func Test_TotalUsageLimits(t *testing.T) {
@ -167,3 +170,94 @@ func Test_DailyUsage(t *testing.T) {
require.GreaterOrEqual(t, output.SettledBandwidthUsage[0].Value, 5*memory.KiB)
})
}
func Test_TotalUsageReport(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1, StorageNodeCount: 1, UplinkCount: 1,
Reconfigure: testplanet.Reconfigure{
Satellite: func(log *zap.Logger, index int, config *satellite.Config) {
config.Console.OpenRegistrationEnabled = true
config.Console.RateLimit.Burst = 10
},
},
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
var (
satelliteSys = planet.Satellites[0]
uplink = planet.Uplinks[0]
now = time.Now()
inFiveMinutes = now.Add(5 * time.Minute)
inAnHour = now.Add(1 * time.Hour)
since = fmt.Sprintf("%d", now.Unix())
before = fmt.Sprintf("%d", inAnHour.Unix())
expectedCSVValue = fmt.Sprintf("%f", float64(0))
)
newUser := console.CreateUser{
FullName: "Total Usage Report Test",
ShortName: "",
Email: "ur@test.test",
}
user, err := satelliteSys.AddUser(ctx, newUser, 3)
require.NoError(t, err)
project1, err := satelliteSys.AddProject(ctx, user.ID, "testProject1")
require.NoError(t, err)
project2, err := satelliteSys.AddProject(ctx, user.ID, "testProject2")
require.NoError(t, err)
bucketName := "bucket"
err = uplink.CreateBucket(ctx, satelliteSys, bucketName)
require.NoError(t, err)
bucketLoc1 := metabase.BucketLocation{
ProjectID: project1.ID,
BucketName: bucketName,
}
tally1 := &accounting.BucketTally{
BucketLocation: bucketLoc1,
}
bucketLoc2 := metabase.BucketLocation{
ProjectID: project2.ID,
BucketName: bucketName,
}
tally2 := &accounting.BucketTally{
BucketLocation: bucketLoc2,
}
bucketTallies := map[metabase.BucketLocation]*accounting.BucketTally{
bucketLoc1: tally1,
bucketLoc2: tally2,
}
err = satelliteSys.DB.ProjectAccounting().SaveTallies(ctx, inFiveMinutes, bucketTallies)
require.NoError(t, err)
endpoint := fmt.Sprintf("projects/total-usage-report?since=%s&before=%s", since, before)
body, status, err := doRequestWithAuth(ctx, t, satelliteSys, user, http.MethodGet, endpoint, nil)
require.NoError(t, err)
require.Equal(t, http.StatusOK, status)
content := string(body)
reader := csv.NewReader(strings.NewReader(content))
records, err := reader.ReadAll()
require.NoError(t, err)
require.Len(t, records, 3)
expectedHeaders := []string{"ProjectID", "BucketName", "TotalStoredData GB-hour", "TotalSegments GB-hour", "ObjectCount GB-hour", "MetadataSize GB-hour", "RepairEgress GB", "GetEgress GB", "AuditEgress GB", "Since", "Before"}
for i, header := range expectedHeaders {
require.Equal(t, header, records[0][i])
}
require.Equal(t, project1.PublicID.String(), records[1][0])
require.Equal(t, project2.PublicID.String(), records[2][0])
require.Equal(t, bucketName, records[1][1])
require.Equal(t, bucketName, records[2][1])
for i := 2; i < 9; i++ {
require.Equal(t, expectedCSVValue, records[1][i])
require.Equal(t, expectedCSVValue, records[2][i])
}
})
}

1
satellite/console/consoleweb/server.go
View File

@ -296,6 +296,7 @@ func NewServer(logger *zap.Logger, config Config, service *console.Service, oidc
projectsRouter.Handle("/{id}/usage-limits", http.HandlerFunc(usageLimitsController.ProjectUsageLimits)).Methods(http.MethodGet, http.MethodOptions)
projectsRouter.Handle("/usage-limits", http.HandlerFunc(usageLimitsController.TotalUsageLimits)).Methods(http.MethodGet, http.MethodOptions)
projectsRouter.Handle("/{id}/daily-usage", http.HandlerFunc(usageLimitsController.DailyUsage)).Methods(http.MethodGet, http.MethodOptions)
projectsRouter.Handle("/total-usage-report", http.HandlerFunc(usageLimitsController.TotalUsageReport)).Methods(http.MethodGet, http.MethodOptions)
authController := consoleapi.NewAuth(logger, service, accountFreezeService, mailService, server.cookieAuth, server.analytics, config.SatelliteName, server.config.ExternalAddress, config.LetUsKnowURL, config.TermsAndConditionsURL, config.ContactInfoURL, config.GeneralRequestURL)
authRouter := router.PathPrefix("/api/v0/auth").Subrouter()

32
satellite/console/service.go
View File

@ -2684,6 +2684,38 @@ func (s *Service) GetAllBucketNames(ctx context.Context, projectID uuid.UUID) (_
return list, nil
}
// GetTotalUsageReport retrieves usage rollups for every bucket of all user owned projects for a given period.
func (s *Service) GetTotalUsageReport(ctx context.Context, since, before time.Time) ([]accounting.BucketUsageRollup, error) {
var err error
defer mon.Task()(&ctx)(&err)
user, err := s.getUserAndAuditLog(ctx, "get user report")
if err != nil {
return nil, Error.Wrap(err)
}
projects, err := s.store.Projects().GetOwn(ctx, user.ID)
if err != nil {
return nil, Error.Wrap(err)
}
usage := make([]accounting.BucketUsageRollup, 0)
for _, p := range projects {
rollups, err := s.projectAccounting.GetBucketUsageRollups(ctx, p.ID, since, before)
if err != nil {
return nil, Error.Wrap(err)
}
for i := range rollups {
rollups[i].ProjectID = p.PublicID
usage = append(usage, rollups[i])
}
}
return usage, nil
}
// GenGetBucketUsageRollups retrieves summed usage rollups for every bucket of particular project for a given period for generated api.
func (s *Service) GenGetBucketUsageRollups(ctx context.Context, reqProjectID uuid.UUID, since, before time.Time) (rollups []accounting.BucketUsageRollup, httpError api.HTTPError) {
var err error

4
satellite/satellitedb/projectaccounting.go
View File

@ -727,7 +727,7 @@ func (db *ProjectAccounting) GetSingleBucketUsageRollup(ctx context.Context, pro
}
func (db *ProjectAccounting) getSingleBucketRollup(ctx context.Context, projectID uuid.UUID, bucket string, since, before time.Time) (*accounting.BucketUsageRollup, error) {
roullupsQuery := db.db.Rebind(`SELECT SUM(settled), SUM(inline), action
rollupsQuery := db.db.Rebind(`SELECT SUM(settled), SUM(inline), action
FROM bucket_bandwidth_rollups
WHERE project_id = ? AND bucket_name = ? AND interval_start >= ? AND interval_start <= ?
GROUP BY action`)
@ -743,7 +743,7 @@ func (db *ProjectAccounting) getSingleBucketRollup(ctx context.Context, projectI
}
// get bucket_bandwidth_rollup
rollupRows, err := db.db.QueryContext(ctx, roullupsQuery, projectID[:], []byte(bucket), since, before)
rollupRows, err := db.db.QueryContext(ctx, rollupsQuery, projectID[:], []byte(bucket), since, before)
if err != nil {
return nil, err
}

11
web/satellite/src/api/projects.ts
View File

@ -199,6 +199,17 @@ export class ProjectsHttpApi implements ProjectsApi {
);
}
/**
* Get link to download total usage report for all the projects that user owns.
*
* @throws Error
*/
public getTotalUsageReportLink(start: Date, end: Date): string {
const since = Time.toUnixTimestamp(start).toString();
const before = Time.toUnixTimestamp(end).toString();
return `${this.ROOT_PATH}/total-usage-report?since=${since}&before=${before}`;
}
/**
* Get project daily usage for specific date range.
*

45
web/satellite/src/components/account/billing/billingTabs/Overview.vue
View File

@ -2,7 +2,6 @@
// See LICENSE for copying information.
<template>
<div>
<div class="total-cost">
<div class="total-cost__header-container">
<h3 class="total-cost__header-container__title">Total Cost</h3>
@ -52,6 +51,18 @@
<p class="total-cost__card__label-text">Legacy STORJ Payments and Bonuses</p>
</div>
</div>
<div class="total-cost__report">
<h3 class="total-cost__report__title">Detailed Usage Report</h3>
<p class="total-cost__report__info">Get a complete usage report for all your projects.</p>
<v-button
class="total-cost__report__button"
label="Download Report"
width="fit-content"
height="30px"
is-transparent
:on-press="downloadUsageReport"
/>
</div>
</div>
<div v-if="isDataFetching">
<v-loader />
@ -88,7 +99,6 @@
/>
<router-view />
</div>
</div>
</template>
<script setup lang="ts">
@ -104,6 +114,7 @@ import { useNotify } from '@/utils/hooks';
import { useBillingStore } from '@/store/modules/billingStore';
import { useProjectsStore } from '@/store/modules/projectsStore';
import { useAnalyticsStore } from '@/store/modules/analyticsStore';
import { Download } from '@/utils/download';
import UsageAndChargesItem from '@/components/account/billing/billingTabs/UsageAndChargesItem.vue';
import VButton from '@/components/common/VButton.vue';
@ -172,6 +183,14 @@ function balanceClicked(): void {
});
}
/**
* Handles download usage report click logic.
*/
function downloadUsageReport(): void {
const link = projectsStore.getTotalUsageReportLink();
Download.fileByLink(link);
}
/**
* Lifecycle hook after initial render.
* Fetches projects and usage rollup.
@ -208,11 +227,32 @@ onMounted(async () => {
font-family: 'font_regular', sans-serif;
margin: 20px 0;
&__report {
box-shadow: 0 0 20px rgb(0 0 0 / 4%);
border-radius: 10px;
background-color: #fff;
padding: 20px;
margin-top: 20px;
&__title,
&__info {
margin-bottom: 10px;
}
&__button {
padding: 0 16px;
}
}
&__header-container {
display: flex;
justify-content: space-between;
align-items: center;
&__title {
padding-bottom: 10px;
}
&__date {
display: flex;
justify-content: space-between;
@ -230,7 +270,6 @@ onMounted(async () => {
display: grid;
grid-template-columns: 1fr 1fr 1fr;
gap: 10px;
margin-top: 20px;
@media screen and (width <= 786px) {
grid-template-columns: 1fr 1fr;

9
web/satellite/src/store/modules/projectsStore.ts
View File

@ -49,6 +49,14 @@ export const useProjectsStore = defineStore('projects', () => {
const api: ProjectsApi = new ProjectsHttpApi();
function getTotalUsageReportLink(): string {
const now = new Date();
const endUTC = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), now.getUTCHours(), now.getUTCMinutes()));
const startUTC = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1, 0, 0));
return api.getTotalUsageReportLink(startUTC, endUTC);
}
async function getProjects(): Promise<Project[]> {
const projects = await api.get();
@ -341,6 +349,7 @@ export const useProjectsStore = defineStore('projects', () => {
requestLimitIncrease,
getProjectLimits,
getTotalLimits,
getTotalUsageReportLink,
getProjectSalt,
getUserInvitations,
respondToInvitation,

7
web/satellite/src/types/projects.ts
View File

@ -64,6 +64,13 @@ export interface ProjectsApi {
*/
getTotalLimits(): Promise<ProjectLimits>;
/**
* Get link to download total usage report for all the projects that user owns.
*
* @throws Error
*/
getTotalUsageReportLink(start: Date, end: Date): string
/**
* Get project daily usage by specific date range.
*

6
web/satellite/src/utils/download.ts
View File

@ -14,4 +14,10 @@ export class Download {
elem.click();
document.body.removeChild(elem);
}
public static fileByLink(link: string): void {
const elem = window.document.createElement('a');
elem.href = link;
elem.click();
}
}