satellite/projectaccounting: query to get daily project usage by date range

Finished implementing queries for both bandwidth and storage using pgx.Batch. Fixed CSP styling issue. Change-Id: I5f9e10abe8096be3115b4e1f6ed3b13f1e7232df
2021-12-09 17:05:21 +02:00 · 2021-12-09 17:05:21 +02:00 · b3e1be37ff
commit b3e1be37ff
parent 4a26f0c4f1
9 changed files with 195 additions and 54 deletions
--- a/satellite/accounting/db.go
+++ b/satellite/accounting/db.go
@ -208,10 +208,8 @@ type ProjectAccounting interface {
 	GetProjectDailyBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, day int) (int64, int64, int64, error)
 	// DeleteProjectBandwidthBefore deletes project bandwidth rollups before the given time
 	DeleteProjectBandwidthBefore(ctx context.Context, before time.Time) error
-	// GetProjectDailyBandwidthByDateRange returns daily settled bandwidth usage for the specified date range.
-	GetProjectDailyBandwidthByDateRange(ctx context.Context, projectID uuid.UUID, from, to time.Time) ([]ProjectUsageByDay, error)
-	// GetProjectDailyStorageByDateRange returns daily storage usage for the specified date range.
-	GetProjectDailyStorageByDateRange(ctx context.Context, projectID uuid.UUID, from, to time.Time) ([]ProjectUsageByDay, error)
+	// GetProjectDailyUsageByDateRange returns daily allocated bandwidth and storage usage for the specified date range.
+	GetProjectDailyUsageByDateRange(ctx context.Context, projectID uuid.UUID, from, to time.Time, crdbInterval time.Duration) (*ProjectDailyUsage, error)

 	// UpdateProjectUsageLimit updates project usage limit.
 	UpdateProjectUsageLimit(ctx context.Context, projectID uuid.UUID, limit memory.Size) error
--- a/satellite/console/consoleweb/server.go
+++ b/satellite/console/consoleweb/server.go
@ -344,6 +344,8 @@ func (server *Server) appHandler(w http.ResponseWriter, r *http.Request) {
 			"frame-ancestors " + server.config.FrameAncestors,
 			"frame-src 'self' *.stripe.com https://www.google.com/recaptcha/ https://recaptcha.google.com/recaptcha/",
 			"img-src 'self' data: *.tardigradeshare.io *.storjshare.io",
+			// Those are hashes of charts custom tooltip inline styles. They have to be updated if styles are updated.
+			"style-src 'unsafe-hashes' 'sha256-7mY2NKmZ4PuyjGUa4FYC5u36SxXdoUM/zxrlr3BEToo=' 'sha256-PRTMwLUW5ce9tdiUrVCGKqj6wPeuOwGogb1pmyuXhgI=' 'sha256-kwpt3lQZ21rs4cld7/uEm9qI5yAbjYzx+9FGm/XmwNU=' 'self'",
 			"media-src 'self' *.tardigradeshare.io *.storjshare.io",
 			"script-src 'sha256-wAqYV6m2PHGd1WDyFBnZmSoyfCK0jxFAns0vGbdiWUA=' 'self' *.stripe.com https://www.google.com/recaptcha/ https://www.gstatic.com/recaptcha/",
 		}
--- a/satellite/console/service.go
+++ b/satellite/console/service.go
@ -134,6 +134,7 @@ type Config struct {
 	OpenRegistrationEnabled bool          `help:"enable open registration" default:"false" testDefault:"true"`
 	DefaultProjectLimit     int           `help:"default project limits for users" default:"1" testDefault:"5"`
 	TokenExpirationTime     time.Duration `help:"expiration time for auth tokens, account recovery tokens, and activation tokens" default:"24h"`
+	AsOfSystemTimeDuration  time.Duration `help:"default duration for AS OF SYSTEM TIME" devDefault:"-5m" releaseDefault:"-5m" testDefault:"0"`
 	UsageLimits             UsageLimitsConfig
 	Recaptcha               RecaptchaConfig
 }
@ -1676,20 +1677,12 @@ func (s *Service) GetDailyProjectUsage(ctx context.Context, projectID uuid.UUID,
 		return nil, Error.Wrap(err)
 	}

-	bandwidthUsage, err := s.projectAccounting.GetProjectDailyBandwidthByDateRange(ctx, projectID, from, to)
+	usage, err := s.projectAccounting.GetProjectDailyUsageByDateRange(ctx, projectID, from, to, s.config.AsOfSystemTimeDuration)
 	if err != nil {
 		return nil, Error.Wrap(err)
 	}

-	storageUsage, err := s.projectAccounting.GetProjectDailyStorageByDateRange(ctx, projectID, from, to)
-	if err != nil {
-		return nil, Error.Wrap(err)
-	}
-
-	return &accounting.ProjectDailyUsage{
-		StorageUsage:   storageUsage,
-		BandwidthUsage: bandwidthUsage,
-	}, nil
+	return usage, nil
 }

 // GetProjectUsageLimits returns project limits and current usage.
--- a/satellite/satellitedb/projectaccounting.go
+++ b/satellite/satellitedb/projectaccounting.go
@ -10,6 +10,7 @@ import (
 	"fmt"
 	"time"

+	"github.com/jackc/pgx/v4"
 	"github.com/zeebo/errs"

 	"storj.io/common/memory"
@ -17,6 +18,7 @@ import (
 	"storj.io/common/uuid"
 	"storj.io/private/dbutil"
 	"storj.io/private/dbutil/pgutil"
+	"storj.io/private/dbutil/pgxutil"
 	"storj.io/storj/satellite/accounting"
 	"storj.io/storj/satellite/metabase"
 	"storj.io/storj/satellite/orders"
@ -199,44 +201,100 @@ func (db *ProjectAccounting) GetProjectDailyBandwidth(ctx context.Context, proje
 	return allocated, settled, dead, err
 }

-// GetProjectDailyBandwidthByDateRange returns project daily settled bandwidth usage by specific date range.
-func (db *ProjectAccounting) GetProjectDailyBandwidthByDateRange(ctx context.Context, projectID uuid.UUID, from, to time.Time) (_ []accounting.ProjectUsageByDay, err error) {
+// GetProjectDailyUsageByDateRange returns project daily allocated bandwidth and storage usage by specific date range.
+func (db *ProjectAccounting) GetProjectDailyUsageByDateRange(ctx context.Context, projectID uuid.UUID, from, to time.Time, crdbInterval time.Duration) (_ *accounting.ProjectDailyUsage, err error) {
 	defer mon.Task()(&ctx)(&err)

-	usage := make([]accounting.ProjectUsageByDay, 0)
-	query := db.db.Rebind(`SELECT interval_day, COALESCE(egress_allocated, 0) FROM project_bandwidth_daily_rollups WHERE project_id = ? AND (interval_day BETWEEN ? AND ?)`)
-	rows, err := db.db.QueryContext(ctx, query, projectID[:], from, to)
-	if err != nil {
-		return nil, err
-	}
+	// use end of the day for 'to' caveat.
+	endOfDay := time.Date(to.Year(), to.Month(), to.Day(), 23, 59, 59, 0, time.UTC)

-	for rows.Next() {
-		var day time.Time
-		var amount int64
+	allocatedBandwidth := make([]accounting.ProjectUsageByDay, 0)
+	storage := make([]accounting.ProjectUsageByDay, 0)

-		err = rows.Scan(&day, &amount)
-		if err != nil {
-			return nil, errs.Combine(err, rows.Close())
+	err = pgxutil.Conn(ctx, db.db, func(conn *pgx.Conn) error {
+		var batch pgx.Batch
+
+		batch.Queue(db.db.Rebind(`
+			SELECT interval_day, COALESCE(egress_allocated, 0)
+			FROM project_bandwidth_daily_rollups
+			WHERE project_id = $1 AND (interval_day BETWEEN $2 AND $3)
+		`), projectID, from, endOfDay)
+
+		storageQuery := db.db.Rebind(`
+			WITH project_usage AS (
+			SELECT
+				interval_start,
+				DATE_TRUNC('day',interval_start) AS interval_day,
+				project_id,
+				bucket_name,
+				total_bytes
+			FROM bucket_storage_tallies
+			WHERE project_id = $1 AND
+				interval_start >= $2 AND
+				interval_start <= $3
+			)
+			-- Sum all buckets usage in the same project.
+			SELECT
+				interval_day,
+				SUM(total_bytes) AS total_bytes
+			FROM
+				(SELECT 
+					DISTINCT ON (project_id, bucket_name, interval_day)
+					project_id,
+					bucket_name,
+					total_bytes,
+					interval_day,
+					interval_start
+				FROM project_usage
+				ORDER BY project_id, bucket_name, interval_day, interval_start DESC) pu
+			` + db.db.impl.AsOfSystemInterval(crdbInterval) + `
+			GROUP BY project_id, bucket_name, interval_day
+		`)
+		batch.Queue(storageQuery, projectID, from, endOfDay)
+
+		results := conn.SendBatch(ctx, &batch)
+		defer func() { err = errs.Combine(err, results.Close()) }()
+
+		handleResult := func(results pgx.BatchResults, data *[]accounting.ProjectUsageByDay) error {
+			rows, err := results.Query()
+			if err != nil {
+				return err
+			}
+
+			for rows.Next() {
+				var day time.Time
+				var amount int64
+
+				err = rows.Scan(&day, &amount)
+				if err != nil {
+					return err
+				}
+
+				*data = append(*data, accounting.ProjectUsageByDay{
+					Date:  day,
+					Value: amount,
+				})
+			}
+
+			defer func() { rows.Close() }()
+
+			return rows.Err()
 		}

-		usage = append(usage, accounting.ProjectUsageByDay{
-			Date:  day,
-			Value: amount,
-		})
+		var errlist errs.Group
+		errlist.Add(handleResult(results, &allocatedBandwidth))
+		errlist.Add(handleResult(results, &storage))
+
+		return errlist.Err()
+	})
+	if err != nil {
+		return nil, Error.New("unable to get project daily usage: %w", err)
 	}

-	err = errs.Combine(rows.Err(), rows.Close())
-
-	return usage, err
-}
-
-// GetProjectDailyStorageByDateRange returns project daily storage usage by specific date range.
-func (db *ProjectAccounting) GetProjectDailyStorageByDateRange(ctx context.Context, _ uuid.UUID, _, _ time.Time) (_ []accounting.ProjectUsageByDay, err error) {
-	defer mon.Task()(&ctx)(&err)
-
-	usage := make([]accounting.ProjectUsageByDay, 0)
-
-	return usage, err
+	return &accounting.ProjectDailyUsage{
+		StorageUsage:   storage,
+		BandwidthUsage: allocatedBandwidth,
+	}, nil
 }

 // DeleteProjectBandwidthBefore deletes project bandwidth rollups before the given time.
--- a/satellite/satellitedb/projectaccounting_test.go
+++ b/satellite/satellitedb/projectaccounting_test.go
@ -0,0 +1,81 @@
+// Copyright (C) 2022 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package satellitedb_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"storj.io/common/memory"
+	"storj.io/common/testcontext"
+	"storj.io/common/testrand"
+	"storj.io/storj/private/testplanet"
+	"storj.io/storj/satellite/console"
+)
+
+func Test_DailyUsage(t *testing.T) {
+	testplanet.Run(t, testplanet.Config{SatelliteCount: 1, StorageNodeCount: 1, UplinkCount: 1},
+		func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
+			const (
+				bucketName = "testbucket"
+				firstPath  = "path"
+				secondPath = "another_path"
+			)
+
+			now := time.Now()
+			inFiveMinutes := time.Now().Add(5 * time.Minute)
+
+			var (
+				satelliteSys = planet.Satellites[0]
+				uplink       = planet.Uplinks[0]
+				projectID    = uplink.Projects[0].ID
+			)
+
+			newUser := console.CreateUser{
+				FullName:  "Project Daily Usage Test",
+				ShortName: "",
+				Email:     "du@test.test",
+			}
+
+			user, err := satelliteSys.AddUser(ctx, newUser, 3)
+			require.NoError(t, err)
+
+			_, err = satelliteSys.DB.Console().ProjectMembers().Insert(ctx, user.ID, projectID)
+			require.NoError(t, err)
+
+			planet.Satellites[0].Orders.Chore.Loop.Pause()
+			satelliteSys.Accounting.Tally.Loop.Pause()
+
+			usage0, err := satelliteSys.DB.ProjectAccounting().GetProjectDailyUsageByDateRange(ctx, projectID, now, inFiveMinutes, 0)
+			require.NoError(t, err)
+			require.Zero(t, len(usage0.BandwidthUsage))
+			require.Zero(t, len(usage0.StorageUsage))
+
+			firstSegment := testrand.Bytes(5 * memory.KiB)
+			secondSegment := testrand.Bytes(10 * memory.KiB)
+
+			err = uplink.Upload(ctx, satelliteSys, bucketName, firstPath, firstSegment)
+			require.NoError(t, err)
+			err = uplink.Upload(ctx, satelliteSys, bucketName, secondPath, secondSegment)
+			require.NoError(t, err)
+
+			_, err = uplink.Download(ctx, satelliteSys, bucketName, firstPath)
+			require.NoError(t, err)
+
+			require.NoError(t, planet.WaitForStorageNodeEndpoints(ctx))
+			tomorrow := time.Now().Add(24 * time.Hour)
+			planet.StorageNodes[0].Storage2.Orders.SendOrders(ctx, tomorrow)
+
+			planet.Satellites[0].Orders.Chore.Loop.TriggerWait()
+			satelliteSys.Accounting.Tally.Loop.TriggerWait()
+
+			usage1, err := satelliteSys.DB.ProjectAccounting().GetProjectDailyUsageByDateRange(ctx, projectID, now, inFiveMinutes, 0)
+			require.NoError(t, err)
+			require.GreaterOrEqual(t, usage1.StorageUsage[0].Value, 15*memory.KiB)
+			require.GreaterOrEqual(t, usage1.BandwidthUsage[0].Value, 5*memory.KiB)
+		},
+	)
+}
--- a/scripts/testdata/satellite-config.yaml.lock
+++ b/scripts/testdata/satellite-config.yaml.lock
@ -1,6 +1,9 @@
 # admin peer http listening address
 # admin.address: ""

+# default duration for AS OF SYSTEM TIME
+# admin.console-config.as-of-system-time-duration: -5m0s
+
 # default project limits for users
 # admin.console-config.default-project-limit: 1

@ -127,6 +130,9 @@ compensation.withheld-percents: 75,75,75,50,50,50,25,25,25,0,0,0,0,0,0
 # server address of the graphql api gateway and frontend app
 # console.address: :10100

+# default duration for AS OF SYSTEM TIME
+# console.as-of-system-time-duration: -5m0s
+
 # auth token needed for access to registration token creation endpoint
 # console.auth-token: ""

--- a/web/satellite/package-lock.json
+++ b/web/satellite/package-lock.json
@ -40,6 +40,7 @@
      "devDependencies": {
        "@babel/core": "7.14.8",
        "@babel/plugin-proposal-object-rest-spread": "7.11.0",
+        "@types/chart.js": "2.9.34",
        "@types/node": "13.11.1",
        "@types/pbkdf2": "3.1.0",
        "@types/qrcode": "1.4.1",
@ -8856,9 +8857,9 @@
      }
    },
    "node_modules/caniuse-lite": {
-      "version": "1.0.30001248",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001248.tgz",
-      "integrity": "sha512-NwlQbJkxUFJ8nMErnGtT0QTM2TJ33xgz4KXJSMIrjXIbDVdaYueGyjOrLKRtJC+rTiWfi6j5cnZN1NBiSBJGNw==",
+      "version": "1.0.30001301",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001301.tgz",
+      "integrity": "sha512-csfD/GpHMqgEL3V3uIgosvh+SVIQvCh43SNu9HRbP1lnxkKm1kjDG4f32PP571JplkLjfS+mg2p1gxR7MYrrIA==",
      "dev": true,
      "funding": {
        "type": "opencollective",
@ -37731,9 +37732,9 @@
      }
    },
    "caniuse-lite": {
-      "version": "1.0.30001248",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001248.tgz",
-      "integrity": "sha512-NwlQbJkxUFJ8nMErnGtT0QTM2TJ33xgz4KXJSMIrjXIbDVdaYueGyjOrLKRtJC+rTiWfi6j5cnZN1NBiSBJGNw==",
+      "version": "1.0.30001301",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001301.tgz",
+      "integrity": "sha512-csfD/GpHMqgEL3V3uIgosvh+SVIQvCh43SNu9HRbP1lnxkKm1kjDG4f32PP571JplkLjfS+mg2p1gxR7MYrrIA==",
      "dev": true
    },
    "capture-exit": {
--- a/web/satellite/package.json
+++ b/web/satellite/package.json
@ -45,6 +45,7 @@
  "devDependencies": {
    "@babel/core": "7.14.8",
    "@babel/plugin-proposal-object-rest-spread": "7.11.0",
+    "@types/chart.js": "2.9.34",
    "@types/node": "13.11.1",
    "@types/pbkdf2": "3.1.0",
    "@types/qrcode": "1.4.1",
--- a/web/satellite/src/components/project/newProjectDashboard/NewProjectDashboard.vue
+++ b/web/satellite/src/components/project/newProjectDashboard/NewProjectDashboard.vue
@ -243,8 +243,9 @@ export default class NewProjectDashboard extends Vue {
     * Used container size recalculation for charts resizing.
     */
    public recalculateChartWidth(): void {
-        const fiftyPixels = 50;
-        this.chartWidth = this.$refs.dashboard.getBoundingClientRect().width / 2 - fiftyPixels;
+        // sixty pixels.
+        const additionalPaddingRight = 60;
+        this.chartWidth = this.$refs.dashboard.getBoundingClientRect().width / 2 - additionalPaddingRight;
    }

    /**
@ -383,7 +384,7 @@ export default class NewProjectDashboard extends Vue {
    .project-dashboard {
        padding: 56px 55px 56px 40px;
        height: calc(100% - 112px);
-        max-width: calc(100vw - 280px - 80px);
+        max-width: calc(100vw - 280px - 95px);
        background-image: url('../../../../static/images/project/background.png');
        background-position: top right;
        background-size: 70%;
@ -526,7 +527,7 @@ export default class NewProjectDashboard extends Vue {
    @media screen and (max-width: 1280px) {

        .project-dashboard {
-            max-width: calc(100vw - 86px - 80px);
+            max-width: calc(100vw - 86px - 95px);
        }
    }
 </style>