From a00ec7af40914e37b6ce185b6ef93c29e48baa8f Mon Sep 17 00:00:00 2001 From: Jeremy Wharton Date: Tue, 8 Aug 2023 00:32:25 -0500 Subject: [PATCH] satellite/console: create http endpoint for getting bucket usage totals This change introduces an HTTP endpoint for retrieving bucket usage totals. In the future, this will replace its GraphQL counterpart. References #6141 Change-Id: Ic6a0069a7e58b90dc2b6c55f164393f036c6acf4 --- satellite/accounting/db.go | 30 ++++---- .../console/consoleweb/consoleapi/buckets.go | 77 +++++++++++++++++++ .../console/consoleweb/endpoints_test.go | 26 +++++++ satellite/console/consoleweb/server.go | 1 + satellite/console/service.go | 4 +- 5 files changed, 121 insertions(+), 17 deletions(-) diff --git a/satellite/accounting/db.go b/satellite/accounting/db.go index 74e666d61..b9ae36658 100644 --- a/satellite/accounting/db.go +++ b/satellite/accounting/db.go @@ -111,16 +111,16 @@ type ProjectUsageByDay struct { // BucketUsage consist of total bucket usage for period. type BucketUsage struct { - ProjectID uuid.UUID - BucketName string + ProjectID uuid.UUID `json:"projectID"` + BucketName string `json:"bucketName"` - Storage float64 - Egress float64 - ObjectCount int64 - SegmentCount int64 + Storage float64 `json:"storage"` + Egress float64 `json:"egress"` + ObjectCount int64 `json:"objectCount"` + SegmentCount int64 `json:"segmentCount"` - Since time.Time - Before time.Time + Since time.Time `json:"since"` + Before time.Time `json:"before"` } // BucketUsageCursor holds info for bucket usage @@ -133,15 +133,15 @@ type BucketUsageCursor struct { // BucketUsagePage represents bucket usage page result. type BucketUsagePage struct { - BucketUsages []BucketUsage + BucketUsages []BucketUsage `json:"bucketUsages"` - Search string - Limit uint - Offset uint64 + Search string `json:"search"` + Limit uint `json:"limit"` + Offset uint64 `json:"offset"` - PageCount uint - CurrentPage uint - TotalCount uint64 + PageCount uint `json:"pageCount"` + CurrentPage uint `json:"currentPage"` + TotalCount uint64 `json:"totalCount"` } // BucketUsageRollup is total bucket usage info diff --git a/satellite/console/consoleweb/consoleapi/buckets.go b/satellite/console/consoleweb/consoleapi/buckets.go index 3dacf67e6..0087330f0 100644 --- a/satellite/console/consoleweb/consoleapi/buckets.go +++ b/satellite/console/consoleweb/consoleapi/buckets.go @@ -7,15 +7,23 @@ import ( "context" "encoding/json" "net/http" + "strconv" + "time" "github.com/zeebo/errs" "go.uber.org/zap" "storj.io/common/uuid" "storj.io/storj/private/web" + "storj.io/storj/satellite/accounting" "storj.io/storj/satellite/console" ) +const ( + missingParamErrMsg = "missing '%s' query parameter" + invalidParamErrMsg = "invalid value '%s' for query parameter '%s': %w" +) + var ( // ErrBucketsAPI - console buckets api error type. ErrBucketsAPI = errs.Class("console api buckets") @@ -81,6 +89,75 @@ func (b *Buckets) AllBucketNames(w http.ResponseWriter, r *http.Request) { } } +// GetBucketTotals returns a page of bucket usage totals since project creation. +func (b *Buckets) GetBucketTotals(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + var err error + defer mon.Task()(&ctx)(&err) + + w.Header().Set("Content-Type", "application/json") + + projectIDString := r.URL.Query().Get("projectID") + if projectIDString == "" { + b.serveJSONError(ctx, w, http.StatusBadRequest, errs.New(missingParamErrMsg, "projectID")) + return + } + projectID, err := uuid.FromString(projectIDString) + if err != nil { + b.serveJSONError(ctx, w, http.StatusBadRequest, errs.New(invalidParamErrMsg, projectIDString, "projectID", err)) + return + } + + beforeString := r.URL.Query().Get("before") + if beforeString == "" { + b.serveJSONError(ctx, w, http.StatusBadRequest, errs.New(missingParamErrMsg, "before")) + return + } + before, err := time.Parse(dateLayout, beforeString) + if err != nil { + b.serveJSONError(ctx, w, http.StatusBadRequest, errs.New(invalidParamErrMsg, beforeString, "before", err)) + return + } + + limitString := r.URL.Query().Get("limit") + if limitString == "" { + b.serveJSONError(ctx, w, http.StatusBadRequest, errs.New(missingParamErrMsg, "limit")) + return + } + limitU64, err := strconv.ParseUint(limitString, 10, 32) + if err != nil { + b.serveJSONError(ctx, w, http.StatusBadRequest, errs.New(invalidParamErrMsg, limitString, "limit", err)) + return + } + limit := uint(limitU64) + + pageString := r.URL.Query().Get("page") + if pageString == "" { + b.serveJSONError(ctx, w, http.StatusBadRequest, errs.New(missingParamErrMsg, "page")) + return + } + pageU64, err := strconv.ParseUint(pageString, 10, 32) + if err != nil { + b.serveJSONError(ctx, w, http.StatusBadRequest, errs.New(invalidParamErrMsg, pageString, "page", err)) + return + } + page := uint(pageU64) + + totals, err := b.service.GetBucketTotals(ctx, projectID, accounting.BucketUsageCursor{ + Limit: limit, + Search: r.URL.Query().Get("search"), + Page: page, + }, before) + if err != nil { + b.serveJSONError(ctx, w, http.StatusInternalServerError, err) + } + + err = json.NewEncoder(w).Encode(totals) + if err != nil { + b.log.Error("failed to write json bucket totals response", zap.Error(ErrBucketsAPI.Wrap(err))) + } +} + // serveJSONError writes JSON error to response output stream. func (b *Buckets) serveJSONError(ctx context.Context, w http.ResponseWriter, status int, err error) { web.ServeJSONError(ctx, b.log, w, status, err) diff --git a/satellite/console/consoleweb/endpoints_test.go b/satellite/console/consoleweb/endpoints_test.go index 96251267b..eeb954098 100644 --- a/satellite/console/consoleweb/endpoints_test.go +++ b/satellite/console/consoleweb/endpoints_test.go @@ -19,7 +19,9 @@ import ( "storj.io/common/testcontext" "storj.io/common/uuid" + "storj.io/storj/private/apigen" "storj.io/storj/private/testplanet" + "storj.io/storj/satellite/accounting" "storj.io/storj/satellite/console" "storj.io/storj/satellite/payments/storjscan/blockchaintest" ) @@ -368,6 +370,30 @@ func TestBuckets(t *testing.T) { }`})) require.Contains(t, body, "bucketUsagePage") require.Equal(t, http.StatusOK, resp.StatusCode) + + params := url.Values{ + "projectID": {test.defaultProjectID()}, + "before": {time.Now().Add(time.Second).Format(apigen.DateFormat)}, + "limit": {"1"}, + "search": {""}, + "page": {"1"}, + } + + resp, body = test.request(http.MethodGet, "/buckets/usage-totals?"+params.Encode(), nil) + require.Equal(t, http.StatusOK, resp.StatusCode) + var page accounting.BucketUsagePage + require.NoError(t, json.Unmarshal([]byte(body), &page)) + require.Empty(t, page.BucketUsages) + + const bucketName = "my-bucket" + require.NoError(t, planet.Uplinks[0].CreateBucket(ctx, planet.Satellites[0], bucketName)) + + resp, body = test.request(http.MethodGet, "/buckets/usage-totals?"+params.Encode(), nil) + require.Equal(t, http.StatusOK, resp.StatusCode) + + require.NoError(t, json.Unmarshal([]byte(body), &page)) + require.NotEmpty(t, page.BucketUsages) + require.Equal(t, bucketName, page.BucketUsages[0].BucketName) } }) } diff --git a/satellite/console/consoleweb/server.go b/satellite/console/consoleweb/server.go index bd1057629..b49524e4e 100644 --- a/satellite/console/consoleweb/server.go +++ b/satellite/console/consoleweb/server.go @@ -352,6 +352,7 @@ func NewServer(logger *zap.Logger, config Config, service *console.Service, oidc bucketsRouter.Use(server.withCORS) bucketsRouter.Use(server.withAuth) bucketsRouter.HandleFunc("/bucket-names", bucketsController.AllBucketNames).Methods(http.MethodGet, http.MethodOptions) + bucketsRouter.HandleFunc("/usage-totals", bucketsController.GetBucketTotals).Methods(http.MethodGet, http.MethodOptions) apiKeysController := consoleapi.NewAPIKeys(logger, service) apiKeysRouter := router.PathPrefix("/api/v0/api-keys").Subrouter() diff --git a/satellite/console/service.go b/satellite/console/service.go index a2ee4a55a..d53788861 100644 --- a/satellite/console/service.go +++ b/satellite/console/service.go @@ -2567,12 +2567,12 @@ func (s *Service) GetBucketTotals(ctx context.Context, projectID uuid.UUID, curs return nil, Error.Wrap(err) } - _, err = s.isProjectMember(ctx, user.ID, projectID) + isMember, err := s.isProjectMember(ctx, user.ID, projectID) if err != nil { return nil, Error.Wrap(err) } - usage, err := s.projectAccounting.GetBucketTotals(ctx, projectID, cursor, before) + usage, err := s.projectAccounting.GetBucketTotals(ctx, isMember.project.ID, cursor, before) if err != nil { return nil, Error.Wrap(err) }