JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/console: project usage limits api (#3702)

Browse Source
This commit is contained in:
Yaroslav Vorobiov 2019-12-12 14:58:15 +02:00 committed by GitHub
parent 9d8f6a6d39
commit 77839dd41b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 211 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
satellite/accounting/projectusage.go
View File

@ -57,20 +57,16 @@ func (usage *Service) ExceedsBandwidthUsage(ctx context.Context, projectID uuid.
var group errgroup.Group
var bandwidthGetTotal int64
limit = usage.maxAlphaUsage
// TODO(michal): to reduce db load, consider using a cache to retrieve the project.UsageLimit value if needed
group.Go(func() error {
projectLimit, err := usage.projectAccountingDB.GetProjectBandwidthLimit(ctx, projectID)
if projectLimit > 0 {
limit = projectLimit
}
var err error
limit, err = usage.GetProjectBandwidthLimit(ctx, projectID)
return err
})
group.Go(func() error {
var err error
from := time.Now().AddDate(0, 0, -AverageDaysInMonth) // past 30 days
bandwidthGetTotal, err = usage.projectAccountingDB.GetAllocatedBandwidthTotal(ctx, projectID, from)
bandwidthGetTotal, err = usage.GetProjectBandwidthTotals(ctx, projectID)
return err
})
@ -96,21 +92,19 @@ func (usage *Service) ExceedsStorageUsage(ctx context.Context, projectID uuid.UU
var group errgroup.Group
var totalUsed int64
limit = usage.maxAlphaUsage
// TODO(michal): to reduce db load, consider using a cache to retrieve the project.UsageLimit value if needed
group.Go(func() error {
projectLimit, err := usage.projectAccountingDB.GetProjectStorageLimit(ctx, projectID)
if projectLimit > 0 {
limit = projectLimit
}
var err error
limit, err = usage.GetProjectStorageLimit(ctx, projectID)
return err
})
group.Go(func() error {
var err error
totalUsed, err = usage.getProjectStorageTotals(ctx, projectID)
totalUsed, err = usage.GetProjectStorageTotals(ctx, projectID)
return err
})
err = group.Wait()
if err != nil {
return false, 0, ErrProjectUsage.Wrap(err)
@ -124,20 +118,61 @@ func (usage *Service) ExceedsStorageUsage(ctx context.Context, projectID uuid.UU
return false, limit, nil
}
func (usage *Service) getProjectStorageTotals(ctx context.Context, projectID uuid.UUID) (total int64, err error) {
defer mon.Task()(&ctx)(&err)
// GetProjectStorageTotals returns total amount of storage used by project.
func (usage *Service) GetProjectStorageTotals(ctx context.Context, projectID uuid.UUID) (total int64, err error) {
defer mon.Task()(&ctx, projectID)(&err)
lastCountInline, lastCountRemote, err := usage.projectAccountingDB.GetStorageTotals(ctx, projectID)
if err != nil {
return 0, err
return 0, ErrProjectUsage.Wrap(err)
}
cachedTotal, err := usage.liveAccounting.GetProjectStorageUsage(ctx, projectID)
if err != nil {
return 0, err
return 0, ErrProjectUsage.Wrap(err)
}
return lastCountInline + lastCountRemote + cachedTotal, nil
}
// GetProjectBandwidthTotals returns total amount of allocated bandwidth used for past 30 days.
func (usage *Service) GetProjectBandwidthTotals(ctx context.Context, projectID uuid.UUID) (_ int64, err error) {
defer mon.Task()(&ctx, projectID)(&err)
from := time.Now().AddDate(0, 0, -AverageDaysInMonth) // past 30 days
total, err := usage.projectAccountingDB.GetAllocatedBandwidthTotal(ctx, projectID, from)
return total, ErrProjectUsage.Wrap(err)
}
// GetProjectStorageLimit returns current project storage limit.
func (usage *Service) GetProjectStorageLimit(ctx context.Context, projectID uuid.UUID) (_ memory.Size, err error) {
defer mon.Task()(&ctx, projectID)(&err)
limit, err := usage.projectAccountingDB.GetProjectStorageLimit(ctx, projectID)
if err != nil {
return 0, ErrProjectUsage.Wrap(err)
}
if limit == 0 {
return usage.maxAlphaUsage, nil
}
return limit, nil
}
// GetProjectBandwidthLimit returns current project bandwidth limit.
func (usage *Service) GetProjectBandwidthLimit(ctx context.Context, projectID uuid.UUID) (_ memory.Size, err error) {
defer mon.Task()(&ctx, projectID)(&err)
limit, err := usage.projectAccountingDB.GetProjectBandwidthLimit(ctx, projectID)
if err != nil {
return 0, ErrProjectUsage.Wrap(err)
}
if limit == 0 {
return usage.maxAlphaUsage, nil
}
return limit, nil
}
// AddProjectStorageUsage lets the live accounting know that the given
// project has just added inlineSpaceUsed bytes of inline space usage
// and remoteSpaceUsed bytes of remote space usage.

1
satellite/api.go
View File

@ -457,6 +457,7 @@ func NewAPI(log *zap.Logger, full *identity.FullIdentity, db DB, pointerDB metai
&consoleauth.Hmac{Secret: []byte(consoleConfig.AuthTokenSecret)},
peer.DB.Console(),
peer.DB.ProjectAccounting(),
peer.Accounting.ProjectUsage,
peer.DB.Rewards(),
peer.Marketing.PartnersService,
peer.Payments.Accounts,

20
satellite/console/consoleweb/consoleql/mutation_test.go
View File

@ -20,6 +20,8 @@ import (
"storj.io/storj/private/post"
"storj.io/storj/private/testcontext"
"storj.io/storj/satellite"
"storj.io/storj/satellite/accounting"
"storj.io/storj/satellite/accounting/live"
"storj.io/storj/satellite/console"
"storj.io/storj/satellite/console/consoleauth"
"storj.io/storj/satellite/console/consoleweb/consoleql"
@ -59,14 +61,26 @@ func TestGrapqhlMutation(t *testing.T) {
},
)
paymentsConfig := stripecoinpayments.Config{}
payments := stripecoinpayments.NewService(log, paymentsConfig, db.StripeCoinPayments(), db.Console().Projects(), db.ProjectAccounting(), 0, 0, 0)
payments := stripecoinpayments.NewService(
log.Named("payments"),
stripecoinpayments.Config{},
db.StripeCoinPayments(),
db.Console().Projects(),
db.ProjectAccounting(),
0, 0, 0,
)
cache, err := live.NewCache(log.Named("cache"), live.Config{StorageBackend: "memory"})
require.NoError(t, err)
projectUsage := accounting.NewService(db.ProjectAccounting(), cache, 0)
service, err := console.NewService(
log,
log.Named("console"),
&consoleauth.Hmac{Secret: []byte("my-suppa-secret-key")},
db.Console(),
db.ProjectAccounting(),
projectUsage,
db.Rewards(),
partnersService,
payments.Accounts(),

20
satellite/console/consoleweb/consoleql/query_test.go
View File

@ -16,6 +16,8 @@ import (
"storj.io/storj/pkg/auth"
"storj.io/storj/private/testcontext"
"storj.io/storj/satellite"
"storj.io/storj/satellite/accounting"
"storj.io/storj/satellite/accounting/live"
"storj.io/storj/satellite/console"
"storj.io/storj/satellite/console/consoleauth"
"storj.io/storj/satellite/console/consoleweb/consoleql"
@ -42,14 +44,26 @@ func TestGraphqlQuery(t *testing.T) {
},
)
paymentsConfig := stripecoinpayments.Config{}
payments := stripecoinpayments.NewService(log, paymentsConfig, db.StripeCoinPayments(), db.Console().Projects(), db.ProjectAccounting(), 0, 0, 0)
payments := stripecoinpayments.NewService(
log.Named("payments"),
stripecoinpayments.Config{},
db.StripeCoinPayments(),
db.Console().Projects(),
db.ProjectAccounting(),
0, 0, 0,
)
cache, err := live.NewCache(log.Named("cache"), live.Config{StorageBackend: "memory"})
require.NoError(t, err)
projectUsage := accounting.NewService(db.ProjectAccounting(), cache, 0)
service, err := console.NewService(
log,
log.Named("console"),
&consoleauth.Hmac{Secret: []byte("my-suppa-secret-key")},
db.Console(),
db.ProjectAccounting(),
projectUsage,
db.Rewards(),
partnersService,
payments.Accounts(),

84
satellite/console/consoleweb/server.go
View File

@ -130,6 +130,11 @@ func NewServer(logger *zap.Logger, config Config, service *console.Service, mail
router.HandleFunc("/registrationToken/", server.createRegistrationTokenHandler)
router.HandleFunc("/robots.txt", server.seoHandler)
router.Handle(
"/api/v0/projects/{id}/usage-limits",
server.withAuth(http.HandlerFunc(server.projectUsageLimitsHandler)),
).Methods(http.MethodGet)
referralsController := consoleapi.NewReferrals(logger, referralsService, service, mailService, server.config.ExternalAddress)
referralsRouter := router.PathPrefix("/api/v0/referrals").Subrouter()
referralsRouter.Handle("/tokens", server.withAuth(http.HandlerFunc(referralsController.GetTokens))).Methods(http.MethodGet)
@ -444,21 +449,60 @@ func (server *Server) cancelPasswordRecoveryHandler(w http.ResponseWriter, r *ht
http.Redirect(w, r, "https://storjlabs.atlassian.net/servicedesk/customer/portals", http.StatusSeeOther)
}
func (server *Server) serveError(w http.ResponseWriter, status int) {
w.WriteHeader(status)
// projectUsageLimitsHandler api handler for project usage limits.
func (server *Server) projectUsageLimitsHandler(w http.ResponseWriter, r *http.Request) {
err := error(nil)
ctx := r.Context()
switch status {
case http.StatusInternalServerError:
err := server.templates.internalServerError.Execute(w, nil)
if err != nil {
server.log.Error("cannot parse internalServerError template", zap.Error(Error.Wrap(err)))
defer mon.Task()(&ctx)(&err)
var ok bool
var idParam string
handleError := func(code int, err error) {
w.WriteHeader(code)
var jsonError struct {
Error string `json:"error"`
}
default:
err := server.templates.notFound.Execute(w, nil)
if err != nil {
server.log.Error("cannot parse pageNotFound template", zap.Error(Error.Wrap(err)))
jsonError.Error = err.Error()
if err := json.NewEncoder(w).Encode(err); err != nil {
server.log.Error("error encoding project usage limits error", zap.Error(err))
}
}
handleServiceError := func(err error) {
switch {
case console.ErrUnauthorized.Has(err):
handleError(http.StatusUnauthorized, err)
default:
handleError(http.StatusInternalServerError, err)
}
}
if idParam, ok = mux.Vars(r)["id"]; !ok {
handleError(http.StatusBadRequest, errs.New("missing project id route param"))
return
}
projectID, err := uuid.Parse(idParam)
if err != nil {
handleError(http.StatusBadRequest, errs.New("invalid project id: %v", err))
return
}
limits, err := server.service.GetProjectUsageLimits(ctx, *projectID)
if err != nil {
handleServiceError(err)
return
}
if err := json.NewEncoder(w).Encode(limits); err != nil {
server.log.Error("error encoding project usage limits", zap.Error(err))
return
}
}
// grapqlHandler is graphql endpoint http handler function
@ -512,6 +556,24 @@ func (server *Server) grapqlHandler(w http.ResponseWriter, r *http.Request) {
sugar.Debug(result)
}
// serveError serves error static pages.
func (server *Server) serveError(w http.ResponseWriter, status int) {
w.WriteHeader(status)
switch status {
case http.StatusInternalServerError:
err := server.templates.internalServerError.Execute(w, nil)
if err != nil {
server.log.Error("cannot parse internalServerError template", zap.Error(Error.Wrap(err)))
}
default:
err := server.templates.notFound.Execute(w, nil)
if err != nil {
server.log.Error("cannot parse pageNotFound template", zap.Error(Error.Wrap(err)))
}
}
}
// seoHandler used to communicate with web crawlers and other web robots
func (server *Server) seoHandler(w http.ResponseWriter, req *http.Request) {
header := w.Header()

12
satellite/console/projectusagelimits.go Normal file
View File

@ -0,0 +1,12 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package console
// ProjectUsageLimits holds project usage limits and current usage.
type ProjectUsageLimits struct {
StorageLimit int64 `json:"storageLimit"`
BandwidthLimit int64 `json:"bandwidthLimit"`
StorageUsed int64 `json:"storageUsed"`
BandwidthUsed int64 `json:"bandwidthUsed"`
}

41
satellite/console/service.go
View File

@ -13,7 +13,7 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap"
"golang.org/x/crypto/bcrypt"
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/auth"
"storj.io/storj/pkg/macaroon"
@ -72,6 +72,7 @@ type Service struct {
log *zap.Logger
store DB
projectAccounting accounting.ProjectAccounting
projectUsage *accounting.Service
rewards rewards.DB
partners *rewards.PartnersService
accounts payments.Accounts
@ -85,7 +86,7 @@ type PaymentsService struct {
}
// NewService returns new instance of Service.
func NewService(log *zap.Logger, signer Signer, store DB, projectAccounting accounting.ProjectAccounting, rewards rewards.DB, partners *rewards.PartnersService, accounts payments.Accounts, passwordCost int) (*Service, error) {
func NewService(log *zap.Logger, signer Signer, store DB, projectAccounting accounting.ProjectAccounting, projectUsage *accounting.Service, rewards rewards.DB, partners *rewards.PartnersService, accounts payments.Accounts, passwordCost int) (*Service, error) {
if signer == nil {
return nil, errs.New("signer can't be nil")
}
@ -104,6 +105,7 @@ func NewService(log *zap.Logger, signer Signer, store DB, projectAccounting acco
Signer: signer,
store: store,
projectAccounting: projectAccounting,
projectUsage: projectUsage,
rewards: rewards,
partners: partners,
accounts: accounts,
@ -1203,6 +1205,41 @@ func (s *Service) GetBucketUsageRollups(ctx context.Context, projectID uuid.UUID
return result, nil
}
// GetProjectUsageLimits returns project limits and current usage.
func (s *Service) GetProjectUsageLimits(ctx context.Context, projectID uuid.UUID) (_ *ProjectUsageLimits, err error) {
defer mon.Task()(&ctx)(&err)
_, err = GetAuth(ctx)
if err != nil {
return nil, err
}
storageLimit, err := s.projectUsage.GetProjectStorageLimit(ctx, projectID)
if err != nil {
return nil, err
}
bandwidthLimit, err := s.projectUsage.GetProjectBandwidthLimit(ctx, projectID)
if err != nil {
return nil, err
}
storageUsed, err := s.projectUsage.GetProjectStorageTotals(ctx, projectID)
if err != nil {
return nil, err
}
bandwidthUsed, err := s.projectUsage.GetProjectBandwidthTotals(ctx, projectID)
if err != nil {
return nil, err
}
return &ProjectUsageLimits{
StorageLimit: storageLimit.Int64(),
BandwidthLimit: bandwidthLimit.Int64(),
StorageUsed: storageUsed,
BandwidthUsed: bandwidthUsed,
}, nil
}
// Authorize validates token from context and returns authorized Authorization
func (s *Service) Authorize(ctx context.Context) (a Authorization, err error) {
defer mon.Task()(&ctx)(&err)