From fde5c3542b40fb95a6fa4af3e4060316480942fd Mon Sep 17 00:00:00 2001 From: crawter Date: Mon, 16 Mar 2020 06:28:03 +0200 Subject: [PATCH] storagenode/console/api: period payStub api extended Change-Id: I624bbf7a9640f9df97789bea109201cbfb556753 --- storagenode/console/consoleapi/heldamount.go | 79 +++++++++++++ storagenode/console/consoleserver/server.go | 2 + storagenode/heldamount/db_test.go | 110 ++++++++++++++++++- storagenode/heldamount/heldamount.go | 5 + storagenode/heldamount/service.go | 103 +++++++++++++++++ storagenode/heldamount/service_test.go | 43 ++++++++ storagenode/storagenodedb/heldamount.go | 4 + 7 files changed, 345 insertions(+), 1 deletion(-) create mode 100644 storagenode/heldamount/service_test.go diff --git a/storagenode/console/consoleapi/heldamount.go b/storagenode/console/consoleapi/heldamount.go index 9a80aa53a..7398d4b33 100644 --- a/storagenode/console/consoleapi/heldamount.go +++ b/storagenode/console/consoleapi/heldamount.go @@ -100,6 +100,85 @@ func (heldAmount *HeldAmount) AllPayStubsMonthly(w http.ResponseWriter, r *http. } } +// SatellitePayStubPeriod retrieves held amount for all satellites for selected months from storagenode database. +func (heldAmount *HeldAmount) SatellitePayStubPeriod(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + var err error + defer mon.Task()(&ctx)(&err) + + w.Header().Set(contentType, applicationJSON) + + params := mux.Vars(r) + + id, ok := params["satelliteID"] + if !ok { + heldAmount.serveJSONError(w, http.StatusBadRequest, ErrNotificationsAPI.Wrap(err)) + return + } + satelliteID, err := storj.NodeIDFromString(id) + if err != nil { + heldAmount.serveJSONError(w, http.StatusBadRequest, ErrHeldAmountPI.Wrap(err)) + return + } + + start, ok := params["start"] + if !ok { + heldAmount.serveJSONError(w, http.StatusBadRequest, ErrNotificationsAPI.Wrap(err)) + return + } + + end, ok := params["end"] + if !ok { + heldAmount.serveJSONError(w, http.StatusBadRequest, ErrNotificationsAPI.Wrap(err)) + return + } + + payStubs, err := heldAmount.service.SatellitePayStubPeriodCached(ctx, satelliteID, start, end) + if err != nil { + heldAmount.serveJSONError(w, http.StatusInternalServerError, ErrHeldAmountPI.Wrap(err)) + return + } + + if err := json.NewEncoder(w).Encode(payStubs); err != nil { + heldAmount.log.Error("failed to encode json response", zap.Error(ErrHeldAmountPI.Wrap(err))) + return + } +} + +// AllPayStubsPeriod retrieves held amount for all satellites for selected range of months from storagenode database. +func (heldAmount *HeldAmount) AllPayStubsPeriod(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + var err error + defer mon.Task()(&ctx)(&err) + + w.Header().Set(contentType, applicationJSON) + + params := mux.Vars(r) + + start, ok := params["start"] + if !ok { + heldAmount.serveJSONError(w, http.StatusBadRequest, ErrNotificationsAPI.Wrap(err)) + return + } + + end, ok := params["end"] + if !ok { + heldAmount.serveJSONError(w, http.StatusBadRequest, ErrNotificationsAPI.Wrap(err)) + return + } + + payStubs, err := heldAmount.service.AllPayStubsPeriodCached(ctx, start, end) + if err != nil { + heldAmount.serveJSONError(w, http.StatusInternalServerError, ErrHeldAmountPI.Wrap(err)) + return + } + + if err := json.NewEncoder(w).Encode(payStubs); err != nil { + heldAmount.log.Error("failed to encode json response", zap.Error(ErrHeldAmountPI.Wrap(err))) + return + } +} + // GetMonthlyPayment returns payment data from satellite for specific month. func (heldAmount *HeldAmount) GetMonthlyPayment(w http.ResponseWriter, r *http.Request) { ctx := r.Context() diff --git a/storagenode/console/consoleserver/server.go b/storagenode/console/consoleserver/server.go index cec081b1e..257356ddc 100644 --- a/storagenode/console/consoleserver/server.go +++ b/storagenode/console/consoleserver/server.go @@ -79,6 +79,8 @@ func NewServer(logger *zap.Logger, assets http.FileSystem, notifications *notifi heldAmountRouter.StrictSlash(true) heldAmountRouter.HandleFunc("/paystub/{period}/{satelliteID}", heldAmountController.SatellitePayStubMonthly).Methods(http.MethodGet) heldAmountRouter.HandleFunc("/paystub/{period}", heldAmountController.AllPayStubsMonthly).Methods(http.MethodGet) + heldAmountRouter.HandleFunc("/paystub/{start}/{end}/{satelliteID}", heldAmountController.SatellitePayStubPeriod).Methods(http.MethodGet) + heldAmountRouter.HandleFunc("/paystub/{start}/{end}", heldAmountController.AllPayStubsPeriod).Methods(http.MethodGet) heldAmountRouter.HandleFunc("/payment/{period}/{satelliteID}", heldAmountController.GetMonthlyPayment).Methods(http.MethodGet) if assets != nil { diff --git a/storagenode/heldamount/db_test.go b/storagenode/heldamount/db_test.go index 3fabc7fd2..6efaebba7 100644 --- a/storagenode/heldamount/db_test.go +++ b/storagenode/heldamount/db_test.go @@ -4,11 +4,14 @@ package heldamount_test import ( + "fmt" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "storj.io/common/rpc" "storj.io/common/storj" "storj.io/common/testcontext" "storj.io/storj/storagenode" @@ -77,10 +80,12 @@ func TestHeldAmountDB(t *testing.T) { stub, err = heldAmount.GetPayStub(ctx, satelliteID, "") assert.Error(t, err) + assert.Equal(t, true, heldamount.ErrNoPayStubForPeriod.Has(err)) assert.Nil(t, stub) stub, err = heldAmount.GetPayStub(ctx, storj.NodeID{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, period) assert.Error(t, err) + assert.Equal(t, true, heldamount.ErrNoPayStubForPeriod.Has(err)) assert.Nil(t, stub) }) @@ -112,8 +117,8 @@ func TestHeldAmountDB(t *testing.T) { assert.Equal(t, stubs[0].UsagePutRepair, paystub.UsagePutRepair) stubs, err = heldAmount.AllPayStubs(ctx, "") - assert.NoError(t, err) assert.Equal(t, len(stubs), 0) + assert.NoError(t, err) }) payment := heldamount.Payment{ @@ -169,3 +174,106 @@ func TestHeldAmountDB(t *testing.T) { }) }) } + +func TestSatellitePayStubPeriodCached(t *testing.T) { + storagenodedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db storagenode.DB) { + heldAmountDB := db.HeldAmount() + service := heldamount.NewService(nil, heldAmountDB, rpc.Dialer{}, nil) + + payStub := heldamount.PayStub{ + SatelliteID: storj.NodeID{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + Created: time.Now().UTC(), + Codes: "code", + UsageAtRest: 1, + UsageGet: 2, + UsagePut: 3, + UsageGetRepair: 4, + UsagePutRepair: 5, + UsageGetAudit: 6, + CompAtRest: 7, + CompGet: 8, + CompPut: 9, + CompGetRepair: 10, + CompPutRepair: 11, + CompGetAudit: 12, + SurgePercent: 13, + Held: 14, + Owed: 15, + Disposed: 16, + Paid: 17, + } + + for i := 1; i < 4; i++ { + payStub.Period = fmt.Sprintf("2020-0%d", i) + err := heldAmountDB.StorePayStub(ctx, payStub) + require.NoError(t, err) + } + + payStubs, err := service.SatellitePayStubPeriodCached(ctx, payStub.SatelliteID, "2020-01", "2020-03") + require.NoError(t, err) + require.Equal(t, 3, len(payStubs)) + + payStubs, err = service.SatellitePayStubPeriodCached(ctx, payStub.SatelliteID, "2019-01", "2021-03") + require.NoError(t, err) + require.Equal(t, 3, len(payStubs)) + + payStubs, err = service.SatellitePayStubPeriodCached(ctx, payStub.SatelliteID, "2019-01", "2020-01") + require.NoError(t, err) + require.Equal(t, 1, len(payStubs)) + }) +} + +func TestAllPayStubPeriodCached(t *testing.T) { + storagenodedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db storagenode.DB) { + heldAmountDB := db.HeldAmount() + service := heldamount.NewService(nil, heldAmountDB, rpc.Dialer{}, nil) + + payStub := heldamount.PayStub{ + SatelliteID: storj.NodeID{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + Created: time.Now().UTC(), + Codes: "code", + UsageAtRest: 1, + UsageGet: 2, + UsagePut: 3, + UsageGetRepair: 4, + UsagePutRepair: 5, + UsageGetAudit: 6, + CompAtRest: 7, + CompGet: 8, + CompPut: 9, + CompGetRepair: 10, + CompPutRepair: 11, + CompGetAudit: 12, + SurgePercent: 13, + Held: 14, + Owed: 15, + Disposed: 16, + Paid: 17, + } + + for i := 1; i < 4; i++ { + payStub.SatelliteID[0] += byte(i) + for j := 1; j < 4; j++ { + payStub.Period = fmt.Sprintf("2020-0%d", j) + err := heldAmountDB.StorePayStub(ctx, payStub) + require.NoError(t, err) + } + } + + payStubs, err := service.AllPayStubsPeriodCached(ctx, "2020-01", "2020-03") + require.NoError(t, err) + require.Equal(t, 9, len(payStubs)) + + payStubs, err = service.AllPayStubsPeriodCached(ctx, "2019-01", "2021-03") + require.NoError(t, err) + require.Equal(t, 9, len(payStubs)) + + payStubs, err = service.AllPayStubsPeriodCached(ctx, "2019-01", "2020-01") + require.NoError(t, err) + require.Equal(t, 3, len(payStubs)) + + payStubs, err = service.AllPayStubsPeriodCached(ctx, "2019-01", "2019-01") + require.NoError(t, err) + require.Equal(t, 0, len(payStubs)) + }) +} diff --git a/storagenode/heldamount/heldamount.go b/storagenode/heldamount/heldamount.go index f67c7e5dd..2da078fc6 100644 --- a/storagenode/heldamount/heldamount.go +++ b/storagenode/heldamount/heldamount.go @@ -7,6 +7,8 @@ import ( "context" "time" + "github.com/zeebo/errs" + "storj.io/common/storj" ) @@ -28,6 +30,9 @@ type DB interface { AllPayments(ctx context.Context, period string) ([]Payment, error) } +// ErrNoPayStubForPeriod represents errors from the heldamount database. +var ErrNoPayStubForPeriod = errs.Class("no payStub for period error") + // PayStub is node heldamount data for satellite by specific period. type PayStub struct { SatelliteID storj.NodeID `json:"satelliteId"` diff --git a/storagenode/heldamount/service.go b/storagenode/heldamount/service.go index 5fafbf266..dc0055edb 100644 --- a/storagenode/heldamount/service.go +++ b/storagenode/heldamount/service.go @@ -5,6 +5,9 @@ package heldamount import ( "context" + "fmt" + "strconv" + "strings" "time" "github.com/spacemonkeygo/monkit/v3" @@ -163,6 +166,51 @@ func (service *Service) AllPayStubsMonthlyCached(ctx context.Context, period str return payStubs, nil } +// SatellitePayStubPeriodCached retrieves held amount for all satellites for selected months from storagenode database. +func (service *Service) SatellitePayStubPeriodCached(ctx context.Context, satelliteID storj.NodeID, periodStart, periodEnd string) (payStubs []*PayStub, err error) { + defer mon.Task()(&ctx, &satelliteID, &periodStart, &periodEnd)(&err) + + periods, err := parsePeriodRange(periodStart, periodEnd) + if err != nil { + return nil, err + } + + for _, period := range periods { + payStub, err := service.db.GetPayStub(ctx, satelliteID, period) + if err != nil { + if ErrNoPayStubForPeriod.Has(err) { + continue + } + return nil, ErrHeldAmountService.Wrap(err) + } + + payStubs = append(payStubs, payStub) + } + + return payStubs, nil +} + +// AllPayStubsPeriodCached retrieves held amount for all satellites for selected range of months from storagenode database. +func (service *Service) AllPayStubsPeriodCached(ctx context.Context, periodStart, periodEnd string) (payStubs []PayStub, err error) { + defer mon.Task()(&ctx, &periodStart, &periodEnd)(&err) + + periods, err := parsePeriodRange(periodStart, periodEnd) + if err != nil { + return nil, err + } + + for _, period := range periods { + payStub, err := service.db.AllPayStubs(ctx, period) + if err != nil { + return nil, ErrHeldAmountService.Wrap(err) + } + + payStubs = append(payStubs, payStub...) + } + + return payStubs, nil +} + // GetPaymentCached retrieves payment data from particular satellite from storagenode database. func (service *Service) GetPaymentCached(ctx context.Context, satelliteID storj.NodeID, period string) (_ *Payment, err error) { defer mon.Task()(&ctx, &satelliteID, &period)(&err) @@ -200,3 +248,58 @@ func stringToTime(period string) (_ time.Time, err error) { return result, nil } + +// TODO: improve it. +func parsePeriodRange(periodStart, periodEnd string) (periods []string, err error) { + var yearStart, yearEnd, monthStart, monthEnd int + + start := strings.Split(periodStart, "-") + if len(start) != 2 { + return nil, ErrHeldAmountService.New("period start has wrong format") + } + end := strings.Split(periodEnd, "-") + if len(start) != 2 { + return nil, ErrHeldAmountService.New("period end has wrong format") + } + + yearStart, err = strconv.Atoi(start[0]) + if err != nil { + return nil, ErrHeldAmountService.New("period start has wrong format") + } + monthStart, err = strconv.Atoi(start[1]) + if err != nil || monthStart > 12 || monthStart < 1 { + return nil, ErrHeldAmountService.New("period start has wrong format") + } + yearEnd, err = strconv.Atoi(end[0]) + if err != nil { + return nil, ErrHeldAmountService.New("period end has wrong format") + } + monthEnd, err = strconv.Atoi(end[1]) + if err != nil || monthEnd > 12 || monthEnd < 1 { + return nil, ErrHeldAmountService.New("period end has wrong format") + } + if yearEnd < yearStart { + return nil, ErrHeldAmountService.New("period has wrong format") + } + if yearEnd == yearStart && monthEnd < monthStart { + return nil, ErrHeldAmountService.New("period has wrong format") + } + + for ; yearStart <= yearEnd; yearStart++ { + lastMonth := 12 + if yearStart == yearEnd { + lastMonth = monthEnd + } + for ; monthStart <= lastMonth; monthStart++ { + format := "%d-%d" + if monthStart < 10 { + format = "%d-0%d" + } + periods = append(periods, fmt.Sprintf(format, yearStart, monthStart)) + } + + monthStart = 1 + } + + return periods, nil +} diff --git a/storagenode/heldamount/service_test.go b/storagenode/heldamount/service_test.go new file mode 100644 index 000000000..d9929280d --- /dev/null +++ b/storagenode/heldamount/service_test.go @@ -0,0 +1,43 @@ +// Copyright (C) 2020 Storj Labs, Inc. +// See LICENSE for copying information. + +package heldamount + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParsePeriodRange(t *testing.T) { + testCases := [...]struct { + periodStart string + periodEnd string + periods []string + }{ + {"2020-01", "2020-02", []string{"2020-01", "2020-02"}}, + {"2020-01", "2020-01", []string{"2020-01"}}, + {"2019-11", "2020-02", []string{"2019-11", "2019-12", "2020-01", "2020-02"}}, + {"", "2020-02", nil}, + {"2020-01", "", nil}, + {"2020-01-01", "2020-02", nil}, + {"2020-44", "2020-02", nil}, + {"2020-01", "2020-44", nil}, + {"2020-01", "2019-01", nil}, + {"2020-02", "2020-01", nil}, + } + + for _, tc := range testCases { + periods, err := parsePeriodRange(tc.periodStart, tc.periodEnd) + require.Equal(t, len(periods), len(tc.periods)) + if periods != nil { + for i := 0; i < len(periods); i++ { + require.Equal(t, periods[i], tc.periods[i]) + require.NoError(t, err) + } + } else { + require.Error(t, err) + } + + } +} diff --git a/storagenode/storagenodedb/heldamount.go b/storagenode/storagenodedb/heldamount.go index 78175a2da..8e4d51f1c 100644 --- a/storagenode/storagenodedb/heldamount.go +++ b/storagenode/storagenodedb/heldamount.go @@ -5,6 +5,7 @@ package storagenodedb import ( "context" + "database/sql" "github.com/zeebo/errs" @@ -136,6 +137,9 @@ func (db *heldamountDB) GetPayStub(ctx context.Context, satelliteID storj.NodeID &result.Paid, ) if err != nil { + if sql.ErrNoRows == err { + return nil, heldamount.ErrNoPayStubForPeriod.Wrap(err) + } return nil, ErrHeldAmount.Wrap(err) }