diff --git a/private/date/utils.go b/private/date/utils.go index 53d500fca..27e06e81d 100644 --- a/private/date/utils.go +++ b/private/date/utils.go @@ -30,3 +30,25 @@ func PeriodToTime(period string) (_ time.Time, err error) { return result, nil } + +// MonthsCountSince calculates the months between now and the createdAtTime time.Time value passed. +func MonthsCountSince(from time.Time) int { + now := time.Now().UTC() + return MonthsBetweenDates(from, now) +} + +// MonthsBetweenDates calculates amount of months between two dates +func MonthsBetweenDates(from time.Time, to time.Time) int { + months := 0 + month := from.Month() + for from.Before(to) { + from = from.Add(time.Hour * 24) + nextMonth := from.Month() + if nextMonth != month { + months++ + } + month = nextMonth + } + + return months +} diff --git a/private/date/utils_test.go b/private/date/utils_test.go index 56bade55d..bf7415d1c 100644 --- a/private/date/utils_test.go +++ b/private/date/utils_test.go @@ -47,3 +47,23 @@ func TestPeriodToTime(t *testing.T) { require.Equal(t, periodTime.String(), tc.periodTime.String()) } } + +func TestMonthsBetweenDates(t *testing.T) { + testCases := [...]struct { + from time.Time + to time.Time + monthsAmount int + }{ + {time.Date(2020, 2, 13, 0, 0, 0, 0, &time.Location{}), time.Date(2020, 05, 13, 0, 0, 0, 0, &time.Location{}), 3}, + {time.Date(2015, 7, 30, 0, 0, 0, 0, &time.Location{}), time.Date(2020, 05, 13, 0, 0, 0, 0, &time.Location{}), 58}, + {time.Date(2017, 1, 28, 0, 0, 0, 0, &time.Location{}), time.Date(2020, 05, 13, 0, 0, 0, 0, &time.Location{}), 40}, + {time.Date(2016, 11, 1, 0, 0, 0, 0, &time.Location{}), time.Date(2020, 05, 13, 0, 0, 0, 0, &time.Location{}), 42}, + {time.Date(2019, 4, 17, 0, 0, 0, 0, &time.Location{}), time.Date(2020, 05, 13, 0, 0, 0, 0, &time.Location{}), 13}, + {time.Date(2018, 9, 11, 0, 0, 0, 0, &time.Location{}), time.Date(2020, 05, 13, 0, 0, 0, 0, &time.Location{}), 20}, + } + + for _, tc := range testCases { + monthDiff := date.MonthsBetweenDates(tc.from, tc.to) + require.Equal(t, monthDiff, tc.monthsAmount) + } +} diff --git a/storagenode/console/consoleapi/storagenode.go b/storagenode/console/consoleapi/storagenode.go index ef16cc151..a877de5f9 100644 --- a/storagenode/console/consoleapi/storagenode.go +++ b/storagenode/console/consoleapi/storagenode.go @@ -111,6 +111,47 @@ func (dashboard *StorageNode) Satellite(w http.ResponseWriter, r *http.Request) } } +// EstimatedPayout returns estimated payout from specific satellite or all satellites if current traffic level remains same. +func (dashboard *StorageNode) EstimatedPayout(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + var err error + defer mon.Task()(&ctx)(&err) + + w.Header().Set(contentType, applicationJSON) + + queryParams := r.URL.Query() + id := queryParams.Get("id") + if id == "" { + data, err := dashboard.service.GetAllSatellitesEstimatedPayout(ctx) + if err != nil { + dashboard.serveJSONError(w, http.StatusInternalServerError, ErrStorageNodeAPI.Wrap(err)) + return + } + + if err := json.NewEncoder(w).Encode(data); err != nil { + dashboard.log.Error("failed to encode json response", zap.Error(ErrHeldAmountAPI.Wrap(err))) + return + } + } else { + satelliteID, err := storj.NodeIDFromString(id) + if err != nil { + dashboard.serveJSONError(w, http.StatusBadRequest, ErrHeldAmountAPI.Wrap(err)) + return + } + + data, err := dashboard.service.GetSatelliteEstimatedPayout(ctx, satelliteID) + if err != nil { + dashboard.serveJSONError(w, http.StatusInternalServerError, ErrStorageNodeAPI.Wrap(err)) + return + } + + if err := json.NewEncoder(w).Encode(data); err != nil { + dashboard.log.Error("failed to encode json response", zap.Error(ErrHeldAmountAPI.Wrap(err))) + return + } + } +} + // serveJSONError writes JSON error to response output stream. func (dashboard *StorageNode) serveJSONError(w http.ResponseWriter, status int, err error) { w.WriteHeader(status) diff --git a/storagenode/console/consoleapi/storagenode_test.go b/storagenode/console/consoleapi/storagenode_test.go new file mode 100644 index 000000000..d7a35710c --- /dev/null +++ b/storagenode/console/consoleapi/storagenode_test.go @@ -0,0 +1,219 @@ +// Copyright (C) 2020 Storj Labs, Inc. +// See LICENSE for copying information. + +package consoleapi_test + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "math" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "storj.io/common/pb" + "storj.io/common/storj" + "storj.io/common/testcontext" + "storj.io/storj/private/date" + "storj.io/storj/private/testplanet" + "storj.io/storj/storagenode/heldamount" + "storj.io/storj/storagenode/pricing" + "storj.io/storj/storagenode/reputation" + "storj.io/storj/storagenode/storageusage" +) + +var ( + actions = []pb.PieceAction{ + pb.PieceAction_INVALID, + + pb.PieceAction_PUT, + pb.PieceAction_GET, + pb.PieceAction_GET_AUDIT, + pb.PieceAction_GET_REPAIR, + pb.PieceAction_PUT_REPAIR, + pb.PieceAction_DELETE, + + pb.PieceAction_PUT, + pb.PieceAction_GET, + pb.PieceAction_GET_AUDIT, + pb.PieceAction_GET_REPAIR, + pb.PieceAction_PUT_REPAIR, + pb.PieceAction_DELETE, + } +) + +func TestStorageNodeApi(t *testing.T) { + testplanet.Run(t, + testplanet.Config{ + SatelliteCount: 2, + StorageNodeCount: 1, + }, + func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { + satellite := planet.Satellites[0] + satellite2 := planet.Satellites[1] + sno := planet.StorageNodes[0] + console := sno.Console + bandwidthdb := sno.DB.Bandwidth() + pricingdb := sno.DB.Pricing() + storageusagedb := sno.DB.StorageUsage() + reputationdb := sno.DB.Reputation() + baseURL := fmt.Sprintf("http://%s/api/sno", console.Listener.Addr()) + + now := time.Now().UTC().Add(-2 * time.Hour) + + randAmount1 := int64(120000000000) + randAmount2 := int64(450000000000) + + for _, action := range actions { + err := bandwidthdb.Add(ctx, satellite.ID(), action, randAmount1, now) + require.NoError(t, err) + + err = bandwidthdb.Add(ctx, satellite2.ID(), action, randAmount2, now.Add(2*time.Hour)) + require.NoError(t, err) + } + var satellites []storj.NodeID + + satellites = append(satellites, satellite.ID(), satellite2.ID()) + stamps, _ := makeStorageUsageStamps(satellites) + + err := storageusagedb.Store(ctx, stamps) + require.NoError(t, err) + + egressPrice, repairPrice, auditPrice, diskPrice := int64(2000), int64(1000), int64(1000), int64(150) + + err = pricingdb.Store(ctx, pricing.Pricing{ + SatelliteID: satellite.ID(), + EgressBandwidth: egressPrice, + RepairBandwidth: repairPrice, + AuditBandwidth: auditPrice, + DiskSpace: diskPrice, + }) + require.NoError(t, err) + + err = pricingdb.Store(ctx, pricing.Pricing{ + SatelliteID: satellite2.ID(), + EgressBandwidth: egressPrice, + RepairBandwidth: repairPrice, + AuditBandwidth: auditPrice, + DiskSpace: diskPrice, + }) + require.NoError(t, err) + + err = reputationdb.Store(ctx, reputation.Stats{ + SatelliteID: satellite.ID(), + JoinedAt: time.Now().UTC(), + }) + require.NoError(t, err) + err = reputationdb.Store(ctx, reputation.Stats{ + SatelliteID: satellite2.ID(), + JoinedAt: time.Now().UTC(), + }) + require.NoError(t, err) + + t.Run("test EstimatedPayout", func(t *testing.T) { + // should return estimated payout for both satellites in current month and empty for previous + url := fmt.Sprintf("%s/estimatedPayout", baseURL) + res, err := http.Get(url) + require.NoError(t, err) + require.NotNil(t, res) + require.Equal(t, http.StatusOK, res.StatusCode) + + defer func() { + err = res.Body.Close() + require.NoError(t, err) + }() + body, err := ioutil.ReadAll(res.Body) + require.NoError(t, err) + + expectedAuditRepairSatellite1 := 4 * (float64(randAmount1*auditPrice) / math.Pow10(12)) + expectedAuditRepairSatellite2 := 4 * float64(randAmount2*repairPrice) / math.Pow10(12) + expectedUsageSatellite1 := 2 * float64(randAmount1*egressPrice) / math.Pow10(12) + expectedUsageSatellite2 := 2 * float64(randAmount2*egressPrice) / math.Pow10(12) + expectedDisk := int64(float64(30000000000000*diskPrice/730)/math.Pow10(12)) * int64(time.Now().UTC().Day()) + + day := int64(time.Now().Day()) + + month := time.Now().UTC() + _, to := date.MonthBoundary(month) + + sum1 := expectedAuditRepairSatellite1 + expectedUsageSatellite1 + float64(expectedDisk) + sum1AfterHeld := math.Round(sum1 / 4) + estimated1 := int64(sum1AfterHeld) * int64(to.Day()) / day + sum2 := expectedAuditRepairSatellite2 + expectedUsageSatellite2 + float64(expectedDisk) + sum2AfterHeld := math.Round(sum2 / 4) + estimated2 := int64(sum2AfterHeld) * int64(to.Day()) / day + + expected, err := json.Marshal(heldamount.EstimatedPayout{ + CurrentMonthEstimatedAmount: estimated1 + estimated2, + PreviousMonthPayout: heldamount.PayoutMonthly{ + EgressBandwidth: 0, + EgressPayout: 0, + EgressRepairAudit: 0, + RepairAuditPayout: 0, + DiskSpace: 0, + DiskSpaceAmount: 0, + HeldPercentRate: 0, + }, + }) + require.NoError(t, err) + require.Equal(t, string(expected)+"\n", string(body)) + + // should return estimated payout for first satellite in current month and empty for previous + url = fmt.Sprintf("%s/estimatedPayout?id=%s", baseURL, satellite.ID().String()) + res2, err := http.Get(url) + require.NoError(t, err) + require.NotNil(t, res) + require.Equal(t, http.StatusOK, res.StatusCode) + + defer func() { + err = res2.Body.Close() + require.NoError(t, err) + }() + body2, err := ioutil.ReadAll(res2.Body) + require.NoError(t, err) + + expected2, err := json.Marshal(heldamount.EstimatedPayout{ + CurrentMonthEstimatedAmount: estimated1, + PreviousMonthPayout: heldamount.PayoutMonthly{ + EgressBandwidth: 0, + EgressPayout: 0, + EgressRepairAudit: 0, + RepairAuditPayout: 0, + DiskSpace: 0, + DiskSpaceAmount: 0, + HeldPercentRate: 75, + }, + }) + require.NoError(t, err) + require.Equal(t, string(expected2)+"\n", string(body2)) + }) + }, + ) +} + +// makeStorageUsageStamps creates storage usage stamps and expected summaries for provided satellites. +// Creates one entry per day for 30 days with last date as beginning of provided endDate. +func makeStorageUsageStamps(satellites []storj.NodeID) ([]storageusage.Stamp, map[storj.NodeID]float64) { + var stamps []storageusage.Stamp + summary := make(map[storj.NodeID]float64) + + now := time.Now().UTC().Day() + + for _, satellite := range satellites { + for i := 0; i < now; i++ { + stamp := storageusage.Stamp{ + SatelliteID: satellite, + AtRestTotal: 30000000000000, + IntervalStart: time.Now().UTC().Add(time.Hour * -24 * time.Duration(i)), + } + + summary[satellite] += stamp.AtRestTotal + stamps = append(stamps, stamp) + } + } + + return stamps, summary +} diff --git a/storagenode/console/consoleserver/server.go b/storagenode/console/consoleserver/server.go index 23c3b6e82..918b5bb51 100644 --- a/storagenode/console/consoleserver/server.go +++ b/storagenode/console/consoleserver/server.go @@ -68,6 +68,7 @@ func NewServer(logger *zap.Logger, assets http.FileSystem, notifications *notifi storageNodeRouter.HandleFunc("/", storageNodeController.StorageNode).Methods(http.MethodGet) storageNodeRouter.HandleFunc("/satellites", storageNodeController.Satellites).Methods(http.MethodGet) storageNodeRouter.HandleFunc("/satellite/{id}", storageNodeController.Satellite).Methods(http.MethodGet) + storageNodeRouter.HandleFunc("/estimatedPayout", storageNodeController.EstimatedPayout).Methods(http.MethodGet) notificationController := consoleapi.NewNotifications(server.log, server.notifications) notificationRouter := router.PathPrefix("/api/notifications").Subrouter() diff --git a/storagenode/console/service.go b/storagenode/console/service.go index 7674e347e..8c5998f6f 100644 --- a/storagenode/console/service.go +++ b/storagenode/console/service.go @@ -5,6 +5,7 @@ package console import ( "context" + "math" "time" "github.com/spacemonkeygo/monkit/v3" @@ -18,6 +19,7 @@ import ( "storj.io/storj/private/version/checker" "storj.io/storj/storagenode/bandwidth" "storj.io/storj/storagenode/contact" + "storj.io/storj/storagenode/heldamount" "storj.io/storj/storagenode/pieces" "storj.io/storj/storagenode/pricing" "storj.io/storj/storagenode/reputation" @@ -365,3 +367,163 @@ func (s *Service) VerifySatelliteID(ctx context.Context, satelliteID storj.NodeI return nil } + +// GetSatelliteEstimatedPayout returns estimated payout for current and previous months from specific satellite with current level of load. +func (s *Service) GetSatelliteEstimatedPayout(ctx context.Context, satelliteID storj.NodeID) (payout heldamount.EstimatedPayout, err error) { + defer mon.Task()(&ctx)(&err) + + currentMonthPayout, err := s.estimatedPayoutCurrentMonth(ctx, satelliteID) + if err != nil { + return heldamount.EstimatedPayout{}, SNOServiceErr.Wrap(err) + } + + previousMonthPayout, err := s.estimatedPayoutPreviousMonth(ctx, satelliteID) + if err != nil { + return heldamount.EstimatedPayout{}, SNOServiceErr.Wrap(err) + } + + payout.CurrentMonthEstimatedAmount = currentMonthPayout + payout.PreviousMonthPayout = previousMonthPayout + + return payout, nil +} + +// GetAllSatellitesEstimatedPayout returns estimated payout for current and previous months from all satellites with current level of load. +func (s *Service) GetAllSatellitesEstimatedPayout(ctx context.Context) (payout heldamount.EstimatedPayout, err error) { + defer mon.Task()(&ctx)(&err) + + satelliteIDs := s.trust.GetSatellites(ctx) + for i := 0; i < len(satelliteIDs); i++ { + current, err := s.estimatedPayoutCurrentMonth(ctx, satelliteIDs[i]) + if err != nil { + return heldamount.EstimatedPayout{}, SNOServiceErr.Wrap(err) + } + + previous, err := s.estimatedPayoutPreviousMonth(ctx, satelliteIDs[i]) + if err != nil { + return heldamount.EstimatedPayout{}, SNOServiceErr.Wrap(err) + } + + payout.CurrentMonthEstimatedAmount += current + payout.PreviousMonthPayout.DiskSpaceAmount += previous.DiskSpaceAmount + payout.PreviousMonthPayout.DiskSpace += previous.DiskSpace + payout.PreviousMonthPayout.EgressBandwidth += previous.EgressBandwidth + payout.PreviousMonthPayout.EgressPayout += previous.EgressPayout + payout.PreviousMonthPayout.RepairAuditPayout += previous.RepairAuditPayout + payout.PreviousMonthPayout.EgressRepairAudit += previous.EgressRepairAudit + } + + return payout, nil +} + +// estimatedPayoutCurrentMonth returns estimated payout for current month from specific satellite with current level of load and previous month. +func (s *Service) estimatedPayoutCurrentMonth(ctx context.Context, satelliteID storj.NodeID) (_ int64, err error) { + defer mon.Task()(&ctx)(&err) + + var totalSum int64 + + stats, err := s.reputationDB.Get(ctx, satelliteID) + if err != nil { + return 0, SNOServiceErr.Wrap(err) + } + + heldRate := s.getHeldRate(stats.JoinedAt) + + month := time.Now().UTC() + from, to := date.MonthBoundary(month) + + priceModel, err := s.pricingDB.Get(ctx, satelliteID) + if err != nil { + return 0, SNOServiceErr.Wrap(err) + } + + bandwidthDaily, err := s.bandwidthDB.GetDailySatelliteRollups(ctx, satelliteID, from, to) + if err != nil { + return 0, SNOServiceErr.Wrap(err) + } + + for i := 0; i < len(bandwidthDaily); i++ { + auditDaily := float64(bandwidthDaily[i].Egress.Audit*priceModel.AuditBandwidth) / math.Pow10(12) + repairDaily := float64(bandwidthDaily[i].Egress.Repair*priceModel.RepairBandwidth) / math.Pow10(12) + usageDaily := float64(bandwidthDaily[i].Egress.Usage*priceModel.EgressBandwidth) / math.Pow10(12) + totalSum += int64(auditDaily + repairDaily + usageDaily) + } + + storageDaily, err := s.storageUsageDB.GetDaily(ctx, satelliteID, from, to) + if err != nil { + return 0, SNOServiceErr.Wrap(err) + } + + for j := 0; j < len(storageDaily); j++ { + diskSpace := (storageDaily[j].AtRestTotal * float64(priceModel.DiskSpace) / 730) / math.Pow10(12) + totalSum += int64(diskSpace) + } + + day := int64(time.Now().UTC().Day()) + amount := totalSum - (totalSum*heldRate)/100 + + return amount * int64(to.Day()) / day, nil +} + +// estimatedPayoutPreviousMonth returns estimated payout data for previous month from specific satellite. +func (s *Service) estimatedPayoutPreviousMonth(ctx context.Context, satelliteID storj.NodeID) (payoutData heldamount.PayoutMonthly, err error) { + defer mon.Task()(&ctx)(&err) + + month := time.Now().UTC().AddDate(0, -1, 0).UTC() + from, to := date.MonthBoundary(month) + + priceModel, err := s.pricingDB.Get(ctx, satelliteID) + if err != nil { + return heldamount.PayoutMonthly{}, SNOServiceErr.Wrap(err) + } + + stats, err := s.reputationDB.Get(ctx, satelliteID) + if err != nil { + return heldamount.PayoutMonthly{}, SNOServiceErr.Wrap(err) + } + + heldRate := s.getHeldRate(stats.JoinedAt) + payoutData.HeldPercentRate = heldRate + + bandwidthDaily, err := s.bandwidthDB.GetDailySatelliteRollups(ctx, satelliteID, from, to) + if err != nil { + return heldamount.PayoutMonthly{}, SNOServiceErr.Wrap(err) + } + + for i := 0; i < len(bandwidthDaily); i++ { + payoutData.EgressBandwidth += bandwidthDaily[i].Egress.Usage + usagePayout := float64(bandwidthDaily[i].Egress.Usage*priceModel.EgressBandwidth*heldRate/100) / math.Pow10(12) + payoutData.EgressPayout += int64(usagePayout) + payoutData.EgressRepairAudit += bandwidthDaily[i].Egress.Audit + bandwidthDaily[i].Egress.Repair + repairAuditPayout := float64((bandwidthDaily[i].Egress.Audit*priceModel.AuditBandwidth+bandwidthDaily[i].Egress.Repair*priceModel.RepairBandwidth)*heldRate/100) / math.Pow10(12) + payoutData.RepairAuditPayout += int64(repairAuditPayout) + } + + storageDaily, err := s.storageUsageDB.GetDaily(ctx, satelliteID, from, to) + if err != nil { + return heldamount.PayoutMonthly{}, SNOServiceErr.Wrap(err) + } + + for j := 0; j < len(storageDaily); j++ { + payoutData.DiskSpace += storageDaily[j].AtRestTotal + payoutData.DiskSpaceAmount += int64(storageDaily[j].AtRestTotal / 730 / math.Pow10(12) * float64(priceModel.DiskSpace*heldRate/100)) + } + + return payoutData, nil +} + +func (s *Service) getHeldRate(joinTime time.Time) (heldRate int64) { + monthsSinceJoin := date.MonthsCountSince(joinTime) + switch monthsSinceJoin { + case 0, 1, 2: + heldRate = 75 + case 3, 4, 5: + heldRate = 50 + case 6, 7, 8: + heldRate = 25 + default: + heldRate = 0 + } + + return heldRate +} diff --git a/storagenode/heldamount/heldamount.go b/storagenode/heldamount/heldamount.go index 8dd10f596..06683be58 100644 --- a/storagenode/heldamount/heldamount.go +++ b/storagenode/heldamount/heldamount.go @@ -63,3 +63,20 @@ type Heldback struct { Period string `json:"period"` Held int64 `json:"held"` } + +// EstimatedPayout contains amount in cents of estimated payout for current and previous months. +type EstimatedPayout struct { + CurrentMonthEstimatedAmount int64 `json:"currentAmount"` + PreviousMonthPayout PayoutMonthly `json:"previousPayout"` +} + +// PayoutMonthly contains bandwidth and payout amount for month. +type PayoutMonthly struct { + EgressBandwidth int64 `json:"egressBandwidth"` + EgressPayout int64 `json:"egressPayout"` + EgressRepairAudit int64 `json:"egressRepairAudit"` + RepairAuditPayout int64 `json:"repairAuditPayout"` + DiskSpace float64 `json:"diskSpace"` + DiskSpaceAmount int64 `json:"diskSpaceAmount"` + HeldPercentRate int64 `json:"heldRate"` +}