storagenode/console: added estimated payout for current month and estimated pay stub for previous month (until there's real data in satellite's table) + heldback percentage rate for previous month.

Change-Id: I9346f6d22ed6fbb7e5346b102fc898467678f384

private/date/utils.go

@@ -30,3 +30,25 @@ func PeriodToTime(period string) (_ time.Time, err error) {
 
 	return result, nil
 }
+
+// MonthsCountSince calculates the months between now and the createdAtTime time.Time value passed.
+func MonthsCountSince(from time.Time) int {
+	now := time.Now().UTC()
+	return MonthsBetweenDates(from, now)
+}
+
+// MonthsBetweenDates calculates amount of months between two dates
+func MonthsBetweenDates(from time.Time, to time.Time) int {
+	months := 0
+	month := from.Month()
+	for from.Before(to) {
+		from = from.Add(time.Hour * 24)
+		nextMonth := from.Month()
+		if nextMonth != month {
+			months++
+		}
+		month = nextMonth
+	}
+
+	return months
+}

private/date/utils_test.go

@@ -47,3 +47,23 @@ func TestPeriodToTime(t *testing.T) {
 		require.Equal(t, periodTime.String(), tc.periodTime.String())
 	}
 }
+
+func TestMonthsBetweenDates(t *testing.T) {
+	testCases := [...]struct {
+		from         time.Time
+		to           time.Time
+		monthsAmount int
+	}{
+		{time.Date(2020, 2, 13, 0, 0, 0, 0, &time.Location{}), time.Date(2020, 05, 13, 0, 0, 0, 0, &time.Location{}), 3},
+		{time.Date(2015, 7, 30, 0, 0, 0, 0, &time.Location{}), time.Date(2020, 05, 13, 0, 0, 0, 0, &time.Location{}), 58},
+		{time.Date(2017, 1, 28, 0, 0, 0, 0, &time.Location{}), time.Date(2020, 05, 13, 0, 0, 0, 0, &time.Location{}), 40},
+		{time.Date(2016, 11, 1, 0, 0, 0, 0, &time.Location{}), time.Date(2020, 05, 13, 0, 0, 0, 0, &time.Location{}), 42},
+		{time.Date(2019, 4, 17, 0, 0, 0, 0, &time.Location{}), time.Date(2020, 05, 13, 0, 0, 0, 0, &time.Location{}), 13},
+		{time.Date(2018, 9, 11, 0, 0, 0, 0, &time.Location{}), time.Date(2020, 05, 13, 0, 0, 0, 0, &time.Location{}), 20},
+	}
+
+	for _, tc := range testCases {
+		monthDiff := date.MonthsBetweenDates(tc.from, tc.to)
+		require.Equal(t, monthDiff, tc.monthsAmount)
+	}
+}

storagenode/console/consoleapi/storagenode.go

@@ -111,6 +111,47 @@ func (dashboard *StorageNode) Satellite(w http.ResponseWriter, r *http.Request)
 	}
 }
 
+// EstimatedPayout returns estimated payout from specific satellite or all satellites if current traffic level remains same.
+func (dashboard *StorageNode) EstimatedPayout(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	var err error
+	defer mon.Task()(&ctx)(&err)
+
+	w.Header().Set(contentType, applicationJSON)
+
+	queryParams := r.URL.Query()
+	id := queryParams.Get("id")
+	if id == "" {
+		data, err := dashboard.service.GetAllSatellitesEstimatedPayout(ctx)
+		if err != nil {
+			dashboard.serveJSONError(w, http.StatusInternalServerError, ErrStorageNodeAPI.Wrap(err))
+			return
+		}
+
+		if err := json.NewEncoder(w).Encode(data); err != nil {
+			dashboard.log.Error("failed to encode json response", zap.Error(ErrHeldAmountAPI.Wrap(err)))
+			return
+		}
+	} else {
+		satelliteID, err := storj.NodeIDFromString(id)
+		if err != nil {
+			dashboard.serveJSONError(w, http.StatusBadRequest, ErrHeldAmountAPI.Wrap(err))
+			return
+		}
+
+		data, err := dashboard.service.GetSatelliteEstimatedPayout(ctx, satelliteID)
+		if err != nil {
+			dashboard.serveJSONError(w, http.StatusInternalServerError, ErrStorageNodeAPI.Wrap(err))
+			return
+		}
+
+		if err := json.NewEncoder(w).Encode(data); err != nil {
+			dashboard.log.Error("failed to encode json response", zap.Error(ErrHeldAmountAPI.Wrap(err)))
+			return
+		}
+	}
+}
+
 // serveJSONError writes JSON error to response output stream.
 func (dashboard *StorageNode) serveJSONError(w http.ResponseWriter, status int, err error) {
 	w.WriteHeader(status)

storagenode/console/consoleapi/storagenode_test.go

@@ -0,0 +1,219 @@
+// Copyright (C) 2020 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package consoleapi_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"math"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"storj.io/common/pb"
+	"storj.io/common/storj"
+	"storj.io/common/testcontext"
+	"storj.io/storj/private/date"
+	"storj.io/storj/private/testplanet"
+	"storj.io/storj/storagenode/heldamount"
+	"storj.io/storj/storagenode/pricing"
+	"storj.io/storj/storagenode/reputation"
+	"storj.io/storj/storagenode/storageusage"
+)
+
+var (
+	actions = []pb.PieceAction{
+		pb.PieceAction_INVALID,
+
+		pb.PieceAction_PUT,
+		pb.PieceAction_GET,
+		pb.PieceAction_GET_AUDIT,
+		pb.PieceAction_GET_REPAIR,
+		pb.PieceAction_PUT_REPAIR,
+		pb.PieceAction_DELETE,
+
+		pb.PieceAction_PUT,
+		pb.PieceAction_GET,
+		pb.PieceAction_GET_AUDIT,
+		pb.PieceAction_GET_REPAIR,
+		pb.PieceAction_PUT_REPAIR,
+		pb.PieceAction_DELETE,
+	}
+)
+
+func TestStorageNodeApi(t *testing.T) {
+	testplanet.Run(t,
+		testplanet.Config{
+			SatelliteCount:   2,
+			StorageNodeCount: 1,
+		},
+		func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
+			satellite := planet.Satellites[0]
+			satellite2 := planet.Satellites[1]
+			sno := planet.StorageNodes[0]
+			console := sno.Console
+			bandwidthdb := sno.DB.Bandwidth()
+			pricingdb := sno.DB.Pricing()
+			storageusagedb := sno.DB.StorageUsage()
+			reputationdb := sno.DB.Reputation()
+			baseURL := fmt.Sprintf("http://%s/api/sno", console.Listener.Addr())
+
+			now := time.Now().UTC().Add(-2 * time.Hour)
+
+			randAmount1 := int64(120000000000)
+			randAmount2 := int64(450000000000)
+
+			for _, action := range actions {
+				err := bandwidthdb.Add(ctx, satellite.ID(), action, randAmount1, now)
+				require.NoError(t, err)
+
+				err = bandwidthdb.Add(ctx, satellite2.ID(), action, randAmount2, now.Add(2*time.Hour))
+				require.NoError(t, err)
+			}
+			var satellites []storj.NodeID
+
+			satellites = append(satellites, satellite.ID(), satellite2.ID())
+			stamps, _ := makeStorageUsageStamps(satellites)
+
+			err := storageusagedb.Store(ctx, stamps)
+			require.NoError(t, err)
+
+			egressPrice, repairPrice, auditPrice, diskPrice := int64(2000), int64(1000), int64(1000), int64(150)
+
+			err = pricingdb.Store(ctx, pricing.Pricing{
+				SatelliteID:     satellite.ID(),
+				EgressBandwidth: egressPrice,
+				RepairBandwidth: repairPrice,
+				AuditBandwidth:  auditPrice,
+				DiskSpace:       diskPrice,
+			})
+			require.NoError(t, err)
+
+			err = pricingdb.Store(ctx, pricing.Pricing{
+				SatelliteID:     satellite2.ID(),
+				EgressBandwidth: egressPrice,
+				RepairBandwidth: repairPrice,
+				AuditBandwidth:  auditPrice,
+				DiskSpace:       diskPrice,
+			})
+			require.NoError(t, err)
+
+			err = reputationdb.Store(ctx, reputation.Stats{
+				SatelliteID: satellite.ID(),
+				JoinedAt:    time.Now().UTC(),
+			})
+			require.NoError(t, err)
+			err = reputationdb.Store(ctx, reputation.Stats{
+				SatelliteID: satellite2.ID(),
+				JoinedAt:    time.Now().UTC(),
+			})
+			require.NoError(t, err)
+
+			t.Run("test EstimatedPayout", func(t *testing.T) {
+				// should return estimated payout for both satellites in current month and empty for previous
+				url := fmt.Sprintf("%s/estimatedPayout", baseURL)
+				res, err := http.Get(url)
+				require.NoError(t, err)
+				require.NotNil(t, res)
+				require.Equal(t, http.StatusOK, res.StatusCode)
+
+				defer func() {
+					err = res.Body.Close()
+					require.NoError(t, err)
+				}()
+				body, err := ioutil.ReadAll(res.Body)
+				require.NoError(t, err)
+
+				expectedAuditRepairSatellite1 := 4 * (float64(randAmount1*auditPrice) / math.Pow10(12))
+				expectedAuditRepairSatellite2 := 4 * float64(randAmount2*repairPrice) / math.Pow10(12)
+				expectedUsageSatellite1 := 2 * float64(randAmount1*egressPrice) / math.Pow10(12)
+				expectedUsageSatellite2 := 2 * float64(randAmount2*egressPrice) / math.Pow10(12)
+				expectedDisk := int64(float64(30000000000000*diskPrice/730)/math.Pow10(12)) * int64(time.Now().UTC().Day())
+
+				day := int64(time.Now().Day())
+
+				month := time.Now().UTC()
+				_, to := date.MonthBoundary(month)
+
+				sum1 := expectedAuditRepairSatellite1 + expectedUsageSatellite1 + float64(expectedDisk)
+				sum1AfterHeld := math.Round(sum1 / 4)
+				estimated1 := int64(sum1AfterHeld) * int64(to.Day()) / day
+				sum2 := expectedAuditRepairSatellite2 + expectedUsageSatellite2 + float64(expectedDisk)
+				sum2AfterHeld := math.Round(sum2 / 4)
+				estimated2 := int64(sum2AfterHeld) * int64(to.Day()) / day
+
+				expected, err := json.Marshal(heldamount.EstimatedPayout{
+					CurrentMonthEstimatedAmount: estimated1 + estimated2,
+					PreviousMonthPayout: heldamount.PayoutMonthly{
+						EgressBandwidth:   0,
+						EgressPayout:      0,
+						EgressRepairAudit: 0,
+						RepairAuditPayout: 0,
+						DiskSpace:         0,
+						DiskSpaceAmount:   0,
+						HeldPercentRate:   0,
+					},
+				})
+				require.NoError(t, err)
+				require.Equal(t, string(expected)+"\n", string(body))
+
+				// should return estimated payout for first satellite in current month and empty for previous
+				url = fmt.Sprintf("%s/estimatedPayout?id=%s", baseURL, satellite.ID().String())
+				res2, err := http.Get(url)
+				require.NoError(t, err)
+				require.NotNil(t, res)
+				require.Equal(t, http.StatusOK, res.StatusCode)
+
+				defer func() {
+					err = res2.Body.Close()
+					require.NoError(t, err)
+				}()
+				body2, err := ioutil.ReadAll(res2.Body)
+				require.NoError(t, err)
+
+				expected2, err := json.Marshal(heldamount.EstimatedPayout{
+					CurrentMonthEstimatedAmount: estimated1,
+					PreviousMonthPayout: heldamount.PayoutMonthly{
+						EgressBandwidth:   0,
+						EgressPayout:      0,
+						EgressRepairAudit: 0,
+						RepairAuditPayout: 0,
+						DiskSpace:         0,
+						DiskSpaceAmount:   0,
+						HeldPercentRate:   75,
+					},
+				})
+				require.NoError(t, err)
+				require.Equal(t, string(expected2)+"\n", string(body2))
+			})
+		},
+	)
+}
+
+// makeStorageUsageStamps creates storage usage stamps and expected summaries for provided satellites.
+// Creates one entry per day for 30 days with last date as beginning of provided endDate.
+func makeStorageUsageStamps(satellites []storj.NodeID) ([]storageusage.Stamp, map[storj.NodeID]float64) {
+	var stamps []storageusage.Stamp
+	summary := make(map[storj.NodeID]float64)
+
+	now := time.Now().UTC().Day()
+
+	for _, satellite := range satellites {
+		for i := 0; i < now; i++ {
+			stamp := storageusage.Stamp{
+				SatelliteID:   satellite,
+				AtRestTotal:   30000000000000,
+				IntervalStart: time.Now().UTC().Add(time.Hour * -24 * time.Duration(i)),
+			}
+
+			summary[satellite] += stamp.AtRestTotal
+			stamps = append(stamps, stamp)
+		}
+	}
+
+	return stamps, summary
+}

storagenode/console/consoleserver/server.go

@@ -68,6 +68,7 @@ func NewServer(logger *zap.Logger, assets http.FileSystem, notifications *notifi
 	storageNodeRouter.HandleFunc("/", storageNodeController.StorageNode).Methods(http.MethodGet)
 	storageNodeRouter.HandleFunc("/satellites", storageNodeController.Satellites).Methods(http.MethodGet)
 	storageNodeRouter.HandleFunc("/satellite/{id}", storageNodeController.Satellite).Methods(http.MethodGet)
+	storageNodeRouter.HandleFunc("/estimatedPayout", storageNodeController.EstimatedPayout).Methods(http.MethodGet)
 
 	notificationController := consoleapi.NewNotifications(server.log, server.notifications)
 	notificationRouter := router.PathPrefix("/api/notifications").Subrouter()

storagenode/console/service.go

@@ -5,6 +5,7 @@ package console
 
 import (
 	"context"
+	"math"
 	"time"
 
 	"github.com/spacemonkeygo/monkit/v3"
@@ -18,6 +19,7 @@ import (
 	"storj.io/storj/private/version/checker"
 	"storj.io/storj/storagenode/bandwidth"
 	"storj.io/storj/storagenode/contact"
+	"storj.io/storj/storagenode/heldamount"
 	"storj.io/storj/storagenode/pieces"
 	"storj.io/storj/storagenode/pricing"
 	"storj.io/storj/storagenode/reputation"
@@ -365,3 +367,163 @@ func (s *Service) VerifySatelliteID(ctx context.Context, satelliteID storj.NodeI
 
 	return nil
 }
+
+// GetSatelliteEstimatedPayout returns estimated payout for current and previous months from specific satellite with current level of load.
+func (s *Service) GetSatelliteEstimatedPayout(ctx context.Context, satelliteID storj.NodeID) (payout heldamount.EstimatedPayout, err error) {
+	defer mon.Task()(&ctx)(&err)
+
+	currentMonthPayout, err := s.estimatedPayoutCurrentMonth(ctx, satelliteID)
+	if err != nil {
+		return heldamount.EstimatedPayout{}, SNOServiceErr.Wrap(err)
+	}
+
+	previousMonthPayout, err := s.estimatedPayoutPreviousMonth(ctx, satelliteID)
+	if err != nil {
+		return heldamount.EstimatedPayout{}, SNOServiceErr.Wrap(err)
+	}
+
+	payout.CurrentMonthEstimatedAmount = currentMonthPayout
+	payout.PreviousMonthPayout = previousMonthPayout
+
+	return payout, nil
+}
+
+// GetAllSatellitesEstimatedPayout returns estimated payout for current and previous months from all satellites with current level of load.
+func (s *Service) GetAllSatellitesEstimatedPayout(ctx context.Context) (payout heldamount.EstimatedPayout, err error) {
+	defer mon.Task()(&ctx)(&err)
+
+	satelliteIDs := s.trust.GetSatellites(ctx)
+	for i := 0; i < len(satelliteIDs); i++ {
+		current, err := s.estimatedPayoutCurrentMonth(ctx, satelliteIDs[i])
+		if err != nil {
+			return heldamount.EstimatedPayout{}, SNOServiceErr.Wrap(err)
+		}
+
+		previous, err := s.estimatedPayoutPreviousMonth(ctx, satelliteIDs[i])
+		if err != nil {
+			return heldamount.EstimatedPayout{}, SNOServiceErr.Wrap(err)
+		}
+
+		payout.CurrentMonthEstimatedAmount += current
+		payout.PreviousMonthPayout.DiskSpaceAmount += previous.DiskSpaceAmount
+		payout.PreviousMonthPayout.DiskSpace += previous.DiskSpace
+		payout.PreviousMonthPayout.EgressBandwidth += previous.EgressBandwidth
+		payout.PreviousMonthPayout.EgressPayout += previous.EgressPayout
+		payout.PreviousMonthPayout.RepairAuditPayout += previous.RepairAuditPayout
+		payout.PreviousMonthPayout.EgressRepairAudit += previous.EgressRepairAudit
+	}
+
+	return payout, nil
+}
+
+// estimatedPayoutCurrentMonth returns estimated payout for current month from specific satellite with current level of load and previous month.
+func (s *Service) estimatedPayoutCurrentMonth(ctx context.Context, satelliteID storj.NodeID) (_ int64, err error) {
+	defer mon.Task()(&ctx)(&err)
+
+	var totalSum int64
+
+	stats, err := s.reputationDB.Get(ctx, satelliteID)
+	if err != nil {
+		return 0, SNOServiceErr.Wrap(err)
+	}
+
+	heldRate := s.getHeldRate(stats.JoinedAt)
+
+	month := time.Now().UTC()
+	from, to := date.MonthBoundary(month)
+
+	priceModel, err := s.pricingDB.Get(ctx, satelliteID)
+	if err != nil {
+		return 0, SNOServiceErr.Wrap(err)
+	}
+
+	bandwidthDaily, err := s.bandwidthDB.GetDailySatelliteRollups(ctx, satelliteID, from, to)
+	if err != nil {
+		return 0, SNOServiceErr.Wrap(err)
+	}
+
+	for i := 0; i < len(bandwidthDaily); i++ {
+		auditDaily := float64(bandwidthDaily[i].Egress.Audit*priceModel.AuditBandwidth) / math.Pow10(12)
+		repairDaily := float64(bandwidthDaily[i].Egress.Repair*priceModel.RepairBandwidth) / math.Pow10(12)
+		usageDaily := float64(bandwidthDaily[i].Egress.Usage*priceModel.EgressBandwidth) / math.Pow10(12)
+		totalSum += int64(auditDaily + repairDaily + usageDaily)
+	}
+
+	storageDaily, err := s.storageUsageDB.GetDaily(ctx, satelliteID, from, to)
+	if err != nil {
+		return 0, SNOServiceErr.Wrap(err)
+	}
+
+	for j := 0; j < len(storageDaily); j++ {
+		diskSpace := (storageDaily[j].AtRestTotal * float64(priceModel.DiskSpace) / 730) / math.Pow10(12)
+		totalSum += int64(diskSpace)
+	}
+
+	day := int64(time.Now().UTC().Day())
+	amount := totalSum - (totalSum*heldRate)/100
+
+	return amount * int64(to.Day()) / day, nil
+}
+
+// estimatedPayoutPreviousMonth returns estimated payout data for previous month from specific satellite.
+func (s *Service) estimatedPayoutPreviousMonth(ctx context.Context, satelliteID storj.NodeID) (payoutData heldamount.PayoutMonthly, err error) {
+	defer mon.Task()(&ctx)(&err)
+
+	month := time.Now().UTC().AddDate(0, -1, 0).UTC()
+	from, to := date.MonthBoundary(month)
+
+	priceModel, err := s.pricingDB.Get(ctx, satelliteID)
+	if err != nil {
+		return heldamount.PayoutMonthly{}, SNOServiceErr.Wrap(err)
+	}
+
+	stats, err := s.reputationDB.Get(ctx, satelliteID)
+	if err != nil {
+		return heldamount.PayoutMonthly{}, SNOServiceErr.Wrap(err)
+	}
+
+	heldRate := s.getHeldRate(stats.JoinedAt)
+	payoutData.HeldPercentRate = heldRate
+
+	bandwidthDaily, err := s.bandwidthDB.GetDailySatelliteRollups(ctx, satelliteID, from, to)
+	if err != nil {
+		return heldamount.PayoutMonthly{}, SNOServiceErr.Wrap(err)
+	}
+
+	for i := 0; i < len(bandwidthDaily); i++ {
+		payoutData.EgressBandwidth += bandwidthDaily[i].Egress.Usage
+		usagePayout := float64(bandwidthDaily[i].Egress.Usage*priceModel.EgressBandwidth*heldRate/100) / math.Pow10(12)
+		payoutData.EgressPayout += int64(usagePayout)
+		payoutData.EgressRepairAudit += bandwidthDaily[i].Egress.Audit + bandwidthDaily[i].Egress.Repair
+		repairAuditPayout := float64((bandwidthDaily[i].Egress.Audit*priceModel.AuditBandwidth+bandwidthDaily[i].Egress.Repair*priceModel.RepairBandwidth)*heldRate/100) / math.Pow10(12)
+		payoutData.RepairAuditPayout += int64(repairAuditPayout)
+	}
+
+	storageDaily, err := s.storageUsageDB.GetDaily(ctx, satelliteID, from, to)
+	if err != nil {
+		return heldamount.PayoutMonthly{}, SNOServiceErr.Wrap(err)
+	}
+
+	for j := 0; j < len(storageDaily); j++ {
+		payoutData.DiskSpace += storageDaily[j].AtRestTotal
+		payoutData.DiskSpaceAmount += int64(storageDaily[j].AtRestTotal / 730 / math.Pow10(12) * float64(priceModel.DiskSpace*heldRate/100))
+	}
+
+	return payoutData, nil
+}
+
+func (s *Service) getHeldRate(joinTime time.Time) (heldRate int64) {
+	monthsSinceJoin := date.MonthsCountSince(joinTime)
+	switch monthsSinceJoin {
+	case 0, 1, 2:
+		heldRate = 75
+	case 3, 4, 5:
+		heldRate = 50
+	case 6, 7, 8:
+		heldRate = 25
+	default:
+		heldRate = 0
+	}
+
+	return heldRate
+}

storagenode/heldamount/heldamount.go

@@ -63,3 +63,20 @@ type Heldback struct {
 	Period string `json:"period"`
 	Held   int64  `json:"held"`
 }
+
+// EstimatedPayout contains amount in cents of estimated payout for current and previous months.
+type EstimatedPayout struct {
+	CurrentMonthEstimatedAmount int64         `json:"currentAmount"`
+	PreviousMonthPayout         PayoutMonthly `json:"previousPayout"`
+}
+
+// PayoutMonthly contains bandwidth and payout amount for month.
+type PayoutMonthly struct {
+	EgressBandwidth   int64   `json:"egressBandwidth"`
+	EgressPayout      int64   `json:"egressPayout"`
+	EgressRepairAudit int64   `json:"egressRepairAudit"`
+	RepairAuditPayout int64   `json:"repairAuditPayout"`
+	DiskSpace         float64 `json:"diskSpace"`
+	DiskSpaceAmount   int64   `json:"diskSpaceAmount"`
+	HeldPercentRate   int64   `json:"heldRate"`
+}