246 lines
7.3 KiB
Go
246 lines
7.3 KiB
Go
// Copyright (C) 2019 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package accounting_test
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/skyrings/skyring-common/tools/uuid"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"storj.io/storj/pkg/storj"
|
|
"storj.io/storj/private/memory"
|
|
"storj.io/storj/private/testcontext"
|
|
"storj.io/storj/private/testrand"
|
|
"storj.io/storj/satellite"
|
|
"storj.io/storj/satellite/accounting"
|
|
"storj.io/storj/satellite/satellitedb/satellitedbtest"
|
|
)
|
|
|
|
func TestSaveBucketTallies(t *testing.T) {
|
|
satellitedbtest.Run(t, func(t *testing.T, db satellite.DB) {
|
|
ctx := testcontext.New(t)
|
|
defer ctx.Cleanup()
|
|
|
|
// Setup: create bucket storage tallies
|
|
projectID := testrand.UUID()
|
|
|
|
bucketTallies, expectedTallies, err := createBucketStorageTallies(projectID)
|
|
require.NoError(t, err)
|
|
|
|
// Execute test: retrieve the save tallies and confirm they contains the expected data
|
|
intervalStart := time.Now()
|
|
pdb := db.ProjectAccounting()
|
|
|
|
err = pdb.SaveTallies(ctx, intervalStart, bucketTallies)
|
|
require.NoError(t, err)
|
|
|
|
tallies, err := pdb.GetTallies(ctx)
|
|
require.NoError(t, err)
|
|
for _, tally := range tallies {
|
|
require.Contains(t, expectedTallies, tally)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestStorageNodeUsage(t *testing.T) {
|
|
satellitedbtest.Run(t, func(t *testing.T, db satellite.DB) {
|
|
ctx := testcontext.New(t)
|
|
defer ctx.Cleanup()
|
|
|
|
const days = 30
|
|
|
|
now := time.Now().UTC()
|
|
|
|
nodeID := testrand.NodeID()
|
|
startDate := now.Add(time.Hour * 24 * -days)
|
|
|
|
var nodes storj.NodeIDList
|
|
nodes = append(nodes, nodeID, testrand.NodeID(), testrand.NodeID(), testrand.NodeID())
|
|
|
|
rollups, tallies, lastDate := makeRollupsAndStorageNodeStorageTallies(nodes, startDate, days)
|
|
|
|
lastRollup := rollups[lastDate]
|
|
delete(rollups, lastDate)
|
|
|
|
accountingDB := db.StoragenodeAccounting()
|
|
|
|
// create last rollup timestamp
|
|
_, err := accountingDB.LastTimestamp(ctx, accounting.LastRollup)
|
|
require.NoError(t, err)
|
|
|
|
// save tallies
|
|
for latestTally, tallies := range tallies {
|
|
err = accountingDB.SaveTallies(ctx, latestTally, tallies)
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
// save rollup
|
|
err = accountingDB.SaveRollup(ctx, lastDate.Add(time.Hour*-24), rollups)
|
|
require.NoError(t, err)
|
|
|
|
t.Run("usage with pending tallies", func(t *testing.T) {
|
|
nodeStorageUsages, err := accountingDB.QueryStorageNodeUsage(ctx, nodeID, time.Time{}, now)
|
|
require.NoError(t, err)
|
|
assert.NotNil(t, nodeStorageUsages)
|
|
assert.Equal(t, days, len(nodeStorageUsages))
|
|
|
|
// check usage from rollups
|
|
for _, usage := range nodeStorageUsages[:len(nodeStorageUsages)-1] {
|
|
assert.Equal(t, nodeID, usage.NodeID)
|
|
dateRollup, ok := rollups[usage.Timestamp.UTC()]
|
|
if assert.True(t, ok) {
|
|
nodeRollup, ok := dateRollup[nodeID]
|
|
if assert.True(t, ok) {
|
|
assert.Equal(t, nodeRollup.AtRestTotal, usage.StorageUsed)
|
|
}
|
|
}
|
|
}
|
|
|
|
// check last usage that calculated from tallies
|
|
lastUsage := nodeStorageUsages[len(nodeStorageUsages)-1]
|
|
|
|
assert.Equal(t, nodeID, lastUsage.NodeID)
|
|
assert.Equal(t, lastRollup[nodeID].StartTime, lastUsage.Timestamp.UTC())
|
|
assert.Equal(t, lastRollup[nodeID].AtRestTotal, lastUsage.StorageUsed)
|
|
})
|
|
|
|
t.Run("usage entirely from rollups", func(t *testing.T) {
|
|
const (
|
|
start = 10
|
|
// should be greater than 2
|
|
// not to include tallies into result
|
|
end = 2
|
|
)
|
|
|
|
startDate, endDate := now.Add(time.Hour*24*-start), now.Add(time.Hour*24*-end)
|
|
|
|
nodeStorageUsages, err := accountingDB.QueryStorageNodeUsage(ctx, nodeID, startDate, endDate)
|
|
require.NoError(t, err)
|
|
assert.NotNil(t, nodeStorageUsages)
|
|
assert.Equal(t, start-end, len(nodeStorageUsages))
|
|
|
|
for _, usage := range nodeStorageUsages {
|
|
assert.Equal(t, nodeID, usage.NodeID)
|
|
dateRollup, ok := rollups[usage.Timestamp.UTC()]
|
|
if assert.True(t, ok) {
|
|
nodeRollup, ok := dateRollup[nodeID]
|
|
if assert.True(t, ok) {
|
|
assert.Equal(t, nodeRollup.AtRestTotal, usage.StorageUsed)
|
|
}
|
|
}
|
|
}
|
|
})
|
|
})
|
|
}
|
|
|
|
func TestProjectLimits(t *testing.T) {
|
|
satellitedbtest.Run(t, func(t *testing.T, db satellite.DB) {
|
|
ctx := testcontext.New(t)
|
|
defer ctx.Cleanup()
|
|
|
|
projectID := testrand.UUID()
|
|
|
|
err := db.ProjectAccounting().UpdateProjectStorageLimit(ctx, projectID, 1)
|
|
require.NoError(t, err)
|
|
|
|
err = db.ProjectAccounting().UpdateProjectBandwidthLimit(ctx, projectID, 2)
|
|
require.NoError(t, err)
|
|
|
|
t.Run("get", func(t *testing.T) {
|
|
storageLimit, err := db.ProjectAccounting().GetProjectStorageLimit(ctx, projectID)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, memory.Size(1), storageLimit)
|
|
|
|
bandwidthLimit, err := db.ProjectAccounting().GetProjectBandwidthLimit(ctx, projectID)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, memory.Size(2), bandwidthLimit)
|
|
})
|
|
|
|
t.Run("update", func(t *testing.T) {
|
|
err := db.ProjectAccounting().UpdateProjectStorageLimit(ctx, projectID, 3)
|
|
require.NoError(t, err)
|
|
err = db.ProjectAccounting().UpdateProjectBandwidthLimit(ctx, projectID, 4)
|
|
require.NoError(t, err)
|
|
|
|
storageLimit, err := db.ProjectAccounting().GetProjectStorageLimit(ctx, projectID)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, memory.Size(3), storageLimit)
|
|
|
|
bandwidthLimit, err := db.ProjectAccounting().GetProjectBandwidthLimit(ctx, projectID)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, memory.Size(4), bandwidthLimit)
|
|
})
|
|
})
|
|
}
|
|
|
|
func createBucketStorageTallies(projectID uuid.UUID) (map[string]*accounting.BucketTally, []accounting.BucketTally, error) {
|
|
bucketTallies := make(map[string]*accounting.BucketTally)
|
|
var expectedTallies []accounting.BucketTally
|
|
|
|
for i := 0; i < 4; i++ {
|
|
bucketName := fmt.Sprintf("%s%d", "testbucket", i)
|
|
bucketID := storj.JoinPaths(projectID.String(), bucketName)
|
|
|
|
// Setup: The data in this tally should match the pointer that the uplink.upload created
|
|
tally := accounting.BucketTally{
|
|
BucketName: []byte(bucketName),
|
|
ProjectID: projectID,
|
|
ObjectCount: int64(1),
|
|
InlineSegments: int64(1),
|
|
RemoteSegments: int64(1),
|
|
InlineBytes: int64(1),
|
|
RemoteBytes: int64(1),
|
|
MetadataSize: int64(1),
|
|
}
|
|
bucketTallies[bucketID] = &tally
|
|
expectedTallies = append(expectedTallies, tally)
|
|
|
|
}
|
|
return bucketTallies, expectedTallies, nil
|
|
}
|
|
|
|
// make rollups and tallies for specified nodes and date range.
|
|
func makeRollupsAndStorageNodeStorageTallies(nodes []storj.NodeID, start time.Time, days int) (accounting.RollupStats, map[time.Time]map[storj.NodeID]float64, time.Time) {
|
|
rollups := make(accounting.RollupStats)
|
|
tallies := make(map[time.Time]map[storj.NodeID]float64)
|
|
|
|
const (
|
|
hours = 12
|
|
)
|
|
|
|
for i := 0; i < days; i++ {
|
|
startDay := time.Date(start.Year(), start.Month(), start.Day()+i, 0, 0, 0, 0, start.Location())
|
|
if rollups[startDay] == nil {
|
|
rollups[startDay] = make(map[storj.NodeID]*accounting.Rollup)
|
|
}
|
|
|
|
for _, node := range nodes {
|
|
rollup := &accounting.Rollup{
|
|
NodeID: node,
|
|
StartTime: startDay,
|
|
}
|
|
|
|
for h := 0; h < hours; h++ {
|
|
startTime := startDay.Add(time.Hour * time.Duration(h))
|
|
if tallies[startTime] == nil {
|
|
tallies[startTime] = make(map[storj.NodeID]float64)
|
|
}
|
|
|
|
tallieAtRest := math.Round(testrand.Float64n(1000))
|
|
tallies[startTime][node] = tallieAtRest
|
|
rollup.AtRestTotal += tallieAtRest
|
|
}
|
|
|
|
rollups[startDay][node] = rollup
|
|
}
|
|
}
|
|
|
|
return rollups, tallies, time.Date(start.Year(), start.Month(), start.Day()+days-1, 0, 0, 0, 0, start.Location())
|
|
}
|