JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

use allocated instead of settled (#1700)

Browse Source 
* use allocated instead of settled

* add expansion factor

* changes per CR
This commit is contained in:
Jess G 2019-04-08 14:35:54 -07:00 committed by GitHub
parent 061deb6add
commit 5dfe28a8c3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 130 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/accounting/db.go
View File

@ -68,8 +68,8 @@ type DB interface {
DeleteRawBefore(ctx context.Context, latestRollup time.Time) error
// CreateBucketStorageTally creates a record for BucketStorageTally in the accounting DB table
CreateBucketStorageTally(ctx context.Context, tally BucketStorageTally) error
// ProjectBandwidthTotal returns the sum of GET bandwidth usage for a projectID in the past time frame
ProjectBandwidthTotal(ctx context.Context, bucketID []byte, from time.Time) (int64, error)
// ProjectAllocatedBandwidthTotal returns the sum of GET bandwidth usage allocated for a projectID in the past time frame
ProjectAllocatedBandwidthTotal(ctx context.Context, bucketID []byte, from time.Time) (int64, error)
// ProjectStorageTotals returns the current inline and remote storage usage for a projectID
ProjectStorageTotals(ctx context.Context, projectID uuid.UUID) (int64, int64, error)
}

13
pkg/accounting/projectusage.go
View File

@ -10,18 +10,23 @@ import (
const (
// AverageDaysInMonth is how many days in a month
AverageDaysInMonth = 30
// ExpansionFactor is the expansion for redundancy, based on the default
// redundancy scheme for the uplink.
ExpansionFactor = 3
)
// ExceedsAlphaUsage returns true if more than 25GB of storage is currently in use
// or if 25GB of bandwidth or has been used in the past month (30 days)
// ExceedsAlphaUsage returns true if the storage or bandwidth usage limits have been exceeded
// for a project in the past month (30 days). The usage limit is 25GB multiplied by the redundancy
// expansion factor, so that the uplinks have a raw limit of 25GB.
// TODO(jg): remove this code once we no longer need usage limiting for alpha release
// Ref: https://storjlabs.atlassian.net/browse/V3-1274
func ExceedsAlphaUsage(bandwidthGetTotal, inlineTotal, remoteTotal int64, maxAlphaUsageGB memory.Size) (bool, string) {
if bandwidthGetTotal >= maxAlphaUsageGB.Int64() {
maxUsage := maxAlphaUsageGB.Int64() * int64(ExpansionFactor)
if bandwidthGetTotal >= maxUsage {
return true, "bandwidth"
}
if inlineTotal+remoteTotal >= maxAlphaUsageGB.Int64() {
if inlineTotal+remoteTotal >= maxUsage {
return true, "storage"
}

149
pkg/accounting/projectusage_test.go
View File

@ -19,9 +19,10 @@ import (
"storj.io/storj/pkg/accounting"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/storj"
"storj.io/storj/satellite/orders"
)
func TestProjectUsage(t *testing.T) {
func TestProjectUsageStorage(t *testing.T) {
cases := []struct {
name string
expectedExceeded bool
@ -30,7 +31,63 @@ func TestProjectUsage(t *testing.T) {
}{
{name: "doesn't exceed storage or bandwidth project limit", expectedExceeded: false, expectedErrMsg: ""},
{name: "exceeds storage project limit", expectedExceeded: true, expectedResource: "storage", expectedErrMsg: "segment error: metainfo error: rpc error: code = ResourceExhausted desc = Exceeded Alpha Usage Limit; segment error: metainfo error: rpc error: code = ResourceExhausted desc = Exceeded Alpha Usage Limit"},
{name: "exceeds bandwidth project limit", expectedExceeded: true, expectedResource: "bandwidth", expectedErrMsg: "segment error: metainfo error: rpc error: code = ResourceExhausted desc = Exceeded Alpha Usage Limit; segment error: metainfo error: rpc error: code = ResourceExhausted desc = Exceeded Alpha Usage Limit"},
}
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1, StorageNodeCount: 6, UplinkCount: 1,
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
saDB := planet.Satellites[0].DB
acctDB := saDB.Accounting()
// Setup: create a new project to use the projectID
projects, err := planet.Satellites[0].DB.Console().Projects().GetAll(ctx)
projectID := projects[0].ID
require.NoError(t, err)
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
// Setup: create BucketStorageTally records to test exceeding storage project limit
if tt.expectedResource == "storage" {
now := time.Now()
err := setUpStorageTallies(ctx, projectID, acctDB, now)
require.NoError(t, err)
}
// Execute test: get storage totals for a project, then check if that exceeds the max usage limit
inlineTotal, remoteTotal, err := acctDB.ProjectStorageTotals(ctx, projectID)
require.NoError(t, err)
maxAlphaUsage := 25 * memory.GB
actualExceeded, actualResource := accounting.ExceedsAlphaUsage(0, inlineTotal, remoteTotal, maxAlphaUsage)
require.Equal(t, tt.expectedExceeded, actualExceeded)
require.Equal(t, tt.expectedResource, actualResource)
// Setup: create some bytes for the uplink to upload
expectedData := make([]byte, 50*memory.KiB)
_, err = rand.Read(expectedData)
require.NoError(t, err)
// Execute test: check that the uplink gets an error when they have exceeded storage limits and try to upload a file
actualErr := planet.Uplinks[0].Upload(ctx, planet.Satellites[0], "testbucket", "test/path", expectedData)
if tt.expectedResource == "storage" {
assert.EqualError(t, actualErr, tt.expectedErrMsg)
} else {
require.NoError(t, actualErr)
}
})
}
})
}
func TestProjectUsageBandwidth(t *testing.T) {
cases := []struct {
name string
expectedExceeded bool
expectedResource string
expectedErrMsg string
}{
{name: "doesn't exceed storage or bandwidth project limit", expectedExceeded: false, expectedErrMsg: ""},
{name: "exceeds bandwidth project limit", expectedExceeded: true, expectedResource: "bandwidth", expectedErrMsg: "segment error: metainfo error: rpc error: code = ResourceExhausted desc = Exceeded Alpha Usage Limit"},
}
testplanet.Run(t, testplanet.Config{
@ -40,59 +97,50 @@ func TestProjectUsage(t *testing.T) {
orderDB := saDB.Orders()
acctDB := saDB.Accounting()
// Setup: This date represents the past 30 days so that we can check
// if the alpha max usage has been exceeded in the past month
from := time.Now().AddDate(0, 0, -accounting.AverageDaysInMonth)
// Setup: get projectID and create bucketID
projects, err := planet.Satellites[0].DB.Console().Projects().GetAll(ctx)
projectID := projects[0].ID
require.NoError(t, err)
bucketName := "testbucket"
bucketID := createBucketID(projectID, []byte(bucketName))
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
// Setup: create a new project to use the projectID
projects, err := planet.Satellites[0].DB.Console().Projects().GetAll(ctx)
projectID := projects[0].ID
require.NoError(t, err)
bucketName := "testbucket"
bucketID := createBucketID(projectID, []byte(bucketName))
// Setup: create BucketStorageTally records to test exceeding storage project limit
if tt.expectedResource == "storage" {
now := time.Now()
err := setUpCreateTallies(ctx, projectID, acctDB, now)
require.NoError(t, err)
}
// Setup: create a BucketBandwidthRollup record to test exceeding bandwidth project limit
if tt.expectedResource == "bandwidth" {
amount := 26 * memory.GB.Int64()
action := pb.PieceAction_GET
now := time.Now()
intervalStart := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, now.Location())
err := orderDB.UpdateBucketBandwidthSettle(ctx, bucketID, action, amount, intervalStart)
err := setUpBucketBandwidthAllocations(ctx, projectID, orderDB, now)
require.NoError(t, err)
}
// Execute test: get storage and bandwidth totals for a project, then check if that exceeds the max usage limit
inlineTotal, remoteTotal, err := acctDB.ProjectStorageTotals(ctx, projectID)
require.NoError(t, err)
bandwidthTotal, err := acctDB.ProjectBandwidthTotal(ctx, bucketID, from)
require.NoError(t, err)
maxAlphaUsage := 25 * memory.GB
actualExceeded, actualResource := accounting.ExceedsAlphaUsage(bandwidthTotal, inlineTotal, remoteTotal, maxAlphaUsage)
require.Equal(t, tt.expectedExceeded, actualExceeded)
require.Equal(t, tt.expectedResource, actualResource)
// Execute test: does the uplink get an error message when the usage limits are exceeded
// Setup: create some bytes for the uplink to upload to test the download later
expectedData := make([]byte, 50*memory.KiB)
_, err = rand.Read(expectedData)
require.NoError(t, err)
err := planet.Uplinks[0].Upload(ctx, planet.Satellites[0], bucketName, "test/path", expectedData)
require.NoError(t, err)
actualErr := planet.Uplinks[0].Upload(ctx, planet.Satellites[0], bucketName, "test/path", expectedData)
if tt.expectedResource == "bandwidth" || tt.expectedResource == "storage" {
// Setup: This date represents the past 30 days so that we can check
// if the alpha max usage has been exceeded in the past month
from := time.Now().AddDate(0, 0, -accounting.AverageDaysInMonth)
// Execute test: get bandwidth totals for a project, then check if that exceeds the max usage limit
bandwidthTotal, err := acctDB.ProjectAllocatedBandwidthTotal(ctx, bucketID, from)
require.NoError(t, err)
maxAlphaUsage := 25 * memory.GB
actualExceeded, actualResource := accounting.ExceedsAlphaUsage(bandwidthTotal, 0, 0, maxAlphaUsage)
require.Equal(t, tt.expectedExceeded, actualExceeded)
require.Equal(t, tt.expectedResource, actualResource)
// Execute test: check that the uplink gets an error when they have exceeded bandwidth limits and try to download a file
_, actualErr := planet.Uplinks[0].Download(ctx, planet.Satellites[0], bucketName, "test/path")
if tt.expectedResource == "bandwidth" {
assert.EqualError(t, actualErr, tt.expectedErrMsg)
} else {
require.NoError(t, actualErr)
}
})
}
})
@ -105,7 +153,7 @@ func createBucketID(projectID uuid.UUID, bucket []byte) []byte {
return []byte(storj.JoinPaths(entries...))
}
func setUpCreateTallies(ctx *testcontext.Context, projectID uuid.UUID, acctDB accounting.DB, time time.Time) error {
func setUpStorageTallies(ctx *testcontext.Context, projectID uuid.UUID, acctDB accounting.DB, time time.Time) error {
// Create many records that sum greater than project usage limit of 25GB
for i := 0; i < 4; i++ {
@ -114,7 +162,10 @@ func setUpCreateTallies(ctx *testcontext.Context, projectID uuid.UUID, acctDB ac
BucketName: bucketName,
ProjectID: projectID,
IntervalStart: time,
RemoteBytes: 10 * memory.GB.Int64(),
// In order to exceed the project limits, create storage tally records
// that sum greater than the maxAlphaUsage * expansionFactor
RemoteBytes: 10 * memory.GB.Int64() * accounting.ExpansionFactor,
}
err := acctDB.CreateBucketStorageTally(ctx, tally)
if err != nil {
@ -123,3 +174,23 @@ func setUpCreateTallies(ctx *testcontext.Context, projectID uuid.UUID, acctDB ac
}
return nil
}
func setUpBucketBandwidthAllocations(ctx *testcontext.Context, projectID uuid.UUID, orderDB orders.DB, now time.Time) error {
// Create many records that sum greater than project usage limit of 25GB
for i := 0; i < 4; i++ {
bucketName := fmt.Sprintf("%s%d", "testbucket", i)
bucketID := createBucketID(projectID, []byte(bucketName))
// In order to exceed the project limits, create bandwidth allocation records
// that sum greater than the maxAlphaUsage * expansionFactor
amount := 10 * memory.GB.Int64() * accounting.ExpansionFactor
action := pb.PieceAction_GET
intervalStart := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, now.Location())
err := orderDB.UpdateBucketBandwidthAllocation(ctx, bucketID, action, amount, intervalStart)
if err != nil {
return err
}
}
return nil
}

7
satellite/metainfo/metainfo.go
View File

@ -249,19 +249,18 @@ func (endpoint *Endpoint) DownloadSegment(ctx context.Context, req *pb.SegmentDo
return nil, status.Errorf(codes.InvalidArgument, err.Error())
}
// Check if this projectID has exceeded alpha usage limits, i.e. 25GB of bandwidth or storage used in the past month
// Check if this projectID has exceeded alpha usage limits for bandwidth or storage used in the past month
// TODO: remove this code once we no longer need usage limiting for alpha release
// Ref: https://storjlabs.atlassian.net/browse/V3-1274
bucketID := createBucketID(keyInfo.ProjectID, req.Bucket)
from := time.Now().AddDate(0, 0, -accounting.AverageDaysInMonth) // past 30 days
bandwidthTotal, err := endpoint.accountingDB.ProjectBandwidthTotal(ctx, bucketID, from)
bandwidthTotal, err := endpoint.accountingDB.ProjectAllocatedBandwidthTotal(ctx, bucketID, from)
if err != nil {
endpoint.log.Error("retrieving ProjectBandwidthTotal", zap.Error(err))
}
exceeded, resource := accounting.ExceedsAlphaUsage(bandwidthTotal, 0, 0, endpoint.maxAlphaUsage)
if exceeded {
endpoint.log.Sugar().Errorf("monthly project limits are %s of storage and bandwidth usage. This limit has been exceeded for %s for projectID %s.",
endpoint.maxAlphaUsage.String(),
endpoint.log.Sugar().Errorf("monthly project usage limit has been exceeded for resource: %s, for project: %d. Contact customer support to increase the limit.",
resource, keyInfo.ProjectID,
)
return nil, status.Errorf(codes.ResourceExhausted, "Exceeded Alpha Usage Limit")

6
satellite/satellitedb/accounting.go
View File

@ -23,12 +23,12 @@ type accountingDB struct {
db *dbx.DB
}
// ProjectBandwidthTotal returns the sum of GET bandwidth usage for a projectID for a time frame
func (db *accountingDB) ProjectBandwidthTotal(ctx context.Context, bucketID []byte, from time.Time) (int64, error) {
// ProjectAllocatedBandwidthTotal returns the sum of GET bandwidth usage allocated for a projectID for a time frame
func (db *accountingDB) ProjectAllocatedBandwidthTotal(ctx context.Context, bucketID []byte, from time.Time) (int64, error) {
pathEl := bytes.Split(bucketID, []byte("/"))
_, projectID := pathEl[1], pathEl[0]
var sum *int64
query := `SELECT SUM(settled) FROM bucket_bandwidth_rollups WHERE project_id = ? AND action = ? AND interval_start > ?;`
query := `SELECT SUM(allocated) FROM bucket_bandwidth_rollups WHERE project_id = ? AND action = ? AND interval_start > ?;`
err := db.db.QueryRow(db.db.Rebind(query), projectID, pb.PieceAction_GET, from).Scan(&sum)
if err == sql.ErrNoRows || sum == nil {
return 0, nil

6
satellite/satellitedb/locked.go
View File

@ -92,11 +92,11 @@ func (m *lockedAccounting) LastTimestamp(ctx context.Context, timestampType stri
return m.db.LastTimestamp(ctx, timestampType)
}
// ProjectBandwidthTotal returns the sum of GET bandwidth usage for a projectID in the past time frame
func (m *lockedAccounting) ProjectBandwidthTotal(ctx context.Context, bucketID []byte, from time.Time) (int64, error) {
// ProjectAllocatedBandwidthTotal returns the sum of GET bandwidth usage allocated for a projectID in the past time frame
func (m *lockedAccounting) ProjectAllocatedBandwidthTotal(ctx context.Context, bucketID []byte, from time.Time) (int64, error) {
m.Lock()
defer m.Unlock()
return m.db.ProjectBandwidthTotal(ctx, bucketID, from)
return m.db.ProjectAllocatedBandwidthTotal(ctx, bucketID, from)
}
// ProjectStorageTotals returns the current inline and remote storage usage for a projectID