207 lines
7.9 KiB
Go
207 lines
7.9 KiB
Go
// Copyright (C) 2019 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package accounting
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"storj.io/common/memory"
|
|
"storj.io/common/storj"
|
|
"storj.io/common/uuid"
|
|
"storj.io/storj/satellite/compensation"
|
|
"storj.io/storj/satellite/metainfo/metabase"
|
|
)
|
|
|
|
// RollupStats is a convenience alias.
|
|
type RollupStats map[time.Time]map[storj.NodeID]*Rollup
|
|
|
|
// StoragenodeStorageTally mirrors dbx.StoragenodeStorageTally, allowing us to use that struct without leaking dbx.
|
|
type StoragenodeStorageTally struct {
|
|
ID int64
|
|
NodeID storj.NodeID
|
|
IntervalEndTime time.Time
|
|
DataTotal float64
|
|
}
|
|
|
|
// StoragenodeBandwidthRollup mirrors dbx.StoragenodeBandwidthRollup, allowing us to use the struct without leaking dbx.
|
|
type StoragenodeBandwidthRollup struct {
|
|
NodeID storj.NodeID
|
|
IntervalStart time.Time
|
|
Action uint
|
|
Settled uint64
|
|
}
|
|
|
|
// Rollup mirrors dbx.AccountingRollup, allowing us to use that struct without leaking dbx.
|
|
type Rollup struct {
|
|
ID int64
|
|
NodeID storj.NodeID
|
|
StartTime time.Time
|
|
PutTotal int64
|
|
GetTotal int64
|
|
GetAuditTotal int64
|
|
GetRepairTotal int64
|
|
PutRepairTotal int64
|
|
AtRestTotal float64
|
|
}
|
|
|
|
// StorageNodePeriodUsage represents a statement for a node for a compensation period.
|
|
type StorageNodePeriodUsage struct {
|
|
NodeID storj.NodeID
|
|
AtRestTotal float64
|
|
GetTotal int64
|
|
PutTotal int64
|
|
GetRepairTotal int64
|
|
PutRepairTotal int64
|
|
GetAuditTotal int64
|
|
}
|
|
|
|
// StorageNodeUsage is node at rest space usage over a period of time.
|
|
type StorageNodeUsage struct {
|
|
NodeID storj.NodeID
|
|
StorageUsed float64
|
|
|
|
Timestamp time.Time
|
|
}
|
|
|
|
// ProjectUsage consist of period total storage, egress
|
|
// and objects count per hour for certain Project in bytes.
|
|
type ProjectUsage struct {
|
|
Storage float64 `json:"storage"`
|
|
Egress int64 `json:"egress"`
|
|
ObjectCount float64 `json:"objectCount"`
|
|
|
|
Since time.Time `json:"since"`
|
|
Before time.Time `json:"before"`
|
|
}
|
|
|
|
// BucketUsage consist of total bucket usage for period.
|
|
type BucketUsage struct {
|
|
ProjectID uuid.UUID
|
|
BucketName string
|
|
|
|
Storage float64
|
|
Egress float64
|
|
ObjectCount int64
|
|
|
|
Since time.Time
|
|
Before time.Time
|
|
}
|
|
|
|
// BucketUsageCursor holds info for bucket usage
|
|
// cursor pagination.
|
|
type BucketUsageCursor struct {
|
|
Search string
|
|
Limit uint
|
|
Page uint
|
|
}
|
|
|
|
// BucketUsagePage represents bucket usage page result.
|
|
type BucketUsagePage struct {
|
|
BucketUsages []BucketUsage
|
|
|
|
Search string
|
|
Limit uint
|
|
Offset uint64
|
|
|
|
PageCount uint
|
|
CurrentPage uint
|
|
TotalCount uint64
|
|
}
|
|
|
|
// BucketUsageRollup is total bucket usage info
|
|
// for certain period.
|
|
type BucketUsageRollup struct {
|
|
ProjectID uuid.UUID
|
|
BucketName []byte
|
|
|
|
RemoteStoredData float64
|
|
InlineStoredData float64
|
|
|
|
RemoteSegments float64
|
|
InlineSegments float64
|
|
ObjectCount float64
|
|
MetadataSize float64
|
|
|
|
RepairEgress float64
|
|
GetEgress float64
|
|
AuditEgress float64
|
|
|
|
Since time.Time
|
|
Before time.Time
|
|
}
|
|
|
|
// StoragenodeAccounting stores information about bandwidth and storage usage for storage nodes
|
|
//
|
|
// architecture: Database
|
|
type StoragenodeAccounting interface {
|
|
// SaveTallies records tallies of data at rest
|
|
SaveTallies(ctx context.Context, latestTally time.Time, nodeData map[storj.NodeID]float64) error
|
|
// GetTallies retrieves all tallies
|
|
GetTallies(ctx context.Context) ([]*StoragenodeStorageTally, error)
|
|
// GetTalliesSince retrieves all tallies since latestRollup
|
|
GetTalliesSince(ctx context.Context, latestRollup time.Time) ([]*StoragenodeStorageTally, error)
|
|
// GetBandwidthSince retrieves all bandwidth rollup entires since latestRollup
|
|
GetBandwidthSince(ctx context.Context, latestRollup time.Time) ([]*StoragenodeBandwidthRollup, error)
|
|
// SaveRollup records tally and bandwidth rollup aggregations to the database
|
|
SaveRollup(ctx context.Context, latestTally time.Time, stats RollupStats) error
|
|
// LastTimestamp records and returns the latest last tallied time.
|
|
LastTimestamp(ctx context.Context, timestampType string) (time.Time, error)
|
|
// QueryPaymentInfo queries Nodes and Accounting_Rollup on nodeID
|
|
QueryPaymentInfo(ctx context.Context, start time.Time, end time.Time) ([]*CSVRow, error)
|
|
// QueryStorageNodePeriodUsage returns accounting statements for nodes for a given compensation period
|
|
QueryStorageNodePeriodUsage(ctx context.Context, period compensation.Period) ([]StorageNodePeriodUsage, error)
|
|
// QueryStorageNodeUsage returns slice of StorageNodeUsage for given period
|
|
QueryStorageNodeUsage(ctx context.Context, nodeID storj.NodeID, start time.Time, end time.Time) ([]StorageNodeUsage, error)
|
|
// DeleteTalliesBefore deletes all tallies prior to some time
|
|
DeleteTalliesBefore(ctx context.Context, latestRollup time.Time) error
|
|
}
|
|
|
|
// ProjectAccounting stores information about bandwidth and storage usage for projects
|
|
//
|
|
// architecture: Database
|
|
type ProjectAccounting interface {
|
|
// SaveTallies saves the latest project info
|
|
SaveTallies(ctx context.Context, intervalStart time.Time, bucketTallies map[metabase.BucketLocation]*BucketTally) error
|
|
// GetTallies retrieves all tallies
|
|
GetTallies(ctx context.Context) ([]BucketTally, error)
|
|
// CreateStorageTally creates a record for BucketStorageTally in the accounting DB table
|
|
CreateStorageTally(ctx context.Context, tally BucketStorageTally) error
|
|
// GetAllocatedBandwidthTotal returns the sum of GET bandwidth usage allocated for a projectID in the past time frame
|
|
GetAllocatedBandwidthTotal(ctx context.Context, projectID uuid.UUID, from time.Time) (int64, error)
|
|
// GetProjectAllocatedBandwidth returns project allocated bandwidth for the specified year and month.
|
|
GetProjectAllocatedBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month) (int64, error)
|
|
// DeleteProjectAllocatedBandwidthBefore deletes project bandwidth rollups before the given time
|
|
DeleteProjectAllocatedBandwidthBefore(ctx context.Context, before time.Time) error
|
|
|
|
// GetStorageTotals returns the current inline and remote storage usage for a projectID
|
|
GetStorageTotals(ctx context.Context, projectID uuid.UUID) (int64, int64, error)
|
|
// UpdateProjectUsageLimit updates project usage limit.
|
|
UpdateProjectUsageLimit(ctx context.Context, projectID uuid.UUID, limit memory.Size) error
|
|
// UpdateProjectBandwidthLimit updates project bandwidth limit.
|
|
UpdateProjectBandwidthLimit(ctx context.Context, projectID uuid.UUID, limit memory.Size) error
|
|
// GetProjectStorageLimit returns project storage usage limit.
|
|
GetProjectStorageLimit(ctx context.Context, projectID uuid.UUID) (memory.Size, error)
|
|
// GetProjectBandwidthLimit returns project bandwidth usage limit.
|
|
GetProjectBandwidthLimit(ctx context.Context, projectID uuid.UUID) (memory.Size, error)
|
|
// GetProjectTotal returns project usage summary for specified period of time.
|
|
GetProjectTotal(ctx context.Context, projectID uuid.UUID, since, before time.Time) (*ProjectUsage, error)
|
|
// GetBucketUsageRollups returns usage rollup per each bucket for specified period of time.
|
|
GetBucketUsageRollups(ctx context.Context, projectID uuid.UUID, since, before time.Time) ([]BucketUsageRollup, error)
|
|
// GetBucketTotals returns per bucket usage summary for specified period of time.
|
|
GetBucketTotals(ctx context.Context, projectID uuid.UUID, cursor BucketUsageCursor, since, before time.Time) (*BucketUsagePage, error)
|
|
}
|
|
|
|
// Cache stores live information about project storage which has not yet been synced to ProjectAccounting.
|
|
//
|
|
// architecture: Database
|
|
type Cache interface {
|
|
GetProjectStorageUsage(ctx context.Context, projectID uuid.UUID) (totalUsed int64, err error)
|
|
GetProjectBandwidthUsage(ctx context.Context, projectID uuid.UUID, now time.Time) (currentUsed int64, err error)
|
|
UpdateProjectBandwidthUsage(ctx context.Context, projectID uuid.UUID, increment int64, ttl time.Duration, now time.Time) error
|
|
AddProjectStorageUsage(ctx context.Context, projectID uuid.UUID, spaceUsed int64) error
|
|
GetAllProjectTotals(ctx context.Context) (map[uuid.UUID]int64, error)
|
|
Close() error
|
|
}
|