satellite/accounting/tally: use objects iterator instead metaloop
Bucket tally calculation will be removed from metaloop and will use metabase objects iterator directly. At the moment only bucket tally needs objects so it make no sense to implement separate objects loop. Change-Id: Iee60059fc8b9a1bf64d01cafe9659b69b0e27eb1
This commit is contained in:
parent
4c7384ca2f
commit
27a714e8b0
@ -24,8 +24,7 @@ import (
|
|||||||
var headers = []string{
|
var headers = []string{
|
||||||
"projectID",
|
"projectID",
|
||||||
"bucketName",
|
"bucketName",
|
||||||
"byte-hours:Remote",
|
"byte-hours:Total",
|
||||||
"byte-hours:Inline",
|
|
||||||
"bytes:BWEgress",
|
"bytes:BWEgress",
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -81,14 +80,12 @@ func csvRowToStringSlice(p *attribution.CSVRow) ([]string, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errs.New("Invalid Project ID")
|
return nil, errs.New("Invalid Project ID")
|
||||||
}
|
}
|
||||||
remoteGBPerHour := memory.Size(p.RemoteBytesPerHour).GB()
|
totalGBPerHour := memory.Size(p.TotalBytesPerHour).GB()
|
||||||
inlineGBPerHour := memory.Size(p.InlineBytesPerHour).GB()
|
|
||||||
egressGBData := memory.Size(p.EgressData).GB()
|
egressGBData := memory.Size(p.EgressData).GB()
|
||||||
record := []string{
|
record := []string{
|
||||||
projectID.String(),
|
projectID.String(),
|
||||||
string(p.BucketName),
|
string(p.BucketName),
|
||||||
strconv.FormatFloat(remoteGBPerHour, 'f', 4, 64),
|
strconv.FormatFloat(totalGBPerHour, 'f', 4, 64),
|
||||||
strconv.FormatFloat(inlineGBPerHour, 'f', 4, 64),
|
|
||||||
strconv.FormatFloat(egressGBData, 'f', 4, 64),
|
strconv.FormatFloat(egressGBData, 'f', 4, 64),
|
||||||
}
|
}
|
||||||
return record, nil
|
return record, nil
|
||||||
|
@ -1,18 +1,10 @@
|
|||||||
storj.io/storj/private/lifecycle."slow_shutdown" Event
|
storj.io/storj/private/lifecycle."slow_shutdown" Event
|
||||||
storj.io/storj/private/lifecycle."unexpected_shutdown" Event
|
storj.io/storj/private/lifecycle."unexpected_shutdown" Event
|
||||||
storj.io/storj/satellite/accounting."bucket_bytes" IntVal
|
storj.io/storj/satellite/accounting."bucket_bytes" IntVal
|
||||||
storj.io/storj/satellite/accounting."bucket_inline_bytes" IntVal
|
|
||||||
storj.io/storj/satellite/accounting."bucket_inline_segments" IntVal
|
|
||||||
storj.io/storj/satellite/accounting."bucket_objects" IntVal
|
storj.io/storj/satellite/accounting."bucket_objects" IntVal
|
||||||
storj.io/storj/satellite/accounting."bucket_remote_bytes" IntVal
|
|
||||||
storj.io/storj/satellite/accounting."bucket_remote_segments" IntVal
|
|
||||||
storj.io/storj/satellite/accounting."bucket_segments" IntVal
|
storj.io/storj/satellite/accounting."bucket_segments" IntVal
|
||||||
storj.io/storj/satellite/accounting."total_bytes" IntVal
|
storj.io/storj/satellite/accounting."total_bytes" IntVal
|
||||||
storj.io/storj/satellite/accounting."total_inline_bytes" IntVal
|
|
||||||
storj.io/storj/satellite/accounting."total_inline_segments" IntVal
|
|
||||||
storj.io/storj/satellite/accounting."total_objects" IntVal
|
storj.io/storj/satellite/accounting."total_objects" IntVal
|
||||||
storj.io/storj/satellite/accounting."total_remote_bytes" IntVal
|
|
||||||
storj.io/storj/satellite/accounting."total_remote_segments" IntVal
|
|
||||||
storj.io/storj/satellite/accounting."total_segments" IntVal
|
storj.io/storj/satellite/accounting."total_segments" IntVal
|
||||||
storj.io/storj/satellite/accounting/tally."nodetallies.totalsum" IntVal
|
storj.io/storj/satellite/accounting/tally."nodetallies.totalsum" IntVal
|
||||||
storj.io/storj/satellite/audit."audit_contained_nodes" IntVal
|
storj.io/storj/satellite/audit."audit_contained_nodes" IntVal
|
||||||
|
@ -14,12 +14,7 @@ type BucketTally struct {
|
|||||||
ObjectCount int64
|
ObjectCount int64
|
||||||
|
|
||||||
TotalSegments int64
|
TotalSegments int64
|
||||||
InlineSegments int64
|
|
||||||
RemoteSegments int64
|
|
||||||
|
|
||||||
TotalBytes int64
|
TotalBytes int64
|
||||||
InlineBytes int64
|
|
||||||
RemoteBytes int64
|
|
||||||
|
|
||||||
MetadataSize int64
|
MetadataSize int64
|
||||||
}
|
}
|
||||||
@ -29,26 +24,16 @@ func (s *BucketTally) Combine(o *BucketTally) {
|
|||||||
s.ObjectCount += o.ObjectCount
|
s.ObjectCount += o.ObjectCount
|
||||||
|
|
||||||
s.TotalSegments += o.TotalSegments
|
s.TotalSegments += o.TotalSegments
|
||||||
s.InlineSegments += o.InlineSegments
|
|
||||||
s.RemoteSegments += o.RemoteSegments
|
|
||||||
|
|
||||||
s.TotalBytes += o.TotalBytes
|
s.TotalBytes += o.TotalBytes
|
||||||
s.InlineBytes += o.InlineBytes
|
|
||||||
s.RemoteBytes += o.RemoteBytes
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Segments returns total number of segments.
|
// Segments returns total number of segments.
|
||||||
func (s *BucketTally) Segments() int64 {
|
func (s *BucketTally) Segments() int64 {
|
||||||
if s.TotalSegments != 0 {
|
|
||||||
return s.TotalSegments
|
return s.TotalSegments
|
||||||
}
|
}
|
||||||
return s.InlineSegments + s.RemoteSegments
|
|
||||||
}
|
|
||||||
|
|
||||||
// Bytes returns total bytes.
|
// Bytes returns total bytes.
|
||||||
func (s *BucketTally) Bytes() int64 {
|
func (s *BucketTally) Bytes() int64 {
|
||||||
if s.TotalBytes != 0 {
|
|
||||||
return s.TotalBytes
|
return s.TotalBytes
|
||||||
}
|
}
|
||||||
return s.InlineBytes + s.RemoteBytes
|
|
||||||
}
|
|
||||||
|
@ -18,19 +18,12 @@ type BucketStorageTally struct {
|
|||||||
ObjectCount int64
|
ObjectCount int64
|
||||||
|
|
||||||
TotalSegmentCount int64
|
TotalSegmentCount int64
|
||||||
InlineSegmentCount int64
|
|
||||||
RemoteSegmentCount int64
|
|
||||||
|
|
||||||
TotalBytes int64
|
TotalBytes int64
|
||||||
InlineBytes int64
|
|
||||||
RemoteBytes int64
|
|
||||||
MetadataSize int64
|
MetadataSize int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// Bytes returns total bytes.
|
// Bytes returns total bytes.
|
||||||
func (s *BucketStorageTally) Bytes() int64 {
|
func (s *BucketStorageTally) Bytes() int64 {
|
||||||
if s.TotalBytes != 0 {
|
|
||||||
return s.TotalBytes
|
return s.TotalBytes
|
||||||
}
|
}
|
||||||
return s.InlineBytes + s.RemoteBytes
|
|
||||||
}
|
|
||||||
|
@ -232,10 +232,8 @@ func createBucketStorageTallies(projectID uuid.UUID) (map[metabase.BucketLocatio
|
|||||||
BucketName: bucketName,
|
BucketName: bucketName,
|
||||||
},
|
},
|
||||||
ObjectCount: int64(1),
|
ObjectCount: int64(1),
|
||||||
InlineSegments: int64(1),
|
TotalSegments: int64(2),
|
||||||
RemoteSegments: int64(1),
|
TotalBytes: int64(2),
|
||||||
InlineBytes: int64(1),
|
|
||||||
RemoteBytes: int64(1),
|
|
||||||
MetadataSize: int64(1),
|
MetadataSize: int64(1),
|
||||||
}
|
}
|
||||||
bucketTallies[bucketLocation] = &tally
|
bucketTallies[bucketLocation] = &tally
|
||||||
|
@ -467,20 +467,16 @@ func TestUsageRollups(t *testing.T) {
|
|||||||
tally1 := &accounting.BucketTally{
|
tally1 := &accounting.BucketTally{
|
||||||
BucketLocation: bucketLoc1,
|
BucketLocation: bucketLoc1,
|
||||||
ObjectCount: value1,
|
ObjectCount: value1,
|
||||||
InlineSegments: value1,
|
TotalSegments: value1 + value1,
|
||||||
RemoteSegments: value1,
|
TotalBytes: value1 + value1,
|
||||||
InlineBytes: value1,
|
|
||||||
RemoteBytes: value1,
|
|
||||||
MetadataSize: value1,
|
MetadataSize: value1,
|
||||||
}
|
}
|
||||||
|
|
||||||
tally2 := &accounting.BucketTally{
|
tally2 := &accounting.BucketTally{
|
||||||
BucketLocation: bucketLoc2,
|
BucketLocation: bucketLoc2,
|
||||||
ObjectCount: value2,
|
ObjectCount: value2,
|
||||||
InlineSegments: value2,
|
TotalSegments: value2 + value2,
|
||||||
RemoteSegments: value2,
|
TotalBytes: value2 + value2,
|
||||||
InlineBytes: value2,
|
|
||||||
RemoteBytes: value2,
|
|
||||||
MetadataSize: value2,
|
MetadataSize: value2,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -15,7 +15,6 @@ import (
|
|||||||
"storj.io/common/uuid"
|
"storj.io/common/uuid"
|
||||||
"storj.io/storj/satellite/accounting"
|
"storj.io/storj/satellite/accounting"
|
||||||
"storj.io/storj/satellite/metabase"
|
"storj.io/storj/satellite/metabase"
|
||||||
"storj.io/storj/satellite/metabase/metaloop"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Error is a standard error class for this package.
|
// Error is a standard error class for this package.
|
||||||
@ -29,6 +28,9 @@ type Config struct {
|
|||||||
Interval time.Duration `help:"how frequently the tally service should run" releaseDefault:"1h" devDefault:"30s" testDefault:"$TESTINTERVAL"`
|
Interval time.Duration `help:"how frequently the tally service should run" releaseDefault:"1h" devDefault:"30s" testDefault:"$TESTINTERVAL"`
|
||||||
SaveRollupBatchSize int `help:"how large of batches SaveRollup should process at a time" default:"1000"`
|
SaveRollupBatchSize int `help:"how large of batches SaveRollup should process at a time" default:"1000"`
|
||||||
ReadRollupBatchSize int `help:"how large of batches GetBandwidthSince should process at a time" default:"10000"`
|
ReadRollupBatchSize int `help:"how large of batches GetBandwidthSince should process at a time" default:"10000"`
|
||||||
|
|
||||||
|
ListLimit int `help:"how many objects to query in a batch" default:"2500"`
|
||||||
|
AsOfSystemInterval time.Duration `help:"as of system interval" releaseDefault:"-5m" devDefault:"-1us" testDefault:"-1us"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Service is the tally service for data stored on each storage node.
|
// Service is the tally service for data stored on each storage node.
|
||||||
@ -36,9 +38,10 @@ type Config struct {
|
|||||||
// architecture: Chore
|
// architecture: Chore
|
||||||
type Service struct {
|
type Service struct {
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
|
config Config
|
||||||
Loop *sync2.Cycle
|
Loop *sync2.Cycle
|
||||||
|
|
||||||
metainfoLoop *metaloop.Service
|
metabase *metabase.DB
|
||||||
liveAccounting accounting.Cache
|
liveAccounting accounting.Cache
|
||||||
storagenodeAccountingDB accounting.StoragenodeAccounting
|
storagenodeAccountingDB accounting.StoragenodeAccounting
|
||||||
projectAccountingDB accounting.ProjectAccounting
|
projectAccountingDB accounting.ProjectAccounting
|
||||||
@ -46,12 +49,13 @@ type Service struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new tally Service.
|
// New creates a new tally Service.
|
||||||
func New(log *zap.Logger, sdb accounting.StoragenodeAccounting, pdb accounting.ProjectAccounting, liveAccounting accounting.Cache, metainfoLoop *metaloop.Service, interval time.Duration) *Service {
|
func New(log *zap.Logger, sdb accounting.StoragenodeAccounting, pdb accounting.ProjectAccounting, liveAccounting accounting.Cache, metabase *metabase.DB, config Config) *Service {
|
||||||
return &Service{
|
return &Service{
|
||||||
log: log,
|
log: log,
|
||||||
Loop: sync2.NewCycle(interval),
|
config: config,
|
||||||
|
Loop: sync2.NewCycle(config.Interval),
|
||||||
|
|
||||||
metainfoLoop: metainfoLoop,
|
metabase: metabase,
|
||||||
liveAccounting: liveAccounting,
|
liveAccounting: liveAccounting,
|
||||||
storagenodeAccountingDB: sdb,
|
storagenodeAccountingDB: sdb,
|
||||||
projectAccountingDB: pdb,
|
projectAccountingDB: pdb,
|
||||||
@ -164,9 +168,9 @@ func (service *Service) Tally(ctx context.Context) (err error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// add up all nodes and buckets
|
// add up all buckets
|
||||||
observer := NewObserver(service.log.Named("observer"), service.nowFn())
|
collector := NewBucketTallyCollector(service.log.Named("observer"), service.nowFn(), service.metabase, service.config)
|
||||||
err = service.metainfoLoop.Join(ctx, observer)
|
err = collector.Run(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Error.Wrap(err)
|
return Error.Wrap(err)
|
||||||
}
|
}
|
||||||
@ -174,66 +178,97 @@ func (service *Service) Tally(ctx context.Context) (err error) {
|
|||||||
|
|
||||||
// save the new results
|
// save the new results
|
||||||
var errAtRest error
|
var errAtRest error
|
||||||
if len(observer.Bucket) > 0 {
|
if len(collector.Bucket) > 0 {
|
||||||
// record bucket tallies to DB
|
// record bucket tallies to DB
|
||||||
err = service.projectAccountingDB.SaveTallies(ctx, finishTime, observer.Bucket)
|
err = service.projectAccountingDB.SaveTallies(ctx, finishTime, collector.Bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errAtRest = Error.New("ProjectAccounting.SaveTallies failed: %v", err)
|
errAtRest = Error.New("ProjectAccounting.SaveTallies failed: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
updateLiveAccountingTotals(projectTotalsFromBuckets(observer.Bucket))
|
updateLiveAccountingTotals(projectTotalsFromBuckets(collector.Bucket))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO move commented metrics in a different place or wait until metabase
|
||||||
|
// object will contains such informations
|
||||||
// report bucket metrics
|
// report bucket metrics
|
||||||
if len(observer.Bucket) > 0 {
|
if len(collector.Bucket) > 0 {
|
||||||
var total accounting.BucketTally
|
var total accounting.BucketTally
|
||||||
for _, bucket := range observer.Bucket {
|
for _, bucket := range collector.Bucket {
|
||||||
monAccounting.IntVal("bucket_objects").Observe(bucket.ObjectCount) //mon:locked
|
monAccounting.IntVal("bucket_objects").Observe(bucket.ObjectCount) //mon:locked
|
||||||
monAccounting.IntVal("bucket_segments").Observe(bucket.Segments()) //mon:locked
|
monAccounting.IntVal("bucket_segments").Observe(bucket.Segments()) //mon:locked
|
||||||
monAccounting.IntVal("bucket_inline_segments").Observe(bucket.InlineSegments) //mon:locked
|
// monAccounting.IntVal("bucket_inline_segments").Observe(bucket.InlineSegments) //mon:locked
|
||||||
monAccounting.IntVal("bucket_remote_segments").Observe(bucket.RemoteSegments) //mon:locked
|
// monAccounting.IntVal("bucket_remote_segments").Observe(bucket.RemoteSegments) //mon:locked
|
||||||
|
|
||||||
monAccounting.IntVal("bucket_bytes").Observe(bucket.Bytes()) //mon:locked
|
monAccounting.IntVal("bucket_bytes").Observe(bucket.Bytes()) //mon:locked
|
||||||
monAccounting.IntVal("bucket_inline_bytes").Observe(bucket.InlineBytes) //mon:locked
|
// monAccounting.IntVal("bucket_inline_bytes").Observe(bucket.InlineBytes) //mon:locked
|
||||||
monAccounting.IntVal("bucket_remote_bytes").Observe(bucket.RemoteBytes) //mon:locked
|
// monAccounting.IntVal("bucket_remote_bytes").Observe(bucket.RemoteBytes) //mon:locked
|
||||||
total.Combine(bucket)
|
total.Combine(bucket)
|
||||||
}
|
}
|
||||||
monAccounting.IntVal("total_objects").Observe(total.ObjectCount) //mon:locked
|
monAccounting.IntVal("total_objects").Observe(total.ObjectCount) //mon:locked
|
||||||
|
|
||||||
monAccounting.IntVal("total_segments").Observe(total.Segments()) //mon:locked
|
monAccounting.IntVal("total_segments").Observe(total.Segments()) //mon:locked
|
||||||
monAccounting.IntVal("total_inline_segments").Observe(total.InlineSegments) //mon:locked
|
// monAccounting.IntVal("total_inline_segments").Observe(total.InlineSegments) //mon:locked
|
||||||
monAccounting.IntVal("total_remote_segments").Observe(total.RemoteSegments) //mon:locked
|
// monAccounting.IntVal("total_remote_segments").Observe(total.RemoteSegments) //mon:locked
|
||||||
|
|
||||||
monAccounting.IntVal("total_bytes").Observe(total.Bytes()) //mon:locked
|
monAccounting.IntVal("total_bytes").Observe(total.Bytes()) //mon:locked
|
||||||
monAccounting.IntVal("total_inline_bytes").Observe(total.InlineBytes) //mon:locked
|
// monAccounting.IntVal("total_inline_bytes").Observe(total.InlineBytes) //mon:locked
|
||||||
monAccounting.IntVal("total_remote_bytes").Observe(total.RemoteBytes) //mon:locked
|
// monAccounting.IntVal("total_remote_bytes").Observe(total.RemoteBytes) //mon:locked
|
||||||
}
|
}
|
||||||
|
|
||||||
// return errors if something went wrong.
|
// return errors if something went wrong.
|
||||||
return errAtRest
|
return errAtRest
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ metaloop.Observer = (*Observer)(nil)
|
// BucketTallyCollector collects and adds up tallies for buckets.
|
||||||
|
type BucketTallyCollector struct {
|
||||||
// Observer observes metainfo and adds up tallies for nodes and buckets.
|
|
||||||
type Observer struct {
|
|
||||||
Now time.Time
|
Now time.Time
|
||||||
Log *zap.Logger
|
Log *zap.Logger
|
||||||
Bucket map[metabase.BucketLocation]*accounting.BucketTally
|
Bucket map[metabase.BucketLocation]*accounting.BucketTally
|
||||||
|
|
||||||
|
metabase *metabase.DB
|
||||||
|
config Config
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewObserver returns an metainfo loop observer that adds up totals for buckets and nodes.
|
// NewBucketTallyCollector returns an collector that adds up totals for buckets.
|
||||||
// The now argument controls when the observer considers pointers to be expired.
|
// The now argument controls when the collector considers objects to be expired.
|
||||||
func NewObserver(log *zap.Logger, now time.Time) *Observer {
|
func NewBucketTallyCollector(log *zap.Logger, now time.Time, db *metabase.DB, config Config) *BucketTallyCollector {
|
||||||
return &Observer{
|
return &BucketTallyCollector{
|
||||||
Now: now,
|
Now: now,
|
||||||
Log: log,
|
Log: log,
|
||||||
Bucket: make(map[metabase.BucketLocation]*accounting.BucketTally),
|
Bucket: make(map[metabase.BucketLocation]*accounting.BucketTally),
|
||||||
|
|
||||||
|
metabase: db,
|
||||||
|
config: config,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Run runs collecting bucket tallies.
|
||||||
|
func (observer *BucketTallyCollector) Run(ctx context.Context) (err error) {
|
||||||
|
defer mon.Task()(&ctx)(&err)
|
||||||
|
|
||||||
|
startingTime, err := observer.metabase.Now(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return observer.metabase.IterateLoopObjects(ctx, metabase.IterateLoopObjects{
|
||||||
|
BatchSize: observer.config.ListLimit,
|
||||||
|
AsOfSystemTime: startingTime,
|
||||||
|
AsOfSystemInterval: observer.config.AsOfSystemInterval,
|
||||||
|
}, func(ctx context.Context, it metabase.LoopObjectsIterator) (err error) {
|
||||||
|
var entry metabase.LoopObjectEntry
|
||||||
|
for it.Next(ctx, &entry) {
|
||||||
|
err = observer.object(ctx, entry)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// ensureBucket returns bucket corresponding to the passed in path.
|
// ensureBucket returns bucket corresponding to the passed in path.
|
||||||
func (observer *Observer) ensureBucket(ctx context.Context, location metabase.ObjectLocation) *accounting.BucketTally {
|
func (observer *BucketTallyCollector) ensureBucket(ctx context.Context, location metabase.ObjectLocation) *accounting.BucketTally {
|
||||||
bucketLocation := location.Bucket()
|
bucketLocation := location.Bucket()
|
||||||
bucket, exists := observer.Bucket[bucketLocation]
|
bucket, exists := observer.Bucket[bucketLocation]
|
||||||
if !exists {
|
if !exists {
|
||||||
@ -245,13 +280,8 @@ func (observer *Observer) ensureBucket(ctx context.Context, location metabase.Ob
|
|||||||
return bucket
|
return bucket
|
||||||
}
|
}
|
||||||
|
|
||||||
// LoopStarted is called at each start of a loop.
|
|
||||||
func (observer *Observer) LoopStarted(context.Context, metaloop.LoopInfo) (err error) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Object is called for each object once.
|
// Object is called for each object once.
|
||||||
func (observer *Observer) Object(ctx context.Context, object *metaloop.Object) (err error) {
|
func (observer *BucketTallyCollector) object(ctx context.Context, object metabase.LoopObjectEntry) (err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
|
|
||||||
if object.Expired(observer.Now) {
|
if object.Expired(observer.Now) {
|
||||||
@ -259,46 +289,18 @@ func (observer *Observer) Object(ctx context.Context, object *metaloop.Object) (
|
|||||||
}
|
}
|
||||||
|
|
||||||
bucket := observer.ensureBucket(ctx, object.ObjectStream.Location())
|
bucket := observer.ensureBucket(ctx, object.ObjectStream.Location())
|
||||||
|
bucket.TotalSegments += int64(object.SegmentCount)
|
||||||
|
bucket.TotalBytes += object.TotalEncryptedSize
|
||||||
bucket.MetadataSize += int64(object.EncryptedMetadataSize)
|
bucket.MetadataSize += int64(object.EncryptedMetadataSize)
|
||||||
bucket.ObjectCount++
|
bucket.ObjectCount++
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// InlineSegment is called for each inline segment.
|
|
||||||
func (observer *Observer) InlineSegment(ctx context.Context, segment *metaloop.Segment) (err error) {
|
|
||||||
defer mon.Task()(&ctx)(&err)
|
|
||||||
|
|
||||||
if segment.Expired(observer.Now) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
bucket := observer.ensureBucket(ctx, segment.Location.Object())
|
|
||||||
bucket.InlineSegments++
|
|
||||||
bucket.InlineBytes += int64(segment.EncryptedSize)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// RemoteSegment is called for each remote segment.
|
|
||||||
func (observer *Observer) RemoteSegment(ctx context.Context, segment *metaloop.Segment) (err error) {
|
|
||||||
defer mon.Task()(&ctx)(&err)
|
|
||||||
|
|
||||||
if segment.Expired(observer.Now) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
bucket := observer.ensureBucket(ctx, segment.Location.Object())
|
|
||||||
bucket.RemoteSegments++
|
|
||||||
bucket.RemoteBytes += int64(segment.EncryptedSize)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func projectTotalsFromBuckets(buckets map[metabase.BucketLocation]*accounting.BucketTally) map[uuid.UUID]int64 {
|
func projectTotalsFromBuckets(buckets map[metabase.BucketLocation]*accounting.BucketTally) map[uuid.UUID]int64 {
|
||||||
projectTallyTotals := make(map[uuid.UUID]int64)
|
projectTallyTotals := make(map[uuid.UUID]int64)
|
||||||
for _, bucket := range buckets {
|
for _, bucket := range buckets {
|
||||||
projectTallyTotals[bucket.ProjectID] += (bucket.InlineBytes + bucket.RemoteBytes)
|
projectTallyTotals[bucket.ProjectID] += bucket.TotalBytes
|
||||||
}
|
}
|
||||||
return projectTallyTotals
|
return projectTallyTotals
|
||||||
}
|
}
|
||||||
|
@ -85,8 +85,8 @@ func TestOnlyInline(t *testing.T) {
|
|||||||
BucketName: expectedBucketName,
|
BucketName: expectedBucketName,
|
||||||
},
|
},
|
||||||
ObjectCount: 1,
|
ObjectCount: 1,
|
||||||
InlineSegments: 1,
|
TotalSegments: 1,
|
||||||
InlineBytes: int64(expectedTotalBytes),
|
TotalBytes: int64(expectedTotalBytes),
|
||||||
MetadataSize: 0,
|
MetadataSize: 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -96,16 +96,16 @@ func TestOnlyInline(t *testing.T) {
|
|||||||
|
|
||||||
// run multiple times to ensure we add tallies
|
// run multiple times to ensure we add tallies
|
||||||
for i := 0; i < 2; i++ {
|
for i := 0; i < 2; i++ {
|
||||||
obs := tally.NewObserver(planet.Satellites[0].Log.Named("observer"), time.Now())
|
collector := tally.NewBucketTallyCollector(planet.Satellites[0].Log.Named("bucket tally"), time.Now(), planet.Satellites[0].Metainfo.Metabase, planet.Satellites[0].Config.Tally)
|
||||||
err := planet.Satellites[0].Metainfo.Loop.Join(ctx, obs)
|
err := collector.Run(ctx)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
now := time.Now().Add(time.Duration(i) * time.Second)
|
now := time.Now().Add(time.Duration(i) * time.Second)
|
||||||
err = planet.Satellites[0].DB.ProjectAccounting().SaveTallies(ctx, now, obs.Bucket)
|
err = planet.Satellites[0].DB.ProjectAccounting().SaveTallies(ctx, now, collector.Bucket)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
assert.Equal(t, 1, len(obs.Bucket))
|
assert.Equal(t, 1, len(collector.Bucket))
|
||||||
for _, actualTally := range obs.Bucket {
|
for _, actualTally := range collector.Bucket {
|
||||||
// checking the exact metadata size is brittle, instead, verify that it's not zero
|
// checking the exact metadata size is brittle, instead, verify that it's not zero
|
||||||
assert.NotZero(t, actualTally.MetadataSize)
|
assert.NotZero(t, actualTally.MetadataSize)
|
||||||
actualTally.MetadataSize = expectedTally.MetadataSize
|
actualTally.MetadataSize = expectedTally.MetadataSize
|
||||||
@ -166,20 +166,15 @@ func TestCalculateBucketAtRestData(t *testing.T) {
|
|||||||
for _, segment := range segments {
|
for _, segment := range segments {
|
||||||
loc := streamLocation[segment.StreamID]
|
loc := streamLocation[segment.StreamID]
|
||||||
t := ensure(loc)
|
t := ensure(loc)
|
||||||
if len(segment.Pieces) > 0 {
|
t.TotalSegments++
|
||||||
t.RemoteSegments++
|
t.TotalBytes += int64(segment.EncryptedSize)
|
||||||
t.RemoteBytes += int64(segment.EncryptedSize)
|
|
||||||
} else {
|
|
||||||
t.InlineSegments++
|
|
||||||
t.InlineBytes += int64(segment.EncryptedSize)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
require.Len(t, expectedTotal, 3)
|
require.Len(t, expectedTotal, 3)
|
||||||
|
|
||||||
obs := tally.NewObserver(satellite.Log.Named("observer"), time.Now())
|
collector := tally.NewBucketTallyCollector(satellite.Log.Named("bucket tally"), time.Now(), satellite.Metainfo.Metabase, planet.Satellites[0].Config.Tally)
|
||||||
err = satellite.Metainfo.Loop.Join(ctx, obs)
|
err = collector.Run(ctx)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, expectedTotal, obs.Bucket)
|
require.Equal(t, expectedTotal, collector.Bucket)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -193,12 +188,12 @@ func TestTallyIgnoresExpiredPointers(t *testing.T) {
|
|||||||
err := planet.Uplinks[0].UploadWithExpiration(ctx, planet.Satellites[0], "bucket", "path", []byte{1}, now.Add(12*time.Hour))
|
err := planet.Uplinks[0].UploadWithExpiration(ctx, planet.Satellites[0], "bucket", "path", []byte{1}, now.Add(12*time.Hour))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
obs := tally.NewObserver(satellite.Log.Named("observer"), now.Add(24*time.Hour))
|
collector := tally.NewBucketTallyCollector(satellite.Log.Named("bucket tally"), now.Add(24*time.Hour), satellite.Metainfo.Metabase, planet.Satellites[0].Config.Tally)
|
||||||
err = satellite.Metainfo.Loop.Join(ctx, obs)
|
err = collector.Run(ctx)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// there should be no observed buckets because all of the pointers are expired
|
// there should be no observed buckets because all of the objects are expired
|
||||||
require.Equal(t, obs.Bucket, map[metabase.BucketLocation]*accounting.BucketTally{})
|
require.Equal(t, collector.Bucket, map[metabase.BucketLocation]*accounting.BucketTally{})
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -330,10 +330,8 @@ func TestCheckUsageWithUsage(t *testing.T) {
|
|||||||
ProjectID: projectID,
|
ProjectID: projectID,
|
||||||
IntervalStart: time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 1, time.UTC),
|
IntervalStart: time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 1, time.UTC),
|
||||||
ObjectCount: 1,
|
ObjectCount: 1,
|
||||||
InlineSegmentCount: 1,
|
TotalSegmentCount: 2,
|
||||||
RemoteSegmentCount: 1,
|
TotalBytes: 640000,
|
||||||
InlineBytes: 10,
|
|
||||||
RemoteBytes: 640000,
|
|
||||||
MetadataSize: 2,
|
MetadataSize: 2,
|
||||||
}
|
}
|
||||||
err = planet.Satellites[0].DB.ProjectAccounting().CreateStorageTally(ctx, tally)
|
err = planet.Satellites[0].DB.ProjectAccounting().CreateStorageTally(ctx, tally)
|
||||||
@ -343,10 +341,8 @@ func TestCheckUsageWithUsage(t *testing.T) {
|
|||||||
ProjectID: projectID,
|
ProjectID: projectID,
|
||||||
IntervalStart: time.Date(now.Year(), now.Month(), 1, 0, 1, 0, 1, time.UTC),
|
IntervalStart: time.Date(now.Year(), now.Month(), 1, 0, 1, 0, 1, time.UTC),
|
||||||
ObjectCount: 1,
|
ObjectCount: 1,
|
||||||
InlineSegmentCount: 1,
|
TotalSegmentCount: 2,
|
||||||
RemoteSegmentCount: 1,
|
TotalBytes: 640000,
|
||||||
InlineBytes: 10,
|
|
||||||
RemoteBytes: 640000,
|
|
||||||
MetadataSize: 2,
|
MetadataSize: 2,
|
||||||
}
|
}
|
||||||
err = planet.Satellites[0].DB.ProjectAccounting().CreateStorageTally(ctx, tally)
|
err = planet.Satellites[0].DB.ProjectAccounting().CreateStorageTally(ctx, tally)
|
||||||
@ -404,10 +400,8 @@ func TestCheckUsageLastMonthUnappliedInvoice(t *testing.T) {
|
|||||||
ProjectID: projectID,
|
ProjectID: projectID,
|
||||||
IntervalStart: time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 1, time.UTC),
|
IntervalStart: time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 1, time.UTC),
|
||||||
ObjectCount: 1,
|
ObjectCount: 1,
|
||||||
InlineSegmentCount: 1,
|
TotalSegmentCount: 2,
|
||||||
RemoteSegmentCount: 1,
|
TotalBytes: 640000,
|
||||||
InlineBytes: 10,
|
|
||||||
RemoteBytes: 640000,
|
|
||||||
MetadataSize: 2,
|
MetadataSize: 2,
|
||||||
}
|
}
|
||||||
err = planet.Satellites[0].DB.ProjectAccounting().CreateStorageTally(ctx, tally)
|
err = planet.Satellites[0].DB.ProjectAccounting().CreateStorageTally(ctx, tally)
|
||||||
@ -417,10 +411,8 @@ func TestCheckUsageLastMonthUnappliedInvoice(t *testing.T) {
|
|||||||
ProjectID: projectID,
|
ProjectID: projectID,
|
||||||
IntervalStart: time.Date(now.Year(), now.Month(), 1, 0, 1, 0, 1, time.UTC),
|
IntervalStart: time.Date(now.Year(), now.Month(), 1, 0, 1, 0, 1, time.UTC),
|
||||||
ObjectCount: 1,
|
ObjectCount: 1,
|
||||||
InlineSegmentCount: 1,
|
TotalSegmentCount: 2,
|
||||||
RemoteSegmentCount: 1,
|
TotalBytes: 640000,
|
||||||
InlineBytes: 10,
|
|
||||||
RemoteBytes: 640000,
|
|
||||||
MetadataSize: 2,
|
MetadataSize: 2,
|
||||||
}
|
}
|
||||||
err = planet.Satellites[0].DB.ProjectAccounting().CreateStorageTally(ctx, tally)
|
err = planet.Satellites[0].DB.ProjectAccounting().CreateStorageTally(ctx, tally)
|
||||||
@ -478,10 +470,8 @@ func TestDeleteProjectWithUsageCurrentMonth(t *testing.T) {
|
|||||||
ProjectID: projectID,
|
ProjectID: projectID,
|
||||||
IntervalStart: time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 1, time.UTC),
|
IntervalStart: time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 1, time.UTC),
|
||||||
ObjectCount: 1,
|
ObjectCount: 1,
|
||||||
InlineSegmentCount: 1,
|
TotalSegmentCount: 2,
|
||||||
RemoteSegmentCount: 1,
|
TotalBytes: 640000,
|
||||||
InlineBytes: 10,
|
|
||||||
RemoteBytes: 640000,
|
|
||||||
MetadataSize: 2,
|
MetadataSize: 2,
|
||||||
}
|
}
|
||||||
err = planet.Satellites[0].DB.ProjectAccounting().CreateStorageTally(ctx, tally)
|
err = planet.Satellites[0].DB.ProjectAccounting().CreateStorageTally(ctx, tally)
|
||||||
@ -491,10 +481,8 @@ func TestDeleteProjectWithUsageCurrentMonth(t *testing.T) {
|
|||||||
ProjectID: projectID,
|
ProjectID: projectID,
|
||||||
IntervalStart: time.Date(now.Year(), now.Month(), 1, 0, 1, 0, 1, time.UTC),
|
IntervalStart: time.Date(now.Year(), now.Month(), 1, 0, 1, 0, 1, time.UTC),
|
||||||
ObjectCount: 1,
|
ObjectCount: 1,
|
||||||
InlineSegmentCount: 1,
|
TotalSegmentCount: 2,
|
||||||
RemoteSegmentCount: 1,
|
TotalBytes: 640000,
|
||||||
InlineBytes: 10,
|
|
||||||
RemoteBytes: 640000,
|
|
||||||
MetadataSize: 2,
|
MetadataSize: 2,
|
||||||
}
|
}
|
||||||
err = planet.Satellites[0].DB.ProjectAccounting().CreateStorageTally(ctx, tally)
|
err = planet.Satellites[0].DB.ProjectAccounting().CreateStorageTally(ctx, tally)
|
||||||
@ -548,10 +536,8 @@ func TestDeleteProjectWithUsagePreviousMonth(t *testing.T) {
|
|||||||
ProjectID: projectID,
|
ProjectID: projectID,
|
||||||
IntervalStart: accTime,
|
IntervalStart: accTime,
|
||||||
ObjectCount: 1,
|
ObjectCount: 1,
|
||||||
InlineSegmentCount: 1,
|
TotalSegmentCount: 2,
|
||||||
RemoteSegmentCount: 1,
|
TotalBytes: 640000,
|
||||||
InlineBytes: 10,
|
|
||||||
RemoteBytes: 640000,
|
|
||||||
MetadataSize: 2,
|
MetadataSize: 2,
|
||||||
}
|
}
|
||||||
err = planet.Satellites[0].DB.ProjectAccounting().CreateStorageTally(ctx, tally)
|
err = planet.Satellites[0].DB.ProjectAccounting().CreateStorageTally(ctx, tally)
|
||||||
@ -561,10 +547,8 @@ func TestDeleteProjectWithUsagePreviousMonth(t *testing.T) {
|
|||||||
ProjectID: projectID,
|
ProjectID: projectID,
|
||||||
IntervalStart: accTime.AddDate(0, 0, 1),
|
IntervalStart: accTime.AddDate(0, 0, 1),
|
||||||
ObjectCount: 1,
|
ObjectCount: 1,
|
||||||
InlineSegmentCount: 1,
|
TotalSegmentCount: 2,
|
||||||
RemoteSegmentCount: 1,
|
TotalBytes: 640000,
|
||||||
InlineBytes: 10,
|
|
||||||
RemoteBytes: 640000,
|
|
||||||
MetadataSize: 2,
|
MetadataSize: 2,
|
||||||
}
|
}
|
||||||
err = planet.Satellites[0].DB.ProjectAccounting().CreateStorageTally(ctx, tally)
|
err = planet.Satellites[0].DB.ProjectAccounting().CreateStorageTally(ctx, tally)
|
||||||
|
@ -29,8 +29,7 @@ type CSVRow struct {
|
|||||||
PartnerID []byte
|
PartnerID []byte
|
||||||
ProjectID []byte
|
ProjectID []byte
|
||||||
BucketName []byte
|
BucketName []byte
|
||||||
RemoteBytesPerHour float64
|
TotalBytesPerHour float64
|
||||||
InlineBytesPerHour float64
|
|
||||||
EgressData int64
|
EgressData int64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -47,8 +47,7 @@ type AttributionTestData struct {
|
|||||||
egressSize int64
|
egressSize int64
|
||||||
|
|
||||||
dataCounter int
|
dataCounter int
|
||||||
expectedRemoteBytes int64
|
expectedTotalBytes int64
|
||||||
expectedInlineBytes int64
|
|
||||||
expectedEgress int64
|
expectedEgress int64
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -187,8 +186,7 @@ func verifyData(ctx *testcontext.Context, t *testing.T, attributionDB attributio
|
|||||||
assert.Equal(t, testData.partnerID[:], r.PartnerID, testData.name)
|
assert.Equal(t, testData.partnerID[:], r.PartnerID, testData.name)
|
||||||
assert.Equal(t, testData.projectID[:], r.ProjectID, testData.name)
|
assert.Equal(t, testData.projectID[:], r.ProjectID, testData.name)
|
||||||
assert.Equal(t, testData.bucketName, r.BucketName, testData.name)
|
assert.Equal(t, testData.bucketName, r.BucketName, testData.name)
|
||||||
assert.Equal(t, float64(testData.expectedRemoteBytes/testData.hours), r.RemoteBytesPerHour, testData.name)
|
assert.Equal(t, float64(testData.expectedTotalBytes/testData.hours), r.TotalBytesPerHour, testData.name)
|
||||||
assert.Equal(t, float64(testData.expectedInlineBytes/testData.hours), r.InlineBytesPerHour, testData.name)
|
|
||||||
assert.Equal(t, testData.expectedEgress, r.EgressData, testData.name)
|
assert.Equal(t, testData.expectedEgress, r.EgressData, testData.name)
|
||||||
}
|
}
|
||||||
require.NotEqual(t, 0, count, "Results were returned, but did not match all of the the projectIDs.")
|
require.NotEqual(t, 0, count, "Results were returned, but did not match all of the the projectIDs.")
|
||||||
@ -218,8 +216,7 @@ func createData(ctx *testcontext.Context, t *testing.T, db satellite.DB, testDat
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
if (testData.dataInterval.After(testData.start) || testData.dataInterval.Equal(testData.start)) && testData.dataInterval.Before(testData.end) {
|
if (testData.dataInterval.After(testData.start) || testData.dataInterval.Equal(testData.start)) && testData.dataInterval.Before(testData.end) {
|
||||||
testData.expectedRemoteBytes += tally.RemoteBytes
|
testData.expectedTotalBytes += tally.TotalBytes
|
||||||
testData.expectedInlineBytes += tally.InlineBytes
|
|
||||||
testData.expectedEgress += testData.egressSize
|
testData.expectedEgress += testData.egressSize
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -232,11 +229,9 @@ func createTallyData(ctx *testcontext.Context, projectAccoutingDB accounting.Pro
|
|||||||
|
|
||||||
ObjectCount: 0,
|
ObjectCount: 0,
|
||||||
|
|
||||||
InlineSegmentCount: 0,
|
TotalSegmentCount: 0,
|
||||||
RemoteSegmentCount: 0,
|
|
||||||
|
|
||||||
InlineBytes: inline,
|
TotalBytes: inline + remote,
|
||||||
RemoteBytes: remote,
|
|
||||||
MetadataSize: 0,
|
MetadataSize: 0,
|
||||||
}
|
}
|
||||||
err = projectAccoutingDB.CreateStorageTally(ctx, tally)
|
err = projectAccoutingDB.CreateStorageTally(ctx, tally)
|
||||||
|
@ -357,7 +357,7 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB,
|
|||||||
}
|
}
|
||||||
|
|
||||||
{ // setup accounting
|
{ // setup accounting
|
||||||
peer.Accounting.Tally = tally.New(peer.Log.Named("accounting:tally"), peer.DB.StoragenodeAccounting(), peer.DB.ProjectAccounting(), peer.LiveAccounting.Cache, peer.Metainfo.Loop, config.Tally.Interval)
|
peer.Accounting.Tally = tally.New(peer.Log.Named("accounting:tally"), peer.DB.StoragenodeAccounting(), peer.DB.ProjectAccounting(), peer.LiveAccounting.Cache, peer.Metainfo.Metabase, config.Tally)
|
||||||
peer.Services.Add(lifecycle.Item{
|
peer.Services.Add(lifecycle.Item{
|
||||||
Name: "accounting:tally",
|
Name: "accounting:tally",
|
||||||
Run: peer.Accounting.Tally.Run,
|
Run: peer.Accounting.Tally.Run,
|
||||||
|
@ -245,7 +245,7 @@ func TestQueryAttribution(t *testing.T) {
|
|||||||
|
|
||||||
rows, err := planet.Satellites[0].DB.Attribution().QueryAttribution(ctx, partner.UUID, before, after)
|
rows, err := planet.Satellites[0].DB.Attribution().QueryAttribution(ctx, partner.UUID, before, after)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NotZero(t, rows[0].RemoteBytesPerHour)
|
require.NotZero(t, rows[0].TotalBytesPerHour)
|
||||||
require.Equal(t, rows[0].EgressData, usage.Egress)
|
require.Equal(t, rows[0].EgressData, usage.Egress)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@ -316,7 +316,7 @@ func TestAttributionReport(t *testing.T) {
|
|||||||
|
|
||||||
rows, err := planet.Satellites[0].DB.Attribution().QueryAttribution(ctx, partner.UUID, before, after)
|
rows, err := planet.Satellites[0].DB.Attribution().QueryAttribution(ctx, partner.UUID, before, after)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NotZero(t, rows[0].RemoteBytesPerHour)
|
require.NotZero(t, rows[0].TotalBytesPerHour)
|
||||||
require.Equal(t, rows[0].EgressData, usage.Egress)
|
require.Equal(t, rows[0].EgressData, usage.Egress)
|
||||||
|
|
||||||
// Minio should have no attribution because bucket was created by Zenko
|
// Minio should have no attribution because bucket was created by Zenko
|
||||||
|
@ -148,7 +148,7 @@ func TestService_InvoiceUserWithManyProjects(t *testing.T) {
|
|||||||
ProjectID: projects[i].ID,
|
ProjectID: projects[i].ID,
|
||||||
BucketName: "testbucket",
|
BucketName: "testbucket",
|
||||||
},
|
},
|
||||||
RemoteBytes: projectsStorage[i],
|
TotalBytes: projectsStorage[i],
|
||||||
ObjectCount: int64(i + 1),
|
ObjectCount: int64(i + 1),
|
||||||
}
|
}
|
||||||
tallies := map[metabase.BucketLocation]*accounting.BucketTally{
|
tallies := map[metabase.BucketLocation]*accounting.BucketTally{
|
||||||
@ -259,7 +259,7 @@ func TestService_InvoiceUserWithManyCoupons(t *testing.T) {
|
|||||||
ProjectID: project.ID,
|
ProjectID: project.ID,
|
||||||
BucketName: "testbucket",
|
BucketName: "testbucket",
|
||||||
},
|
},
|
||||||
RemoteBytes: memory.TiB.Int64(),
|
TotalBytes: memory.TiB.Int64(),
|
||||||
ObjectCount: 45,
|
ObjectCount: 45,
|
||||||
}
|
}
|
||||||
tallies := map[metabase.BucketLocation]*accounting.BucketTally{
|
tallies := map[metabase.BucketLocation]*accounting.BucketTally{
|
||||||
|
@ -24,6 +24,7 @@ const (
|
|||||||
o.partner_id as partner_id,
|
o.partner_id as partner_id,
|
||||||
o.project_id as project_id,
|
o.project_id as project_id,
|
||||||
o.bucket_name as bucket_name,
|
o.bucket_name as bucket_name,
|
||||||
|
SUM(o.total) / SUM(o.hours) as total,
|
||||||
SUM(o.remote) / SUM(o.hours) as remote,
|
SUM(o.remote) / SUM(o.hours) as remote,
|
||||||
SUM(o.inline) / SUM(o.hours) as inline,
|
SUM(o.inline) / SUM(o.hours) as inline,
|
||||||
SUM(o.settled) as settled
|
SUM(o.settled) as settled
|
||||||
@ -35,6 +36,7 @@ const (
|
|||||||
bsti.partner_id as partner_id,
|
bsti.partner_id as partner_id,
|
||||||
bsto.project_id as project_id,
|
bsto.project_id as project_id,
|
||||||
bsto.bucket_name as bucket_name,
|
bsto.bucket_name as bucket_name,
|
||||||
|
SUM(bsto.total_bytes) as total,
|
||||||
SUM(bsto.remote) as remote,
|
SUM(bsto.remote) as remote,
|
||||||
SUM(bsto.inline) as inline,
|
SUM(bsto.inline) as inline,
|
||||||
0 as settled,
|
0 as settled,
|
||||||
@ -82,6 +84,7 @@ const (
|
|||||||
va.partner_id as partner_id,
|
va.partner_id as partner_id,
|
||||||
bbr.project_id as project_id,
|
bbr.project_id as project_id,
|
||||||
bbr.bucket_name as bucket_name,
|
bbr.bucket_name as bucket_name,
|
||||||
|
0 as total,
|
||||||
0 as remote,
|
0 as remote,
|
||||||
0 as inline,
|
0 as inline,
|
||||||
SUM(settled)::integer as settled,
|
SUM(settled)::integer as settled,
|
||||||
@ -165,10 +168,16 @@ func (keys *attributionDB) QueryAttribution(ctx context.Context, partnerID uuid.
|
|||||||
results := []*attribution.CSVRow{}
|
results := []*attribution.CSVRow{}
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
r := &attribution.CSVRow{}
|
r := &attribution.CSVRow{}
|
||||||
err := rows.Scan(&r.PartnerID, &r.ProjectID, &r.BucketName, &r.RemoteBytesPerHour, &r.InlineBytesPerHour, &r.EgressData)
|
var inline, remote float64
|
||||||
|
err := rows.Scan(&r.PartnerID, &r.ProjectID, &r.BucketName, &r.TotalBytesPerHour, &inline, &remote, &r.EgressData)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return results, Error.Wrap(err)
|
return results, Error.Wrap(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if r.TotalBytesPerHour == 0 {
|
||||||
|
r.TotalBytesPerHour = inline + remote
|
||||||
|
}
|
||||||
|
|
||||||
results = append(results, r)
|
results = append(results, r)
|
||||||
}
|
}
|
||||||
return results, Error.Wrap(rows.Err())
|
return results, Error.Wrap(rows.Err())
|
||||||
|
@ -40,17 +40,13 @@ func (db *ProjectAccounting) SaveTallies(ctx context.Context, intervalStart time
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
var bucketNames, projectIDs [][]byte
|
var bucketNames, projectIDs [][]byte
|
||||||
var totalBytes, inlineBytes, remoteBytes, metadataSizes []int64
|
var totalBytes, metadataSizes []int64
|
||||||
var totalSegments, remoteSegments, inlineSegments, objectCounts []int64
|
var totalSegments, objectCounts []int64
|
||||||
for _, info := range bucketTallies {
|
for _, info := range bucketTallies {
|
||||||
bucketNames = append(bucketNames, []byte(info.BucketName))
|
bucketNames = append(bucketNames, []byte(info.BucketName))
|
||||||
projectIDs = append(projectIDs, info.ProjectID[:])
|
projectIDs = append(projectIDs, info.ProjectID[:])
|
||||||
totalBytes = append(totalBytes, info.TotalBytes)
|
totalBytes = append(totalBytes, info.TotalBytes)
|
||||||
inlineBytes = append(inlineBytes, info.InlineBytes)
|
|
||||||
remoteBytes = append(remoteBytes, info.RemoteBytes)
|
|
||||||
totalSegments = append(totalSegments, info.TotalSegments)
|
totalSegments = append(totalSegments, info.TotalSegments)
|
||||||
remoteSegments = append(remoteSegments, info.RemoteSegments)
|
|
||||||
inlineSegments = append(inlineSegments, info.InlineSegments)
|
|
||||||
objectCounts = append(objectCounts, info.ObjectCount)
|
objectCounts = append(objectCounts, info.ObjectCount)
|
||||||
metadataSizes = append(metadataSizes, info.MetadataSize)
|
metadataSizes = append(metadataSizes, info.MetadataSize)
|
||||||
}
|
}
|
||||||
@ -64,13 +60,13 @@ func (db *ProjectAccounting) SaveTallies(ctx context.Context, intervalStart time
|
|||||||
SELECT
|
SELECT
|
||||||
$1,
|
$1,
|
||||||
unnest($2::bytea[]), unnest($3::bytea[]),
|
unnest($2::bytea[]), unnest($3::bytea[]),
|
||||||
unnest($4::int8[]), unnest($5::int8[]), unnest($6::int8[]),
|
unnest($4::int8[]), $5, $6,
|
||||||
unnest($7::int8[]), unnest($8::int8[]), unnest($9::int8[]),
|
unnest($7::int8[]), $8, $9,
|
||||||
unnest($10::int8[]), unnest($11::int8[])`),
|
unnest($10::int8[]), unnest($11::int8[])`),
|
||||||
intervalStart,
|
intervalStart,
|
||||||
pgutil.ByteaArray(bucketNames), pgutil.ByteaArray(projectIDs),
|
pgutil.ByteaArray(bucketNames), pgutil.ByteaArray(projectIDs),
|
||||||
pgutil.Int8Array(totalBytes), pgutil.Int8Array(inlineBytes), pgutil.Int8Array(remoteBytes),
|
pgutil.Int8Array(totalBytes), 0, 0,
|
||||||
pgutil.Int8Array(totalSegments), pgutil.Int8Array(remoteSegments), pgutil.Int8Array(inlineSegments),
|
pgutil.Int8Array(totalSegments), 0, 0,
|
||||||
pgutil.Int8Array(objectCounts), pgutil.Int8Array(metadataSizes))
|
pgutil.Int8Array(objectCounts), pgutil.Int8Array(metadataSizes))
|
||||||
|
|
||||||
return Error.Wrap(err)
|
return Error.Wrap(err)
|
||||||
@ -91,18 +87,24 @@ func (db *ProjectAccounting) GetTallies(ctx context.Context) (tallies []accounti
|
|||||||
return nil, Error.Wrap(err)
|
return nil, Error.Wrap(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
totalBytes := dbxTally.TotalBytes
|
||||||
|
if totalBytes == 0 {
|
||||||
|
totalBytes = dbxTally.Inline + dbxTally.Remote
|
||||||
|
}
|
||||||
|
|
||||||
|
totalSegments := dbxTally.TotalSegmentsCount
|
||||||
|
if totalSegments == 0 {
|
||||||
|
totalSegments = dbxTally.InlineSegmentsCount + dbxTally.RemoteSegmentsCount
|
||||||
|
}
|
||||||
|
|
||||||
tallies = append(tallies, accounting.BucketTally{
|
tallies = append(tallies, accounting.BucketTally{
|
||||||
BucketLocation: metabase.BucketLocation{
|
BucketLocation: metabase.BucketLocation{
|
||||||
ProjectID: projectID,
|
ProjectID: projectID,
|
||||||
BucketName: string(dbxTally.BucketName),
|
BucketName: string(dbxTally.BucketName),
|
||||||
},
|
},
|
||||||
ObjectCount: int64(dbxTally.ObjectCount),
|
ObjectCount: int64(dbxTally.ObjectCount),
|
||||||
TotalSegments: int64(dbxTally.TotalSegmentsCount),
|
TotalSegments: int64(totalSegments),
|
||||||
InlineSegments: int64(dbxTally.InlineSegmentsCount),
|
TotalBytes: int64(totalBytes),
|
||||||
RemoteSegments: int64(dbxTally.RemoteSegmentsCount),
|
|
||||||
TotalBytes: int64(dbxTally.TotalBytes),
|
|
||||||
InlineBytes: int64(dbxTally.Inline),
|
|
||||||
RemoteBytes: int64(dbxTally.Remote),
|
|
||||||
MetadataSize: int64(dbxTally.MetadataSize),
|
MetadataSize: int64(dbxTally.MetadataSize),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@ -129,8 +131,8 @@ func (db *ProjectAccounting) CreateStorageTally(ctx context.Context, tally accou
|
|||||||
?, ?
|
?, ?
|
||||||
)`), tally.IntervalStart,
|
)`), tally.IntervalStart,
|
||||||
[]byte(tally.BucketName), tally.ProjectID,
|
[]byte(tally.BucketName), tally.ProjectID,
|
||||||
tally.TotalBytes, tally.InlineBytes, tally.RemoteBytes,
|
tally.TotalBytes, 0, 0,
|
||||||
tally.TotalSegmentCount, tally.RemoteSegmentCount, tally.InlineSegmentCount,
|
tally.TotalSegmentCount, 0, 0,
|
||||||
tally.ObjectCount, tally.MetadataSize,
|
tally.ObjectCount, tally.MetadataSize,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -301,10 +303,15 @@ func (db *ProjectAccounting) GetProjectTotal(ctx context.Context, projectID uuid
|
|||||||
for storageTalliesRows.Next() {
|
for storageTalliesRows.Next() {
|
||||||
tally := accounting.BucketStorageTally{}
|
tally := accounting.BucketStorageTally{}
|
||||||
|
|
||||||
err = storageTalliesRows.Scan(&tally.IntervalStart, &tally.TotalBytes, &tally.InlineBytes, &tally.RemoteBytes, &tally.ObjectCount)
|
var inline, remote int64
|
||||||
|
err = storageTalliesRows.Scan(&tally.IntervalStart, &tally.TotalBytes, &inline, &remote, &tally.ObjectCount)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errs.Combine(err, storageTalliesRows.Close())
|
return nil, errs.Combine(err, storageTalliesRows.Close())
|
||||||
}
|
}
|
||||||
|
if tally.TotalBytes == 0 {
|
||||||
|
tally.TotalBytes = inline + remote
|
||||||
|
}
|
||||||
|
|
||||||
tally.BucketName = bucket
|
tally.BucketName = bucket
|
||||||
storageTallies = append(storageTallies, &tally)
|
storageTallies = append(storageTallies, &tally)
|
||||||
}
|
}
|
||||||
@ -638,13 +645,18 @@ func (db *ProjectAccounting) GetBucketTotals(ctx context.Context, projectID uuid
|
|||||||
storageRow := db.db.QueryRowContext(ctx, storageQuery, projectID[:], []byte(bucket), since, before)
|
storageRow := db.db.QueryRowContext(ctx, storageQuery, projectID[:], []byte(bucket), since, before)
|
||||||
|
|
||||||
var tally accounting.BucketStorageTally
|
var tally accounting.BucketStorageTally
|
||||||
err = storageRow.Scan(&tally.TotalBytes, &tally.InlineBytes, &tally.RemoteBytes, &tally.ObjectCount)
|
var inline, remote int64
|
||||||
|
err = storageRow.Scan(&tally.TotalBytes, &inline, &remote, &tally.ObjectCount)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, sql.ErrNoRows) {
|
if !errors.Is(err, sql.ErrNoRows) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if tally.TotalBytes == 0 {
|
||||||
|
tally.TotalBytes = inline + remote
|
||||||
|
}
|
||||||
|
|
||||||
// fill storage and object count
|
// fill storage and object count
|
||||||
bucketUsage.Storage = memory.Size(tally.Bytes()).GB()
|
bucketUsage.Storage = memory.Size(tally.Bytes()).GB()
|
||||||
bucketUsage.ObjectCount = tally.ObjectCount
|
bucketUsage.ObjectCount = tally.ObjectCount
|
||||||
|
6
scripts/testdata/satellite-config.yaml.lock
vendored
6
scripts/testdata/satellite-config.yaml.lock
vendored
@ -718,9 +718,15 @@ server.private-address: 127.0.0.1:7778
|
|||||||
# length of time a node can go without contacting satellite before being disqualified
|
# length of time a node can go without contacting satellite before being disqualified
|
||||||
# stray-nodes.max-duration-without-contact: 720h0m0s
|
# stray-nodes.max-duration-without-contact: 720h0m0s
|
||||||
|
|
||||||
|
# as of system interval
|
||||||
|
# tally.as-of-system-interval: -5m0s
|
||||||
|
|
||||||
# how frequently the tally service should run
|
# how frequently the tally service should run
|
||||||
# tally.interval: 1h0m0s
|
# tally.interval: 1h0m0s
|
||||||
|
|
||||||
|
# how many objects to query in a batch
|
||||||
|
# tally.list-limit: 2500
|
||||||
|
|
||||||
# how large of batches GetBandwidthSince should process at a time
|
# how large of batches GetBandwidthSince should process at a time
|
||||||
# tally.read-rollup-batch-size: 10000
|
# tally.read-rollup-batch-size: 10000
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user