73a279235a
After infrastructure changes redis instance is not neccessay close to core instance (where tally is calculated) and round trips to get data from redis can be very costly. From less than hour calculation can take few hours for larger satellite. This change combines 'segment' and 'storage' usage requests into batches to reduce latency impact on tally calculation. https://github.com/storj/storj/issues/5800 Change-Id: I87e57ec09e88fd167060a4ed51dc8b0274a095c5
58 lines
2.0 KiB
Go
58 lines
2.0 KiB
Go
// Copyright (C) 2019 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package live
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/spacemonkeygo/monkit/v3"
|
|
"github.com/zeebo/errs"
|
|
"go.uber.org/zap"
|
|
|
|
"storj.io/storj/satellite/accounting"
|
|
)
|
|
|
|
var (
|
|
// Error is the default error class for live-accounting.
|
|
Error = errs.Class("live-accounting")
|
|
mon = monkit.Package()
|
|
)
|
|
|
|
// Config contains configurable values for the live accounting service.
|
|
type Config struct {
|
|
StorageBackend string `help:"what to use for storing real-time accounting data"`
|
|
BandwidthCacheTTL time.Duration `default:"5m" help:"bandwidth cache key time to live"`
|
|
AsOfSystemInterval time.Duration `default:"-10s" help:"as of system interval"`
|
|
BatchSize int `default:"5000" help:"how much projects usage should be requested from redis cache at once"`
|
|
}
|
|
|
|
// OpenCache creates a new accounting.Cache instance using the type specified backend in
|
|
// the provided config.
|
|
//
|
|
// The cache instance may be returned despite of returning the
|
|
// accounting.ErrSystemOrNetError because some backends allows to reconnect on
|
|
// each operation if the connection was not established or it was disconnected,
|
|
// which is what it could happen at the moment to instance it and the cache will
|
|
// work one the backend system will be reachable later on.
|
|
// For this reason, the components that uses the cache should operate despite
|
|
// the backend is not responding successfully although their service is
|
|
// degraded.
|
|
func OpenCache(ctx context.Context, log *zap.Logger, config Config) (accounting.Cache, error) {
|
|
parts := strings.SplitN(config.StorageBackend, ":", 2)
|
|
var backendType string
|
|
if len(parts) == 0 || parts[0] == "" {
|
|
return nil, Error.New("please specify a backend for live accounting")
|
|
}
|
|
|
|
backendType = parts[0]
|
|
switch backendType {
|
|
case "redis":
|
|
return openRedisLiveAccounting(ctx, config.StorageBackend, config.BatchSize)
|
|
default:
|
|
return nil, Error.New("unrecognized live accounting backend specifier %q. Currently only redis is supported", backendType)
|
|
}
|
|
}
|