storagenode/console: added list of all audits per satellite to sno dashboard/satellites
Change-Id: I52e58748d6467f372d9a308347fc77e400d137e2
This commit is contained in:
parent
373934efb2
commit
4ee1b2d45a
2
go.mod
2
go.mod
@ -41,7 +41,7 @@ require (
|
||||
golang.org/x/sys v0.0.0-20200610111108-226ff32320da
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0
|
||||
golang.org/x/tools v0.0.0-20200428211428-0c9eba77bc32 // indirect
|
||||
storj.io/common v0.0.0-20200729140050-4c1ddac6fa63
|
||||
storj.io/common v0.0.0-20200810083036-275dbbe0bd21
|
||||
storj.io/drpc v0.0.14
|
||||
storj.io/monkit-jaeger v0.0.0-20200518165323-80778fc3f91b
|
||||
storj.io/private v0.0.0-20200729145012-46794d335b51
|
||||
|
2
go.sum
2
go.sum
@ -714,6 +714,8 @@ storj.io/common v0.0.0-20200710113903-e5030388ee6a h1:8lMuSiBcU3u6Y3YzlG53GKvyG0
|
||||
storj.io/common v0.0.0-20200710113903-e5030388ee6a/go.mod h1:vMAnlNbkgW6i+w/OT1h4X8w6TajOHWAT+SvFHUFCpq0=
|
||||
storj.io/common v0.0.0-20200729140050-4c1ddac6fa63 h1:BkRvlginTJGi0yAkpN+4ZKm2YpG63bDSDFLQtXYxxdg=
|
||||
storj.io/common v0.0.0-20200729140050-4c1ddac6fa63/go.mod h1:ILr54ISCqCQ6MmIwT7eaR/fEGrBfgfxiPt8nmpWqnUM=
|
||||
storj.io/common v0.0.0-20200810083036-275dbbe0bd21 h1:Qjrd4KslLBdI76VHwO6+S7tkkYa6/K/IgPFzTvA6M8k=
|
||||
storj.io/common v0.0.0-20200810083036-275dbbe0bd21/go.mod h1:ILr54ISCqCQ6MmIwT7eaR/fEGrBfgfxiPt8nmpWqnUM=
|
||||
storj.io/drpc v0.0.11/go.mod h1:TiFc2obNjL9/3isMW1Rpxjy8V9uE0B2HMeMFGiiI7Iw=
|
||||
storj.io/drpc v0.0.11/go.mod h1:TiFc2obNjL9/3isMW1Rpxjy8V9uE0B2HMeMFGiiI7Iw=
|
||||
storj.io/drpc v0.0.13 h1:EDR3WiwVcIHtg+8M5vqBFmUAuJvmM2erVHIfqPPSAoc=
|
||||
|
@ -70,7 +70,7 @@ func TestAPI(t *testing.T) {
|
||||
require.Equal(t, http.StatusOK, response.StatusCode)
|
||||
require.NoError(t, response.Body.Close())
|
||||
|
||||
assertGet(t, link, `{"usage":{"amount":"1.0 GB","bytes":1000000000},"bandwidth":{"amount":"0 B","bytes":0},"rate":{"rps":0},"maxBuckets":0}`)
|
||||
assertGet(t, link, `{"usage":{"amount":"1.00 GB","bytes":1000000000},"bandwidth":{"amount":"0 B","bytes":0},"rate":{"rps":0},"maxBuckets":0}`)
|
||||
})
|
||||
|
||||
t.Run("UpdateBandwidth", func(t *testing.T) {
|
||||
@ -83,7 +83,7 @@ func TestAPI(t *testing.T) {
|
||||
require.Equal(t, http.StatusOK, response.StatusCode)
|
||||
require.NoError(t, response.Body.Close())
|
||||
|
||||
assertGet(t, link, `{"usage":{"amount":"1.0 GB","bytes":1000000000},"bandwidth":{"amount":"1.0 MB","bytes":1000000},"rate":{"rps":0},"maxBuckets":0}`)
|
||||
assertGet(t, link, `{"usage":{"amount":"1.00 GB","bytes":1000000000},"bandwidth":{"amount":"1.00 MB","bytes":1000000},"rate":{"rps":0},"maxBuckets":0}`)
|
||||
})
|
||||
|
||||
t.Run("UpdateRate", func(t *testing.T) {
|
||||
@ -96,7 +96,7 @@ func TestAPI(t *testing.T) {
|
||||
require.Equal(t, http.StatusOK, response.StatusCode)
|
||||
require.NoError(t, response.Body.Close())
|
||||
|
||||
assertGet(t, link, `{"usage":{"amount":"1.0 GB","bytes":1000000000},"bandwidth":{"amount":"1.0 MB","bytes":1000000},"rate":{"rps":100},"maxBuckets":0}`)
|
||||
assertGet(t, link, `{"usage":{"amount":"1.00 GB","bytes":1000000000},"bandwidth":{"amount":"1.00 MB","bytes":1000000},"rate":{"rps":100},"maxBuckets":0}`)
|
||||
})
|
||||
t.Run("UpdateBuckets", func(t *testing.T) {
|
||||
req, err := http.NewRequest(http.MethodPut, link+"?buckets=2000", nil)
|
||||
@ -108,7 +108,7 @@ func TestAPI(t *testing.T) {
|
||||
require.Equal(t, http.StatusOK, response.StatusCode)
|
||||
require.NoError(t, response.Body.Close())
|
||||
|
||||
assertGet(t, link, `{"usage":{"amount":"1.0 GB","bytes":1000000000},"bandwidth":{"amount":"1.0 MB","bytes":1000000},"rate":{"rps":100},"maxBuckets":2000}`)
|
||||
assertGet(t, link, `{"usage":{"amount":"1.00 GB","bytes":1000000000},"bandwidth":{"amount":"1.00 MB","bytes":1000000},"rate":{"rps":100},"maxBuckets":2000}`)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
@ -26,7 +26,7 @@ const (
|
||||
// RSConfig is a configuration struct that keeps details about default
|
||||
// redundancy strategy information.
|
||||
type RSConfig struct {
|
||||
MaxBufferMem memory.Size `help:"maximum buffer memory to be allocated for read buffers" default:"4MiB"`
|
||||
MaxBufferMem memory.Size `help:"maximum buffer memory to be allocated for read buffers" default:"4.00MiB"`
|
||||
ErasureShareSize memory.Size `help:"the size of each new erasure share in bytes" default:"256B"`
|
||||
MinThreshold int `help:"the minimum pieces required to recover a segment. k." releaseDefault:"29" devDefault:"4"`
|
||||
RepairThreshold int `help:"the minimum safe pieces before a repair is triggered. m." releaseDefault:"35" devDefault:"6"`
|
||||
@ -50,8 +50,8 @@ type RateLimiterConfig struct {
|
||||
// ProjectLimitConfig is a configuration struct for default project limits.
|
||||
type ProjectLimitConfig struct {
|
||||
MaxBuckets int `help:"max bucket count for a project." default:"100"`
|
||||
DefaultMaxUsage memory.Size `help:"the default storage usage limit" releaseDefault:"50GB" devDefault:"200GB"`
|
||||
DefaultMaxBandwidth memory.Size `help:"the default bandwidth usage limit" releaseDefault:"50GB" devDefault:"200GB"`
|
||||
DefaultMaxUsage memory.Size `help:"the default storage usage limit" releaseDefault:"50.00GB" devDefault:"200GB"`
|
||||
DefaultMaxBandwidth memory.Size `help:"the default bandwidth usage limit" releaseDefault:"50.00GB" devDefault:"200GB"`
|
||||
}
|
||||
|
||||
// Config is a configuration struct that is everything you need to start a metainfo.
|
||||
|
@ -35,7 +35,7 @@ type NodeSelectionConfig struct {
|
||||
MinimumVersion string `help:"the minimum node software version for node selection queries" default:""`
|
||||
OnlineWindow time.Duration `help:"the amount of time without seeing a node before its considered offline" default:"4h"`
|
||||
DistinctIP bool `help:"require distinct IPs when choosing nodes for upload" releaseDefault:"true" devDefault:"false"`
|
||||
MinimumDiskSpace memory.Size `help:"how much disk space a node at minimum must have to be selected for upload" default:"500MB"`
|
||||
MinimumDiskSpace memory.Size `help:"how much disk space a node at minimum must have to be selected for upload" default:"500.00MB"`
|
||||
|
||||
AuditReputationRepairWeight float64 `help:"weight to apply to audit reputation for total repair reputation calculation" default:"1.0"`
|
||||
AuditReputationUplinkWeight float64 `help:"weight to apply to audit reputation for total uplink reputation calculation" default:"1.0"`
|
||||
|
8
scripts/testdata/satellite-config.yaml.lock
vendored
Executable file → Normal file
8
scripts/testdata/satellite-config.yaml.lock
vendored
Executable file → Normal file
@ -368,10 +368,10 @@ identity.key-path: /root/.local/share/storj/identity/satellite/identity.key
|
||||
# metainfo.piece-deletion.request-timeout: 1m0s
|
||||
|
||||
# the default bandwidth usage limit
|
||||
# metainfo.project-limits.default-max-bandwidth: 50.0 GB
|
||||
# metainfo.project-limits.default-max-bandwidth: 50.00 GB
|
||||
|
||||
# the default storage usage limit
|
||||
# metainfo.project-limits.default-max-usage: 50.0 GB
|
||||
# metainfo.project-limits.default-max-usage: 50.00 GB
|
||||
|
||||
# max bucket count for a project.
|
||||
# metainfo.project-limits.max-buckets: 100
|
||||
@ -494,7 +494,7 @@ identity.key-path: /root/.local/share/storj/identity/satellite/identity.key
|
||||
# overlay.node.distinct-ip: true
|
||||
|
||||
# how much disk space a node at minimum must have to be selected for upload
|
||||
# overlay.node.minimum-disk-space: 500.0 MB
|
||||
# overlay.node.minimum-disk-space: 500.00 MB
|
||||
|
||||
# the minimum node software version for node selection queries
|
||||
# overlay.node.minimum-version: ""
|
||||
@ -602,7 +602,7 @@ identity.key-path: /root/.local/share/storj/identity/satellite/identity.key
|
||||
# repairer.interval: 5m0s
|
||||
|
||||
# maximum buffer memory (in bytes) to be allocated for read buffers
|
||||
# repairer.max-buffer-mem: 4.0 MB
|
||||
# repairer.max-buffer-mem: 4.00 MB
|
||||
|
||||
# ratio applied to the optimal threshold to calculate the excess of the maximum number of repaired pieces to upload
|
||||
# repairer.max-excess-rate-optimal-threshold: 0.05
|
||||
|
@ -294,6 +294,13 @@ type Satellites struct {
|
||||
EgressSummary int64 `json:"egressSummary"`
|
||||
IngressSummary int64 `json:"ingressSummary"`
|
||||
EarliestJoinedAt time.Time `json:"earliestJoinedAt"`
|
||||
Audits []Audits `json:"audits"`
|
||||
}
|
||||
|
||||
// Audits represents audit metrics across all satellites.
|
||||
type Audits struct {
|
||||
Audit reputation.Metric
|
||||
SatelliteID storj.NodeID
|
||||
}
|
||||
|
||||
// GetAllSatellitesData returns bandwidth and storage daily usage consolidate
|
||||
@ -302,6 +309,8 @@ func (s *Service) GetAllSatellitesData(ctx context.Context) (_ *Satellites, err
|
||||
defer mon.Task()(&ctx)(nil)
|
||||
from, to := date.MonthBoundary(time.Now().UTC())
|
||||
|
||||
var audits []Audits
|
||||
|
||||
bandwidthDaily, err := s.bandwidthDB.GetDailyRollups(ctx, from, to)
|
||||
if err != nil {
|
||||
return nil, SNOServiceErr.Wrap(err)
|
||||
@ -334,12 +343,17 @@ func (s *Service) GetAllSatellitesData(ctx context.Context) (_ *Satellites, err
|
||||
|
||||
satellitesIDs := s.trust.GetSatellites(ctx)
|
||||
joinedAt := time.Now().UTC()
|
||||
|
||||
for i := 0; i < len(satellitesIDs); i++ {
|
||||
stats, err := s.reputationDB.Get(ctx, satellitesIDs[i])
|
||||
if err != nil {
|
||||
return nil, SNOServiceErr.Wrap(err)
|
||||
}
|
||||
|
||||
audits = append(audits, Audits{
|
||||
Audit: stats.Audit,
|
||||
SatelliteID: satellitesIDs[i],
|
||||
})
|
||||
if !stats.JoinedAt.IsZero() && stats.JoinedAt.Before(joinedAt) {
|
||||
joinedAt = stats.JoinedAt
|
||||
}
|
||||
@ -353,6 +367,7 @@ func (s *Service) GetAllSatellitesData(ctx context.Context) (_ *Satellites, err
|
||||
EgressSummary: egressSummary.Total(),
|
||||
IngressSummary: ingressSummary.Total(),
|
||||
EarliestJoinedAt: joinedAt,
|
||||
Audits: audits,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user