JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

storagenode/console: added list of all audits per satellite to sno dashboard/satellites

Browse Source 
Change-Id: I52e58748d6467f372d9a308347fc77e400d137e2
This commit is contained in:
Qweder93 2020-08-10 13:13:21 +03:00 committed by Nikolai Siedov
parent 373934efb2
commit 4ee1b2d45a
7 changed files with 30 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -41,7 +41,7 @@ require (
golang.org/x/sys v0.0.0-20200610111108-226ff32320da golang.org/x/sys v0.0.0-20200610111108-226ff32320da
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 golang.org/x/time v0.0.0-20191024005414-555d28b269f0
golang.org/x/tools v0.0.0-20200428211428-0c9eba77bc32 // indirect golang.org/x/tools v0.0.0-20200428211428-0c9eba77bc32 // indirect
storj.io/common v0.0.0-20200729140050-4c1ddac6fa63 storj.io/common v0.0.0-20200810083036-275dbbe0bd21
storj.io/drpc v0.0.14 storj.io/drpc v0.0.14
storj.io/monkit-jaeger v0.0.0-20200518165323-80778fc3f91b storj.io/monkit-jaeger v0.0.0-20200518165323-80778fc3f91b
storj.io/private v0.0.0-20200729145012-46794d335b51 storj.io/private v0.0.0-20200729145012-46794d335b51

2
go.sum
View File

@ -714,6 +714,8 @@ storj.io/common v0.0.0-20200710113903-e5030388ee6a h1:8lMuSiBcU3u6Y3YzlG53GKvyG0
storj.io/common v0.0.0-20200710113903-e5030388ee6a/go.mod h1:vMAnlNbkgW6i+w/OT1h4X8w6TajOHWAT+SvFHUFCpq0= storj.io/common v0.0.0-20200710113903-e5030388ee6a/go.mod h1:vMAnlNbkgW6i+w/OT1h4X8w6TajOHWAT+SvFHUFCpq0=
storj.io/common v0.0.0-20200729140050-4c1ddac6fa63 h1:BkRvlginTJGi0yAkpN+4ZKm2YpG63bDSDFLQtXYxxdg= storj.io/common v0.0.0-20200729140050-4c1ddac6fa63 h1:BkRvlginTJGi0yAkpN+4ZKm2YpG63bDSDFLQtXYxxdg=
storj.io/common v0.0.0-20200729140050-4c1ddac6fa63/go.mod h1:ILr54ISCqCQ6MmIwT7eaR/fEGrBfgfxiPt8nmpWqnUM= storj.io/common v0.0.0-20200729140050-4c1ddac6fa63/go.mod h1:ILr54ISCqCQ6MmIwT7eaR/fEGrBfgfxiPt8nmpWqnUM=
storj.io/common v0.0.0-20200810083036-275dbbe0bd21 h1:Qjrd4KslLBdI76VHwO6+S7tkkYa6/K/IgPFzTvA6M8k=
storj.io/common v0.0.0-20200810083036-275dbbe0bd21/go.mod h1:ILr54ISCqCQ6MmIwT7eaR/fEGrBfgfxiPt8nmpWqnUM=
storj.io/drpc v0.0.11/go.mod h1:TiFc2obNjL9/3isMW1Rpxjy8V9uE0B2HMeMFGiiI7Iw= storj.io/drpc v0.0.11/go.mod h1:TiFc2obNjL9/3isMW1Rpxjy8V9uE0B2HMeMFGiiI7Iw=
storj.io/drpc v0.0.11/go.mod h1:TiFc2obNjL9/3isMW1Rpxjy8V9uE0B2HMeMFGiiI7Iw= storj.io/drpc v0.0.11/go.mod h1:TiFc2obNjL9/3isMW1Rpxjy8V9uE0B2HMeMFGiiI7Iw=
storj.io/drpc v0.0.13 h1:EDR3WiwVcIHtg+8M5vqBFmUAuJvmM2erVHIfqPPSAoc= storj.io/drpc v0.0.13 h1:EDR3WiwVcIHtg+8M5vqBFmUAuJvmM2erVHIfqPPSAoc=

8
satellite/admin/project_test.go
View File

@ -70,7 +70,7 @@ func TestAPI(t *testing.T) {
require.Equal(t, http.StatusOK, response.StatusCode) require.Equal(t, http.StatusOK, response.StatusCode)
require.NoError(t, response.Body.Close()) require.NoError(t, response.Body.Close())
assertGet(t, link, `{"usage":{"amount":"1.0 GB","bytes":1000000000},"bandwidth":{"amount":"0 B","bytes":0},"rate":{"rps":0},"maxBuckets":0}`) assertGet(t, link, `{"usage":{"amount":"1.00 GB","bytes":1000000000},"bandwidth":{"amount":"0 B","bytes":0},"rate":{"rps":0},"maxBuckets":0}`)
}) })
t.Run("UpdateBandwidth", func(t *testing.T) { t.Run("UpdateBandwidth", func(t *testing.T) {
@ -83,7 +83,7 @@ func TestAPI(t *testing.T) {
require.Equal(t, http.StatusOK, response.StatusCode) require.Equal(t, http.StatusOK, response.StatusCode)
require.NoError(t, response.Body.Close()) require.NoError(t, response.Body.Close())
assertGet(t, link, `{"usage":{"amount":"1.0 GB","bytes":1000000000},"bandwidth":{"amount":"1.0 MB","bytes":1000000},"rate":{"rps":0},"maxBuckets":0}`) assertGet(t, link, `{"usage":{"amount":"1.00 GB","bytes":1000000000},"bandwidth":{"amount":"1.00 MB","bytes":1000000},"rate":{"rps":0},"maxBuckets":0}`)
}) })
t.Run("UpdateRate", func(t *testing.T) { t.Run("UpdateRate", func(t *testing.T) {
@ -96,7 +96,7 @@ func TestAPI(t *testing.T) {
require.Equal(t, http.StatusOK, response.StatusCode) require.Equal(t, http.StatusOK, response.StatusCode)
require.NoError(t, response.Body.Close()) require.NoError(t, response.Body.Close())
assertGet(t, link, `{"usage":{"amount":"1.0 GB","bytes":1000000000},"bandwidth":{"amount":"1.0 MB","bytes":1000000},"rate":{"rps":100},"maxBuckets":0}`) assertGet(t, link, `{"usage":{"amount":"1.00 GB","bytes":1000000000},"bandwidth":{"amount":"1.00 MB","bytes":1000000},"rate":{"rps":100},"maxBuckets":0}`)
}) })
t.Run("UpdateBuckets", func(t *testing.T) { t.Run("UpdateBuckets", func(t *testing.T) {
req, err := http.NewRequest(http.MethodPut, link+"?buckets=2000", nil) req, err := http.NewRequest(http.MethodPut, link+"?buckets=2000", nil)
@ -108,7 +108,7 @@ func TestAPI(t *testing.T) {
require.Equal(t, http.StatusOK, response.StatusCode) require.Equal(t, http.StatusOK, response.StatusCode)
require.NoError(t, response.Body.Close()) require.NoError(t, response.Body.Close())
assertGet(t, link, `{"usage":{"amount":"1.0 GB","bytes":1000000000},"bandwidth":{"amount":"1.0 MB","bytes":1000000},"rate":{"rps":100},"maxBuckets":2000}`) assertGet(t, link, `{"usage":{"amount":"1.00 GB","bytes":1000000000},"bandwidth":{"amount":"1.00 MB","bytes":1000000},"rate":{"rps":100},"maxBuckets":2000}`)
}) })
}) })
} }

6
satellite/metainfo/config.go
View File

@ -26,7 +26,7 @@ const (
// RSConfig is a configuration struct that keeps details about default // RSConfig is a configuration struct that keeps details about default
// redundancy strategy information. // redundancy strategy information.
type RSConfig struct { type RSConfig struct {
MaxBufferMem memory.Size `help:"maximum buffer memory to be allocated for read buffers" default:"4MiB"` MaxBufferMem memory.Size `help:"maximum buffer memory to be allocated for read buffers" default:"4.00MiB"`
ErasureShareSize memory.Size `help:"the size of each new erasure share in bytes" default:"256B"` ErasureShareSize memory.Size `help:"the size of each new erasure share in bytes" default:"256B"`
MinThreshold int `help:"the minimum pieces required to recover a segment. k." releaseDefault:"29" devDefault:"4"` MinThreshold int `help:"the minimum pieces required to recover a segment. k." releaseDefault:"29" devDefault:"4"`
RepairThreshold int `help:"the minimum safe pieces before a repair is triggered. m." releaseDefault:"35" devDefault:"6"` RepairThreshold int `help:"the minimum safe pieces before a repair is triggered. m." releaseDefault:"35" devDefault:"6"`
@ -50,8 +50,8 @@ type RateLimiterConfig struct {
// ProjectLimitConfig is a configuration struct for default project limits. // ProjectLimitConfig is a configuration struct for default project limits.
type ProjectLimitConfig struct { type ProjectLimitConfig struct {
MaxBuckets int `help:"max bucket count for a project." default:"100"` MaxBuckets int `help:"max bucket count for a project." default:"100"`
DefaultMaxUsage memory.Size `help:"the default storage usage limit" releaseDefault:"50GB" devDefault:"200GB"` DefaultMaxUsage memory.Size `help:"the default storage usage limit" releaseDefault:"50.00GB" devDefault:"200GB"`
DefaultMaxBandwidth memory.Size `help:"the default bandwidth usage limit" releaseDefault:"50GB" devDefault:"200GB"` DefaultMaxBandwidth memory.Size `help:"the default bandwidth usage limit" releaseDefault:"50.00GB" devDefault:"200GB"`
} }
// Config is a configuration struct that is everything you need to start a metainfo. // Config is a configuration struct that is everything you need to start a metainfo.

2
satellite/overlay/config.go
View File

@ -35,7 +35,7 @@ type NodeSelectionConfig struct {
MinimumVersion string `help:"the minimum node software version for node selection queries" default:""` MinimumVersion string `help:"the minimum node software version for node selection queries" default:""`
OnlineWindow time.Duration `help:"the amount of time without seeing a node before its considered offline" default:"4h"` OnlineWindow time.Duration `help:"the amount of time without seeing a node before its considered offline" default:"4h"`
DistinctIP bool `help:"require distinct IPs when choosing nodes for upload" releaseDefault:"true" devDefault:"false"` DistinctIP bool `help:"require distinct IPs when choosing nodes for upload" releaseDefault:"true" devDefault:"false"`
MinimumDiskSpace memory.Size `help:"how much disk space a node at minimum must have to be selected for upload" default:"500MB"` MinimumDiskSpace memory.Size `help:"how much disk space a node at minimum must have to be selected for upload" default:"500.00MB"`
AuditReputationRepairWeight float64 `help:"weight to apply to audit reputation for total repair reputation calculation" default:"1.0"` AuditReputationRepairWeight float64 `help:"weight to apply to audit reputation for total repair reputation calculation" default:"1.0"`
AuditReputationUplinkWeight float64 `help:"weight to apply to audit reputation for total uplink reputation calculation" default:"1.0"` AuditReputationUplinkWeight float64 `help:"weight to apply to audit reputation for total uplink reputation calculation" default:"1.0"`

8
scripts/testdata/satellite-config.yaml.lock vendored Executable file → Normal file
View File

@ -368,10 +368,10 @@ identity.key-path: /root/.local/share/storj/identity/satellite/identity.key
# metainfo.piece-deletion.request-timeout: 1m0s # metainfo.piece-deletion.request-timeout: 1m0s
# the default bandwidth usage limit # the default bandwidth usage limit
# metainfo.project-limits.default-max-bandwidth: 50.0 GB # metainfo.project-limits.default-max-bandwidth: 50.00 GB
# the default storage usage limit # the default storage usage limit
# metainfo.project-limits.default-max-usage: 50.0 GB # metainfo.project-limits.default-max-usage: 50.00 GB
# max bucket count for a project. # max bucket count for a project.
# metainfo.project-limits.max-buckets: 100 # metainfo.project-limits.max-buckets: 100
@ -494,7 +494,7 @@ identity.key-path: /root/.local/share/storj/identity/satellite/identity.key
# overlay.node.distinct-ip: true # overlay.node.distinct-ip: true
# how much disk space a node at minimum must have to be selected for upload # how much disk space a node at minimum must have to be selected for upload
# overlay.node.minimum-disk-space: 500.0 MB # overlay.node.minimum-disk-space: 500.00 MB
# the minimum node software version for node selection queries # the minimum node software version for node selection queries
# overlay.node.minimum-version: "" # overlay.node.minimum-version: ""
@ -602,7 +602,7 @@ identity.key-path: /root/.local/share/storj/identity/satellite/identity.key
# repairer.interval: 5m0s # repairer.interval: 5m0s
# maximum buffer memory (in bytes) to be allocated for read buffers # maximum buffer memory (in bytes) to be allocated for read buffers
# repairer.max-buffer-mem: 4.0 MB # repairer.max-buffer-mem: 4.00 MB
# ratio applied to the optimal threshold to calculate the excess of the maximum number of repaired pieces to upload # ratio applied to the optimal threshold to calculate the excess of the maximum number of repaired pieces to upload
# repairer.max-excess-rate-optimal-threshold: 0.05 # repairer.max-excess-rate-optimal-threshold: 0.05

15
storagenode/console/service.go
View File

@ -294,6 +294,13 @@ type Satellites struct {
EgressSummary int64 `json:"egressSummary"` EgressSummary int64 `json:"egressSummary"`
IngressSummary int64 `json:"ingressSummary"` IngressSummary int64 `json:"ingressSummary"`
EarliestJoinedAt time.Time `json:"earliestJoinedAt"` EarliestJoinedAt time.Time `json:"earliestJoinedAt"`
Audits []Audits `json:"audits"`
}
// Audits represents audit metrics across all satellites.
type Audits struct {
Audit reputation.Metric
SatelliteID storj.NodeID
} }
// GetAllSatellitesData returns bandwidth and storage daily usage consolidate // GetAllSatellitesData returns bandwidth and storage daily usage consolidate
@ -302,6 +309,8 @@ func (s *Service) GetAllSatellitesData(ctx context.Context) (_ *Satellites, err
defer mon.Task()(&ctx)(nil) defer mon.Task()(&ctx)(nil)
from, to := date.MonthBoundary(time.Now().UTC()) from, to := date.MonthBoundary(time.Now().UTC())
var audits []Audits
bandwidthDaily, err := s.bandwidthDB.GetDailyRollups(ctx, from, to) bandwidthDaily, err := s.bandwidthDB.GetDailyRollups(ctx, from, to)
if err != nil { if err != nil {
return nil, SNOServiceErr.Wrap(err) return nil, SNOServiceErr.Wrap(err)
@ -334,12 +343,17 @@ func (s *Service) GetAllSatellitesData(ctx context.Context) (_ *Satellites, err
satellitesIDs := s.trust.GetSatellites(ctx) satellitesIDs := s.trust.GetSatellites(ctx)
joinedAt := time.Now().UTC() joinedAt := time.Now().UTC()
for i := 0; i < len(satellitesIDs); i++ { for i := 0; i < len(satellitesIDs); i++ {
stats, err := s.reputationDB.Get(ctx, satellitesIDs[i]) stats, err := s.reputationDB.Get(ctx, satellitesIDs[i])
if err != nil { if err != nil {
return nil, SNOServiceErr.Wrap(err) return nil, SNOServiceErr.Wrap(err)
} }
audits = append(audits, Audits{
Audit: stats.Audit,
SatelliteID: satellitesIDs[i],
})
if !stats.JoinedAt.IsZero() && stats.JoinedAt.Before(joinedAt) { if !stats.JoinedAt.IsZero() && stats.JoinedAt.Before(joinedAt) {
joinedAt = stats.JoinedAt joinedAt = stats.JoinedAt
} }
@ -353,6 +367,7 @@ func (s *Service) GetAllSatellitesData(ctx context.Context) (_ *Satellites, err
EgressSummary: egressSummary.Total(), EgressSummary: egressSummary.Total(),
IngressSummary: ingressSummary.Total(), IngressSummary: ingressSummary.Total(),
EarliestJoinedAt: joinedAt, EarliestJoinedAt: joinedAt,
Audits: audits,
}, nil }, nil
} }