JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/metainfo: support a wider range of values for RS.Total in satellite metainfo validation (#3431)

Browse Source 
change uplink RS default configuration from 130 to 95
This commit is contained in:
Maximillian von Briesen 2019-10-31 15:04:33 -04:00 committed by Yingrong Zhao
parent 8ce09700af
commit d9bb25b4b9
7 changed files with 73 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
internal/testplanet/satellite.go
View File

@ -290,14 +290,15 @@ func (planet *Planet) newSatellites(count int) ([]*SatelliteSystem, error) {
MaxCommitInterval: 1 * time.Hour,
Overlay: true,
RS: metainfo.RSConfig{
MaxSegmentSize: 64 * memory.MiB,
MaxBufferMem: memory.Size(256),
ErasureShareSize: memory.Size(256),
MinThreshold: (planet.config.StorageNodeCount * 1 / 5),
RepairThreshold: (planet.config.StorageNodeCount * 2 / 5),
SuccessThreshold: (planet.config.StorageNodeCount * 3 / 5),
MaxThreshold: (planet.config.StorageNodeCount * 4 / 5),
Validate: false,
MaxSegmentSize: 64 * memory.MiB,
MaxBufferMem: memory.Size(256),
ErasureShareSize: memory.Size(256),
MinThreshold: (planet.config.StorageNodeCount * 1 / 5),
RepairThreshold: (planet.config.StorageNodeCount * 2 / 5),
SuccessThreshold: (planet.config.StorageNodeCount * 3 / 5),
MinTotalThreshold: (planet.config.StorageNodeCount * 4 / 5),
MaxTotalThreshold: (planet.config.StorageNodeCount * 4 / 5),
Validate: false,
},
Loop: metainfo.LoopConfig{
CoalesceDuration: 1 * time.Second,

2
lib/uplink/project.go
View File

@ -79,7 +79,7 @@ func (cfg *BucketConfig) setDefaults() {
cfg.Volatile.RedundancyScheme.OptimalShares = 80
}
if cfg.Volatile.RedundancyScheme.TotalShares == 0 {
cfg.Volatile.RedundancyScheme.TotalShares = 130
cfg.Volatile.RedundancyScheme.TotalShares = 95
}
if cfg.Volatile.RedundancyScheme.ShareSize == 0 {
cfg.Volatile.RedundancyScheme.ShareSize = 256 * memory.B.Int32()

17
satellite/metainfo/config.go
View File

@ -23,14 +23,15 @@ const (
// RSConfig is a configuration struct that keeps details about default
// redundancy strategy information
type RSConfig struct {
MaxSegmentSize memory.Size `help:"maximum segment size" default:"64MiB"`
MaxBufferMem memory.Size `help:"maximum buffer memory to be allocated for read buffers" default:"4MiB"`
ErasureShareSize memory.Size `help:"the size of each new erasure share in bytes" default:"256B"`
MinThreshold int `help:"the minimum pieces required to recover a segment. k." releaseDefault:"29" devDefault:"4"`
RepairThreshold int `help:"the minimum safe pieces before a repair is triggered. m." releaseDefault:"35" devDefault:"6"`
SuccessThreshold int `help:"the desired total pieces for a segment. o." releaseDefault:"80" devDefault:"8"`
MaxThreshold int `help:"the largest amount of pieces to encode to. n." releaseDefault:"130" devDefault:"10"`
Validate bool `help:"validate redundancy scheme configuration" default:"true"`
MaxSegmentSize memory.Size `help:"maximum segment size" default:"64MiB"`
MaxBufferMem memory.Size `help:"maximum buffer memory to be allocated for read buffers" default:"4MiB"`
ErasureShareSize memory.Size `help:"the size of each new erasure share in bytes" default:"256B"`
MinThreshold int `help:"the minimum pieces required to recover a segment. k." releaseDefault:"29" devDefault:"4"`
RepairThreshold int `help:"the minimum safe pieces before a repair is triggered. m." releaseDefault:"35" devDefault:"6"`
SuccessThreshold int `help:"the desired total pieces for a segment. o." releaseDefault:"80" devDefault:"8"`
MinTotalThreshold int `help:"the largest amount of pieces to encode to. n (lower bound for validation)." releaseDefault:"95" devDefault:"10"`
MaxTotalThreshold int `help:"the largest amount of pieces to encode to. n (upper bound for validation)." releaseDefault:"130" devDefault:"10"`
Validate bool `help:"validate redundancy scheme configuration" default:"true"`
}
// Config is a configuration struct that is everything you need to start a metainfo

43
satellite/metainfo/metainfo_test.go
View File

@ -28,6 +28,7 @@ import (
"storj.io/storj/pkg/signing"
"storj.io/storj/pkg/storj"
"storj.io/storj/satellite"
"storj.io/storj/uplink"
"storj.io/storj/uplink/eestream"
"storj.io/storj/uplink/metainfo"
)
@ -1681,3 +1682,45 @@ func TestBatch(t *testing.T) {
}
})
}
func TestValidateRS(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1, StorageNodeCount: 6, UplinkCount: 1,
Reconfigure: testplanet.Reconfigure{
Satellite: func(log *zap.Logger, index int, config *satellite.Config) {
config.Metainfo.RS.MinTotalThreshold = 4
config.Metainfo.RS.MaxTotalThreshold = 5
config.Metainfo.RS.Validate = true
},
},
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
ul := planet.Uplinks[0]
satellite := planet.Satellites[0]
testData := testrand.Bytes(8 * memory.KiB)
rs := &uplink.RSConfig{
MinThreshold: 1,
RepairThreshold: 2,
SuccessThreshold: 3,
MaxThreshold: 3,
}
// test below permitted total value
err := ul.UploadWithConfig(ctx, satellite, rs, "testbucket", "test/path/below", testData)
require.Error(t, err)
// test above permitted total value
rs.MaxThreshold = 6
err = ul.UploadWithConfig(ctx, satellite, rs, "testbucket", "test/path/above", testData)
require.Error(t, err)
// test minimum permitted total value
rs.MaxThreshold = 4
err = ul.UploadWithConfig(ctx, satellite, rs, "testbucket", "test/path/min", testData)
require.NoError(t, err)
// test maximum permitted total value
rs.MaxThreshold = 5
err = ul.UploadWithConfig(ctx, satellite, rs, "testbucket", "test/path/max", testData)
require.NoError(t, err)
})
}

8
satellite/metainfo/validation.go
View File

@ -343,15 +343,17 @@ func (endpoint *Endpoint) validateRedundancy(ctx context.Context, redundancy *pb
if endpoint.requiredRSConfig.Validate {
if endpoint.requiredRSConfig.ErasureShareSize.Int32() != redundancy.ErasureShareSize ||
endpoint.requiredRSConfig.MaxThreshold != int(redundancy.Total) ||
endpoint.requiredRSConfig.MinTotalThreshold > int(redundancy.Total) ||
endpoint.requiredRSConfig.MaxTotalThreshold < int(redundancy.Total) ||
endpoint.requiredRSConfig.MinThreshold != int(redundancy.MinReq) ||
endpoint.requiredRSConfig.RepairThreshold != int(redundancy.RepairThreshold) ||
endpoint.requiredRSConfig.SuccessThreshold != int(redundancy.SuccessThreshold) {
return Error.New("provided redundancy scheme parameters not allowed: want [%d, %d, %d, %d, %d] got [%d, %d, %d, %d, %d]",
return Error.New("provided redundancy scheme parameters not allowed: want [%d, %d, %d, %d-%d, %d] got [%d, %d, %d, %d, %d]",
endpoint.requiredRSConfig.MinThreshold,
endpoint.requiredRSConfig.RepairThreshold,
endpoint.requiredRSConfig.SuccessThreshold,
endpoint.requiredRSConfig.MaxThreshold,
endpoint.requiredRSConfig.MinTotalThreshold,
endpoint.requiredRSConfig.MaxTotalThreshold,
endpoint.requiredRSConfig.ErasureShareSize.Int32(),
redundancy.MinReq,

7
scripts/testdata/satellite-config.yaml.lock vendored
View File

@ -223,12 +223,15 @@ identity.key-path: /root/.local/share/storj/identity/satellite/identity.key
# maximum segment size
# metainfo.rs.max-segment-size: 64.0 MiB
# the largest amount of pieces to encode to. n.
# metainfo.rs.max-threshold: 130
# the largest amount of pieces to encode to. n (upper bound for validation).
# metainfo.rs.max-total-threshold: 130
# the minimum pieces required to recover a segment. k.
# metainfo.rs.min-threshold: 29
# the largest amount of pieces to encode to. n (lower bound for validation).
# metainfo.rs.min-total-threshold: 95
# the minimum safe pieces before a repair is triggered. m.
# metainfo.rs.repair-threshold: 35

2
uplink/config.go
View File

@ -27,7 +27,7 @@ type RSConfig struct {
MinThreshold int `help:"the minimum pieces required to recover a segment. k." releaseDefault:"29" devDefault:"4" hidden:"true"`
RepairThreshold int `help:"the minimum safe pieces before a repair is triggered. m." releaseDefault:"35" devDefault:"6" hidden:"true"`
SuccessThreshold int `help:"the desired total pieces for a segment. o." releaseDefault:"80" devDefault:"8" hidden:"true"`
MaxThreshold int `help:"the largest amount of pieces to encode to. n." releaseDefault:"130" devDefault:"10" hidden:"true"`
MaxThreshold int `help:"the largest amount of pieces to encode to. n." releaseDefault:"95" devDefault:"10" hidden:"true"`
}
// EncryptionConfig is a configuration struct that keeps details about