JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/metainfo: add max segment size and max inline size to

Browse Source 
BeginObject response

We want to control inline segment size and segment size on satellite
side. We need to return such information to uplink like with redundancy
scheme.

Change-Id: If04b0a45a2757a01c0cc046432c115f475e9323c
This commit is contained in:
Michal Niewrzal 2020-04-01 11:15:24 +02:00
parent 4a79b609e9
commit c178a08cb8
9 changed files with 71 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -43,7 +43,7 @@ require (
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5
golang.org/x/time v0.0.0-20191024005414-555d28b269f0
google.golang.org/grpc v1.28.0
storj.io/common v0.0.0-20200401062549-3fa2945ebe52
storj.io/common v0.0.0-20200401122855-4ad28fd2218f
storj.io/drpc v0.0.11
storj.io/private v0.0.0-20200327035409-e9d82e7e0c6b
storj.io/uplink v1.0.3

6
go.sum
View File

@ -623,8 +623,10 @@ storj.io/common v0.0.0-20200323134045-2bd4d6e2dd7d/go.mod h1:I0QTs7z1rI+ZEN95GGY
storj.io/common v0.0.0-20200331095257-30ebbdbbba88/go.mod h1:RBaNRmk/lqyZ7h1MAH4N9zld0z+tO4M9sLOFT30K+cE=
storj.io/common v0.0.0-20200331124657-a4f8265946f2 h1:Fctei5lPPAfbvtpijLQZTjQeeuh+MCkacLYau7nyxKA=
storj.io/common v0.0.0-20200331124657-a4f8265946f2/go.mod h1:RBaNRmk/lqyZ7h1MAH4N9zld0z+tO4M9sLOFT30K+cE=
storj.io/common v0.0.0-20200401062549-3fa2945ebe52 h1:Oulb5CY00odkUrddYONRYkr7FefoJx1mbiIh8kOTh+0=
storj.io/common v0.0.0-20200401062549-3fa2945ebe52/go.mod h1:RBaNRmk/lqyZ7h1MAH4N9zld0z+tO4M9sLOFT30K+cE=
storj.io/common v0.0.0-20200401095230-4fe9b2ad3ec0 h1:gFsVck24e/eU3j9qmfXwsyBUwvtcjuTD5JXvvv9NdFE=
storj.io/common v0.0.0-20200401095230-4fe9b2ad3ec0/go.mod h1:RBaNRmk/lqyZ7h1MAH4N9zld0z+tO4M9sLOFT30K+cE=
storj.io/common v0.0.0-20200401122855-4ad28fd2218f h1:17QhsiCE8PFKoVaxRk+8tfGazlm2r5mMsXEtFTT2F9w=
storj.io/common v0.0.0-20200401122855-4ad28fd2218f/go.mod h1:RBaNRmk/lqyZ7h1MAH4N9zld0z+tO4M9sLOFT30K+cE=
storj.io/drpc v0.0.7-0.20191115031725-2171c57838d2/go.mod h1:/ascUDbzNAv0A3Jj7wUIKFBH2JdJ2uJIBO/b9+2yHgQ=
storj.io/drpc v0.0.11 h1:6vLxfpSbwCLtqzAoXzXx/SxBqBtbzbmquXPqfcWKqfw=
storj.io/drpc v0.0.11/go.mod h1:TiFc2obNjL9/3isMW1Rpxjy8V9uE0B2HMeMFGiiI7Iw=

4
private/testplanet/satellite.go
View File

@ -332,11 +332,11 @@ func (planet *Planet) newSatellites(count int, satelliteDatabases satellitedbtes
Metainfo: metainfo.Config{
DatabaseURL: "", // not used
MinRemoteSegmentSize: 0, // TODO: fix tests to work with 1024
MaxInlineSegmentSize: 8000,
MaxInlineSegmentSize: 4 * memory.KiB,
MaxSegmentSize: 64 * memory.MiB,
MaxCommitInterval: 1 * time.Hour,
Overlay: true,
RS: metainfo.RSConfig{
MaxSegmentSize: 64 * memory.MiB,
MaxBufferMem: memory.Size(256),
ErasureShareSize: memory.Size(256),
MinThreshold: atLeastOne(planet.config.StorageNodeCount * 1 / 5),

4
satellite/api.go
View File

@ -425,10 +425,8 @@ func NewAPI(log *zap.Logger, full *identity.FullIdentity, db DB,
peer.DB.Console().APIKeys(),
peer.Accounting.ProjectUsage,
peer.DB.Console().Projects(),
config.Metainfo.RS,
signing.SignerFromFullIdentity(peer.Identity),
config.Metainfo.MaxCommitInterval,
config.Metainfo.RateLimiter,
config.Metainfo,
)
pbgrpc.RegisterMetainfoServer(peer.Server.GRPC(), peer.Metainfo.Endpoint2)
if err := pb.DRPCRegisterMetainfo(peer.Server.DRPC(), peer.Metainfo.Endpoint2); err != nil {

4
satellite/metainfo/config.go
View File

@ -24,7 +24,6 @@ const (
// RSConfig is a configuration struct that keeps details about default
// redundancy strategy information
type RSConfig struct {
MaxSegmentSize memory.Size `help:"maximum segment size" default:"64MiB"`
MaxBufferMem memory.Size `help:"maximum buffer memory to be allocated for read buffers" default:"4MiB"`
ErasureShareSize memory.Size `help:"the size of each new erasure share in bytes" default:"256B"`
MinThreshold int `help:"the minimum pieces required to recover a segment. k." releaseDefault:"29" devDefault:"4"`
@ -50,7 +49,8 @@ type RateLimiterConfig struct {
type Config struct {
DatabaseURL string `help:"the database connection string to use" default:"postgres://"`
MinRemoteSegmentSize memory.Size `default:"1240" help:"minimum remote segment size"`
MaxInlineSegmentSize memory.Size `default:"8000" help:"maximum inline segment size"`
MaxInlineSegmentSize memory.Size `default:"4KiB" help:"maximum inline segment size"`
MaxSegmentSize memory.Size `default:"64MiB" help:"maximum segment size"`
MaxCommitInterval time.Duration `default:"48h" help:"maximum time allowed to pass between creating and committing a segment"`
Overlay bool `default:"true" help:"toggle flag if overlay is enabled"`
RS RSConfig `help:"redundancy scheme configuration"`

44
satellite/metainfo/metainfo.go
View File

@ -83,11 +83,9 @@ type Endpoint struct {
projects console.Projects
apiKeys APIKeys
createRequests *createRequests
requiredRSConfig RSConfig
satellite signing.Signer
maxCommitInterval time.Duration
limiterCache *lrucache.ExpiringLRU
limiterConfig RateLimiterConfig
config Config
}
// NewEndpoint creates new metainfo endpoint instance.
@ -95,8 +93,7 @@ func NewEndpoint(log *zap.Logger, metainfo *Service, deletePieces *piecedeletion
orders *orders.Service, cache *overlay.Service, attributions attribution.DB,
partners *rewards.PartnersService, peerIdentities overlay.PeerIdentities,
apiKeys APIKeys, projectUsage *accounting.Service, projects console.Projects,
rsConfig RSConfig, satellite signing.Signer, maxCommitInterval time.Duration,
limiterConfig RateLimiterConfig) *Endpoint {
satellite signing.Signer, config Config) *Endpoint {
// TODO do something with too many params
return &Endpoint{
log: log,
@ -111,14 +108,12 @@ func NewEndpoint(log *zap.Logger, metainfo *Service, deletePieces *piecedeletion
projectUsage: projectUsage,
projects: projects,
createRequests: newCreateRequests(),
requiredRSConfig: rsConfig,
satellite: satellite,
maxCommitInterval: maxCommitInterval,
limiterCache: lrucache.New(lrucache.Options{
Capacity: limiterConfig.CacheCapacity,
Expiration: limiterConfig.CacheExpiration,
Capacity: config.RateLimiter.CacheCapacity,
Expiration: config.RateLimiter.CacheExpiration,
}),
limiterConfig: limiterConfig,
config: config,
}
}
@ -1053,11 +1048,13 @@ func (endpoint *Endpoint) BeginObject(ctx context.Context, req *pb.ObjectBeginRe
mon.Meter("req_put_object").Mark(1)
return &pb.ObjectBeginResponse{
Bucket: req.Bucket,
EncryptedPath: req.EncryptedPath,
Version: req.Version,
StreamId: streamID,
RedundancyScheme: pbRS,
Bucket: req.Bucket,
EncryptedPath: req.EncryptedPath,
Version: req.Version,
StreamId: streamID,
RedundancyScheme: pbRS,
MaxInlineSegmentSize: endpoint.config.MaxInlineSegmentSize.Int64(),
MaxSegmentSize: endpoint.config.MaxSegmentSize.Int64(),
}, nil
}
@ -1697,6 +1694,11 @@ func (endpoint *Endpoint) makeInlineSegment(ctx context.Context, req *pb.Segment
return nil, nil, rpcstatus.Error(rpcstatus.InvalidArgument, "segment index must be greater then 0")
}
inlineUsed := int64(len(req.EncryptedInlineData))
if inlineUsed > endpoint.config.MaxInlineSegmentSize.Int64() {
return nil, nil, rpcstatus.Error(rpcstatus.InvalidArgument, fmt.Sprintf("inline segment size cannot be larger than %s", endpoint.config.MaxInlineSegmentSize))
}
exceeded, limit, err := endpoint.projectUsage.ExceedsStorageUsage(ctx, keyInfo.ProjectID)
if err != nil {
return nil, nil, rpcstatus.Error(rpcstatus.Internal, err.Error())
@ -1708,8 +1710,6 @@ func (endpoint *Endpoint) makeInlineSegment(ctx context.Context, req *pb.Segment
return nil, nil, rpcstatus.Error(rpcstatus.ResourceExhausted, "Exceeded Usage Limit")
}
inlineUsed := int64(len(req.EncryptedInlineData))
if err := endpoint.projectUsage.AddProjectStorageUsage(ctx, keyInfo.ProjectID, inlineUsed); err != nil {
endpoint.log.Sugar().Errorf("Could not track new storage usage by project %v: %v", keyInfo.ProjectID, err)
// but continue. it's most likely our own fault that we couldn't track it, and the only thing
@ -2502,10 +2502,10 @@ func (endpoint *Endpoint) findIndexPreviousLastSegmentWhenNotKnowingNumSegments(
func (endpoint *Endpoint) redundancyScheme() *pb.RedundancyScheme {
return &pb.RedundancyScheme{
Type: pb.RedundancyScheme_RS,
MinReq: int32(endpoint.requiredRSConfig.MinThreshold),
RepairThreshold: int32(endpoint.requiredRSConfig.RepairThreshold),
SuccessThreshold: int32(endpoint.requiredRSConfig.SuccessThreshold),
Total: int32(endpoint.requiredRSConfig.TotalThreshold),
ErasureShareSize: endpoint.requiredRSConfig.ErasureShareSize.Int32(),
MinReq: int32(endpoint.config.RS.MinThreshold),
RepairThreshold: int32(endpoint.config.RS.RepairThreshold),
SuccessThreshold: int32(endpoint.config.RS.SuccessThreshold),
Total: int32(endpoint.config.RS.TotalThreshold),
ErasureShareSize: endpoint.config.RS.ErasureShareSize.Int32(),
}
}

17
satellite/metainfo/metainfo_test.go
View File

@ -760,6 +760,23 @@ func TestInlineSegment(t *testing.T) {
})
require.NoError(t, err)
{ // test max inline segment size 4KiB
beginObjectResp, err := metainfoClient.BeginObject(ctx, metainfo.BeginObjectParams{
Bucket: []byte(bucket.Name),
EncryptedPath: []byte("too-large-inline-segment"),
})
require.NoError(t, err)
data := testrand.Bytes(5 * memory.KiB)
err = metainfoClient.MakeInlineSegment(ctx, metainfo.MakeInlineSegmentParams{
StreamID: beginObjectResp.StreamID,
Position: storj.SegmentPosition{
Index: 0,
},
EncryptedInlineData: data,
})
require.Error(t, err)
}
{ // test listing inline segments
for _, test := range []struct {
Index int32

36
satellite/metainfo/validation.go
View File

@ -168,11 +168,11 @@ func (endpoint *Endpoint) validateAuth(ctx context.Context, header *pb.RequestHe
func (endpoint *Endpoint) checkRate(ctx context.Context, projectID uuid.UUID) (err error) {
defer mon.Task()(&ctx)(&err)
if !endpoint.limiterConfig.Enabled {
if !endpoint.config.RateLimiter.Enabled {
return nil
}
limiter, err := endpoint.limiterCache.Get(projectID.String(), func() (interface{}, error) {
limit := rate.Limit(endpoint.limiterConfig.Rate)
limit := rate.Limit(endpoint.config.RateLimiter.Rate)
project, err := endpoint.projects.Get(ctx, projectID)
if err != nil {
@ -320,7 +320,7 @@ func (endpoint *Endpoint) validatePointer(ctx context.Context, pointer *pb.Point
return Error.New("invalid no order limit for piece")
}
maxAllowed, err := encryption.CalcEncryptedSize(endpoint.requiredRSConfig.MaxSegmentSize.Int64(), storj.EncryptionParameters{
maxAllowed, err := encryption.CalcEncryptedSize(endpoint.config.MaxSegmentSize.Int64(), storj.EncryptionParameters{
CipherSuite: storj.EncAESGCM,
BlockSize: 128, // intentionally low block size to allow maximum possible encryption overhead
})
@ -351,8 +351,8 @@ func (endpoint *Endpoint) validatePointer(ctx context.Context, pointer *pb.Point
}
// expect that too much time has not passed between order limit creation and now
if time.Since(limit.OrderCreation) > endpoint.maxCommitInterval {
return Error.New("Segment not committed before max commit interval of %f minutes.", endpoint.maxCommitInterval.Minutes())
if time.Since(limit.OrderCreation) > endpoint.config.MaxCommitInterval {
return Error.New("Segment not committed before max commit interval of %f minutes.", endpoint.config.MaxCommitInterval.Minutes())
}
derivedPieceID := remote.RootPieceId.Derive(piece.NodeId, piece.PieceNum)
@ -382,20 +382,20 @@ func (endpoint *Endpoint) validatePointer(ctx context.Context, pointer *pb.Point
func (endpoint *Endpoint) validateRedundancy(ctx context.Context, redundancy *pb.RedundancyScheme) (err error) {
defer mon.Task()(&ctx)(&err)
if endpoint.requiredRSConfig.Validate {
if endpoint.requiredRSConfig.ErasureShareSize.Int32() != redundancy.ErasureShareSize ||
endpoint.requiredRSConfig.MinTotalThreshold > int(redundancy.Total) ||
endpoint.requiredRSConfig.MaxTotalThreshold < int(redundancy.Total) ||
endpoint.requiredRSConfig.MinThreshold != int(redundancy.MinReq) ||
endpoint.requiredRSConfig.RepairThreshold != int(redundancy.RepairThreshold) ||
endpoint.requiredRSConfig.SuccessThreshold != int(redundancy.SuccessThreshold) {
if endpoint.config.RS.Validate {
if endpoint.config.RS.ErasureShareSize.Int32() != redundancy.ErasureShareSize ||
endpoint.config.RS.MinTotalThreshold > int(redundancy.Total) ||
endpoint.config.RS.MaxTotalThreshold < int(redundancy.Total) ||
endpoint.config.RS.MinThreshold != int(redundancy.MinReq) ||
endpoint.config.RS.RepairThreshold != int(redundancy.RepairThreshold) ||
endpoint.config.RS.SuccessThreshold != int(redundancy.SuccessThreshold) {
return Error.New("provided redundancy scheme parameters not allowed: want [%d, %d, %d, %d-%d, %d] got [%d, %d, %d, %d, %d]",
endpoint.requiredRSConfig.MinThreshold,
endpoint.requiredRSConfig.RepairThreshold,
endpoint.requiredRSConfig.SuccessThreshold,
endpoint.requiredRSConfig.MinTotalThreshold,
endpoint.requiredRSConfig.MaxTotalThreshold,
endpoint.requiredRSConfig.ErasureShareSize.Int32(),
endpoint.config.RS.MinThreshold,
endpoint.config.RS.RepairThreshold,
endpoint.config.RS.SuccessThreshold,
endpoint.config.RS.MinTotalThreshold,
endpoint.config.RS.MaxTotalThreshold,
endpoint.config.RS.ErasureShareSize.Int32(),
redundancy.MinReq,
redundancy.RepairThreshold,

8
scripts/testdata/satellite-config.yaml.lock vendored
View File

@ -281,7 +281,10 @@ identity.key-path: /root/.local/share/storj/identity/satellite/identity.key
# metainfo.max-commit-interval: 48h0m0s
# maximum inline segment size
# metainfo.max-inline-segment-size: 8.0 KB
# metainfo.max-inline-segment-size: 4.0 KiB
# maximum segment size
# metainfo.max-segment-size: 64.0 MiB
# minimum remote segment size
# metainfo.min-remote-segment-size: 1.2 KiB
@ -325,9 +328,6 @@ identity.key-path: /root/.local/share/storj/identity/satellite/identity.key
# maximum buffer memory to be allocated for read buffers
# metainfo.rs.max-buffer-mem: 4.0 MiB
# maximum segment size
# metainfo.rs.max-segment-size: 64.0 MiB
# the largest amount of pieces to encode to. n (upper bound for validation).
# metainfo.rs.max-total-threshold: 130