From c178a08cb8f2fa477338200d1ca8cf053e1ea15f Mon Sep 17 00:00:00 2001 From: Michal Niewrzal Date: Wed, 1 Apr 2020 11:15:24 +0200 Subject: [PATCH] satellite/metainfo: add max segment size and max inline size to BeginObject response We want to control inline segment size and segment size on satellite side. We need to return such information to uplink like with redundancy scheme. Change-Id: If04b0a45a2757a01c0cc046432c115f475e9323c --- go.mod | 2 +- go.sum | 6 ++- private/testplanet/satellite.go | 4 +- satellite/api.go | 4 +- satellite/metainfo/config.go | 4 +- satellite/metainfo/metainfo.go | 44 ++++++++++----------- satellite/metainfo/metainfo_test.go | 17 ++++++++ satellite/metainfo/validation.go | 36 ++++++++--------- scripts/testdata/satellite-config.yaml.lock | 8 ++-- 9 files changed, 71 insertions(+), 54 deletions(-) diff --git a/go.mod b/go.mod index 003456f85..5f346dd2d 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 google.golang.org/grpc v1.28.0 - storj.io/common v0.0.0-20200401062549-3fa2945ebe52 + storj.io/common v0.0.0-20200401122855-4ad28fd2218f storj.io/drpc v0.0.11 storj.io/private v0.0.0-20200327035409-e9d82e7e0c6b storj.io/uplink v1.0.3 diff --git a/go.sum b/go.sum index 941926a66..859192136 100644 --- a/go.sum +++ b/go.sum @@ -623,8 +623,10 @@ storj.io/common v0.0.0-20200323134045-2bd4d6e2dd7d/go.mod h1:I0QTs7z1rI+ZEN95GGY storj.io/common v0.0.0-20200331095257-30ebbdbbba88/go.mod h1:RBaNRmk/lqyZ7h1MAH4N9zld0z+tO4M9sLOFT30K+cE= storj.io/common v0.0.0-20200331124657-a4f8265946f2 h1:Fctei5lPPAfbvtpijLQZTjQeeuh+MCkacLYau7nyxKA= storj.io/common v0.0.0-20200331124657-a4f8265946f2/go.mod h1:RBaNRmk/lqyZ7h1MAH4N9zld0z+tO4M9sLOFT30K+cE= -storj.io/common v0.0.0-20200401062549-3fa2945ebe52 h1:Oulb5CY00odkUrddYONRYkr7FefoJx1mbiIh8kOTh+0= -storj.io/common v0.0.0-20200401062549-3fa2945ebe52/go.mod h1:RBaNRmk/lqyZ7h1MAH4N9zld0z+tO4M9sLOFT30K+cE= +storj.io/common v0.0.0-20200401095230-4fe9b2ad3ec0 h1:gFsVck24e/eU3j9qmfXwsyBUwvtcjuTD5JXvvv9NdFE= +storj.io/common v0.0.0-20200401095230-4fe9b2ad3ec0/go.mod h1:RBaNRmk/lqyZ7h1MAH4N9zld0z+tO4M9sLOFT30K+cE= +storj.io/common v0.0.0-20200401122855-4ad28fd2218f h1:17QhsiCE8PFKoVaxRk+8tfGazlm2r5mMsXEtFTT2F9w= +storj.io/common v0.0.0-20200401122855-4ad28fd2218f/go.mod h1:RBaNRmk/lqyZ7h1MAH4N9zld0z+tO4M9sLOFT30K+cE= storj.io/drpc v0.0.7-0.20191115031725-2171c57838d2/go.mod h1:/ascUDbzNAv0A3Jj7wUIKFBH2JdJ2uJIBO/b9+2yHgQ= storj.io/drpc v0.0.11 h1:6vLxfpSbwCLtqzAoXzXx/SxBqBtbzbmquXPqfcWKqfw= storj.io/drpc v0.0.11/go.mod h1:TiFc2obNjL9/3isMW1Rpxjy8V9uE0B2HMeMFGiiI7Iw= diff --git a/private/testplanet/satellite.go b/private/testplanet/satellite.go index 2a2a835e9..1b1be2a40 100644 --- a/private/testplanet/satellite.go +++ b/private/testplanet/satellite.go @@ -332,11 +332,11 @@ func (planet *Planet) newSatellites(count int, satelliteDatabases satellitedbtes Metainfo: metainfo.Config{ DatabaseURL: "", // not used MinRemoteSegmentSize: 0, // TODO: fix tests to work with 1024 - MaxInlineSegmentSize: 8000, + MaxInlineSegmentSize: 4 * memory.KiB, + MaxSegmentSize: 64 * memory.MiB, MaxCommitInterval: 1 * time.Hour, Overlay: true, RS: metainfo.RSConfig{ - MaxSegmentSize: 64 * memory.MiB, MaxBufferMem: memory.Size(256), ErasureShareSize: memory.Size(256), MinThreshold: atLeastOne(planet.config.StorageNodeCount * 1 / 5), diff --git a/satellite/api.go b/satellite/api.go index 7a41b7432..2790abfbd 100644 --- a/satellite/api.go +++ b/satellite/api.go @@ -425,10 +425,8 @@ func NewAPI(log *zap.Logger, full *identity.FullIdentity, db DB, peer.DB.Console().APIKeys(), peer.Accounting.ProjectUsage, peer.DB.Console().Projects(), - config.Metainfo.RS, signing.SignerFromFullIdentity(peer.Identity), - config.Metainfo.MaxCommitInterval, - config.Metainfo.RateLimiter, + config.Metainfo, ) pbgrpc.RegisterMetainfoServer(peer.Server.GRPC(), peer.Metainfo.Endpoint2) if err := pb.DRPCRegisterMetainfo(peer.Server.DRPC(), peer.Metainfo.Endpoint2); err != nil { diff --git a/satellite/metainfo/config.go b/satellite/metainfo/config.go index 5694020c7..f3b30c737 100644 --- a/satellite/metainfo/config.go +++ b/satellite/metainfo/config.go @@ -24,7 +24,6 @@ const ( // RSConfig is a configuration struct that keeps details about default // redundancy strategy information type RSConfig struct { - MaxSegmentSize memory.Size `help:"maximum segment size" default:"64MiB"` MaxBufferMem memory.Size `help:"maximum buffer memory to be allocated for read buffers" default:"4MiB"` ErasureShareSize memory.Size `help:"the size of each new erasure share in bytes" default:"256B"` MinThreshold int `help:"the minimum pieces required to recover a segment. k." releaseDefault:"29" devDefault:"4"` @@ -50,7 +49,8 @@ type RateLimiterConfig struct { type Config struct { DatabaseURL string `help:"the database connection string to use" default:"postgres://"` MinRemoteSegmentSize memory.Size `default:"1240" help:"minimum remote segment size"` - MaxInlineSegmentSize memory.Size `default:"8000" help:"maximum inline segment size"` + MaxInlineSegmentSize memory.Size `default:"4KiB" help:"maximum inline segment size"` + MaxSegmentSize memory.Size `default:"64MiB" help:"maximum segment size"` MaxCommitInterval time.Duration `default:"48h" help:"maximum time allowed to pass between creating and committing a segment"` Overlay bool `default:"true" help:"toggle flag if overlay is enabled"` RS RSConfig `help:"redundancy scheme configuration"` diff --git a/satellite/metainfo/metainfo.go b/satellite/metainfo/metainfo.go index 65e9040b4..911d1e7d5 100644 --- a/satellite/metainfo/metainfo.go +++ b/satellite/metainfo/metainfo.go @@ -83,11 +83,9 @@ type Endpoint struct { projects console.Projects apiKeys APIKeys createRequests *createRequests - requiredRSConfig RSConfig satellite signing.Signer - maxCommitInterval time.Duration limiterCache *lrucache.ExpiringLRU - limiterConfig RateLimiterConfig + config Config } // NewEndpoint creates new metainfo endpoint instance. @@ -95,8 +93,7 @@ func NewEndpoint(log *zap.Logger, metainfo *Service, deletePieces *piecedeletion orders *orders.Service, cache *overlay.Service, attributions attribution.DB, partners *rewards.PartnersService, peerIdentities overlay.PeerIdentities, apiKeys APIKeys, projectUsage *accounting.Service, projects console.Projects, - rsConfig RSConfig, satellite signing.Signer, maxCommitInterval time.Duration, - limiterConfig RateLimiterConfig) *Endpoint { + satellite signing.Signer, config Config) *Endpoint { // TODO do something with too many params return &Endpoint{ log: log, @@ -111,14 +108,12 @@ func NewEndpoint(log *zap.Logger, metainfo *Service, deletePieces *piecedeletion projectUsage: projectUsage, projects: projects, createRequests: newCreateRequests(), - requiredRSConfig: rsConfig, satellite: satellite, - maxCommitInterval: maxCommitInterval, limiterCache: lrucache.New(lrucache.Options{ - Capacity: limiterConfig.CacheCapacity, - Expiration: limiterConfig.CacheExpiration, + Capacity: config.RateLimiter.CacheCapacity, + Expiration: config.RateLimiter.CacheExpiration, }), - limiterConfig: limiterConfig, + config: config, } } @@ -1053,11 +1048,13 @@ func (endpoint *Endpoint) BeginObject(ctx context.Context, req *pb.ObjectBeginRe mon.Meter("req_put_object").Mark(1) return &pb.ObjectBeginResponse{ - Bucket: req.Bucket, - EncryptedPath: req.EncryptedPath, - Version: req.Version, - StreamId: streamID, - RedundancyScheme: pbRS, + Bucket: req.Bucket, + EncryptedPath: req.EncryptedPath, + Version: req.Version, + StreamId: streamID, + RedundancyScheme: pbRS, + MaxInlineSegmentSize: endpoint.config.MaxInlineSegmentSize.Int64(), + MaxSegmentSize: endpoint.config.MaxSegmentSize.Int64(), }, nil } @@ -1697,6 +1694,11 @@ func (endpoint *Endpoint) makeInlineSegment(ctx context.Context, req *pb.Segment return nil, nil, rpcstatus.Error(rpcstatus.InvalidArgument, "segment index must be greater then 0") } + inlineUsed := int64(len(req.EncryptedInlineData)) + if inlineUsed > endpoint.config.MaxInlineSegmentSize.Int64() { + return nil, nil, rpcstatus.Error(rpcstatus.InvalidArgument, fmt.Sprintf("inline segment size cannot be larger than %s", endpoint.config.MaxInlineSegmentSize)) + } + exceeded, limit, err := endpoint.projectUsage.ExceedsStorageUsage(ctx, keyInfo.ProjectID) if err != nil { return nil, nil, rpcstatus.Error(rpcstatus.Internal, err.Error()) @@ -1708,8 +1710,6 @@ func (endpoint *Endpoint) makeInlineSegment(ctx context.Context, req *pb.Segment return nil, nil, rpcstatus.Error(rpcstatus.ResourceExhausted, "Exceeded Usage Limit") } - inlineUsed := int64(len(req.EncryptedInlineData)) - if err := endpoint.projectUsage.AddProjectStorageUsage(ctx, keyInfo.ProjectID, inlineUsed); err != nil { endpoint.log.Sugar().Errorf("Could not track new storage usage by project %v: %v", keyInfo.ProjectID, err) // but continue. it's most likely our own fault that we couldn't track it, and the only thing @@ -2502,10 +2502,10 @@ func (endpoint *Endpoint) findIndexPreviousLastSegmentWhenNotKnowingNumSegments( func (endpoint *Endpoint) redundancyScheme() *pb.RedundancyScheme { return &pb.RedundancyScheme{ Type: pb.RedundancyScheme_RS, - MinReq: int32(endpoint.requiredRSConfig.MinThreshold), - RepairThreshold: int32(endpoint.requiredRSConfig.RepairThreshold), - SuccessThreshold: int32(endpoint.requiredRSConfig.SuccessThreshold), - Total: int32(endpoint.requiredRSConfig.TotalThreshold), - ErasureShareSize: endpoint.requiredRSConfig.ErasureShareSize.Int32(), + MinReq: int32(endpoint.config.RS.MinThreshold), + RepairThreshold: int32(endpoint.config.RS.RepairThreshold), + SuccessThreshold: int32(endpoint.config.RS.SuccessThreshold), + Total: int32(endpoint.config.RS.TotalThreshold), + ErasureShareSize: endpoint.config.RS.ErasureShareSize.Int32(), } } diff --git a/satellite/metainfo/metainfo_test.go b/satellite/metainfo/metainfo_test.go index ebbb62edd..762b94f10 100644 --- a/satellite/metainfo/metainfo_test.go +++ b/satellite/metainfo/metainfo_test.go @@ -760,6 +760,23 @@ func TestInlineSegment(t *testing.T) { }) require.NoError(t, err) + { // test max inline segment size 4KiB + beginObjectResp, err := metainfoClient.BeginObject(ctx, metainfo.BeginObjectParams{ + Bucket: []byte(bucket.Name), + EncryptedPath: []byte("too-large-inline-segment"), + }) + require.NoError(t, err) + + data := testrand.Bytes(5 * memory.KiB) + err = metainfoClient.MakeInlineSegment(ctx, metainfo.MakeInlineSegmentParams{ + StreamID: beginObjectResp.StreamID, + Position: storj.SegmentPosition{ + Index: 0, + }, + EncryptedInlineData: data, + }) + require.Error(t, err) + } { // test listing inline segments for _, test := range []struct { Index int32 diff --git a/satellite/metainfo/validation.go b/satellite/metainfo/validation.go index 8d3ddb70f..8ab9b0a41 100644 --- a/satellite/metainfo/validation.go +++ b/satellite/metainfo/validation.go @@ -168,11 +168,11 @@ func (endpoint *Endpoint) validateAuth(ctx context.Context, header *pb.RequestHe func (endpoint *Endpoint) checkRate(ctx context.Context, projectID uuid.UUID) (err error) { defer mon.Task()(&ctx)(&err) - if !endpoint.limiterConfig.Enabled { + if !endpoint.config.RateLimiter.Enabled { return nil } limiter, err := endpoint.limiterCache.Get(projectID.String(), func() (interface{}, error) { - limit := rate.Limit(endpoint.limiterConfig.Rate) + limit := rate.Limit(endpoint.config.RateLimiter.Rate) project, err := endpoint.projects.Get(ctx, projectID) if err != nil { @@ -320,7 +320,7 @@ func (endpoint *Endpoint) validatePointer(ctx context.Context, pointer *pb.Point return Error.New("invalid no order limit for piece") } - maxAllowed, err := encryption.CalcEncryptedSize(endpoint.requiredRSConfig.MaxSegmentSize.Int64(), storj.EncryptionParameters{ + maxAllowed, err := encryption.CalcEncryptedSize(endpoint.config.MaxSegmentSize.Int64(), storj.EncryptionParameters{ CipherSuite: storj.EncAESGCM, BlockSize: 128, // intentionally low block size to allow maximum possible encryption overhead }) @@ -351,8 +351,8 @@ func (endpoint *Endpoint) validatePointer(ctx context.Context, pointer *pb.Point } // expect that too much time has not passed between order limit creation and now - if time.Since(limit.OrderCreation) > endpoint.maxCommitInterval { - return Error.New("Segment not committed before max commit interval of %f minutes.", endpoint.maxCommitInterval.Minutes()) + if time.Since(limit.OrderCreation) > endpoint.config.MaxCommitInterval { + return Error.New("Segment not committed before max commit interval of %f minutes.", endpoint.config.MaxCommitInterval.Minutes()) } derivedPieceID := remote.RootPieceId.Derive(piece.NodeId, piece.PieceNum) @@ -382,20 +382,20 @@ func (endpoint *Endpoint) validatePointer(ctx context.Context, pointer *pb.Point func (endpoint *Endpoint) validateRedundancy(ctx context.Context, redundancy *pb.RedundancyScheme) (err error) { defer mon.Task()(&ctx)(&err) - if endpoint.requiredRSConfig.Validate { - if endpoint.requiredRSConfig.ErasureShareSize.Int32() != redundancy.ErasureShareSize || - endpoint.requiredRSConfig.MinTotalThreshold > int(redundancy.Total) || - endpoint.requiredRSConfig.MaxTotalThreshold < int(redundancy.Total) || - endpoint.requiredRSConfig.MinThreshold != int(redundancy.MinReq) || - endpoint.requiredRSConfig.RepairThreshold != int(redundancy.RepairThreshold) || - endpoint.requiredRSConfig.SuccessThreshold != int(redundancy.SuccessThreshold) { + if endpoint.config.RS.Validate { + if endpoint.config.RS.ErasureShareSize.Int32() != redundancy.ErasureShareSize || + endpoint.config.RS.MinTotalThreshold > int(redundancy.Total) || + endpoint.config.RS.MaxTotalThreshold < int(redundancy.Total) || + endpoint.config.RS.MinThreshold != int(redundancy.MinReq) || + endpoint.config.RS.RepairThreshold != int(redundancy.RepairThreshold) || + endpoint.config.RS.SuccessThreshold != int(redundancy.SuccessThreshold) { return Error.New("provided redundancy scheme parameters not allowed: want [%d, %d, %d, %d-%d, %d] got [%d, %d, %d, %d, %d]", - endpoint.requiredRSConfig.MinThreshold, - endpoint.requiredRSConfig.RepairThreshold, - endpoint.requiredRSConfig.SuccessThreshold, - endpoint.requiredRSConfig.MinTotalThreshold, - endpoint.requiredRSConfig.MaxTotalThreshold, - endpoint.requiredRSConfig.ErasureShareSize.Int32(), + endpoint.config.RS.MinThreshold, + endpoint.config.RS.RepairThreshold, + endpoint.config.RS.SuccessThreshold, + endpoint.config.RS.MinTotalThreshold, + endpoint.config.RS.MaxTotalThreshold, + endpoint.config.RS.ErasureShareSize.Int32(), redundancy.MinReq, redundancy.RepairThreshold, diff --git a/scripts/testdata/satellite-config.yaml.lock b/scripts/testdata/satellite-config.yaml.lock index a6b6f07bf..247e87cd2 100644 --- a/scripts/testdata/satellite-config.yaml.lock +++ b/scripts/testdata/satellite-config.yaml.lock @@ -281,7 +281,10 @@ identity.key-path: /root/.local/share/storj/identity/satellite/identity.key # metainfo.max-commit-interval: 48h0m0s # maximum inline segment size -# metainfo.max-inline-segment-size: 8.0 KB +# metainfo.max-inline-segment-size: 4.0 KiB + +# maximum segment size +# metainfo.max-segment-size: 64.0 MiB # minimum remote segment size # metainfo.min-remote-segment-size: 1.2 KiB @@ -325,9 +328,6 @@ identity.key-path: /root/.local/share/storj/identity/satellite/identity.key # maximum buffer memory to be allocated for read buffers # metainfo.rs.max-buffer-mem: 4.0 MiB -# maximum segment size -# metainfo.rs.max-segment-size: 64.0 MiB - # the largest amount of pieces to encode to. n (upper bound for validation). # metainfo.rs.max-total-threshold: 130