diff --git a/go.mod b/go.mod index 003456f85..5f346dd2d 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 google.golang.org/grpc v1.28.0 - storj.io/common v0.0.0-20200401062549-3fa2945ebe52 + storj.io/common v0.0.0-20200401122855-4ad28fd2218f storj.io/drpc v0.0.11 storj.io/private v0.0.0-20200327035409-e9d82e7e0c6b storj.io/uplink v1.0.3 diff --git a/go.sum b/go.sum index 941926a66..859192136 100644 --- a/go.sum +++ b/go.sum @@ -623,8 +623,10 @@ storj.io/common v0.0.0-20200323134045-2bd4d6e2dd7d/go.mod h1:I0QTs7z1rI+ZEN95GGY storj.io/common v0.0.0-20200331095257-30ebbdbbba88/go.mod h1:RBaNRmk/lqyZ7h1MAH4N9zld0z+tO4M9sLOFT30K+cE= storj.io/common v0.0.0-20200331124657-a4f8265946f2 h1:Fctei5lPPAfbvtpijLQZTjQeeuh+MCkacLYau7nyxKA= storj.io/common v0.0.0-20200331124657-a4f8265946f2/go.mod h1:RBaNRmk/lqyZ7h1MAH4N9zld0z+tO4M9sLOFT30K+cE= -storj.io/common v0.0.0-20200401062549-3fa2945ebe52 h1:Oulb5CY00odkUrddYONRYkr7FefoJx1mbiIh8kOTh+0= -storj.io/common v0.0.0-20200401062549-3fa2945ebe52/go.mod h1:RBaNRmk/lqyZ7h1MAH4N9zld0z+tO4M9sLOFT30K+cE= +storj.io/common v0.0.0-20200401095230-4fe9b2ad3ec0 h1:gFsVck24e/eU3j9qmfXwsyBUwvtcjuTD5JXvvv9NdFE= +storj.io/common v0.0.0-20200401095230-4fe9b2ad3ec0/go.mod h1:RBaNRmk/lqyZ7h1MAH4N9zld0z+tO4M9sLOFT30K+cE= +storj.io/common v0.0.0-20200401122855-4ad28fd2218f h1:17QhsiCE8PFKoVaxRk+8tfGazlm2r5mMsXEtFTT2F9w= +storj.io/common v0.0.0-20200401122855-4ad28fd2218f/go.mod h1:RBaNRmk/lqyZ7h1MAH4N9zld0z+tO4M9sLOFT30K+cE= storj.io/drpc v0.0.7-0.20191115031725-2171c57838d2/go.mod h1:/ascUDbzNAv0A3Jj7wUIKFBH2JdJ2uJIBO/b9+2yHgQ= storj.io/drpc v0.0.11 h1:6vLxfpSbwCLtqzAoXzXx/SxBqBtbzbmquXPqfcWKqfw= storj.io/drpc v0.0.11/go.mod h1:TiFc2obNjL9/3isMW1Rpxjy8V9uE0B2HMeMFGiiI7Iw= diff --git a/private/testplanet/satellite.go b/private/testplanet/satellite.go index 2a2a835e9..1b1be2a40 100644 --- a/private/testplanet/satellite.go +++ b/private/testplanet/satellite.go @@ -332,11 +332,11 @@ func (planet *Planet) newSatellites(count int, satelliteDatabases satellitedbtes Metainfo: metainfo.Config{ DatabaseURL: "", // not used MinRemoteSegmentSize: 0, // TODO: fix tests to work with 1024 - MaxInlineSegmentSize: 8000, + MaxInlineSegmentSize: 4 * memory.KiB, + MaxSegmentSize: 64 * memory.MiB, MaxCommitInterval: 1 * time.Hour, Overlay: true, RS: metainfo.RSConfig{ - MaxSegmentSize: 64 * memory.MiB, MaxBufferMem: memory.Size(256), ErasureShareSize: memory.Size(256), MinThreshold: atLeastOne(planet.config.StorageNodeCount * 1 / 5), diff --git a/satellite/api.go b/satellite/api.go index 7a41b7432..2790abfbd 100644 --- a/satellite/api.go +++ b/satellite/api.go @@ -425,10 +425,8 @@ func NewAPI(log *zap.Logger, full *identity.FullIdentity, db DB, peer.DB.Console().APIKeys(), peer.Accounting.ProjectUsage, peer.DB.Console().Projects(), - config.Metainfo.RS, signing.SignerFromFullIdentity(peer.Identity), - config.Metainfo.MaxCommitInterval, - config.Metainfo.RateLimiter, + config.Metainfo, ) pbgrpc.RegisterMetainfoServer(peer.Server.GRPC(), peer.Metainfo.Endpoint2) if err := pb.DRPCRegisterMetainfo(peer.Server.DRPC(), peer.Metainfo.Endpoint2); err != nil { diff --git a/satellite/metainfo/config.go b/satellite/metainfo/config.go index 5694020c7..f3b30c737 100644 --- a/satellite/metainfo/config.go +++ b/satellite/metainfo/config.go @@ -24,7 +24,6 @@ const ( // RSConfig is a configuration struct that keeps details about default // redundancy strategy information type RSConfig struct { - MaxSegmentSize memory.Size `help:"maximum segment size" default:"64MiB"` MaxBufferMem memory.Size `help:"maximum buffer memory to be allocated for read buffers" default:"4MiB"` ErasureShareSize memory.Size `help:"the size of each new erasure share in bytes" default:"256B"` MinThreshold int `help:"the minimum pieces required to recover a segment. k." releaseDefault:"29" devDefault:"4"` @@ -50,7 +49,8 @@ type RateLimiterConfig struct { type Config struct { DatabaseURL string `help:"the database connection string to use" default:"postgres://"` MinRemoteSegmentSize memory.Size `default:"1240" help:"minimum remote segment size"` - MaxInlineSegmentSize memory.Size `default:"8000" help:"maximum inline segment size"` + MaxInlineSegmentSize memory.Size `default:"4KiB" help:"maximum inline segment size"` + MaxSegmentSize memory.Size `default:"64MiB" help:"maximum segment size"` MaxCommitInterval time.Duration `default:"48h" help:"maximum time allowed to pass between creating and committing a segment"` Overlay bool `default:"true" help:"toggle flag if overlay is enabled"` RS RSConfig `help:"redundancy scheme configuration"` diff --git a/satellite/metainfo/metainfo.go b/satellite/metainfo/metainfo.go index 65e9040b4..911d1e7d5 100644 --- a/satellite/metainfo/metainfo.go +++ b/satellite/metainfo/metainfo.go @@ -83,11 +83,9 @@ type Endpoint struct { projects console.Projects apiKeys APIKeys createRequests *createRequests - requiredRSConfig RSConfig satellite signing.Signer - maxCommitInterval time.Duration limiterCache *lrucache.ExpiringLRU - limiterConfig RateLimiterConfig + config Config } // NewEndpoint creates new metainfo endpoint instance. @@ -95,8 +93,7 @@ func NewEndpoint(log *zap.Logger, metainfo *Service, deletePieces *piecedeletion orders *orders.Service, cache *overlay.Service, attributions attribution.DB, partners *rewards.PartnersService, peerIdentities overlay.PeerIdentities, apiKeys APIKeys, projectUsage *accounting.Service, projects console.Projects, - rsConfig RSConfig, satellite signing.Signer, maxCommitInterval time.Duration, - limiterConfig RateLimiterConfig) *Endpoint { + satellite signing.Signer, config Config) *Endpoint { // TODO do something with too many params return &Endpoint{ log: log, @@ -111,14 +108,12 @@ func NewEndpoint(log *zap.Logger, metainfo *Service, deletePieces *piecedeletion projectUsage: projectUsage, projects: projects, createRequests: newCreateRequests(), - requiredRSConfig: rsConfig, satellite: satellite, - maxCommitInterval: maxCommitInterval, limiterCache: lrucache.New(lrucache.Options{ - Capacity: limiterConfig.CacheCapacity, - Expiration: limiterConfig.CacheExpiration, + Capacity: config.RateLimiter.CacheCapacity, + Expiration: config.RateLimiter.CacheExpiration, }), - limiterConfig: limiterConfig, + config: config, } } @@ -1053,11 +1048,13 @@ func (endpoint *Endpoint) BeginObject(ctx context.Context, req *pb.ObjectBeginRe mon.Meter("req_put_object").Mark(1) return &pb.ObjectBeginResponse{ - Bucket: req.Bucket, - EncryptedPath: req.EncryptedPath, - Version: req.Version, - StreamId: streamID, - RedundancyScheme: pbRS, + Bucket: req.Bucket, + EncryptedPath: req.EncryptedPath, + Version: req.Version, + StreamId: streamID, + RedundancyScheme: pbRS, + MaxInlineSegmentSize: endpoint.config.MaxInlineSegmentSize.Int64(), + MaxSegmentSize: endpoint.config.MaxSegmentSize.Int64(), }, nil } @@ -1697,6 +1694,11 @@ func (endpoint *Endpoint) makeInlineSegment(ctx context.Context, req *pb.Segment return nil, nil, rpcstatus.Error(rpcstatus.InvalidArgument, "segment index must be greater then 0") } + inlineUsed := int64(len(req.EncryptedInlineData)) + if inlineUsed > endpoint.config.MaxInlineSegmentSize.Int64() { + return nil, nil, rpcstatus.Error(rpcstatus.InvalidArgument, fmt.Sprintf("inline segment size cannot be larger than %s", endpoint.config.MaxInlineSegmentSize)) + } + exceeded, limit, err := endpoint.projectUsage.ExceedsStorageUsage(ctx, keyInfo.ProjectID) if err != nil { return nil, nil, rpcstatus.Error(rpcstatus.Internal, err.Error()) @@ -1708,8 +1710,6 @@ func (endpoint *Endpoint) makeInlineSegment(ctx context.Context, req *pb.Segment return nil, nil, rpcstatus.Error(rpcstatus.ResourceExhausted, "Exceeded Usage Limit") } - inlineUsed := int64(len(req.EncryptedInlineData)) - if err := endpoint.projectUsage.AddProjectStorageUsage(ctx, keyInfo.ProjectID, inlineUsed); err != nil { endpoint.log.Sugar().Errorf("Could not track new storage usage by project %v: %v", keyInfo.ProjectID, err) // but continue. it's most likely our own fault that we couldn't track it, and the only thing @@ -2502,10 +2502,10 @@ func (endpoint *Endpoint) findIndexPreviousLastSegmentWhenNotKnowingNumSegments( func (endpoint *Endpoint) redundancyScheme() *pb.RedundancyScheme { return &pb.RedundancyScheme{ Type: pb.RedundancyScheme_RS, - MinReq: int32(endpoint.requiredRSConfig.MinThreshold), - RepairThreshold: int32(endpoint.requiredRSConfig.RepairThreshold), - SuccessThreshold: int32(endpoint.requiredRSConfig.SuccessThreshold), - Total: int32(endpoint.requiredRSConfig.TotalThreshold), - ErasureShareSize: endpoint.requiredRSConfig.ErasureShareSize.Int32(), + MinReq: int32(endpoint.config.RS.MinThreshold), + RepairThreshold: int32(endpoint.config.RS.RepairThreshold), + SuccessThreshold: int32(endpoint.config.RS.SuccessThreshold), + Total: int32(endpoint.config.RS.TotalThreshold), + ErasureShareSize: endpoint.config.RS.ErasureShareSize.Int32(), } } diff --git a/satellite/metainfo/metainfo_test.go b/satellite/metainfo/metainfo_test.go index ebbb62edd..762b94f10 100644 --- a/satellite/metainfo/metainfo_test.go +++ b/satellite/metainfo/metainfo_test.go @@ -760,6 +760,23 @@ func TestInlineSegment(t *testing.T) { }) require.NoError(t, err) + { // test max inline segment size 4KiB + beginObjectResp, err := metainfoClient.BeginObject(ctx, metainfo.BeginObjectParams{ + Bucket: []byte(bucket.Name), + EncryptedPath: []byte("too-large-inline-segment"), + }) + require.NoError(t, err) + + data := testrand.Bytes(5 * memory.KiB) + err = metainfoClient.MakeInlineSegment(ctx, metainfo.MakeInlineSegmentParams{ + StreamID: beginObjectResp.StreamID, + Position: storj.SegmentPosition{ + Index: 0, + }, + EncryptedInlineData: data, + }) + require.Error(t, err) + } { // test listing inline segments for _, test := range []struct { Index int32 diff --git a/satellite/metainfo/validation.go b/satellite/metainfo/validation.go index 8d3ddb70f..8ab9b0a41 100644 --- a/satellite/metainfo/validation.go +++ b/satellite/metainfo/validation.go @@ -168,11 +168,11 @@ func (endpoint *Endpoint) validateAuth(ctx context.Context, header *pb.RequestHe func (endpoint *Endpoint) checkRate(ctx context.Context, projectID uuid.UUID) (err error) { defer mon.Task()(&ctx)(&err) - if !endpoint.limiterConfig.Enabled { + if !endpoint.config.RateLimiter.Enabled { return nil } limiter, err := endpoint.limiterCache.Get(projectID.String(), func() (interface{}, error) { - limit := rate.Limit(endpoint.limiterConfig.Rate) + limit := rate.Limit(endpoint.config.RateLimiter.Rate) project, err := endpoint.projects.Get(ctx, projectID) if err != nil { @@ -320,7 +320,7 @@ func (endpoint *Endpoint) validatePointer(ctx context.Context, pointer *pb.Point return Error.New("invalid no order limit for piece") } - maxAllowed, err := encryption.CalcEncryptedSize(endpoint.requiredRSConfig.MaxSegmentSize.Int64(), storj.EncryptionParameters{ + maxAllowed, err := encryption.CalcEncryptedSize(endpoint.config.MaxSegmentSize.Int64(), storj.EncryptionParameters{ CipherSuite: storj.EncAESGCM, BlockSize: 128, // intentionally low block size to allow maximum possible encryption overhead }) @@ -351,8 +351,8 @@ func (endpoint *Endpoint) validatePointer(ctx context.Context, pointer *pb.Point } // expect that too much time has not passed between order limit creation and now - if time.Since(limit.OrderCreation) > endpoint.maxCommitInterval { - return Error.New("Segment not committed before max commit interval of %f minutes.", endpoint.maxCommitInterval.Minutes()) + if time.Since(limit.OrderCreation) > endpoint.config.MaxCommitInterval { + return Error.New("Segment not committed before max commit interval of %f minutes.", endpoint.config.MaxCommitInterval.Minutes()) } derivedPieceID := remote.RootPieceId.Derive(piece.NodeId, piece.PieceNum) @@ -382,20 +382,20 @@ func (endpoint *Endpoint) validatePointer(ctx context.Context, pointer *pb.Point func (endpoint *Endpoint) validateRedundancy(ctx context.Context, redundancy *pb.RedundancyScheme) (err error) { defer mon.Task()(&ctx)(&err) - if endpoint.requiredRSConfig.Validate { - if endpoint.requiredRSConfig.ErasureShareSize.Int32() != redundancy.ErasureShareSize || - endpoint.requiredRSConfig.MinTotalThreshold > int(redundancy.Total) || - endpoint.requiredRSConfig.MaxTotalThreshold < int(redundancy.Total) || - endpoint.requiredRSConfig.MinThreshold != int(redundancy.MinReq) || - endpoint.requiredRSConfig.RepairThreshold != int(redundancy.RepairThreshold) || - endpoint.requiredRSConfig.SuccessThreshold != int(redundancy.SuccessThreshold) { + if endpoint.config.RS.Validate { + if endpoint.config.RS.ErasureShareSize.Int32() != redundancy.ErasureShareSize || + endpoint.config.RS.MinTotalThreshold > int(redundancy.Total) || + endpoint.config.RS.MaxTotalThreshold < int(redundancy.Total) || + endpoint.config.RS.MinThreshold != int(redundancy.MinReq) || + endpoint.config.RS.RepairThreshold != int(redundancy.RepairThreshold) || + endpoint.config.RS.SuccessThreshold != int(redundancy.SuccessThreshold) { return Error.New("provided redundancy scheme parameters not allowed: want [%d, %d, %d, %d-%d, %d] got [%d, %d, %d, %d, %d]", - endpoint.requiredRSConfig.MinThreshold, - endpoint.requiredRSConfig.RepairThreshold, - endpoint.requiredRSConfig.SuccessThreshold, - endpoint.requiredRSConfig.MinTotalThreshold, - endpoint.requiredRSConfig.MaxTotalThreshold, - endpoint.requiredRSConfig.ErasureShareSize.Int32(), + endpoint.config.RS.MinThreshold, + endpoint.config.RS.RepairThreshold, + endpoint.config.RS.SuccessThreshold, + endpoint.config.RS.MinTotalThreshold, + endpoint.config.RS.MaxTotalThreshold, + endpoint.config.RS.ErasureShareSize.Int32(), redundancy.MinReq, redundancy.RepairThreshold, diff --git a/scripts/testdata/satellite-config.yaml.lock b/scripts/testdata/satellite-config.yaml.lock index a6b6f07bf..247e87cd2 100644 --- a/scripts/testdata/satellite-config.yaml.lock +++ b/scripts/testdata/satellite-config.yaml.lock @@ -281,7 +281,10 @@ identity.key-path: /root/.local/share/storj/identity/satellite/identity.key # metainfo.max-commit-interval: 48h0m0s # maximum inline segment size -# metainfo.max-inline-segment-size: 8.0 KB +# metainfo.max-inline-segment-size: 4.0 KiB + +# maximum segment size +# metainfo.max-segment-size: 64.0 MiB # minimum remote segment size # metainfo.min-remote-segment-size: 1.2 KiB @@ -325,9 +328,6 @@ identity.key-path: /root/.local/share/storj/identity/satellite/identity.key # maximum buffer memory to be allocated for read buffers # metainfo.rs.max-buffer-mem: 4.0 MiB -# maximum segment size -# metainfo.rs.max-segment-size: 64.0 MiB - # the largest amount of pieces to encode to. n (upper bound for validation). # metainfo.rs.max-total-threshold: 130