satellite/metainfo: adjust max inline segment size validation to

potential encryption overhead.

This is the same approach we have for validating remote segment size.

https://storjlabs.atlassian.net/browse/USR-619

Change-Id: I2597ee734313a3068fd986001680bbedbf1bed2a

satellite/api.go

@@ -413,7 +413,7 @@ func NewAPI(log *zap.Logger, full *identity.FullIdentity, db DB,
 			Close: peer.Metainfo.PieceDeletion.Close,
 		})
 
-		peer.Metainfo.Endpoint2 = metainfo.NewEndpoint(
+		peer.Metainfo.Endpoint2, err = metainfo.NewEndpoint(
 			peer.Log.Named("metainfo:endpoint"),
 			peer.Metainfo.Service,
 			peer.Metainfo.PieceDeletion,
@@ -428,6 +428,10 @@ func NewAPI(log *zap.Logger, full *identity.FullIdentity, db DB,
 			signing.SignerFromFullIdentity(peer.Identity),
 			config.Metainfo,
 		)
+		if err != nil {
+			return nil, errs.Combine(err, peer.Close())
+		}
+
 		pbgrpc.RegisterMetainfoServer(peer.Server.GRPC(), peer.Metainfo.Endpoint2)
 		if err := pb.DRPCRegisterMetainfo(peer.Server.DRPC(), peer.Metainfo.Endpoint2); err != nil {
 			return nil, errs.Combine(err, peer.Close())

satellite/metainfo/metainfo.go

@@ -16,6 +16,7 @@ import (
 	"go.uber.org/zap"
 
 	"storj.io/common/context2"
+	"storj.io/common/encryption"
 	"storj.io/common/errs2"
 	"storj.io/common/pb"
 	"storj.io/common/rpc/rpcstatus"
@@ -70,21 +71,22 @@ type Revocations interface {
 //
 // architecture: Endpoint
 type Endpoint struct {
-	log                 *zap.Logger
-	metainfo            *Service
-	deletePieces        *piecedeletion.Service
-	orders              *orders.Service
-	overlay             *overlay.Service
-	attributions        attribution.DB
-	partners            *rewards.PartnersService
-	pointerVerification *pointerverification.Service
-	projectUsage        *accounting.Service
-	projects            console.Projects
-	apiKeys             APIKeys
-	createRequests      *createRequests
-	satellite           signing.Signer
-	limiterCache        *lrucache.ExpiringLRU
-	config              Config
+	log                  *zap.Logger
+	metainfo             *Service
+	deletePieces         *piecedeletion.Service
+	orders               *orders.Service
+	overlay              *overlay.Service
+	attributions         attribution.DB
+	partners             *rewards.PartnersService
+	pointerVerification  *pointerverification.Service
+	projectUsage         *accounting.Service
+	projects             console.Projects
+	apiKeys              APIKeys
+	createRequests       *createRequests
+	satellite            signing.Signer
+	limiterCache         *lrucache.ExpiringLRU
+	encInlineSegmentSize int64 // max inline segment size + encryption overhead
+	config               Config
 }
 
 // NewEndpoint creates new metainfo endpoint instance.
@@ -92,8 +94,16 @@ func NewEndpoint(log *zap.Logger, metainfo *Service, deletePieces *piecedeletion
 	orders *orders.Service, cache *overlay.Service, attributions attribution.DB,
 	partners *rewards.PartnersService, peerIdentities overlay.PeerIdentities,
 	apiKeys APIKeys, projectUsage *accounting.Service, projects console.Projects,
-	satellite signing.Signer, config Config) *Endpoint {
+	satellite signing.Signer, config Config) (*Endpoint, error) {
 	// TODO do something with too many params
+
+	encInlineSegmentSize, err := encryption.CalcEncryptedSize(config.MaxInlineSegmentSize.Int64(), storj.EncryptionParameters{
+		CipherSuite: storj.EncAESGCM,
+		BlockSize:   128, // intentionally low block size to allow maximum possible encryption overhead
+	})
+	if err != nil {
+		return nil, err
+	}
 	return &Endpoint{
 		log:                 log,
 		metainfo:            metainfo,
@@ -112,8 +122,9 @@ func NewEndpoint(log *zap.Logger, metainfo *Service, deletePieces *piecedeletion
 			Capacity:   config.RateLimiter.CacheCapacity,
 			Expiration: config.RateLimiter.CacheExpiration,
 		}),
-		config: config,
-	}
+		encInlineSegmentSize: encInlineSegmentSize,
+		config:               config,
+	}, nil
 }
 
 // Close closes resources
@@ -1692,7 +1703,7 @@ func (endpoint *Endpoint) makeInlineSegment(ctx context.Context, req *pb.Segment
 	}
 
 	inlineUsed := int64(len(req.EncryptedInlineData))
-	if inlineUsed > endpoint.config.MaxInlineSegmentSize.Int64() {
+	if inlineUsed > endpoint.encInlineSegmentSize {
 		return nil, nil, rpcstatus.Error(rpcstatus.InvalidArgument, fmt.Sprintf("inline segment size cannot be larger than %s", endpoint.config.MaxInlineSegmentSize))
 	}
 

satellite/metainfo/metainfo_test.go

@@ -26,7 +26,9 @@ import (
 	"storj.io/common/testrand"
 	"storj.io/storj/private/testplanet"
 	"storj.io/storj/satellite"
+	satMetainfo "storj.io/storj/satellite/metainfo"
 	"storj.io/uplink/private/metainfo"
+	"storj.io/uplink/private/storage/meta"
 )
 
 func TestInvalidAPIKey(t *testing.T) {
@@ -759,14 +761,14 @@ func TestInlineSegment(t *testing.T) {
 		})
 		require.NoError(t, err)
 
-		{ // test max inline segment size 4KiB
+		{ // Confirm data larger than our configured max inline segment size of 4 KiB cannot be inlined
 			beginObjectResp, err := metainfoClient.BeginObject(ctx, metainfo.BeginObjectParams{
 				Bucket:        []byte(bucket.Name),
 				EncryptedPath: []byte("too-large-inline-segment"),
 			})
 			require.NoError(t, err)
 
-			data := testrand.Bytes(5 * memory.KiB)
+			data := testrand.Bytes(10 * memory.KiB)
 			err = metainfoClient.MakeInlineSegment(ctx, metainfo.MakeInlineSegmentParams{
 				StreamID: beginObjectResp.StreamID,
 				Position: storj.SegmentPosition{
@@ -1421,3 +1423,45 @@ func TestDeleteBatchWithoutPermission(t *testing.T) {
 		require.Equal(t, 2, len(responses))
 	})
 }
+
+func TestInlineSegmentThreshold(t *testing.T) {
+	testplanet.Run(t, testplanet.Config{
+		SatelliteCount: 1, StorageNodeCount: 4, UplinkCount: 1,
+	}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
+		projectID := planet.Uplinks[0].ProjectID[planet.Satellites[0].ID()]
+
+		{ // limit is max inline segment size + encryption overhead
+			err := planet.Uplinks[0].Upload(ctx, planet.Satellites[0], "test-bucket-inline", "inline-object", testrand.Bytes(4*memory.KiB))
+			require.NoError(t, err)
+
+			// we don't know encrypted path
+			prefix, err := satMetainfo.CreatePath(ctx, projectID, -1, []byte("test-bucket-inline"), []byte{})
+			require.NoError(t, err)
+
+			items, _, err := planet.Satellites[0].Metainfo.Service.List(ctx, prefix, "", false, 0, meta.All)
+			require.NoError(t, err)
+			require.Equal(t, 1, len(items))
+
+			pointer, err := planet.Satellites[0].Metainfo.Service.Get(ctx, prefix+"/"+items[0].Path)
+			require.NoError(t, err)
+			require.Equal(t, pb.Pointer_INLINE, pointer.Type)
+		}
+
+		{ // one more byte over limit should enough to create remote segment
+			err := planet.Uplinks[0].Upload(ctx, planet.Satellites[0], "test-bucket-remote", "remote-object", testrand.Bytes(4*memory.KiB+1))
+			require.NoError(t, err)
+
+			// we don't know encrypted path
+			prefix, err := satMetainfo.CreatePath(ctx, projectID, -1, []byte("test-bucket-remote"), []byte{})
+			require.NoError(t, err)
+
+			items, _, err := planet.Satellites[0].Metainfo.Service.List(ctx, prefix, "", false, 0, meta.All)
+			require.NoError(t, err)
+			require.Equal(t, 1, len(items))
+
+			pointer, err := planet.Satellites[0].Metainfo.Service.Get(ctx, prefix+"/"+items[0].Path)
+			require.NoError(t, err)
+			require.Equal(t, pb.Pointer_REMOTE, pointer.Type)
+		}
+	})
+}