satellite/metainfo: add TestEnableBucketVersioning flag

Additional feature flag (onyly for testing) to set versioning enabled
for all new create buckets. We need it until we will have support
for enabling/disabling versioning for bucket on metainfo API.

In addition this change is fixing also two small issues which makes
testing this flag imposible:
* metabase Status list was not aligned with protobuf definition
* object retruned by metainfo API didn't have correct status set in some
cases

Change-Id: I0d63dff6a08efa588c8999af1e17db476943e067

satellite/metabase/common.go

@@ -412,10 +412,11 @@ const (
 	CommittedUnversioned = ObjectStatus(3)
 	// CommittedVersioned means that the object is finished and should be visible for general listing.
 	CommittedVersioned = ObjectStatus(4)
-	// DeleteMarkerUnversioned is inserted when an unversioned object is deleted in a versioning suspended bucket.
-	DeleteMarkerUnversioned = ObjectStatus(5)
 	// DeleteMarkerVersioned is inserted when an object is deleted in a versioning enabled bucket.
-	DeleteMarkerVersioned = ObjectStatus(6)
+	DeleteMarkerVersioned = ObjectStatus(5)
+	// DeleteMarkerUnversioned is inserted when an unversioned object is deleted in a versioning suspended bucket.
+	DeleteMarkerUnversioned = ObjectStatus(6)
+
 	// Prefix is an ephemeral status used during non-recursive listing.
 	Prefix = ObjectStatus(7)
 
@@ -424,8 +425,8 @@ const (
 	statusCommittedUnversioned    = "3"
 	statusCommittedVersioned      = "4"
 	statusesCommitted             = "(" + statusCommittedUnversioned + "," + statusCommittedVersioned + ")"
-	statusDeleteMarkerUnversioned = "5"
-	statusDeleteMarkerVersioned   = "6"
+	statusDeleteMarkerVersioned   = "5"
+	statusDeleteMarkerUnversioned = "6"
 	statusesDeleteMarker          = "(" + statusDeleteMarkerUnversioned + "," + statusDeleteMarkerVersioned + ")"
 	statusesUnversioned           = "(" + statusCommittedUnversioned + "," + statusDeleteMarkerUnversioned + ")"
 )

satellite/metainfo/config.go

@@ -157,6 +157,9 @@ type Config struct {
 
 	// TODO remove when we benchmarking are done and decision is made.
 	TestListingQuery bool `default:"false" help:"test the new query for non-recursive listing"`
+
+	// flag will be effective only if UseBucketLevelObjectVersioning or UseBucketLevelObjectVersioningProjects are enabled
+	TestEnableBucketVersioning bool `default:"false" help:"if enabled all new created buckets will have versioning enabled (use only for testing)" hidden:"true"`
 }
 
 // Metabase constructs Metabase configuration based on Metainfo configuration with specific application name.

satellite/metainfo/endpoint_bucket.go

@@ -144,7 +144,11 @@ func (endpoint *Endpoint) CreateBucket(ctx context.Context, req *pb.BucketCreate
 	bucketReq.Placement = project.DefaultPlacement
 
 	if endpoint.config.UseBucketLevelObjectVersioningByProject(keyInfo.ProjectID) {
-		bucketReq.Versioning = buckets.Unversioned
+		if endpoint.config.TestEnableBucketVersioning {
+			bucketReq.Versioning = buckets.VersioningEnabled
+		} else {
+			bucketReq.Versioning = buckets.Unversioned
+		}
 	}
 	bucket, err := endpoint.buckets.CreateBucket(ctx, bucketReq)
 	if err != nil {

satellite/metainfo/endpoint_object.go

@@ -922,6 +922,12 @@ func (endpoint *Endpoint) ListObjects(ctx context.Context, req *pb.ObjectListReq
 	}
 	metabase.ListLimit.Ensure(&limit)
 
+	// TODO(ver): this is only temporary logic for testing, will be cleanup later
+	useListObjects := false
+	if endpoint.config.UseBucketLevelObjectVersioningByProject(keyInfo.ProjectID) && endpoint.config.TestEnableBucketVersioning {
+		useListObjects = !req.IncludeAllVersions
+	}
+
 	var prefix metabase.ObjectKey
 	if len(req.EncryptedPrefix) != 0 {
 		prefix = metabase.ObjectKey(req.EncryptedPrefix)
@@ -965,7 +971,7 @@ func (endpoint *Endpoint) ListObjects(ctx context.Context, req *pb.ObjectListReq
 	}
 
 	resp = &pb.ObjectListResponse{}
-	if endpoint.config.TestListingQuery {
+	if endpoint.config.TestListingQuery || useListObjects {
 		result, err := endpoint.metabase.ListObjects(ctx,
 			metabase.ListObjects{
 				ProjectID:  keyInfo.ProjectID,
@@ -1599,6 +1605,7 @@ func (endpoint *Endpoint) objectToProto(ctx context.Context, object metabase.Obj
 		Version:            int32(object.Version), // TODO incompatible types
 		ObjectVersion:      object.Version.Encode(),
 		StreamId:           streamID,
+		Status:             pb.Object_Status(object.Status),
 		ExpiresAt:          expires,
 		CreatedAt:          object.CreatedAt,
 
@@ -1806,24 +1813,36 @@ func (endpoint *Endpoint) DeleteCommittedObject(
 	}
 
 	var result metabase.DeleteObjectResult
-	if endpoint.config.ServerSideCopy {
-		if len(version) == 0 {
-			result, err = endpoint.metabase.DeleteObjectLastCommitted(ctx, metabase.DeleteObjectLastCommitted{
-				ObjectLocation: req,
-			})
-		} else {
-			var v metabase.Version
-			v, err = metabase.VersionFromBytes(version)
+	if len(version) == 0 {
+		versioned := false
+		suspended := false
+		if endpoint.config.UseBucketLevelObjectVersioningByProject(projectID) {
+			// TODO(ver): for production we need to avoid somehow additional GetBucket call
+			bucket, err := endpoint.buckets.GetBucket(ctx, []byte(bucket), projectID)
 			if err != nil {
-				return nil, err
+				endpoint.log.Error("unable to check bucket", zap.Error(err))
+				return nil, rpcstatus.Error(rpcstatus.Internal, "unable to get bucket versioning state")
 			}
-			result, err = endpoint.metabase.DeleteObjectExactVersion(ctx, metabase.DeleteObjectExactVersion{
-				ObjectLocation: req,
-				Version:        v,
-			})
+
+			versioned = bucket.Versioning == buckets.VersioningEnabled
+			suspended = bucket.Versioning == buckets.VersioningSuspended
 		}
+
+		result, err = endpoint.metabase.DeleteObjectLastCommitted(ctx, metabase.DeleteObjectLastCommitted{
+			ObjectLocation: req,
+			Versioned:      versioned,
+			Suspended:      suspended,
+		})
 	} else {
-		result, err = endpoint.metabase.DeleteObjectsAllVersions(ctx, metabase.DeleteObjectsAllVersions{Locations: []metabase.ObjectLocation{req}})
+		var v metabase.Version
+		v, err = metabase.VersionFromBytes(version)
+		if err != nil {
+			return nil, err
+		}
+		result, err = endpoint.metabase.DeleteObjectExactVersion(ctx, metabase.DeleteObjectExactVersion{
+			ObjectLocation: req,
+			Version:        v,
+		})
 	}
 	if err != nil {
 		return nil, Error.Wrap(err)

satellite/metainfo/endpoint_object_test.go

@@ -3130,6 +3130,7 @@ func TestEndpoint_Object_No_StorageNodes_Versioning(t *testing.T) {
 		Reconfigure: testplanet.Reconfigure{
 			Satellite: func(log *zap.Logger, index int, config *satellite.Config) {
 				config.Metainfo.UseBucketLevelObjectVersioning = true
+				config.Metainfo.TestEnableBucketVersioning = true
 			},
 		},
 	}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
@@ -3140,15 +3141,16 @@ func TestEndpoint_Object_No_StorageNodes_Versioning(t *testing.T) {
 		bucketName := "versioned-bucket"
 		objectKey := "versioned-object"
 
+		project, err := planet.Uplinks[0].OpenProject(ctx, satelliteSys)
+		require.NoError(t, err)
+		defer ctx.Check(project.Close)
+
 		createBucket := func(name string) error {
 			_, err := satelliteSys.API.Metainfo.Endpoint.CreateBucket(ctx, &pb.CreateBucketRequest{
 				Header: &pb.RequestHeader{ApiKey: apiKey},
 				Name:   []byte(name),
 			})
-			if err != nil {
-				return err
-			}
-			return satelliteSys.API.DB.Buckets().EnableBucketVersioning(ctx, []byte(bucketName), projectID)
+			return err
 		}
 
 		deleteBucket := func(name string) func() error {
@@ -3167,7 +3169,11 @@ func TestEndpoint_Object_No_StorageNodes_Versioning(t *testing.T) {
 
 			require.NoError(t, createBucket(bucketName))
 
-			err := planet.Uplinks[0].Upload(ctx, satelliteSys, bucketName, objectKey, testrand.Bytes(100))
+			state, err := planet.Satellites[0].API.Buckets.Service.GetBucketVersioningState(ctx, []byte(bucketName), projectID)
+			require.NoError(t, err)
+			require.Equal(t, buckets.VersioningEnabled, state)
+
+			err = planet.Uplinks[0].Upload(ctx, satelliteSys, bucketName, objectKey, testrand.Bytes(100))
 			require.NoError(t, err)
 
 			err = planet.Uplinks[0].Upload(ctx, satelliteSys, bucketName, objectKey, testrand.Bytes(100))
@@ -3176,6 +3182,19 @@ func TestEndpoint_Object_No_StorageNodes_Versioning(t *testing.T) {
 			objects, err := satelliteSys.Metabase.DB.TestingAllObjects(ctx)
 			require.NoError(t, err)
 			require.Len(t, objects, 2)
+
+			response, err := satelliteSys.API.Metainfo.Endpoint.BeginDeleteObject(ctx, &pb.BeginDeleteObjectRequest{
+				Header:             &pb.RequestHeader{ApiKey: apiKey},
+				Bucket:             []byte(bucketName),
+				EncryptedObjectKey: []byte(objects[0].ObjectKey),
+			})
+
+			require.NoError(t, err)
+			require.Equal(t, pb.Object_DELETE_MARKER_VERSIONED, response.Object.Status)
+
+			objects, err = satelliteSys.Metabase.DB.TestingAllObjects(ctx)
+			require.NoError(t, err)
+			require.Len(t, objects, 3)
 		})
 	})
 }