diff --git a/satellite/metainfo/metabase/commit.go b/satellite/metainfo/metabase/commit.go index 5773c8725..599cb87b2 100644 --- a/satellite/metainfo/metabase/commit.go +++ b/satellite/metainfo/metabase/commit.go @@ -391,8 +391,9 @@ func (db *DB) CommitInlineSegment(ctx context.Context, opts CommitInlineSegment) type CommitObject struct { ObjectStream - EncryptedMetadata []byte - EncryptedMetadataNonce []byte + EncryptedMetadata []byte + EncryptedMetadataNonce []byte + EncryptedMetadataEncryptedKey []byte // TODO: proof Proofs []SegmentProof @@ -491,11 +492,12 @@ func (db *DB) commitObjectWithoutProofs(ctx context.Context, opts CommitObject) status = 1, -- committed segment_count = $6, - encrypted_metadata_nonce = $7, - encrypted_metadata = $8, + encrypted_metadata_nonce = $7, + encrypted_metadata = $8, + encrypted_metadata_encrypted_key = $9, - total_encrypted_size = $9, - fixed_segment_size = $10, + total_encrypted_size = $10, + fixed_segment_size = $11, zombie_deletion_deadline = NULL WHERE project_id = $1 AND @@ -509,7 +511,7 @@ func (db *DB) commitObjectWithoutProofs(ctx context.Context, opts CommitObject) encryption; `, opts.ProjectID, opts.BucketName, []byte(opts.ObjectKey), opts.Version, opts.StreamID, len(segments), - opts.EncryptedMetadataNonce, opts.EncryptedMetadata, + opts.EncryptedMetadataNonce, opts.EncryptedMetadata, opts.EncryptedMetadataEncryptedKey, totalEncryptedSize, fixedSegmentSize, ). @@ -533,6 +535,7 @@ func (db *DB) commitObjectWithoutProofs(ctx context.Context, opts CommitObject) object.SegmentCount = int32(len(segments)) object.EncryptedMetadataNonce = opts.EncryptedMetadataNonce object.EncryptedMetadata = opts.EncryptedMetadata + object.EncryptedMetadataEncryptedKey = opts.EncryptedMetadataEncryptedKey object.TotalEncryptedSize = totalEncryptedSize object.FixedSegmentSize = fixedSegmentSize return nil @@ -552,8 +555,9 @@ func (db *DB) commitObjectWithProofs(ctx context.Context, opts CommitObject) (ob type UpdateObjectMetadata struct { ObjectStream - EncryptedMetadata []byte - EncryptedMetadataNonce []byte + EncryptedMetadata []byte + EncryptedMetadataNonce []byte + EncryptedMetadataEncryptedKey []byte } // UpdateObjectMetadata updates an object metadata. @@ -575,8 +579,9 @@ func (db *DB) UpdateObjectMetadata(ctx context.Context, opts UpdateObjectMetadat // during commit object. result, err := db.db.ExecContext(ctx, ` UPDATE objects SET - encrypted_metadata_nonce = $6, - encrypted_metadata = $7 + encrypted_metadata_nonce = $6, + encrypted_metadata = $7, + encrypted_metadata_encrypted_key = $8 WHERE project_id = $1 AND bucket_name = $2 AND @@ -585,7 +590,7 @@ func (db *DB) UpdateObjectMetadata(ctx context.Context, opts UpdateObjectMetadat stream_id = $5 AND status = 1 `, opts.ProjectID, opts.BucketName, []byte(opts.ObjectKey), opts.Version, opts.StreamID, - opts.EncryptedMetadataNonce, opts.EncryptedMetadata) + opts.EncryptedMetadataNonce, opts.EncryptedMetadata, opts.EncryptedMetadataEncryptedKey) if err != nil { return Error.New("unable to update object metadata: %w", err) } diff --git a/satellite/metainfo/metabase/commit_test.go b/satellite/metainfo/metabase/commit_test.go index 63b2fe352..ffda56c30 100644 --- a/satellite/metainfo/metabase/commit_test.go +++ b/satellite/metainfo/metabase/commit_test.go @@ -1467,6 +1467,7 @@ func TestCommitObject(t *testing.T) { encryptedMetadata := testrand.Bytes(1024) encryptedMetadataNonce := testrand.Nonce() + encryptedMetadataKey := testrand.Bytes(265) CommitObject{ Opts: metabase.CommitObject{ @@ -1477,8 +1478,9 @@ func TestCommitObject(t *testing.T) { Version: 5, StreamID: obj.StreamID, }, - EncryptedMetadataNonce: encryptedMetadataNonce[:], - EncryptedMetadata: encryptedMetadata, + EncryptedMetadataNonce: encryptedMetadataNonce[:], + EncryptedMetadata: encryptedMetadata, + EncryptedMetadataEncryptedKey: encryptedMetadataKey, }, }.Check(ctx, t, db) @@ -1510,8 +1512,9 @@ func TestCommitObject(t *testing.T) { CreatedAt: now, Status: metabase.Committed, - EncryptedMetadataNonce: encryptedMetadataNonce[:], - EncryptedMetadata: encryptedMetadata, + EncryptedMetadataNonce: encryptedMetadataNonce[:], + EncryptedMetadata: encryptedMetadata, + EncryptedMetadataEncryptedKey: encryptedMetadataKey, Encryption: defaultTestEncryption, }, @@ -1580,6 +1583,7 @@ func TestUpdateObjectMetadata(t *testing.T) { encryptedMetadata := testrand.Bytes(1024) encryptedMetadataNonce := testrand.Nonce() + encryptedMetadataKey := testrand.Bytes(265) Verify{ Objects: []metabase.RawObject{ @@ -1594,9 +1598,10 @@ func TestUpdateObjectMetadata(t *testing.T) { UpdateObjectMetadata{ Opts: metabase.UpdateObjectMetadata{ - ObjectStream: obj, - EncryptedMetadata: encryptedMetadata, - EncryptedMetadataNonce: encryptedMetadataNonce[:], + ObjectStream: obj, + EncryptedMetadata: encryptedMetadata, + EncryptedMetadataNonce: encryptedMetadataNonce[:], + EncryptedMetadataEncryptedKey: encryptedMetadataKey, }, }.Check(ctx, t, db) @@ -1608,8 +1613,9 @@ func TestUpdateObjectMetadata(t *testing.T) { Status: metabase.Committed, Encryption: defaultTestEncryption, - EncryptedMetadata: encryptedMetadata, - EncryptedMetadataNonce: encryptedMetadataNonce[:], + EncryptedMetadata: encryptedMetadata, + EncryptedMetadataNonce: encryptedMetadataNonce[:], + EncryptedMetadataEncryptedKey: encryptedMetadataKey, }, }, }.Check(ctx, t, db) diff --git a/satellite/metainfo/metabase/db.go b/satellite/metainfo/metabase/db.go index d718f46a9..0b2fda84a 100644 --- a/satellite/metainfo/metabase/db.go +++ b/satellite/metainfo/metabase/db.go @@ -96,8 +96,9 @@ func (db *DB) PostgresMigration() *migrate.Migration { status INT2 NOT NULL default 0, segment_count INT4 NOT NULL default 0, - encrypted_metadata_nonce BYTEA default NULL, - encrypted_metadata BYTEA default NULL, + encrypted_metadata_nonce BYTEA default NULL, + encrypted_metadata BYTEA default NULL, + encrypted_metadata_encrypted_key BYTEA default NULL, total_encrypted_size INT4 NOT NULL default 0, fixed_segment_size INT4 NOT NULL default 0, diff --git a/satellite/metainfo/metabase/delete.go b/satellite/metainfo/metabase/delete.go index 1a81a36b7..9982583ce 100644 --- a/satellite/metainfo/metabase/delete.go +++ b/satellite/metainfo/metabase/delete.go @@ -115,7 +115,7 @@ func (db *DB) DeleteObjectExactVersion(ctx context.Context, opts DeleteObjectExa version, stream_id, created_at, expires_at, status, segment_count, - encrypted_metadata_nonce, encrypted_metadata, + encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key, total_encrypted_size, fixed_segment_size, encryption; `, opts.ProjectID, opts.BucketName, []byte(opts.ObjectKey), opts.Version) @@ -194,7 +194,7 @@ func (db *DB) DeleteObjectLatestVersion(ctx context.Context, opts DeleteObjectLa version, stream_id, created_at, expires_at, status, segment_count, - encrypted_metadata_nonce, encrypted_metadata, + encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key, total_encrypted_size, fixed_segment_size, encryption; `, opts.ProjectID, opts.BucketName, []byte(opts.ObjectKey)) @@ -250,7 +250,7 @@ func (db *DB) DeleteObjectAllVersions(ctx context.Context, opts DeleteObjectAllV version, stream_id, created_at, expires_at, status, segment_count, - encrypted_metadata_nonce, encrypted_metadata, + encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key, total_encrypted_size, fixed_segment_size, encryption; `, opts.ProjectID, opts.BucketName, []byte(opts.ObjectKey)) @@ -328,7 +328,7 @@ func (db *DB) DeleteObjectsAllVersions(ctx context.Context, opts DeleteObjectsAl object_key, version, stream_id, created_at, expires_at, status, segment_count, - encrypted_metadata_nonce, encrypted_metadata, + encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key, total_encrypted_size, fixed_segment_size, encryption; `, projectID, bucketName, pgutil.ByteaArray(objectKeys)) @@ -378,7 +378,7 @@ func scanObjectDeletion(location ObjectLocation, rows tagsql.Rows) (objects []Ob err = rows.Scan(&object.Version, &object.StreamID, &object.CreatedAt, &object.ExpiresAt, &object.Status, &object.SegmentCount, - &object.EncryptedMetadataNonce, &object.EncryptedMetadata, + &object.EncryptedMetadataNonce, &object.EncryptedMetadata, &object.EncryptedMetadataEncryptedKey, &object.TotalEncryptedSize, &object.FixedSegmentSize, encryptionParameters{&object.Encryption}) if err != nil { @@ -405,7 +405,7 @@ func scanMultipleObjectsDeletion(rows tagsql.Rows) (objects []Object, err error) &object.ObjectKey, &object.Version, &object.StreamID, &object.CreatedAt, &object.ExpiresAt, &object.Status, &object.SegmentCount, - &object.EncryptedMetadataNonce, &object.EncryptedMetadata, + &object.EncryptedMetadataNonce, &object.EncryptedMetadata, &object.EncryptedMetadataEncryptedKey, &object.TotalEncryptedSize, &object.FixedSegmentSize, encryptionParameters{&object.Encryption}) if err != nil { diff --git a/satellite/metainfo/metabase/delete_test.go b/satellite/metainfo/metabase/delete_test.go index 827e18f68..7b7873409 100644 --- a/satellite/metainfo/metabase/delete_test.go +++ b/satellite/metainfo/metabase/delete_test.go @@ -160,7 +160,18 @@ func TestDeleteObjectExactVersion(t *testing.T) { t.Run("Delete object without segments", func(t *testing.T) { defer DeleteAll{}.Check(ctx, t, db) - object := createObject(ctx, t, db, obj, 0) + encryptedMetadata := testrand.Bytes(1024) + encryptedMetadataNonce := testrand.Nonce() + encryptedMetadataKey := testrand.Bytes(265) + + object := CreateTestObject{ + CommitObject: &metabase.CommitObject{ + ObjectStream: obj, + EncryptedMetadataNonce: encryptedMetadataNonce[:], + EncryptedMetadata: encryptedMetadata, + EncryptedMetadataEncryptedKey: encryptedMetadataKey, + }, + }.Run(ctx, t, db, obj, 0) DeleteObjectExactVersion{ Opts: metabase.DeleteObjectExactVersion{ @@ -324,7 +335,18 @@ func TestDeleteObjectLatestVersion(t *testing.T) { t.Run("Delete object without segments", func(t *testing.T) { defer DeleteAll{}.Check(ctx, t, db) - object := createObject(ctx, t, db, obj, 0) + encryptedMetadata := testrand.Bytes(1024) + encryptedMetadataNonce := testrand.Nonce() + encryptedMetadataKey := testrand.Bytes(265) + + object := CreateTestObject{ + CommitObject: &metabase.CommitObject{ + ObjectStream: obj, + EncryptedMetadataNonce: encryptedMetadataNonce[:], + EncryptedMetadata: encryptedMetadata, + EncryptedMetadataEncryptedKey: encryptedMetadataKey, + }, + }.Run(ctx, t, db, obj, 0) DeleteObjectLatestVersion{ Opts: metabase.DeleteObjectLatestVersion{ @@ -552,7 +574,18 @@ func TestDeleteObjectAllVersions(t *testing.T) { t.Run("Delete object without segments", func(t *testing.T) { defer DeleteAll{}.Check(ctx, t, db) - object := createObject(ctx, t, db, obj, 0) + encryptedMetadata := testrand.Bytes(1024) + encryptedMetadataNonce := testrand.Nonce() + encryptedMetadataKey := testrand.Bytes(265) + + object := CreateTestObject{ + CommitObject: &metabase.CommitObject{ + ObjectStream: obj, + EncryptedMetadataNonce: encryptedMetadataNonce[:], + EncryptedMetadata: encryptedMetadata, + EncryptedMetadataEncryptedKey: encryptedMetadataKey, + }, + }.Run(ctx, t, db, obj, 0) DeleteObjectAllVersions{ Opts: metabase.DeleteObjectAllVersions{ObjectLocation: obj.Location()}, diff --git a/satellite/metainfo/metabase/get.go b/satellite/metainfo/metabase/get.go index 067e67c86..c0b933edf 100644 --- a/satellite/metainfo/metabase/get.go +++ b/satellite/metainfo/metabase/get.go @@ -53,7 +53,7 @@ func (db *DB) GetObjectExactVersion(ctx context.Context, opts GetObjectExactVers stream_id, created_at, expires_at, segment_count, - encrypted_metadata_nonce, encrypted_metadata, + encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key, total_encrypted_size, fixed_segment_size, encryption FROM objects @@ -68,7 +68,7 @@ func (db *DB) GetObjectExactVersion(ctx context.Context, opts GetObjectExactVers &object.StreamID, &object.CreatedAt, &object.ExpiresAt, &object.SegmentCount, - &object.EncryptedMetadataNonce, &object.EncryptedMetadata, + &object.EncryptedMetadataNonce, &object.EncryptedMetadata, &object.EncryptedMetadataEncryptedKey, &object.TotalEncryptedSize, &object.FixedSegmentSize, encryptionParameters{&object.Encryption}, ) @@ -109,7 +109,7 @@ func (db *DB) GetObjectLatestVersion(ctx context.Context, opts GetObjectLatestVe stream_id, version, created_at, expires_at, segment_count, - encrypted_metadata_nonce, encrypted_metadata, + encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key, total_encrypted_size, fixed_segment_size, encryption FROM objects @@ -125,7 +125,7 @@ func (db *DB) GetObjectLatestVersion(ctx context.Context, opts GetObjectLatestVe &object.StreamID, &object.Version, &object.CreatedAt, &object.ExpiresAt, &object.SegmentCount, - &object.EncryptedMetadataNonce, &object.EncryptedMetadata, + &object.EncryptedMetadataNonce, &object.EncryptedMetadata, &object.EncryptedMetadataEncryptedKey, &object.TotalEncryptedSize, &object.FixedSegmentSize, encryptionParameters{&object.Encryption}, ) diff --git a/satellite/metainfo/metabase/get_test.go b/satellite/metainfo/metabase/get_test.go index 697450715..4f779696b 100644 --- a/satellite/metainfo/metabase/get_test.go +++ b/satellite/metainfo/metabase/get_test.go @@ -9,6 +9,7 @@ import ( "storj.io/common/storj" "storj.io/common/testcontext" + "storj.io/common/testrand" "storj.io/storj/satellite/metainfo/metabase" ) @@ -233,11 +234,16 @@ func TestGetObjectLatestVersion(t *testing.T) { t.Run("Get object", func(t *testing.T) { defer DeleteAll{}.Check(ctx, t, db) + encryptedMetadata := testrand.Bytes(1024) + encryptedMetadataNonce := testrand.Nonce() + encryptedMetadataKey := testrand.Bytes(265) + CreateTestObject{ CommitObject: &metabase.CommitObject{ - ObjectStream: obj, - EncryptedMetadataNonce: []byte("nonce"), - EncryptedMetadata: []byte("metadata"), + ObjectStream: obj, + EncryptedMetadataNonce: encryptedMetadataNonce[:], + EncryptedMetadata: encryptedMetadata, + EncryptedMetadataEncryptedKey: encryptedMetadataKey, }, }.Run(ctx, t, db, obj, 0) @@ -252,8 +258,9 @@ func TestGetObjectLatestVersion(t *testing.T) { Encryption: defaultTestEncryption, - EncryptedMetadataNonce: []byte("nonce"), - EncryptedMetadata: []byte("metadata"), + EncryptedMetadataNonce: encryptedMetadataNonce[:], + EncryptedMetadata: encryptedMetadata, + EncryptedMetadataEncryptedKey: encryptedMetadataKey, }, }.Check(ctx, t, db) @@ -265,8 +272,9 @@ func TestGetObjectLatestVersion(t *testing.T) { Encryption: defaultTestEncryption, - EncryptedMetadataNonce: []byte("nonce"), - EncryptedMetadata: []byte("metadata"), + EncryptedMetadataNonce: encryptedMetadataNonce[:], + EncryptedMetadata: encryptedMetadata, + EncryptedMetadataEncryptedKey: encryptedMetadataKey, }, }}.Check(ctx, t, db) }) diff --git a/satellite/metainfo/metabase/iterator.go b/satellite/metainfo/metabase/iterator.go index 780b049bc..593485a4e 100644 --- a/satellite/metainfo/metabase/iterator.go +++ b/satellite/metainfo/metabase/iterator.go @@ -146,7 +146,7 @@ func (it *objectsIterator) doNextQuery(ctx context.Context) (_ tagsql.Rows, err object_key, stream_id, version, status, created_at, expires_at, segment_count, - encrypted_metadata_nonce, encrypted_metadata, + encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key, total_encrypted_size, fixed_segment_size, encryption FROM objects @@ -171,7 +171,7 @@ func (it *objectsIterator) scanItem(item *ObjectEntry) error { &item.ObjectKey, &item.StreamID, &item.Version, &item.Status, &item.CreatedAt, &item.ExpiresAt, &item.SegmentCount, - &item.EncryptedMetadataNonce, &item.EncryptedMetadata, + &item.EncryptedMetadataNonce, &item.EncryptedMetadata, &item.EncryptedMetadataEncryptedKey, &item.TotalEncryptedSize, &item.FixedSegmentSize, encryptionParameters{&item.Encryption}, ) diff --git a/satellite/metainfo/metabase/list_test.go b/satellite/metainfo/metabase/list_test.go index a54f7b5da..5788f283a 100644 --- a/satellite/metainfo/metabase/list_test.go +++ b/satellite/metainfo/metabase/list_test.go @@ -9,6 +9,7 @@ import ( "time" "storj.io/common/testcontext" + "storj.io/common/testrand" "storj.io/common/uuid" "storj.io/storj/satellite/metainfo/metabase" ) @@ -110,6 +111,10 @@ func TestIterateObjects(t *testing.T) { Version: 1, }.Check(ctx, t, db) + encryptedMetadata := testrand.Bytes(1024) + encryptedMetadataNonce := testrand.Nonce() + encryptedMetadataKey := testrand.Bytes(265) + BeginObjectExactVersion{ Opts: metabase.BeginObjectExactVersion{ ObjectStream: committed, @@ -119,7 +124,10 @@ func TestIterateObjects(t *testing.T) { }.Check(ctx, t, db) CommitObject{ Opts: metabase.CommitObject{ - ObjectStream: committed, + ObjectStream: committed, + EncryptedMetadataNonce: encryptedMetadataNonce[:], + EncryptedMetadata: encryptedMetadata, + EncryptedMetadataEncryptedKey: encryptedMetadataKey, }, }.Check(ctx, t, db) @@ -131,10 +139,13 @@ func TestIterateObjects(t *testing.T) { Status: metabase.Committed, }, Result: []metabase.ObjectEntry{{ - ObjectStream: committed, - CreatedAt: now, - Status: metabase.Committed, - Encryption: defaultTestEncryption, + ObjectStream: committed, + CreatedAt: now, + Status: metabase.Committed, + Encryption: defaultTestEncryption, + EncryptedMetadataNonce: encryptedMetadataNonce[:], + EncryptedMetadata: encryptedMetadata, + EncryptedMetadataEncryptedKey: encryptedMetadataKey, }}, }.Check(ctx, t, db) diff --git a/satellite/metainfo/metabase/raw.go b/satellite/metainfo/metabase/raw.go index 8039696ce..d22d68f21 100644 --- a/satellite/metainfo/metabase/raw.go +++ b/satellite/metainfo/metabase/raw.go @@ -23,8 +23,9 @@ type RawObject struct { Status ObjectStatus SegmentCount int32 - EncryptedMetadataNonce []byte - EncryptedMetadata []byte + EncryptedMetadataNonce []byte + EncryptedMetadata []byte + EncryptedMetadataEncryptedKey []byte TotalEncryptedSize int64 FixedSegmentSize int32 @@ -98,7 +99,7 @@ func (db *DB) testingGetAllObjects(ctx context.Context) (_ []RawObject, err erro project_id, bucket_name, object_key, version, stream_id, created_at, expires_at, status, segment_count, - encrypted_metadata_nonce, encrypted_metadata, + encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key, total_encrypted_size, fixed_segment_size, encryption, zombie_deletion_deadline @@ -126,6 +127,7 @@ func (db *DB) testingGetAllObjects(ctx context.Context) (_ []RawObject, err erro &obj.EncryptedMetadataNonce, &obj.EncryptedMetadata, + &obj.EncryptedMetadataEncryptedKey, &obj.TotalEncryptedSize, &obj.FixedSegmentSize,