JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/metainfo/metabase: add EncryptedMetainfoEncryptedKey parameter

Browse Source 
to metabase

* EncryptedMetainfoEncryptedKey added to CommitSegment and
UpdateMetadata request
* EncryptedMetainfoEncryptedKey returned with GetObject response and all
delete responses
* EncryptedMetainfoEncryptedKey returned with object iterator results

Change-Id: I917541ab5f3e1863bc8f238d17a15fbf72a23025
This commit is contained in:
Michal Niewrzal 2020-11-16 17:46:47 +01:00 committed by Fadila
parent 402cfcb7c6
commit b7df41a0a1
10 changed files with 119 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
satellite/metainfo/metabase/commit.go
View File

@ -391,8 +391,9 @@ func (db *DB) CommitInlineSegment(ctx context.Context, opts CommitInlineSegment)
type CommitObject struct {
ObjectStream
EncryptedMetadata []byte
EncryptedMetadataNonce []byte
EncryptedMetadata []byte
EncryptedMetadataNonce []byte
EncryptedMetadataEncryptedKey []byte
// TODO: proof
Proofs []SegmentProof
@ -491,11 +492,12 @@ func (db *DB) commitObjectWithoutProofs(ctx context.Context, opts CommitObject)
status = 1, -- committed
segment_count = $6,
encrypted_metadata_nonce = $7,
encrypted_metadata = $8,
encrypted_metadata_nonce = $7,
encrypted_metadata = $8,
encrypted_metadata_encrypted_key = $9,
total_encrypted_size = $9,
fixed_segment_size = $10,
total_encrypted_size = $10,
fixed_segment_size = $11,
zombie_deletion_deadline = NULL
WHERE
project_id = $1 AND
@ -509,7 +511,7 @@ func (db *DB) commitObjectWithoutProofs(ctx context.Context, opts CommitObject)
encryption;
`, opts.ProjectID, opts.BucketName, []byte(opts.ObjectKey), opts.Version, opts.StreamID,
len(segments),
opts.EncryptedMetadataNonce, opts.EncryptedMetadata,
opts.EncryptedMetadataNonce, opts.EncryptedMetadata, opts.EncryptedMetadataEncryptedKey,
totalEncryptedSize,
fixedSegmentSize,
).
@ -533,6 +535,7 @@ func (db *DB) commitObjectWithoutProofs(ctx context.Context, opts CommitObject)
object.SegmentCount = int32(len(segments))
object.EncryptedMetadataNonce = opts.EncryptedMetadataNonce
object.EncryptedMetadata = opts.EncryptedMetadata
object.EncryptedMetadataEncryptedKey = opts.EncryptedMetadataEncryptedKey
object.TotalEncryptedSize = totalEncryptedSize
object.FixedSegmentSize = fixedSegmentSize
return nil
@ -552,8 +555,9 @@ func (db *DB) commitObjectWithProofs(ctx context.Context, opts CommitObject) (ob
type UpdateObjectMetadata struct {
ObjectStream
EncryptedMetadata []byte
EncryptedMetadataNonce []byte
EncryptedMetadata []byte
EncryptedMetadataNonce []byte
EncryptedMetadataEncryptedKey []byte
}
// UpdateObjectMetadata updates an object metadata.
@ -575,8 +579,9 @@ func (db *DB) UpdateObjectMetadata(ctx context.Context, opts UpdateObjectMetadat
// during commit object.
result, err := db.db.ExecContext(ctx, `
UPDATE objects SET
encrypted_metadata_nonce = $6,
encrypted_metadata = $7
encrypted_metadata_nonce = $6,
encrypted_metadata = $7,
encrypted_metadata_encrypted_key = $8
WHERE
project_id = $1 AND
bucket_name = $2 AND
@ -585,7 +590,7 @@ func (db *DB) UpdateObjectMetadata(ctx context.Context, opts UpdateObjectMetadat
stream_id = $5 AND
status = 1
`, opts.ProjectID, opts.BucketName, []byte(opts.ObjectKey), opts.Version, opts.StreamID,
opts.EncryptedMetadataNonce, opts.EncryptedMetadata)
opts.EncryptedMetadataNonce, opts.EncryptedMetadata, opts.EncryptedMetadataEncryptedKey)
if err != nil {
return Error.New("unable to update object metadata: %w", err)
}

24
satellite/metainfo/metabase/commit_test.go
View File

@ -1467,6 +1467,7 @@ func TestCommitObject(t *testing.T) {
encryptedMetadata := testrand.Bytes(1024)
encryptedMetadataNonce := testrand.Nonce()
encryptedMetadataKey := testrand.Bytes(265)
CommitObject{
Opts: metabase.CommitObject{
@ -1477,8 +1478,9 @@ func TestCommitObject(t *testing.T) {
Version: 5,
StreamID: obj.StreamID,
},
EncryptedMetadataNonce: encryptedMetadataNonce[:],
EncryptedMetadata: encryptedMetadata,
EncryptedMetadataNonce: encryptedMetadataNonce[:],
EncryptedMetadata: encryptedMetadata,
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
},
}.Check(ctx, t, db)
@ -1510,8 +1512,9 @@ func TestCommitObject(t *testing.T) {
CreatedAt: now,
Status: metabase.Committed,
EncryptedMetadataNonce: encryptedMetadataNonce[:],
EncryptedMetadata: encryptedMetadata,
EncryptedMetadataNonce: encryptedMetadataNonce[:],
EncryptedMetadata: encryptedMetadata,
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
Encryption: defaultTestEncryption,
},
@ -1580,6 +1583,7 @@ func TestUpdateObjectMetadata(t *testing.T) {
encryptedMetadata := testrand.Bytes(1024)
encryptedMetadataNonce := testrand.Nonce()
encryptedMetadataKey := testrand.Bytes(265)
Verify{
Objects: []metabase.RawObject{
@ -1594,9 +1598,10 @@ func TestUpdateObjectMetadata(t *testing.T) {
UpdateObjectMetadata{
Opts: metabase.UpdateObjectMetadata{
ObjectStream: obj,
EncryptedMetadata: encryptedMetadata,
EncryptedMetadataNonce: encryptedMetadataNonce[:],
ObjectStream: obj,
EncryptedMetadata: encryptedMetadata,
EncryptedMetadataNonce: encryptedMetadataNonce[:],
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
},
}.Check(ctx, t, db)
@ -1608,8 +1613,9 @@ func TestUpdateObjectMetadata(t *testing.T) {
Status: metabase.Committed,
Encryption: defaultTestEncryption,
EncryptedMetadata: encryptedMetadata,
EncryptedMetadataNonce: encryptedMetadataNonce[:],
EncryptedMetadata: encryptedMetadata,
EncryptedMetadataNonce: encryptedMetadataNonce[:],
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
},
},
}.Check(ctx, t, db)

5
satellite/metainfo/metabase/db.go
View File

@ -96,8 +96,9 @@ func (db *DB) PostgresMigration() *migrate.Migration {
status INT2 NOT NULL default 0,
segment_count INT4 NOT NULL default 0,
encrypted_metadata_nonce BYTEA default NULL,
encrypted_metadata BYTEA default NULL,
encrypted_metadata_nonce BYTEA default NULL,
encrypted_metadata BYTEA default NULL,
encrypted_metadata_encrypted_key BYTEA default NULL,
total_encrypted_size INT4 NOT NULL default 0,
fixed_segment_size INT4 NOT NULL default 0,

12
satellite/metainfo/metabase/delete.go
View File

@ -115,7 +115,7 @@ func (db *DB) DeleteObjectExactVersion(ctx context.Context, opts DeleteObjectExa
version, stream_id,
created_at, expires_at,
status, segment_count,
encrypted_metadata_nonce, encrypted_metadata,
encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key,
total_encrypted_size, fixed_segment_size,
encryption;
`, opts.ProjectID, opts.BucketName, []byte(opts.ObjectKey), opts.Version)
@ -194,7 +194,7 @@ func (db *DB) DeleteObjectLatestVersion(ctx context.Context, opts DeleteObjectLa
version, stream_id,
created_at, expires_at,
status, segment_count,
encrypted_metadata_nonce, encrypted_metadata,
encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key,
total_encrypted_size, fixed_segment_size,
encryption;
`, opts.ProjectID, opts.BucketName, []byte(opts.ObjectKey))
@ -250,7 +250,7 @@ func (db *DB) DeleteObjectAllVersions(ctx context.Context, opts DeleteObjectAllV
version, stream_id,
created_at, expires_at,
status, segment_count,
encrypted_metadata_nonce, encrypted_metadata,
encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key,
total_encrypted_size, fixed_segment_size,
encryption;
`, opts.ProjectID, opts.BucketName, []byte(opts.ObjectKey))
@ -328,7 +328,7 @@ func (db *DB) DeleteObjectsAllVersions(ctx context.Context, opts DeleteObjectsAl
object_key, version, stream_id,
created_at, expires_at,
status, segment_count,
encrypted_metadata_nonce, encrypted_metadata,
encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key,
total_encrypted_size, fixed_segment_size,
encryption;
`, projectID, bucketName, pgutil.ByteaArray(objectKeys))
@ -378,7 +378,7 @@ func scanObjectDeletion(location ObjectLocation, rows tagsql.Rows) (objects []Ob
err = rows.Scan(&object.Version, &object.StreamID,
&object.CreatedAt, &object.ExpiresAt,
&object.Status, &object.SegmentCount,
&object.EncryptedMetadataNonce, &object.EncryptedMetadata,
&object.EncryptedMetadataNonce, &object.EncryptedMetadata, &object.EncryptedMetadataEncryptedKey,
&object.TotalEncryptedSize, &object.FixedSegmentSize,
encryptionParameters{&object.Encryption})
if err != nil {
@ -405,7 +405,7 @@ func scanMultipleObjectsDeletion(rows tagsql.Rows) (objects []Object, err error)
&object.ObjectKey, &object.Version, &object.StreamID,
&object.CreatedAt, &object.ExpiresAt,
&object.Status, &object.SegmentCount,
&object.EncryptedMetadataNonce, &object.EncryptedMetadata,
&object.EncryptedMetadataNonce, &object.EncryptedMetadata, &object.EncryptedMetadataEncryptedKey,
&object.TotalEncryptedSize, &object.FixedSegmentSize,
encryptionParameters{&object.Encryption})
if err != nil {

39
satellite/metainfo/metabase/delete_test.go
View File

@ -160,7 +160,18 @@ func TestDeleteObjectExactVersion(t *testing.T) {
t.Run("Delete object without segments", func(t *testing.T) {
defer DeleteAll{}.Check(ctx, t, db)
object := createObject(ctx, t, db, obj, 0)
encryptedMetadata := testrand.Bytes(1024)
encryptedMetadataNonce := testrand.Nonce()
encryptedMetadataKey := testrand.Bytes(265)
object := CreateTestObject{
CommitObject: &metabase.CommitObject{
ObjectStream: obj,
EncryptedMetadataNonce: encryptedMetadataNonce[:],
EncryptedMetadata: encryptedMetadata,
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
},
}.Run(ctx, t, db, obj, 0)
DeleteObjectExactVersion{
Opts: metabase.DeleteObjectExactVersion{
@ -324,7 +335,18 @@ func TestDeleteObjectLatestVersion(t *testing.T) {
t.Run("Delete object without segments", func(t *testing.T) {
defer DeleteAll{}.Check(ctx, t, db)
object := createObject(ctx, t, db, obj, 0)
encryptedMetadata := testrand.Bytes(1024)
encryptedMetadataNonce := testrand.Nonce()
encryptedMetadataKey := testrand.Bytes(265)
object := CreateTestObject{
CommitObject: &metabase.CommitObject{
ObjectStream: obj,
EncryptedMetadataNonce: encryptedMetadataNonce[:],
EncryptedMetadata: encryptedMetadata,
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
},
}.Run(ctx, t, db, obj, 0)
DeleteObjectLatestVersion{
Opts: metabase.DeleteObjectLatestVersion{
@ -552,7 +574,18 @@ func TestDeleteObjectAllVersions(t *testing.T) {
t.Run("Delete object without segments", func(t *testing.T) {
defer DeleteAll{}.Check(ctx, t, db)
object := createObject(ctx, t, db, obj, 0)
encryptedMetadata := testrand.Bytes(1024)
encryptedMetadataNonce := testrand.Nonce()
encryptedMetadataKey := testrand.Bytes(265)
object := CreateTestObject{
CommitObject: &metabase.CommitObject{
ObjectStream: obj,
EncryptedMetadataNonce: encryptedMetadataNonce[:],
EncryptedMetadata: encryptedMetadata,
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
},
}.Run(ctx, t, db, obj, 0)
DeleteObjectAllVersions{
Opts: metabase.DeleteObjectAllVersions{ObjectLocation: obj.Location()},

8
satellite/metainfo/metabase/get.go
View File

@ -53,7 +53,7 @@ func (db *DB) GetObjectExactVersion(ctx context.Context, opts GetObjectExactVers
stream_id,
created_at, expires_at,
segment_count,
encrypted_metadata_nonce, encrypted_metadata,
encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key,
total_encrypted_size, fixed_segment_size,
encryption
FROM objects
@ -68,7 +68,7 @@ func (db *DB) GetObjectExactVersion(ctx context.Context, opts GetObjectExactVers
&object.StreamID,
&object.CreatedAt, &object.ExpiresAt,
&object.SegmentCount,
&object.EncryptedMetadataNonce, &object.EncryptedMetadata,
&object.EncryptedMetadataNonce, &object.EncryptedMetadata, &object.EncryptedMetadataEncryptedKey,
&object.TotalEncryptedSize, &object.FixedSegmentSize,
encryptionParameters{&object.Encryption},
)
@ -109,7 +109,7 @@ func (db *DB) GetObjectLatestVersion(ctx context.Context, opts GetObjectLatestVe
stream_id, version,
created_at, expires_at,
segment_count,
encrypted_metadata_nonce, encrypted_metadata,
encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key,
total_encrypted_size, fixed_segment_size,
encryption
FROM objects
@ -125,7 +125,7 @@ func (db *DB) GetObjectLatestVersion(ctx context.Context, opts GetObjectLatestVe
&object.StreamID, &object.Version,
&object.CreatedAt, &object.ExpiresAt,
&object.SegmentCount,
&object.EncryptedMetadataNonce, &object.EncryptedMetadata,
&object.EncryptedMetadataNonce, &object.EncryptedMetadata, &object.EncryptedMetadataEncryptedKey,
&object.TotalEncryptedSize, &object.FixedSegmentSize,
encryptionParameters{&object.Encryption},
)

22
satellite/metainfo/metabase/get_test.go
View File

@ -9,6 +9,7 @@ import (
"storj.io/common/storj"
"storj.io/common/testcontext"
"storj.io/common/testrand"
"storj.io/storj/satellite/metainfo/metabase"
)
@ -233,11 +234,16 @@ func TestGetObjectLatestVersion(t *testing.T) {
t.Run("Get object", func(t *testing.T) {
defer DeleteAll{}.Check(ctx, t, db)
encryptedMetadata := testrand.Bytes(1024)
encryptedMetadataNonce := testrand.Nonce()
encryptedMetadataKey := testrand.Bytes(265)
CreateTestObject{
CommitObject: &metabase.CommitObject{
ObjectStream: obj,
EncryptedMetadataNonce: []byte("nonce"),
EncryptedMetadata: []byte("metadata"),
ObjectStream: obj,
EncryptedMetadataNonce: encryptedMetadataNonce[:],
EncryptedMetadata: encryptedMetadata,
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
},
}.Run(ctx, t, db, obj, 0)
@ -252,8 +258,9 @@ func TestGetObjectLatestVersion(t *testing.T) {
Encryption: defaultTestEncryption,
EncryptedMetadataNonce: []byte("nonce"),
EncryptedMetadata: []byte("metadata"),
EncryptedMetadataNonce: encryptedMetadataNonce[:],
EncryptedMetadata: encryptedMetadata,
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
},
}.Check(ctx, t, db)
@ -265,8 +272,9 @@ func TestGetObjectLatestVersion(t *testing.T) {
Encryption: defaultTestEncryption,
EncryptedMetadataNonce: []byte("nonce"),
EncryptedMetadata: []byte("metadata"),
EncryptedMetadataNonce: encryptedMetadataNonce[:],
EncryptedMetadata: encryptedMetadata,
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
},
}}.Check(ctx, t, db)
})

4
satellite/metainfo/metabase/iterator.go
View File

@ -146,7 +146,7 @@ func (it *objectsIterator) doNextQuery(ctx context.Context) (_ tagsql.Rows, err
object_key, stream_id, version, status,
created_at, expires_at,
segment_count,
encrypted_metadata_nonce, encrypted_metadata,
encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key,
total_encrypted_size, fixed_segment_size,
encryption
FROM objects
@ -171,7 +171,7 @@ func (it *objectsIterator) scanItem(item *ObjectEntry) error {
&item.ObjectKey, &item.StreamID, &item.Version, &item.Status,
&item.CreatedAt, &item.ExpiresAt,
&item.SegmentCount,
&item.EncryptedMetadataNonce, &item.EncryptedMetadata,
&item.EncryptedMetadataNonce, &item.EncryptedMetadata, &item.EncryptedMetadataEncryptedKey,
&item.TotalEncryptedSize, &item.FixedSegmentSize,
encryptionParameters{&item.Encryption},
)

21
satellite/metainfo/metabase/list_test.go
View File

@ -9,6 +9,7 @@ import (
"time"
"storj.io/common/testcontext"
"storj.io/common/testrand"
"storj.io/common/uuid"
"storj.io/storj/satellite/metainfo/metabase"
)
@ -110,6 +111,10 @@ func TestIterateObjects(t *testing.T) {
Version: 1,
}.Check(ctx, t, db)
encryptedMetadata := testrand.Bytes(1024)
encryptedMetadataNonce := testrand.Nonce()
encryptedMetadataKey := testrand.Bytes(265)
BeginObjectExactVersion{
Opts: metabase.BeginObjectExactVersion{
ObjectStream: committed,
@ -119,7 +124,10 @@ func TestIterateObjects(t *testing.T) {
}.Check(ctx, t, db)
CommitObject{
Opts: metabase.CommitObject{
ObjectStream: committed,
ObjectStream: committed,
EncryptedMetadataNonce: encryptedMetadataNonce[:],
EncryptedMetadata: encryptedMetadata,
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
},
}.Check(ctx, t, db)
@ -131,10 +139,13 @@ func TestIterateObjects(t *testing.T) {
Status: metabase.Committed,
},
Result: []metabase.ObjectEntry{{
ObjectStream: committed,
CreatedAt: now,
Status: metabase.Committed,
Encryption: defaultTestEncryption,
ObjectStream: committed,
CreatedAt: now,
Status: metabase.Committed,
Encryption: defaultTestEncryption,
EncryptedMetadataNonce: encryptedMetadataNonce[:],
EncryptedMetadata: encryptedMetadata,
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
}},
}.Check(ctx, t, db)

8
satellite/metainfo/metabase/raw.go
View File

@ -23,8 +23,9 @@ type RawObject struct {
Status ObjectStatus
SegmentCount int32
EncryptedMetadataNonce []byte
EncryptedMetadata []byte
EncryptedMetadataNonce []byte
EncryptedMetadata []byte
EncryptedMetadataEncryptedKey []byte
TotalEncryptedSize int64
FixedSegmentSize int32
@ -98,7 +99,7 @@ func (db *DB) testingGetAllObjects(ctx context.Context) (_ []RawObject, err erro
project_id, bucket_name, object_key, version, stream_id,
created_at, expires_at,
status, segment_count,
encrypted_metadata_nonce, encrypted_metadata,
encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key,
total_encrypted_size, fixed_segment_size,
encryption,
zombie_deletion_deadline
@ -126,6 +127,7 @@ func (db *DB) testingGetAllObjects(ctx context.Context) (_ []RawObject, err erro
&obj.EncryptedMetadataNonce,
&obj.EncryptedMetadata,
&obj.EncryptedMetadataEncryptedKey,
&obj.TotalEncryptedSize,
&obj.FixedSegmentSize,