JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/orders: use smaller encrypted metadata

Browse Source 
Avoid using project uuid string representation, because
it uses more bandwidth.

This reduces the encrypted metadata size from 118 -> 97 bytes.

Change-Id: Ic53a81b83acc065f24f28cd404f9c0b1fe592594
This commit is contained in:
Egon Elbre 2021-01-08 18:04:46 +02:00
parent 6507f3ebc6
commit 51731db121
11 changed files with 206 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
private/testuplink/client_planet_test.go
View File

@ -23,6 +23,7 @@ import (
"storj.io/common/testcontext"
"storj.io/common/testrand"
"storj.io/storj/private/testplanet"
"storj.io/storj/satellite/internalpb"
"storj.io/uplink/private/ecclient"
"storj.io/uplink/private/eestream"
)
@ -132,8 +133,8 @@ func newAddressedOrderLimit(ctx context.Context, action pb.PieceAction, satellit
key := satellite.Config.Orders.EncryptionKeys.Default
encrypted, err := key.EncryptMetadata(
serialNumber,
&pb.OrderLimitMetadata{
ProjectBucketPrefix: []byte("testprojectid/testbucketname"),
&internalpb.OrderLimitMetadata{
CompactProjectBucketPrefix: []byte("0000111122223333testbucketname"),
},
)
if err != nil {

100
satellite/internalpb/ordersmeta.pb.go Normal file
View File

@ -0,0 +1,100 @@
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// source: ordersmeta.proto
package internalpb
import (
fmt "fmt"
math "math"
proto "github.com/gogo/protobuf/proto"
)
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
// OrderLimitMetadata is used to transmit meta information about an order limit.
// This data will be encrypted.
type OrderLimitMetadata struct {
BucketId []byte `protobuf:"bytes,1,opt,name=bucket_id,json=bucketId,proto3" json:"bucket_id,omitempty"`
ProjectBucketPrefix []byte `protobuf:"bytes,2,opt,name=project_bucket_prefix,json=projectBucketPrefix,proto3" json:"project_bucket_prefix,omitempty"`
CompactProjectBucketPrefix []byte `protobuf:"bytes,3,opt,name=compact_project_bucket_prefix,json=compactProjectBucketPrefix,proto3" json:"compact_project_bucket_prefix,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *OrderLimitMetadata) Reset() { *m = OrderLimitMetadata{} }
func (m *OrderLimitMetadata) String() string { return proto.CompactTextString(m) }
func (*OrderLimitMetadata) ProtoMessage() {}
func (*OrderLimitMetadata) Descriptor() ([]byte, []int) {
return fileDescriptor_e00ef1afe54bb544, []int{0}
}
func (m *OrderLimitMetadata) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_OrderLimitMetadata.Unmarshal(m, b)
}
func (m *OrderLimitMetadata) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_OrderLimitMetadata.Marshal(b, m, deterministic)
}
func (m *OrderLimitMetadata) XXX_Merge(src proto.Message) {
xxx_messageInfo_OrderLimitMetadata.Merge(m, src)
}
func (m *OrderLimitMetadata) XXX_Size() int {
return xxx_messageInfo_OrderLimitMetadata.Size(m)
}
func (m *OrderLimitMetadata) XXX_DiscardUnknown() {
xxx_messageInfo_OrderLimitMetadata.DiscardUnknown(m)
}
var xxx_messageInfo_OrderLimitMetadata proto.InternalMessageInfo
func (m *OrderLimitMetadata) GetBucketId() []byte {
if m != nil {
return m.BucketId
}
return nil
}
func (m *OrderLimitMetadata) GetProjectBucketPrefix() []byte {
if m != nil {
return m.ProjectBucketPrefix
}
return nil
}
func (m *OrderLimitMetadata) GetCompactProjectBucketPrefix() []byte {
if m != nil {
return m.CompactProjectBucketPrefix
}
return nil
}
func init() {
proto.RegisterType((*OrderLimitMetadata)(nil), "satellite.ordersmeta.OrderLimitMetadata")
}
func init() { proto.RegisterFile("ordersmeta.proto", fileDescriptor_e00ef1afe54bb544) }
var fileDescriptor_e00ef1afe54bb544 = []byte{
// 192 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0xc8, 0x2f, 0x4a, 0x49,
0x2d, 0x2a, 0xce, 0x4d, 0x2d, 0x49, 0xd4, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0x29, 0x4e,
0x2c, 0x49, 0xcd, 0xc9, 0xc9, 0x2c, 0x49, 0xd5, 0x43, 0xc8, 0x29, 0xad, 0x60, 0xe4, 0x12, 0xf2,
0x07, 0x71, 0x7d, 0x32, 0x73, 0x33, 0x4b, 0x7c, 0x53, 0x4b, 0x12, 0x53, 0x12, 0x4b, 0x12, 0x85,
0xa4, 0xb9, 0x38, 0x93, 0x4a, 0x93, 0xb3, 0x53, 0x4b, 0xe2, 0x33, 0x53, 0x24, 0x18, 0x15, 0x18,
0x35, 0x78, 0x82, 0x38, 0x20, 0x02, 0x9e, 0x29, 0x42, 0x46, 0x5c, 0xa2, 0x05, 0x45, 0xf9, 0x59,
0xa9, 0xc9, 0x25, 0xf1, 0x50, 0x45, 0x05, 0x45, 0xa9, 0x69, 0x99, 0x15, 0x12, 0x4c, 0x60, 0x85,
0xc2, 0x50, 0x49, 0x27, 0xb0, 0x5c, 0x00, 0x58, 0x4a, 0xc8, 0x91, 0x4b, 0x36, 0x39, 0x3f, 0xb7,
0x20, 0x31, 0x19, 0xa4, 0x18, 0x9b, 0x5e, 0x66, 0xb0, 0x5e, 0x29, 0xa8, 0xa2, 0x00, 0x4c, 0x23,
0x9c, 0x54, 0xa3, 0x94, 0x8b, 0x4b, 0xf2, 0x8b, 0xb2, 0xf4, 0x32, 0xf3, 0xf5, 0xc1, 0x0c, 0x7d,
0xb8, 0x8f, 0xf4, 0x33, 0xf3, 0x4a, 0x52, 0x8b, 0xf2, 0x12, 0x73, 0x0a, 0x92, 0x92, 0xd8, 0xc0,
0xde, 0x35, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0x00, 0xcc, 0x15, 0xb0, 0x02, 0x01, 0x00, 0x00,
}

15
satellite/internalpb/ordersmeta.proto Normal file
View File

@ -0,0 +1,15 @@
// Copyright (C) 2021 Storj Labs, Inc.
// See LICENSE for copying information.
syntax = "proto3";
option go_package = "storj.io/storj/satellite/internalpb";
package satellite.ordersmeta;
// OrderLimitMetadata is used to transmit meta information about an order limit.
// This data will be encrypted.
message OrderLimitMetadata {
bytes bucket_id = 1;
bytes project_bucket_prefix = 2;
bytes compact_project_bucket_prefix = 3;
}

20
satellite/metainfo/metabase/common.go
View File

@ -50,11 +50,31 @@ func ParseBucketPrefix(prefix BucketPrefix) (BucketLocation, error) {
}, nil
}
// ParseCompactBucketPrefix parses BucketPrefix.
func ParseCompactBucketPrefix(compactPrefix []byte) (BucketLocation, error) {
if len(compactPrefix) < 16 {
return BucketLocation{}, Error.New("invalid prefix %q", compactPrefix)
}
var loc BucketLocation
copy(loc.ProjectID[:], compactPrefix)
loc.BucketName = string(compactPrefix[16:])
return loc, nil
}
// Prefix converts bucket location into bucket prefix.
func (loc BucketLocation) Prefix() BucketPrefix {
return BucketPrefix(loc.ProjectID.String() + "/" + loc.BucketName)
}
// CompactPrefix converts bucket location into bucket prefix with compact project ID.
func (loc BucketLocation) CompactPrefix() []byte {
xs := make([]byte, 0, 16+len(loc.BucketName))
xs = append(xs, loc.ProjectID[:]...)
xs = append(xs, []byte(loc.BucketName)...)
return xs
}
// ObjectKey is an encrypted object key encoded using Path Component Encoding.
// It is not ascii safe.
type ObjectKey string

7
satellite/orders/encryptionkey.go
View File

@ -13,6 +13,7 @@ import (
"storj.io/common/pb"
"storj.io/common/storj"
"storj.io/storj/satellite/internalpb"
)
// ErrEncryptionKey is error class used for keys.
@ -82,7 +83,7 @@ func (key *EncryptionKey) Decrypt(ciphertext []byte, nonce storj.SerialNumber) (
}
// EncryptMetadata encrypts order limit metadata.
func (key *EncryptionKey) EncryptMetadata(serial storj.SerialNumber, metadata *pb.OrderLimitMetadata) ([]byte, error) {
func (key *EncryptionKey) EncryptMetadata(serial storj.SerialNumber, metadata *internalpb.OrderLimitMetadata) ([]byte, error) {
marshaled, err := pb.Marshal(metadata)
if err != nil {
return nil, ErrEncryptionKey.Wrap(err)
@ -91,13 +92,13 @@ func (key *EncryptionKey) EncryptMetadata(serial storj.SerialNumber, metadata *p
}
// DecryptMetadata decrypts order limit metadata.
func (key *EncryptionKey) DecryptMetadata(serial storj.SerialNumber, encrypted []byte) (*pb.OrderLimitMetadata, error) {
func (key *EncryptionKey) DecryptMetadata(serial storj.SerialNumber, encrypted []byte) (*internalpb.OrderLimitMetadata, error) {
decrypted, err := key.Decrypt(encrypted, serial)
if err != nil {
return nil, ErrEncryptionKey.Wrap(err)
}
metadata := &pb.OrderLimitMetadata{}
metadata := &internalpb.OrderLimitMetadata{}
err = pb.Unmarshal(decrypted, metadata)
if err != nil {
return nil, ErrEncryptionKey.Wrap(err)

27
satellite/orders/endpoint.go
View File

@ -649,14 +649,29 @@ func (endpoint *Endpoint) SettlementWithWindowFinal(stream pb.DRPCOrders_Settlem
mon.Event("bucketinfo_from_orders_metadata_error_1")
continue
}
bucketInfo, err := metabase.ParseBucketPrefix(
metabase.BucketPrefix(metadata.GetProjectBucketPrefix()),
)
if err != nil {
log.Debug("decrypt order: ParseBucketPrefix", zap.Error(err))
mon.Event("bucketinfo_from_orders_metadata_error_2")
var bucketInfo metabase.BucketLocation
switch {
case len(metadata.CompactProjectBucketPrefix) > 0:
bucketInfo, err = metabase.ParseCompactBucketPrefix(metadata.GetCompactProjectBucketPrefix())
if err != nil {
log.Debug("decrypt order: ParseCompactBucketPrefix", zap.Error(err))
mon.Event("bucketinfo_from_orders_metadata_error_compact")
continue
}
case len(metadata.ProjectBucketPrefix) > 0:
bucketInfo, err = metabase.ParseBucketPrefix(metabase.BucketPrefix(metadata.GetProjectBucketPrefix()))
if err != nil {
log.Debug("decrypt order: ParseBucketPrefix", zap.Error(err))
mon.Event("bucketinfo_from_orders_metadata_error_uncompact")
continue
}
default:
log.Debug("decrypt order: project bucket prefix missing", zap.Error(err))
mon.Event("bucketinfo_from_orders_metadata_error_default")
continue
}
if bucketInfo.BucketName == "" || bucketInfo.ProjectID.IsZero() {
log.Info("decrypt order: bucketName or projectID not set",
zap.String("bucketName", bucketInfo.BucketName),

49
satellite/orders/endpoint_test.go
View File

@ -20,6 +20,8 @@ import (
"storj.io/common/testrand"
"storj.io/storj/private/testplanet"
"storj.io/storj/satellite"
"storj.io/storj/satellite/internalpb"
"storj.io/storj/satellite/metainfo/metabase"
"storj.io/storj/satellite/orders"
)
@ -52,7 +54,10 @@ func TestSettlementWithWindowEndpointManyOrders(t *testing.T) {
now := time.Now()
projectID := testrand.UUID()
bucketname := "testbucket"
bucketID := storj.JoinPaths(projectID.String(), bucketname)
bucketLocation := metabase.BucketLocation{
ProjectID: projectID,
BucketName: bucketname,
}
key := satellite.Config.Orders.EncryptionKeys.Default
// stop any async flushes because we want to be sure when some values are
@ -84,8 +89,8 @@ func TestSettlementWithWindowEndpointManyOrders(t *testing.T) {
serialNumber1 := testrand.SerialNumber()
encrypted1, err := key.EncryptMetadata(
serialNumber1,
&pb.OrderLimitMetadata{
ProjectBucketPrefix: []byte(bucketID),
&internalpb.OrderLimitMetadata{
CompactProjectBucketPrefix: bucketLocation.CompactPrefix(),
},
)
require.NoError(t, err)
@ -93,8 +98,8 @@ func TestSettlementWithWindowEndpointManyOrders(t *testing.T) {
serialNumber2 := testrand.SerialNumber()
encrypted2, err := key.EncryptMetadata(
serialNumber2,
&pb.OrderLimitMetadata{
ProjectBucketPrefix: []byte(bucketID),
&internalpb.OrderLimitMetadata{
CompactProjectBucketPrefix: bucketLocation.CompactPrefix(),
},
)
require.NoError(t, err)
@ -210,7 +215,10 @@ func TestSettlementWithWindowEndpointSingleOrder(t *testing.T) {
now := time.Now()
projectID := testrand.UUID()
bucketname := "testbucket"
bucketID := storj.JoinPaths(projectID.String(), bucketname)
bucketLocation := metabase.BucketLocation{
ProjectID: projectID,
BucketName: bucketname,
}
key := satellite.Config.Orders.EncryptionKeys.Default
// stop any async flushes because we want to be sure when some values are
@ -231,8 +239,8 @@ func TestSettlementWithWindowEndpointSingleOrder(t *testing.T) {
serialNumber := testrand.SerialNumber()
encrypted, err := key.EncryptMetadata(
serialNumber,
&pb.OrderLimitMetadata{
ProjectBucketPrefix: []byte(bucketID),
&internalpb.OrderLimitMetadata{
CompactProjectBucketPrefix: bucketLocation.CompactPrefix(),
},
)
require.NoError(t, err)
@ -333,7 +341,10 @@ func TestSettlementWithWindowEndpointErrors(t *testing.T) {
now := time.Now()
projectID := testrand.UUID()
bucketname := "testbucket"
bucketID := storj.JoinPaths(projectID.String(), bucketname)
bucketLocation := metabase.BucketLocation{
ProjectID: projectID,
BucketName: bucketname,
}
// stop any async flushes because we want to be sure when some values are
// written to avoid races
@ -351,11 +362,11 @@ func TestSettlementWithWindowEndpointErrors(t *testing.T) {
// create serial number to use in test
serialNumber1 := testrand.SerialNumber()
err = ordersDB.CreateSerialInfo(ctx, serialNumber1, []byte(bucketID), now.AddDate(1, 0, 10))
err = ordersDB.CreateSerialInfo(ctx, serialNumber1, []byte(bucketLocation.Prefix()), now.AddDate(1, 0, 10))
require.NoError(t, err)
serialNumber2 := testrand.SerialNumber()
err = ordersDB.CreateSerialInfo(ctx, serialNumber2, []byte(bucketID), now.AddDate(1, 0, 10))
err = ordersDB.CreateSerialInfo(ctx, serialNumber2, []byte(bucketLocation.Prefix()), now.AddDate(1, 0, 10))
require.NoError(t, err)
piecePublicKey1, piecePrivateKey1, err := storj.NewPieceKey()
@ -366,8 +377,8 @@ func TestSettlementWithWindowEndpointErrors(t *testing.T) {
key := satellite.Config.Orders.EncryptionKeys.Default
encrypted, err := key.EncryptMetadata(
serialNumber1,
&pb.OrderLimitMetadata{
ProjectBucketPrefix: []byte(bucketID),
&internalpb.OrderLimitMetadata{
CompactProjectBucketPrefix: bucketLocation.CompactPrefix(),
},
)
require.NoError(t, err)
@ -468,8 +479,10 @@ func TestSettlementEndpointSingleOrder(t *testing.T) {
now := time.Now()
projectID := testrand.UUID()
bucketname := "testbucket"
bucketID := storj.JoinPaths(projectID.String(), bucketname)
bucketLocation := metabase.BucketLocation{
ProjectID: projectID,
BucketName: bucketname,
}
// stop any async flushes because we want to be sure when some values are
// written to avoid races
satellite.Orders.Chore.Loop.Pause()
@ -486,7 +499,7 @@ func TestSettlementEndpointSingleOrder(t *testing.T) {
// create serial number to use in test
serialNumber := testrand.SerialNumber()
err = ordersDB.CreateSerialInfo(ctx, serialNumber, []byte(bucketID), now.AddDate(1, 0, 10))
err = ordersDB.CreateSerialInfo(ctx, serialNumber, []byte(bucketLocation.Prefix()), now.AddDate(1, 0, 10))
require.NoError(t, err)
piecePublicKey, piecePrivateKey, err := storj.NewPieceKey()
@ -494,8 +507,8 @@ func TestSettlementEndpointSingleOrder(t *testing.T) {
key := satellite.Config.Orders.EncryptionKeys.Default
encrypted, err := key.EncryptMetadata(
serialNumber,
&pb.OrderLimitMetadata{
ProjectBucketPrefix: []byte(bucketID),
&internalpb.OrderLimitMetadata{
CompactProjectBucketPrefix: bucketLocation.CompactPrefix(),
},
)
require.NoError(t, err)

3
satellite/orders/service.go
View File

@ -17,6 +17,7 @@ import (
"storj.io/common/signing"
"storj.io/common/storj"
"storj.io/common/uuid"
"storj.io/storj/satellite/internalpb"
"storj.io/storj/satellite/metainfo/metabase"
"storj.io/storj/satellite/overlay"
"storj.io/uplink/private/eestream"
@ -566,7 +567,7 @@ func (service *Service) UpdatePutInlineOrder(ctx context.Context, bucket metabas
}
// DecryptOrderMetadata decrypts the order metadata.
func (service *Service) DecryptOrderMetadata(ctx context.Context, order *pb.OrderLimit) (_ *pb.OrderLimitMetadata, err error) {
func (service *Service) DecryptOrderMetadata(ctx context.Context, order *pb.OrderLimit) (_ *internalpb.OrderLimitMetadata, err error) {
defer mon.Task()(&ctx)(&err)
var orderKeyID EncryptionKeyID

4
satellite/orders/service_test.go
View File

@ -53,9 +53,7 @@ func TestOrderLimitsEncryptedMetadata(t *testing.T) {
require.Error(t, err)
actualOrderMetadata, err := satellitePeer.Orders.Service.DecryptOrderMetadata(ctx, orderLimit1)
require.NoError(t, err)
actualBucketInfo, err := metabase.ParseBucketPrefix(
metabase.BucketPrefix(actualOrderMetadata.GetProjectBucketPrefix()),
)
actualBucketInfo, err := metabase.ParseCompactBucketPrefix(actualOrderMetadata.GetCompactProjectBucketPrefix())
require.NoError(t, err)
require.Equal(t, bucketName, actualBucketInfo.BucketName)
require.Equal(t, projectID, actualBucketInfo.ProjectID)

5
satellite/orders/signer.go
View File

@ -14,6 +14,7 @@ import (
"storj.io/common/pb"
"storj.io/common/signing"
"storj.io/common/storj"
"storj.io/storj/satellite/internalpb"
"storj.io/storj/satellite/metainfo/metabase"
)
@ -148,8 +149,8 @@ func (signer *Signer) Sign(ctx context.Context, node storj.NodeURL, pieceNum int
encrypted, err := encryptionKey.EncryptMetadata(
signer.Serial,
&pb.OrderLimitMetadata{
ProjectBucketPrefix: []byte(signer.Bucket.Prefix()),
&internalpb.OrderLimitMetadata{
CompactProjectBucketPrefix: signer.Bucket.CompactPrefix(),
},
)
if err != nil {

10
satellite/orders/signer_test.go
View File

@ -41,7 +41,7 @@ func TestSigner_EncryptedMetadata(t *testing.T) {
project, err := uplink.GetProject(ctx, satellite)
require.NoError(t, err)
bucketName := "testbucket"
bucketName := "123456789012345678901234567890123456789012345678901234567890123"
bucketLocation := metabase.BucketLocation{
ProjectID: uplink.Projects[0].ID,
BucketName: bucketName,
@ -70,7 +70,7 @@ func TestSigner_EncryptedMetadata(t *testing.T) {
metadata, err := ekeys.Default.DecryptMetadata(addressedLimit.Limit.SerialNumber, addressedLimit.Limit.EncryptedMetadata)
require.NoError(t, err)
bucketInfo, err := metabase.ParseBucketPrefix(metabase.BucketPrefix(metadata.ProjectBucketPrefix))
bucketInfo, err := metabase.ParseCompactBucketPrefix(metadata.CompactProjectBucketPrefix)
require.NoError(t, err)
require.Equal(t, bucketInfo.BucketName, bucketName)
require.Equal(t, bucketInfo.ProjectID, uplink.Projects[0].ID)
@ -96,11 +96,13 @@ func TestSigner_EncryptedMetadata_UploadDownload(t *testing.T) {
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
satellite, uplink := planet.Satellites[0], planet.Uplinks[0]
const bucket = "123456789012345678901234567890123456789012345678901234567890123"
testdata := testrand.Bytes(8 * memory.KiB)
err := uplink.Upload(ctx, satellite, "testbucket", "data", testdata)
err := uplink.Upload(ctx, satellite, bucket, "data", testdata)
require.NoError(t, err)
downdata, err := uplink.Download(ctx, satellite, "testbucket", "data")
downdata, err := uplink.Download(ctx, satellite, bucket, "data")
require.NoError(t, err)
require.Equal(t, testdata, downdata)