diff --git a/internal/testrand/rand.go b/internal/testrand/rand.go index 8a03bb314..51724fb76 100644 --- a/internal/testrand/rand.go +++ b/internal/testrand/rand.go @@ -98,6 +98,11 @@ func StreamID(size int) storj.StreamID { return storj.StreamID(BytesInt(size)) } +// SegmentID creates a random segment ID +func SegmentID(size int) storj.SegmentID { + return storj.SegmentID(BytesInt(size)) +} + // UUID creates a random uuid. func UUID() uuid.UUID { var uuid uuid.UUID diff --git a/pkg/pb/metainfo.pb.go b/pkg/pb/metainfo.pb.go index 2af4cdab8..c8dc9f745 100644 --- a/pkg/pb/metainfo.pb.go +++ b/pkg/pb/metainfo.pb.go @@ -2360,16 +2360,18 @@ var xxx_messageInfo_ObjectFinishDeleteResponse proto.InternalMessageInfo // only for satellite use type SatStreamID struct { - Bucket []byte `protobuf:"bytes,1,opt,name=bucket,proto3" json:"bucket,omitempty"` - EncryptedPath []byte `protobuf:"bytes,2,opt,name=encrypted_path,json=encryptedPath,proto3" json:"encrypted_path,omitempty"` - Version int32 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` - Redundancy *RedundancyScheme `protobuf:"bytes,4,opt,name=redundancy,proto3" json:"redundancy,omitempty"` - CreationDate time.Time `protobuf:"bytes,5,opt,name=creation_date,json=creationDate,proto3,stdtime" json:"creation_date"` - ExpirationDate time.Time `protobuf:"bytes,6,opt,name=expiration_date,json=expirationDate,proto3,stdtime" json:"expiration_date"` - SatelliteSignature []byte `protobuf:"bytes,7,opt,name=satellite_signature,json=satelliteSignature,proto3" json:"satellite_signature,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Bucket []byte `protobuf:"bytes,1,opt,name=bucket,proto3" json:"bucket,omitempty"` + EncryptedPath []byte `protobuf:"bytes,2,opt,name=encrypted_path,json=encryptedPath,proto3" json:"encrypted_path,omitempty"` + Version int32 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` + Redundancy *RedundancyScheme `protobuf:"bytes,4,opt,name=redundancy,proto3" json:"redundancy,omitempty"` + CreationDate time.Time `protobuf:"bytes,5,opt,name=creation_date,json=creationDate,proto3,stdtime" json:"creation_date"` + ExpirationDate time.Time `protobuf:"bytes,6,opt,name=expiration_date,json=expirationDate,proto3,stdtime" json:"expiration_date"` + EncryptedMetadataNonce Nonce `protobuf:"bytes,7,opt,name=encrypted_metadata_nonce,json=encryptedMetadataNonce,proto3,customtype=Nonce" json:"encrypted_metadata_nonce"` + EncryptedMetadata []byte `protobuf:"bytes,8,opt,name=encrypted_metadata,json=encryptedMetadata,proto3" json:"encrypted_metadata,omitempty"` + SatelliteSignature []byte `protobuf:"bytes,9,opt,name=satellite_signature,json=satelliteSignature,proto3" json:"satellite_signature,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *SatStreamID) Reset() { *m = SatStreamID{} } @@ -2438,6 +2440,13 @@ func (m *SatStreamID) GetExpirationDate() time.Time { return time.Time{} } +func (m *SatStreamID) GetEncryptedMetadata() []byte { + if m != nil { + return m.EncryptedMetadata + } + return nil +} + func (m *SatStreamID) GetSatelliteSignature() []byte { if m != nil { return m.SatelliteSignature @@ -3497,194 +3506,194 @@ func init() { func init() { proto.RegisterFile("metainfo.proto", fileDescriptor_631e2f30a93cd64e) } var fileDescriptor_631e2f30a93cd64e = []byte{ - // 2978 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3a, 0x4b, 0x6f, 0x23, 0xc7, - 0xd1, 0x1e, 0xbe, 0x55, 0xa4, 0x48, 0xaa, 0xa5, 0xd5, 0x72, 0xa9, 0x95, 0x25, 0xcf, 0x7a, 0xd7, - 0x32, 0x60, 0x73, 0x0d, 0xf9, 0xfb, 0x00, 0x03, 0xb6, 0x81, 0x48, 0xe2, 0x7a, 0x45, 0x7b, 0xf5, - 0xf0, 0xc8, 0xaf, 0x18, 0x0e, 0x06, 0x23, 0x4d, 0x4b, 0x9a, 0x2c, 0x39, 0xc3, 0xcc, 0x0c, 0x37, - 0x5a, 0x9f, 0x03, 0x04, 0x81, 0x73, 0xc8, 0xd1, 0x27, 0x5f, 0x82, 0x9c, 0xf2, 0x0b, 0x72, 0xc9, - 0x35, 0x41, 0x12, 0xe4, 0x90, 0x63, 0x02, 0x38, 0x41, 0x7e, 0x40, 0x2e, 0xb9, 0xe4, 0x12, 0x20, - 0xe8, 0xd7, 0x4c, 0xcf, 0x4b, 0x24, 0x57, 0xd4, 0xfa, 0x36, 0x5d, 0x55, 0x5d, 0x5d, 0x5d, 0xaf, - 0xae, 0xae, 0x69, 0xa8, 0x0f, 0xb0, 0x6f, 0x58, 0xf6, 0xa9, 0xd3, 0x19, 0xba, 0x8e, 0xef, 0xa0, - 0x8a, 0x18, 0xb7, 0x9b, 0xd8, 0x3e, 0x71, 0x9f, 0x0e, 0x7d, 0xcb, 0xb1, 0x19, 0xae, 0x0d, 0x67, - 0xce, 0x19, 0xa7, 0x6b, 0xaf, 0x9d, 0x39, 0xce, 0x59, 0x1f, 0xdf, 0xa7, 0xa3, 0xe3, 0xd1, 0xe9, - 0x7d, 0xdf, 0x1a, 0x60, 0xcf, 0x37, 0x06, 0x43, 0x41, 0x6c, 0x3b, 0x26, 0xe6, 0xdf, 0x8d, 0xa1, - 0x63, 0xd9, 0x3e, 0x76, 0xcd, 0x63, 0x0e, 0xa8, 0x39, 0xae, 0x89, 0x5d, 0x8f, 0x8d, 0xd4, 0x5f, - 0xe7, 0xa1, 0xb4, 0x3d, 0x3a, 0x79, 0x8c, 0x7d, 0x84, 0xa0, 0x60, 0x1b, 0x03, 0xdc, 0x52, 0xd6, - 0x95, 0x8d, 0x9a, 0x46, 0xbf, 0xd1, 0x5b, 0x50, 0x1d, 0x1a, 0xfe, 0xb9, 0x7e, 0x62, 0x0d, 0xcf, - 0xb1, 0xdb, 0xca, 0xad, 0x2b, 0x1b, 0xf5, 0xcd, 0x9b, 0x1d, 0x49, 0xbc, 0x1d, 0x8a, 0x39, 0x1a, - 0x59, 0x3e, 0xd6, 0x80, 0xd0, 0x32, 0x00, 0xda, 0x01, 0x38, 0x71, 0xb1, 0xe1, 0x63, 0x53, 0x37, - 0xfc, 0x56, 0x7e, 0x5d, 0xd9, 0xa8, 0x6e, 0xb6, 0x3b, 0x4c, 0xf2, 0x8e, 0x90, 0xbc, 0xf3, 0x91, - 0x90, 0x7c, 0xbb, 0xf2, 0xfb, 0x6f, 0xd7, 0x5e, 0xf8, 0xc5, 0xdf, 0xd7, 0x14, 0x6d, 0x8e, 0xcf, - 0xdb, 0xf2, 0xd1, 0x1b, 0xb0, 0x64, 0xe2, 0x53, 0x63, 0xd4, 0xf7, 0x75, 0x0f, 0x9f, 0x0d, 0xb0, - 0xed, 0xeb, 0x9e, 0xf5, 0x25, 0x6e, 0x15, 0xd6, 0x95, 0x8d, 0xbc, 0x86, 0x38, 0xee, 0x88, 0xa1, - 0x8e, 0xac, 0x2f, 0x31, 0xfa, 0x14, 0x6e, 0x89, 0x19, 0x2e, 0x36, 0x47, 0xb6, 0x69, 0xd8, 0x27, - 0x4f, 0x75, 0xef, 0xe4, 0x1c, 0x0f, 0x70, 0xab, 0x48, 0xa5, 0x58, 0xe9, 0x84, 0x2a, 0xd1, 0x02, - 0x9a, 0x23, 0x4a, 0xa2, 0xdd, 0xe4, 0xb3, 0xe3, 0x08, 0x64, 0xc2, 0xaa, 0x60, 0x1c, 0xee, 0x5e, - 0x1f, 0x1a, 0xae, 0x31, 0xc0, 0x3e, 0x76, 0xbd, 0x56, 0x89, 0x32, 0x5f, 0x97, 0x75, 0xf3, 0x20, - 0xf8, 0x3c, 0x0c, 0xe8, 0xb4, 0x15, 0xce, 0x26, 0x0d, 0x89, 0x56, 0x01, 0x86, 0x86, 0xeb, 0xdb, - 0xd8, 0xd5, 0x2d, 0xb3, 0x55, 0xa6, 0x96, 0x98, 0xe3, 0x90, 0x9e, 0xa9, 0x5a, 0x50, 0x67, 0xc6, - 0x7a, 0x64, 0x79, 0x7e, 0xcf, 0xc7, 0x83, 0x54, 0xa3, 0x45, 0x55, 0x9f, 0x7b, 0x26, 0xd5, 0xab, - 0xff, 0xce, 0xc1, 0x22, 0x5b, 0x6b, 0x87, 0xc2, 0x34, 0xfc, 0xa3, 0x11, 0xf6, 0x66, 0xed, 0x25, - 0x59, 0x06, 0xce, 0x3f, 0x9b, 0x81, 0x0b, 0xd7, 0x69, 0xe0, 0xe2, 0xec, 0x0d, 0x5c, 0x8a, 0x1b, - 0xf8, 0x7b, 0xb0, 0x14, 0x55, 0xba, 0x37, 0x74, 0x6c, 0x0f, 0xa3, 0x0d, 0x28, 0x1d, 0x53, 0x38, - 0xd5, 0x7b, 0x75, 0xb3, 0xd9, 0x09, 0x72, 0x07, 0xa3, 0xd7, 0x38, 0x5e, 0xbd, 0x07, 0x4d, 0x06, - 0x79, 0x88, 0xfd, 0x4b, 0x6c, 0xa6, 0xbe, 0x0b, 0x0b, 0x12, 0xdd, 0xd4, 0xcb, 0xbc, 0x2a, 0xbc, - 0xa3, 0x8b, 0xfb, 0xf8, 0x52, 0xef, 0x50, 0x97, 0xc5, 0x9e, 0x04, 0x29, 0x5b, 0x4c, 0xd5, 0x85, - 0x04, 0xc4, 0x99, 0x05, 0x83, 0x65, 0x28, 0x9d, 0x8c, 0x5c, 0xcf, 0x71, 0x39, 0x0b, 0x3e, 0x42, - 0x4b, 0x50, 0xec, 0x5b, 0x03, 0x8b, 0xb9, 0x73, 0x51, 0x63, 0x03, 0x74, 0x1b, 0xe6, 0x4c, 0xcb, - 0xc5, 0x27, 0x44, 0xc9, 0xd4, 0x67, 0x8a, 0x5a, 0x08, 0x50, 0x3f, 0x03, 0x24, 0x2f, 0xc0, 0xf7, - 0xd8, 0x81, 0xa2, 0xe5, 0xe3, 0x81, 0xd7, 0x52, 0xd6, 0xf3, 0x1b, 0xd5, 0xcd, 0x56, 0x7c, 0x8b, - 0x22, 0xb4, 0x34, 0x46, 0x46, 0xb6, 0x34, 0x70, 0x5c, 0x4c, 0x17, 0xae, 0x68, 0xf4, 0x5b, 0xfd, - 0x0c, 0x56, 0x18, 0xf1, 0x11, 0xf6, 0xb7, 0x7c, 0xdf, 0xb5, 0x8e, 0x47, 0x64, 0xc5, 0xcb, 0x62, - 0xe4, 0x2e, 0xd4, 0x8d, 0x90, 0x92, 0x18, 0x3f, 0x47, 0xb1, 0xf3, 0x12, 0xb4, 0x67, 0xaa, 0x2f, - 0xc2, 0xed, 0x74, 0xce, 0x5c, 0x69, 0x3f, 0x51, 0x60, 0x71, 0xcb, 0x34, 0x5d, 0xec, 0x79, 0xd8, - 0x3c, 0x20, 0x99, 0xfc, 0x11, 0xd5, 0xc4, 0x86, 0xd0, 0x0f, 0x33, 0x1c, 0xea, 0xf0, 0x2c, 0x1f, - 0x92, 0x08, 0x9d, 0xed, 0xc0, 0x92, 0xe7, 0x3b, 0xae, 0x71, 0x86, 0x75, 0x72, 0x4c, 0xe8, 0x06, - 0xe3, 0xc6, 0xf3, 0xc4, 0x42, 0x87, 0x9e, 0x1d, 0xfb, 0x8e, 0x89, 0xf9, 0x32, 0x1a, 0xe2, 0xe4, - 0x12, 0x4c, 0xfd, 0x26, 0x07, 0xcb, 0x3c, 0x2a, 0x3f, 0x75, 0xad, 0xc0, 0xfe, 0x07, 0x7d, 0x93, - 0x58, 0x50, 0xf2, 0xa1, 0x9a, 0xf0, 0x18, 0xa2, 0x14, 0x12, 0xf8, 0x7c, 0xdb, 0xf4, 0x1b, 0xb5, - 0xa0, 0xcc, 0xc3, 0x9e, 0x47, 0xbc, 0x18, 0xa2, 0xb7, 0x01, 0xc2, 0xf0, 0x9e, 0x24, 0xae, 0x25, - 0x72, 0xf4, 0x36, 0xb4, 0x07, 0xc6, 0x85, 0x08, 0x63, 0x6c, 0x46, 0x73, 0x4b, 0x91, 0xae, 0x74, - 0x73, 0x60, 0x5c, 0x3c, 0x10, 0x04, 0x72, 0x82, 0xe9, 0x02, 0xe0, 0x8b, 0xa1, 0xe5, 0x1a, 0xd4, - 0xa9, 0x4a, 0x53, 0x64, 0x4f, 0x69, 0x9e, 0xfa, 0x17, 0x05, 0x6e, 0x46, 0x15, 0xc4, 0x0c, 0x48, - 0x34, 0xb4, 0x0b, 0x4d, 0x43, 0x98, 0x50, 0xa7, 0x46, 0x11, 0xce, 0xb8, 0x1a, 0x3a, 0x63, 0x8a, - 0x91, 0xb5, 0x46, 0x30, 0x8d, 0x8e, 0x3d, 0xf4, 0x26, 0xcc, 0xbb, 0x8e, 0xe3, 0xeb, 0x43, 0x0b, - 0x9f, 0xe0, 0xc0, 0xa7, 0xb6, 0x1b, 0x44, 0xa4, 0xbf, 0x7e, 0xbb, 0x56, 0x3e, 0x24, 0xf0, 0x5e, - 0x57, 0xab, 0x12, 0x2a, 0x36, 0x30, 0x69, 0xb6, 0x76, 0xad, 0x27, 0x86, 0x8f, 0xf5, 0xc7, 0xf8, - 0x29, 0x55, 0x7c, 0x6d, 0xfb, 0x26, 0x9f, 0xd2, 0xa0, 0x54, 0x87, 0x0c, 0xff, 0x01, 0x7e, 0xaa, - 0xc1, 0x30, 0xf8, 0x56, 0xff, 0x10, 0x6e, 0x6a, 0xc7, 0x19, 0x10, 0x89, 0x66, 0x6d, 0xf6, 0xd7, - 0xa0, 0xcc, 0x6d, 0xcc, 0x6d, 0x8e, 0x24, 0x9b, 0x1f, 0xb2, 0x2f, 0x4d, 0x90, 0xa0, 0xb7, 0xa1, - 0xe1, 0xb8, 0xd6, 0x99, 0x65, 0x1b, 0x7d, 0xa1, 0xc7, 0x22, 0xd5, 0x63, 0x9a, 0xfb, 0xd7, 0x05, - 0x29, 0xd3, 0x9d, 0xba, 0x0b, 0xad, 0xd8, 0x5e, 0x42, 0x0b, 0x49, 0x62, 0x28, 0x63, 0xc5, 0x50, - 0x0d, 0xb8, 0xc5, 0x39, 0x75, 0x9d, 0x1f, 0xdb, 0x7d, 0xc7, 0x30, 0x67, 0xad, 0x17, 0xf5, 0xcf, - 0x0a, 0xb4, 0x13, 0x6b, 0x5c, 0x87, 0x47, 0x49, 0x3b, 0xcf, 0x8d, 0x37, 0xc0, 0xb3, 0xbb, 0xd2, - 0x0f, 0xe0, 0x06, 0xdf, 0x4f, 0xcf, 0x3e, 0x75, 0x66, 0xae, 0xaf, 0xf7, 0x82, 0xf4, 0xc4, 0xd8, - 0xa7, 0x9a, 0x76, 0xfc, 0x06, 0x55, 0x3d, 0x70, 0xf8, 0xc8, 0x39, 0x37, 0x3b, 0x41, 0xbf, 0x51, - 0x02, 0x37, 0x8c, 0x1e, 0x8f, 0xb3, 0x35, 0x6b, 0xcc, 0x50, 0xb9, 0xc9, 0x0d, 0xf5, 0x37, 0x05, - 0x96, 0xc9, 0x91, 0xc8, 0x85, 0xf4, 0x26, 0xd0, 0xc0, 0x32, 0x94, 0x86, 0x2e, 0x3e, 0xb5, 0x2e, - 0xb8, 0x0e, 0xf8, 0x08, 0xad, 0x41, 0xd5, 0xf3, 0x0d, 0xd7, 0xd7, 0x8d, 0x53, 0xa2, 0x7e, 0xea, - 0x2d, 0x1a, 0x50, 0xd0, 0x16, 0x81, 0x90, 0xe2, 0x08, 0xdb, 0xa6, 0x7e, 0x8c, 0x4f, 0xc9, 0x81, - 0x5b, 0x60, 0xc5, 0x11, 0xb6, 0xcd, 0x6d, 0x0a, 0x20, 0xa7, 0xbd, 0x8b, 0x49, 0x3d, 0x60, 0x3d, - 0x61, 0x59, 0xbc, 0xa2, 0x85, 0x80, 0xb0, 0x42, 0x28, 0xc9, 0x15, 0xc2, 0x2a, 0x00, 0xd1, 0x94, - 0x7e, 0xda, 0x37, 0xce, 0x3c, 0x5a, 0x50, 0x97, 0xb5, 0x39, 0x02, 0x79, 0x8f, 0x00, 0x68, 0x9a, - 0x8e, 0xee, 0x2e, 0xd4, 0xfe, 0x3b, 0xd1, 0x42, 0xe1, 0x5e, 0xa8, 0xf2, 0x8c, 0x19, 0x9d, 0x31, - 0x65, 0x43, 0x1b, 0x43, 0x41, 0x14, 0xed, 0xd4, 0x45, 0x14, 0xc9, 0x45, 0xa6, 0x0b, 0xbc, 0x15, - 0x98, 0xb3, 0x3c, 0x9d, 0x6b, 0x39, 0x4f, 0x97, 0xa8, 0x58, 0xde, 0x21, 0x1d, 0xab, 0x9f, 0x13, - 0x97, 0x4a, 0xa9, 0x4b, 0xc8, 0xa6, 0xd6, 0xa0, 0xca, 0xac, 0xa4, 0x4b, 0x15, 0x0a, 0x30, 0xd0, - 0x3e, 0xa9, 0x53, 0xa2, 0x05, 0x6a, 0x2e, 0x5e, 0xa0, 0xae, 0x90, 0x5c, 0x97, 0x56, 0x99, 0x1c, - 0xf4, 0x4d, 0x75, 0x09, 0xd0, 0xa1, 0xeb, 0xfc, 0x10, 0x9f, 0xc8, 0x41, 0xad, 0xbe, 0x05, 0x8b, - 0x11, 0x28, 0xaf, 0xc3, 0x5e, 0x82, 0xda, 0x90, 0x81, 0x75, 0xcf, 0xe8, 0x0b, 0x1f, 0xaa, 0x72, - 0xd8, 0x91, 0xd1, 0xf7, 0xd5, 0x9f, 0x95, 0xa1, 0x74, 0x70, 0x4c, 0x86, 0x99, 0xbe, 0x76, 0x17, - 0xea, 0xe1, 0x31, 0x2f, 0xc5, 0xdd, 0x7c, 0x00, 0x3d, 0xe4, 0x01, 0xf8, 0x04, 0xbb, 0x5e, 0x58, - 0x26, 0x8a, 0x21, 0xba, 0x0f, 0x25, 0xcf, 0x37, 0xfc, 0x91, 0x47, 0xfd, 0x8d, 0x5c, 0x5b, 0x02, - 0x33, 0xb3, 0xa5, 0x3b, 0x47, 0x14, 0xad, 0x71, 0x32, 0xf4, 0x3a, 0xcc, 0x79, 0xbe, 0x8b, 0x8d, - 0x01, 0xd1, 0x4f, 0x91, 0x06, 0x52, 0x93, 0x07, 0x52, 0xe5, 0x88, 0x22, 0x7a, 0x5d, 0xad, 0xc2, - 0x48, 0x7a, 0x66, 0xec, 0x32, 0x56, 0x7a, 0xb6, 0x7b, 0xf0, 0x16, 0x59, 0x93, 0xac, 0x4e, 0x78, - 0x94, 0xa7, 0xe0, 0x51, 0x61, 0xd3, 0xb6, 0x48, 0xd9, 0xc7, 0xca, 0x13, 0x4c, 0x79, 0x54, 0xa6, - 0x91, 0x83, 0xcf, 0xdb, 0xf2, 0xd1, 0x43, 0x68, 0x85, 0xda, 0x26, 0x7a, 0x32, 0x0d, 0xdf, 0xd0, - 0x6d, 0xc7, 0x3e, 0xc1, 0xad, 0x39, 0xaa, 0x8a, 0x79, 0xae, 0x8a, 0xe2, 0x3e, 0x01, 0x6a, 0xcb, - 0x01, 0xf9, 0x1e, 0xa7, 0xa6, 0x70, 0xf4, 0x3a, 0xa0, 0x24, 0xa3, 0x16, 0x50, 0xd3, 0x2d, 0x24, - 0xe6, 0xa0, 0xd7, 0x00, 0x9d, 0x5a, 0x17, 0xf1, 0x42, 0xae, 0x4a, 0x53, 0x69, 0x93, 0x62, 0xe4, - 0x0a, 0x6e, 0x17, 0x16, 0x92, 0x57, 0xc3, 0xda, 0xf8, 0x12, 0xb2, 0xe9, 0xc6, 0xef, 0x84, 0x1f, - 0xc3, 0x8d, 0xf4, 0xbb, 0xe0, 0xfc, 0x84, 0x77, 0xc1, 0x25, 0x9c, 0x71, 0x09, 0xf4, 0x1d, 0xdf, - 0xe8, 0xb3, 0x6d, 0xd4, 0xe9, 0x36, 0xe6, 0x28, 0x84, 0xca, 0xbf, 0x06, 0x55, 0xcb, 0xee, 0x5b, - 0x36, 0x66, 0xf8, 0x06, 0xc5, 0x03, 0x03, 0x09, 0x02, 0x17, 0x0f, 0x1c, 0x9f, 0x13, 0x34, 0x19, - 0x01, 0x03, 0x11, 0x02, 0xf5, 0x43, 0x28, 0x31, 0xaf, 0x45, 0x55, 0x28, 0xf7, 0xf6, 0x3f, 0xd9, - 0x7a, 0xd4, 0xeb, 0x36, 0x5f, 0x40, 0xf3, 0x30, 0xf7, 0xf1, 0xe1, 0xa3, 0x83, 0xad, 0x6e, 0x6f, - 0xff, 0x61, 0x53, 0x41, 0x75, 0x80, 0x9d, 0x83, 0xbd, 0xbd, 0xde, 0x47, 0x1f, 0x91, 0x71, 0x8e, - 0xa0, 0xf9, 0xf8, 0x41, 0xb7, 0x99, 0x47, 0x35, 0xa8, 0x74, 0x1f, 0x3c, 0x7a, 0x40, 0x91, 0x05, - 0xf5, 0x8f, 0x79, 0x40, 0x2c, 0x20, 0xb6, 0xf1, 0x99, 0x65, 0x4b, 0xf7, 0xb5, 0xeb, 0x89, 0xcb, - 0xa8, 0xbf, 0x16, 0x66, 0xef, 0xaf, 0xc5, 0xab, 0xfb, 0x6b, 0x29, 0xcb, 0x5f, 0x53, 0x3d, 0xb0, - 0x3c, 0x53, 0x0f, 0xac, 0x5c, 0xc5, 0x03, 0xd5, 0xdf, 0xe6, 0x60, 0x31, 0x62, 0x4d, 0x9e, 0x94, - 0xaf, 0xcd, 0x9c, 0x91, 0xac, 0x59, 0x18, 0x9b, 0x35, 0x53, 0x15, 0x58, 0x9c, 0xa9, 0x02, 0x4b, - 0x57, 0x52, 0x60, 0x57, 0xe8, 0x2f, 0x72, 0x11, 0x8a, 0x6e, 0x53, 0x19, 0xb7, 0x4d, 0x75, 0x19, - 0x96, 0xa2, 0x5c, 0xf8, 0x2d, 0xff, 0x31, 0x34, 0x19, 0x5c, 0x6a, 0xe2, 0x5c, 0x97, 0x69, 0xd4, - 0x77, 0x61, 0x41, 0x5a, 0x2c, 0xec, 0x04, 0x39, 0x14, 0x98, 0xec, 0x04, 0x31, 0x62, 0x8d, 0xe3, - 0xd5, 0x7f, 0x2a, 0x62, 0x7e, 0xac, 0x8f, 0x93, 0x2a, 0xed, 0xab, 0xd0, 0x94, 0xa4, 0x95, 0xab, - 0xc4, 0x46, 0x28, 0x2f, 0x2b, 0x17, 0x23, 0xa4, 0xbc, 0x29, 0x94, 0x8f, 0x91, 0xee, 0xc4, 0xba, - 0x43, 0x05, 0xb9, 0xf6, 0xeb, 0x41, 0x83, 0xc9, 0xa8, 0x5b, 0xf6, 0x49, 0x7f, 0x64, 0xe2, 0xb0, - 0x87, 0x17, 0xdb, 0x8c, 0xe8, 0xf9, 0xf4, 0x38, 0x9d, 0x56, 0x67, 0x13, 0xc5, 0x58, 0xfd, 0x4c, - 0x24, 0xbf, 0x09, 0x5b, 0x49, 0x51, 0xb6, 0x97, 0xb5, 0x92, 0x7e, 0x97, 0x87, 0x7a, 0x94, 0x3a, - 0xc5, 0xa2, 0xca, 0x18, 0x8b, 0xe6, 0xb2, 0x6a, 0x9a, 0xfc, 0x64, 0x35, 0x4d, 0xb4, 0x48, 0x29, - 0xcc, 0xa0, 0x48, 0x29, 0xce, 0xa0, 0x48, 0x29, 0xcd, 0x3e, 0xe9, 0x97, 0xaf, 0x9e, 0xf4, 0x2b, - 0x19, 0x49, 0x5f, 0xfd, 0x3f, 0x58, 0x4e, 0xf7, 0x26, 0xd4, 0x86, 0x4a, 0x30, 0x5d, 0x61, 0xc5, - 0xba, 0x18, 0xab, 0x1e, 0xb4, 0xa4, 0x44, 0x1c, 0xed, 0xa6, 0x5e, 0x5b, 0xc8, 0xbf, 0x0f, 0xb7, - 0x52, 0x16, 0xe5, 0x5e, 0x3d, 0x65, 0x0e, 0x0b, 0x78, 0xbd, 0x67, 0xd9, 0x96, 0x77, 0x1e, 0xdd, - 0xc1, 0x94, 0xbc, 0x6e, 0x43, 0x3b, 0x8d, 0x17, 0xcf, 0x8a, 0xff, 0xca, 0x41, 0xf5, 0xc8, 0xf0, - 0xc5, 0xbc, 0xeb, 0x3b, 0xac, 0xae, 0xd4, 0x7c, 0xec, 0xc1, 0x3c, 0x8d, 0x09, 0x72, 0xdc, 0x98, - 0x86, 0x8f, 0xa7, 0x0a, 0x85, 0x9a, 0x98, 0xda, 0x35, 0x7c, 0x8c, 0xf6, 0xa0, 0x11, 0xb6, 0x14, - 0x19, 0xb3, 0x69, 0x62, 0xa2, 0x1e, 0x4e, 0xa6, 0xec, 0xee, 0xc3, 0xa2, 0x67, 0xf8, 0xb8, 0xdf, - 0xb7, 0x68, 0xe5, 0x78, 0x66, 0x1b, 0xfe, 0xc8, 0xe5, 0x31, 0xa1, 0xa1, 0x00, 0x75, 0x24, 0x30, - 0xea, 0x3f, 0x72, 0x50, 0xe6, 0x85, 0xf5, 0x94, 0x96, 0x44, 0xff, 0x0f, 0x95, 0xa1, 0xe3, 0x59, - 0xbe, 0xc8, 0x4e, 0xd5, 0xcd, 0x5b, 0x61, 0x12, 0xe2, 0x3c, 0x0f, 0x39, 0x81, 0x16, 0x90, 0xa2, - 0x77, 0x61, 0x31, 0x34, 0xdd, 0x63, 0xfc, 0x94, 0x87, 0x6d, 0x3e, 0x2d, 0x6c, 0xc3, 0x10, 0xfc, - 0x00, 0x3f, 0x65, 0x11, 0x7b, 0x07, 0xe6, 0x23, 0xd3, 0x79, 0x0f, 0xa1, 0x26, 0x53, 0xa2, 0x0e, - 0x2c, 0x92, 0xb2, 0x59, 0x6a, 0x0f, 0xd3, 0xc0, 0x64, 0x6d, 0xe1, 0x05, 0x82, 0x0a, 0xfa, 0xc2, - 0x5d, 0x52, 0xcc, 0x6d, 0x06, 0x15, 0x04, 0x36, 0x75, 0x5e, 0x98, 0x4b, 0xe5, 0x5f, 0x28, 0x70, - 0x8f, 0xe2, 0xe8, 0x9c, 0x57, 0xa0, 0x44, 0x7b, 0xb2, 0x5e, 0xab, 0x4c, 0x8f, 0x86, 0x46, 0xb8, - 0x79, 0xda, 0x6c, 0xd1, 0x38, 0x5a, 0xdd, 0x85, 0x22, 0x05, 0x90, 0x1b, 0x3d, 0xeb, 0xe2, 0xda, - 0xa3, 0x01, 0xd5, 0x6f, 0x51, 0xab, 0x50, 0xc0, 0xfe, 0x68, 0x80, 0x54, 0x28, 0xd8, 0x8e, 0x89, - 0x79, 0xdf, 0xa6, 0xce, 0xf5, 0x50, 0xda, 0x77, 0x4c, 0xdc, 0xeb, 0x6a, 0x14, 0xa7, 0xee, 0x42, - 0x23, 0xa6, 0x57, 0x72, 0x4f, 0x20, 0x37, 0x77, 0xc2, 0xf2, 0x98, 0xb7, 0x32, 0x8b, 0x1a, 0xbd, - 0xde, 0xef, 0x53, 0x08, 0x39, 0x37, 0x2d, 0xdb, 0xc4, 0x17, 0xe2, 0xaf, 0x0a, 0x1d, 0xa8, 0xbf, - 0x54, 0x60, 0x91, 0xb3, 0x8a, 0xd4, 0xfa, 0xcf, 0xc7, 0x05, 0xee, 0x41, 0x63, 0x60, 0x5c, 0xe8, - 0xb4, 0x81, 0xcb, 0x9a, 0x5e, 0xbc, 0x67, 0x36, 0x3f, 0x30, 0x2e, 0xc2, 0x1e, 0x97, 0xfa, 0x27, - 0x05, 0x96, 0xa2, 0x52, 0xf2, 0xfc, 0xf5, 0x06, 0x80, 0xb8, 0x26, 0x06, 0x72, 0x2e, 0x70, 0x39, - 0xe7, 0x44, 0x57, 0xb0, 0xab, 0xcd, 0x71, 0xa2, 0x5e, 0x7a, 0x9f, 0x2d, 0x37, 0x8b, 0x3e, 0xdb, - 0x14, 0x0d, 0xd1, 0x5f, 0xe5, 0x82, 0xed, 0x44, 0x4b, 0xca, 0xe9, 0xb7, 0x93, 0x11, 0x44, 0xb9, - 0x67, 0x0d, 0xa2, 0xfc, 0xe4, 0x41, 0x54, 0xc8, 0x0a, 0xa2, 0x87, 0x30, 0x3f, 0x1a, 0xf6, 0x1d, - 0xc3, 0xd4, 0x5d, 0xec, 0x8d, 0xfa, 0x3e, 0x6f, 0xd4, 0xab, 0x49, 0x8f, 0x20, 0x3a, 0xfa, 0x78, - 0xc8, 0x3b, 0xdc, 0xa3, 0xbe, 0xaf, 0xd5, 0x46, 0xd2, 0x48, 0xfd, 0x69, 0xd8, 0x30, 0x4d, 0x90, - 0x5e, 0x1e, 0x44, 0xaf, 0x40, 0x99, 0xfe, 0xf0, 0x0a, 0x7e, 0x93, 0xc4, 0xe3, 0xa8, 0x44, 0xd0, - 0x3d, 0x13, 0xdd, 0x85, 0xc2, 0xb9, 0xe1, 0x9d, 0xf3, 0x47, 0x0b, 0x0b, 0xe2, 0x5f, 0x02, 0x5d, - 0x6e, 0xd7, 0xf0, 0xce, 0x35, 0x8a, 0x56, 0xff, 0x9b, 0x83, 0x1a, 0x39, 0x8e, 0x84, 0x09, 0xd0, - 0x66, 0x3c, 0x3e, 0xaa, 0x9b, 0x37, 0xa4, 0xfd, 0x85, 0x27, 0x97, 0x14, 0x24, 0xb1, 0x10, 0xcd, - 0x65, 0x87, 0x68, 0x5e, 0x0a, 0xd1, 0xe4, 0x8f, 0x9f, 0xe2, 0x04, 0x3f, 0x7e, 0x3e, 0x84, 0x1b, - 0xc1, 0xef, 0x12, 0x29, 0xbc, 0xc8, 0x55, 0x68, 0x02, 0x5f, 0x5f, 0x14, 0x73, 0x43, 0x98, 0x97, - 0x3c, 0xec, 0xca, 0xcf, 0x7c, 0xd8, 0x65, 0x9c, 0x4e, 0x95, 0xcc, 0xd3, 0xe9, 0x66, 0xf0, 0x0b, - 0x21, 0x76, 0x7b, 0xfa, 0x3a, 0x17, 0xb8, 0xc8, 0x9e, 0xf1, 0x18, 0xb3, 0xb4, 0xfc, 0x7c, 0x93, - 0xd8, 0xf3, 0x38, 0xc7, 0x32, 0xcf, 0xa5, 0x62, 0xe6, 0xb9, 0xc4, 0xda, 0xb7, 0x09, 0xcd, 0x70, - 0xbd, 0x39, 0x01, 0x32, 0xa5, 0x16, 0x5d, 0x49, 0xe8, 0xed, 0xca, 0x5a, 0x52, 0xbf, 0x0e, 0xff, - 0x6a, 0xa5, 0x15, 0xa2, 0xdf, 0x61, 0x22, 0x57, 0x7f, 0x1e, 0x8a, 0x96, 0x56, 0xd7, 0x4e, 0x2f, - 0xda, 0x3b, 0x50, 0x66, 0x99, 0x4f, 0x48, 0x94, 0x91, 0xfa, 0x02, 0x1d, 0x90, 0xd4, 0x27, 0xa6, - 0x24, 0xb2, 0x9e, 0x4c, 0xf5, 0x7c, 0xb3, 0xde, 0x2a, 0xac, 0xa4, 0xea, 0x85, 0xfb, 0xd0, 0x57, - 0x0a, 0x20, 0x8e, 0x97, 0xdb, 0x01, 0x97, 0x7a, 0xcf, 0x36, 0x34, 0xd8, 0xf5, 0x5e, 0x9f, 0xdc, - 0x89, 0xea, 0x6c, 0x46, 0x50, 0xea, 0x04, 0x1d, 0x80, 0xbc, 0xd4, 0x01, 0x50, 0x3f, 0x0f, 0x0a, - 0x99, 0xc8, 0xbd, 0xfd, 0x7e, 0xf4, 0xde, 0x9e, 0x5c, 0x66, 0x92, 0x8b, 0x7b, 0x58, 0x6f, 0x05, - 0x17, 0x77, 0x39, 0x0c, 0x94, 0xc9, 0xc3, 0xe0, 0x2b, 0x25, 0xf8, 0x5b, 0x19, 0xfb, 0x81, 0x3c, - 0x6d, 0xb6, 0x9a, 0x81, 0x26, 0xd5, 0xff, 0x84, 0x3f, 0xf9, 0xe3, 0xbf, 0x9a, 0xbf, 0xd3, 0xd2, - 0x2a, 0x33, 0xdd, 0xe5, 0xb3, 0xcb, 0xf0, 0xd7, 0xa1, 0x60, 0xe3, 0x0b, 0xd1, 0xd1, 0xb8, 0x44, - 0x09, 0x94, 0x6c, 0xf3, 0x37, 0x0b, 0x50, 0xd9, 0xe3, 0x24, 0x68, 0x0f, 0x6a, 0xec, 0x11, 0x16, - 0x7f, 0x1e, 0xb9, 0x1a, 0x7f, 0x28, 0x14, 0x79, 0x17, 0xd7, 0x7e, 0x31, 0x0b, 0xcd, 0x55, 0xd7, - 0x85, 0xb9, 0x87, 0xd8, 0xe7, 0xbc, 0xda, 0x71, 0xe2, 0xb0, 0xcf, 0xd7, 0x5e, 0x49, 0xc5, 0x71, - 0x2e, 0x7b, 0x50, 0x63, 0x01, 0x97, 0x25, 0x54, 0x24, 0x4d, 0x25, 0x85, 0x8a, 0x65, 0xd8, 0x5d, - 0xa8, 0x12, 0xe7, 0x65, 0x38, 0x0f, 0xad, 0xa4, 0xbd, 0x85, 0x12, 0xbc, 0x6e, 0xa7, 0x23, 0x39, - 0x27, 0x4c, 0xaa, 0x57, 0xce, 0x48, 0xfa, 0x3b, 0x88, 0xee, 0xc6, 0x67, 0xa5, 0xfe, 0x99, 0x6c, - 0xdf, 0x1b, 0x47, 0xc6, 0x97, 0x79, 0x1f, 0xaa, 0xf4, 0xa4, 0xe0, 0x7f, 0x05, 0x6f, 0xc7, 0x1b, - 0x5b, 0xf2, 0x7d, 0xa5, 0xbd, 0x9a, 0x81, 0x0d, 0x75, 0xc9, 0x0a, 0x07, 0xce, 0x2c, 0x41, 0x1e, - 0xa9, 0xc3, 0x65, 0x5d, 0xa6, 0xf5, 0x6c, 0xb9, 0x81, 0x39, 0xaf, 0x76, 0x9c, 0x38, 0xdd, 0xc0, - 0xc9, 0xbe, 0x2b, 0xb7, 0x08, 0x43, 0x44, 0x2c, 0x92, 0xe8, 0xb1, 0xb6, 0x6f, 0xa7, 0x23, 0x39, - 0xa7, 0x2f, 0x60, 0x41, 0x3a, 0x54, 0xb9, 0x5c, 0x6a, 0xaa, 0x4a, 0xa2, 0x4e, 0x73, 0xe7, 0x52, - 0x1a, 0xce, 0x5d, 0x07, 0x24, 0xe7, 0x7f, 0xce, 0x3e, 0x31, 0x35, 0xe5, 0xec, 0x6c, 0xbf, 0x7c, - 0x39, 0x51, 0x68, 0x1d, 0xba, 0xae, 0xe8, 0x3f, 0xac, 0x26, 0x82, 0x37, 0x62, 0xeb, 0x17, 0xb3, - 0xd0, 0x9c, 0xdd, 0x21, 0xcc, 0x33, 0x7b, 0x09, 0x7e, 0xc9, 0x09, 0x51, 0x73, 0xaf, 0x65, 0xe2, - 0x43, 0xfd, 0x86, 0x35, 0x94, 0xe0, 0x9a, 0x3c, 0xd4, 0x13, 0x15, 0xa8, 0xac, 0xdf, 0xcc, 0x5a, - 0x8c, 0xe8, 0x57, 0x52, 0xbb, 0x60, 0x7f, 0x27, 0x7d, 0x97, 0x99, 0xfa, 0xbd, 0xa4, 0xb8, 0x3a, - 0x86, 0x45, 0x59, 0xef, 0x62, 0x85, 0xe4, 0xe4, 0x34, 0x13, 0xde, 0x1d, 0x43, 0xc5, 0xd7, 0xf8, - 0x00, 0x6a, 0xf2, 0x53, 0x09, 0x39, 0x5c, 0x93, 0x35, 0x42, 0x7b, 0x35, 0x03, 0xcb, 0x99, 0x7d, - 0x02, 0x0d, 0x71, 0x1e, 0x09, 0x61, 0xd7, 0x13, 0x33, 0x62, 0xe7, 0x67, 0xfb, 0xa5, 0x4b, 0x28, - 0x38, 0xdf, 0x4f, 0xa1, 0xc9, 0x52, 0x35, 0x27, 0x38, 0xe8, 0x9b, 0x29, 0x8c, 0x63, 0xaf, 0x1c, - 0x53, 0x18, 0x27, 0x9e, 0xf9, 0x7d, 0x1f, 0x9a, 0x11, 0x97, 0x23, 0xb0, 0x97, 0x2e, 0xf7, 0x3a, - 0xc2, 0x59, 0x1d, 0xe3, 0x78, 0x84, 0xcd, 0x11, 0xd4, 0xa5, 0xe7, 0x4d, 0xf4, 0x5d, 0x47, 0x62, - 0x56, 0xf4, 0x5d, 0x55, 0x7b, 0x3d, 0x83, 0x20, 0x64, 0xaa, 0x03, 0x8a, 0x29, 0x98, 0x40, 0xef, - 0x8c, 0xd3, 0x31, 0x61, 0xfe, 0xf2, 0x58, 0x35, 0x73, 0x85, 0x44, 0x9c, 0x2d, 0x5d, 0x21, 0xf1, - 0x87, 0x56, 0x29, 0x0a, 0x49, 0xbe, 0x94, 0xfa, 0x04, 0x1a, 0xb2, 0xa7, 0xc5, 0x6c, 0x98, 0xfe, - 0x7e, 0x49, 0xb6, 0x61, 0xd6, 0x1b, 0xa0, 0x2f, 0x60, 0x21, 0x7a, 0x12, 0x11, 0x60, 0x44, 0xa0, - 0xf4, 0x77, 0x36, 0xd1, 0x20, 0xcf, 0x78, 0x2f, 0x43, 0x4e, 0x33, 0xe9, 0x65, 0x8c, 0x1c, 0x1e, - 0xc9, 0x67, 0x34, 0x72, 0x78, 0xa4, 0x3c, 0xa7, 0xd9, 0x2e, 0x7c, 0x9e, 0x1b, 0x1e, 0x1f, 0x97, - 0xe8, 0x85, 0xfb, 0xcd, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x64, 0xa4, 0x88, 0x3f, 0x5b, 0x32, - 0x00, 0x00, + // 2984 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x5a, 0x4b, 0x6f, 0x23, 0xc7, + 0xf1, 0xf7, 0xf0, 0xcd, 0x22, 0x45, 0x52, 0x2d, 0xad, 0x96, 0x4b, 0xad, 0x2c, 0x79, 0xd6, 0xbb, + 0x96, 0x01, 0x9b, 0x6b, 0xc8, 0xff, 0x3f, 0x60, 0xc0, 0x36, 0x10, 0x49, 0x5c, 0xaf, 0x68, 0xaf, + 0x1e, 0x1e, 0xf9, 0x15, 0xc3, 0xc1, 0x60, 0xa4, 0x69, 0x49, 0x93, 0x25, 0x67, 0x98, 0x99, 0xa1, + 0xa3, 0xf5, 0x39, 0x40, 0x10, 0x38, 0x87, 0x1c, 0x7d, 0xf2, 0x25, 0xc8, 0x29, 0x9f, 0x20, 0x97, + 0x5c, 0x13, 0x24, 0x41, 0x0e, 0x39, 0x26, 0x80, 0x13, 0xe4, 0x90, 0x2f, 0x90, 0x4b, 0x2e, 0x01, + 0x82, 0x7e, 0xcd, 0xf4, 0xbc, 0x44, 0x71, 0x45, 0xed, 0xde, 0xa6, 0xbb, 0xaa, 0xab, 0xbb, 0x7f, + 0xf5, 0xe8, 0xea, 0x9a, 0x86, 0xc6, 0x10, 0xfb, 0x86, 0x65, 0x9f, 0x38, 0xdd, 0x91, 0xeb, 0xf8, + 0x0e, 0xaa, 0x88, 0x76, 0xa7, 0x85, 0xed, 0x63, 0xf7, 0xc9, 0xc8, 0xb7, 0x1c, 0x9b, 0xd1, 0x3a, + 0x70, 0xea, 0x9c, 0x72, 0xbe, 0xce, 0xea, 0xa9, 0xe3, 0x9c, 0x0e, 0xf0, 0x7d, 0xda, 0x3a, 0x1a, + 0x9f, 0xdc, 0xf7, 0xad, 0x21, 0xf6, 0x7c, 0x63, 0x38, 0x12, 0xcc, 0xb6, 0x63, 0x62, 0xfe, 0xdd, + 0x1c, 0x39, 0x96, 0xed, 0x63, 0xd7, 0x3c, 0xe2, 0x1d, 0x75, 0xc7, 0x35, 0xb1, 0xeb, 0xb1, 0x96, + 0xfa, 0xeb, 0x3c, 0x94, 0xb6, 0xc6, 0xc7, 0x8f, 0xb1, 0x8f, 0x10, 0x14, 0x6c, 0x63, 0x88, 0xdb, + 0xca, 0x9a, 0xb2, 0x5e, 0xd7, 0xe8, 0x37, 0x7a, 0x0b, 0x6a, 0x23, 0xc3, 0x3f, 0xd3, 0x8f, 0xad, + 0xd1, 0x19, 0x76, 0xdb, 0xb9, 0x35, 0x65, 0xbd, 0xb1, 0x71, 0xb3, 0x2b, 0x2d, 0x6f, 0x9b, 0x52, + 0x0e, 0xc7, 0x96, 0x8f, 0x35, 0x20, 0xbc, 0xac, 0x03, 0x6d, 0x03, 0x1c, 0xbb, 0xd8, 0xf0, 0xb1, + 0xa9, 0x1b, 0x7e, 0x3b, 0xbf, 0xa6, 0xac, 0xd7, 0x36, 0x3a, 0x5d, 0xb6, 0xf2, 0xae, 0x58, 0x79, + 0xf7, 0x23, 0xb1, 0xf2, 0xad, 0xca, 0xef, 0xbf, 0x5b, 0x7d, 0xe1, 0x17, 0x7f, 0x5f, 0x55, 0xb4, + 0x2a, 0x1f, 0xb7, 0xe9, 0xa3, 0x37, 0x60, 0xd1, 0xc4, 0x27, 0xc6, 0x78, 0xe0, 0xeb, 0x1e, 0x3e, + 0x1d, 0x62, 0xdb, 0xd7, 0x3d, 0xeb, 0x2b, 0xdc, 0x2e, 0xac, 0x29, 0xeb, 0x79, 0x0d, 0x71, 0xda, + 0x21, 0x23, 0x1d, 0x5a, 0x5f, 0x61, 0xf4, 0x29, 0xdc, 0x12, 0x23, 0x5c, 0x6c, 0x8e, 0x6d, 0xd3, + 0xb0, 0x8f, 0x9f, 0xe8, 0xde, 0xf1, 0x19, 0x1e, 0xe2, 0x76, 0x91, 0xae, 0x62, 0xb9, 0x1b, 0x42, + 0xa2, 0x05, 0x3c, 0x87, 0x94, 0x45, 0xbb, 0xc9, 0x47, 0xc7, 0x09, 0xc8, 0x84, 0x15, 0x21, 0x38, + 0xdc, 0xbd, 0x3e, 0x32, 0x5c, 0x63, 0x88, 0x7d, 0xec, 0x7a, 0xed, 0x12, 0x15, 0xbe, 0x26, 0x63, + 0xf3, 0x20, 0xf8, 0x3c, 0x08, 0xf8, 0xb4, 0x65, 0x2e, 0x26, 0x8d, 0x88, 0x56, 0x00, 0x46, 0x86, + 0xeb, 0xdb, 0xd8, 0xd5, 0x2d, 0xb3, 0x5d, 0xa6, 0x9a, 0xa8, 0xf2, 0x9e, 0xbe, 0xa9, 0x5a, 0xd0, + 0x60, 0xca, 0x7a, 0x64, 0x79, 0x7e, 0xdf, 0xc7, 0xc3, 0x54, 0xa5, 0x45, 0xa1, 0xcf, 0x3d, 0x15, + 0xf4, 0xea, 0xbf, 0x73, 0xb0, 0xc0, 0xe6, 0xda, 0xa6, 0x7d, 0x1a, 0xfe, 0xd1, 0x18, 0x7b, 0xb3, + 0xb6, 0x92, 0x2c, 0x05, 0xe7, 0x9f, 0x4e, 0xc1, 0x85, 0xeb, 0x54, 0x70, 0x71, 0xf6, 0x0a, 0x2e, + 0xc5, 0x15, 0xfc, 0x3d, 0x58, 0x8c, 0x82, 0xee, 0x8d, 0x1c, 0xdb, 0xc3, 0x68, 0x1d, 0x4a, 0x47, + 0xb4, 0x9f, 0xe2, 0x5e, 0xdb, 0x68, 0x75, 0x83, 0xd8, 0xc1, 0xf8, 0x35, 0x4e, 0x57, 0xef, 0x41, + 0x8b, 0xf5, 0x3c, 0xc4, 0xfe, 0x05, 0x3a, 0x53, 0xdf, 0x85, 0x79, 0x89, 0x6f, 0xea, 0x69, 0x5e, + 0x15, 0xd6, 0xd1, 0xc3, 0x03, 0x7c, 0xa1, 0x75, 0xa8, 0x4b, 0x62, 0x4f, 0x82, 0x95, 0x4d, 0xa6, + 0xea, 0x62, 0x05, 0xc4, 0x98, 0x85, 0x80, 0x25, 0x28, 0x1d, 0x8f, 0x5d, 0xcf, 0x71, 0xb9, 0x08, + 0xde, 0x42, 0x8b, 0x50, 0x1c, 0x58, 0x43, 0x8b, 0x99, 0x73, 0x51, 0x63, 0x0d, 0x74, 0x1b, 0xaa, + 0xa6, 0xe5, 0xe2, 0x63, 0x02, 0x32, 0xb5, 0x99, 0xa2, 0x16, 0x76, 0xa8, 0x9f, 0x01, 0x92, 0x27, + 0xe0, 0x7b, 0xec, 0x42, 0xd1, 0xf2, 0xf1, 0xd0, 0x6b, 0x2b, 0x6b, 0xf9, 0xf5, 0xda, 0x46, 0x3b, + 0xbe, 0x45, 0xe1, 0x5a, 0x1a, 0x63, 0x23, 0x5b, 0x1a, 0x3a, 0x2e, 0xa6, 0x13, 0x57, 0x34, 0xfa, + 0xad, 0x7e, 0x06, 0xcb, 0x8c, 0xf9, 0x10, 0xfb, 0x9b, 0xbe, 0xef, 0x5a, 0x47, 0x63, 0x32, 0xe3, + 0x45, 0x3e, 0x72, 0x17, 0x1a, 0x46, 0xc8, 0x49, 0x94, 0x9f, 0xa3, 0xd4, 0x39, 0xa9, 0xb7, 0x6f, + 0xaa, 0x2f, 0xc2, 0xed, 0x74, 0xc9, 0x1c, 0xb4, 0x9f, 0x28, 0xb0, 0xb0, 0x69, 0x9a, 0x2e, 0xf6, + 0x3c, 0x6c, 0xee, 0x93, 0x48, 0xfe, 0x88, 0x22, 0xb1, 0x2e, 0xf0, 0x61, 0x8a, 0x43, 0x5d, 0x1e, + 0xe5, 0x43, 0x16, 0x81, 0xd9, 0x36, 0x2c, 0x7a, 0xbe, 0xe3, 0x1a, 0xa7, 0x58, 0x27, 0xc7, 0x84, + 0x6e, 0x30, 0x69, 0x3c, 0x4e, 0xcc, 0x77, 0xe9, 0xd9, 0xb1, 0xe7, 0x98, 0x98, 0x4f, 0xa3, 0x21, + 0xce, 0x2e, 0xf5, 0xa9, 0xdf, 0xe6, 0x60, 0x89, 0x7b, 0xe5, 0xa7, 0xae, 0x15, 0xe8, 0x7f, 0x7f, + 0x60, 0x12, 0x0d, 0x4a, 0x36, 0x54, 0x17, 0x16, 0x43, 0x40, 0x21, 0x8e, 0xcf, 0xb7, 0x4d, 0xbf, + 0x51, 0x1b, 0xca, 0xdc, 0xed, 0xb9, 0xc7, 0x8b, 0x26, 0x7a, 0x1b, 0x20, 0x74, 0xef, 0xcb, 0xf8, + 0xb5, 0xc4, 0x8e, 0xde, 0x86, 0xce, 0xd0, 0x38, 0x17, 0x6e, 0x8c, 0xcd, 0x68, 0x6c, 0x29, 0xd2, + 0x99, 0x6e, 0x0e, 0x8d, 0xf3, 0x07, 0x82, 0x41, 0x0e, 0x30, 0x3d, 0x00, 0x7c, 0x3e, 0xb2, 0x5c, + 0x83, 0x1a, 0x55, 0x69, 0x8a, 0xe8, 0x29, 0x8d, 0x53, 0xff, 0xa2, 0xc0, 0xcd, 0x28, 0x40, 0x4c, + 0x81, 0x04, 0xa1, 0x1d, 0x68, 0x19, 0x42, 0x85, 0x3a, 0x55, 0x8a, 0x30, 0xc6, 0x95, 0xd0, 0x18, + 0x53, 0x94, 0xac, 0x35, 0x83, 0x61, 0xb4, 0xed, 0xa1, 0x37, 0x61, 0xce, 0x75, 0x1c, 0x5f, 0x1f, + 0x59, 0xf8, 0x18, 0x07, 0x36, 0xb5, 0xd5, 0x24, 0x4b, 0xfa, 0xeb, 0x77, 0xab, 0xe5, 0x03, 0xd2, + 0xdf, 0xef, 0x69, 0x35, 0xc2, 0xc5, 0x1a, 0x26, 0x8d, 0xd6, 0xae, 0xf5, 0xa5, 0xe1, 0x63, 0xfd, + 0x31, 0x7e, 0x42, 0x81, 0xaf, 0x6f, 0xdd, 0xe4, 0x43, 0x9a, 0x94, 0xeb, 0x80, 0xd1, 0x3f, 0xc0, + 0x4f, 0x34, 0x18, 0x05, 0xdf, 0xea, 0x1f, 0xc2, 0x4d, 0x6d, 0x3b, 0x43, 0xb2, 0xa2, 0x59, 0xab, + 0xfd, 0x35, 0x28, 0x73, 0x1d, 0x73, 0x9d, 0x23, 0x49, 0xe7, 0x07, 0xec, 0x4b, 0x13, 0x2c, 0xe8, + 0x6d, 0x68, 0x3a, 0xae, 0x75, 0x6a, 0xd9, 0xc6, 0x40, 0xe0, 0x58, 0xa4, 0x38, 0xa6, 0x99, 0x7f, + 0x43, 0xb0, 0x32, 0xec, 0xd4, 0x1d, 0x68, 0xc7, 0xf6, 0x12, 0x6a, 0x48, 0x5a, 0x86, 0x32, 0x71, + 0x19, 0xaa, 0x01, 0xb7, 0xb8, 0xa4, 0x9e, 0xf3, 0x63, 0x7b, 0xe0, 0x18, 0xe6, 0xac, 0x71, 0x51, + 0xff, 0xac, 0x40, 0x27, 0x31, 0xc7, 0x75, 0x58, 0x94, 0xb4, 0xf3, 0xdc, 0x64, 0x05, 0x3c, 0xbd, + 0x29, 0xfd, 0x00, 0x6e, 0xf0, 0xfd, 0xf4, 0xed, 0x13, 0x67, 0xe6, 0x78, 0xbd, 0x17, 0x84, 0x27, + 0x26, 0x3e, 0x55, 0xb5, 0x93, 0x37, 0xa8, 0xea, 0x81, 0xc1, 0x47, 0xce, 0xb9, 0xd9, 0x2d, 0xf4, + 0x5b, 0x25, 0x30, 0xc3, 0xe8, 0xf1, 0x38, 0x5b, 0xb5, 0xc6, 0x14, 0x95, 0xbb, 0xbc, 0xa2, 0xfe, + 0xa6, 0xc0, 0x12, 0x39, 0x12, 0xf9, 0x22, 0xbd, 0x4b, 0x20, 0xb0, 0x04, 0xa5, 0x91, 0x8b, 0x4f, + 0xac, 0x73, 0x8e, 0x01, 0x6f, 0xa1, 0x55, 0xa8, 0x79, 0xbe, 0xe1, 0xfa, 0xba, 0x71, 0x42, 0xe0, + 0xa7, 0xd6, 0xa2, 0x01, 0xed, 0xda, 0x24, 0x3d, 0x24, 0x39, 0xc2, 0xb6, 0xa9, 0x1f, 0xe1, 0x13, + 0x72, 0xe0, 0x16, 0x58, 0x72, 0x84, 0x6d, 0x73, 0x8b, 0x76, 0x90, 0xd3, 0xde, 0xc5, 0x24, 0x1f, + 0xb0, 0xbe, 0x64, 0x51, 0xbc, 0xa2, 0x85, 0x1d, 0x61, 0x86, 0x50, 0x92, 0x33, 0x84, 0x15, 0x00, + 0x82, 0x94, 0x7e, 0x32, 0x30, 0x4e, 0x3d, 0x9a, 0x50, 0x97, 0xb5, 0x2a, 0xe9, 0x79, 0x8f, 0x74, + 0xd0, 0x30, 0x1d, 0xdd, 0x5d, 0x88, 0xfe, 0x3b, 0xd1, 0x44, 0xe1, 0x5e, 0x08, 0x79, 0xc6, 0x88, + 0xee, 0x84, 0xb4, 0xa1, 0x83, 0xa1, 0x20, 0x92, 0x76, 0x6a, 0x22, 0x8a, 0x64, 0x22, 0xd3, 0x39, + 0xde, 0x32, 0x54, 0x2d, 0x4f, 0xe7, 0x28, 0xe7, 0xe9, 0x14, 0x15, 0xcb, 0x3b, 0xa0, 0x6d, 0xf5, + 0x73, 0x62, 0x52, 0x29, 0x79, 0x09, 0xd9, 0xd4, 0x2a, 0xd4, 0x98, 0x96, 0x74, 0x29, 0x43, 0x01, + 0xd6, 0xb5, 0x47, 0xf2, 0x94, 0x68, 0x82, 0x9a, 0x8b, 0x27, 0xa8, 0xcb, 0x24, 0xd6, 0xa5, 0x65, + 0x26, 0xfb, 0x03, 0x53, 0x5d, 0x04, 0x74, 0xe0, 0x3a, 0x3f, 0xc4, 0xc7, 0xb2, 0x53, 0xab, 0x6f, + 0xc1, 0x42, 0xa4, 0x97, 0xe7, 0x61, 0x2f, 0x41, 0x7d, 0xc4, 0xba, 0x75, 0xcf, 0x18, 0x08, 0x1b, + 0xaa, 0xf1, 0xbe, 0x43, 0x63, 0xe0, 0xab, 0x3f, 0x2b, 0x43, 0x69, 0xff, 0x88, 0x34, 0x33, 0x6d, + 0xed, 0x2e, 0x34, 0xc2, 0x63, 0x5e, 0xf2, 0xbb, 0xb9, 0xa0, 0xf7, 0x80, 0x3b, 0xe0, 0x97, 0xd8, + 0xf5, 0xc2, 0x34, 0x51, 0x34, 0xd1, 0x7d, 0x28, 0x79, 0xbe, 0xe1, 0x8f, 0x3d, 0x6a, 0x6f, 0xe4, + 0xda, 0x12, 0xa8, 0x99, 0x4d, 0xdd, 0x3d, 0xa4, 0x64, 0x8d, 0xb3, 0xa1, 0xd7, 0xa1, 0xea, 0xf9, + 0x2e, 0x36, 0x86, 0x04, 0x9f, 0x22, 0x75, 0xa4, 0x16, 0x77, 0xa4, 0xca, 0x21, 0x25, 0xf4, 0x7b, + 0x5a, 0x85, 0xb1, 0xf4, 0xcd, 0xd8, 0x65, 0xac, 0xf4, 0x74, 0xf7, 0xe0, 0x4d, 0x32, 0x27, 0x99, + 0x9d, 0xc8, 0x28, 0x4f, 0x21, 0xa3, 0xc2, 0x86, 0x6d, 0x92, 0xb4, 0x8f, 0xa5, 0x27, 0x98, 0xca, + 0xa8, 0x4c, 0xb3, 0x0e, 0x3e, 0x6e, 0xd3, 0x47, 0x0f, 0xa1, 0x1d, 0xa2, 0x4d, 0x70, 0x32, 0x0d, + 0xdf, 0xd0, 0x6d, 0xc7, 0x3e, 0xc6, 0xed, 0x2a, 0x85, 0x62, 0x8e, 0x43, 0x51, 0xdc, 0x23, 0x9d, + 0xda, 0x52, 0xc0, 0xbe, 0xcb, 0xb9, 0x69, 0x3f, 0x7a, 0x1d, 0x50, 0x52, 0x50, 0x1b, 0xa8, 0xea, + 0xe6, 0x13, 0x63, 0xd0, 0x6b, 0x80, 0x4e, 0xac, 0xf3, 0x78, 0x22, 0x57, 0xa3, 0xa1, 0xb4, 0x45, + 0x29, 0x72, 0x06, 0xb7, 0x03, 0xf3, 0xc9, 0xab, 0x61, 0x7d, 0x72, 0x0a, 0xd9, 0x72, 0xe3, 0x77, + 0xc2, 0x8f, 0xe1, 0x46, 0xfa, 0x5d, 0x70, 0xee, 0x92, 0x77, 0xc1, 0x45, 0x9c, 0x71, 0x09, 0xf4, + 0x1d, 0xdf, 0x18, 0xb0, 0x6d, 0x34, 0xe8, 0x36, 0xaa, 0xb4, 0x87, 0xae, 0x7f, 0x15, 0x6a, 0x96, + 0x3d, 0xb0, 0x6c, 0xcc, 0xe8, 0x4d, 0x4a, 0x07, 0xd6, 0x25, 0x18, 0x5c, 0x3c, 0x74, 0x7c, 0xce, + 0xd0, 0x62, 0x0c, 0xac, 0x8b, 0x30, 0xa8, 0x1f, 0x42, 0x89, 0x59, 0x2d, 0xaa, 0x41, 0xb9, 0xbf, + 0xf7, 0xc9, 0xe6, 0xa3, 0x7e, 0xaf, 0xf5, 0x02, 0x9a, 0x83, 0xea, 0xc7, 0x07, 0x8f, 0xf6, 0x37, + 0x7b, 0xfd, 0xbd, 0x87, 0x2d, 0x05, 0x35, 0x00, 0xb6, 0xf7, 0x77, 0x77, 0xfb, 0x1f, 0x7d, 0x44, + 0xda, 0x39, 0x42, 0xe6, 0xed, 0x07, 0xbd, 0x56, 0x1e, 0xd5, 0xa1, 0xd2, 0x7b, 0xf0, 0xe8, 0x01, + 0x25, 0x16, 0xd4, 0x3f, 0xe6, 0x01, 0x31, 0x87, 0xd8, 0xc2, 0xa7, 0x96, 0x2d, 0xdd, 0xd7, 0xae, + 0xc7, 0x2f, 0xa3, 0xf6, 0x5a, 0x98, 0xbd, 0xbd, 0x16, 0xaf, 0x6e, 0xaf, 0xa5, 0x2c, 0x7b, 0x4d, + 0xb5, 0xc0, 0xf2, 0x4c, 0x2d, 0xb0, 0x72, 0x15, 0x0b, 0x54, 0x7f, 0x9b, 0x83, 0x85, 0x88, 0x36, + 0x79, 0x50, 0xbe, 0x36, 0x75, 0x46, 0xa2, 0x66, 0x61, 0x62, 0xd4, 0x4c, 0x05, 0xb0, 0x38, 0x53, + 0x00, 0x4b, 0x57, 0x02, 0xb0, 0x27, 0xf0, 0x8b, 0x5c, 0x84, 0xa2, 0xdb, 0x54, 0x26, 0x6d, 0x53, + 0x5d, 0x82, 0xc5, 0xa8, 0x14, 0x7e, 0xcb, 0x7f, 0x0c, 0x2d, 0xd6, 0x2f, 0x15, 0x71, 0xae, 0x4b, + 0x35, 0xea, 0xbb, 0x30, 0x2f, 0x4d, 0x16, 0x56, 0x82, 0x1c, 0xda, 0x99, 0xac, 0x04, 0x31, 0x66, + 0x8d, 0xd3, 0xd5, 0x7f, 0x2a, 0x62, 0x7c, 0xac, 0x8e, 0x93, 0xba, 0xda, 0x57, 0xa1, 0x25, 0xad, + 0x56, 0xce, 0x12, 0x9b, 0xe1, 0x7a, 0x59, 0xba, 0x18, 0x61, 0xe5, 0x45, 0xa1, 0x7c, 0x8c, 0x75, + 0x3b, 0x56, 0x1d, 0x2a, 0xc8, 0xb9, 0x5f, 0x1f, 0x9a, 0x6c, 0x8d, 0xba, 0x65, 0x1f, 0x0f, 0xc6, + 0x26, 0x0e, 0x6b, 0x78, 0xb1, 0xcd, 0x88, 0x9a, 0x4f, 0x9f, 0xf3, 0x69, 0x0d, 0x36, 0x50, 0xb4, + 0xd5, 0xcf, 0x44, 0xf0, 0xbb, 0x64, 0x29, 0x29, 0x2a, 0xf6, 0xa2, 0x52, 0xd2, 0xef, 0xf2, 0xd0, + 0x88, 0x72, 0xa7, 0x68, 0x54, 0x99, 0xa0, 0xd1, 0x5c, 0x56, 0x4e, 0x93, 0xbf, 0x5c, 0x4e, 0x13, + 0x4d, 0x52, 0x0a, 0x33, 0x48, 0x52, 0x8a, 0x33, 0x48, 0x52, 0x4a, 0xb3, 0x0f, 0xfa, 0xe5, 0xab, + 0x07, 0xfd, 0x4a, 0x46, 0xd0, 0x57, 0xff, 0x0f, 0x96, 0xd2, 0xad, 0x09, 0x75, 0xa0, 0x12, 0x0c, + 0x57, 0x58, 0xb2, 0x2e, 0xda, 0xaa, 0x07, 0x6d, 0x29, 0x10, 0x47, 0xab, 0xa9, 0xd7, 0xe6, 0xf2, + 0xef, 0xc3, 0xad, 0x94, 0x49, 0xb9, 0x55, 0x4f, 0x19, 0xc3, 0x02, 0x59, 0xef, 0x59, 0xb6, 0xe5, + 0x9d, 0x45, 0x77, 0x30, 0xa5, 0xac, 0xdb, 0xd0, 0x49, 0x93, 0xc5, 0xa3, 0xe2, 0xbf, 0xf2, 0x50, + 0x3b, 0x34, 0x7c, 0x31, 0xee, 0xfa, 0x0e, 0xab, 0x2b, 0x15, 0x1f, 0xfb, 0x30, 0x47, 0x7d, 0x82, + 0x1c, 0x37, 0xa6, 0xe1, 0xe3, 0xa9, 0x5c, 0xa1, 0x2e, 0x86, 0xf6, 0x0c, 0x1f, 0xa3, 0x5d, 0x68, + 0x86, 0x25, 0x45, 0x26, 0x6c, 0x1a, 0x9f, 0x68, 0x84, 0x83, 0xa9, 0xb8, 0xe7, 0xe4, 0x18, 0xe8, + 0x3e, 0x2c, 0x78, 0x86, 0x8f, 0x07, 0x03, 0x8b, 0x66, 0xac, 0xa7, 0xb6, 0xe1, 0x8f, 0x5d, 0x7e, + 0x61, 0xd0, 0x50, 0x40, 0x3a, 0x14, 0x14, 0xf5, 0x1f, 0x39, 0x28, 0xf3, 0x84, 0x7e, 0x4a, 0x0b, + 0x42, 0xff, 0x0f, 0x95, 0x91, 0xe3, 0x59, 0xbe, 0x88, 0x8a, 0xb5, 0x8d, 0x5b, 0x61, 0xf0, 0xe3, + 0x32, 0x0f, 0x38, 0x83, 0x16, 0xb0, 0xa2, 0x77, 0x61, 0x21, 0xdc, 0xd1, 0x63, 0xfc, 0x84, 0xa3, + 0x92, 0x4f, 0x43, 0x25, 0xdc, 0xe1, 0x07, 0xf8, 0x09, 0x03, 0xe4, 0x0e, 0xcc, 0x45, 0x86, 0xf3, + 0xda, 0x45, 0x5d, 0xe6, 0x44, 0x5d, 0x58, 0x20, 0xe9, 0xba, 0x54, 0x96, 0xa6, 0xb0, 0xb1, 0x72, + 0xf4, 0x3c, 0x21, 0x05, 0xf5, 0xe8, 0x1e, 0x81, 0x6d, 0x23, 0xc8, 0x5c, 0xb0, 0xa9, 0xf3, 0x0b, + 0x81, 0x94, 0x76, 0x86, 0x0b, 0xee, 0x53, 0x1a, 0x1d, 0xf3, 0x0a, 0x94, 0x68, 0x2d, 0xd8, 0x6b, + 0x97, 0xe9, 0x91, 0xd4, 0x0c, 0x37, 0x4f, 0x8b, 0x3c, 0x1a, 0x27, 0xab, 0x3b, 0x50, 0xa4, 0x1d, + 0x68, 0x19, 0xaa, 0xac, 0x7a, 0x6c, 0x8f, 0x87, 0x14, 0xdf, 0xa2, 0x56, 0xa1, 0x1d, 0x7b, 0xe3, + 0x21, 0x52, 0xa1, 0x60, 0x3b, 0x26, 0xe6, 0xf5, 0xa2, 0x06, 0xc7, 0xa1, 0xb4, 0xe7, 0x98, 0xb8, + 0xdf, 0xd3, 0x28, 0x4d, 0xdd, 0x81, 0x66, 0x0c, 0x57, 0x72, 0x3f, 0x19, 0x19, 0xae, 0x4f, 0x44, + 0x1e, 0xf1, 0x12, 0x6a, 0x51, 0xa3, 0x65, 0x85, 0x3d, 0xda, 0x43, 0xce, 0x6b, 0xcb, 0x36, 0xf1, + 0xb9, 0xf8, 0x9b, 0x43, 0x1b, 0xea, 0x2f, 0x15, 0x58, 0xe0, 0xa2, 0x22, 0x77, 0x8c, 0x67, 0x63, + 0x02, 0xf7, 0xa0, 0x39, 0x34, 0xce, 0x75, 0x5a, 0x38, 0x66, 0xc5, 0x36, 0x5e, 0xab, 0x9b, 0x1b, + 0x1a, 0xe7, 0x61, 0x6d, 0x4d, 0xfd, 0x93, 0x02, 0x8b, 0xd1, 0x55, 0xf2, 0xb8, 0xf9, 0x06, 0x80, + 0xb8, 0x9e, 0x06, 0xeb, 0x9c, 0xe7, 0xeb, 0xac, 0x8a, 0x6a, 0x64, 0x4f, 0xab, 0x72, 0xa6, 0x7e, + 0x7a, 0x7d, 0x2f, 0x37, 0x8b, 0xfa, 0xde, 0x14, 0x85, 0xd8, 0x5f, 0xe5, 0x82, 0xed, 0x44, 0x53, + 0xd9, 0xe9, 0xb7, 0x93, 0xe1, 0x44, 0xb9, 0xa7, 0x75, 0xa2, 0xfc, 0xe5, 0x9d, 0xa8, 0x90, 0xe5, + 0x44, 0x0f, 0x61, 0x6e, 0x3c, 0x1a, 0x38, 0x86, 0xa9, 0xbb, 0xd8, 0x1b, 0x0f, 0x7c, 0xfe, 0x83, + 0x40, 0x4d, 0x5a, 0x04, 0xc1, 0xe8, 0xe3, 0x11, 0xaf, 0xac, 0x8f, 0x07, 0xbe, 0x56, 0x1f, 0x4b, + 0x2d, 0xf5, 0xa7, 0x61, 0xa1, 0x36, 0xc1, 0x7a, 0xb1, 0x13, 0xbd, 0x02, 0x65, 0xfa, 0xa3, 0x2d, + 0xf8, 0x3d, 0x13, 0xf7, 0xa3, 0x12, 0x21, 0xf7, 0x4d, 0x74, 0x17, 0x0a, 0x67, 0x86, 0x77, 0xc6, + 0x1f, 0x4b, 0xcc, 0x8b, 0x7f, 0x18, 0x74, 0xba, 0x1d, 0xc3, 0x3b, 0xd3, 0x28, 0x59, 0xfd, 0x6f, + 0x0e, 0xea, 0xe4, 0x18, 0x14, 0x2a, 0x40, 0x1b, 0x71, 0xff, 0xa8, 0x6d, 0xdc, 0x90, 0xf6, 0x17, + 0x9e, 0x98, 0x92, 0x93, 0xc4, 0x5c, 0x34, 0x97, 0xed, 0xa2, 0x79, 0xc9, 0x45, 0x93, 0x3f, 0x9c, + 0x8a, 0x97, 0xf8, 0xe1, 0xf4, 0x21, 0xdc, 0x08, 0x7e, 0xd3, 0x48, 0xee, 0x45, 0xae, 0x60, 0x97, + 0xb0, 0xf5, 0x05, 0x31, 0x36, 0xec, 0xf3, 0x92, 0x87, 0x6c, 0xf9, 0xa9, 0x0f, 0xd9, 0x8c, 0xd3, + 0xa9, 0x92, 0x79, 0x3a, 0xdd, 0x0c, 0x7e, 0x5d, 0xc4, 0x6e, 0x6d, 0xdf, 0xe4, 0x02, 0x13, 0xd9, + 0x35, 0x1e, 0x63, 0x16, 0x96, 0x9f, 0x6d, 0x10, 0x7b, 0x16, 0xe7, 0x58, 0xe6, 0xb9, 0x54, 0xcc, + 0x3c, 0x97, 0x58, 0xd9, 0x38, 0x81, 0x0c, 0xc7, 0xcd, 0x09, 0x88, 0x29, 0x39, 0xf0, 0x72, 0x02, + 0xb7, 0x2b, 0xa3, 0xa4, 0x7e, 0x13, 0xfe, 0x4d, 0x4b, 0x4b, 0x80, 0x9f, 0x63, 0x20, 0x57, 0x7f, + 0x1e, 0x2e, 0x2d, 0x2d, 0x9f, 0x9e, 0x7e, 0x69, 0xef, 0x40, 0x99, 0x45, 0x3e, 0xb1, 0xa2, 0x8c, + 0xd0, 0x17, 0x60, 0x40, 0x42, 0x9f, 0x18, 0x92, 0x88, 0x7a, 0x32, 0xd7, 0xb3, 0x8d, 0x7a, 0x2b, + 0xb0, 0x9c, 0x8a, 0x0b, 0xb7, 0xa1, 0xaf, 0x15, 0x40, 0x9c, 0x2e, 0x97, 0x21, 0x2e, 0xb4, 0x9e, + 0x2d, 0x68, 0xb2, 0xb2, 0x82, 0x7e, 0x79, 0x23, 0x6a, 0xb0, 0x11, 0x41, 0xaa, 0x13, 0x54, 0x1e, + 0xf2, 0x52, 0xe5, 0x41, 0xfd, 0x3c, 0x48, 0x64, 0x22, 0xf5, 0x82, 0xfb, 0xd1, 0x7a, 0x41, 0x72, + 0x9a, 0xcb, 0x14, 0x0c, 0xc2, 0x7c, 0x2b, 0x28, 0x18, 0xc8, 0x6e, 0xa0, 0x5c, 0xde, 0x0d, 0xbe, + 0x56, 0x82, 0xbf, 0xa4, 0xb1, 0x1f, 0xd7, 0xd3, 0x46, 0xab, 0x19, 0x20, 0xa9, 0xfe, 0x27, 0x7c, + 0x5c, 0x10, 0xff, 0xc5, 0xfd, 0x5c, 0x53, 0xab, 0xcc, 0x70, 0x97, 0xcf, 0x4e, 0xc3, 0x5f, 0x87, + 0x82, 0x8d, 0xcf, 0x45, 0x25, 0xe5, 0x02, 0x10, 0x28, 0xdb, 0xc6, 0x6f, 0xe6, 0xa1, 0xb2, 0xcb, + 0x59, 0xd0, 0x2e, 0xd4, 0xd9, 0xe3, 0x2f, 0xfe, 0x2c, 0x73, 0x25, 0xfe, 0x40, 0x29, 0xf2, 0x1e, + 0xaf, 0xf3, 0x62, 0x16, 0x99, 0x43, 0xd7, 0x83, 0xea, 0x43, 0xec, 0x73, 0x59, 0x9d, 0x38, 0x73, + 0x58, 0x5f, 0xec, 0x2c, 0xa7, 0xd2, 0xb8, 0x94, 0x5d, 0xa8, 0x33, 0x87, 0xcb, 0x5a, 0x54, 0x24, + 0x4c, 0x25, 0x17, 0x15, 0x8b, 0xb0, 0x3b, 0x50, 0x23, 0xc6, 0xcb, 0x68, 0x1e, 0x5a, 0x4e, 0x7b, + 0x83, 0x25, 0x64, 0xdd, 0x4e, 0x27, 0x72, 0x49, 0x98, 0x64, 0xaf, 0x5c, 0x90, 0xf4, 0x57, 0x12, + 0xdd, 0x8d, 0x8f, 0x4a, 0xfd, 0x23, 0xda, 0xb9, 0x37, 0x89, 0x8d, 0x4f, 0xf3, 0x3e, 0xd4, 0xe8, + 0x49, 0xc1, 0xff, 0x46, 0xde, 0x8e, 0x17, 0xd4, 0xe4, 0xfb, 0x4a, 0x67, 0x25, 0x83, 0x1a, 0x62, + 0xc9, 0x12, 0x07, 0x2e, 0x2c, 0xc1, 0x1e, 0xc9, 0xc3, 0x65, 0x2c, 0xd3, 0x6a, 0xc5, 0x5c, 0xc1, + 0x5c, 0x56, 0x27, 0xce, 0x9c, 0xae, 0xe0, 0x64, 0xbd, 0x97, 0x6b, 0x84, 0x11, 0x22, 0x1a, 0x49, + 0xd4, 0x76, 0x3b, 0xb7, 0xd3, 0x89, 0x5c, 0xd2, 0x17, 0x30, 0x2f, 0x1d, 0xaa, 0x7c, 0x5d, 0x6a, + 0x2a, 0x24, 0x51, 0xa3, 0xb9, 0x73, 0x21, 0x0f, 0x97, 0xae, 0x03, 0x92, 0xe3, 0x3f, 0x17, 0x9f, + 0x18, 0x9a, 0x72, 0x76, 0x76, 0x5e, 0xbe, 0x98, 0x29, 0xd4, 0x0e, 0x9d, 0x57, 0xd4, 0x1f, 0x56, + 0x12, 0xce, 0x1b, 0xd1, 0xf5, 0x8b, 0x59, 0x64, 0x2e, 0xee, 0x00, 0xe6, 0x98, 0xbe, 0x84, 0xbc, + 0xe4, 0x80, 0xa8, 0xba, 0x57, 0x33, 0xe9, 0x21, 0xbe, 0x61, 0x0e, 0x25, 0xa4, 0x26, 0x0f, 0xf5, + 0x44, 0x06, 0x2a, 0xe3, 0x9b, 0x99, 0x8b, 0x11, 0x7c, 0x25, 0xd8, 0x85, 0xf8, 0x3b, 0xe9, 0xbb, + 0xcc, 0xc4, 0xf7, 0x82, 0xe4, 0xea, 0x08, 0x16, 0x64, 0xdc, 0xc5, 0x0c, 0xc9, 0xc1, 0x69, 0x2a, + 0xbc, 0x3b, 0x81, 0x8b, 0xcf, 0xf1, 0x01, 0xd4, 0xe5, 0x27, 0x1a, 0xb2, 0xbb, 0x26, 0x73, 0x84, + 0xce, 0x4a, 0x06, 0x95, 0x0b, 0xfb, 0x04, 0x9a, 0xe2, 0x3c, 0x12, 0x8b, 0x5d, 0x4b, 0x8c, 0x88, + 0x9d, 0x9f, 0x9d, 0x97, 0x2e, 0xe0, 0xe0, 0x72, 0x3f, 0x85, 0x16, 0x0b, 0xd5, 0x9c, 0x61, 0x7f, + 0x60, 0xa6, 0x08, 0x8e, 0xbd, 0xae, 0x4c, 0x11, 0x9c, 0x78, 0x5e, 0xf8, 0x7d, 0x68, 0x45, 0x4c, + 0x8e, 0xf4, 0xbd, 0x74, 0xb1, 0xd5, 0x11, 0xc9, 0xea, 0x04, 0xc3, 0x23, 0x62, 0x0e, 0xa1, 0x21, + 0x3d, 0xab, 0xa2, 0xef, 0x49, 0x12, 0xa3, 0xa2, 0xef, 0xb9, 0x3a, 0x6b, 0x19, 0x0c, 0xa1, 0x50, + 0x1d, 0x50, 0x0c, 0x60, 0xd2, 0x7b, 0x67, 0x12, 0xc6, 0x44, 0xf8, 0xcb, 0x13, 0x61, 0xe6, 0x80, + 0x44, 0x8c, 0x2d, 0x1d, 0x90, 0xf8, 0x03, 0xaf, 0x14, 0x40, 0x92, 0x2f, 0xb4, 0x3e, 0x81, 0xa6, + 0x6c, 0x69, 0x31, 0x1d, 0xa6, 0xbf, 0x9b, 0x92, 0x75, 0x98, 0xf5, 0xf6, 0xe8, 0x0b, 0x98, 0x8f, + 0x9e, 0x44, 0xa4, 0x33, 0xb2, 0xa0, 0xf4, 0xf7, 0x3d, 0x51, 0x27, 0xcf, 0x78, 0xa7, 0x43, 0x4e, + 0x33, 0xe9, 0x45, 0x8e, 0xec, 0x1e, 0xc9, 0xe7, 0x3b, 0xb2, 0x7b, 0xa4, 0x3c, 0xe3, 0xd9, 0x2a, + 0x7c, 0x9e, 0x1b, 0x1d, 0x1d, 0x95, 0xe8, 0x85, 0xfb, 0xcd, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, + 0x91, 0xea, 0xb2, 0xa0, 0xd3, 0x32, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/pkg/pb/metainfo.proto b/pkg/pb/metainfo.proto index 24ba1451d..c92350069 100644 --- a/pkg/pb/metainfo.proto +++ b/pkg/pb/metainfo.proto @@ -352,7 +352,10 @@ message SatStreamID { google.protobuf.Timestamp creation_date = 5 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false]; google.protobuf.Timestamp expiration_date = 6 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false]; - bytes satellite_signature = 7; + bytes encrypted_metadata_nonce = 7 [(gogoproto.customtype) = "Nonce", (gogoproto.nullable) = false]; + bytes encrypted_metadata = 8; + + bytes satellite_signature = 9; } //--------------------------- diff --git a/pkg/storj/segment.go b/pkg/storj/segment.go new file mode 100644 index 000000000..ebcc52a5a --- /dev/null +++ b/pkg/storj/segment.go @@ -0,0 +1,22 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package storj + +// SegmentPosition segment position in object +type SegmentPosition struct { + PartNumber int32 + Index int32 +} + +// SegmentListItem represents listed segment +type SegmentListItem struct { + Position SegmentPosition +} + +// SegmentDownloadInfo represents segment download information inline/remote +type SegmentDownloadInfo struct { + SegmentID SegmentID + EncryptedInlineData []byte + Next SegmentPosition +} diff --git a/proto.lock b/proto.lock index d59b4f42d..502c86284 100644 --- a/proto.lock +++ b/proto.lock @@ -2738,6 +2738,26 @@ }, { "id": 7, + "name": "encrypted_metadata_nonce", + "type": "bytes", + "options": [ + { + "name": "(gogoproto.customtype)", + "value": "Nonce" + }, + { + "name": "(gogoproto.nullable)", + "value": "false" + } + ] + }, + { + "id": 8, + "name": "encrypted_metadata", + "type": "bytes" + }, + { + "id": 9, "name": "satellite_signature", "type": "bytes" } diff --git a/satellite/metainfo/metainfo.go b/satellite/metainfo/metainfo.go index e600252ac..ea6b76d82 100644 --- a/satellite/metainfo/metainfo.go +++ b/satellite/metainfo/metainfo.go @@ -36,6 +36,8 @@ import ( const ( pieceHashExpiration = 2 * time.Hour satIDExpiration = 24 * time.Hour + lastSegment = -1 + listLimit = 1000 ) var ( @@ -488,15 +490,8 @@ func (endpoint *Endpoint) filterValidPieces(ctx context.Context, pointer *pb.Poi allSizesValid := true lastPieceSize := int64(0) for _, piece := range remote.RemotePieces { - // TODO enable verification - // err := auth.VerifyMsg(piece.Hash, piece.NodeId) - // if err == nil { - // remotePieces = append(remotePieces, piece) - // } else { - // // TODO satellite should send Delete request for piece that failed - // s.logger.Warn("unable to verify piece hash: %v", zap.Error(err)) - // } + // TODO enable piece hash signature verification err = endpoint.validatePieceHash(ctx, piece, limits) if err != nil { @@ -702,6 +697,8 @@ func (endpoint *Endpoint) CreateBucket(ctx context.Context, req *pb.BucketCreate return nil, status.Errorf(codes.InvalidArgument, err.Error()) } + // TODO set default Redundancy if not set + err = endpoint.validateRedundancy(ctx, req.GetDefaultRedundancyScheme()) if err != nil { return nil, status.Errorf(codes.InvalidArgument, err.Error()) @@ -964,26 +961,16 @@ func (endpoint *Endpoint) BeginObject(ctx context.Context, req *pb.ObjectBeginRe pbEP.BlockSize = int64(bucket.DefaultEncryptionParameters.BlockSize) } - satStreamID := &pb.SatStreamID{ - Bucket: req.Bucket, - EncryptedPath: req.EncryptedPath, - Version: req.Version, - Redundancy: pbRS, - CreationDate: time.Now(), - ExpirationDate: req.ExpiresAt, - } - - satStreamID, err = signing.SignStreamID(ctx, endpoint.satellite, satStreamID) - if err != nil { - return nil, status.Errorf(codes.Internal, err.Error()) - } - - encodedStreamID, err := proto.Marshal(satStreamID) - if err != nil { - return nil, status.Errorf(codes.Internal, err.Error()) - } - - streamID, err := storj.StreamIDFromBytes(encodedStreamID) + streamID, err := endpoint.packStreamID(ctx, &pb.SatStreamID{ + Bucket: req.Bucket, + EncryptedPath: req.EncryptedPath, + Version: req.Version, + Redundancy: pbRS, + CreationDate: time.Now(), + ExpirationDate: req.ExpiresAt, + EncryptedMetadataNonce: req.EncryptedMetadataNonce, + EncryptedMetadata: req.EncryptedMetadata, + }) if err != nil { return nil, status.Errorf(codes.Internal, err.Error()) } @@ -1064,24 +1051,12 @@ func (endpoint *Endpoint) GetObject(ctx context.Context, req *pb.ObjectGetReques return nil, status.Errorf(codes.Internal, err.Error()) } - satStreamID := &pb.SatStreamID{ + streamID, err := endpoint.packStreamID(ctx, &pb.SatStreamID{ Bucket: req.Bucket, EncryptedPath: req.EncryptedPath, Version: req.Version, CreationDate: time.Now(), - } - - satStreamID, err = signing.SignStreamID(ctx, endpoint.satellite, satStreamID) - if err != nil { - return nil, status.Errorf(codes.Internal, err.Error()) - } - - encodedStreamID, err := proto.Marshal(satStreamID) - if err != nil { - return nil, status.Errorf(codes.Internal, err.Error()) - } - - streamID, err := storj.StreamIDFromBytes(encodedStreamID) + }) if err != nil { return nil, status.Errorf(codes.Internal, err.Error()) } @@ -1136,9 +1111,10 @@ func (endpoint *Endpoint) ListObjects(ctx context.Context, req *pb.ObjectListReq items := make([]*pb.ObjectListItem, len(segments)) for i, segment := range segments { items[i] = &pb.ObjectListItem{ - EncryptedPath: []byte(segment.Path), - CreatedAt: segment.Pointer.CreationDate, - ExpiresAt: segment.Pointer.ExpirationDate, + EncryptedPath: []byte(segment.Path), + EncryptedMetadata: segment.Pointer.Metadata, + CreatedAt: segment.Pointer.CreationDate, + ExpiresAt: segment.Pointer.ExpirationDate, } } @@ -1237,7 +1213,7 @@ func (endpoint *Endpoint) BeginSegment(ctx context.Context, req *pb.SegmentBegin return nil, status.Errorf(codes.InvalidArgument, err.Error()) } - _, err = endpoint.validateAuth(ctx, macaroon.Action{ + keyInfo, err := endpoint.validateAuth(ctx, macaroon.Action{ Op: macaroon.ActionWrite, Bucket: streamID.Bucket, EncryptedPath: streamID.EncryptedPath, @@ -1247,9 +1223,55 @@ func (endpoint *Endpoint) BeginSegment(ctx context.Context, req *pb.SegmentBegin return nil, status.Errorf(codes.Unauthenticated, err.Error()) } - // TODO implement logic + // no need to validate streamID fields because it was validated during BeginObject - return &pb.SegmentBeginResponse{}, status.Error(codes.Unimplemented, "not implemented") + exceeded, limit, err := endpoint.projectUsage.ExceedsStorageUsage(ctx, keyInfo.ProjectID) + if err != nil { + endpoint.log.Error("retrieving project storage totals", zap.Error(err)) + } + if exceeded { + endpoint.log.Sugar().Errorf("monthly project limits are %s of storage and bandwidth usage. This limit has been exceeded for storage for projectID %s", + limit, keyInfo.ProjectID, + ) + return nil, status.Errorf(codes.ResourceExhausted, "Exceeded Usage Limit") + } + + redundancy, err := eestream.NewRedundancyStrategyFromProto(streamID.Redundancy) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, err.Error()) + } + + maxPieceSize := eestream.CalcPieceSize(req.MaxOrderLimit, redundancy) + + request := overlay.FindStorageNodesRequest{ + RequestedCount: redundancy.TotalCount(), + FreeBandwidth: maxPieceSize, + FreeDisk: maxPieceSize, + } + nodes, err := endpoint.cache.FindStorageNodes(ctx, request) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + + bucketID := createBucketID(keyInfo.ProjectID, streamID.Bucket) + rootPieceID, addressedLimits, piecePrivateKey, err := endpoint.orders.CreatePutOrderLimits(ctx, bucketID, nodes, streamID.ExpirationDate, maxPieceSize) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + + segmentID, err := endpoint.packSegmentID(ctx, &pb.SatSegmentID{ + StreamId: streamID, + OriginalOrderLimits: addressedLimits, + RootPieceId: rootPieceID, + Index: req.Position.Index, + CreationDate: time.Now(), + }) + + return &pb.SegmentBeginResponse{ + SegmentId: segmentID, + AddressedLimits: addressedLimits, + PrivateKey: piecePrivateKey, + }, nil } // CommitSegment commits segment after uploading @@ -1263,7 +1285,7 @@ func (endpoint *Endpoint) CommitSegment(ctx context.Context, req *pb.SegmentComm streamID := segmentID.StreamId - _, err = endpoint.validateAuth(ctx, macaroon.Action{ + keyInfo, err := endpoint.validateAuth(ctx, macaroon.Action{ Op: macaroon.ActionWrite, Bucket: streamID.Bucket, EncryptedPath: streamID.EncryptedPath, @@ -1273,9 +1295,93 @@ func (endpoint *Endpoint) CommitSegment(ctx context.Context, req *pb.SegmentComm return nil, status.Errorf(codes.Unauthenticated, err.Error()) } - // TODO implement logic + if len(segmentID.OriginalOrderLimits) < len(req.UploadResult) { + return nil, status.Errorf(codes.InvalidArgument, "invalid number of upload results: wanted max %d got %d", + len(segmentID.OriginalOrderLimits), len(req.UploadResult)) + } - return &pb.SegmentCommitResponse{}, status.Error(codes.Unimplemented, "not implemented") + pieces := make([]*pb.RemotePiece, len(req.UploadResult)) + for i, result := range req.UploadResult { + pieces[i] = &pb.RemotePiece{ + PieceNum: result.PieceNum, + NodeId: result.NodeId, + Hash: result.Hash, + } + } + remote := &pb.RemoteSegment{ + Redundancy: streamID.Redundancy, + RootPieceId: segmentID.RootPieceId, + RemotePieces: pieces, + } + + pointer := &pb.Pointer{ + Type: pb.Pointer_REMOTE, + Remote: remote, + SegmentSize: req.SizeEncryptedData, + + CreationDate: streamID.CreationDate, + ExpirationDate: streamID.ExpirationDate, + } + + if segmentID.Index == lastSegment { + pointer.Metadata = streamID.EncryptedMetadata + } + + orderLimits := make([]*pb.OrderLimit, len(segmentID.OriginalOrderLimits)) + for i, orderLimit := range segmentID.OriginalOrderLimits { + orderLimits[i] = orderLimit.Limit + } + + err = endpoint.validatePointer(ctx, pointer, orderLimits) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, err.Error()) + } + + err = endpoint.filterValidPieces(ctx, pointer, orderLimits) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + + path, err := CreatePath(ctx, keyInfo.ProjectID, int64(segmentID.Index), streamID.Bucket, streamID.EncryptedPath) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, err.Error()) + } + + exceeded, limit, err := endpoint.projectUsage.ExceedsStorageUsage(ctx, keyInfo.ProjectID) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + if exceeded { + endpoint.log.Sugar().Errorf("monthly project limits are %s of storage and bandwidth usage. This limit has been exceeded for storage for projectID %s.", + limit, keyInfo.ProjectID, + ) + return nil, status.Errorf(codes.ResourceExhausted, "Exceeded Usage Limit") + } + + inlineUsed, remoteUsed := calculateSpaceUsed(pointer) + + // ToDo: Replace with hash & signature validation + // Ensure neither uplink or storage nodes are cheating on us + if pointer.Type == pb.Pointer_REMOTE { + //We cannot have more redundancy than total/min + if float64(remoteUsed) > (float64(pointer.SegmentSize)/float64(pointer.Remote.Redundancy.MinReq))*float64(pointer.Remote.Redundancy.Total) { + endpoint.log.Sugar().Debugf("data size mismatch, got segment: %d, pieces: %d, RS Min, Total: %d,%d", pointer.SegmentSize, remoteUsed, pointer.Remote.Redundancy.MinReq, pointer.Remote.Redundancy.Total) + return nil, status.Errorf(codes.InvalidArgument, "mismatched segment size and piece usage") + } + } + + if err := endpoint.projectUsage.AddProjectStorageUsage(ctx, keyInfo.ProjectID, inlineUsed, remoteUsed); err != nil { + endpoint.log.Sugar().Errorf("Could not track new storage usage by project %v: %v", keyInfo.ProjectID, err) + // but continue. it's most likely our own fault that we couldn't track it, and the only thing + // that will be affected is our per-project bandwidth and storage limits. + } + + err = endpoint.metainfo.Put(ctx, path, pointer) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + + return &pb.SegmentCommitResponse{}, nil } // MakeInlineSegment makes inline segment on satellite @@ -1287,7 +1393,7 @@ func (endpoint *Endpoint) MakeInlineSegment(ctx context.Context, req *pb.Segment return nil, status.Errorf(codes.InvalidArgument, err.Error()) } - _, err = endpoint.validateAuth(ctx, macaroon.Action{ + keyInfo, err := endpoint.validateAuth(ctx, macaroon.Action{ Op: macaroon.ActionWrite, Bucket: streamID.Bucket, EncryptedPath: streamID.EncryptedPath, @@ -1297,9 +1403,58 @@ func (endpoint *Endpoint) MakeInlineSegment(ctx context.Context, req *pb.Segment return nil, status.Errorf(codes.Unauthenticated, err.Error()) } - // TODO implement logic + path, err := CreatePath(ctx, keyInfo.ProjectID, int64(req.Position.Index), streamID.Bucket, streamID.EncryptedPath) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, err.Error()) + } - return &pb.SegmentMakeInlineResponse{}, status.Error(codes.Unimplemented, "not implemented") + exceeded, limit, err := endpoint.projectUsage.ExceedsStorageUsage(ctx, keyInfo.ProjectID) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + if exceeded { + endpoint.log.Sugar().Errorf("monthly project limits are %s of storage and bandwidth usage. This limit has been exceeded for storage for projectID %s.", + limit, keyInfo.ProjectID, + ) + return nil, status.Errorf(codes.ResourceExhausted, "Exceeded Usage Limit") + } + + inlineUsed := int64(len(req.EncryptedInlineData)) + + if err := endpoint.projectUsage.AddProjectStorageUsage(ctx, keyInfo.ProjectID, inlineUsed, 0); err != nil { + endpoint.log.Sugar().Errorf("Could not track new storage usage by project %v: %v", keyInfo.ProjectID, err) + // but continue. it's most likely our own fault that we couldn't track it, and the only thing + // that will be affected is our per-project bandwidth and storage limits. + } + + pointer := &pb.Pointer{ + Type: pb.Pointer_INLINE, + SegmentSize: inlineUsed, + CreationDate: streamID.CreationDate, + ExpirationDate: streamID.ExpirationDate, + InlineSegment: req.EncryptedInlineData, + } + + if req.Position.Index == lastSegment { + pointer.Metadata = streamID.EncryptedMetadata + } + + err = endpoint.metainfo.Put(ctx, path, pointer) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + + err = endpoint.orders.UpdatePutInlineOrder(ctx, keyInfo.ProjectID, streamID.Bucket, inlineUsed) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + + pointer, err = endpoint.metainfo.Get(ctx, path) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + + return &pb.SegmentMakeInlineResponse{}, nil } // BeginDeleteSegment begins segment deletion process @@ -1311,7 +1466,7 @@ func (endpoint *Endpoint) BeginDeleteSegment(ctx context.Context, req *pb.Segmen return nil, status.Errorf(codes.InvalidArgument, err.Error()) } - _, err = endpoint.validateAuth(ctx, macaroon.Action{ + keyInfo, err := endpoint.validateAuth(ctx, macaroon.Action{ Op: macaroon.ActionDelete, Bucket: streamID.Bucket, EncryptedPath: streamID.EncryptedPath, @@ -1321,9 +1476,39 @@ func (endpoint *Endpoint) BeginDeleteSegment(ctx context.Context, req *pb.Segmen return nil, status.Errorf(codes.Unauthenticated, err.Error()) } - // TODO implement logic + path, err := CreatePath(ctx, keyInfo.ProjectID, int64(req.Position.Index), streamID.Bucket, streamID.EncryptedPath) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, err.Error()) + } - return &pb.SegmentBeginDeleteResponse{}, status.Error(codes.Unimplemented, "not implemented") + pointer, err := endpoint.metainfo.Get(ctx, path) + if err != nil { + if storage.ErrKeyNotFound.Has(err) { + return nil, status.Errorf(codes.NotFound, err.Error()) + } + return nil, status.Errorf(codes.Internal, err.Error()) + } + + var limits []*pb.AddressedOrderLimit + if pointer.Type == pb.Pointer_REMOTE && pointer.Remote != nil { + bucketID := createBucketID(keyInfo.ProjectID, streamID.Bucket) + limits, _, err = endpoint.orders.CreateDeleteOrderLimits(ctx, bucketID, pointer) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + } + + segmentID, err := endpoint.packSegmentID(ctx, &pb.SatSegmentID{ + StreamId: streamID, + OriginalOrderLimits: limits, + Index: req.Position.Index, + CreationDate: time.Now(), + }) + + return &pb.SegmentBeginDeleteResponse{ + SegmentId: segmentID, + AddressedLimits: limits, + }, nil } // FinishDeleteSegment finishes segment deletion process @@ -1337,7 +1522,7 @@ func (endpoint *Endpoint) FinishDeleteSegment(ctx context.Context, req *pb.Segme streamID := segmentID.StreamId - _, err = endpoint.validateAuth(ctx, macaroon.Action{ + keyInfo, err := endpoint.validateAuth(ctx, macaroon.Action{ Op: macaroon.ActionDelete, Bucket: streamID.Bucket, EncryptedPath: streamID.EncryptedPath, @@ -1347,12 +1532,35 @@ func (endpoint *Endpoint) FinishDeleteSegment(ctx context.Context, req *pb.Segme return nil, status.Errorf(codes.Unauthenticated, err.Error()) } - // TODO implement logic + path, err := CreatePath(ctx, keyInfo.ProjectID, int64(segmentID.Index), streamID.Bucket, streamID.EncryptedPath) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, err.Error()) + } - return &pb.SegmentFinishDeleteResponse{}, status.Error(codes.Unimplemented, "not implemented") + pointer, err := endpoint.metainfo.Get(ctx, path) + if err != nil { + if storage.ErrKeyNotFound.Has(err) { + return nil, status.Errorf(codes.NotFound, err.Error()) + } + return nil, status.Errorf(codes.Internal, err.Error()) + } + + for _, piece := range pointer.GetRemote().GetRemotePieces() { + _, err := endpoint.containment.Delete(ctx, piece.NodeId) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + } + + err = endpoint.metainfo.Delete(ctx, path) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + + return &pb.SegmentFinishDeleteResponse{}, nil } -// ListSegments list segments +// ListSegments list object segments func (endpoint *Endpoint) ListSegments(ctx context.Context, req *pb.SegmentListRequest) (resp *pb.SegmentListResponse, err error) { defer mon.Task()(&ctx)(&err) @@ -1361,7 +1569,7 @@ func (endpoint *Endpoint) ListSegments(ctx context.Context, req *pb.SegmentListR return nil, status.Errorf(codes.InvalidArgument, err.Error()) } - _, err = endpoint.validateAuth(ctx, macaroon.Action{ + keyInfo, err := endpoint.validateAuth(ctx, macaroon.Action{ Op: macaroon.ActionList, Bucket: streamID.Bucket, EncryptedPath: streamID.EncryptedPath, @@ -1371,9 +1579,52 @@ func (endpoint *Endpoint) ListSegments(ctx context.Context, req *pb.SegmentListR return nil, status.Errorf(codes.Unauthenticated, err.Error()) } - // TODO implement logic + limit := req.Limit + if limit == 0 || limit > listLimit { + limit = listLimit + } - return &pb.SegmentListResponse{}, status.Error(codes.Unimplemented, "not implemented") + index := int64(req.CursorPosition.Index) + more := false + segmentItems := make([]*pb.SegmentListItem, 0) + // TODO think about better implementation + for { + path, err := CreatePath(ctx, keyInfo.ProjectID, index, streamID.Bucket, streamID.EncryptedPath) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + _, err = endpoint.metainfo.Get(ctx, path) + if err != nil { + if storage.ErrKeyNotFound.Has(err) { + if index == lastSegment { + break + } + index = lastSegment + continue + } + return nil, status.Errorf(codes.Internal, err.Error()) + } + if limit == 0 { + more = true + break + } + segmentItems = append(segmentItems, &pb.SegmentListItem{ + Position: &pb.SegmentPosition{ + Index: int32(index), + }, + }) + + if index == lastSegment { + break + } + index++ + limit-- + } + + return &pb.SegmentListResponse{ + Items: segmentItems, + More: more, + }, nil } // DownloadSegment returns data necessary to download segment @@ -1385,7 +1636,7 @@ func (endpoint *Endpoint) DownloadSegment(ctx context.Context, req *pb.SegmentDo return nil, status.Errorf(codes.InvalidArgument, err.Error()) } - _, err = endpoint.validateAuth(ctx, macaroon.Action{ + keyInfo, err := endpoint.validateAuth(ctx, macaroon.Action{ Op: macaroon.ActionRead, Bucket: streamID.Bucket, EncryptedPath: streamID.EncryptedPath, @@ -1395,14 +1646,103 @@ func (endpoint *Endpoint) DownloadSegment(ctx context.Context, req *pb.SegmentDo return nil, status.Errorf(codes.Unauthenticated, err.Error()) } - // TODO implement logic + bucketID := createBucketID(keyInfo.ProjectID, streamID.Bucket) - return &pb.SegmentDownloadResponse{}, status.Error(codes.Unimplemented, "not implemented") + exceeded, limit, err := endpoint.projectUsage.ExceedsBandwidthUsage(ctx, keyInfo.ProjectID, bucketID) + if err != nil { + endpoint.log.Error("retrieving project bandwidth total", zap.Error(err)) + } + if exceeded { + endpoint.log.Sugar().Errorf("monthly project limits are %s of storage and bandwidth usage. This limit has been exceeded for bandwidth for projectID %s.", + limit, keyInfo.ProjectID, + ) + return nil, status.Errorf(codes.ResourceExhausted, "Exceeded Usage Limit") + } + + path, err := CreatePath(ctx, keyInfo.ProjectID, int64(req.CursorPosition.Index), streamID.Bucket, streamID.EncryptedPath) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, err.Error()) + } + + pointer, err := endpoint.metainfo.Get(ctx, path) + if err != nil { + if storage.ErrKeyNotFound.Has(err) { + return nil, status.Errorf(codes.NotFound, err.Error()) + } + return nil, status.Errorf(codes.Internal, err.Error()) + } + + segmentID, err := endpoint.packSegmentID(ctx, &pb.SatSegmentID{}) + + if pointer.Type == pb.Pointer_INLINE { + err := endpoint.orders.UpdateGetInlineOrder(ctx, keyInfo.ProjectID, streamID.Bucket, int64(len(pointer.InlineSegment))) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + return &pb.SegmentDownloadResponse{ + SegmentId: segmentID, + EncryptedInlineData: pointer.InlineSegment, + }, nil + } else if pointer.Type == pb.Pointer_REMOTE && pointer.Remote != nil { + limits, _, err := endpoint.orders.CreateGetOrderLimits(ctx, bucketID, pointer) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + + return &pb.SegmentDownloadResponse{ + SegmentId: segmentID, + AddressedLimits: limits, + }, nil + } + + return &pb.SegmentDownloadResponse{}, status.Errorf(codes.Internal, "invalid type of pointer") } -func (endpoint *Endpoint) unmarshalSatStreamID(ctx context.Context, streamID storj.StreamID) (*pb.SatStreamID, error) { +func (endpoint *Endpoint) packStreamID(ctx context.Context, satStreamID *pb.SatStreamID) (streamID storj.StreamID, err error) { + defer mon.Task()(&ctx)(&err) + + signedStreamID, err := signing.SignStreamID(ctx, endpoint.satellite, satStreamID) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + + encodedStreamID, err := proto.Marshal(signedStreamID) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + + streamID, err = storj.StreamIDFromBytes(encodedStreamID) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + return streamID, nil +} + +func (endpoint *Endpoint) packSegmentID(ctx context.Context, satSegmentID *pb.SatSegmentID) (segmentID storj.SegmentID, err error) { + defer mon.Task()(&ctx)(&err) + + signedSegmentID, err := signing.SignSegmentID(ctx, endpoint.satellite, satSegmentID) + if err != nil { + return nil, err + } + + encodedSegmentID, err := proto.Marshal(signedSegmentID) + if err != nil { + return nil, err + } + + segmentID, err = storj.SegmentIDFromBytes(encodedSegmentID) + if err != nil { + return nil, err + } + return segmentID, nil +} + +func (endpoint *Endpoint) unmarshalSatStreamID(ctx context.Context, streamID storj.StreamID) (_ *pb.SatStreamID, err error) { + defer mon.Task()(&ctx)(&err) + satStreamID := &pb.SatStreamID{} - err := proto.Unmarshal(streamID, satStreamID) + err = proto.Unmarshal(streamID, satStreamID) if err != nil { return nil, err } @@ -1419,9 +1759,11 @@ func (endpoint *Endpoint) unmarshalSatStreamID(ctx context.Context, streamID sto return satStreamID, nil } -func (endpoint *Endpoint) unmarshalSatSegmentID(ctx context.Context, segmentID storj.SegmentID) (*pb.SatSegmentID, error) { +func (endpoint *Endpoint) unmarshalSatSegmentID(ctx context.Context, segmentID storj.SegmentID) (_ *pb.SatSegmentID, err error) { + defer mon.Task()(&ctx)(&err) + satSegmentID := &pb.SatSegmentID{} - err := proto.Unmarshal(segmentID, satSegmentID) + err = proto.Unmarshal(segmentID, satSegmentID) if err != nil { return nil, err } diff --git a/satellite/metainfo/metainfo_test.go b/satellite/metainfo/metainfo_test.go index 8607852f5..1ec7e9642 100644 --- a/satellite/metainfo/metainfo_test.go +++ b/satellite/metainfo/metainfo_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/gogo/protobuf/proto" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/zeebo/errs" @@ -21,6 +22,7 @@ import ( "storj.io/storj/internal/testcontext" "storj.io/storj/internal/testplanet" "storj.io/storj/internal/testrand" + "storj.io/storj/pkg/auth/signing" "storj.io/storj/pkg/eestream" "storj.io/storj/pkg/macaroon" "storj.io/storj/pkg/pb" @@ -771,7 +773,7 @@ func createTestPointer(t *testing.T) *pb.Pointer { func TestBucketNameValidation(t *testing.T) { testplanet.Run(t, testplanet.Config{ - SatelliteCount: 1, StorageNodeCount: 6, UplinkCount: 1, + SatelliteCount: 1, StorageNodeCount: 4, UplinkCount: 1, }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { apiKey := planet.Uplinks[0].APIKey[planet.Satellites[0].ID()] @@ -839,24 +841,22 @@ func TestBeginCommitObject(t *testing.T) { _, err = metainfoService.CreateBucket(ctx, bucket) require.NoError(t, err) - metainfo, err := planet.Uplinks[0].DialMetainfo(ctx, planet.Satellites[0], apiKey) + metainfoClient, err := planet.Uplinks[0].DialMetainfo(ctx, planet.Satellites[0], apiKey) require.NoError(t, err) - defer ctx.Check(metainfo.Close) + defer ctx.Check(metainfoClient.Close) - streamID, err := metainfo.BeginObject( - ctx, - []byte(bucket.Name), - []byte("encrypted-path"), - 1, - storj.RedundancyScheme{}, - storj.EncryptionParameters{}, - time.Time{}, - testrand.Nonce(), - testrand.Bytes(memory.KiB), - ) + streamID, err := metainfoClient.BeginObject(ctx, metainfo.BeginObjectParams{ + Bucket: []byte(bucket.Name), + EncryptedPath: []byte("encrypted-path"), + Redundancy: storj.RedundancyScheme{}, + EncryptionParameters: storj.EncryptionParameters{}, + ExpiresAt: time.Time{}, + EncryptedMetadataNonce: testrand.Nonce(), + EncryptedMetadata: testrand.Bytes(memory.KiB), + }) require.NoError(t, err) - err = metainfo.CommitObject(ctx, streamID) + err = metainfoClient.CommitObject(ctx, streamID) require.NoError(t, err) }) } @@ -867,19 +867,17 @@ func TestBeginFinishDeleteObject(t *testing.T) { }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { apiKey := planet.Uplinks[0].APIKey[planet.Satellites[0].ID()] - metainfo, err := planet.Uplinks[0].DialMetainfo(ctx, planet.Satellites[0], apiKey) + metainfoClient, err := planet.Uplinks[0].DialMetainfo(ctx, planet.Satellites[0], apiKey) require.NoError(t, err) - defer ctx.Check(metainfo.Close) + defer ctx.Check(metainfoClient.Close) - streamID, err := metainfo.BeginDeleteObject( - ctx, - []byte("initial-bucket"), - []byte("encrypted-path"), - 1, - ) + streamID, err := metainfoClient.BeginDeleteObject(ctx, metainfo.BeginDeleteObjectParams{ + Bucket: []byte("initial-bucket"), + EncryptedPath: []byte("encrypted-path"), + }) require.NoError(t, err) - err = metainfo.FinishDeleteObject(ctx, streamID) + err = metainfoClient.FinishDeleteObject(ctx, streamID) require.NoError(t, err) }) } @@ -899,27 +897,459 @@ func TestListGetObjects(t *testing.T) { require.NoError(t, err) } - metainfo, err := planet.Uplinks[0].DialMetainfo(ctx, planet.Satellites[0], apiKey) + metainfoClient, err := planet.Uplinks[0].DialMetainfo(ctx, planet.Satellites[0], apiKey) require.NoError(t, err) - defer ctx.Check(metainfo.Close) + defer ctx.Check(metainfoClient.Close) - expectedBucketName := []byte("testbucket") - items, _, err := metainfo.ListObjects(ctx, expectedBucketName, []byte(""), []byte(""), 0) + expectedBucketName := "testbucket" + items, _, err := metainfoClient.ListObjects(ctx, metainfo.ListObjectsParams{ + Bucket: []byte(expectedBucketName), + }) require.NoError(t, err) require.Equal(t, len(files), len(items)) for _, item := range items { require.NotEmpty(t, item.EncryptedPath) require.True(t, item.CreatedAt.Before(time.Now())) - object, streamID, err := metainfo.GetObject(ctx, expectedBucketName, item.EncryptedPath, -1) + object, streamID, err := metainfoClient.GetObject(ctx, metainfo.GetObjectParams{ + Bucket: []byte(expectedBucketName), + EncryptedPath: item.EncryptedPath, + }) require.NoError(t, err) require.Equal(t, item.EncryptedPath, []byte(object.Path)) require.NotEmpty(t, streamID) } - items, _, err = metainfo.ListObjects(ctx, expectedBucketName, []byte(""), []byte(""), 3) + items, _, err = metainfoClient.ListObjects(ctx, metainfo.ListObjectsParams{ + Bucket: []byte(expectedBucketName), + Limit: 3, + }) require.NoError(t, err) require.Equal(t, 3, len(items)) }) } + +func TestBeginCommitListSegment(t *testing.T) { + testplanet.Run(t, testplanet.Config{ + SatelliteCount: 1, StorageNodeCount: 4, UplinkCount: 1, + }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { + apiKey := planet.Uplinks[0].APIKey[planet.Satellites[0].ID()] + uplink := planet.Uplinks[0] + + config := uplink.GetConfig(planet.Satellites[0]) + metainfoService := planet.Satellites[0].Metainfo.Service + + projects, err := planet.Satellites[0].DB.Console().Projects().GetAll(ctx) + require.NoError(t, err) + projectID := projects[0].ID + + bucket := storj.Bucket{ + Name: "initial-bucket", + ProjectID: projectID, + PathCipher: config.GetEncryptionParameters().CipherSuite, + } + _, err = metainfoService.CreateBucket(ctx, bucket) + require.NoError(t, err) + + metainfoClient, err := planet.Uplinks[0].DialMetainfo(ctx, planet.Satellites[0], apiKey) + require.NoError(t, err) + defer ctx.Check(metainfoClient.Close) + + params := metainfo.BeginObjectParams{ + Bucket: []byte(bucket.Name), + EncryptedPath: []byte("encrypted-path"), + Redundancy: storj.RedundancyScheme{ + Algorithm: storj.ReedSolomon, + ShareSize: 256, + RequiredShares: 1, + RepairShares: 1, + OptimalShares: 3, + TotalShares: 4, + }, + EncryptionParameters: storj.EncryptionParameters{}, + ExpiresAt: time.Now().UTC().Add(24 * time.Hour), + EncryptedMetadataNonce: testrand.Nonce(), + EncryptedMetadata: testrand.Bytes(memory.KiB), + } + streamID, err := metainfoClient.BeginObject(ctx, params) + require.NoError(t, err) + + segmentID, limits, _, err := metainfoClient.BeginSegment(ctx, metainfo.BeginSegmentParams{ + StreamID: streamID, + Position: storj.SegmentPosition{ + Index: -1, + }, + MaxOderLimit: memory.MiB.Int64(), + }) + require.NoError(t, err) + + makeResult := func(num int32) *pb.SegmentPieceUploadResult { + return &pb.SegmentPieceUploadResult{ + PieceNum: num, + NodeId: limits[num].Limit.StorageNodeId, + Hash: &pb.PieceHash{ + PieceId: limits[num].Limit.PieceId, + PieceSize: 1048832, + Timestamp: time.Now(), + // TODO we still not verifying signature in metainfo + }, + } + } + err = metainfoClient.CommitSegment2(ctx, metainfo.CommitSegmentParams{ + SegmentID: segmentID, + SizeEncryptedData: memory.MiB.Int64(), + UploadResult: []*pb.SegmentPieceUploadResult{ + makeResult(0), + makeResult(1), + }, + }) + require.NoError(t, err) + + err = metainfoClient.CommitObject(ctx, streamID) + require.NoError(t, err) + + objects, _, err := metainfoClient.ListObjects(ctx, metainfo.ListObjectsParams{ + Bucket: []byte(bucket.Name), + }) + require.NoError(t, err) + require.Len(t, objects, 1) + + require.Equal(t, params.EncryptedPath, objects[0].EncryptedPath) + require.Equal(t, params.EncryptedMetadata, objects[0].EncryptedMetadata) + require.Equal(t, params.ExpiresAt, objects[0].ExpiresAt) + + _, streamID, err = metainfoClient.GetObject(ctx, metainfo.GetObjectParams{ + Bucket: []byte(bucket.Name), + EncryptedPath: objects[0].EncryptedPath, + }) + require.NoError(t, err) + + segments, _, err := metainfoClient.ListSegments2(ctx, metainfo.ListSegmentsParams{ + StreamID: streamID, + }) + require.NoError(t, err) + require.Len(t, segments, 1) + }) +} + +func TestInlineSegment(t *testing.T) { + testplanet.Run(t, testplanet.Config{ + SatelliteCount: 1, StorageNodeCount: 0, UplinkCount: 1, + }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { + apiKey := planet.Uplinks[0].APIKey[planet.Satellites[0].ID()] + uplink := planet.Uplinks[0] + + config := uplink.GetConfig(planet.Satellites[0]) + metainfoService := planet.Satellites[0].Metainfo.Service + + projects, err := planet.Satellites[0].DB.Console().Projects().GetAll(ctx) + require.NoError(t, err) + projectID := projects[0].ID + + // TODO maybe split into separate cases + // Test: + // * create bucket + // * begin object + // * send several inline segments + // * commit object + // * list created object + // * list object segments + // * download segments + // * delete segments and object + + bucket := storj.Bucket{ + Name: "inline-segments-bucket", + ProjectID: projectID, + PathCipher: config.GetEncryptionParameters().CipherSuite, + } + _, err = metainfoService.CreateBucket(ctx, bucket) + require.NoError(t, err) + + metainfoClient, err := planet.Uplinks[0].DialMetainfo(ctx, planet.Satellites[0], apiKey) + require.NoError(t, err) + defer ctx.Check(metainfoClient.Close) + + params := metainfo.BeginObjectParams{ + Bucket: []byte(bucket.Name), + EncryptedPath: []byte("encrypted-path"), + Redundancy: storj.RedundancyScheme{ + Algorithm: storj.ReedSolomon, + ShareSize: 256, + RequiredShares: 1, + RepairShares: 1, + OptimalShares: 3, + TotalShares: 4, + }, + EncryptionParameters: storj.EncryptionParameters{}, + ExpiresAt: time.Now().UTC().Add(24 * time.Hour), + EncryptedMetadataNonce: testrand.Nonce(), + EncryptedMetadata: testrand.Bytes(memory.KiB), + } + streamID, err := metainfoClient.BeginObject(ctx, params) + require.NoError(t, err) + + segments := []int32{0, 1, 2, 3, 4, 5, -1} + segmentsData := make([][]byte, len(segments)) + for i, segment := range segments { + segmentsData[i] = testrand.Bytes(memory.KiB) + err = metainfoClient.MakeInlineSegment(ctx, metainfo.MakeInlineSegmentParams{ + StreamID: streamID, + Position: storj.SegmentPosition{ + Index: segment, + }, + EncryptedInlineData: segmentsData[i], + }) + require.NoError(t, err) + } + + err = metainfoClient.CommitObject(ctx, streamID) + require.NoError(t, err) + + objects, _, err := metainfoClient.ListObjects(ctx, metainfo.ListObjectsParams{ + Bucket: []byte(bucket.Name), + }) + require.NoError(t, err) + require.Len(t, objects, 1) + + require.Equal(t, params.EncryptedPath, objects[0].EncryptedPath) + require.Equal(t, params.EncryptedMetadata, objects[0].EncryptedMetadata) + require.Equal(t, params.ExpiresAt, objects[0].ExpiresAt) + + _, streamID, err = metainfoClient.GetObject(ctx, metainfo.GetObjectParams{ + Bucket: params.Bucket, + EncryptedPath: params.EncryptedPath, + }) + require.NoError(t, err) + + { // test listing inline segments + for _, test := range []struct { + Index int32 + Limit int + Result int + More bool + }{ + {Index: 0, Result: len(segments), More: false}, + {Index: 2, Result: len(segments) - 2, More: false}, + {Index: 0, Result: 3, More: true, Limit: 3}, + {Index: 0, Result: len(segments), More: false, Limit: len(segments)}, + {Index: 0, Result: len(segments) - 1, More: true, Limit: len(segments) - 1}, + } { + items, more, err := metainfoClient.ListSegments2(ctx, metainfo.ListSegmentsParams{ + StreamID: streamID, + CursorPosition: storj.SegmentPosition{ + Index: test.Index, + }, + Limit: int32(test.Limit), + }) + require.NoError(t, err) + require.Equal(t, test.Result, len(items)) + require.Equal(t, test.More, more) + } + } + + { // test download inline segments + for i, segment := range segments { + info, limits, err := metainfoClient.DownloadSegment(ctx, metainfo.DownloadSegmentParams{ + StreamID: streamID, + Position: storj.SegmentPosition{ + Index: segment, + }, + }) + require.NoError(t, err) + require.Nil(t, limits) + require.Equal(t, segmentsData[i], info.EncryptedInlineData) + } + } + + { // test deleting segments + streamID, err = metainfoClient.BeginDeleteObject(ctx, metainfo.BeginDeleteObjectParams{ + Bucket: params.Bucket, + EncryptedPath: params.EncryptedPath, + }) + require.NoError(t, err) + + items, _, err := metainfoClient.ListSegments2(ctx, metainfo.ListSegmentsParams{ + StreamID: streamID, + }) + require.NoError(t, err) + for _, item := range items { + segmentID, limits, err := metainfoClient.BeginDeleteSegment(ctx, metainfo.BeginDeleteSegmentParams{ + StreamID: streamID, + Position: storj.SegmentPosition{ + Index: item.Position.Index, + }, + }) + require.NoError(t, err) + require.Nil(t, limits) + + err = metainfoClient.FinishDeleteSegment(ctx, metainfo.FinishDeleteSegmentParams{ + SegmentID: segmentID, + }) + require.NoError(t, err) + } + + items, _, err = metainfoClient.ListSegments2(ctx, metainfo.ListSegmentsParams{ + StreamID: streamID, + }) + require.NoError(t, err) + require.Empty(t, items) + + err = metainfoClient.FinishDeleteObject(ctx, streamID) + require.NoError(t, err) + } + }) +} + +func TestRemoteSegment(t *testing.T) { + testplanet.Run(t, testplanet.Config{ + SatelliteCount: 1, StorageNodeCount: 4, UplinkCount: 1, + }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { + apiKey := planet.Uplinks[0].APIKey[planet.Satellites[0].ID()] + uplink := planet.Uplinks[0] + + expectedBucketName := "remote-segments-bucket" + err := uplink.Upload(ctx, planet.Satellites[0], expectedBucketName, "file-object", testrand.Bytes(10*memory.KiB)) + require.NoError(t, err) + + metainfoClient, err := planet.Uplinks[0].DialMetainfo(ctx, planet.Satellites[0], apiKey) + require.NoError(t, err) + defer ctx.Check(metainfoClient.Close) + + items, _, err := metainfoClient.ListObjects(ctx, metainfo.ListObjectsParams{ + Bucket: []byte(expectedBucketName), + }) + require.NoError(t, err) + require.Len(t, items, 1) + + { + // Get object + // List segments + // Download segment + + _, streamID, err := metainfoClient.GetObject(ctx, metainfo.GetObjectParams{ + Bucket: []byte(expectedBucketName), + EncryptedPath: items[0].EncryptedPath, + }) + require.NoError(t, err) + + segments, _, err := metainfoClient.ListSegments2(ctx, metainfo.ListSegmentsParams{ + StreamID: streamID, + }) + require.NoError(t, err) + require.Len(t, segments, 1) + + _, limits, err := metainfoClient.DownloadSegment(ctx, metainfo.DownloadSegmentParams{ + StreamID: streamID, + Position: storj.SegmentPosition{ + Index: segments[0].Position.Index, + }, + }) + require.NoError(t, err) + require.NotEmpty(t, limits) + } + + { + // Begin deleting object + // List segments + // Begin/Finish deleting segment + // List objects + + streamID, err := metainfoClient.BeginDeleteObject(ctx, metainfo.BeginDeleteObjectParams{ + Bucket: []byte(expectedBucketName), + EncryptedPath: items[0].EncryptedPath, + }) + require.NoError(t, err) + + segments, _, err := metainfoClient.ListSegments2(ctx, metainfo.ListSegmentsParams{ + StreamID: streamID, + }) + require.NoError(t, err) + + for _, segment := range segments { + segmentID, limits, err := metainfoClient.BeginDeleteSegment(ctx, metainfo.BeginDeleteSegmentParams{ + StreamID: streamID, + Position: storj.SegmentPosition{ + Index: segment.Position.Index, + }, + }) + require.NoError(t, err) + require.NotEmpty(t, limits) + + err = metainfoClient.FinishDeleteSegment(ctx, metainfo.FinishDeleteSegmentParams{ + SegmentID: segmentID, + }) + require.NoError(t, err) + } + + err = metainfoClient.FinishDeleteObject(ctx, streamID) + require.NoError(t, err) + + items, _, err = metainfoClient.ListObjects(ctx, metainfo.ListObjectsParams{ + Bucket: []byte(expectedBucketName), + }) + require.NoError(t, err) + require.Len(t, items, 0) + } + }) +} + +func TestIDs(t *testing.T) { + testplanet.Run(t, testplanet.Config{ + SatelliteCount: 1, StorageNodeCount: 0, UplinkCount: 1, + }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { + apiKey := planet.Uplinks[0].APIKey[planet.Satellites[0].ID()] + + metainfoClient, err := planet.Uplinks[0].DialMetainfo(ctx, planet.Satellites[0], apiKey) + require.NoError(t, err) + defer ctx.Check(metainfoClient.Close) + + { + streamID := testrand.StreamID(256) + err = metainfoClient.CommitObject(ctx, streamID) + require.Error(t, err) // invalid streamID + + segmentID := testrand.SegmentID(512) + err = metainfoClient.CommitSegment2(ctx, metainfo.CommitSegmentParams{ + SegmentID: segmentID, + }) + require.Error(t, err) // invalid segmentID + } + + satellitePeer := signing.SignerFromFullIdentity(planet.Satellites[0].Identity) + + { // streamID expired + signedStreamID, err := signing.SignStreamID(ctx, satellitePeer, &pb.SatStreamID{ + CreationDate: time.Now().Add(-24 * time.Hour), + }) + require.NoError(t, err) + + encodedStreamID, err := proto.Marshal(signedStreamID) + require.NoError(t, err) + + streamID, err := storj.StreamIDFromBytes(encodedStreamID) + require.NoError(t, err) + + err = metainfoClient.CommitObject(ctx, streamID) + require.Error(t, err) + } + + { // segmentID expired + signedSegmentID, err := signing.SignSegmentID(ctx, satellitePeer, &pb.SatSegmentID{ + CreationDate: time.Now().Add(-24 * time.Hour), + }) + require.NoError(t, err) + + encodedSegmentID, err := proto.Marshal(signedSegmentID) + require.NoError(t, err) + + segmentID, err := storj.SegmentIDFromBytes(encodedSegmentID) + require.NoError(t, err) + + err = metainfoClient.CommitSegment2(ctx, metainfo.CommitSegmentParams{ + SegmentID: segmentID, + }) + require.Error(t, err) + } + }) +} diff --git a/satellite/metainfo/validation.go b/satellite/metainfo/validation.go index f07823668..13bb62f0c 100644 --- a/satellite/metainfo/validation.go +++ b/satellite/metainfo/validation.go @@ -174,54 +174,11 @@ func (endpoint *Endpoint) validateCommitSegment(ctx context.Context, req *pb.Seg return err } - err = endpoint.validatePointer(ctx, req.Pointer) + err = endpoint.validatePointer(ctx, req.Pointer, req.OriginalLimits) if err != nil { return err } - if req.Pointer.Type == pb.Pointer_REMOTE { - remote := req.Pointer.Remote - - if len(req.OriginalLimits) == 0 { - return Error.New("no order limits") - } - if int32(len(req.OriginalLimits)) != remote.Redundancy.Total { - return Error.New("invalid no order limit for piece") - } - - maxAllowed, err := encryption.CalcEncryptedSize(endpoint.rsConfig.MaxSegmentSize.Int64(), storj.EncryptionParameters{ - CipherSuite: storj.EncAESGCM, - BlockSize: 128, // intentionally low block size to allow maximum possible encryption overhead - }) - if err != nil { - return err - } - - if req.Pointer.SegmentSize > maxAllowed || req.Pointer.SegmentSize < 0 { - return Error.New("segment size %v is out of range, maximum allowed is %v", req.Pointer.SegmentSize, maxAllowed) - } - - for _, piece := range remote.RemotePieces { - limit := req.OriginalLimits[piece.PieceNum] - - err := endpoint.orders.VerifyOrderLimitSignature(ctx, limit) - if err != nil { - return err - } - - if limit == nil { - return Error.New("empty order limit for piece") - } - derivedPieceID := remote.RootPieceId.Derive(piece.NodeId, piece.PieceNum) - if limit.PieceId.IsZero() || limit.PieceId != derivedPieceID { - return Error.New("invalid order limit piece id") - } - if bytes.Compare(piece.NodeId.Bytes(), limit.StorageNodeId.Bytes()) != 0 { - return Error.New("piece NodeID != order limit NodeID") - } - } - } - if len(req.OriginalLimits) > 0 { createRequest, found := endpoint.createRequests.Load(req.OriginalLimits[0].SerialNumber) @@ -296,7 +253,7 @@ func isDigit(r byte) bool { return r >= '0' && r <= '9' } -func (endpoint *Endpoint) validatePointer(ctx context.Context, pointer *pb.Pointer) (err error) { +func (endpoint *Endpoint) validatePointer(ctx context.Context, pointer *pb.Pointer, originalLimits []*pb.OrderLimit) (err error) { defer mon.Task()(&ctx)(&err) if pointer == nil { @@ -316,7 +273,50 @@ func (endpoint *Endpoint) validatePointer(ctx context.Context, pointer *pb.Point case pointer.Remote.Redundancy == nil: return Error.New("no redundancy scheme specified") } + + remote := pointer.Remote + + if len(originalLimits) == 0 { + return Error.New("no order limits") + } + if int32(len(originalLimits)) != remote.Redundancy.Total { + return Error.New("invalid no order limit for piece") + } + + maxAllowed, err := encryption.CalcEncryptedSize(endpoint.rsConfig.MaxSegmentSize.Int64(), storj.EncryptionParameters{ + CipherSuite: storj.EncAESGCM, + BlockSize: 128, // intentionally low block size to allow maximum possible encryption overhead + }) + if err != nil { + return err + } + + if pointer.SegmentSize > maxAllowed || pointer.SegmentSize < 0 { + return Error.New("segment size %v is out of range, maximum allowed is %v", pointer.SegmentSize, maxAllowed) + } + + for _, piece := range remote.RemotePieces { + limit := originalLimits[piece.PieceNum] + + if limit == nil { + return Error.New("empty order limit for piece") + } + + err := endpoint.orders.VerifyOrderLimitSignature(ctx, limit) + if err != nil { + return err + } + + derivedPieceID := remote.RootPieceId.Derive(piece.NodeId, piece.PieceNum) + if limit.PieceId.IsZero() || limit.PieceId != derivedPieceID { + return Error.New("invalid order limit piece id") + } + if bytes.Compare(piece.NodeId.Bytes(), limit.StorageNodeId.Bytes()) != 0 { + return Error.New("piece NodeID != order limit NodeID") + } + } } + return nil } diff --git a/uplink/metainfo/client.go b/uplink/metainfo/client.go index 13126d43e..002bc4ce5 100644 --- a/uplink/metainfo/client.go +++ b/uplink/metainfo/client.go @@ -365,30 +365,40 @@ func convertProtoToBucket(pbBucket *pb.Bucket) (bucket storj.Bucket, err error) }, nil } +// BeginObjectParams parmaters for BeginObject method +type BeginObjectParams struct { + Bucket []byte + EncryptedPath []byte + Version int32 + Redundancy storj.RedundancyScheme + EncryptionParameters storj.EncryptionParameters + ExpiresAt time.Time + EncryptedMetadataNonce storj.Nonce + EncryptedMetadata []byte +} + // BeginObject begins object creation -func (client *Client) BeginObject(ctx context.Context, bucket []byte, encryptedPath []byte, version int32, - rs storj.RedundancyScheme, ep storj.EncryptionParameters, expiresAt time.Time, nonce storj.Nonce, encryptedMetadata []byte) (_ storj.StreamID, err error) { +func (client *Client) BeginObject(ctx context.Context, params BeginObjectParams) (_ storj.StreamID, err error) { defer mon.Task()(&ctx)(&err) - // TODO do proper algorithm conversion response, err := client.client.BeginObject(ctx, &pb.ObjectBeginRequest{ - Bucket: bucket, - EncryptedPath: encryptedPath, - Version: version, - ExpiresAt: expiresAt, - EncryptedMetadataNonce: nonce, - EncryptedMetadata: encryptedMetadata, + Bucket: params.Bucket, + EncryptedPath: params.EncryptedPath, + Version: params.Version, + ExpiresAt: params.ExpiresAt, + EncryptedMetadataNonce: params.EncryptedMetadataNonce, + EncryptedMetadata: params.EncryptedMetadata, RedundancyScheme: &pb.RedundancyScheme{ - Type: pb.RedundancyScheme_RS, - ErasureShareSize: rs.ShareSize, - MinReq: int32(rs.RequiredShares), - RepairThreshold: int32(rs.RepairShares), - SuccessThreshold: int32(rs.OptimalShares), - Total: int32(rs.TotalShares), + Type: pb.RedundancyScheme_SchemeType(params.Redundancy.Algorithm), + ErasureShareSize: params.Redundancy.ShareSize, + MinReq: int32(params.Redundancy.RequiredShares), + RepairThreshold: int32(params.Redundancy.RepairShares), + SuccessThreshold: int32(params.Redundancy.OptimalShares), + Total: int32(params.Redundancy.TotalShares), }, EncryptionParameters: &pb.EncryptionParameters{ - CipherSuite: pb.CipherSuite(ep.CipherSuite), - BlockSize: int64(ep.BlockSize), + CipherSuite: pb.CipherSuite(params.EncryptionParameters.CipherSuite), + BlockSize: int64(params.EncryptionParameters.BlockSize), }, }) if err != nil { @@ -408,14 +418,21 @@ func (client *Client) CommitObject(ctx context.Context, streamID storj.StreamID) return Error.Wrap(err) } +// GetObjectParams parameters for GetObject method +type GetObjectParams struct { + Bucket []byte + EncryptedPath []byte + Version int32 +} + // GetObject gets single object -func (client *Client) GetObject(ctx context.Context, bucket []byte, encryptedPath []byte, version int32) (_ storj.Object, _ storj.StreamID, err error) { +func (client *Client) GetObject(ctx context.Context, params GetObjectParams) (_ storj.Object, _ storj.StreamID, err error) { defer mon.Task()(&ctx)(&err) response, err := client.client.GetObject(ctx, &pb.ObjectGetRequest{ - Bucket: bucket, - EncryptedPath: encryptedPath, - Version: version, + Bucket: params.Bucket, + EncryptedPath: params.EncryptedPath, + Version: params.Version, }) if err != nil { return storj.Object{}, storj.StreamID{}, Error.Wrap(err) @@ -434,14 +451,21 @@ func (client *Client) GetObject(ctx context.Context, bucket []byte, encryptedPat return object, response.Object.StreamId, nil } +// BeginDeleteObjectParams parameters for BeginDeleteObject method +type BeginDeleteObjectParams struct { + Bucket []byte + EncryptedPath []byte + Version int32 +} + // BeginDeleteObject begins object deletion process -func (client *Client) BeginDeleteObject(ctx context.Context, bucket []byte, encryptedPath []byte, version int32) (_ storj.StreamID, err error) { +func (client *Client) BeginDeleteObject(ctx context.Context, params BeginDeleteObjectParams) (_ storj.StreamID, err error) { defer mon.Task()(&ctx)(&err) response, err := client.client.BeginDeleteObject(ctx, &pb.ObjectBeginDeleteRequest{ - Bucket: bucket, - EncryptedPath: encryptedPath, - Version: version, + Bucket: params.Bucket, + EncryptedPath: params.EncryptedPath, + Version: params.Version, }) if err != nil { return storj.StreamID{}, Error.Wrap(err) @@ -460,15 +484,27 @@ func (client *Client) FinishDeleteObject(ctx context.Context, streamID storj.Str return Error.Wrap(err) } +// ListObjectsParams parameters for ListObjects method +type ListObjectsParams struct { + Bucket []byte + EncryptedPrefix []byte + EncryptedCursor []byte + Limit int32 + IncludeMetadata bool +} + // ListObjects lists objects according to specific parameters -func (client *Client) ListObjects(ctx context.Context, bucket []byte, encryptedPrefix []byte, encryptedCursor []byte, limit int32) (_ []storj.ObjectListItem, more bool, err error) { +func (client *Client) ListObjects(ctx context.Context, params ListObjectsParams) (_ []storj.ObjectListItem, more bool, err error) { defer mon.Task()(&ctx)(&err) response, err := client.client.ListObjects(ctx, &pb.ObjectListRequest{ - Bucket: bucket, - EncryptedPrefix: encryptedPrefix, - EncryptedCursor: encryptedCursor, - Limit: limit, + Bucket: params.Bucket, + EncryptedPrefix: params.EncryptedPrefix, + EncryptedCursor: params.EncryptedCursor, + Limit: params.Limit, + ObjectIncludes: &pb.ObjectListItemIncludes{ + Metadata: params.IncludeMetadata, + }, }) if err != nil { return []storj.ObjectListItem{}, false, Error.Wrap(err) @@ -490,3 +526,201 @@ func (client *Client) ListObjects(ctx context.Context, bucket []byte, encryptedP return objects, response.More, Error.Wrap(err) } + +// BeginSegmentParams parameters for BeginSegment method +type BeginSegmentParams struct { + StreamID storj.StreamID + Position storj.SegmentPosition + MaxOderLimit int64 +} + +// BeginSegment begins segment upload +func (client *Client) BeginSegment(ctx context.Context, params BeginSegmentParams) (_ storj.SegmentID, limits []*pb.AddressedOrderLimit, piecePrivateKey storj.PiecePrivateKey, err error) { + defer mon.Task()(&ctx)(&err) + + response, err := client.client.BeginSegment(ctx, &pb.SegmentBeginRequest{ + StreamId: params.StreamID, + Position: &pb.SegmentPosition{ + PartNumber: params.Position.PartNumber, + Index: params.Position.Index, + }, + MaxOrderLimit: params.MaxOderLimit, + }) + if err != nil { + return storj.SegmentID{}, nil, storj.PiecePrivateKey{}, Error.Wrap(err) + } + + return response.SegmentId, response.AddressedLimits, response.PrivateKey, nil +} + +// CommitSegmentParams parameters for CommitSegment method +type CommitSegmentParams struct { + SegmentID storj.SegmentID + EncryptedKeyNonce storj.Nonce + EncryptedKey []byte + SizeEncryptedData int64 + // TODO find better way for this + UploadResult []*pb.SegmentPieceUploadResult +} + +// CommitSegment2 commits segment after upload +func (client *Client) CommitSegment2(ctx context.Context, params CommitSegmentParams) (err error) { + // TODO method name will be changed when new methods will be fully integrated with client side + defer mon.Task()(&ctx)(&err) + + _, err = client.client.CommitSegment(ctx, &pb.SegmentCommitRequest{ + SegmentId: params.SegmentID, + EncryptedKeyNonce: params.EncryptedKeyNonce, + EncryptedKey: params.EncryptedKey, + SizeEncryptedData: params.SizeEncryptedData, + UploadResult: params.UploadResult, + }) + if err != nil { + return Error.Wrap(err) + } + + return nil +} + +// MakeInlineSegmentParams parameters for MakeInlineSegment method +type MakeInlineSegmentParams struct { + StreamID storj.StreamID + Position storj.SegmentPosition + EncryptedKeyNonce storj.Nonce + EncryptedKey []byte + EncryptedInlineData []byte +} + +// MakeInlineSegment commits segment after upload +func (client *Client) MakeInlineSegment(ctx context.Context, params MakeInlineSegmentParams) (err error) { + defer mon.Task()(&ctx)(&err) + + _, err = client.client.MakeInlineSegment(ctx, &pb.SegmentMakeInlineRequest{ + StreamId: params.StreamID, + Position: &pb.SegmentPosition{ + PartNumber: params.Position.PartNumber, + Index: params.Position.Index, + }, + EncryptedKeyNonce: params.EncryptedKeyNonce, + EncryptedKey: params.EncryptedKey, + EncryptedInlineData: params.EncryptedInlineData, + }) + if err != nil { + return Error.Wrap(err) + } + + return nil +} + +// BeginDeleteSegmentParams parameters for BeginDeleteSegment method +type BeginDeleteSegmentParams struct { + StreamID storj.StreamID + Position storj.SegmentPosition +} + +// BeginDeleteSegment begins segment upload process +func (client *Client) BeginDeleteSegment(ctx context.Context, params BeginDeleteSegmentParams) (_ storj.SegmentID, limits []*pb.AddressedOrderLimit, err error) { + defer mon.Task()(&ctx)(&err) + + response, err := client.client.BeginDeleteSegment(ctx, &pb.SegmentBeginDeleteRequest{ + StreamId: params.StreamID, + Position: &pb.SegmentPosition{ + PartNumber: params.Position.PartNumber, + Index: params.Position.Index, + }, + }) + if err != nil { + return storj.SegmentID{}, nil, Error.Wrap(err) + } + + return response.SegmentId, response.AddressedLimits, nil +} + +// FinishDeleteSegmentParams parameters for FinishDeleteSegment method +type FinishDeleteSegmentParams struct { + SegmentID storj.SegmentID + // TODO find better way to pass this + DeleteResults []*pb.SegmentPieceDeleteResult +} + +// FinishDeleteSegment finishes segment upload process +func (client *Client) FinishDeleteSegment(ctx context.Context, params FinishDeleteSegmentParams) (err error) { + defer mon.Task()(&ctx)(&err) + + _, err = client.client.FinishDeleteSegment(ctx, &pb.SegmentFinishDeleteRequest{ + SegmentId: params.SegmentID, + Results: params.DeleteResults, + }) + return Error.Wrap(err) +} + +// DownloadSegmentParams parameters for DownloadSegment method +type DownloadSegmentParams struct { + StreamID storj.StreamID + Position storj.SegmentPosition +} + +// DownloadSegment gets info for downloading remote segment or data from inline segment +func (client *Client) DownloadSegment(ctx context.Context, params DownloadSegmentParams) (_ storj.SegmentDownloadInfo, _ []*pb.AddressedOrderLimit, err error) { + defer mon.Task()(&ctx)(&err) + + response, err := client.client.DownloadSegment(ctx, &pb.SegmentDownloadRequest{ + StreamId: params.StreamID, + CursorPosition: &pb.SegmentPosition{ + PartNumber: params.Position.PartNumber, + Index: params.Position.Index, + }, + }) + if err != nil { + return storj.SegmentDownloadInfo{}, nil, Error.Wrap(err) + } + + info := storj.SegmentDownloadInfo{ + SegmentID: response.SegmentId, + EncryptedInlineData: response.EncryptedInlineData, + } + if response.Next != nil { + info.Next = storj.SegmentPosition{ + PartNumber: response.Next.PartNumber, + Index: response.Next.Index, + } + } + + return info, response.AddressedLimits, nil +} + +// ListSegmentsParams parameters for ListSegment method +type ListSegmentsParams struct { + StreamID storj.StreamID + CursorPosition storj.SegmentPosition + Limit int32 +} + +// ListSegments2 lists object segments +func (client *Client) ListSegments2(ctx context.Context, params ListSegmentsParams) (_ []storj.SegmentListItem, more bool, err error) { + // TODO method name will be changed when new methods will be fully integrated with client side + defer mon.Task()(&ctx)(&err) + + response, err := client.client.ListSegments(ctx, &pb.SegmentListRequest{ + StreamId: params.StreamID, + CursorPosition: &pb.SegmentPosition{ + PartNumber: params.CursorPosition.PartNumber, + Index: params.CursorPosition.Index, + }, + Limit: params.Limit, + }) + if err != nil { + return []storj.SegmentListItem{}, false, Error.Wrap(err) + } + + items := make([]storj.SegmentListItem, len(response.Items)) + for i, responseItem := range response.Items { + items[i] = storj.SegmentListItem{ + Position: storj.SegmentPosition{ + PartNumber: responseItem.Position.PartNumber, + Index: responseItem.Position.Index, + }, + } + } + return items, response.More, Error.Wrap(err) +}