storj/pkg/pb/metainfo.proto
JT Olio 946ec201e2
metainfo: move api keys to part of the request (#3069)
What: we move api keys out of the grpc connection-level metadata on the client side and into the request protobufs directly. the server side still supports both mechanisms for backwards compatibility.

Why: dRPC won't support connection-level metadata. the only thing we currently use connection-level metadata for is api keys. we need to move all information needed by a request into the request protobuf itself for drpc support. check out the .proto changes for the main details.

One fun side-fact: Did you know that protobuf fields 1-15 are special and only use one byte for both the field number and type? Additionally did you know we don't use field 15 anywhere yet? So the new request header will use field 15, and should use field 15 on all protobufs going forward.

Please describe the tests: all existing tests should pass

Please describe the performance impact: none
2019-09-19 10:19:29 -06:00

626 lines
18 KiB
Protocol Buffer

// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
syntax = "proto3";
option go_package = "pb";
package metainfo;
import "encryption.proto";
import "gogo.proto";
import "google/protobuf/timestamp.proto";
import "node.proto";
import "pointerdb.proto";
import "orders.proto";
// Metainfo it's a satellite RPC service
service Metainfo {
// Bucket
rpc CreateBucket(BucketCreateRequest) returns (BucketCreateResponse);
rpc GetBucket(BucketGetRequest) returns (BucketGetResponse);
rpc DeleteBucket(BucketDeleteRequest) returns (BucketDeleteResponse);
rpc ListBuckets(BucketListRequest) returns (BucketListResponse);
rpc SetBucketAttribution(BucketSetAttributionRequest) returns (BucketSetAttributionResponse);
// Object
rpc BeginObject(ObjectBeginRequest) returns (ObjectBeginResponse);
rpc CommitObject(ObjectCommitRequest) returns (ObjectCommitResponse);
rpc GetObject(ObjectGetRequest) returns (ObjectGetResponse);
rpc ListObjects(ObjectListRequest) returns (ObjectListResponse);
rpc BeginDeleteObject(ObjectBeginDeleteRequest) returns (ObjectBeginDeleteResponse);
rpc FinishDeleteObject(ObjectFinishDeleteRequest) returns (ObjectFinishDeleteResponse);
rpc BeginSegment(SegmentBeginRequest) returns (SegmentBeginResponse);
rpc CommitSegment(SegmentCommitRequest) returns (SegmentCommitResponse);
rpc MakeInlineSegment(SegmentMakeInlineRequest) returns (SegmentMakeInlineResponse);
rpc BeginDeleteSegment(SegmentBeginDeleteRequest) returns (SegmentBeginDeleteResponse);
rpc FinishDeleteSegment(SegmentFinishDeleteRequest) returns (SegmentFinishDeleteResponse);
rpc ListSegments(SegmentListRequest) returns (SegmentListResponse);
rpc DownloadSegment(SegmentDownloadRequest) returns (SegmentDownloadResponse);
rpc Batch(BatchRequest) returns (BatchResponse);
rpc CreateSegmentOld(SegmentWriteRequestOld) returns (SegmentWriteResponseOld);
rpc CommitSegmentOld(SegmentCommitRequestOld) returns (SegmentCommitResponseOld);
rpc SegmentInfoOld(SegmentInfoRequestOld) returns (SegmentInfoResponseOld);
rpc DownloadSegmentOld(SegmentDownloadRequestOld) returns (SegmentDownloadResponseOld);
rpc DeleteSegmentOld(SegmentDeleteRequestOld) returns (SegmentDeleteResponseOld);
rpc ListSegmentsOld(ListSegmentsRequestOld) returns (ListSegmentsResponseOld);
rpc SetAttributionOld(SetAttributionRequestOld) returns (SetAttributionResponseOld);
rpc ProjectInfo(ProjectInfoRequest) returns (ProjectInfoResponse);
}
message RequestHeader {
bytes api_key = 1;
}
message Bucket {
bytes name = 1;
encryption.CipherSuite path_cipher = 2;
google.protobuf.Timestamp created_at = 3 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
int64 default_segment_size = 4;
pointerdb.RedundancyScheme default_redundancy_scheme = 5;
encryption.EncryptionParameters default_encryption_parameters = 6;
bytes partner_id = 7;
}
message BucketListItem {
bytes name = 1;
google.protobuf.Timestamp created_at = 2 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
}
message BucketCreateRequest {
RequestHeader header = 15;
bytes name = 1;
encryption.CipherSuite path_cipher = 2;
int64 default_segment_size = 3;
pointerdb.RedundancyScheme default_redundancy_scheme = 4;
encryption.EncryptionParameters default_encryption_parameters = 5;
bytes partner_id = 6;
}
message BucketCreateResponse {
Bucket bucket = 1;
}
message BucketGetRequest {
RequestHeader header = 15;
bytes name = 1;
}
message BucketGetResponse {
Bucket bucket = 1;
}
message BucketDeleteRequest {
RequestHeader header = 15;
bytes name = 1;
}
message BucketDeleteResponse {
}
message BucketListRequest {
RequestHeader header = 15;
bytes cursor = 1;
int32 limit = 2;
int32 direction = 3;
}
message BucketListResponse {
repeated BucketListItem items = 1;
bool more = 2;
}
message BucketSetAttributionRequest {
RequestHeader header = 15;
bytes name = 1;
bytes partner_id = 2;
}
message BucketSetAttributionResponse {
}
message AddressedOrderLimit {
orders.OrderLimit limit = 1;
node.NodeAddress storage_node_address = 2;
}
message SegmentWriteRequestOld {
RequestHeader header = 15;
bytes bucket = 1;
bytes path = 2;
int64 segment = 3;
pointerdb.RedundancyScheme redundancy = 4;
int64 max_encrypted_segment_size = 5;
google.protobuf.Timestamp expiration = 6 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
}
message SegmentWriteResponseOld {
repeated AddressedOrderLimit addressed_limits = 1;
bytes root_piece_id = 2 [(gogoproto.customtype) = "PieceID", (gogoproto.nullable) = false];
bytes private_key = 3 [(gogoproto.customtype) = "PiecePrivateKey", (gogoproto.nullable) = false];
}
message SegmentCommitRequestOld {
RequestHeader header = 15;
bytes bucket = 1;
bytes path = 2;
int64 segment = 3;
pointerdb.Pointer pointer = 4;
repeated orders.OrderLimit original_limits = 5;
}
message SegmentCommitResponseOld {
pointerdb.Pointer pointer = 1;
}
message SegmentDownloadRequestOld {
RequestHeader header = 15;
bytes bucket = 1;
bytes path = 2;
int64 segment = 3;
}
message SegmentDownloadResponseOld {
repeated AddressedOrderLimit addressed_limits = 1;
pointerdb.Pointer pointer = 2;
bytes private_key = 3 [(gogoproto.customtype) = "PiecePrivateKey", (gogoproto.nullable) = false];
}
message SegmentInfoRequestOld {
RequestHeader header = 15;
bytes bucket = 1;
bytes path = 2;
int64 segment = 3;
}
message SegmentInfoResponseOld {
pointerdb.Pointer pointer = 2;
}
message SegmentDeleteRequestOld {
RequestHeader header = 15;
bytes bucket = 1;
bytes path = 2;
int64 segment = 3;
}
message SegmentDeleteResponseOld {
repeated AddressedOrderLimit addressed_limits = 1;
bytes private_key = 2 [(gogoproto.customtype) = "PiecePrivateKey", (gogoproto.nullable) = false];
}
message ListSegmentsRequestOld {
RequestHeader header = 15;
bytes bucket = 1;
bytes prefix = 2;
bytes start_after = 3;
bytes end_before = 4;
bool recursive = 5;
int32 limit = 6;
fixed32 meta_flags = 7;
}
message ListSegmentsResponseOld {
message Item {
bytes path = 1;
pointerdb.Pointer pointer = 2;
bool is_prefix = 3;
}
repeated Item items = 1;
bool more = 2;
}
message SetAttributionRequestOld {
RequestHeader header = 15;
bytes bucket_name = 1;
bytes partner_id = 2 ;
}
message SetAttributionResponseOld {
}
message ProjectInfoRequest {
RequestHeader header = 15;
}
message ProjectInfoResponse {
bytes project_salt = 1;
}
//---------------------------
// Object
//---------------------------
message Object {
enum Status {
INVALID = 0;
UPLOADING = 1;
COMMITTING = 2;
COMMITTED = 3;
DELETING = 4;
}
bytes bucket = 1;
bytes encrypted_path = 2;
int32 version = 3;
Status status = 4;
bytes stream_id = 5 [(gogoproto.customtype) = "StreamID", (gogoproto.nullable) = false];
google.protobuf.Timestamp created_at = 6 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
google.protobuf.Timestamp status_at = 7 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
google.protobuf.Timestamp expires_at = 8 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
bytes encrypted_metadata_nonce = 9 [(gogoproto.customtype) = "Nonce", (gogoproto.nullable) = false];
bytes encrypted_metadata = 10;
int64 fixed_segment_size = 11;
pointerdb.RedundancyScheme redundancy_scheme = 12;
encryption.EncryptionParameters encryption_parameters = 13;
int64 total_size = 14; // total size of object
int64 inline_size = 15; // size of inline part of object
int64 remote_size = 16; // size of remote part of object
}
message ObjectBeginRequest {
RequestHeader header = 15;
bytes bucket = 1;
bytes encrypted_path = 2;
int32 version = 3;
google.protobuf.Timestamp expires_at = 4 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
pointerdb.RedundancyScheme redundancy_scheme = 7; // can be zero
encryption.EncryptionParameters encryption_parameters = 8; // can be zero
}
message ObjectBeginResponse {
bytes bucket = 1;
bytes encrypted_path = 2;
int32 version = 3;
bytes stream_id = 4 [(gogoproto.customtype) = "StreamID", (gogoproto.nullable) = false];
pointerdb.RedundancyScheme redundancy_scheme = 5;
encryption.EncryptionParameters encryption_parameters = 6;
}
message ObjectCommitRequest {
RequestHeader header = 15;
bytes stream_id = 1 [(gogoproto.customtype) = "StreamID", (gogoproto.nullable) = false];
bytes encrypted_metadata_nonce = 2 [(gogoproto.customtype) = "Nonce", (gogoproto.nullable) = false];
bytes encrypted_metadata = 3; // TODO: set maximum size limit
}
message ObjectCommitResponse {
}
message ObjectGetRequest {
RequestHeader header = 15;
bytes bucket = 1;
bytes encrypted_path = 2;
int32 version = 3;
}
message ObjectGetResponse {
Object object = 1;
}
message ObjectListRequest {
RequestHeader header = 15;
bytes bucket = 1;
bytes encrypted_prefix = 2;
bytes encrypted_cursor = 3;
bool recursive = 4;
int32 limit = 5;
ObjectListItemIncludes object_includes = 6;
}
message ObjectListResponse {
repeated ObjectListItem items = 1;
bool more = 2;
}
message ObjectListItem {
bytes encrypted_path = 1;
int32 version = 2;
Object.Status status = 3;
google.protobuf.Timestamp created_at = 4 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
google.protobuf.Timestamp status_at = 5 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
google.protobuf.Timestamp expires_at = 6 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
bytes encrypted_metadata_nonce = 7 [(gogoproto.customtype) = "Nonce", (gogoproto.nullable) = false];
bytes encrypted_metadata = 8;
}
message ObjectListItemIncludes {
bool metadata = 1;
}
message ObjectBeginDeleteRequest {
RequestHeader header = 15;
bytes bucket = 1;
bytes encrypted_path = 2;
int32 version = 3;
}
message ObjectBeginDeleteResponse {
bytes stream_id = 1 [(gogoproto.customtype) = "StreamID", (gogoproto.nullable) = false];
}
message ObjectFinishDeleteRequest {
RequestHeader header = 15;
bytes stream_id = 1 [(gogoproto.customtype) = "StreamID", (gogoproto.nullable) = false];
}
message ObjectFinishDeleteResponse {
}
// only for satellite use
message SatStreamID {
bytes bucket = 1;
bytes encrypted_path = 2;
int32 version = 3;
pointerdb.RedundancyScheme redundancy = 4;
google.protobuf.Timestamp creation_date = 5 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
google.protobuf.Timestamp expiration_date = 6 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
bytes satellite_signature = 9;
}
//---------------------------
// Segment
//---------------------------
message Segment {
bytes stream_id = 1 [(gogoproto.customtype) = "StreamID", (gogoproto.nullable) = false];
SegmentPosition position = 2;
bytes encrypted_key_nonce = 3 [(gogoproto.customtype) = "Nonce", (gogoproto.nullable) = false];
bytes encrypted_key = 4;
int64 size_encrypted_data = 5; // refers to segment size not piece size
bytes encrypted_inline_data = 6;
repeated Piece pieces = 7;
}
message Piece {
int32 piece_num = 1;
bytes node = 2[(gogoproto.customtype) = "NodeID", (gogoproto.nullable) = false];
}
message SegmentPosition {
int32 part_number = 1;
int32 index = 2;
}
message SegmentBeginRequest {
RequestHeader header = 15;
bytes stream_id = 1 [(gogoproto.customtype) = "StreamID", (gogoproto.nullable) = false];
SegmentPosition position = 2;
int64 max_order_limit = 3;
}
message SegmentBeginResponse {
bytes segment_id = 1 [(gogoproto.customtype) = "SegmentID", (gogoproto.nullable) = false];
repeated AddressedOrderLimit addressed_limits = 2;
bytes private_key = 3 [(gogoproto.customtype) = "PiecePrivateKey", (gogoproto.nullable) = false];
}
message SegmentCommitRequest {
RequestHeader header = 15;
bytes segment_id = 1 [(gogoproto.customtype) = "SegmentID", (gogoproto.nullable) = false];
bytes encrypted_key_nonce = 2 [(gogoproto.customtype) = "Nonce", (gogoproto.nullable) = false];
bytes encrypted_key = 3;
int64 size_encrypted_data = 4; // refers to segment size not piece size
repeated SegmentPieceUploadResult upload_result = 5;
}
message SegmentPieceUploadResult {
int32 piece_num = 1;
bytes node_id = 2 [(gogoproto.customtype) = "NodeID", (gogoproto.nullable) = false];
orders.PieceHash hash = 3;
}
// only for satellite use
message SatSegmentID {
SatStreamID stream_id = 1;
int32 part_number = 2;
int32 index = 3;
// TODO we have redundancy in SatStreamID, do we need it here?
// pointerdb.RedundancyScheme redundancy = 4;
bytes root_piece_id = 5 [(gogoproto.customtype) = "PieceID", (gogoproto.nullable) = false];
repeated AddressedOrderLimit original_order_limits = 6;
google.protobuf.Timestamp creation_date = 7 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
bytes satellite_signature = 8;
}
message SegmentCommitResponse {
int32 successful_pieces = 1;
}
message SegmentMakeInlineRequest {
RequestHeader header = 15;
bytes stream_id = 1 [(gogoproto.customtype) = "StreamID", (gogoproto.nullable) = false];
SegmentPosition position = 2;
bytes encrypted_key_nonce = 3 [(gogoproto.customtype) = "Nonce", (gogoproto.nullable) = false];
bytes encrypted_key = 4;
bytes encrypted_inline_data = 5;
}
message SegmentMakeInlineResponse {}
message SegmentBeginDeleteRequest {
RequestHeader header = 15;
bytes stream_id = 1;
SegmentPosition position = 2;
}
message SegmentBeginDeleteResponse {
bytes segment_id = 1 [(gogoproto.customtype) = "SegmentID", (gogoproto.nullable) = false];
repeated AddressedOrderLimit addressed_limits = 2;
bytes private_key = 3 [(gogoproto.customtype) = "PiecePrivateKey", (gogoproto.nullable) = false];
}
message SegmentFinishDeleteRequest {
RequestHeader header = 15;
bytes segment_id = 1 [(gogoproto.customtype) = "SegmentID", (gogoproto.nullable) = false];
repeated SegmentPieceDeleteResult results = 2;
}
message SegmentPieceDeleteResult {
int32 piece_num = 1;
bytes node_id = 2 [(gogoproto.customtype) = "NodeID", (gogoproto.nullable) = false];
orders.PieceHash hash = 3;
}
message SegmentFinishDeleteResponse {}
message SegmentListRequest {
RequestHeader header = 15;
bytes stream_id = 1;
SegmentPosition cursor_position = 2;
int32 limit = 3;
}
message SegmentListResponse {
repeated SegmentListItem items = 1;
bool more = 2;
}
message SegmentListItem {
SegmentPosition position = 1;
}
message SegmentDownloadRequest {
RequestHeader header = 15;
bytes stream_id = 1 [(gogoproto.customtype) = "StreamID", (gogoproto.nullable) = false];
SegmentPosition cursor_position = 2;
}
message SegmentDownloadResponse {
bytes segment_id = 1 [(gogoproto.customtype) = "SegmentID", (gogoproto.nullable) = false];
repeated AddressedOrderLimit addressed_limits = 2;
bytes private_key = 3 [(gogoproto.customtype) = "PiecePrivateKey", (gogoproto.nullable) = false];
bytes encrypted_inline_data = 4;
int64 segment_size = 5;
bytes encrypted_key_nonce = 6 [(gogoproto.customtype) = "Nonce", (gogoproto.nullable) = false];
bytes encrypted_key = 7;
SegmentPosition next = 8; // can be nil
}
message BatchRequest {
RequestHeader header = 15; // the only header that matters in a batch.
// headers for specific BatchRequestItems are ignored entirely
repeated BatchRequestItem requests = 1;
}
message BatchRequestItem {
oneof Request {
BucketCreateRequest bucket_create = 1;
BucketGetRequest bucket_get = 2;
BucketDeleteRequest bucket_delete = 3;
BucketListRequest bucket_list = 4;
BucketSetAttributionRequest bucket_set_attribution = 5;
ObjectBeginRequest object_begin = 6;
ObjectCommitRequest object_commit = 7;
ObjectGetRequest object_get = 8;
ObjectListRequest object_list = 9;
ObjectBeginDeleteRequest object_begin_delete = 10;
ObjectFinishDeleteRequest object_finish_delete = 11;
SegmentBeginRequest segment_begin = 12;
SegmentCommitRequest segment_commit = 13;
SegmentMakeInlineRequest segment_make_inline = 14;
SegmentBeginDeleteRequest segment_begin_delete = 15;
SegmentFinishDeleteRequest segment_finish_delete = 16;
SegmentListRequest segment_list = 17;
SegmentDownloadRequest segment_download = 18;
}
}
message BatchResponse {
repeated BatchResponseItem responses = 1;
}
message BatchResponseItem {
oneof Response {
BucketCreateResponse bucket_create = 1;
BucketGetResponse bucket_get = 2;
BucketDeleteResponse bucket_delete = 3;
BucketListResponse bucket_list = 4;
BucketSetAttributionResponse bucket_set_attribution = 5;
ObjectBeginResponse object_begin = 6;
ObjectCommitResponse object_commit = 7;
ObjectGetResponse object_get = 8;
ObjectListResponse object_list = 9;
ObjectBeginDeleteResponse object_begin_delete = 10;
ObjectFinishDeleteResponse object_finish_delete = 11;
SegmentBeginResponse segment_begin = 12;
SegmentCommitResponse segment_commit = 13;
SegmentMakeInlineResponse segment_make_inline = 14;
SegmentBeginDeleteResponse segment_begin_delete = 15;
SegmentFinishDeleteResponse segment_finish_delete = 16;
SegmentListResponse segment_list = 17;
SegmentDownloadResponse segment_download = 18;
}
}