17bdb5e9e5
Deprecate the pieceinfo database, and start storing piece info as a header to piece files. Institute a "storage format version" concept allowing us to handle pieces stored under multiple different types of storage. Add a piece_expirations table which will still be used to track expiration times, so we can query it, but which should be much smaller than the pieceinfo database would be for the same number of pieces. (Only pieces with expiration times need to be stored in piece_expirations, and we don't need to store large byte blobs like the serialized order limit, etc.) Use specialized names for accessing any functionality related only to dealing with V0 pieces (e.g., `store.V0PieceInfo()`). Move SpaceUsed- type functionality under the purview of the piece store. Add some generic interfaces for traversing all blobs or all pieces. Add lots of tests.
114 lines
3.3 KiB
Protocol Buffer
114 lines
3.3 KiB
Protocol Buffer
// Copyright (C) 2019 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
syntax = "proto3";
|
|
option go_package = "pb";
|
|
|
|
package piecestore;
|
|
|
|
import "gogo.proto";
|
|
import "google/protobuf/timestamp.proto";
|
|
import "orders.proto";
|
|
|
|
service Piecestore {
|
|
rpc Upload(stream PieceUploadRequest) returns (PieceUploadResponse) {}
|
|
rpc Download(stream PieceDownloadRequest) returns (stream PieceDownloadResponse) {}
|
|
rpc Delete(PieceDeleteRequest) returns (PieceDeleteResponse) {}
|
|
rpc Retain(RetainRequest) returns (RetainResponse);
|
|
}
|
|
|
|
// Expected order of messages from uplink:
|
|
// OrderLimit ->
|
|
// repeated
|
|
// Order ->
|
|
// Chunk ->
|
|
// PieceHash signed by uplink ->
|
|
// <- PieceHash signed by storage node
|
|
//
|
|
message PieceUploadRequest {
|
|
// first message to show that we are allowed to upload
|
|
orders.OrderLimit limit = 1;
|
|
// order for uploading
|
|
orders.Order order = 2;
|
|
|
|
// data message
|
|
message Chunk {
|
|
int64 offset = 1;
|
|
bytes data = 2;
|
|
}
|
|
Chunk chunk = 3;
|
|
// final message
|
|
orders.PieceHash done = 4;
|
|
}
|
|
|
|
message PieceUploadResponse {
|
|
orders.PieceHash done = 1;
|
|
}
|
|
|
|
// Expected order of messages from uplink:
|
|
// {OrderLimit, Chunk} ->
|
|
// go repeated
|
|
// Order -> (async)
|
|
// go repeated
|
|
// <- PieceDownloadResponse.Chunk
|
|
message PieceDownloadRequest {
|
|
// first message to show that we are allowed to upload
|
|
orders.OrderLimit limit = 1;
|
|
// order for downloading
|
|
orders.Order order = 2;
|
|
|
|
// Chunk that we wish to download
|
|
message Chunk {
|
|
int64 offset = 1;
|
|
int64 chunk_size = 2;
|
|
}
|
|
|
|
// request for the chunk
|
|
Chunk chunk = 3;
|
|
}
|
|
|
|
message PieceDownloadResponse {
|
|
// Chunk response for download request
|
|
message Chunk {
|
|
int64 offset = 1;
|
|
bytes data = 2;
|
|
}
|
|
Chunk chunk = 1;
|
|
}
|
|
|
|
message PieceDeleteRequest {
|
|
orders.OrderLimit limit = 1;
|
|
}
|
|
|
|
message PieceDeleteResponse {
|
|
}
|
|
|
|
message RetainRequest {
|
|
google.protobuf.Timestamp creation_date = 1 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
|
|
bytes filter = 2;
|
|
}
|
|
|
|
message RetainResponse {
|
|
}
|
|
|
|
// PieceHeader is used in piece storage to keep track of piece attributes.
|
|
message PieceHeader {
|
|
enum FormatVersion {
|
|
FORMAT_V0 = 0;
|
|
FORMAT_V1 = 1;
|
|
}
|
|
// the storage format version being used for this piece. The piece filename should agree with this.
|
|
// The inclusion of this field is intended to aid repairability when filenames are damaged.
|
|
FormatVersion format_version = 1;
|
|
// content hash of the piece
|
|
bytes hash = 2;
|
|
// timestamp when upload occurred, as given by the "timestamp" field in the original orders.PieceHash
|
|
google.protobuf.Timestamp creation_time = 3 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
|
|
// signature from uplink over the original orders.PieceHash (the corresponding PieceHashSigning
|
|
// is reconstructable using the piece id from the piecestore, the piece size from the
|
|
// filesystem (minus the piece header size), and these (hash, upload_time, signature) fields).
|
|
bytes signature = 4;
|
|
// the OrderLimit authorizing storage of this piece, as signed by the satellite and sent by
|
|
// the uplink
|
|
orders.OrderLimit order_limit = 5 [(gogoproto.nullable) = false];
|
|
} |