From 8f26f66da0cd513b40a141ed5abc821571e8fd2f Mon Sep 17 00:00:00 2001 From: Michal Niewrzal Date: Thu, 29 Oct 2020 17:16:25 +0100 Subject: [PATCH] internalpb: move satellite specific protobuf types storj/storj We have some types that are only valid for satellite usage. Such types are SatStreamID and SatSegmentID. This change moves those types to storj/storj and adds basic infrastructure for generating code. Change-Id: I1e643844f947ce06b13e51ff16b7e671267cea64 --- go.mod | 2 +- satellite/internalpb/gen.go | 124 ++++++++++++ satellite/internalpb/gogo.proto | 143 ++++++++++++++ satellite/internalpb/metainfo_sat.pb.go | 242 ++++++++++++++++++++++++ satellite/internalpb/metainfo_sat.proto | 41 ++++ satellite/internalpb/types.go | 9 + satellite/metainfo/metainfo.go | 23 +-- satellite/metainfo/signing.go | 90 +++++++++ 8 files changed, 662 insertions(+), 12 deletions(-) create mode 100644 satellite/internalpb/gen.go create mode 100644 satellite/internalpb/gogo.proto create mode 100644 satellite/internalpb/metainfo_sat.pb.go create mode 100644 satellite/internalpb/metainfo_sat.proto create mode 100644 satellite/internalpb/types.go create mode 100644 satellite/metainfo/signing.go diff --git a/go.mod b/go.mod index 8496d5dbc..3f6b533c9 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/cheggaaa/pb/v3 v3.0.5 github.com/fatih/color v1.9.0 github.com/go-redis/redis v6.15.9+incompatible - github.com/gogo/protobuf v1.3.1 // indirect + github.com/gogo/protobuf v1.3.1 github.com/golang-migrate/migrate/v4 v4.7.0 github.com/google/go-cmp v0.5.2 github.com/gorilla/mux v1.8.0 diff --git a/satellite/internalpb/gen.go b/satellite/internalpb/gen.go new file mode 100644 index 000000000..8643e3f76 --- /dev/null +++ b/satellite/internalpb/gen.go @@ -0,0 +1,124 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +// +build ignore + +//go:generate go run gen.go + +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" +) + +var ( + mainpkg = flag.String("pkg", "storj.io/storj/internalpb", "main package name") + protoc = flag.String("protoc", "protoc", "protoc compiler") +) + +var ignoreProto = map[string]bool{ + "gogo.proto": true, +} + +func ignore(files []string) []string { + xs := []string{} + for _, file := range files { + if !ignoreProto[file] { + xs = append(xs, file) + } + } + return xs +} + +// Programs needed for code generation: +// +// github.com/ckaznocha/protoc-gen-lint +// storj.io/drpc/cmd/protoc-gen-drpc +// github.com/nilslice/protolock/cmd/protolock + +func main() { + flag.Parse() + + // TODO: protolock + + { + // cleanup previous files + localfiles, err := filepath.Glob("*.pb.go") + check(err) + + all := []string{} + all = append(all, localfiles...) + for _, match := range all { + _ = os.Remove(match) + } + } + + { + protofiles, err := filepath.Glob("*.proto") + check(err) + + protofiles = ignore(protofiles) + + commonPb := os.Getenv("STORJ_COMMON_PB") + if commonPb == "" { + commonPb = "../../common/pb" + } + + overrideImports := ",Mgoogle/protobuf/timestamp.proto=storj.io/storj/internalpb" + args := []string{ + "--lint_out=.", + "--drpc_out=plugins=drpc,paths=source_relative" + overrideImports + ":.", + "-I=.", + "-I=" + commonPb, + } + args = append(args, protofiles...) + + // generate new code + cmd := exec.Command(*protoc, args...) + fmt.Println(strings.Join(cmd.Args, " ")) + out, err := cmd.CombinedOutput() + fmt.Println(string(out)) + check(err) + } + + { + files, err := filepath.Glob("*.pb.go") + check(err) + for _, file := range files { + process(file) + } + } + + { + // format code to get rid of extra imports + out, err := exec.Command("goimports", "-local", "storj.io", "-w", ".").CombinedOutput() + fmt.Println(string(out)) + check(err) + } +} + +func process(file string) { + data, err := ioutil.ReadFile(file) + check(err) + + source := string(data) + + // When generating code to the same path as proto, it will + // end up generating an `import _ "."`, the following replace removes it. + source = strings.Replace(source, `_ "."`, "", -1) + + err = ioutil.WriteFile(file, []byte(source), 0644) + check(err) +} + +func check(err error) { + if err != nil { + panic(err) + } +} diff --git a/satellite/internalpb/gogo.proto b/satellite/internalpb/gogo.proto new file mode 100644 index 000000000..937487bf8 --- /dev/null +++ b/satellite/internalpb/gogo.proto @@ -0,0 +1,143 @@ +// Protocol Buffers for Go with Gadgets +// +// Copyright (c) 2013, The GoGo Authors. All rights reserved. +// http://github.com/gogo/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; +package gogoproto; + +import "google/protobuf/descriptor.proto"; + +option java_package = "com.google.protobuf"; +option java_outer_classname = "GoGoProtos"; + +extend google.protobuf.EnumOptions { + optional bool goproto_enum_prefix = 62001; + optional bool goproto_enum_stringer = 62021; + optional bool enum_stringer = 62022; + optional string enum_customname = 62023; + optional bool enumdecl = 62024; +} + +extend google.protobuf.EnumValueOptions { + optional string enumvalue_customname = 66001; +} + +extend google.protobuf.FileOptions { + optional bool goproto_getters_all = 63001; + optional bool goproto_enum_prefix_all = 63002; + optional bool goproto_stringer_all = 63003; + optional bool verbose_equal_all = 63004; + optional bool face_all = 63005; + optional bool gostring_all = 63006; + optional bool populate_all = 63007; + optional bool stringer_all = 63008; + optional bool onlyone_all = 63009; + + optional bool equal_all = 63013; + optional bool description_all = 63014; + optional bool testgen_all = 63015; + optional bool benchgen_all = 63016; + optional bool marshaler_all = 63017; + optional bool unmarshaler_all = 63018; + optional bool stable_marshaler_all = 63019; + + optional bool sizer_all = 63020; + + optional bool goproto_enum_stringer_all = 63021; + optional bool enum_stringer_all = 63022; + + optional bool unsafe_marshaler_all = 63023; + optional bool unsafe_unmarshaler_all = 63024; + + optional bool goproto_extensions_map_all = 63025; + optional bool goproto_unrecognized_all = 63026; + optional bool gogoproto_import = 63027; + optional bool protosizer_all = 63028; + optional bool compare_all = 63029; + optional bool typedecl_all = 63030; + optional bool enumdecl_all = 63031; + + optional bool goproto_registration = 63032; + optional bool messagename_all = 63033; + + optional bool goproto_sizecache_all = 63034; + optional bool goproto_unkeyed_all = 63035; +} + +extend google.protobuf.MessageOptions { + optional bool goproto_getters = 64001; + optional bool goproto_stringer = 64003; + optional bool verbose_equal = 64004; + optional bool face = 64005; + optional bool gostring = 64006; + optional bool populate = 64007; + optional bool stringer = 67008; + optional bool onlyone = 64009; + + optional bool equal = 64013; + optional bool description = 64014; + optional bool testgen = 64015; + optional bool benchgen = 64016; + optional bool marshaler = 64017; + optional bool unmarshaler = 64018; + optional bool stable_marshaler = 64019; + + optional bool sizer = 64020; + + optional bool unsafe_marshaler = 64023; + optional bool unsafe_unmarshaler = 64024; + + optional bool goproto_extensions_map = 64025; + optional bool goproto_unrecognized = 64026; + + optional bool protosizer = 64028; + + optional bool typedecl = 64030; + + optional bool messagename = 64033; + + optional bool goproto_sizecache = 64034; + optional bool goproto_unkeyed = 64035; +} + +extend google.protobuf.FieldOptions { + optional bool nullable = 65001; + optional bool embed = 65002; + optional string customtype = 65003; + optional string customname = 65004; + optional string jsontag = 65005; + optional string moretags = 65006; + optional string casttype = 65007; + optional string castkey = 65008; + optional string castvalue = 65009; + + optional bool stdtime = 65010; + optional bool stdduration = 65011; + optional bool wktpointer = 65012; + optional bool compare = 65013; + +} \ No newline at end of file diff --git a/satellite/internalpb/metainfo_sat.pb.go b/satellite/internalpb/metainfo_sat.pb.go new file mode 100644 index 000000000..019200242 --- /dev/null +++ b/satellite/internalpb/metainfo_sat.pb.go @@ -0,0 +1,242 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: metainfo_sat.proto + +package internalpb + +import ( + fmt "fmt" + math "math" + time "time" + + proto "github.com/gogo/protobuf/proto" + + pb "storj.io/common/pb" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf +var _ = time.Kitchen + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +type StreamID struct { + Bucket []byte `protobuf:"bytes,1,opt,name=bucket,proto3" json:"bucket,omitempty"` + EncryptedPath []byte `protobuf:"bytes,2,opt,name=encrypted_path,json=encryptedPath,proto3" json:"encrypted_path,omitempty"` + Version int32 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` + Redundancy *pb.RedundancyScheme `protobuf:"bytes,4,opt,name=redundancy,proto3" json:"redundancy,omitempty"` + CreationDate time.Time `protobuf:"bytes,5,opt,name=creation_date,json=creationDate,proto3,stdtime" json:"creation_date"` + ExpirationDate time.Time `protobuf:"bytes,6,opt,name=expiration_date,json=expirationDate,proto3,stdtime" json:"expiration_date"` + SatelliteSignature []byte `protobuf:"bytes,9,opt,name=satellite_signature,json=satelliteSignature,proto3" json:"satellite_signature,omitempty"` + StreamId []byte `protobuf:"bytes,10,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *StreamID) Reset() { *m = StreamID{} } +func (m *StreamID) String() string { return proto.CompactTextString(m) } +func (*StreamID) ProtoMessage() {} +func (*StreamID) Descriptor() ([]byte, []int) { + return fileDescriptor_47c60bd892d94aaf, []int{0} +} +func (m *StreamID) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_StreamID.Unmarshal(m, b) +} +func (m *StreamID) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_StreamID.Marshal(b, m, deterministic) +} +func (m *StreamID) XXX_Merge(src proto.Message) { + xxx_messageInfo_StreamID.Merge(m, src) +} +func (m *StreamID) XXX_Size() int { + return xxx_messageInfo_StreamID.Size(m) +} +func (m *StreamID) XXX_DiscardUnknown() { + xxx_messageInfo_StreamID.DiscardUnknown(m) +} + +var xxx_messageInfo_StreamID proto.InternalMessageInfo + +func (m *StreamID) GetBucket() []byte { + if m != nil { + return m.Bucket + } + return nil +} + +func (m *StreamID) GetEncryptedPath() []byte { + if m != nil { + return m.EncryptedPath + } + return nil +} + +func (m *StreamID) GetVersion() int32 { + if m != nil { + return m.Version + } + return 0 +} + +func (m *StreamID) GetRedundancy() *pb.RedundancyScheme { + if m != nil { + return m.Redundancy + } + return nil +} + +func (m *StreamID) GetCreationDate() time.Time { + if m != nil { + return m.CreationDate + } + return time.Time{} +} + +func (m *StreamID) GetExpirationDate() time.Time { + if m != nil { + return m.ExpirationDate + } + return time.Time{} +} + +func (m *StreamID) GetSatelliteSignature() []byte { + if m != nil { + return m.SatelliteSignature + } + return nil +} + +func (m *StreamID) GetStreamId() []byte { + if m != nil { + return m.StreamId + } + return nil +} + +// only for satellite use +type SegmentID struct { + StreamId *StreamID `protobuf:"bytes,1,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` + PartNumber int32 `protobuf:"varint,2,opt,name=part_number,json=partNumber,proto3" json:"part_number,omitempty"` + Index int32 `protobuf:"varint,3,opt,name=index,proto3" json:"index,omitempty"` + RootPieceId PieceID `protobuf:"bytes,5,opt,name=root_piece_id,json=rootPieceId,proto3,customtype=PieceID" json:"root_piece_id"` + OriginalOrderLimits []*pb.AddressedOrderLimit `protobuf:"bytes,6,rep,name=original_order_limits,json=originalOrderLimits,proto3" json:"original_order_limits,omitempty"` + CreationDate time.Time `protobuf:"bytes,7,opt,name=creation_date,json=creationDate,proto3,stdtime" json:"creation_date"` + SatelliteSignature []byte `protobuf:"bytes,8,opt,name=satellite_signature,json=satelliteSignature,proto3" json:"satellite_signature,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SegmentID) Reset() { *m = SegmentID{} } +func (m *SegmentID) String() string { return proto.CompactTextString(m) } +func (*SegmentID) ProtoMessage() {} +func (*SegmentID) Descriptor() ([]byte, []int) { + return fileDescriptor_47c60bd892d94aaf, []int{1} +} +func (m *SegmentID) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SegmentID.Unmarshal(m, b) +} +func (m *SegmentID) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SegmentID.Marshal(b, m, deterministic) +} +func (m *SegmentID) XXX_Merge(src proto.Message) { + xxx_messageInfo_SegmentID.Merge(m, src) +} +func (m *SegmentID) XXX_Size() int { + return xxx_messageInfo_SegmentID.Size(m) +} +func (m *SegmentID) XXX_DiscardUnknown() { + xxx_messageInfo_SegmentID.DiscardUnknown(m) +} + +var xxx_messageInfo_SegmentID proto.InternalMessageInfo + +func (m *SegmentID) GetStreamId() *StreamID { + if m != nil { + return m.StreamId + } + return nil +} + +func (m *SegmentID) GetPartNumber() int32 { + if m != nil { + return m.PartNumber + } + return 0 +} + +func (m *SegmentID) GetIndex() int32 { + if m != nil { + return m.Index + } + return 0 +} + +func (m *SegmentID) GetOriginalOrderLimits() []*pb.AddressedOrderLimit { + if m != nil { + return m.OriginalOrderLimits + } + return nil +} + +func (m *SegmentID) GetCreationDate() time.Time { + if m != nil { + return m.CreationDate + } + return time.Time{} +} + +func (m *SegmentID) GetSatelliteSignature() []byte { + if m != nil { + return m.SatelliteSignature + } + return nil +} + +func init() { + proto.RegisterType((*StreamID)(nil), "metainfo.StreamID") + proto.RegisterType((*SegmentID)(nil), "metainfo.SegmentID") +} + +func init() { proto.RegisterFile("metainfo_sat.proto", fileDescriptor_47c60bd892d94aaf) } + +var fileDescriptor_47c60bd892d94aaf = []byte{ + // 492 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x52, 0xcb, 0x8e, 0xd3, 0x40, + 0x10, 0xc4, 0x1b, 0xf2, 0x9a, 0xbc, 0xa4, 0x59, 0x40, 0x26, 0x11, 0x4a, 0xb4, 0x12, 0x52, 0x4e, + 0xb6, 0xb4, 0x7b, 0xe4, 0x44, 0x94, 0x8b, 0x25, 0x1e, 0x8b, 0xc3, 0x89, 0x8b, 0x35, 0xf6, 0xf4, + 0x3a, 0x03, 0xf6, 0x8c, 0x35, 0xd3, 0x46, 0xbb, 0x7f, 0xc0, 0x91, 0xcf, 0xe2, 0x1b, 0x38, 0x2c, + 0x9f, 0xc1, 0x15, 0x79, 0x1c, 0xc7, 0x91, 0xd0, 0x1e, 0xe0, 0xe6, 0xaa, 0xae, 0xea, 0xe9, 0xee, + 0x32, 0xa1, 0x39, 0x20, 0x13, 0xf2, 0x46, 0x45, 0x86, 0xa1, 0x57, 0x68, 0x85, 0x8a, 0x0e, 0x1a, + 0x6e, 0x4e, 0x52, 0x95, 0xaa, 0x9a, 0x9d, 0x2f, 0x53, 0xa5, 0xd2, 0x0c, 0x7c, 0x8b, 0xe2, 0xf2, + 0xc6, 0x47, 0x91, 0x83, 0x41, 0x96, 0x17, 0x07, 0xc1, 0xac, 0x50, 0x42, 0x22, 0x68, 0x1e, 0x1f, + 0x88, 0x69, 0xd3, 0xa7, 0xc6, 0x17, 0xdf, 0x3a, 0x64, 0xb0, 0x43, 0x0d, 0x2c, 0x0f, 0xb6, 0xf4, + 0x19, 0xe9, 0xc5, 0x65, 0xf2, 0x05, 0xd0, 0x75, 0x56, 0xce, 0x7a, 0x1c, 0x1e, 0x10, 0x7d, 0x49, + 0xa6, 0x20, 0x13, 0x7d, 0x57, 0x20, 0xf0, 0xa8, 0x60, 0xb8, 0x77, 0xcf, 0x6c, 0x7d, 0x72, 0x64, + 0xaf, 0x19, 0xee, 0xa9, 0x4b, 0xfa, 0x5f, 0x41, 0x1b, 0xa1, 0xa4, 0xdb, 0x59, 0x39, 0xeb, 0x6e, + 0xd8, 0x40, 0xfa, 0x8a, 0x10, 0x0d, 0xbc, 0x94, 0x9c, 0xc9, 0xe4, 0xce, 0x7d, 0xbc, 0x72, 0xd6, + 0xa3, 0xcb, 0x85, 0xd7, 0xce, 0x16, 0x1e, 0x8b, 0xbb, 0x64, 0x0f, 0x39, 0x84, 0x27, 0x72, 0x1a, + 0x90, 0x49, 0xa2, 0x81, 0xa1, 0x50, 0x32, 0xe2, 0x0c, 0xc1, 0xed, 0x5a, 0xff, 0xdc, 0xab, 0x97, + 0xf7, 0x9a, 0xe5, 0xbd, 0x8f, 0xcd, 0xf2, 0x9b, 0xc1, 0x8f, 0xfb, 0xe5, 0xa3, 0xef, 0xbf, 0x96, + 0x4e, 0x38, 0x6e, 0xac, 0x5b, 0x86, 0x40, 0xdf, 0x92, 0x19, 0xdc, 0x16, 0x42, 0x9f, 0x34, 0xeb, + 0xfd, 0x43, 0xb3, 0x69, 0x6b, 0xb6, 0xed, 0x7c, 0x72, 0x6e, 0x18, 0x42, 0x96, 0x09, 0x84, 0xc8, + 0x88, 0x54, 0x32, 0x2c, 0x35, 0xb8, 0x43, 0x7b, 0x1c, 0x7a, 0x2c, 0xed, 0x9a, 0x0a, 0x5d, 0x90, + 0xa1, 0xb1, 0xc7, 0x8e, 0x04, 0x77, 0x89, 0x95, 0x0d, 0x6a, 0x22, 0xe0, 0x17, 0xbf, 0xcf, 0xc8, + 0x70, 0x07, 0x69, 0x0e, 0x12, 0x83, 0x2d, 0xf5, 0x4f, 0xa5, 0x8e, 0x1d, 0x92, 0x7a, 0xc7, 0xf0, + 0x9a, 0xc8, 0x5a, 0x3b, 0x5d, 0x92, 0x51, 0xc1, 0x34, 0x46, 0xb2, 0xcc, 0x63, 0xd0, 0x36, 0xa1, + 0x6e, 0x48, 0x2a, 0xea, 0x9d, 0x65, 0xe8, 0x13, 0xd2, 0x15, 0x92, 0xc3, 0xed, 0x21, 0x9c, 0x1a, + 0xd0, 0x2b, 0x32, 0xd1, 0x4a, 0x61, 0x54, 0x08, 0x48, 0xa0, 0x7a, 0xab, 0xba, 0xee, 0x78, 0x33, + 0xab, 0x96, 0xfe, 0x79, 0xbf, 0xec, 0x5f, 0x57, 0x7c, 0xb0, 0x0d, 0x47, 0x95, 0xaa, 0x06, 0x9c, + 0x7e, 0x20, 0x4f, 0x95, 0x16, 0xa9, 0x90, 0x2c, 0x8b, 0x94, 0xe6, 0xa0, 0xa3, 0x4c, 0xe4, 0x02, + 0x8d, 0xdb, 0x5b, 0x75, 0xd6, 0xa3, 0xcb, 0x17, 0xed, 0xa0, 0xaf, 0x39, 0xd7, 0x60, 0x0c, 0xf0, + 0xf7, 0x95, 0xec, 0x4d, 0xa5, 0x0a, 0xcf, 0x1b, 0x6f, 0xcb, 0x99, 0xbf, 0x53, 0xee, 0xff, 0x77, + 0xca, 0x0f, 0xc4, 0x32, 0x78, 0x28, 0x96, 0xcd, 0xe2, 0xd3, 0x73, 0x83, 0x4a, 0x7f, 0xf6, 0x84, + 0xf2, 0xed, 0x87, 0x6f, 0x7f, 0x4c, 0xc9, 0xb2, 0x22, 0x8e, 0x7b, 0xf6, 0xe5, 0xab, 0x3f, 0x01, + 0x00, 0x00, 0xff, 0xff, 0xcf, 0xd5, 0x29, 0xca, 0x96, 0x03, 0x00, 0x00, +} diff --git a/satellite/internalpb/metainfo_sat.proto b/satellite/internalpb/metainfo_sat.proto new file mode 100644 index 000000000..9d18276d2 --- /dev/null +++ b/satellite/internalpb/metainfo_sat.proto @@ -0,0 +1,41 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +syntax = "proto3"; +option go_package = "storj.io/storj/internalpb"; + +package metainfo; + +import "gogo.proto"; +import "google/protobuf/timestamp.proto"; +import "pointerdb.proto"; +import "metainfo.proto"; + + +message StreamID { + bytes bucket = 1; + bytes encrypted_path = 2; + int32 version = 3; + + pointerdb.RedundancyScheme redundancy = 4; + + google.protobuf.Timestamp creation_date = 5 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false]; + google.protobuf.Timestamp expiration_date = 6 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false]; + + bytes satellite_signature = 9; + + bytes stream_id = 10; +} + +// only for satellite use +message SegmentID { + StreamID stream_id = 1; + int32 part_number = 2; + int32 index = 3; + + bytes root_piece_id = 5 [(gogoproto.customtype) = "PieceID", (gogoproto.nullable) = false]; + repeated metainfo.AddressedOrderLimit original_order_limits = 6; + google.protobuf.Timestamp creation_date = 7 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false]; + + bytes satellite_signature = 8; +} \ No newline at end of file diff --git a/satellite/internalpb/types.go b/satellite/internalpb/types.go new file mode 100644 index 000000000..726944d3e --- /dev/null +++ b/satellite/internalpb/types.go @@ -0,0 +1,9 @@ +// Copyright (C) 2020 Storj Labs, Inc. +// See LICENSE for copying information. + +package internalpb + +import "storj.io/common/storj" + +// PieceID is an alias to storj.PieceID for use in generated protobuf code. +type PieceID = storj.PieceID diff --git a/satellite/metainfo/metainfo.go b/satellite/metainfo/metainfo.go index acc8743db..bd7c5c8ae 100644 --- a/satellite/metainfo/metainfo.go +++ b/satellite/metainfo/metainfo.go @@ -27,6 +27,7 @@ import ( "storj.io/storj/satellite/accounting" "storj.io/storj/satellite/attribution" "storj.io/storj/satellite/console" + "storj.io/storj/satellite/internalpb" "storj.io/storj/satellite/metainfo/metabase" "storj.io/storj/satellite/metainfo/objectdeletion" "storj.io/storj/satellite/metainfo/piecedeletion" @@ -665,7 +666,7 @@ func (endpoint *Endpoint) BeginObject(ctx context.Context, req *pb.ObjectBeginRe // use only satellite values for Redundancy Scheme pbRS := endpoint.redundancyScheme() - streamID, err := endpoint.packStreamID(ctx, &pb.SatStreamID{ + streamID, err := endpoint.packStreamID(ctx, &internalpb.StreamID{ Bucket: req.Bucket, EncryptedPath: req.EncryptedPath, Version: req.Version, @@ -853,7 +854,7 @@ func (endpoint *Endpoint) getObject(ctx context.Context, projectID uuid.UUID, bu return nil, rpcstatus.Error(rpcstatus.Internal, err.Error()) } - streamID, err := endpoint.packStreamID(ctx, &pb.SatStreamID{ + streamID, err := endpoint.packStreamID(ctx, &internalpb.StreamID{ Bucket: bucket, EncryptedPath: encryptedPath, Version: version, @@ -1200,7 +1201,7 @@ func (endpoint *Endpoint) BeginSegment(ctx context.Context, req *pb.SegmentBegin return nil, rpcstatus.Error(rpcstatus.Internal, err.Error()) } - segmentID, err := endpoint.packSegmentID(ctx, &pb.SatSegmentID{ + segmentID, err := endpoint.packSegmentID(ctx, &internalpb.SegmentID{ StreamId: streamID, Index: req.Position.Index, OriginalOrderLimits: addressedLimits, @@ -1522,7 +1523,7 @@ func (endpoint *Endpoint) DownloadSegment(ctx context.Context, req *pb.SegmentDo return nil, rpcstatus.Error(rpcstatus.Internal, err.Error()) } - segmentID, err := endpoint.packSegmentID(ctx, &pb.SatSegmentID{}) + segmentID, err := endpoint.packSegmentID(ctx, &internalpb.SegmentID{}) if err != nil { return nil, rpcstatus.Error(rpcstatus.Internal, err.Error()) } @@ -1656,10 +1657,10 @@ func getLimitByStorageNodeID(limits []*pb.AddressedOrderLimit, storageNodeID sto return nil } -func (endpoint *Endpoint) packStreamID(ctx context.Context, satStreamID *pb.SatStreamID) (streamID storj.StreamID, err error) { +func (endpoint *Endpoint) packStreamID(ctx context.Context, satStreamID *internalpb.StreamID) (streamID storj.StreamID, err error) { defer mon.Task()(&ctx)(&err) - signedStreamID, err := signing.SignStreamID(ctx, endpoint.satellite, satStreamID) + signedStreamID, err := SignStreamID(ctx, endpoint.satellite, satStreamID) if err != nil { return nil, rpcstatus.Error(rpcstatus.Internal, err.Error()) } @@ -1676,10 +1677,10 @@ func (endpoint *Endpoint) packStreamID(ctx context.Context, satStreamID *pb.SatS return streamID, nil } -func (endpoint *Endpoint) packSegmentID(ctx context.Context, satSegmentID *pb.SatSegmentID) (segmentID storj.SegmentID, err error) { +func (endpoint *Endpoint) packSegmentID(ctx context.Context, satSegmentID *internalpb.SegmentID) (segmentID storj.SegmentID, err error) { defer mon.Task()(&ctx)(&err) - signedSegmentID, err := signing.SignSegmentID(ctx, endpoint.satellite, satSegmentID) + signedSegmentID, err := SignSegmentID(ctx, endpoint.satellite, satSegmentID) if err != nil { return nil, err } @@ -1696,16 +1697,16 @@ func (endpoint *Endpoint) packSegmentID(ctx context.Context, satSegmentID *pb.Sa return segmentID, nil } -func (endpoint *Endpoint) unmarshalSatStreamID(ctx context.Context, streamID storj.StreamID) (_ *pb.SatStreamID, err error) { +func (endpoint *Endpoint) unmarshalSatStreamID(ctx context.Context, streamID storj.StreamID) (_ *internalpb.StreamID, err error) { defer mon.Task()(&ctx)(&err) - satStreamID := &pb.SatStreamID{} + satStreamID := &internalpb.StreamID{} err = pb.Unmarshal(streamID, satStreamID) if err != nil { return nil, err } - err = signing.VerifyStreamID(ctx, endpoint.satellite, satStreamID) + err = VerifyStreamID(ctx, endpoint.satellite, satStreamID) if err != nil { return nil, err } diff --git a/satellite/metainfo/signing.go b/satellite/metainfo/signing.go new file mode 100644 index 000000000..1cdb29968 --- /dev/null +++ b/satellite/metainfo/signing.go @@ -0,0 +1,90 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package metainfo + +import ( + "context" + + "storj.io/common/pb" + "storj.io/common/signing" + "storj.io/storj/satellite/internalpb" +) + +// SignStreamID signs the stream ID using the specified signer. +// Signer is a satellite. +func SignStreamID(ctx context.Context, signer signing.Signer, unsigned *internalpb.StreamID) (_ *internalpb.StreamID, err error) { + defer mon.Task()(&ctx)(&err) + bytes, err := EncodeStreamID(ctx, unsigned) + if err != nil { + return nil, Error.Wrap(err) + } + + signed := *unsigned + signed.SatelliteSignature, err = signer.HashAndSign(ctx, bytes) + if err != nil { + return nil, Error.Wrap(err) + } + + return &signed, nil +} + +// SignSegmentID signs the segment ID using the specified signer. +// Signer is a satellite. +func SignSegmentID(ctx context.Context, signer signing.Signer, unsigned *internalpb.SegmentID) (_ *internalpb.SegmentID, err error) { + defer mon.Task()(&ctx)(&err) + bytes, err := EncodeSegmentID(ctx, unsigned) + if err != nil { + return nil, Error.Wrap(err) + } + + signed := *unsigned + signed.SatelliteSignature, err = signer.HashAndSign(ctx, bytes) + if err != nil { + return nil, Error.Wrap(err) + } + + return &signed, nil +} + +// EncodeStreamID encodes stream ID into bytes for signing. +func EncodeStreamID(ctx context.Context, streamID *internalpb.StreamID) (_ []byte, err error) { + defer mon.Task()(&ctx)(&err) + signature := streamID.SatelliteSignature + streamID.SatelliteSignature = nil + out, err := pb.Marshal(streamID) + streamID.SatelliteSignature = signature + return out, err +} + +// EncodeSegmentID encodes segment ID into bytes for signing. +func EncodeSegmentID(ctx context.Context, segmentID *internalpb.SegmentID) (_ []byte, err error) { + defer mon.Task()(&ctx)(&err) + signature := segmentID.SatelliteSignature + segmentID.SatelliteSignature = nil + out, err := pb.Marshal(segmentID) + segmentID.SatelliteSignature = signature + return out, err +} + +// VerifyStreamID verifies that the signature inside stream ID belongs to the satellite. +func VerifyStreamID(ctx context.Context, satellite signing.Signee, signed *internalpb.StreamID) (err error) { + defer mon.Task()(&ctx)(&err) + bytes, err := EncodeStreamID(ctx, signed) + if err != nil { + return Error.Wrap(err) + } + + return satellite.HashAndVerifySignature(ctx, bytes, signed.SatelliteSignature) +} + +// VerifySegmentID verifies that the signature inside segment ID belongs to the satellite. +func VerifySegmentID(ctx context.Context, satellite signing.Signee, signed *internalpb.SegmentID) (err error) { + defer mon.Task()(&ctx)(&err) + bytes, err := EncodeSegmentID(ctx, signed) + if err != nil { + return Error.Wrap(err) + } + + return satellite.HashAndVerifySignature(ctx, bytes, signed.SatelliteSignature) +}