From 6fc009f6e447e16a2096a0308bc37efd0c201d47 Mon Sep 17 00:00:00 2001 From: Egon Elbre Date: Thu, 19 Dec 2019 17:37:59 +0200 Subject: [PATCH] uplink/eestream: move Pad to encryption package to break dependency to eestream Change-Id: I0c9bc3c65f161d79812196ac8285405e6be04c9e --- examples/eestream/serve-collected/main.go | 2 +- examples/eestream/serve/main.go | 2 +- examples/eestream/store/main.go | 2 +- pkg/encryption/encryption_test.go | 3 +-- {uplink/eestream => pkg/encryption}/pad.go | 7 +------ {uplink/eestream => pkg/encryption}/pad_test.go | 2 +- uplink/ecclient/client.go | 5 +++-- uplink/eestream/encode.go | 1 + uplink/eestream/rs_test.go | 7 ++++--- uplink/storage/streams/store.go | 5 ++--- 10 files changed, 16 insertions(+), 20 deletions(-) rename {uplink/eestream => pkg/encryption}/pad.go (97%) rename {uplink/eestream => pkg/encryption}/pad_test.go (98%) diff --git a/examples/eestream/serve-collected/main.go b/examples/eestream/serve-collected/main.go index 227a5cf5c..35a1ade57 100644 --- a/examples/eestream/serve-collected/main.go +++ b/examples/eestream/serve-collected/main.go @@ -83,7 +83,7 @@ func Main() error { if err != nil { return err } - rr, err = eestream.UnpadSlow(ctx, rr) + rr, err = encryption.UnpadSlow(ctx, rr) if err != nil { return err } diff --git a/examples/eestream/serve/main.go b/examples/eestream/serve/main.go index 7b6b6f1e8..6d53e3f27 100644 --- a/examples/eestream/serve/main.go +++ b/examples/eestream/serve/main.go @@ -84,7 +84,7 @@ func Main() error { return err } ctx := context.Background() - rr, err = eestream.UnpadSlow(ctx, rr) + rr, err = encryption.UnpadSlow(ctx, rr) if err != nil { return err } diff --git a/examples/eestream/store/main.go b/examples/eestream/store/main.go index 3025df103..d4ba60721 100644 --- a/examples/eestream/store/main.go +++ b/examples/eestream/store/main.go @@ -62,7 +62,7 @@ func Main() error { return err } readers, err := eestream.EncodeReader(context.Background(), zap.L(), - encryption.TransformReader(eestream.PadReader(os.Stdin, + encryption.TransformReader(encryption.PadReader(os.Stdin, encrypter.InBlockSize()), encrypter, 0), rs) if err != nil { return err diff --git a/pkg/encryption/encryption_test.go b/pkg/encryption/encryption_test.go index b0d282249..3cca3614e 100644 --- a/pkg/encryption/encryption_test.go +++ b/pkg/encryption/encryption_test.go @@ -17,7 +17,6 @@ import ( "storj.io/storj/private/memory" "storj.io/storj/private/testcontext" "storj.io/storj/private/testrand" - "storj.io/storj/uplink/eestream" ) const ( @@ -49,7 +48,7 @@ func TestCalcEncryptedSize(t *testing.T) { require.NoError(t, err, errTag) randReader := ioutil.NopCloser(io.LimitReader(testrand.Reader(), dataSize)) - reader := encryption.TransformReader(eestream.PadReader(randReader, encrypter.InBlockSize()), encrypter, 0) + reader := encryption.TransformReader(encryption.PadReader(randReader, encrypter.InBlockSize()), encrypter, 0) cipherData, err := ioutil.ReadAll(reader) assert.NoError(t, err, errTag) diff --git a/uplink/eestream/pad.go b/pkg/encryption/pad.go similarity index 97% rename from uplink/eestream/pad.go rename to pkg/encryption/pad.go index 1dc4d68b9..333663211 100644 --- a/uplink/eestream/pad.go +++ b/pkg/encryption/pad.go @@ -1,7 +1,7 @@ // Copyright (C) 2019 Storj Labs, Inc. // See LICENSE for copying information. -package eestream +package encryption import ( "bytes" @@ -14,10 +14,6 @@ import ( "storj.io/storj/private/readcloser" ) -const ( - uint32Size = 4 -) - // makePadding calculates how many bytes of padding are needed to fill // an encryption block then creates a slice of zero bytes that size. // The last byte of the padding slice contains the count of the total padding bytes added. @@ -52,7 +48,6 @@ func Unpad(data ranger.Ranger, padding int) (ranger.Ranger, error) { // UnpadSlow is like Unpad, but does not require the amount of padding. // UnpadSlow will have to do extra work to make up for this missing information. func UnpadSlow(ctx context.Context, data ranger.Ranger) (_ ranger.Ranger, err error) { - defer mon.Task()(&ctx)(&err) r, err := data.Range(ctx, data.Size()-uint32Size, uint32Size) if err != nil { return nil, Error.Wrap(err) diff --git a/uplink/eestream/pad_test.go b/pkg/encryption/pad_test.go similarity index 98% rename from uplink/eestream/pad_test.go rename to pkg/encryption/pad_test.go index 06a46b581..c86efa101 100644 --- a/uplink/eestream/pad_test.go +++ b/pkg/encryption/pad_test.go @@ -1,7 +1,7 @@ // Copyright (C) 2019 Storj Labs, Inc. // See LICENSE for copying information. -package eestream +package encryption import ( "bytes" diff --git a/uplink/ecclient/client.go b/uplink/ecclient/client.go index 4f21f9825..5d129edbd 100644 --- a/uplink/ecclient/client.go +++ b/uplink/ecclient/client.go @@ -15,6 +15,7 @@ import ( "go.uber.org/zap" monkit "gopkg.in/spacemonkeygo/monkit.v2" + "storj.io/storj/pkg/encryption" "storj.io/storj/pkg/identity" "storj.io/storj/pkg/pb" "storj.io/storj/pkg/ranger" @@ -91,7 +92,7 @@ func (ec *ecClient) Put(ctx context.Context, limits []*pb.AddressedOrderLimit, p zap.Int("Optimal Threshold", rs.OptimalThreshold()), ) - padded := eestream.PadReader(ioutil.NopCloser(data), rs.StripeSize()) + padded := encryption.PadReader(ioutil.NopCloser(data), rs.StripeSize()) readers, err := eestream.EncodeReader(ctx, ec.log, padded, rs) if err != nil { return nil, nil, err @@ -299,7 +300,7 @@ func (ec *ecClient) Get(ctx context.Context, limits []*pb.AddressedOrderLimit, p return nil, Error.Wrap(err) } - ranger, err := eestream.Unpad(rr, int(paddedSize-size)) + ranger, err := encryption.Unpad(rr, int(paddedSize-size)) return ranger, Error.Wrap(err) } diff --git a/uplink/eestream/encode.go b/uplink/eestream/encode.go index 771064553..0a66aae39 100644 --- a/uplink/eestream/encode.go +++ b/uplink/eestream/encode.go @@ -307,6 +307,7 @@ func (er *EncodedRanger) Range(ctx context.Context, offset, length int64) (_ []i // CalcPieceSize calculates what would be the piece size of the encoded data // after erasure coding data with dataSize using the given ErasureScheme. func CalcPieceSize(dataSize int64, scheme ErasureScheme) int64 { + const uint32Size = 4 stripeSize := int64(scheme.StripeSize()) stripes := (dataSize + uint32Size + stripeSize - 1) / stripeSize diff --git a/uplink/eestream/rs_test.go b/uplink/eestream/rs_test.go index 9db72ef6e..a89c749d3 100644 --- a/uplink/eestream/rs_test.go +++ b/uplink/eestream/rs_test.go @@ -111,7 +111,7 @@ func TestRSRanger(t *testing.T) { if err != nil { t.Fatal(err) } - readers, err := EncodeReader(ctx, zaptest.NewLogger(t), encryption.TransformReader(PadReader(ioutil.NopCloser( + readers, err := EncodeReader(ctx, zaptest.NewLogger(t), encryption.TransformReader(encryption.PadReader(ioutil.NopCloser( bytes.NewReader(data)), encrypter.InBlockSize()), encrypter, 0), rs) if err != nil { t.Fatal(err) @@ -136,7 +136,7 @@ func TestRSRanger(t *testing.T) { if err != nil { t.Fatal(err) } - rr, err = UnpadSlow(ctx, rr) + rr, err = encryption.UnpadSlow(ctx, rr) if err != nil { t.Fatal(err) } @@ -645,6 +645,7 @@ func BenchmarkReedSolomonErasureScheme(b *testing.B) { } func TestCalcPieceSize(t *testing.T) { + const uint32Size = 4 ctx := testcontext.New(t) defer ctx.Cleanup() @@ -668,7 +669,7 @@ func TestCalcPieceSize(t *testing.T) { calculatedSize := CalcPieceSize(dataSize, es) randReader := ioutil.NopCloser(io.LimitReader(testrand.Reader(), dataSize)) - readers, err := EncodeReader(ctx, zaptest.NewLogger(t), PadReader(randReader, es.StripeSize()), rs) + readers, err := EncodeReader(ctx, zaptest.NewLogger(t), encryption.PadReader(randReader, es.StripeSize()), rs) require.NoError(t, err, errTag) for _, reader := range readers { diff --git a/uplink/storage/streams/store.go b/uplink/storage/streams/store.go index 03a8e1c48..a8422084c 100644 --- a/uplink/storage/streams/store.go +++ b/uplink/storage/streams/store.go @@ -21,7 +21,6 @@ import ( "storj.io/storj/pkg/pb" "storj.io/storj/pkg/ranger" "storj.io/storj/pkg/storj" - "storj.io/storj/uplink/eestream" "storj.io/storj/uplink/metainfo" "storj.io/storj/uplink/storage/segments" ) @@ -219,7 +218,7 @@ func (s *streamStore) upload(ctx context.Context, path Path, pathCipher storj.Ci return Meta{}, err } - paddedReader := eestream.PadReader(ioutil.NopCloser(peekReader), encrypter.InBlockSize()) + paddedReader := encryption.PadReader(ioutil.NopCloser(peekReader), encrypter.InBlockSize()) transformedReader := encryption.TransformReader(paddedReader, encrypter, 0) beginSegment := &metainfo.BeginSegmentParams{ @@ -701,7 +700,7 @@ func decryptRanger(ctx context.Context, rr ranger.Ranger, decryptedSize int64, c if err != nil { return nil, err } - return eestream.Unpad(rd, int(rd.Size()-decryptedSize)) + return encryption.Unpad(rd, int(rd.Size()-decryptedSize)) } // CancelHandler handles clean up of segments on receiving CTRL+C