JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

uplink/eestream: move Pad to encryption package to break dependency to eestream

Browse Source 
Change-Id: I0c9bc3c65f161d79812196ac8285405e6be04c9e
This commit is contained in:
Egon Elbre 2019-12-19 17:37:59 +02:00
parent b959ccbae6
commit 6fc009f6e4
10 changed files with 16 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/eestream/serve-collected/main.go
View File

@ -83,7 +83,7 @@ func Main() error {
if err != nil { if err != nil {
return err return err
} }
rr, err = eestream.UnpadSlow(ctx, rr) rr, err = encryption.UnpadSlow(ctx, rr)
if err != nil { if err != nil {
return err return err
} }

2
examples/eestream/serve/main.go
View File

@ -84,7 +84,7 @@ func Main() error {
return err return err
} }
ctx := context.Background() ctx := context.Background()
rr, err = eestream.UnpadSlow(ctx, rr) rr, err = encryption.UnpadSlow(ctx, rr)
if err != nil { if err != nil {
return err return err
} }

2
examples/eestream/store/main.go
View File

@ -62,7 +62,7 @@ func Main() error {
return err return err
} }
readers, err := eestream.EncodeReader(context.Background(), zap.L(), readers, err := eestream.EncodeReader(context.Background(), zap.L(),
encryption.TransformReader(eestream.PadReader(os.Stdin, encryption.TransformReader(encryption.PadReader(os.Stdin,
encrypter.InBlockSize()), encrypter, 0), rs) encrypter.InBlockSize()), encrypter, 0), rs)
if err != nil { if err != nil {
return err return err

3
pkg/encryption/encryption_test.go
View File

@ -17,7 +17,6 @@ import (
"storj.io/storj/private/memory" "storj.io/storj/private/memory"
"storj.io/storj/private/testcontext" "storj.io/storj/private/testcontext"
"storj.io/storj/private/testrand" "storj.io/storj/private/testrand"
"storj.io/storj/uplink/eestream"
) )
const ( const (
@ -49,7 +48,7 @@ func TestCalcEncryptedSize(t *testing.T) {
require.NoError(t, err, errTag) require.NoError(t, err, errTag)
randReader := ioutil.NopCloser(io.LimitReader(testrand.Reader(), dataSize)) randReader := ioutil.NopCloser(io.LimitReader(testrand.Reader(), dataSize))
reader := encryption.TransformReader(eestream.PadReader(randReader, encrypter.InBlockSize()), encrypter, 0) reader := encryption.TransformReader(encryption.PadReader(randReader, encrypter.InBlockSize()), encrypter, 0)
cipherData, err := ioutil.ReadAll(reader) cipherData, err := ioutil.ReadAll(reader)
assert.NoError(t, err, errTag) assert.NoError(t, err, errTag)

7
uplink/eestream/pad.go → pkg/encryption/pad.go
View File

@ -1,7 +1,7 @@
// Copyright (C) 2019 Storj Labs, Inc. // Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information. // See LICENSE for copying information.
package eestream package encryption
import ( import (
"bytes" "bytes"
@ -14,10 +14,6 @@ import (
"storj.io/storj/private/readcloser" "storj.io/storj/private/readcloser"
) )
const (
uint32Size = 4
)
// makePadding calculates how many bytes of padding are needed to fill // makePadding calculates how many bytes of padding are needed to fill
// an encryption block then creates a slice of zero bytes that size. // an encryption block then creates a slice of zero bytes that size.
// The last byte of the padding slice contains the count of the total padding bytes added. // The last byte of the padding slice contains the count of the total padding bytes added.
@ -52,7 +48,6 @@ func Unpad(data ranger.Ranger, padding int) (ranger.Ranger, error) {
// UnpadSlow is like Unpad, but does not require the amount of padding. // UnpadSlow is like Unpad, but does not require the amount of padding.
// UnpadSlow will have to do extra work to make up for this missing information. // UnpadSlow will have to do extra work to make up for this missing information.
func UnpadSlow(ctx context.Context, data ranger.Ranger) (_ ranger.Ranger, err error) { func UnpadSlow(ctx context.Context, data ranger.Ranger) (_ ranger.Ranger, err error) {
defer mon.Task()(&ctx)(&err)
r, err := data.Range(ctx, data.Size()-uint32Size, uint32Size) r, err := data.Range(ctx, data.Size()-uint32Size, uint32Size)
if err != nil { if err != nil {
return nil, Error.Wrap(err) return nil, Error.Wrap(err)

2
uplink/eestream/pad_test.go → pkg/encryption/pad_test.go
View File

@ -1,7 +1,7 @@
// Copyright (C) 2019 Storj Labs, Inc. // Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information. // See LICENSE for copying information.
package eestream package encryption
import ( import (
"bytes" "bytes"

5
uplink/ecclient/client.go
View File

@ -15,6 +15,7 @@ import (
"go.uber.org/zap" "go.uber.org/zap"
monkit "gopkg.in/spacemonkeygo/monkit.v2" monkit "gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/encryption"
"storj.io/storj/pkg/identity" "storj.io/storj/pkg/identity"
"storj.io/storj/pkg/pb" "storj.io/storj/pkg/pb"
"storj.io/storj/pkg/ranger" "storj.io/storj/pkg/ranger"
@ -91,7 +92,7 @@ func (ec *ecClient) Put(ctx context.Context, limits []*pb.AddressedOrderLimit, p
zap.Int("Optimal Threshold", rs.OptimalThreshold()), zap.Int("Optimal Threshold", rs.OptimalThreshold()),
) )
padded := eestream.PadReader(ioutil.NopCloser(data), rs.StripeSize()) padded := encryption.PadReader(ioutil.NopCloser(data), rs.StripeSize())
readers, err := eestream.EncodeReader(ctx, ec.log, padded, rs) readers, err := eestream.EncodeReader(ctx, ec.log, padded, rs)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
@ -299,7 +300,7 @@ func (ec *ecClient) Get(ctx context.Context, limits []*pb.AddressedOrderLimit, p
return nil, Error.Wrap(err) return nil, Error.Wrap(err)
} }
ranger, err := eestream.Unpad(rr, int(paddedSize-size)) ranger, err := encryption.Unpad(rr, int(paddedSize-size))
return ranger, Error.Wrap(err) return ranger, Error.Wrap(err)
} }

1
uplink/eestream/encode.go
View File

@ -307,6 +307,7 @@ func (er *EncodedRanger) Range(ctx context.Context, offset, length int64) (_ []i
// CalcPieceSize calculates what would be the piece size of the encoded data // CalcPieceSize calculates what would be the piece size of the encoded data
// after erasure coding data with dataSize using the given ErasureScheme. // after erasure coding data with dataSize using the given ErasureScheme.
func CalcPieceSize(dataSize int64, scheme ErasureScheme) int64 { func CalcPieceSize(dataSize int64, scheme ErasureScheme) int64 {
const uint32Size = 4
stripeSize := int64(scheme.StripeSize()) stripeSize := int64(scheme.StripeSize())
stripes := (dataSize + uint32Size + stripeSize - 1) / stripeSize stripes := (dataSize + uint32Size + stripeSize - 1) / stripeSize

7
uplink/eestream/rs_test.go
View File

@ -111,7 +111,7 @@ func TestRSRanger(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
readers, err := EncodeReader(ctx, zaptest.NewLogger(t), encryption.TransformReader(PadReader(ioutil.NopCloser( readers, err := EncodeReader(ctx, zaptest.NewLogger(t), encryption.TransformReader(encryption.PadReader(ioutil.NopCloser(
bytes.NewReader(data)), encrypter.InBlockSize()), encrypter, 0), rs) bytes.NewReader(data)), encrypter.InBlockSize()), encrypter, 0), rs)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -136,7 +136,7 @@ func TestRSRanger(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
rr, err = UnpadSlow(ctx, rr) rr, err = encryption.UnpadSlow(ctx, rr)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -645,6 +645,7 @@ func BenchmarkReedSolomonErasureScheme(b *testing.B) {
} }
func TestCalcPieceSize(t *testing.T) { func TestCalcPieceSize(t *testing.T) {
const uint32Size = 4
ctx := testcontext.New(t) ctx := testcontext.New(t)
defer ctx.Cleanup() defer ctx.Cleanup()
@ -668,7 +669,7 @@ func TestCalcPieceSize(t *testing.T) {
calculatedSize := CalcPieceSize(dataSize, es) calculatedSize := CalcPieceSize(dataSize, es)
randReader := ioutil.NopCloser(io.LimitReader(testrand.Reader(), dataSize)) randReader := ioutil.NopCloser(io.LimitReader(testrand.Reader(), dataSize))
readers, err := EncodeReader(ctx, zaptest.NewLogger(t), PadReader(randReader, es.StripeSize()), rs) readers, err := EncodeReader(ctx, zaptest.NewLogger(t), encryption.PadReader(randReader, es.StripeSize()), rs)
require.NoError(t, err, errTag) require.NoError(t, err, errTag)
for _, reader := range readers { for _, reader := range readers {

5
uplink/storage/streams/store.go
View File

@ -21,7 +21,6 @@ import (
"storj.io/storj/pkg/pb" "storj.io/storj/pkg/pb"
"storj.io/storj/pkg/ranger" "storj.io/storj/pkg/ranger"
"storj.io/storj/pkg/storj" "storj.io/storj/pkg/storj"
"storj.io/storj/uplink/eestream"
"storj.io/storj/uplink/metainfo" "storj.io/storj/uplink/metainfo"
"storj.io/storj/uplink/storage/segments" "storj.io/storj/uplink/storage/segments"
) )
@ -219,7 +218,7 @@ func (s *streamStore) upload(ctx context.Context, path Path, pathCipher storj.Ci
return Meta{}, err return Meta{}, err
} }
paddedReader := eestream.PadReader(ioutil.NopCloser(peekReader), encrypter.InBlockSize()) paddedReader := encryption.PadReader(ioutil.NopCloser(peekReader), encrypter.InBlockSize())
transformedReader := encryption.TransformReader(paddedReader, encrypter, 0) transformedReader := encryption.TransformReader(paddedReader, encrypter, 0)
beginSegment := &metainfo.BeginSegmentParams{ beginSegment := &metainfo.BeginSegmentParams{
@ -701,7 +700,7 @@ func decryptRanger(ctx context.Context, rr ranger.Ranger, decryptedSize int64, c
if err != nil { if err != nil {
return nil, err return nil, err
} }
return eestream.Unpad(rd, int(rd.Size()-decryptedSize)) return encryption.Unpad(rd, int(rd.Size()-decryptedSize))
} }
// CancelHandler handles clean up of segments on receiving CTRL+C // CancelHandler handles clean up of segments on receiving CTRL+C