1c56a4cf5c
* Decoder can tolerate readers with unexpected EOF * Return EOF if required number of inbufs are at EOF * Use existing randData() to generate random data for tests * Test case for io.ErrUnexpectedEOF * Add TransformReaderSize constructor
278 lines
8.1 KiB
Go
278 lines
8.1 KiB
Go
// Copyright (C) 2018 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package eestream
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"math/rand"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/vivint/infectious"
|
|
|
|
"storj.io/storj/internal/pkg/readcloser"
|
|
)
|
|
|
|
func TestRS(t *testing.T) {
|
|
data := randData(32 * 1024)
|
|
fc, err := infectious.NewFEC(2, 4)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rs := NewRSScheme(fc, 8*1024)
|
|
readers := EncodeReader(bytes.NewReader(data), rs)
|
|
readerMap := make(map[int]io.ReadCloser, len(readers))
|
|
for i, reader := range readers {
|
|
readerMap[i] = ioutil.NopCloser(reader)
|
|
}
|
|
data2, err := ioutil.ReadAll(DecodeReaders(readerMap, rs))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Equal(data, data2) {
|
|
t.Fatalf("rs encode/decode failed")
|
|
}
|
|
}
|
|
|
|
// Check that io.ReadFull will return io.ErrUnexpectedEOF
|
|
// if DecodeReaders return less data than expected.
|
|
func TestRSUnexectedEOF(t *testing.T) {
|
|
data := randData(32 * 1024)
|
|
fc, err := infectious.NewFEC(2, 4)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rs := NewRSScheme(fc, 8*1024)
|
|
readers := EncodeReader(bytes.NewReader(data), rs)
|
|
readerMap := make(map[int]io.ReadCloser, len(readers))
|
|
for i, reader := range readers {
|
|
readerMap[i] = ioutil.NopCloser(reader)
|
|
}
|
|
// Try ReadFull more data from DecodeReaders than available
|
|
data2 := make([]byte, len(data)+1024)
|
|
_, err = io.ReadFull(DecodeReaders(readerMap, rs), data2)
|
|
assert.EqualError(t, err, io.ErrUnexpectedEOF.Error())
|
|
}
|
|
|
|
// Some pieces will read error.
|
|
// Test will pass if at least required number of pieces are still good.
|
|
func TestRSErrors(t *testing.T) {
|
|
for i, tt := range []testCase{
|
|
{4 * 1024, 1024, 1, 1, 0, false},
|
|
{4 * 1024, 1024, 1, 1, 1, true},
|
|
{4 * 1024, 1024, 1, 2, 0, false},
|
|
{4 * 1024, 1024, 1, 2, 1, false},
|
|
{4 * 1024, 1024, 1, 2, 2, true},
|
|
{4 * 1024, 1024, 2, 4, 0, false},
|
|
{4 * 1024, 1024, 2, 4, 1, false},
|
|
{4 * 1024, 1024, 2, 4, 2, false},
|
|
{4 * 1024, 1024, 2, 4, 3, true},
|
|
{4 * 1024, 1024, 2, 4, 4, true},
|
|
{6 * 1024, 1024, 3, 7, 0, false},
|
|
{6 * 1024, 1024, 3, 7, 1, false},
|
|
{6 * 1024, 1024, 3, 7, 2, false},
|
|
{6 * 1024, 1024, 3, 7, 3, false},
|
|
{6 * 1024, 1024, 3, 7, 4, false},
|
|
{6 * 1024, 1024, 3, 7, 5, true},
|
|
{6 * 1024, 1024, 3, 7, 6, true},
|
|
{6 * 1024, 1024, 3, 7, 7, true},
|
|
} {
|
|
testRSProblematic(t, tt, i, func(in []byte) io.ReadCloser {
|
|
return readcloser.FatalReadCloser(
|
|
errors.New("I am an error piece"))
|
|
})
|
|
}
|
|
}
|
|
|
|
// Some pieces will read EOF at the beginning (byte 0).
|
|
// Test will pass if those pieces are less than required.
|
|
func TestRSEOF(t *testing.T) {
|
|
for i, tt := range []testCase{
|
|
{4 * 1024, 1024, 1, 1, 0, false},
|
|
{4 * 1024, 1024, 1, 1, 1, true},
|
|
{4 * 1024, 1024, 1, 2, 0, false},
|
|
{4 * 1024, 1024, 1, 2, 1, true},
|
|
{4 * 1024, 1024, 1, 2, 2, true},
|
|
{4 * 1024, 1024, 2, 4, 0, false},
|
|
{4 * 1024, 1024, 2, 4, 1, false},
|
|
{4 * 1024, 1024, 2, 4, 2, true},
|
|
{4 * 1024, 1024, 2, 4, 3, true},
|
|
{4 * 1024, 1024, 2, 4, 4, true},
|
|
{6 * 1024, 1024, 3, 7, 0, false},
|
|
{6 * 1024, 1024, 3, 7, 1, false},
|
|
{6 * 1024, 1024, 3, 7, 2, false},
|
|
{6 * 1024, 1024, 3, 7, 3, true},
|
|
{6 * 1024, 1024, 3, 7, 4, true},
|
|
{6 * 1024, 1024, 3, 7, 5, true},
|
|
{6 * 1024, 1024, 3, 7, 6, true},
|
|
{6 * 1024, 1024, 3, 7, 7, true},
|
|
} {
|
|
testRSProblematic(t, tt, i, func(in []byte) io.ReadCloser {
|
|
return readcloser.LimitReadCloser(
|
|
ioutil.NopCloser(bytes.NewReader(in)), 0)
|
|
})
|
|
}
|
|
}
|
|
|
|
// Some pieces will read EOF earlier than expected
|
|
// Test will pass if those pieces are less than required.
|
|
func TestRSEarlyEOF(t *testing.T) {
|
|
for i, tt := range []testCase{
|
|
{4 * 1024, 1024, 1, 1, 0, false},
|
|
{4 * 1024, 1024, 1, 1, 1, true},
|
|
{4 * 1024, 1024, 1, 2, 0, false},
|
|
{4 * 1024, 1024, 1, 2, 1, true},
|
|
{4 * 1024, 1024, 1, 2, 2, true},
|
|
{4 * 1024, 1024, 2, 4, 0, false},
|
|
{4 * 1024, 1024, 2, 4, 1, false},
|
|
{4 * 1024, 1024, 2, 4, 2, true},
|
|
{4 * 1024, 1024, 2, 4, 3, true},
|
|
{4 * 1024, 1024, 2, 4, 4, true},
|
|
{6 * 1024, 1024, 3, 7, 0, false},
|
|
{6 * 1024, 1024, 3, 7, 1, false},
|
|
{6 * 1024, 1024, 3, 7, 2, false},
|
|
{6 * 1024, 1024, 3, 7, 3, true},
|
|
{6 * 1024, 1024, 3, 7, 4, true},
|
|
{6 * 1024, 1024, 3, 7, 5, true},
|
|
{6 * 1024, 1024, 3, 7, 6, true},
|
|
{6 * 1024, 1024, 3, 7, 7, true},
|
|
} {
|
|
testRSProblematic(t, tt, i, func(in []byte) io.ReadCloser {
|
|
// Read EOF after 500 bytes
|
|
return readcloser.LimitReadCloser(
|
|
ioutil.NopCloser(bytes.NewReader(in)), 500)
|
|
})
|
|
}
|
|
}
|
|
|
|
// Some pieces will read EOF later than expected.
|
|
// Test will pass if at least required number of pieces are still good.
|
|
func TestRSLateEOF(t *testing.T) {
|
|
for i, tt := range []testCase{
|
|
{4 * 1024, 1024, 1, 1, 0, false},
|
|
{4 * 1024, 1024, 1, 1, 1, true},
|
|
{4 * 1024, 1024, 1, 2, 0, false},
|
|
{4 * 1024, 1024, 1, 2, 1, false},
|
|
{4 * 1024, 1024, 1, 2, 2, true},
|
|
{4 * 1024, 1024, 2, 4, 0, false},
|
|
{4 * 1024, 1024, 2, 4, 1, false},
|
|
{4 * 1024, 1024, 2, 4, 2, false},
|
|
{4 * 1024, 1024, 2, 4, 3, true},
|
|
{4 * 1024, 1024, 2, 4, 4, true},
|
|
{6 * 1024, 1024, 3, 7, 0, false},
|
|
{6 * 1024, 1024, 3, 7, 1, false},
|
|
{6 * 1024, 1024, 3, 7, 2, false},
|
|
{6 * 1024, 1024, 3, 7, 3, false},
|
|
{6 * 1024, 1024, 3, 7, 4, false},
|
|
{6 * 1024, 1024, 3, 7, 5, true},
|
|
{6 * 1024, 1024, 3, 7, 6, true},
|
|
{6 * 1024, 1024, 3, 7, 7, true},
|
|
} {
|
|
testRSProblematic(t, tt, i, func(in []byte) io.ReadCloser {
|
|
// extend the input with random number of random bytes
|
|
random := randData(1 + rand.Intn(10000))
|
|
extended := append(in, random...)
|
|
return ioutil.NopCloser(bytes.NewReader(extended))
|
|
})
|
|
}
|
|
}
|
|
|
|
// Some pieces will read random data.
|
|
// Test will pass if there are enough good pieces for error correction.
|
|
func TestRSRandomData(t *testing.T) {
|
|
for i, tt := range []testCase{
|
|
{4 * 1024, 1024, 1, 1, 0, false},
|
|
{4 * 1024, 1024, 1, 1, 1, true},
|
|
{4 * 1024, 1024, 1, 2, 0, false},
|
|
{4 * 1024, 1024, 1, 2, 1, true},
|
|
{4 * 1024, 1024, 1, 2, 2, true},
|
|
{4 * 1024, 1024, 2, 4, 0, false},
|
|
{4 * 1024, 1024, 2, 4, 1, false},
|
|
{4 * 1024, 1024, 2, 4, 2, true},
|
|
{4 * 1024, 1024, 2, 4, 3, true},
|
|
{4 * 1024, 1024, 2, 4, 4, true},
|
|
{6 * 1024, 1024, 3, 7, 0, false},
|
|
{6 * 1024, 1024, 3, 7, 1, false},
|
|
{6 * 1024, 1024, 3, 7, 2, false},
|
|
{6 * 1024, 1024, 3, 7, 3, true},
|
|
{6 * 1024, 1024, 3, 7, 4, true},
|
|
{6 * 1024, 1024, 3, 7, 5, true},
|
|
{6 * 1024, 1024, 3, 7, 6, true},
|
|
{6 * 1024, 1024, 3, 7, 7, true},
|
|
} {
|
|
testRSProblematic(t, tt, i, func(in []byte) io.ReadCloser {
|
|
// return random data instead of expected one
|
|
return ioutil.NopCloser(bytes.NewReader(randData(len(in))))
|
|
})
|
|
}
|
|
}
|
|
|
|
type testCase struct {
|
|
dataSize int
|
|
blockSize int
|
|
required int
|
|
total int
|
|
problematic int
|
|
fail bool
|
|
}
|
|
|
|
type problematicReadCloser func([]byte) io.ReadCloser
|
|
|
|
func testRSProblematic(t *testing.T, tt testCase, i int, fn problematicReadCloser) {
|
|
errTag := fmt.Sprintf("Test case #%d", i)
|
|
data := randData(tt.dataSize)
|
|
fc, err := infectious.NewFEC(tt.required, tt.total)
|
|
if !assert.NoError(t, err, errTag) {
|
|
return
|
|
}
|
|
rs := NewRSScheme(fc, tt.blockSize)
|
|
readers := EncodeReader(bytes.NewReader(data), rs)
|
|
// read all readers in []byte buffers to avoid deadlock if later
|
|
// we don't read in parallel from all of them
|
|
pieces, err := readAll(readers)
|
|
if !assert.NoError(t, err, errTag) {
|
|
return
|
|
}
|
|
readerMap := make(map[int]io.ReadCloser, len(readers))
|
|
// some readers will return EOF later
|
|
for i := 0; i < tt.problematic; i++ {
|
|
readerMap[i] = fn(pieces[i])
|
|
}
|
|
// the rest will operate normally
|
|
for i := tt.problematic; i < tt.total; i++ {
|
|
readerMap[i] = ioutil.NopCloser(bytes.NewReader(pieces[i]))
|
|
}
|
|
data2, err := ioutil.ReadAll(DecodeReaders(readerMap, rs))
|
|
if tt.fail {
|
|
if err == nil && bytes.Compare(data, data2) == 0 {
|
|
assert.Fail(t, "expected to fail, but didn't", errTag)
|
|
}
|
|
} else if assert.NoError(t, err, errTag) {
|
|
assert.Equal(t, data, data2, errTag)
|
|
}
|
|
}
|
|
|
|
func readAll(readers []io.Reader) ([][]byte, error) {
|
|
pieces := make([][]byte, len(readers))
|
|
errs := make(chan error, len(readers))
|
|
var err error
|
|
for i := range readers {
|
|
go func(i int) {
|
|
pieces[i], err = ioutil.ReadAll(readers[i])
|
|
errs <- err
|
|
}(i)
|
|
}
|
|
for range readers {
|
|
err := <-errs
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return pieces, nil
|
|
}
|