storj/pkg/piecestore/pstore_test.go
Alexander Leitner ff8e191a9a
Added rpc server (#40)
* Added piecestore

* gofmt

* Added requested changes

* Added cli

* Removed ranger because I wanted something that can stand alone

* Add example of http server using piece store

* Changed piecestore code to make it more optial for error handelling

* Merged with piecestore

* Added missing package

* Forgot io import

* gofmt

* gofmt

* Forgot io

* Make path by hash exported

* updated to simplify again whoops

* Updated server to work real good

* Forgot ampersand

* Updated to match FilePiece

* Merged in cam's delete code

* Remove unused io

* Added RPC code

* Give the download request a reader

* Removed http server stuff; changed receive stream to say io.reader

* Added expiration date to shardInfo

* Change all instances of Shard to Piece; change protobuf name; moved client insance to outside functions

* added ttl info request

* Move scripts to http server pr; added close method for Retrieve api

* added rpc server tests for getting piece meta data and retrieval routes

* Resolved linter errors, moved to prc server to pkg, updated go.mod to use latest protobuf

* Imported cams test

* Bump gometalinter deadline

* WIP adding tests

* added tests for store and delete routes

* Add changes as requested by Kaloyan, also cleaned up some code

* Get the code actually working whoops

* More cleanup

* Separating database calls from api.go

* need to rename expiration

* Added some changes requested by JT

* Fix read size

* Fixed total amount to read

* added tests

* Simplify protobuf, add store tests, edited api to handle invalid stores properly, return errors instead of messages

* Moved rpc client and server to piece store

* Moved piecestore protobuf to the /protos folder

* Cleaned up messages

* Clean up linter errors

* Added missing sqlite import

* Add ability to do iterative reads and writes to pstore

* Incrementally read data

* Fix linter and import errors

* Solve linter Error

* Change return types

* begin test refactor

* refactored to implement only 1 db connection, moved SQLite row checking into separate function and removed defer on rows.Close(), fixed os.tempDir in rpc_test.go

* Cleaning up tests

* Added retrieve tests

* refactored delete tests

* Deleted old tests

* Updated cmd/piecestore to reflect changes to piecestore

* Refactored server tests and server/client store code

* gofmt

* WIP implementing TTL struct

* Read 4k at a time when Retrieving

* implemented ttl struct

* Accidentally removed fpiece dependency?

* Double resolve merge conflict

* Moved client to the examples since it is an example

* Change hash to id in protobuf. TODO: Change client and server to reflect these changes

* Update client and protobuf

* changed hash to id

* Handle eof properly in retrieve

* Id -> ID

* forgot to change import for client after moving

* Moved client and server main to examples

* Made changes requested by JT

* checkEntries is now private, created currentTime variable for checkEntries, added defer rows.Close()

* Print not fatal

* Handle overflow when reading from server

* added const IDLength

* removed types from comments

* Add reader/writer for download data from and uploading data to server

* goimports and comments

* fixed nits, casts, added OK const, DBCleanup now exits program on error

* Add stream reader and writer for server

* Fix errors

* i beforee except after c lol

* customizable data dir

* Forgot to rename variable

* customizable data dir

* Handle closing of server stream properly

* linter

* pass on inserting the same data twice

* linter

* linter

* Do the easy things JT asked for

* Handle overflow from reads properly; handle custom db path

* Handle overflow for server stream read; TODO Combine server and client stream reads

* Allow for TTL of 0 to stay forever

* Make Client cleaner for user
2018-06-02 14:14:59 -04:00

312 lines
6.8 KiB
Go

// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package pstore
import (
"os"
"path"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestStore(t *testing.T) {
tests := []struct{
it string
id string
size int64
offset int64
content []byte
expectedContent []byte
err string
} {
{
it: "should successfully store data",
id: "0123456789ABCDEFGHIJ",
size: 5,
offset: 0,
content: []byte("butts"),
expectedContent: []byte("butts"),
err: "",
},
{
it: "should successfully store data by offset",
id: "0123456789ABCDEFGHIJ",
size: 5,
offset: 5,
content: []byte("butts"),
expectedContent: []byte("butts"),
err: "",
},
{
it: "should successfully store data by chunk",
id: "0123456789ABCDEFGHIJ",
size: 2,
offset: 3,
content: []byte("butts"),
expectedContent: []byte("bu"),
err: "",
},
{
it: "should return an error when given an invalid id",
id: "012",
size: 5,
offset: 0,
content: []byte("butts"),
expectedContent: []byte("butts"),
err: "argError: Invalid id length",
},
{
it: "should return an error when given negative offset",
id: "0123456789ABCDEFGHIJ",
size: 5,
offset: -1,
content: []byte("butts"),
expectedContent: []byte(""),
err: "argError: Offset is less than 0. Must be greater than or equal to 0",
},
{
it: "should return an error when given negative length",
id: "0123456789ABCDEFGHIJ",
size: -1,
offset: 0,
content: []byte("butts"),
expectedContent: []byte(""),
err: "Invalid Length",
},
}
for _, tt := range tests {
t.Run(tt.it, func(t *testing.T) {
assert := assert.New(t)
storeFile, err := StoreWriter(tt.id, tt.size, tt.offset, os.TempDir())
if tt.err != "" {
if assert.NotNil(err) {
assert.Equal(tt.err, err.Error())
}
return
} else if err != nil {
t.Errorf("Error: %s", err.Error())
return
}
// Write chunk received to disk
_, err = storeFile.Write(tt.content)
assert.Nil(err)
storeFile.Close()
folder1 := string(tt.id[0:2])
folder2 := string(tt.id[2:4])
fileName := string(tt.id[4:])
createdFilePath := path.Join(os.TempDir(), folder1, folder2, fileName)
createdFile, err := os.Open(createdFilePath)
if err != nil {
t.Errorf("Error: %s opening created file %s", err.Error(), createdFilePath)
return
}
buffer := make([]byte, tt.size)
createdFile.Seek(tt.offset, 0)
_, _ = createdFile.Read(buffer)
createdFile.Close()
os.RemoveAll(path.Join(os.TempDir(), folder1))
if string(buffer) != string(tt.expectedContent) {
t.Errorf("Expected data butts does not equal Actual data %s", string(buffer))
return
}
})
}
}
func TestRetrieve(t *testing.T) {
tests := []struct{
it string
id string
size int64
offset int64
content []byte
expectedContent []byte
err string
} {
{
it: "should successfully retrieve data",
id: "0123456789ABCDEFGHIJ",
size: 5,
offset: 0,
content: []byte("butts"),
expectedContent: []byte("butts"),
err: "",
},
{
it: "should successfully retrieve data by offset",
id: "0123456789ABCDEFGHIJ",
size: 5,
offset: 5,
content: []byte("butts"),
expectedContent: []byte("butts"),
err: "",
},
{
it: "should successfully retrieve data by chunk",
id: "0123456789ABCDEFGHIJ",
size: 2,
offset: 5,
content: []byte("bu"),
expectedContent: []byte("bu"),
err: "",
},
{
it: "should return an error when given negative offset",
id: "0123456789ABCDEFGHIJ",
size: 0,
offset: -1337,
content: []byte("butts"),
expectedContent: []byte(""),
err: "argError: Invalid offset: -1337",
},
{
it: "should successfully retrieve data with negative length",
id: "0123456789ABCDEFGHIJ",
size: -1,
offset: 0,
content: []byte("butts"),
expectedContent: []byte("butts"),
err: "",
},
}
for _, tt := range tests {
t.Run(tt.it, func(t *testing.T) {
assert := assert.New(t)
folder1 := string(tt.id[0:2])
folder2 := string(tt.id[2:4])
fileName := string(tt.id[4:])
createdFilePath := path.Join(os.TempDir(), folder1, folder2, fileName)
if err := os.MkdirAll(filepath.Dir(createdFilePath), 0700); err != nil {
t.Errorf("Error: %s when creating dir", err.Error())
return
}
createdFile, err := os.OpenFile(createdFilePath, os.O_RDWR|os.O_CREATE, 0755)
if err != nil {
t.Errorf("Error: %s opening created file %s", err.Error(), createdFilePath)
return
}
createdFile.Seek(tt.offset, 0)
_, err = createdFile.Write(tt.content)
if err != nil {
t.Errorf("Error: %s writing to created file", err.Error())
return
}
createdFile.Close()
storeFile, err := RetrieveReader(tt.id, tt.size, tt.offset, os.TempDir())
if tt.err != "" {
if assert.NotNil(err) {
assert.Equal(tt.err, err.Error())
}
return
} else if err != nil {
t.Errorf("Error: %s", err.Error())
return
}
size := tt.size
if tt.size < 0 {
size = int64(len(tt.content))
}
buffer := make([]byte, size)
storeFile.Read(buffer)
storeFile.Close()
os.RemoveAll(path.Join(os.TempDir(), folder1))
if string(buffer) != string(tt.expectedContent) {
t.Errorf("Expected data butts does not equal Actual data %s", string(buffer))
return
}
})
}
}
func TestDelete(t *testing.T) {
tests := []struct{
it string
id string
err string
} {
{
it: "should successfully delete data",
id: "11111111111111111111",
err: "",
},
{
it: "should return nil-err with non-existent id",
id: "11111111111111111111",
err: "",
},
{
it: "should err with invalid id length",
id: "111111",
err: "argError: Invalid id length",
},
}
for _, tt := range tests {
t.Run(tt.it, func(t *testing.T) {
assert := assert.New(t)
folder1 := string(tt.id[0:2])
folder2 := string(tt.id[2:4])
fileName := string(tt.id[4:])
createdFilePath := path.Join(os.TempDir(), folder1, folder2, fileName)
if err := os.MkdirAll(filepath.Dir(createdFilePath), 0700); err != nil {
t.Errorf("Error: %s when creating dir", err.Error())
return
}
createdFile, err := os.OpenFile(createdFilePath, os.O_RDWR|os.O_CREATE, 0755)
if err != nil {
t.Errorf("Error: %s opening created file %s", err.Error(), createdFilePath)
return
}
createdFile.Close()
err = Delete(tt.id, os.TempDir())
if tt.err != "" {
if assert.NotNil(err) {
assert.Equal(tt.err, err.Error())
}
return
} else if err != nil {
t.Errorf("Error: %s", err.Error())
return
}
if _, err = os.Stat(createdFilePath); os.IsExist(err) {
t.Errorf("Error deleting file")
return
}
return
})
}
}
func TestMain(m *testing.M) {
m.Run()
}