ff8e191a9a
* Added piecestore * gofmt * Added requested changes * Added cli * Removed ranger because I wanted something that can stand alone * Add example of http server using piece store * Changed piecestore code to make it more optial for error handelling * Merged with piecestore * Added missing package * Forgot io import * gofmt * gofmt * Forgot io * Make path by hash exported * updated to simplify again whoops * Updated server to work real good * Forgot ampersand * Updated to match FilePiece * Merged in cam's delete code * Remove unused io * Added RPC code * Give the download request a reader * Removed http server stuff; changed receive stream to say io.reader * Added expiration date to shardInfo * Change all instances of Shard to Piece; change protobuf name; moved client insance to outside functions * added ttl info request * Move scripts to http server pr; added close method for Retrieve api * added rpc server tests for getting piece meta data and retrieval routes * Resolved linter errors, moved to prc server to pkg, updated go.mod to use latest protobuf * Imported cams test * Bump gometalinter deadline * WIP adding tests * added tests for store and delete routes * Add changes as requested by Kaloyan, also cleaned up some code * Get the code actually working whoops * More cleanup * Separating database calls from api.go * need to rename expiration * Added some changes requested by JT * Fix read size * Fixed total amount to read * added tests * Simplify protobuf, add store tests, edited api to handle invalid stores properly, return errors instead of messages * Moved rpc client and server to piece store * Moved piecestore protobuf to the /protos folder * Cleaned up messages * Clean up linter errors * Added missing sqlite import * Add ability to do iterative reads and writes to pstore * Incrementally read data * Fix linter and import errors * Solve linter Error * Change return types * begin test refactor * refactored to implement only 1 db connection, moved SQLite row checking into separate function and removed defer on rows.Close(), fixed os.tempDir in rpc_test.go * Cleaning up tests * Added retrieve tests * refactored delete tests * Deleted old tests * Updated cmd/piecestore to reflect changes to piecestore * Refactored server tests and server/client store code * gofmt * WIP implementing TTL struct * Read 4k at a time when Retrieving * implemented ttl struct * Accidentally removed fpiece dependency? * Double resolve merge conflict * Moved client to the examples since it is an example * Change hash to id in protobuf. TODO: Change client and server to reflect these changes * Update client and protobuf * changed hash to id * Handle eof properly in retrieve * Id -> ID * forgot to change import for client after moving * Moved client and server main to examples * Made changes requested by JT * checkEntries is now private, created currentTime variable for checkEntries, added defer rows.Close() * Print not fatal * Handle overflow when reading from server * added const IDLength * removed types from comments * Add reader/writer for download data from and uploading data to server * goimports and comments * fixed nits, casts, added OK const, DBCleanup now exits program on error * Add stream reader and writer for server * Fix errors * i beforee except after c lol * customizable data dir * Forgot to rename variable * customizable data dir * Handle closing of server stream properly * linter * pass on inserting the same data twice * linter * linter * Do the easy things JT asked for * Handle overflow from reads properly; handle custom db path * Handle overflow for server stream read; TODO Combine server and client stream reads * Allow for TTL of 0 to stay forever * Make Client cleaner for user
312 lines
6.8 KiB
Go
312 lines
6.8 KiB
Go
// Copyright (C) 2018 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package pstore
|
|
|
|
import (
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
func TestStore(t *testing.T) {
|
|
tests := []struct{
|
|
it string
|
|
id string
|
|
size int64
|
|
offset int64
|
|
content []byte
|
|
expectedContent []byte
|
|
err string
|
|
} {
|
|
{
|
|
it: "should successfully store data",
|
|
id: "0123456789ABCDEFGHIJ",
|
|
size: 5,
|
|
offset: 0,
|
|
content: []byte("butts"),
|
|
expectedContent: []byte("butts"),
|
|
err: "",
|
|
},
|
|
{
|
|
it: "should successfully store data by offset",
|
|
id: "0123456789ABCDEFGHIJ",
|
|
size: 5,
|
|
offset: 5,
|
|
content: []byte("butts"),
|
|
expectedContent: []byte("butts"),
|
|
err: "",
|
|
},
|
|
{
|
|
it: "should successfully store data by chunk",
|
|
id: "0123456789ABCDEFGHIJ",
|
|
size: 2,
|
|
offset: 3,
|
|
content: []byte("butts"),
|
|
expectedContent: []byte("bu"),
|
|
err: "",
|
|
},
|
|
{
|
|
it: "should return an error when given an invalid id",
|
|
id: "012",
|
|
size: 5,
|
|
offset: 0,
|
|
content: []byte("butts"),
|
|
expectedContent: []byte("butts"),
|
|
err: "argError: Invalid id length",
|
|
},
|
|
{
|
|
it: "should return an error when given negative offset",
|
|
id: "0123456789ABCDEFGHIJ",
|
|
size: 5,
|
|
offset: -1,
|
|
content: []byte("butts"),
|
|
expectedContent: []byte(""),
|
|
err: "argError: Offset is less than 0. Must be greater than or equal to 0",
|
|
},
|
|
{
|
|
it: "should return an error when given negative length",
|
|
id: "0123456789ABCDEFGHIJ",
|
|
size: -1,
|
|
offset: 0,
|
|
content: []byte("butts"),
|
|
expectedContent: []byte(""),
|
|
err: "Invalid Length",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.it, func(t *testing.T) {
|
|
assert := assert.New(t)
|
|
storeFile, err := StoreWriter(tt.id, tt.size, tt.offset, os.TempDir())
|
|
if tt.err != "" {
|
|
if assert.NotNil(err) {
|
|
assert.Equal(tt.err, err.Error())
|
|
}
|
|
return
|
|
} else if err != nil {
|
|
t.Errorf("Error: %s", err.Error())
|
|
return
|
|
}
|
|
|
|
// Write chunk received to disk
|
|
_, err = storeFile.Write(tt.content)
|
|
assert.Nil(err)
|
|
|
|
storeFile.Close()
|
|
|
|
folder1 := string(tt.id[0:2])
|
|
folder2 := string(tt.id[2:4])
|
|
fileName := string(tt.id[4:])
|
|
|
|
createdFilePath := path.Join(os.TempDir(), folder1, folder2, fileName)
|
|
|
|
createdFile, err := os.Open(createdFilePath)
|
|
if err != nil {
|
|
t.Errorf("Error: %s opening created file %s", err.Error(), createdFilePath)
|
|
return
|
|
}
|
|
|
|
buffer := make([]byte, tt.size)
|
|
createdFile.Seek(tt.offset, 0)
|
|
_, _ = createdFile.Read(buffer)
|
|
|
|
createdFile.Close()
|
|
os.RemoveAll(path.Join(os.TempDir(), folder1))
|
|
|
|
if string(buffer) != string(tt.expectedContent) {
|
|
t.Errorf("Expected data butts does not equal Actual data %s", string(buffer))
|
|
return
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRetrieve(t *testing.T) {
|
|
tests := []struct{
|
|
it string
|
|
id string
|
|
size int64
|
|
offset int64
|
|
content []byte
|
|
expectedContent []byte
|
|
err string
|
|
} {
|
|
{
|
|
it: "should successfully retrieve data",
|
|
id: "0123456789ABCDEFGHIJ",
|
|
size: 5,
|
|
offset: 0,
|
|
content: []byte("butts"),
|
|
expectedContent: []byte("butts"),
|
|
err: "",
|
|
},
|
|
{
|
|
it: "should successfully retrieve data by offset",
|
|
id: "0123456789ABCDEFGHIJ",
|
|
size: 5,
|
|
offset: 5,
|
|
content: []byte("butts"),
|
|
expectedContent: []byte("butts"),
|
|
err: "",
|
|
},
|
|
{
|
|
it: "should successfully retrieve data by chunk",
|
|
id: "0123456789ABCDEFGHIJ",
|
|
size: 2,
|
|
offset: 5,
|
|
content: []byte("bu"),
|
|
expectedContent: []byte("bu"),
|
|
err: "",
|
|
},
|
|
{
|
|
it: "should return an error when given negative offset",
|
|
id: "0123456789ABCDEFGHIJ",
|
|
size: 0,
|
|
offset: -1337,
|
|
content: []byte("butts"),
|
|
expectedContent: []byte(""),
|
|
err: "argError: Invalid offset: -1337",
|
|
},
|
|
{
|
|
it: "should successfully retrieve data with negative length",
|
|
id: "0123456789ABCDEFGHIJ",
|
|
size: -1,
|
|
offset: 0,
|
|
content: []byte("butts"),
|
|
expectedContent: []byte("butts"),
|
|
err: "",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.it, func(t *testing.T) {
|
|
assert := assert.New(t)
|
|
|
|
folder1 := string(tt.id[0:2])
|
|
folder2 := string(tt.id[2:4])
|
|
fileName := string(tt.id[4:])
|
|
|
|
createdFilePath := path.Join(os.TempDir(), folder1, folder2, fileName)
|
|
|
|
if err := os.MkdirAll(filepath.Dir(createdFilePath), 0700); err != nil {
|
|
t.Errorf("Error: %s when creating dir", err.Error())
|
|
return
|
|
}
|
|
|
|
createdFile, err := os.OpenFile(createdFilePath, os.O_RDWR|os.O_CREATE, 0755)
|
|
if err != nil {
|
|
t.Errorf("Error: %s opening created file %s", err.Error(), createdFilePath)
|
|
return
|
|
}
|
|
|
|
createdFile.Seek(tt.offset, 0)
|
|
_, err = createdFile.Write(tt.content)
|
|
if err != nil {
|
|
t.Errorf("Error: %s writing to created file", err.Error())
|
|
return
|
|
}
|
|
createdFile.Close()
|
|
|
|
storeFile, err := RetrieveReader(tt.id, tt.size, tt.offset, os.TempDir())
|
|
if tt.err != "" {
|
|
if assert.NotNil(err) {
|
|
assert.Equal(tt.err, err.Error())
|
|
}
|
|
return
|
|
} else if err != nil {
|
|
t.Errorf("Error: %s", err.Error())
|
|
return
|
|
}
|
|
|
|
size := tt.size
|
|
if tt.size < 0 {
|
|
size = int64(len(tt.content))
|
|
}
|
|
buffer := make([]byte, size)
|
|
storeFile.Read(buffer)
|
|
storeFile.Close()
|
|
|
|
os.RemoveAll(path.Join(os.TempDir(), folder1))
|
|
|
|
if string(buffer) != string(tt.expectedContent) {
|
|
t.Errorf("Expected data butts does not equal Actual data %s", string(buffer))
|
|
return
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelete(t *testing.T) {
|
|
tests := []struct{
|
|
it string
|
|
id string
|
|
err string
|
|
} {
|
|
{
|
|
it: "should successfully delete data",
|
|
id: "11111111111111111111",
|
|
err: "",
|
|
},
|
|
{
|
|
it: "should return nil-err with non-existent id",
|
|
id: "11111111111111111111",
|
|
err: "",
|
|
},
|
|
{
|
|
it: "should err with invalid id length",
|
|
id: "111111",
|
|
err: "argError: Invalid id length",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.it, func(t *testing.T) {
|
|
assert := assert.New(t)
|
|
|
|
folder1 := string(tt.id[0:2])
|
|
folder2 := string(tt.id[2:4])
|
|
fileName := string(tt.id[4:])
|
|
|
|
createdFilePath := path.Join(os.TempDir(), folder1, folder2, fileName)
|
|
|
|
if err := os.MkdirAll(filepath.Dir(createdFilePath), 0700); err != nil {
|
|
t.Errorf("Error: %s when creating dir", err.Error())
|
|
return
|
|
}
|
|
|
|
createdFile, err := os.OpenFile(createdFilePath, os.O_RDWR|os.O_CREATE, 0755)
|
|
if err != nil {
|
|
t.Errorf("Error: %s opening created file %s", err.Error(), createdFilePath)
|
|
return
|
|
}
|
|
|
|
createdFile.Close()
|
|
|
|
err = Delete(tt.id, os.TempDir())
|
|
if tt.err != "" {
|
|
if assert.NotNil(err) {
|
|
assert.Equal(tt.err, err.Error())
|
|
}
|
|
return
|
|
} else if err != nil {
|
|
t.Errorf("Error: %s", err.Error())
|
|
return
|
|
}
|
|
|
|
if _, err = os.Stat(createdFilePath); os.IsExist(err) {
|
|
t.Errorf("Error deleting file")
|
|
return
|
|
}
|
|
return
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMain(m *testing.M) {
|
|
m.Run()
|
|
}
|