2018-05-16 13:40:44 +01:00
|
|
|
// Copyright (C) 2018 Storj Labs, Inc.
|
|
|
|
// See LICENSE for copying information.
|
|
|
|
|
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
2018-06-01 15:07:58 +01:00
|
|
|
"context"
|
2018-05-16 13:40:44 +01:00
|
|
|
"fmt"
|
Added rpc server (#40)
* Added piecestore
* gofmt
* Added requested changes
* Added cli
* Removed ranger because I wanted something that can stand alone
* Add example of http server using piece store
* Changed piecestore code to make it more optial for error handelling
* Merged with piecestore
* Added missing package
* Forgot io import
* gofmt
* gofmt
* Forgot io
* Make path by hash exported
* updated to simplify again whoops
* Updated server to work real good
* Forgot ampersand
* Updated to match FilePiece
* Merged in cam's delete code
* Remove unused io
* Added RPC code
* Give the download request a reader
* Removed http server stuff; changed receive stream to say io.reader
* Added expiration date to shardInfo
* Change all instances of Shard to Piece; change protobuf name; moved client insance to outside functions
* added ttl info request
* Move scripts to http server pr; added close method for Retrieve api
* added rpc server tests for getting piece meta data and retrieval routes
* Resolved linter errors, moved to prc server to pkg, updated go.mod to use latest protobuf
* Imported cams test
* Bump gometalinter deadline
* WIP adding tests
* added tests for store and delete routes
* Add changes as requested by Kaloyan, also cleaned up some code
* Get the code actually working whoops
* More cleanup
* Separating database calls from api.go
* need to rename expiration
* Added some changes requested by JT
* Fix read size
* Fixed total amount to read
* added tests
* Simplify protobuf, add store tests, edited api to handle invalid stores properly, return errors instead of messages
* Moved rpc client and server to piece store
* Moved piecestore protobuf to the /protos folder
* Cleaned up messages
* Clean up linter errors
* Added missing sqlite import
* Add ability to do iterative reads and writes to pstore
* Incrementally read data
* Fix linter and import errors
* Solve linter Error
* Change return types
* begin test refactor
* refactored to implement only 1 db connection, moved SQLite row checking into separate function and removed defer on rows.Close(), fixed os.tempDir in rpc_test.go
* Cleaning up tests
* Added retrieve tests
* refactored delete tests
* Deleted old tests
* Updated cmd/piecestore to reflect changes to piecestore
* Refactored server tests and server/client store code
* gofmt
* WIP implementing TTL struct
* Read 4k at a time when Retrieving
* implemented ttl struct
* Accidentally removed fpiece dependency?
* Double resolve merge conflict
* Moved client to the examples since it is an example
* Change hash to id in protobuf. TODO: Change client and server to reflect these changes
* Update client and protobuf
* changed hash to id
* Handle eof properly in retrieve
* Id -> ID
* forgot to change import for client after moving
* Moved client and server main to examples
* Made changes requested by JT
* checkEntries is now private, created currentTime variable for checkEntries, added defer rows.Close()
* Print not fatal
* Handle overflow when reading from server
* added const IDLength
* removed types from comments
* Add reader/writer for download data from and uploading data to server
* goimports and comments
* fixed nits, casts, added OK const, DBCleanup now exits program on error
* Add stream reader and writer for server
* Fix errors
* i beforee except after c lol
* customizable data dir
* Forgot to rename variable
* customizable data dir
* Handle closing of server stream properly
* linter
* pass on inserting the same data twice
* linter
* linter
* Do the easy things JT asked for
* Handle overflow from reads properly; handle custom db path
* Handle overflow for server stream read; TODO Combine server and client stream reads
* Allow for TTL of 0 to stay forever
* Make Client cleaner for user
2018-06-02 19:14:59 +01:00
|
|
|
"io"
|
2018-05-16 13:40:44 +01:00
|
|
|
"os"
|
|
|
|
|
2018-06-27 09:02:49 +01:00
|
|
|
"github.com/spf13/cobra"
|
|
|
|
|
2018-08-27 18:28:16 +01:00
|
|
|
pstore "storj.io/storj/pkg/piecestore"
|
2018-06-01 15:07:58 +01:00
|
|
|
"storj.io/storj/pkg/process"
|
2018-05-16 13:40:44 +01:00
|
|
|
)
|
|
|
|
|
2018-10-02 15:48:09 +01:00
|
|
|
func main() {
|
|
|
|
cobra.EnableCommandSorting = false
|
|
|
|
root := &cobra.Command{
|
|
|
|
Use: "piecestore-cli",
|
|
|
|
Short: "piecestore example cli",
|
2018-05-16 13:40:44 +01:00
|
|
|
}
|
|
|
|
|
2018-10-02 15:48:09 +01:00
|
|
|
root.AddCommand(
|
|
|
|
storeMain,
|
|
|
|
retrieveMain,
|
|
|
|
deleteMain,
|
|
|
|
)
|
2018-05-16 13:40:44 +01:00
|
|
|
|
2018-10-02 15:48:09 +01:00
|
|
|
process.Exec(root)
|
2018-05-16 13:40:44 +01:00
|
|
|
}
|
2018-06-01 15:07:58 +01:00
|
|
|
|
2018-10-02 15:48:09 +01:00
|
|
|
var storeMain = &cobra.Command{
|
|
|
|
Use: "store [id] [dataPath] [storeDir]",
|
|
|
|
Aliases: []string{"s"},
|
|
|
|
Short: "Store data by id",
|
|
|
|
Args: cobra.ExactArgs(3),
|
|
|
|
ValidArgs: []string{"id", "datapath", "storedir"},
|
|
|
|
|
|
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
|
|
id := args[0]
|
|
|
|
path := args[1]
|
|
|
|
outputDir := args[2]
|
|
|
|
|
|
|
|
file, err := os.Open(path)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Close the file when we are done
|
|
|
|
defer printError(file.Close)
|
|
|
|
|
|
|
|
fileInfo, err := os.Stat(path)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if fileInfo.IsDir() {
|
|
|
|
return fmt.Errorf("Path (%s) is a directory, not a file", path)
|
|
|
|
}
|
|
|
|
|
|
|
|
dataFileChunk, err := pstore.StoreWriter(id, outputDir)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Close when finished
|
|
|
|
defer printError(dataFileChunk.Close)
|
|
|
|
|
|
|
|
_, err = io.Copy(dataFileChunk, file)
|
|
|
|
|
|
|
|
return err
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
var retrieveMain = &cobra.Command{
|
|
|
|
Use: "retrieve [id] [storeDir]",
|
|
|
|
Aliases: []string{"r"},
|
|
|
|
Args: cobra.ExactArgs(2),
|
|
|
|
Short: "Retrieve data by id and print to Stdout",
|
|
|
|
|
|
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
|
|
id := args[0]
|
|
|
|
path := args[1]
|
|
|
|
|
|
|
|
fileInfo, err := os.Stat(path)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if !fileInfo.IsDir() {
|
|
|
|
return fmt.Errorf("Path (%s) is a file, not a directory", path)
|
|
|
|
}
|
|
|
|
|
|
|
|
dataFileChunk, err := pstore.RetrieveReader(context.Background(), id, 0, -1, path)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Close when finished
|
|
|
|
defer printError(dataFileChunk.Close)
|
|
|
|
|
|
|
|
_, err = io.Copy(os.Stdout, dataFileChunk)
|
|
|
|
return err
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
var deleteMain = &cobra.Command{
|
|
|
|
Use: "delete [id] [storeDir]",
|
|
|
|
Aliases: []string{"d"},
|
|
|
|
Args: cobra.ExactArgs(2),
|
|
|
|
Short: "Delete data by id",
|
|
|
|
|
|
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
|
|
id := args[0]
|
|
|
|
directory := args[1]
|
|
|
|
return pstore.Delete(id, directory)
|
|
|
|
},
|
2018-07-19 01:09:38 +01:00
|
|
|
}
|
2018-09-28 19:08:44 +01:00
|
|
|
|
|
|
|
func printError(fn func() error) {
|
|
|
|
err := fn()
|
|
|
|
if err != nil {
|
|
|
|
fmt.Println(err)
|
|
|
|
}
|
|
|
|
}
|