2018-05-16 13:40:44 +01:00
|
|
|
// Copyright (C) 2018 Storj Labs, Inc.
|
|
|
|
// See LICENSE for copying information.
|
|
|
|
|
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
2018-06-01 15:07:58 +01:00
|
|
|
"context"
|
|
|
|
"flag"
|
2018-05-16 13:40:44 +01:00
|
|
|
"fmt"
|
Added rpc server (#40)
* Added piecestore
* gofmt
* Added requested changes
* Added cli
* Removed ranger because I wanted something that can stand alone
* Add example of http server using piece store
* Changed piecestore code to make it more optial for error handelling
* Merged with piecestore
* Added missing package
* Forgot io import
* gofmt
* gofmt
* Forgot io
* Make path by hash exported
* updated to simplify again whoops
* Updated server to work real good
* Forgot ampersand
* Updated to match FilePiece
* Merged in cam's delete code
* Remove unused io
* Added RPC code
* Give the download request a reader
* Removed http server stuff; changed receive stream to say io.reader
* Added expiration date to shardInfo
* Change all instances of Shard to Piece; change protobuf name; moved client insance to outside functions
* added ttl info request
* Move scripts to http server pr; added close method for Retrieve api
* added rpc server tests for getting piece meta data and retrieval routes
* Resolved linter errors, moved to prc server to pkg, updated go.mod to use latest protobuf
* Imported cams test
* Bump gometalinter deadline
* WIP adding tests
* added tests for store and delete routes
* Add changes as requested by Kaloyan, also cleaned up some code
* Get the code actually working whoops
* More cleanup
* Separating database calls from api.go
* need to rename expiration
* Added some changes requested by JT
* Fix read size
* Fixed total amount to read
* added tests
* Simplify protobuf, add store tests, edited api to handle invalid stores properly, return errors instead of messages
* Moved rpc client and server to piece store
* Moved piecestore protobuf to the /protos folder
* Cleaned up messages
* Clean up linter errors
* Added missing sqlite import
* Add ability to do iterative reads and writes to pstore
* Incrementally read data
* Fix linter and import errors
* Solve linter Error
* Change return types
* begin test refactor
* refactored to implement only 1 db connection, moved SQLite row checking into separate function and removed defer on rows.Close(), fixed os.tempDir in rpc_test.go
* Cleaning up tests
* Added retrieve tests
* refactored delete tests
* Deleted old tests
* Updated cmd/piecestore to reflect changes to piecestore
* Refactored server tests and server/client store code
* gofmt
* WIP implementing TTL struct
* Read 4k at a time when Retrieving
* implemented ttl struct
* Accidentally removed fpiece dependency?
* Double resolve merge conflict
* Moved client to the examples since it is an example
* Change hash to id in protobuf. TODO: Change client and server to reflect these changes
* Update client and protobuf
* changed hash to id
* Handle eof properly in retrieve
* Id -> ID
* forgot to change import for client after moving
* Moved client and server main to examples
* Made changes requested by JT
* checkEntries is now private, created currentTime variable for checkEntries, added defer rows.Close()
* Print not fatal
* Handle overflow when reading from server
* added const IDLength
* removed types from comments
* Add reader/writer for download data from and uploading data to server
* goimports and comments
* fixed nits, casts, added OK const, DBCleanup now exits program on error
* Add stream reader and writer for server
* Fix errors
* i beforee except after c lol
* customizable data dir
* Forgot to rename variable
* customizable data dir
* Handle closing of server stream properly
* linter
* pass on inserting the same data twice
* linter
* linter
* Do the easy things JT asked for
* Handle overflow from reads properly; handle custom db path
* Handle overflow for server stream read; TODO Combine server and client stream reads
* Allow for TTL of 0 to stay forever
* Make Client cleaner for user
2018-06-02 19:14:59 +01:00
|
|
|
"io"
|
2018-05-16 13:40:44 +01:00
|
|
|
"os"
|
|
|
|
"sort"
|
|
|
|
|
2018-06-27 09:02:49 +01:00
|
|
|
"github.com/spf13/cobra"
|
2018-05-16 13:40:44 +01:00
|
|
|
"github.com/urfave/cli"
|
|
|
|
"github.com/zeebo/errs"
|
2018-06-27 09:02:49 +01:00
|
|
|
|
2018-05-16 13:40:44 +01:00
|
|
|
"storj.io/storj/pkg/piecestore"
|
2018-06-01 15:07:58 +01:00
|
|
|
"storj.io/storj/pkg/process"
|
2018-05-16 13:40:44 +01:00
|
|
|
)
|
|
|
|
|
|
|
|
var argError = errs.Class("argError")
|
|
|
|
|
2018-06-27 09:02:49 +01:00
|
|
|
func run(ctx context.Context, _ *cobra.Command, _ []string) error {
|
2018-05-16 13:40:44 +01:00
|
|
|
app := cli.NewApp()
|
|
|
|
|
|
|
|
app.Name = "Piece Store CLI"
|
|
|
|
app.Usage = "Store data in hash folder structure"
|
|
|
|
app.Version = "1.0.0"
|
|
|
|
|
|
|
|
app.Flags = []cli.Flag{}
|
|
|
|
|
|
|
|
app.Commands = []cli.Command{
|
|
|
|
{
|
|
|
|
Name: "store",
|
|
|
|
Aliases: []string{"s"},
|
2018-06-27 19:42:54 +01:00
|
|
|
Usage: "Store data by id",
|
|
|
|
ArgsUsage: "[id] [dataPath] [storeDir]",
|
2018-05-16 13:40:44 +01:00
|
|
|
Action: func(c *cli.Context) error {
|
|
|
|
if c.Args().Get(0) == "" {
|
2018-06-27 19:42:54 +01:00
|
|
|
return argError.New("No id specified")
|
2018-05-16 13:40:44 +01:00
|
|
|
}
|
|
|
|
|
2018-06-27 19:42:54 +01:00
|
|
|
id := c.Args().Get(0)
|
|
|
|
|
2018-06-04 17:35:31 +01:00
|
|
|
if c.Args().Get(1) == "" {
|
2018-06-27 19:42:54 +01:00
|
|
|
return argError.New("No input file specified")
|
|
|
|
}
|
|
|
|
|
|
|
|
path := c.Args().Get(1)
|
|
|
|
|
|
|
|
if c.Args().Get(2) == "" {
|
2018-05-16 13:40:44 +01:00
|
|
|
return argError.New("No output directory specified")
|
|
|
|
}
|
|
|
|
|
2018-06-27 19:42:54 +01:00
|
|
|
outputDir := c.Args().Get(2)
|
|
|
|
|
|
|
|
file, err := os.Open(path)
|
2018-05-16 13:40:44 +01:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Close the file when we are done
|
|
|
|
defer file.Close()
|
|
|
|
|
2018-06-27 19:42:54 +01:00
|
|
|
fileInfo, err := os.Stat(path)
|
2018-05-16 13:40:44 +01:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if fileInfo.IsDir() {
|
2018-06-27 19:42:54 +01:00
|
|
|
return argError.New(fmt.Sprintf("Path (%s) is a directory, not a file", path))
|
2018-05-16 13:40:44 +01:00
|
|
|
}
|
|
|
|
|
2018-06-27 19:42:54 +01:00
|
|
|
dataFileChunk, err := pstore.StoreWriter(id, outputDir)
|
Added rpc server (#40)
* Added piecestore
* gofmt
* Added requested changes
* Added cli
* Removed ranger because I wanted something that can stand alone
* Add example of http server using piece store
* Changed piecestore code to make it more optial for error handelling
* Merged with piecestore
* Added missing package
* Forgot io import
* gofmt
* gofmt
* Forgot io
* Make path by hash exported
* updated to simplify again whoops
* Updated server to work real good
* Forgot ampersand
* Updated to match FilePiece
* Merged in cam's delete code
* Remove unused io
* Added RPC code
* Give the download request a reader
* Removed http server stuff; changed receive stream to say io.reader
* Added expiration date to shardInfo
* Change all instances of Shard to Piece; change protobuf name; moved client insance to outside functions
* added ttl info request
* Move scripts to http server pr; added close method for Retrieve api
* added rpc server tests for getting piece meta data and retrieval routes
* Resolved linter errors, moved to prc server to pkg, updated go.mod to use latest protobuf
* Imported cams test
* Bump gometalinter deadline
* WIP adding tests
* added tests for store and delete routes
* Add changes as requested by Kaloyan, also cleaned up some code
* Get the code actually working whoops
* More cleanup
* Separating database calls from api.go
* need to rename expiration
* Added some changes requested by JT
* Fix read size
* Fixed total amount to read
* added tests
* Simplify protobuf, add store tests, edited api to handle invalid stores properly, return errors instead of messages
* Moved rpc client and server to piece store
* Moved piecestore protobuf to the /protos folder
* Cleaned up messages
* Clean up linter errors
* Added missing sqlite import
* Add ability to do iterative reads and writes to pstore
* Incrementally read data
* Fix linter and import errors
* Solve linter Error
* Change return types
* begin test refactor
* refactored to implement only 1 db connection, moved SQLite row checking into separate function and removed defer on rows.Close(), fixed os.tempDir in rpc_test.go
* Cleaning up tests
* Added retrieve tests
* refactored delete tests
* Deleted old tests
* Updated cmd/piecestore to reflect changes to piecestore
* Refactored server tests and server/client store code
* gofmt
* WIP implementing TTL struct
* Read 4k at a time when Retrieving
* implemented ttl struct
* Accidentally removed fpiece dependency?
* Double resolve merge conflict
* Moved client to the examples since it is an example
* Change hash to id in protobuf. TODO: Change client and server to reflect these changes
* Update client and protobuf
* changed hash to id
* Handle eof properly in retrieve
* Id -> ID
* forgot to change import for client after moving
* Moved client and server main to examples
* Made changes requested by JT
* checkEntries is now private, created currentTime variable for checkEntries, added defer rows.Close()
* Print not fatal
* Handle overflow when reading from server
* added const IDLength
* removed types from comments
* Add reader/writer for download data from and uploading data to server
* goimports and comments
* fixed nits, casts, added OK const, DBCleanup now exits program on error
* Add stream reader and writer for server
* Fix errors
* i beforee except after c lol
* customizable data dir
* Forgot to rename variable
* customizable data dir
* Handle closing of server stream properly
* linter
* pass on inserting the same data twice
* linter
* linter
* Do the easy things JT asked for
* Handle overflow from reads properly; handle custom db path
* Handle overflow for server stream read; TODO Combine server and client stream reads
* Allow for TTL of 0 to stay forever
* Make Client cleaner for user
2018-06-02 19:14:59 +01:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Close when finished
|
|
|
|
defer dataFileChunk.Close()
|
|
|
|
|
|
|
|
_, err = io.Copy(dataFileChunk, file)
|
2018-05-16 13:40:44 +01:00
|
|
|
|
|
|
|
return err
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Name: "retrieve",
|
|
|
|
Aliases: []string{"r"},
|
2018-06-27 19:42:54 +01:00
|
|
|
Usage: "Retrieve data by id and print to Stdout",
|
|
|
|
ArgsUsage: "[id] [storeDir]",
|
2018-05-16 13:40:44 +01:00
|
|
|
Action: func(c *cli.Context) error {
|
|
|
|
if c.Args().Get(0) == "" {
|
2018-06-27 19:42:54 +01:00
|
|
|
return argError.New("Missing data id")
|
2018-05-16 13:40:44 +01:00
|
|
|
}
|
|
|
|
if c.Args().Get(1) == "" {
|
|
|
|
return argError.New("Missing file path")
|
|
|
|
}
|
|
|
|
fileInfo, err := os.Stat(c.Args().Get(1))
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if fileInfo.IsDir() != true {
|
|
|
|
return argError.New(fmt.Sprintf("Path (%s) is a file, not a directory", c.Args().Get(1)))
|
|
|
|
}
|
|
|
|
|
2018-06-19 16:59:09 +01:00
|
|
|
dataFileChunk, err := pstore.RetrieveReader(context.Background(),
|
|
|
|
c.Args().Get(0), 0, -1, c.Args().Get(1))
|
2018-05-16 13:40:44 +01:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
Added rpc server (#40)
* Added piecestore
* gofmt
* Added requested changes
* Added cli
* Removed ranger because I wanted something that can stand alone
* Add example of http server using piece store
* Changed piecestore code to make it more optial for error handelling
* Merged with piecestore
* Added missing package
* Forgot io import
* gofmt
* gofmt
* Forgot io
* Make path by hash exported
* updated to simplify again whoops
* Updated server to work real good
* Forgot ampersand
* Updated to match FilePiece
* Merged in cam's delete code
* Remove unused io
* Added RPC code
* Give the download request a reader
* Removed http server stuff; changed receive stream to say io.reader
* Added expiration date to shardInfo
* Change all instances of Shard to Piece; change protobuf name; moved client insance to outside functions
* added ttl info request
* Move scripts to http server pr; added close method for Retrieve api
* added rpc server tests for getting piece meta data and retrieval routes
* Resolved linter errors, moved to prc server to pkg, updated go.mod to use latest protobuf
* Imported cams test
* Bump gometalinter deadline
* WIP adding tests
* added tests for store and delete routes
* Add changes as requested by Kaloyan, also cleaned up some code
* Get the code actually working whoops
* More cleanup
* Separating database calls from api.go
* need to rename expiration
* Added some changes requested by JT
* Fix read size
* Fixed total amount to read
* added tests
* Simplify protobuf, add store tests, edited api to handle invalid stores properly, return errors instead of messages
* Moved rpc client and server to piece store
* Moved piecestore protobuf to the /protos folder
* Cleaned up messages
* Clean up linter errors
* Added missing sqlite import
* Add ability to do iterative reads and writes to pstore
* Incrementally read data
* Fix linter and import errors
* Solve linter Error
* Change return types
* begin test refactor
* refactored to implement only 1 db connection, moved SQLite row checking into separate function and removed defer on rows.Close(), fixed os.tempDir in rpc_test.go
* Cleaning up tests
* Added retrieve tests
* refactored delete tests
* Deleted old tests
* Updated cmd/piecestore to reflect changes to piecestore
* Refactored server tests and server/client store code
* gofmt
* WIP implementing TTL struct
* Read 4k at a time when Retrieving
* implemented ttl struct
* Accidentally removed fpiece dependency?
* Double resolve merge conflict
* Moved client to the examples since it is an example
* Change hash to id in protobuf. TODO: Change client and server to reflect these changes
* Update client and protobuf
* changed hash to id
* Handle eof properly in retrieve
* Id -> ID
* forgot to change import for client after moving
* Moved client and server main to examples
* Made changes requested by JT
* checkEntries is now private, created currentTime variable for checkEntries, added defer rows.Close()
* Print not fatal
* Handle overflow when reading from server
* added const IDLength
* removed types from comments
* Add reader/writer for download data from and uploading data to server
* goimports and comments
* fixed nits, casts, added OK const, DBCleanup now exits program on error
* Add stream reader and writer for server
* Fix errors
* i beforee except after c lol
* customizable data dir
* Forgot to rename variable
* customizable data dir
* Handle closing of server stream properly
* linter
* pass on inserting the same data twice
* linter
* linter
* Do the easy things JT asked for
* Handle overflow from reads properly; handle custom db path
* Handle overflow for server stream read; TODO Combine server and client stream reads
* Allow for TTL of 0 to stay forever
* Make Client cleaner for user
2018-06-02 19:14:59 +01:00
|
|
|
// Close when finished
|
|
|
|
defer dataFileChunk.Close()
|
|
|
|
|
|
|
|
_, err = io.Copy(os.Stdout, dataFileChunk)
|
|
|
|
return err
|
2018-05-16 13:40:44 +01:00
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Name: "delete",
|
|
|
|
Aliases: []string{"d"},
|
2018-06-27 19:42:54 +01:00
|
|
|
Usage: "Delete data by id",
|
|
|
|
ArgsUsage: "[id] [storeDir]",
|
2018-05-16 13:40:44 +01:00
|
|
|
Action: func(c *cli.Context) error {
|
|
|
|
if c.Args().Get(0) == "" {
|
2018-06-27 19:42:54 +01:00
|
|
|
return argError.New("Missing data id")
|
2018-05-16 13:40:44 +01:00
|
|
|
}
|
|
|
|
if c.Args().Get(1) == "" {
|
|
|
|
return argError.New("No directory specified")
|
|
|
|
}
|
|
|
|
err := pstore.Delete(c.Args().Get(0), c.Args().Get(1))
|
|
|
|
|
|
|
|
return err
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
sort.Sort(cli.FlagsByName(app.Flags))
|
|
|
|
sort.Sort(cli.CommandsByName(app.Commands))
|
|
|
|
|
2018-06-01 15:07:58 +01:00
|
|
|
return app.Run(append([]string{os.Args[0]}, flag.Args()...))
|
2018-05-16 13:40:44 +01:00
|
|
|
}
|
2018-06-01 15:07:58 +01:00
|
|
|
|
|
|
|
func main() { process.Must(process.Main(process.ServiceFunc(run))) }
|