storj/examples/piecestore/rpc/client/main.go
Alexander Leitner ff8e191a9a
Added rpc server (#40)
* Added piecestore

* gofmt

* Added requested changes

* Added cli

* Removed ranger because I wanted something that can stand alone

* Add example of http server using piece store

* Changed piecestore code to make it more optial for error handelling

* Merged with piecestore

* Added missing package

* Forgot io import

* gofmt

* gofmt

* Forgot io

* Make path by hash exported

* updated to simplify again whoops

* Updated server to work real good

* Forgot ampersand

* Updated to match FilePiece

* Merged in cam's delete code

* Remove unused io

* Added RPC code

* Give the download request a reader

* Removed http server stuff; changed receive stream to say io.reader

* Added expiration date to shardInfo

* Change all instances of Shard to Piece; change protobuf name; moved client insance to outside functions

* added ttl info request

* Move scripts to http server pr; added close method for Retrieve api

* added rpc server tests for getting piece meta data and retrieval routes

* Resolved linter errors, moved to prc server to pkg, updated go.mod to use latest protobuf

* Imported cams test

* Bump gometalinter deadline

* WIP adding tests

* added tests for store and delete routes

* Add changes as requested by Kaloyan, also cleaned up some code

* Get the code actually working whoops

* More cleanup

* Separating database calls from api.go

* need to rename expiration

* Added some changes requested by JT

* Fix read size

* Fixed total amount to read

* added tests

* Simplify protobuf, add store tests, edited api to handle invalid stores properly, return errors instead of messages

* Moved rpc client and server to piece store

* Moved piecestore protobuf to the /protos folder

* Cleaned up messages

* Clean up linter errors

* Added missing sqlite import

* Add ability to do iterative reads and writes to pstore

* Incrementally read data

* Fix linter and import errors

* Solve linter Error

* Change return types

* begin test refactor

* refactored to implement only 1 db connection, moved SQLite row checking into separate function and removed defer on rows.Close(), fixed os.tempDir in rpc_test.go

* Cleaning up tests

* Added retrieve tests

* refactored delete tests

* Deleted old tests

* Updated cmd/piecestore to reflect changes to piecestore

* Refactored server tests and server/client store code

* gofmt

* WIP implementing TTL struct

* Read 4k at a time when Retrieving

* implemented ttl struct

* Accidentally removed fpiece dependency?

* Double resolve merge conflict

* Moved client to the examples since it is an example

* Change hash to id in protobuf. TODO: Change client and server to reflect these changes

* Update client and protobuf

* changed hash to id

* Handle eof properly in retrieve

* Id -> ID

* forgot to change import for client after moving

* Moved client and server main to examples

* Made changes requested by JT

* checkEntries is now private, created currentTime variable for checkEntries, added defer rows.Close()

* Print not fatal

* Handle overflow when reading from server

* added const IDLength

* removed types from comments

* Add reader/writer for download data from and uploading data to server

* goimports and comments

* fixed nits, casts, added OK const, DBCleanup now exits program on error

* Add stream reader and writer for server

* Fix errors

* i beforee except after c lol

* customizable data dir

* Forgot to rename variable

* customizable data dir

* Handle closing of server stream properly

* linter

* pass on inserting the same data twice

* linter

* linter

* Do the easy things JT asked for

* Handle overflow from reads properly; handle custom db path

* Handle overflow for server stream read; TODO Combine server and client stream reads

* Allow for TTL of 0 to stay forever

* Make Client cleaner for user
2018-06-02 14:14:59 -04:00

176 lines
3.8 KiB
Go

// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package main
import (
"fmt"
"io"
"log"
"os"
"path/filepath"
"time"
"github.com/urfave/cli"
"github.com/zeebo/errs"
"golang.org/x/net/context"
"google.golang.org/grpc"
"storj.io/storj/examples/piecestore/rpc/client/utils"
"storj.io/storj/pkg/piecestore/rpc/client"
)
var argError = errs.Class("argError")
func main() {
app := cli.NewApp()
// Set up connection with rpc server
var conn *grpc.ClientConn
conn, err := grpc.Dial(":7777", grpc.WithInsecure())
if err != nil {
log.Fatalf("did not connect: %s", err)
}
defer conn.Close()
routeClient := client.New(context.Background(), conn)
app.Commands = []cli.Command{
{
Name: "upload",
Aliases: []string{"u"},
Usage: "upload data",
Action: func(c *cli.Context) error {
if c.Args().Get(0) == "" {
return argError.New("No input file specified")
}
file, err := os.Open(c.Args().Get(0))
if err != nil {
return err
}
// Close the file when we are done
defer file.Close()
fileInfo, err := file.Stat()
if err != nil {
return err
}
if fileInfo.IsDir() {
return argError.New(fmt.Sprintf("Path (%s) is a directory, not a file", c.Args().Get(0)))
}
var fileOffset, storeOffset int64 = 0, 0
var length = fileInfo.Size()
var ttl = time.Now().Unix() + 86400
id, err := utils.DetermineID(file, fileOffset, length)
if err != nil {
return err
}
// Created a section reader so that we can concurrently retrieve the same file.
dataSection := io.NewSectionReader(file, fileOffset, length)
writer, err := routeClient.StorePieceRequest(id, fileOffset, length, ttl, storeOffset)
if err != nil {
fmt.Printf("Failed to send meta data to server to store file of id: %s\n", id)
return err
}
_, err = io.Copy(writer, dataSection)
if err != nil {
fmt.Printf("Failed to store file of id: %s\n", id)
} else {
fmt.Printf("successfully stored file of id: %s\n", id)
}
return writer.Close()
},
},
{
Name: "download",
Aliases: []string{"d"},
Usage: "download data",
Action: func(c *cli.Context) error {
if c.Args().Get(0) == "" {
return argError.New("No id specified")
}
id := c.Args().Get(0)
if c.Args().Get(1) == "" {
return argError.New("No output file specified")
}
_, err := os.Stat(c.Args().Get(1))
if err != nil && !os.IsNotExist(err) {
return err
}
if err == nil {
return argError.New("File already exists")
}
dataPath := c.Args().Get(1)
if err = os.MkdirAll(filepath.Dir(dataPath), 0700); err != nil {
return err
}
// Create File on file system
dataFile, err := os.OpenFile(dataPath, os.O_RDWR|os.O_CREATE, 0755)
if err != nil {
return err
}
defer dataFile.Close()
pieceInfo, err := routeClient.PieceMetaRequest(id)
if err != nil {
os.Remove(dataPath)
return err
}
reader, err := routeClient.RetrievePieceRequest(id, 0, pieceInfo.Size)
if err != nil {
fmt.Printf("Failed to retrieve file of id: %s\n", id)
os.Remove(dataPath)
return err
}
_, err = io.Copy(dataFile, reader)
if err != nil {
fmt.Printf("Failed to retrieve file of id: %s\n", id)
os.Remove(dataPath)
} else {
fmt.Printf("Successfully retrieved file of id: %s\n", id)
}
return reader.Close()
},
},
{
Name: "delete",
Aliases: []string{"x"},
Usage: "delete data",
Action: func(c *cli.Context) error {
if c.Args().Get(0) == "" {
return argError.New("Missing data Id")
}
err = routeClient.DeletePieceRequest(c.Args().Get(0))
return err
},
},
}
err = app.Run(os.Args)
if err != nil {
log.Fatal(err)
}
}