Diagnostic tool for Satellite (#586)
* Satellite diagnostic tool initial development * changes for code review and lint warning fixes
This commit is contained in:
parent
810835e6a9
commit
285b70048f
@ -4,18 +4,25 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"text/tabwriter"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/zap"
|
||||
|
||||
// "storj.io/storj/pkg/audit"
|
||||
"storj.io/storj/pkg/auth/grpcauth"
|
||||
"storj.io/storj/pkg/bwagreement"
|
||||
"storj.io/storj/pkg/cfgstruct"
|
||||
"storj.io/storj/pkg/kademlia"
|
||||
"storj.io/storj/pkg/overlay"
|
||||
mockOverlay "storj.io/storj/pkg/overlay/mocks"
|
||||
"storj.io/storj/pkg/pb"
|
||||
"storj.io/storj/pkg/pointerdb"
|
||||
"storj.io/storj/pkg/process"
|
||||
"storj.io/storj/pkg/provider"
|
||||
@ -37,6 +44,11 @@ var (
|
||||
Short: "Create config files",
|
||||
RunE: cmdSetup,
|
||||
}
|
||||
diagCmd = &cobra.Command{
|
||||
Use: "diag",
|
||||
Short: "Diagnostic Tool support",
|
||||
RunE: cmdDiag,
|
||||
}
|
||||
|
||||
runCfg struct {
|
||||
Identity provider.IdentityConfig
|
||||
@ -58,15 +70,21 @@ var (
|
||||
Identity provider.IdentitySetupConfig
|
||||
Overwrite bool `default:"false" help:"whether to overwrite pre-existing configuration files"`
|
||||
}
|
||||
diagCfg struct {
|
||||
BasePath string `default:"$CONFDIR" help:"base path for setup"`
|
||||
}
|
||||
|
||||
defaultConfDir = "$HOME/.storj/satellite"
|
||||
defaultDiagDir = "postgres://postgres@localhost/pointerdb?sslmode=disable"
|
||||
)
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(runCmd)
|
||||
rootCmd.AddCommand(setupCmd)
|
||||
rootCmd.AddCommand(diagCmd)
|
||||
cfgstruct.Bind(runCmd.Flags(), &runCfg, cfgstruct.ConfDir(defaultConfDir))
|
||||
cfgstruct.Bind(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir))
|
||||
cfgstruct.Bind(diagCmd.Flags(), &diagCfg, cfgstruct.ConfDir(defaultDiagDir))
|
||||
}
|
||||
|
||||
func cmdRun(cmd *cobra.Command, args []string) (err error) {
|
||||
@ -130,6 +148,82 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
|
||||
filepath.Join(setupCfg.BasePath, "config.yaml"), o)
|
||||
}
|
||||
|
||||
func cmdDiag(cmd *cobra.Command, args []string) (err error) {
|
||||
// open the psql db
|
||||
dbpath := diagCfg.BasePath
|
||||
s, err := bwagreement.NewServer("postgres", dbpath, zap.NewNop())
|
||||
if err != nil {
|
||||
fmt.Println("Storagenode database couldnt open:", dbpath)
|
||||
return err
|
||||
}
|
||||
//get all bandwidth aggrements rows already ordered
|
||||
baRows, err := s.GetBandwidthAllocations(context.Background())
|
||||
if err != nil {
|
||||
fmt.Printf("error reading satellite database %v: %v\n", dbpath, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Agreement is a struct that contains a bandwidth agreement and the associated signature
|
||||
type UplinkSummary struct {
|
||||
TotalBytes int64
|
||||
PutActionCount int64
|
||||
GetActionCount int64
|
||||
TotalTransactions int64
|
||||
// additional attributes add here ...
|
||||
}
|
||||
|
||||
// attributes per uplinkid
|
||||
summaries := make(map[string]*UplinkSummary)
|
||||
uplinkIDs := []string{}
|
||||
|
||||
for _, baRow := range baRows {
|
||||
// deserializing rbad you get payerbwallocation, total & storage node id
|
||||
rbad := &pb.RenterBandwidthAllocation_Data{}
|
||||
if err := proto.Unmarshal(baRow.Data, rbad); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// deserializing pbad you get satelliteID, uplinkID, max size, exp, serial# & action
|
||||
pbad := &pb.PayerBandwidthAllocation_Data{}
|
||||
if err := proto.Unmarshal(rbad.GetPayerAllocation().GetData(), pbad); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
uplinkID := string(pbad.GetUplinkId())
|
||||
summary, ok := summaries[uplinkID]
|
||||
if !ok {
|
||||
summaries[uplinkID] = &UplinkSummary{}
|
||||
uplinkIDs = append(uplinkIDs, uplinkID)
|
||||
summary = summaries[uplinkID]
|
||||
}
|
||||
|
||||
// fill the summary info
|
||||
summary.TotalBytes += rbad.GetTotal()
|
||||
summary.TotalTransactions++
|
||||
if pbad.GetAction() == pb.PayerBandwidthAllocation_PUT {
|
||||
summary.PutActionCount++
|
||||
} else {
|
||||
summary.GetActionCount++
|
||||
}
|
||||
}
|
||||
|
||||
// initialize the table header (fields)
|
||||
const padding = 3
|
||||
w := tabwriter.NewWriter(os.Stdout, 0, 0, padding, ' ', tabwriter.AlignRight|tabwriter.Debug)
|
||||
fmt.Fprintln(w, "UplinkID\tTotal\t# Of Transactions\tPUT Action\tGET Action\t")
|
||||
|
||||
// populate the row fields
|
||||
sort.Strings(uplinkIDs)
|
||||
for _, uplinkID := range uplinkIDs {
|
||||
summary := summaries[uplinkID]
|
||||
fmt.Fprint(w, uplinkID, "\t", summary.TotalBytes, "\t", summary.TotalTransactions, "\t", summary.PutActionCount, "\t", summary.GetActionCount, "\t\n")
|
||||
}
|
||||
|
||||
// display the data
|
||||
err = w.Flush()
|
||||
return err
|
||||
}
|
||||
|
||||
func main() {
|
||||
runCmd.Flags().String("config",
|
||||
filepath.Join(defaultConfDir, "config.yaml"), "path to configuration")
|
||||
|
@ -18,4 +18,6 @@ read one (
|
||||
read limitoffset (
|
||||
select bwagreement
|
||||
)
|
||||
|
||||
read all (
|
||||
select bwagreement
|
||||
)
|
||||
|
@ -647,6 +647,38 @@ func (obj *postgresImpl) Limited_Bwagreement(ctx context.Context,
|
||||
|
||||
}
|
||||
|
||||
func (obj *postgresImpl) All_Bwagreement(ctx context.Context) (
|
||||
rows []*Bwagreement, err error) {
|
||||
|
||||
var __embed_stmt = __sqlbundle_Literal("SELECT bwagreements.signature, bwagreements.data, bwagreements.created_at FROM bwagreements")
|
||||
|
||||
var __values []interface{}
|
||||
__values = append(__values)
|
||||
|
||||
var __stmt = __sqlbundle_Render(obj.dialect, __embed_stmt)
|
||||
obj.logStmt(__stmt, __values...)
|
||||
|
||||
__rows, err := obj.driver.Query(__stmt, __values...)
|
||||
if err != nil {
|
||||
return nil, obj.makeErr(err)
|
||||
}
|
||||
defer __rows.Close()
|
||||
|
||||
for __rows.Next() {
|
||||
bwagreement := &Bwagreement{}
|
||||
err = __rows.Scan(&bwagreement.Signature, &bwagreement.Data, &bwagreement.CreatedAt)
|
||||
if err != nil {
|
||||
return nil, obj.makeErr(err)
|
||||
}
|
||||
rows = append(rows, bwagreement)
|
||||
}
|
||||
if err := __rows.Err(); err != nil {
|
||||
return nil, obj.makeErr(err)
|
||||
}
|
||||
return rows, nil
|
||||
|
||||
}
|
||||
|
||||
func (obj *postgresImpl) Delete_Bwagreement_By_Signature(ctx context.Context,
|
||||
bwagreement_signature Bwagreement_Signature_Field) (
|
||||
deleted bool, err error) {
|
||||
@ -743,6 +775,15 @@ func (rx *Rx) Rollback() (err error) {
|
||||
return err
|
||||
}
|
||||
|
||||
func (rx *Rx) All_Bwagreement(ctx context.Context) (
|
||||
rows []*Bwagreement, err error) {
|
||||
var tx *Tx
|
||||
if tx, err = rx.getTx(ctx); err != nil {
|
||||
return
|
||||
}
|
||||
return tx.All_Bwagreement(ctx)
|
||||
}
|
||||
|
||||
func (rx *Rx) Create_Bwagreement(ctx context.Context,
|
||||
bwagreement_signature Bwagreement_Signature_Field,
|
||||
bwagreement_data Bwagreement_Data_Field) (
|
||||
@ -786,6 +827,9 @@ func (rx *Rx) Limited_Bwagreement(ctx context.Context,
|
||||
}
|
||||
|
||||
type Methods interface {
|
||||
All_Bwagreement(ctx context.Context) (
|
||||
rows []*Bwagreement, err error)
|
||||
|
||||
Create_Bwagreement(ctx context.Context,
|
||||
bwagreement_signature Bwagreement_Signature_Field,
|
||||
bwagreement_data Bwagreement_Data_Field) (
|
||||
|
@ -6,6 +6,7 @@ package bwagreement
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"google.golang.org/grpc/codes"
|
||||
@ -17,11 +18,17 @@ import (
|
||||
|
||||
// Server is an implementation of the pb.BandwidthServer interface
|
||||
type Server struct {
|
||||
DB *dbx.DB
|
||||
//identity *provider.FullIdentity
|
||||
DB *dbx.DB
|
||||
mu sync.Mutex
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// Agreement is a struct that contains a bandwidth agreement and the associated signature
|
||||
type Agreement struct {
|
||||
Agreement []byte
|
||||
Signature []byte
|
||||
}
|
||||
|
||||
// NewServer creates instance of Server
|
||||
func NewServer(driver, source string, logger *zap.Logger) (*Server, error) {
|
||||
db, err := dbx.Open(driver, source)
|
||||
@ -40,9 +47,15 @@ func NewServer(driver, source string, logger *zap.Logger) (*Server, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Server) locked() func() {
|
||||
s.mu.Lock()
|
||||
return s.mu.Unlock
|
||||
}
|
||||
|
||||
// Create a db entry for the provided storagenode
|
||||
func (s *Server) Create(ctx context.Context, createBwAgreement *pb.RenterBandwidthAllocation) (bwagreement *dbx.Bwagreement, err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
|
||||
s.logger.Debug("entering statdb Create")
|
||||
|
||||
signature := createBwAgreement.GetSignature()
|
||||
@ -64,6 +77,7 @@ func (s *Server) Create(ctx context.Context, createBwAgreement *pb.RenterBandwid
|
||||
func (s *Server) BandwidthAgreements(stream pb.Bandwidth_BandwidthAgreementsServer) (err error) {
|
||||
ctx := stream.Context()
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
defer s.locked()()
|
||||
|
||||
ch := make(chan *pb.RenterBandwidthAllocation, 1)
|
||||
errch := make(chan error, 1)
|
||||
@ -86,14 +100,20 @@ func (s *Server) BandwidthAgreements(stream pb.Bandwidth_BandwidthAgreementsServ
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case agreement := <-ch:
|
||||
go func() {
|
||||
_, err = s.Create(ctx, agreement)
|
||||
if err != nil {
|
||||
s.logger.Error("DB entry creation Error", zap.Error(err))
|
||||
errch <- err
|
||||
}
|
||||
}()
|
||||
_, err = s.Create(ctx, agreement)
|
||||
if err != nil {
|
||||
s.logger.Error("DB entry creation Error", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// GetBandwidthAllocations all bandwidth agreements and sorts by satellite
|
||||
func (s *Server) GetBandwidthAllocations(ctx context.Context) (rows []*dbx.Bwagreement, err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
defer s.locked()()
|
||||
rows, err = s.DB.All_Bwagreement(ctx)
|
||||
return rows, err
|
||||
}
|
||||
|
@ -7,9 +7,10 @@ import (
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/ecdsa"
|
||||
"fmt"
|
||||
"flag"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
@ -26,21 +27,6 @@ var (
|
||||
ctx = context.Background()
|
||||
)
|
||||
|
||||
const (
|
||||
host = "localhost"
|
||||
port = 5432
|
||||
user = "postgres"
|
||||
password = "your-password"
|
||||
dbname = "pointerdb"
|
||||
)
|
||||
|
||||
func getPSQLInfo() string {
|
||||
psqlInfo := fmt.Sprintf("host=%s port=%d user=%s "+
|
||||
"password=%s dbname=%s sslmode=disable",
|
||||
host, port, user, password, dbname)
|
||||
return psqlInfo
|
||||
}
|
||||
|
||||
func TestBandwidthAgreements(t *testing.T) {
|
||||
TS := NewTestServer(t)
|
||||
defer TS.Stop()
|
||||
@ -113,9 +99,22 @@ func NewTestServer(t *testing.T) *TestServer {
|
||||
return ts
|
||||
}
|
||||
|
||||
const (
|
||||
// this connstring is expected to work under the storj-test docker-compose instance
|
||||
defaultPostgresConn = "postgres://pointerdb:pg-secret-pass@test-postgres-pointerdb/pointerdb?sslmode=disable"
|
||||
)
|
||||
|
||||
var (
|
||||
// for travis build support
|
||||
testPostgres = flag.String("postgres-test-db", os.Getenv("STORJ_POSTGRESKV_TEST"), "PostgreSQL test database connection string")
|
||||
)
|
||||
|
||||
func newTestServerStruct(t *testing.T) *Server {
|
||||
psqlInfo := getPSQLInfo()
|
||||
s, err := NewServer("postgres", psqlInfo, zap.NewNop())
|
||||
if *testPostgres == "" {
|
||||
t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", defaultPostgresConn)
|
||||
}
|
||||
|
||||
s, err := NewServer("postgres", *testPostgres, zap.NewNop())
|
||||
assert.NoError(t, err)
|
||||
return s
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user