Diagnostic tool for Satellite (#586)

* Satellite diagnostic tool initial development

* changes for code review and lint warning fixes

cmd/satellite/main.go

@@ -4,18 +4,25 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
+	"text/tabwriter"
 
+	"github.com/golang/protobuf/proto"
 	"github.com/spf13/cobra"
+	"go.uber.org/zap"
 
 	// "storj.io/storj/pkg/audit"
 	"storj.io/storj/pkg/auth/grpcauth"
+	"storj.io/storj/pkg/bwagreement"
 	"storj.io/storj/pkg/cfgstruct"
 	"storj.io/storj/pkg/kademlia"
 	"storj.io/storj/pkg/overlay"
 	mockOverlay "storj.io/storj/pkg/overlay/mocks"
+	"storj.io/storj/pkg/pb"
 	"storj.io/storj/pkg/pointerdb"
 	"storj.io/storj/pkg/process"
 	"storj.io/storj/pkg/provider"
@@ -37,6 +44,11 @@ var (
 		Short: "Create config files",
 		RunE:  cmdSetup,
 	}
+	diagCmd = &cobra.Command{
+		Use:   "diag",
+		Short: "Diagnostic Tool support",
+		RunE:  cmdDiag,
+	}
 
 	runCfg struct {
 		Identity  provider.IdentityConfig
@@ -58,15 +70,21 @@ var (
 		Identity  provider.IdentitySetupConfig
 		Overwrite bool `default:"false" help:"whether to overwrite pre-existing configuration files"`
 	}
+	diagCfg struct {
+		BasePath string `default:"$CONFDIR" help:"base path for setup"`
+	}
 
 	defaultConfDir = "$HOME/.storj/satellite"
+	defaultDiagDir = "postgres://postgres@localhost/pointerdb?sslmode=disable"
 )
 
 func init() {
 	rootCmd.AddCommand(runCmd)
 	rootCmd.AddCommand(setupCmd)
+	rootCmd.AddCommand(diagCmd)
 	cfgstruct.Bind(runCmd.Flags(), &runCfg, cfgstruct.ConfDir(defaultConfDir))
 	cfgstruct.Bind(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir))
+	cfgstruct.Bind(diagCmd.Flags(), &diagCfg, cfgstruct.ConfDir(defaultDiagDir))
 }
 
 func cmdRun(cmd *cobra.Command, args []string) (err error) {
@@ -130,6 +148,82 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
 		filepath.Join(setupCfg.BasePath, "config.yaml"), o)
 }
 
+func cmdDiag(cmd *cobra.Command, args []string) (err error) {
+	// open the psql db
+	dbpath := diagCfg.BasePath
+	s, err := bwagreement.NewServer("postgres", dbpath, zap.NewNop())
+	if err != nil {
+		fmt.Println("Storagenode database couldnt open:", dbpath)
+		return err
+	}
+	//get all bandwidth aggrements rows already ordered
+	baRows, err := s.GetBandwidthAllocations(context.Background())
+	if err != nil {
+		fmt.Printf("error reading satellite database %v: %v\n", dbpath, err)
+		return err
+	}
+
+	// Agreement is a struct that contains a bandwidth agreement and the associated signature
+	type UplinkSummary struct {
+		TotalBytes        int64
+		PutActionCount    int64
+		GetActionCount    int64
+		TotalTransactions int64
+		// additional attributes add here ...
+	}
+
+	// attributes per uplinkid
+	summaries := make(map[string]*UplinkSummary)
+	uplinkIDs := []string{}
+
+	for _, baRow := range baRows {
+		// deserializing rbad you get payerbwallocation, total & storage node id
+		rbad := &pb.RenterBandwidthAllocation_Data{}
+		if err := proto.Unmarshal(baRow.Data, rbad); err != nil {
+			return err
+		}
+
+		// deserializing pbad you get satelliteID, uplinkID, max size, exp, serial# & action
+		pbad := &pb.PayerBandwidthAllocation_Data{}
+		if err := proto.Unmarshal(rbad.GetPayerAllocation().GetData(), pbad); err != nil {
+			return err
+		}
+
+		uplinkID := string(pbad.GetUplinkId())
+		summary, ok := summaries[uplinkID]
+		if !ok {
+			summaries[uplinkID] = &UplinkSummary{}
+			uplinkIDs = append(uplinkIDs, uplinkID)
+			summary = summaries[uplinkID]
+		}
+
+		// fill the summary info
+		summary.TotalBytes += rbad.GetTotal()
+		summary.TotalTransactions++
+		if pbad.GetAction() == pb.PayerBandwidthAllocation_PUT {
+			summary.PutActionCount++
+		} else {
+			summary.GetActionCount++
+		}
+	}
+
+	// initialize the table header (fields)
+	const padding = 3
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, padding, ' ', tabwriter.AlignRight|tabwriter.Debug)
+	fmt.Fprintln(w, "UplinkID\tTotal\t# Of Transactions\tPUT Action\tGET Action\t")
+
+	// populate the row fields
+	sort.Strings(uplinkIDs)
+	for _, uplinkID := range uplinkIDs {
+		summary := summaries[uplinkID]
+		fmt.Fprint(w, uplinkID, "\t", summary.TotalBytes, "\t", summary.TotalTransactions, "\t", summary.PutActionCount, "\t", summary.GetActionCount, "\t\n")
+	}
+
+	// display the data
+	err = w.Flush()
+	return err
+}
+
 func main() {
 	runCmd.Flags().String("config",
 		filepath.Join(defaultConfDir, "config.yaml"), "path to configuration")

pkg/bwagreement/dbx/bwagreement.dbx

@@ -18,4 +18,6 @@ read one (
 read limitoffset (
 	select bwagreement
 )
-
+read all (
+	select bwagreement
+)

pkg/bwagreement/dbx/bwagreement.dbx.go

@@ -647,6 +647,38 @@ func (obj *postgresImpl) Limited_Bwagreement(ctx context.Context,
 
 }
 
+func (obj *postgresImpl) All_Bwagreement(ctx context.Context) (
+	rows []*Bwagreement, err error) {
+
+	var __embed_stmt = __sqlbundle_Literal("SELECT bwagreements.signature, bwagreements.data, bwagreements.created_at FROM bwagreements")
+
+	var __values []interface{}
+	__values = append(__values)
+
+	var __stmt = __sqlbundle_Render(obj.dialect, __embed_stmt)
+	obj.logStmt(__stmt, __values...)
+
+	__rows, err := obj.driver.Query(__stmt, __values...)
+	if err != nil {
+		return nil, obj.makeErr(err)
+	}
+	defer __rows.Close()
+
+	for __rows.Next() {
+		bwagreement := &Bwagreement{}
+		err = __rows.Scan(&bwagreement.Signature, &bwagreement.Data, &bwagreement.CreatedAt)
+		if err != nil {
+			return nil, obj.makeErr(err)
+		}
+		rows = append(rows, bwagreement)
+	}
+	if err := __rows.Err(); err != nil {
+		return nil, obj.makeErr(err)
+	}
+	return rows, nil
+
+}
+
 func (obj *postgresImpl) Delete_Bwagreement_By_Signature(ctx context.Context,
 	bwagreement_signature Bwagreement_Signature_Field) (
 	deleted bool, err error) {
@@ -743,6 +775,15 @@ func (rx *Rx) Rollback() (err error) {
 	return err
 }
 
+func (rx *Rx) All_Bwagreement(ctx context.Context) (
+	rows []*Bwagreement, err error) {
+	var tx *Tx
+	if tx, err = rx.getTx(ctx); err != nil {
+		return
+	}
+	return tx.All_Bwagreement(ctx)
+}
+
 func (rx *Rx) Create_Bwagreement(ctx context.Context,
 	bwagreement_signature Bwagreement_Signature_Field,
 	bwagreement_data Bwagreement_Data_Field) (
@@ -786,6 +827,9 @@ func (rx *Rx) Limited_Bwagreement(ctx context.Context,
 }
 
 type Methods interface {
+	All_Bwagreement(ctx context.Context) (
+		rows []*Bwagreement, err error)
+
 	Create_Bwagreement(ctx context.Context,
 		bwagreement_signature Bwagreement_Signature_Field,
 		bwagreement_data Bwagreement_Data_Field) (

pkg/bwagreement/server.go

@@ -6,6 +6,7 @@ package bwagreement
 import (
 	"context"
 	"strings"
+	"sync"
 
 	"go.uber.org/zap"
 	"google.golang.org/grpc/codes"
@@ -18,10 +19,16 @@ import (
 // Server is an implementation of the pb.BandwidthServer interface
 type Server struct {
 	DB     *dbx.DB
-	//identity *provider.FullIdentity
+	mu     sync.Mutex
 	logger *zap.Logger
 }
 
+// Agreement is a struct that contains a bandwidth agreement and the associated signature
+type Agreement struct {
+	Agreement []byte
+	Signature []byte
+}
+
 // NewServer creates instance of Server
 func NewServer(driver, source string, logger *zap.Logger) (*Server, error) {
 	db, err := dbx.Open(driver, source)
@@ -40,9 +47,15 @@ func NewServer(driver, source string, logger *zap.Logger) (*Server, error) {
 	}, nil
 }
 
+func (s *Server) locked() func() {
+	s.mu.Lock()
+	return s.mu.Unlock
+}
+
 // Create a db entry for the provided storagenode
 func (s *Server) Create(ctx context.Context, createBwAgreement *pb.RenterBandwidthAllocation) (bwagreement *dbx.Bwagreement, err error) {
 	defer mon.Task()(&ctx)(&err)
+
 	s.logger.Debug("entering statdb Create")
 
 	signature := createBwAgreement.GetSignature()
@@ -64,6 +77,7 @@ func (s *Server) Create(ctx context.Context, createBwAgreement *pb.RenterBandwid
 func (s *Server) BandwidthAgreements(stream pb.Bandwidth_BandwidthAgreementsServer) (err error) {
 	ctx := stream.Context()
 	defer mon.Task()(&ctx)(&err)
+	defer s.locked()()
 
 	ch := make(chan *pb.RenterBandwidthAllocation, 1)
 	errch := make(chan error, 1)
@@ -86,14 +100,20 @@ func (s *Server) BandwidthAgreements(stream pb.Bandwidth_BandwidthAgreementsServ
 		case <-ctx.Done():
 			return nil
 		case agreement := <-ch:
-			go func() {
 			_, err = s.Create(ctx, agreement)
 			if err != nil {
 				s.logger.Error("DB entry creation Error", zap.Error(err))
-					errch <- err
+				return err
 			}
-			}()
 		}
 	}
 
 }
+
+// GetBandwidthAllocations all bandwidth agreements and sorts by satellite
+func (s *Server) GetBandwidthAllocations(ctx context.Context) (rows []*dbx.Bwagreement, err error) {
+	defer mon.Task()(&ctx)(&err)
+	defer s.locked()()
+	rows, err = s.DB.All_Bwagreement(ctx)
+	return rows, err
+}

pkg/bwagreement/server_test.go

@@ -7,9 +7,10 @@ import (
 	"context"
 	"crypto"
 	"crypto/ecdsa"
-	"fmt"
+	"flag"
 	"log"
 	"net"
+	"os"
 	"testing"
 
 	"go.uber.org/zap"
@@ -26,21 +27,6 @@ var (
 	ctx = context.Background()
 )
 
-const (
-	host     = "localhost"
-	port     = 5432
-	user     = "postgres"
-	password = "your-password"
-	dbname   = "pointerdb"
-)
-
-func getPSQLInfo() string {
-	psqlInfo := fmt.Sprintf("host=%s port=%d user=%s "+
-		"password=%s dbname=%s sslmode=disable",
-		host, port, user, password, dbname)
-	return psqlInfo
-}
-
 func TestBandwidthAgreements(t *testing.T) {
 	TS := NewTestServer(t)
 	defer TS.Stop()
@@ -113,9 +99,22 @@ func NewTestServer(t *testing.T) *TestServer {
 	return ts
 }
 
+const (
+	// this connstring is expected to work under the storj-test docker-compose instance
+	defaultPostgresConn = "postgres://pointerdb:pg-secret-pass@test-postgres-pointerdb/pointerdb?sslmode=disable"
+)
+
+var (
+	// for travis build support
+	testPostgres = flag.String("postgres-test-db", os.Getenv("STORJ_POSTGRESKV_TEST"), "PostgreSQL test database connection string")
+)
+
 func newTestServerStruct(t *testing.T) *Server {
-	psqlInfo := getPSQLInfo()
+	if *testPostgres == "" {
-	s, err := NewServer("postgres", psqlInfo, zap.NewNop())
+		t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", defaultPostgresConn)
+	}
+
+	s, err := NewServer("postgres", *testPostgres, zap.NewNop())
 	assert.NoError(t, err)
 	return s
 }