V3-1152 Node bootstrap web backend (#1327)

* V3-1152 Node bootstrap

bootstrap/bootstrapweb/bootstrapserver/bootstrapql/query.go

@@ -0,0 +1,48 @@
+// Copyright (C) 2019 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package bootstrapql
+
+import (
+	"github.com/graphql-go/graphql"
+
+	"storj.io/storj/bootstrap/bootstrapweb"
+	"storj.io/storj/pkg/storj"
+)
+
+const (
+	// Query is immutable graphql request
+	Query = "query"
+	// IsNodeUpQuery is a query name for checking if node is up
+	IsNodeUpQuery = "isNodeUp"
+
+	// NodeID is a field name for nodeID
+	NodeID = "nodeID"
+)
+
+// rootQuery creates query for graphql
+func rootQuery(service *bootstrapweb.Service, types Types) *graphql.Object {
+	return graphql.NewObject(graphql.ObjectConfig{
+		Name: Query,
+		Fields: graphql.Fields{
+			IsNodeUpQuery: &graphql.Field{
+				Type: graphql.Boolean,
+				Args: graphql.FieldConfigArgument{
+					NodeID: &graphql.ArgumentConfig{
+						Type: graphql.String,
+					},
+				},
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					inputNodeID, _ := p.Args[NodeID].(string)
+
+					nodeID, err := storj.NodeIDFromString(inputNodeID)
+					if err != nil {
+						return false, err
+					}
+
+					return service.IsNodeAvailable(p.Context, nodeID)
+				},
+			},
+		},
+	})
+}

bootstrap/bootstrapweb/bootstrapserver/bootstrapql/schema.go

@@ -0,0 +1,29 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package bootstrapql
+
+import (
+	"github.com/graphql-go/graphql"
+
+	"storj.io/storj/bootstrap/bootstrapweb"
+	"storj.io/storj/internal/storjql"
+)
+
+// CreateSchema creates a schema for bootstrap graphql api
+func CreateSchema(service *bootstrapweb.Service) (schema graphql.Schema, err error) {
+	storjql.WithLock(func() {
+		creator := TypeCreator{}
+
+		err = creator.Create(service)
+		if err != nil {
+			return
+		}
+
+		schema, err = graphql.NewSchema(graphql.SchemaConfig{
+			Query: creator.RootQuery(),
+		})
+	})
+
+	return schema, err
+}

bootstrap/bootstrapweb/bootstrapserver/bootstrapql/typecreator.go

@@ -0,0 +1,38 @@
+// Copyright (C) 2019 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package bootstrapql
+
+import (
+	"github.com/graphql-go/graphql"
+
+	"storj.io/storj/bootstrap/bootstrapweb"
+)
+
+// Types return graphql type objects
+type Types interface {
+	RootQuery() *graphql.Object
+}
+
+// TypeCreator handles graphql type creation and error checking
+type TypeCreator struct {
+	query *graphql.Object
+}
+
+// Create create types and check for error
+func (c *TypeCreator) Create(service *bootstrapweb.Service) error {
+	// root objects
+	c.query = rootQuery(service, c)
+
+	err := c.query.Error()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// RootQuery returns instance of query *graphql.Object
+func (c *TypeCreator) RootQuery() *graphql.Object {
+	return c.query
+}

bootstrap/bootstrapweb/bootstrapserver/server.go

@@ -0,0 +1,136 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package bootstrapserver
+
+import (
+	"context"
+	"encoding/json"
+	"net"
+	"net/http"
+	"path/filepath"
+
+	"github.com/graphql-go/graphql"
+	"github.com/zeebo/errs"
+	"go.uber.org/zap"
+	"golang.org/x/sync/errgroup"
+
+	"storj.io/storj/bootstrap/bootstrapweb"
+	"storj.io/storj/bootstrap/bootstrapweb/bootstrapserver/bootstrapql"
+)
+
+const (
+	contentType = "Content-Type"
+
+	applicationJSON    = "application/json"
+	applicationGraphql = "application/graphql"
+)
+
+// Error is bootstrap web error type
+var Error = errs.Class("bootstrap web error")
+
+// Config contains configuration for bootstrap web server
+type Config struct {
+	Address   string `help:"server address of the graphql api gateway and frontend app" default:"127.0.0.1:8082"`
+	StaticDir string `help:"path to static resources" default:""`
+}
+
+// Server represents bootstrap web server
+type Server struct {
+	log *zap.Logger
+
+	config   Config
+	service  *bootstrapweb.Service
+	listener net.Listener
+
+	schema graphql.Schema
+	server http.Server
+}
+
+// NewServer creates new instance of bootstrap web server
+func NewServer(logger *zap.Logger, config Config, service *bootstrapweb.Service, listener net.Listener) *Server {
+	server := Server{
+		log:      logger,
+		service:  service,
+		config:   config,
+		listener: listener,
+	}
+
+	mux := http.NewServeMux()
+	fs := http.FileServer(http.Dir(server.config.StaticDir))
+
+	mux.Handle("/api/graphql/v0", http.HandlerFunc(server.grapqlHandler))
+
+	if server.config.StaticDir != "" {
+		mux.Handle("/", http.HandlerFunc(server.appHandler))
+		mux.Handle("/static/", http.StripPrefix("/static", fs))
+	}
+
+	server.server = http.Server{
+		Handler: mux,
+	}
+
+	return &server
+}
+
+// appHandler is web app http handler function
+func (s *Server) appHandler(w http.ResponseWriter, req *http.Request) {
+	http.ServeFile(w, req, filepath.Join(s.config.StaticDir, "dist", "public", "index.html"))
+}
+
+// grapqlHandler is graphql endpoint http handler function
+func (s *Server) grapqlHandler(w http.ResponseWriter, req *http.Request) {
+	w.Header().Set(contentType, applicationJSON)
+
+	query, err := getQuery(req)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	result := graphql.Do(graphql.Params{
+		Schema:         s.schema,
+		Context:        context.Background(),
+		RequestString:  query.Query,
+		VariableValues: query.Variables,
+		OperationName:  query.OperationName,
+		RootObject:     make(map[string]interface{}),
+	})
+
+	err = json.NewEncoder(w).Encode(result)
+	if err != nil {
+		s.log.Error(err.Error())
+		return
+	}
+
+	sugar := s.log.Sugar()
+	sugar.Debug(result)
+}
+
+// Run starts the server that host webapp and api endpoint
+func (s *Server) Run(ctx context.Context) error {
+	var err error
+
+	s.schema, err = bootstrapql.CreateSchema(s.service)
+	if err != nil {
+		return Error.Wrap(err)
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	var group errgroup.Group
+	group.Go(func() error {
+		<-ctx.Done()
+		return s.server.Shutdown(nil)
+	})
+	group.Go(func() error {
+		defer cancel()
+		return s.server.Serve(s.listener)
+	})
+
+	return group.Wait()
+}
+
+// Close closes server and underlying listener
+func (s *Server) Close() error {
+	return s.server.Close()
+}

bootstrap/bootstrapweb/bootstrapserver/utils.go

@@ -0,0 +1,50 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package bootstrapserver
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/zeebo/errs"
+
+	"storj.io/storj/bootstrap/bootstrapweb/bootstrapserver/bootstrapql"
+	"storj.io/storj/pkg/utils"
+)
+
+// JSON request from graphql clients
+type graphqlJSON struct {
+	Query         string
+	OperationName string
+	Variables     map[string]interface{}
+}
+
+// getQuery retrieves graphql query from request
+func getQuery(req *http.Request) (query graphqlJSON, err error) {
+	switch req.Method {
+	case http.MethodGet:
+		query.Query = req.URL.Query().Get(bootstrapql.Query)
+		return query, nil
+	case http.MethodPost:
+		return queryPOST(req)
+	default:
+		return query, errs.New("wrong http request type")
+	}
+}
+
+// queryPOST retrieves graphql query from POST request
+func queryPOST(req *http.Request) (query graphqlJSON, err error) {
+	switch typ := req.Header.Get(contentType); typ {
+	case applicationGraphql:
+		body, err := ioutil.ReadAll(req.Body)
+		query.Query = string(body)
+		return query, utils.CombineErrors(err, req.Body.Close())
+	case applicationJSON:
+		err := json.NewDecoder(req.Body).Decode(&query)
+		return query, utils.CombineErrors(err, req.Body.Close())
+	default:
+		return query, errs.New("can't parse request body of type %s", typ)
+	}
+}

bootstrap/bootstrapweb/service.go

@@ -0,0 +1,42 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package bootstrapweb
+
+import (
+	"context"
+
+	"github.com/zeebo/errs"
+	"go.uber.org/zap"
+
+	"storj.io/storj/pkg/kademlia"
+	"storj.io/storj/pkg/pb"
+)
+
+// Service is handling bootstrap related logic
+type Service struct {
+	log      *zap.Logger
+	kademlia *kademlia.Kademlia
+}
+
+// NewService returns new instance of Service
+func NewService(log *zap.Logger, kademlia *kademlia.Kademlia) (*Service, error) {
+	if log == nil {
+		return nil, errs.New("log can't be nil")
+	}
+
+	if kademlia == nil {
+		return nil, errs.New("kademlia can't be nil")
+	}
+
+	return &Service{log: log, kademlia: kademlia}, nil
+}
+
+// IsNodeAvailable is a method for checking if node is up
+func (s *Service) IsNodeAvailable(ctx context.Context, nodeID pb.NodeID) (bool, error) {
+	_, err := s.kademlia.FetchPeerIdentity(ctx, nodeID)
+
+	isNodeAvailable := err == nil
+
+	return isNodeAvailable, err
+}

bootstrap/peer.go

@@ -6,12 +6,15 @@ package bootstrap
 import (
 	"context"
 	"net"
+	"net/http"
 
 	"github.com/zeebo/errs"
 	"go.uber.org/zap"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
 
+	"storj.io/storj/bootstrap/bootstrapweb"
+	"storj.io/storj/bootstrap/bootstrapweb/bootstrapserver"
 	"storj.io/storj/pkg/identity"
 	"storj.io/storj/pkg/kademlia"
 	"storj.io/storj/pkg/pb"
@@ -39,6 +42,8 @@ type Config struct {
 
 	Server   server.Config
 	Kademlia kademlia.Config
+
+	Web bootstrapserver.Config
 }
 
 // Verify verifies whether configuration is consistent and acceptable.
@@ -68,6 +73,13 @@ type Peer struct {
 		Endpoint     *kademlia.Endpoint
 		Inspector    *kademlia.Inspector
 	}
+
+	// Web server with web UI
+	Web struct {
+		Listener net.Listener
+		Service  *bootstrapweb.Service
+		Endpoint *bootstrapserver.Server
+	}
 }
 
 // New creates a new Bootstrap Node.
@@ -142,6 +154,31 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB, config Config) (*P
 		pb.RegisterKadInspectorServer(peer.Public.Server.GRPC(), peer.Kademlia.Inspector)
 	}
 
+	{ // setup bootstrap web ui
+		config := config.Web
+
+		peer.Web.Listener, err = net.Listen("tcp", config.Address)
+		if err != nil {
+			return nil, errs.Combine(err, peer.Close())
+		}
+
+		peer.Web.Service, err = bootstrapweb.NewService(
+			peer.Log.Named("bootstrapWeb:service"),
+			peer.Kademlia.Service,
+		)
+
+		if err != nil {
+			return nil, errs.Combine(err, peer.Close())
+		}
+
+		peer.Web.Endpoint = bootstrapserver.NewServer(
+			peer.Log.Named("bootstrapWeb:endpoint"),
+			config,
+			peer.Web.Service,
+			peer.Web.Listener,
+		)
+	}
+
 	return peer, nil
 }
 
@@ -160,12 +197,15 @@ func (peer *Peer) Run(ctx context.Context) error {
 		peer.Log.Sugar().Infof("Node %s started on %s", peer.Identity.ID, peer.Public.Server.Addr().String())
 		return ignoreCancel(peer.Public.Server.Run(ctx))
 	})
+	group.Go(func() error {
+		return ignoreCancel(peer.Web.Endpoint.Run(ctx))
+	})
 
 	return group.Wait()
 }
 
 func ignoreCancel(err error) error {
-	if err == context.Canceled || err == grpc.ErrServerStopped {
+	if err == context.Canceled || err == grpc.ErrServerStopped || err == http.ErrServerClosed {
 		return nil
 	}
 	return err
@@ -187,6 +227,14 @@ func (peer *Peer) Close() error {
 		}
 	}
 
+	if peer.Web.Endpoint != nil {
+		errlist.Add(peer.Web.Endpoint.Close())
+	} else {
+		if peer.Web.Listener != nil {
+			errlist.Add(peer.Web.Listener.Close())
+		}
+	}
+
 	// close services in reverse initialization order
 	if peer.Kademlia.Service != nil {
 		errlist.Add(peer.Kademlia.Service.Close())

cmd/storj-sim/network.go

@@ -112,13 +112,14 @@ func newNetwork(flags *Flags) (*Processes, error) {
 
 	processes := NewProcesses()
 	var (
-		configDir       = flags.Directory
-		host            = flags.Host
-		gatewayPort     = 9000
-		bootstrapPort   = 9999
-		satellitePort   = 10000
-		storageNodePort = 11000
-		consolePort     = 10100
+		configDir        = flags.Directory
+		host             = flags.Host
+		gatewayPort      = 9000
+		bootstrapPort    = 9999
+		satellitePort    = 10000
+		storageNodePort  = 11000
+		consolePort      = 10100
+		bootstrapWebPort = 10010
 	)
 
 	bootstrap := processes.New(Info{
@@ -131,6 +132,9 @@ func newNetwork(flags *Flags) (*Processes, error) {
 	bootstrap.Arguments = withCommon(Arguments{
 		"setup": {
 			"--identity-dir", bootstrap.Directory,
+
+			"--web.address", net.JoinHostPort(host, strconv.Itoa(bootstrapWebPort)),
+
 			"--server.address", bootstrap.Address,
 
 			"--kademlia.bootstrap-addr", bootstrap.Address,

internal/storjql/schema.go

@@ -0,0 +1,19 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package storjql
+
+import (
+	"sync"
+)
+
+// mu allows to lock graphql methods, because some of them are not thread-safe
+var mu sync.Mutex
+
+// WithLock locks graphql methods, because some of them are not thread-safe
+func WithLock(fn func()) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	fn()
+}

internal/testplanet/planet.go

@@ -25,6 +25,7 @@ import (
 
 	"storj.io/storj/bootstrap"
 	"storj.io/storj/bootstrap/bootstrapdb"
+	"storj.io/storj/bootstrap/bootstrapweb/bootstrapserver"
 	"storj.io/storj/internal/memory"
 	"storj.io/storj/pkg/accounting/rollup"
 	"storj.io/storj/pkg/accounting/tally"
@@ -619,6 +620,10 @@ func (planet *Planet) newBootstrap() (peer *bootstrap.Peer, err error) {
 				Wallet: "0x" + strings.Repeat("00", 20),
 			},
 		},
+		Web: bootstrapserver.Config{
+			Address:   "127.0.0.1:0",
+			StaticDir: "./web/bootstrap", // TODO: for development only
+		},
 	}
 	if planet.config.Reconfigure.Bootstrap != nil {
 		planet.config.Reconfigure.Bootstrap(0, &config)

satellite/console/consoleweb/consoleql/schema.go

@@ -1,33 +1,31 @@
-// Copyright (C) 2019 Storj Labs, Inc.
+// Copyright (C) 2018 Storj Labs, Inc.
 // See LICENSE for copying information.
 
 package consoleql
 
 import (
-	"sync"
-
 	"github.com/graphql-go/graphql"
 
+	"storj.io/storj/internal/storjql"
 	"storj.io/storj/satellite/console"
 	"storj.io/storj/satellite/mailservice"
 )
 
-// creatingSchemaMutex locks graphql.NewSchema method because it's not thread-safe
-var creatingSchemaMutex sync.Mutex
+// CreateSchema creates a schema for satellites console graphql api
+func CreateSchema(service *console.Service, mailService *mailservice.Service) (schema graphql.Schema, err error) {
+	storjql.WithLock(func() {
+		creator := TypeCreator{}
 
-// CreateSchema creates both type
-func CreateSchema(service *console.Service, mailService *mailservice.Service) (graphql.Schema, error) {
-	creatingSchemaMutex.Lock()
-	defer creatingSchemaMutex.Unlock()
+		err = creator.Create(service, mailService)
+		if err != nil {
+			return
+		}
 
-	creator := TypeCreator{}
-	err := creator.Create(service, mailService)
-	if err != nil {
-		return graphql.Schema{}, err
-	}
-
-	return graphql.NewSchema(graphql.SchemaConfig{
-		Query:    creator.RootQuery(),
-		Mutation: creator.RootMutation(),
+		schema, err = graphql.NewSchema(graphql.SchemaConfig{
+			Query:    creator.RootQuery(),
+			Mutation: creator.RootMutation(),
+		})
 	})
+
+	return schema, err
 }

satellite/console/consoleweb/server.go

@@ -136,6 +136,7 @@ func (s *Server) grapqlHandler(w http.ResponseWriter, req *http.Request) {
 // Run starts the server that host webapp and api endpoint
 func (s *Server) Run(ctx context.Context) error {
 	var err error
+
 	s.schema, err = consoleql.CreateSchema(s.service, s.mailService)
 	if err != nil {
 		return Error.Wrap(err)

satellite/peer.go

@@ -532,8 +532,8 @@ func (peer *Peer) Close() error {
 	if peer.Console.Endpoint != nil {
 		errlist.Add(peer.Console.Endpoint.Close())
 	} else {
-		if peer.Console.Endpoint != nil {
-			errlist.Add(peer.Public.Listener.Close())
+		if peer.Console.Listener != nil {
+			errlist.Add(peer.Console.Listener.Close())
 		}
 	}
 

web/satellite/index.html

@@ -8,5 +8,6 @@
   <body style="margin: 0px !important; height: 100vh; zoom: 100%">
     <div id="app"></div>
     <!-- built files will be auto injected -->
+    <script type="text/javascript" src="/static/dist/build.js"></script>
   </body>
 </html>