cmd/storj-sim: make initial provisioning nicer

the old code to eventually create an api key on a satellite had
some issues. namely, there were some ignored errors, swallowed
errors, incorrect returns of nil errors when there should have
been an error, and it did not handle working against an already
existing database.

this commit fixes the above issues and organizes the code into
a set of methods performing individual steps rather than one big
function. it adds retries and attempts to get existing values
instead of creating them when possible, which means that it will
work if the values already exist. additionally, it removes the
3 second sleep in favor of a bounded retry loop with a small sleep
which improves startup times.

Change-Id: I4de04659e5a62dd3f675fbf3c76f3311c410a03e

cmd/storj-sim/console.go

@@ -4,11 +4,14 @@
 package main
 
 import (
+	"bufio"
 	"bytes"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
 	"time"
 
@@ -18,14 +21,48 @@ import (
 	"storj.io/storj/satellite/console/consoleauth"
 )
 
-func graphqlDo(client *http.Client, request *http.Request, jsonResponse interface{}) error {
-	resp, err := client.Do(request)
+func newConsoleEndpoints(address string) *consoleEndpoints {
+	return &consoleEndpoints{
+		client: http.DefaultClient,
+		base:   "http://" + address,
+	}
+}
+
+type consoleEndpoints struct {
+	client *http.Client
+	base   string
+}
+
+func (ce *consoleEndpoints) appendPath(suffix string) string {
+	return ce.base + suffix
+}
+
+func (ce *consoleEndpoints) RegToken() string {
+	return ce.appendPath("/registrationToken/?projectsLimit=1")
+}
+
+func (ce *consoleEndpoints) Register() string {
+	return ce.appendPath("/api/v0/auth/register")
+}
+
+func (ce *consoleEndpoints) Activation(token string) string {
+	return ce.appendPath("/activation/?token=" + token)
+}
+
+func (ce *consoleEndpoints) Token() string {
+	return ce.appendPath("/api/v0/auth/token")
+}
+
+func (ce *consoleEndpoints) GraphQL() string {
+	return ce.appendPath("/api/v0/graphql")
+}
+
+func (ce *consoleEndpoints) graphqlDo(request *http.Request, jsonResponse interface{}) error {
+	resp, err := ce.client.Do(request)
 	if err != nil {
 		return err
 	}
-	defer func() {
-		err = resp.Body.Close()
-	}()
+	defer func() { err = errs.Combine(err, resp.Body.Close()) }()
 
 	b, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
@@ -42,16 +79,312 @@ func graphqlDo(client *http.Client, request *http.Request, jsonResponse interfac
 	}
 
 	if response.Errors != nil {
-		return errs.New("inner graphql error")
+		return errs.New("inner graphql error: %v", response.Errors)
 	}
 
 	if jsonResponse == nil {
-		return nil
+		return errs.New("empty response: %q", b)
 	}
 
 	return json.NewDecoder(bytes.NewReader(response.Data)).Decode(jsonResponse)
 }
 
+func (ce *consoleEndpoints) createOrGetAPIKey() (string, error) {
+	authToken, err := ce.tryLogin()
+	if err != nil {
+		_ = ce.tryCreateAndActivateUser()
+		authToken, err = ce.tryLogin()
+		if err != nil {
+			return "", errs.Wrap(err)
+		}
+	}
+
+	projectID, err := ce.getOrCreateProject(authToken)
+	if err != nil {
+		return "", errs.Wrap(err)
+	}
+
+	apiKey, err := ce.createAPIKey(authToken, projectID)
+	if err != nil {
+		return "", errs.Wrap(err)
+	}
+
+	return apiKey, nil
+}
+
+func (ce *consoleEndpoints) tryLogin() (string, error) {
+	var authToken struct {
+		Email    string `json:"email"`
+		Password string `json:"password"`
+	}
+
+	authToken.Email = "alice@mail.test"
+	authToken.Password = "123a123"
+
+	res, err := json.Marshal(authToken)
+	if err != nil {
+		return "", errs.Wrap(err)
+	}
+
+	request, err := http.NewRequest(
+		http.MethodPost,
+		ce.Token(),
+		bytes.NewReader(res))
+	if err != nil {
+		return "", errs.Wrap(err)
+	}
+
+	request.Header.Add("Content-Type", "application/json")
+
+	resp, err := ce.client.Do(request)
+	if err != nil {
+		return "", errs.Wrap(err)
+	}
+	defer func() { err = errs.Combine(err, resp.Body.Close()) }()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", errs.New("unexpected status code: %d (%q)",
+			resp.StatusCode, tryReadLine(resp.Body))
+	}
+
+	var token string
+	err = json.NewDecoder(resp.Body).Decode(&token)
+	if err != nil {
+		return "", errs.Wrap(err)
+	}
+
+	return token, nil
+}
+
+func (ce *consoleEndpoints) tryCreateAndActivateUser() error {
+	regToken, err := ce.createRegistrationToken()
+	if err != nil {
+		return errs.Wrap(err)
+	}
+	userID, err := ce.createUser(regToken)
+	if err != nil {
+		return errs.Wrap(err)
+	}
+	return errs.Wrap(ce.activateUser(userID))
+}
+
+func (ce *consoleEndpoints) createRegistrationToken() (string, error) {
+	request, err := http.NewRequest(
+		http.MethodGet,
+		ce.RegToken(),
+		nil)
+	if err != nil {
+		return "", errs.Wrap(err)
+	}
+	request.Header.Set("Authorization", "secure_token")
+
+	resp, err := ce.client.Do(request)
+	if err != nil {
+		return "", errs.Wrap(err)
+	}
+	defer func() { err = errs.Combine(err, resp.Body.Close()) }()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", errs.New("unexpected status code: %d (%q)",
+			resp.StatusCode, tryReadLine(resp.Body))
+	}
+
+	var createTokenResponse struct {
+		Secret string
+		Error  string
+	}
+	if err = json.NewDecoder(resp.Body).Decode(&createTokenResponse); err != nil {
+		return "", errs.Wrap(err)
+	}
+	if createTokenResponse.Error != "" {
+		return "", errs.New("unable to create registration token: %s", createTokenResponse.Error)
+	}
+
+	return createTokenResponse.Secret, nil
+}
+
+func (ce *consoleEndpoints) createUser(regToken string) (string, error) {
+	var registerData struct {
+		FullName  string `json:"fullName"`
+		ShortName string `json:"shortName"`
+		Email     string `json:"email"`
+		Password  string `json:"password"`
+		Secret    string `json:"secret"`
+	}
+
+	registerData.FullName = "Alice"
+	registerData.Email = "alice@mail.test"
+	registerData.Password = "123a123"
+	registerData.ShortName = "al"
+	registerData.Secret = regToken
+
+	res, err := json.Marshal(registerData)
+	if err != nil {
+		return "", errs.Wrap(err)
+	}
+
+	request, err := http.NewRequest(
+		http.MethodPost,
+		ce.Register(),
+		bytes.NewReader(res))
+	if err != nil {
+		return "", errs.Wrap(err)
+	}
+	request.Header.Add("Content-Type", "application/json")
+
+	resp, err := ce.client.Do(request)
+	if err != nil {
+		return "", errs.Wrap(err)
+	}
+	defer func() { err = errs.Combine(err, resp.Body.Close()) }()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", errs.New("unexpected status code: %d (%q)",
+			resp.StatusCode, tryReadLine(resp.Body))
+	}
+
+	var userID string
+	if err = json.NewDecoder(resp.Body).Decode(&userID); err != nil {
+		return "", errs.Wrap(err)
+	}
+
+	return userID, nil
+}
+
+func (ce *consoleEndpoints) activateUser(userID string) error {
+	userUUID, err := uuid.Parse(userID)
+	if err != nil {
+		return errs.Wrap(err)
+	}
+
+	activationToken, err := generateActivationKey(*userUUID, "alice@mail.test", time.Now())
+	if err != nil {
+		return err
+	}
+
+	request, err := http.NewRequest(
+		http.MethodGet,
+		ce.Activation(activationToken),
+		nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := ce.client.Do(request)
+	if err != nil {
+		return err
+	}
+	defer func() { err = errs.Combine(err, resp.Body.Close()) }()
+
+	if resp.StatusCode != http.StatusOK {
+		return errs.New("unexpected status code: %d (%q)",
+			resp.StatusCode, tryReadLine(resp.Body))
+	}
+
+	return nil
+}
+
+func (ce *consoleEndpoints) getOrCreateProject(token string) (string, error) {
+	projectID, err := ce.getProject(token)
+	if err == nil {
+		return projectID, nil
+	}
+	projectID, err = ce.createProject(token)
+	if err == nil {
+		return projectID, nil
+	}
+	return ce.getProject(token)
+}
+
+func (ce *consoleEndpoints) getProject(token string) (string, error) {
+	request, err := http.NewRequest(
+		http.MethodGet,
+		ce.GraphQL(),
+		nil)
+	if err != nil {
+		return "", errs.Wrap(err)
+	}
+
+	q := request.URL.Query()
+	q.Add("query", `query {myProjects{id}}`)
+	request.URL.RawQuery = q.Encode()
+
+	request.Header.Add("Content-Type", "application/graphql")
+	request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
+
+	var getProjects struct {
+		MyProjects []struct {
+			ID string
+		}
+	}
+	if err := ce.graphqlDo(request, &getProjects); err != nil {
+		return "", errs.Wrap(err)
+	}
+	if len(getProjects.MyProjects) == 0 {
+		return "", errs.New("no projects")
+	}
+
+	return getProjects.MyProjects[0].ID, nil
+}
+
+func (ce *consoleEndpoints) createProject(token string) (string, error) {
+	rng := rand.NewSource(time.Now().UnixNano())
+	createProjectQuery := fmt.Sprintf(
+		`mutation {createProject(input:{name:"TestProject-%d",description:""}){id}}`,
+		rng.Int63())
+
+	request, err := http.NewRequest(
+		http.MethodPost,
+		ce.GraphQL(),
+		bytes.NewReader([]byte(createProjectQuery)))
+	if err != nil {
+		return "", errs.Wrap(err)
+	}
+
+	request.Header.Add("Content-Type", "application/graphql")
+	request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
+
+	var createProject struct {
+		CreateProject struct {
+			ID string
+		}
+	}
+	if err := ce.graphqlDo(request, &createProject); err != nil {
+		return "", errs.Wrap(err)
+	}
+
+	return createProject.CreateProject.ID, nil
+}
+
+func (ce *consoleEndpoints) createAPIKey(token, projectID string) (string, error) {
+	rng := rand.NewSource(time.Now().UnixNano())
+	createAPIKeyQuery := fmt.Sprintf(
+		`mutation {createAPIKey(projectID:%q,name:"TestKey-%d"){key}}`,
+		projectID, rng.Int63())
+
+	request, err := http.NewRequest(
+		http.MethodPost,
+		ce.GraphQL(),
+		bytes.NewReader([]byte(createAPIKeyQuery)))
+	if err != nil {
+		return "", errs.Wrap(err)
+	}
+
+	request.Header.Add("Content-Type", "application/graphql")
+	request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
+
+	var createAPIKey struct {
+		CreateAPIKey struct {
+			Key string
+		}
+	}
+	if err := ce.graphqlDo(request, &createAPIKey); err != nil {
+		return "", errs.Wrap(err)
+	}
+
+	return createAPIKey.CreateAPIKey.Key, nil
+}
+
 func generateActivationKey(userID uuid.UUID, email string, createdAt time.Time) (string, error) {
 	claims := consoleauth.Claims{
 		ID:         userID,
@@ -80,221 +413,8 @@ func generateActivationKey(userID uuid.UUID, email string, createdAt time.Time)
 	return token.String(), nil
 }
 
-func addExampleProjectWithKey(key *string, endpoints map[string]string) error {
-	client := http.Client{}
-
-	var createTokenResponse struct {
-		Secret string
-		Error  string
-	}
-	{
-		request, err := http.NewRequest(
-			http.MethodGet,
-			endpoints["regtoken"],
-			nil)
-		if err != nil {
-			return err
-		}
-		request.Header.Set("Authorization", "secure_token")
-
-		resp, err := client.Do(request)
-		if err != nil {
-			return err
-		}
-
-		b, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			return err
-		}
-
-		if err = json.NewDecoder(bytes.NewReader(b)).Decode(&createTokenResponse); err != nil {
-			return err
-		}
-
-		if createTokenResponse.Error != "" {
-			return errs.New(createTokenResponse.Error)
-		}
-
-		err = resp.Body.Close()
-		if err != nil {
-			return err
-		}
-	}
-
-	// create user
-	var user struct {
-		CreateUser struct {
-			Email     string
-			CreatedAt time.Time
-			ID        string
-		}
-	}
-	{
-		var registerData struct {
-			FullName    string `json:"fullName"`
-			ShortName   string `json:"shortName"`
-			Email       string `json:"email"`
-			Password    string `json:"password"`
-			SecretInput string `json:"secret"`
-		}
-
-		registerData.FullName = "Alice"
-		registerData.Email = "alice@mail.test"
-		registerData.Password = "123a123"
-		registerData.ShortName = "al"
-		registerData.SecretInput = createTokenResponse.Secret
-
-		res, _ := json.Marshal(registerData)
-
-		request, err := http.NewRequest(
-			http.MethodPost,
-			endpoints["register"],
-			bytes.NewReader(res))
-		if err != nil {
-			return err
-		}
-		request.Header.Add("Content-Type", "application/json")
-
-		response, err := client.Do(request)
-		if err != nil {
-			return err
-		}
-
-		defer func() { err = errs.Combine(err, response.Body.Close()) }()
-
-		if response.StatusCode != http.StatusOK {
-			return err
-		}
-
-		err = json.NewDecoder(response.Body).Decode(&user.CreateUser.ID)
-		if err != nil {
-			return err
-		}
-
-		user.CreateUser.Email = registerData.Email
-		user.CreateUser.CreatedAt = time.Now()
-	}
-
-	var token string
-	{
-		userID, err := uuid.Parse(user.CreateUser.ID)
-		if err != nil {
-			return err
-		}
-
-		activationToken, err := generateActivationKey(*userID, user.CreateUser.Email, user.CreateUser.CreatedAt)
-		if err != nil {
-			return err
-		}
-
-		request, err := http.NewRequest(
-			http.MethodGet,
-			endpoints["activation"]+activationToken,
-			nil)
-		if err != nil {
-			return err
-		}
-
-		resp, err := client.Do(request)
-		if err != nil {
-			return err
-		}
-
-		defer func() { err = errs.Combine(err, resp.Body.Close()) }()
-
-		var authToken struct {
-			Email    string `json:"email"`
-			Password string `json:"password"`
-		}
-
-		authToken.Email = "alice@mail.test"
-		authToken.Password = "123a123"
-
-		res, _ := json.Marshal(authToken)
-
-		request, err = http.NewRequest(
-			http.MethodPost,
-			endpoints["token"],
-			bytes.NewReader(res))
-		if err != nil {
-			return err
-		}
-
-		request.Header.Add("Content-Type", "application/json")
-
-		response, err := client.Do(request)
-		if err != nil {
-			return err
-		}
-
-		defer func() { err = errs.Combine(err, response.Body.Close()) }()
-
-		if response.StatusCode != http.StatusOK {
-			return err
-		}
-
-		err = json.NewDecoder(response.Body).Decode(&token)
-		if err != nil {
-			return err
-		}
-	}
-
-	// create project
-	var createProject struct {
-		CreateProject struct {
-			ID string
-		}
-	}
-	{
-		createProjectQuery := fmt.Sprintf(
-			"mutation {createProject(input:{name:\"%s\",description:\"\"}){id}}",
-			"TestProject")
-
-		request, err := http.NewRequest(
-			http.MethodPost,
-			endpoints["graphql"],
-			bytes.NewReader([]byte(createProjectQuery)))
-		if err != nil {
-			return err
-		}
-
-		request.Header.Add("Content-Type", "application/graphql")
-		request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
-
-		if err := graphqlDo(&client, request, &createProject); err != nil {
-			return err
-		}
-	}
-
-	// create api key
-	var createAPIKey struct {
-		CreateAPIKey struct {
-			Key string
-		}
-	}
-	{
-		createAPIKeyQuery := fmt.Sprintf(
-			"mutation {createAPIKey(projectID:\"%s\",name:\"%s\"){key}}",
-			createProject.CreateProject.ID,
-			"testKey")
-
-		request, err := http.NewRequest(
-			http.MethodPost,
-			endpoints["graphql"],
-			bytes.NewReader([]byte(createAPIKeyQuery)))
-		if err != nil {
-			return err
-		}
-
-		request.Header.Add("Content-Type", "application/graphql")
-		request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
-
-		if err := graphqlDo(&client, request, &createAPIKey); err != nil {
-			return err
-		}
-	}
-
-	// return key to the caller
-	*key = createAPIKey.CreateAPIKey.Key
-	return nil
+func tryReadLine(r io.Reader) string {
+	scanner := bufio.NewScanner(r)
+	scanner.Scan()
+	return scanner.Text()
 }

cmd/storj-sim/network.go

@@ -442,8 +442,8 @@ func newNetwork(flags *Flags) (*Processes, error) {
 			"run": {},
 		})
 
-		process.ExecBefore["run"] = func(process *Process) error {
-			err := readConfigString(&process.Address, process.Directory, "server.address")
+		process.ExecBefore["run"] = func(process *Process) (err error) {
+			err = readConfigString(&process.Address, process.Directory, "server.address")
 			if err != nil {
 				return err
 			}
@@ -466,22 +466,14 @@ func newNetwork(flags *Flags) (*Processes, error) {
 					return err
 				}
 
-				host := "http://" + consoleAddress
-
-				endpoints := map[string]string{
-					"regtoken":   host + "/registrationToken/?projectsLimit=1",
-					"register":   host + "/api/v0/auth/register",
-					"activation": host + "/activation/?token=",
-					"token":      host + "/api/v0/auth/token",
-					"graphql":    host + "/api/v0/graphql",
-				}
-
-				// wait for console server to start
-				time.Sleep(3 * time.Second)
-
-				var apiKey string
-				if err := addExampleProjectWithKey(&apiKey, endpoints); err != nil {
-					return err
+				// try with 100ms delays until we hit 3s
+				apiKey, start := "", time.Now()
+				for apiKey == "" {
+					apiKey, err = newConsoleEndpoints(consoleAddress).createOrGetAPIKey()
+					if err != nil && time.Since(start) > 3*time.Second {
+						return err
+					}
+					time.Sleep(100 * time.Millisecond)
 				}
 
 				scope, err := uplink.ParseScope(runScopeData)