storj/cmd/storj-sim/network.go

472 lines
13 KiB
Go
Raw Normal View History

2019-01-24 20:15:10 +00:00
// Copyright (C) 2019 Storj Labs, Inc.
2019-01-02 18:07:49 +00:00
// See LICENSE for copying information.
package main
import (
"context"
"errors"
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
"runtime"
"sort"
2019-01-02 18:07:49 +00:00
"strconv"
"strings"
"time"
2019-01-02 18:07:49 +00:00
"github.com/spf13/viper"
2019-01-02 18:07:49 +00:00
"github.com/zeebo/errs"
"golang.org/x/sync/errgroup"
"storj.io/storj/internal/fpath"
"storj.io/storj/internal/processgroup"
)
const folderPermissions = 0744
func networkExec(flags *Flags, args []string, command string) error {
2019-01-08 15:24:15 +00:00
processes, err := newNetwork(flags)
2019-01-02 18:07:49 +00:00
if err != nil {
return err
}
ctx, cancel := NewCLIContext(context.Background())
defer cancel()
if command == "setup" {
identities, err := identitySetup(processes)
if err != nil {
return err
}
err = identities.Exec(ctx, command)
if err != nil {
return err
}
}
2019-01-02 18:07:49 +00:00
err = processes.Exec(ctx, command)
closeErr := processes.Close()
return errs.Combine(err, closeErr)
}
func networkEnv(flags *Flags, args []string) error {
flags.OnlyEnv = true
processes, err := newNetwork(flags)
if err != nil {
return err
}
// run exec before, since it will load env vars from configs
for _, process := range processes.List {
if exec := process.ExecBefore["run"]; exec != nil {
if err := exec(process); err != nil {
return err
}
}
}
if len(args) == 1 {
envprefix := strings.ToUpper(args[0] + "=")
// find the environment value that the environment variable is set to
for _, env := range processes.Env() {
if strings.HasPrefix(strings.ToUpper(env), envprefix) {
fmt.Println(env[len(envprefix):])
return nil
}
}
return nil
}
for _, env := range processes.Env() {
fmt.Println(env)
}
return nil
}
2019-01-02 18:07:49 +00:00
func networkTest(flags *Flags, command string, args []string) error {
2019-01-08 15:24:15 +00:00
processes, err := newNetwork(flags)
2019-01-02 18:07:49 +00:00
if err != nil {
return err
}
ctx, cancel := NewCLIContext(context.Background())
var group errgroup.Group
processes.Start(ctx, &group, "run")
2019-01-08 15:24:15 +00:00
for _, process := range processes.List {
process.Status.Started.Wait()
}
2019-01-02 18:07:49 +00:00
cmd := exec.CommandContext(ctx, command, args...)
cmd.Env = append(os.Environ(), processes.Env()...)
stdout := processes.Output.Prefixed("test:out")
stderr := processes.Output.Prefixed("test:err")
cmd.Stdout, cmd.Stderr = stdout, stderr
processgroup.Setup(cmd)
if printCommands {
fmt.Fprintf(processes.Output, "exec: %v\n", strings.Join(cmd.Args, " "))
}
errRun := cmd.Run()
cancel()
return errs.Combine(errRun, processes.Close(), group.Wait())
}
func networkDestroy(flags *Flags, args []string) error {
if fpath.IsRoot(flags.Directory) {
return errors.New("safety check: disallowed to remove root directory " + flags.Directory)
}
if printCommands {
2019-01-16 23:09:57 +00:00
fmt.Println("sim | exec: rm -rf", flags.Directory)
2019-01-02 18:07:49 +00:00
}
return os.RemoveAll(flags.Directory)
}
// newNetwork creates a default network
2019-01-08 15:24:15 +00:00
func newNetwork(flags *Flags) (*Processes, error) {
// with common adds all common arguments to the process
2019-03-19 09:10:23 +00:00
withCommon := func(dir string, all Arguments) Arguments {
common := []string{"--metrics.app-suffix", "sim", "--log.level", "debug", "--config-dir", dir}
if flags.IsDev {
common = append(common, "--dev")
}
2019-01-08 15:24:15 +00:00
for command, args := range all {
all[command] = append(append(common, command), args...)
2019-01-08 15:24:15 +00:00
}
return all
}
2019-01-02 18:07:49 +00:00
2019-03-19 09:10:23 +00:00
processes := NewProcesses(flags.Directory)
2019-01-08 15:24:15 +00:00
var (
host = flags.Host
gatewayPort = 9000
bootstrapPort = 9999
bootstrapPrivatePort = 9988
satellitePort = 10000
satellitePrivatePort = 11000
storageNodePort = 12000
storageNodePrivatePort = 13000
consolePort = 10100
bootstrapWebPort = 10010
Add Versioning Server (#1576) * Initial Webserver Draft for Version Controlling * Rename type to avoid confusion * Move Function Calls into Version Package * Fix Linting and Language Typos * Fix Linting and Spelling Mistakes * Include Copyright * Include Copyright * Adjust Version-Control Server to return list of Versions * Linting * Improve Request Handling and Readability * Add Configuration File Option Add Systemd Service file * Add Logging to File * Smaller Changes * Add Semantic Versioning and refuses outdated Software from Startup (#1612) * implements internal Semantic Version library * adds version logging + reporting to process * Advance SemVer struct for easier handling * Add Accepted Version Store * Fix Function * Restructure * Type Conversion * Handle Version String properly * Add Note about array index * Set temporary Default Version * Add Copyright * Adding Version to Dashboard * Adding Version Info Log * Renaming and adding CheckerProcess * Iteration Sync * Iteration V2 * linting * made LogAndReportVersion a go routine * Refactor to Go Routine * Add Context to Go Routine and allow Operation if Lookup to Control Server fails * Handle Unmarshal properly * Linting * Relocate Version Checks * Relocating Version Check and specified default Version for now * Linting Error Prevention * Refuse Startup on outdated Version * Add Startup Check Function * Straighten Logging * Dont force Shutdown if --dev flag is set * Create full Service/Peer Structure for ControlServer * Linting * Straighting Naming * Finish VersionControl Service Layout * Improve Error Handling * Change Listening Address * Move Checker Function * Remove VersionControl Peer * Linting * Linting * Create VersionClient Service * Renaming * Add Version Client to Peer Definitions * Linting and Renaming * Linting * Remove Transport Checks for now * Move to Client Side Flag * Remove check * Linting * Transport Client Version Intro * Adding Version Client to Transport Client * Add missing parameter * Adding Version Check, to set Allowed = true * Set Default to true, testing * Restructuring Code * Uplink Changes * Add more proper Defaults * Renaming of Version struct * Dont pass Service use Pointer * Set Defaults for Versioning Checks * Put HTTP Server in go routine * Add Versioncontrol to Storj-Sim * Testplanet Fixes * Linting * Add Error Handling and new Server Struct * Move Lock slightly * Reduce Race Potentials * Remove unnecessary files * Linting * Add Proper Transport Handling * small fixes * add fence for allowed check * Add Startup Version Check and Service Naming * make errormessage private * Add Comments about VersionedClient * Linting * Remove Checks that refuse outgoing connections * Remove release cmd * Add Release Script * Linting * Update to use correct Values * Move vars private and set minimum default versions for testing builds * Remove VersionedClient * Better Error Handling and naked return removal * Straighten the Regex and string conversion * Change Check to allows testplanet and storj-sim to run without the need to pass an LDFlag * Cosmetic Change to Dashboard * Cleanup Returns and remove commented code * Remove Version Check if no build options are passed in * Pass in Config Values instead of Pointers * Handle missed Error * Update Endpoint URL * Change Type of Release Flag * Add additional Logging * Remove Versions Logging of other Services * minor fixes Change-Id: I5cc04a410ea6b2008d14dffd63eb5f36dd348a8b
2019-04-03 20:13:39 +01:00
versioncontrolPort = 10011
2019-01-02 18:07:49 +00:00
)
Add Versioning Server (#1576) * Initial Webserver Draft for Version Controlling * Rename type to avoid confusion * Move Function Calls into Version Package * Fix Linting and Language Typos * Fix Linting and Spelling Mistakes * Include Copyright * Include Copyright * Adjust Version-Control Server to return list of Versions * Linting * Improve Request Handling and Readability * Add Configuration File Option Add Systemd Service file * Add Logging to File * Smaller Changes * Add Semantic Versioning and refuses outdated Software from Startup (#1612) * implements internal Semantic Version library * adds version logging + reporting to process * Advance SemVer struct for easier handling * Add Accepted Version Store * Fix Function * Restructure * Type Conversion * Handle Version String properly * Add Note about array index * Set temporary Default Version * Add Copyright * Adding Version to Dashboard * Adding Version Info Log * Renaming and adding CheckerProcess * Iteration Sync * Iteration V2 * linting * made LogAndReportVersion a go routine * Refactor to Go Routine * Add Context to Go Routine and allow Operation if Lookup to Control Server fails * Handle Unmarshal properly * Linting * Relocate Version Checks * Relocating Version Check and specified default Version for now * Linting Error Prevention * Refuse Startup on outdated Version * Add Startup Check Function * Straighten Logging * Dont force Shutdown if --dev flag is set * Create full Service/Peer Structure for ControlServer * Linting * Straighting Naming * Finish VersionControl Service Layout * Improve Error Handling * Change Listening Address * Move Checker Function * Remove VersionControl Peer * Linting * Linting * Create VersionClient Service * Renaming * Add Version Client to Peer Definitions * Linting and Renaming * Linting * Remove Transport Checks for now * Move to Client Side Flag * Remove check * Linting * Transport Client Version Intro * Adding Version Client to Transport Client * Add missing parameter * Adding Version Check, to set Allowed = true * Set Default to true, testing * Restructuring Code * Uplink Changes * Add more proper Defaults * Renaming of Version struct * Dont pass Service use Pointer * Set Defaults for Versioning Checks * Put HTTP Server in go routine * Add Versioncontrol to Storj-Sim * Testplanet Fixes * Linting * Add Error Handling and new Server Struct * Move Lock slightly * Reduce Race Potentials * Remove unnecessary files * Linting * Add Proper Transport Handling * small fixes * add fence for allowed check * Add Startup Version Check and Service Naming * make errormessage private * Add Comments about VersionedClient * Linting * Remove Checks that refuse outgoing connections * Remove release cmd * Add Release Script * Linting * Update to use correct Values * Move vars private and set minimum default versions for testing builds * Remove VersionedClient * Better Error Handling and naked return removal * Straighten the Regex and string conversion * Change Check to allows testplanet and storj-sim to run without the need to pass an LDFlag * Cosmetic Change to Dashboard * Cleanup Returns and remove commented code * Remove Version Check if no build options are passed in * Pass in Config Values instead of Pointers * Handle missed Error * Update Endpoint URL * Change Type of Release Flag * Add additional Logging * Remove Versions Logging of other Services * minor fixes Change-Id: I5cc04a410ea6b2008d14dffd63eb5f36dd348a8b
2019-04-03 20:13:39 +01:00
versioncontrol := processes.New(Info{
Name: "versioncontrol/0",
Executable: "versioncontrol",
Directory: filepath.Join(processes.Directory, "versioncontrol", "0"),
Address: net.JoinHostPort(host, strconv.Itoa(versioncontrolPort)),
})
versioncontrol.Arguments = withCommon(versioncontrol.Directory, Arguments{
"setup": {
"--address", versioncontrol.Address,
},
"run": {},
})
versioncontrol.ExecBefore["run"] = func(process *Process) error {
return readConfigString(&versioncontrol.Address, versioncontrol.Directory, "address")
}
bootstrap := processes.New(Info{
Name: "bootstrap/0",
Executable: "bootstrap",
2019-03-19 09:10:23 +00:00
Directory: filepath.Join(processes.Directory, "bootstrap", "0"),
Address: net.JoinHostPort(host, strconv.Itoa(bootstrapPort)),
})
Add Versioning Server (#1576) * Initial Webserver Draft for Version Controlling * Rename type to avoid confusion * Move Function Calls into Version Package * Fix Linting and Language Typos * Fix Linting and Spelling Mistakes * Include Copyright * Include Copyright * Adjust Version-Control Server to return list of Versions * Linting * Improve Request Handling and Readability * Add Configuration File Option Add Systemd Service file * Add Logging to File * Smaller Changes * Add Semantic Versioning and refuses outdated Software from Startup (#1612) * implements internal Semantic Version library * adds version logging + reporting to process * Advance SemVer struct for easier handling * Add Accepted Version Store * Fix Function * Restructure * Type Conversion * Handle Version String properly * Add Note about array index * Set temporary Default Version * Add Copyright * Adding Version to Dashboard * Adding Version Info Log * Renaming and adding CheckerProcess * Iteration Sync * Iteration V2 * linting * made LogAndReportVersion a go routine * Refactor to Go Routine * Add Context to Go Routine and allow Operation if Lookup to Control Server fails * Handle Unmarshal properly * Linting * Relocate Version Checks * Relocating Version Check and specified default Version for now * Linting Error Prevention * Refuse Startup on outdated Version * Add Startup Check Function * Straighten Logging * Dont force Shutdown if --dev flag is set * Create full Service/Peer Structure for ControlServer * Linting * Straighting Naming * Finish VersionControl Service Layout * Improve Error Handling * Change Listening Address * Move Checker Function * Remove VersionControl Peer * Linting * Linting * Create VersionClient Service * Renaming * Add Version Client to Peer Definitions * Linting and Renaming * Linting * Remove Transport Checks for now * Move to Client Side Flag * Remove check * Linting * Transport Client Version Intro * Adding Version Client to Transport Client * Add missing parameter * Adding Version Check, to set Allowed = true * Set Default to true, testing * Restructuring Code * Uplink Changes * Add more proper Defaults * Renaming of Version struct * Dont pass Service use Pointer * Set Defaults for Versioning Checks * Put HTTP Server in go routine * Add Versioncontrol to Storj-Sim * Testplanet Fixes * Linting * Add Error Handling and new Server Struct * Move Lock slightly * Reduce Race Potentials * Remove unnecessary files * Linting * Add Proper Transport Handling * small fixes * add fence for allowed check * Add Startup Version Check and Service Naming * make errormessage private * Add Comments about VersionedClient * Linting * Remove Checks that refuse outgoing connections * Remove release cmd * Add Release Script * Linting * Update to use correct Values * Move vars private and set minimum default versions for testing builds * Remove VersionedClient * Better Error Handling and naked return removal * Straighten the Regex and string conversion * Change Check to allows testplanet and storj-sim to run without the need to pass an LDFlag * Cosmetic Change to Dashboard * Cleanup Returns and remove commented code * Remove Version Check if no build options are passed in * Pass in Config Values instead of Pointers * Handle missed Error * Update Endpoint URL * Change Type of Release Flag * Add additional Logging * Remove Versions Logging of other Services * minor fixes Change-Id: I5cc04a410ea6b2008d14dffd63eb5f36dd348a8b
2019-04-03 20:13:39 +01:00
// gateway must wait for the versioncontrol to start up
bootstrap.WaitForStart(versioncontrol)
2019-03-19 09:10:23 +00:00
bootstrap.Arguments = withCommon(bootstrap.Directory, Arguments{
"setup": {
"--identity-dir", bootstrap.Directory,
"--web.address", net.JoinHostPort(host, strconv.Itoa(bootstrapWebPort)),
2019-01-28 14:48:49 +00:00
"--server.address", bootstrap.Address,
"--server.private-address", net.JoinHostPort(host, strconv.Itoa(bootstrapPrivatePort)),
2019-01-28 14:48:49 +00:00
"--kademlia.bootstrap-addr", bootstrap.Address,
"--kademlia.operator.email", "bootstrap@example.com",
"--kademlia.operator.wallet", "0x0123456789012345678901234567890123456789",
"--server.extensions.revocation=false",
"--server.use-peer-ca-whitelist=false",
Add Versioning Server (#1576) * Initial Webserver Draft for Version Controlling * Rename type to avoid confusion * Move Function Calls into Version Package * Fix Linting and Language Typos * Fix Linting and Spelling Mistakes * Include Copyright * Include Copyright * Adjust Version-Control Server to return list of Versions * Linting * Improve Request Handling and Readability * Add Configuration File Option Add Systemd Service file * Add Logging to File * Smaller Changes * Add Semantic Versioning and refuses outdated Software from Startup (#1612) * implements internal Semantic Version library * adds version logging + reporting to process * Advance SemVer struct for easier handling * Add Accepted Version Store * Fix Function * Restructure * Type Conversion * Handle Version String properly * Add Note about array index * Set temporary Default Version * Add Copyright * Adding Version to Dashboard * Adding Version Info Log * Renaming and adding CheckerProcess * Iteration Sync * Iteration V2 * linting * made LogAndReportVersion a go routine * Refactor to Go Routine * Add Context to Go Routine and allow Operation if Lookup to Control Server fails * Handle Unmarshal properly * Linting * Relocate Version Checks * Relocating Version Check and specified default Version for now * Linting Error Prevention * Refuse Startup on outdated Version * Add Startup Check Function * Straighten Logging * Dont force Shutdown if --dev flag is set * Create full Service/Peer Structure for ControlServer * Linting * Straighting Naming * Finish VersionControl Service Layout * Improve Error Handling * Change Listening Address * Move Checker Function * Remove VersionControl Peer * Linting * Linting * Create VersionClient Service * Renaming * Add Version Client to Peer Definitions * Linting and Renaming * Linting * Remove Transport Checks for now * Move to Client Side Flag * Remove check * Linting * Transport Client Version Intro * Adding Version Client to Transport Client * Add missing parameter * Adding Version Check, to set Allowed = true * Set Default to true, testing * Restructuring Code * Uplink Changes * Add more proper Defaults * Renaming of Version struct * Dont pass Service use Pointer * Set Defaults for Versioning Checks * Put HTTP Server in go routine * Add Versioncontrol to Storj-Sim * Testplanet Fixes * Linting * Add Error Handling and new Server Struct * Move Lock slightly * Reduce Race Potentials * Remove unnecessary files * Linting * Add Proper Transport Handling * small fixes * add fence for allowed check * Add Startup Version Check and Service Naming * make errormessage private * Add Comments about VersionedClient * Linting * Remove Checks that refuse outgoing connections * Remove release cmd * Add Release Script * Linting * Update to use correct Values * Move vars private and set minimum default versions for testing builds * Remove VersionedClient * Better Error Handling and naked return removal * Straighten the Regex and string conversion * Change Check to allows testplanet and storj-sim to run without the need to pass an LDFlag * Cosmetic Change to Dashboard * Cleanup Returns and remove commented code * Remove Version Check if no build options are passed in * Pass in Config Values instead of Pointers * Handle missed Error * Update Endpoint URL * Change Type of Release Flag * Add additional Logging * Remove Versions Logging of other Services * minor fixes Change-Id: I5cc04a410ea6b2008d14dffd63eb5f36dd348a8b
2019-04-03 20:13:39 +01:00
"--version.server-address", fmt.Sprintf("http://%s/", versioncontrol.Address),
},
2019-01-28 14:48:49 +00:00
"run": {},
})
2019-02-06 12:47:00 +00:00
bootstrap.ExecBefore["run"] = func(process *Process) error {
return readConfigString(&bootstrap.Address, bootstrap.Directory, "server.address")
}
2019-01-08 15:24:15 +00:00
2019-01-17 20:59:26 +00:00
// Create satellites making all satellites wait for bootstrap to start
var satellites []*Process
2019-01-08 15:24:15 +00:00
for i := 0; i < flags.SatelliteCount; i++ {
process := processes.New(Info{
Name: fmt.Sprintf("satellite/%d", i),
Executable: "satellite",
2019-03-19 09:10:23 +00:00
Directory: filepath.Join(processes.Directory, "satellite", fmt.Sprint(i)),
2019-01-08 15:24:15 +00:00
Address: net.JoinHostPort(host, strconv.Itoa(satellitePort+i)),
})
satellites = append(satellites, process)
2019-01-08 15:24:15 +00:00
// satellite must wait for bootstrap to start
process.WaitForStart(bootstrap)
2019-01-02 18:07:49 +00:00
// TODO: find source file, to set static path
_, filename, _, ok := runtime.Caller(0)
if !ok {
return nil, errs.Combine(processes.Close(), errs.New("no caller information"))
}
storjRoot := strings.TrimSuffix(filename, "/cmd/storj-sim/network.go")
consoleAuthToken := "secure_token"
2019-03-19 09:10:23 +00:00
process.Arguments = withCommon(process.Directory, Arguments{
"setup": {
"--identity-dir", process.Directory,
2019-01-24 16:26:36 +00:00
"--console.address", net.JoinHostPort(host, strconv.Itoa(consolePort+i)),
"--console.static-dir", filepath.Join(storjRoot, "web/satellite/"),
// TODO: remove console.auth-token after vanguard release
"--console.auth-token", consoleAuthToken,
2019-01-08 15:24:15 +00:00
"--server.address", process.Address,
"--server.private-address", net.JoinHostPort(host, strconv.Itoa(satellitePrivatePort+i)),
2019-01-08 23:41:01 +00:00
2019-01-28 14:48:49 +00:00
"--kademlia.bootstrap-addr", bootstrap.Address,
"--server.extensions.revocation=false",
"--server.use-peer-ca-whitelist=false",
"--mail.smtp-server-address", "smtp.gmail.com:587",
"--mail.from", "Storj <yaroslav-satellite-test@storj.io>",
"--mail.template-path", filepath.Join(storjRoot, "web/satellite/static/emails"),
Add Versioning Server (#1576) * Initial Webserver Draft for Version Controlling * Rename type to avoid confusion * Move Function Calls into Version Package * Fix Linting and Language Typos * Fix Linting and Spelling Mistakes * Include Copyright * Include Copyright * Adjust Version-Control Server to return list of Versions * Linting * Improve Request Handling and Readability * Add Configuration File Option Add Systemd Service file * Add Logging to File * Smaller Changes * Add Semantic Versioning and refuses outdated Software from Startup (#1612) * implements internal Semantic Version library * adds version logging + reporting to process * Advance SemVer struct for easier handling * Add Accepted Version Store * Fix Function * Restructure * Type Conversion * Handle Version String properly * Add Note about array index * Set temporary Default Version * Add Copyright * Adding Version to Dashboard * Adding Version Info Log * Renaming and adding CheckerProcess * Iteration Sync * Iteration V2 * linting * made LogAndReportVersion a go routine * Refactor to Go Routine * Add Context to Go Routine and allow Operation if Lookup to Control Server fails * Handle Unmarshal properly * Linting * Relocate Version Checks * Relocating Version Check and specified default Version for now * Linting Error Prevention * Refuse Startup on outdated Version * Add Startup Check Function * Straighten Logging * Dont force Shutdown if --dev flag is set * Create full Service/Peer Structure for ControlServer * Linting * Straighting Naming * Finish VersionControl Service Layout * Improve Error Handling * Change Listening Address * Move Checker Function * Remove VersionControl Peer * Linting * Linting * Create VersionClient Service * Renaming * Add Version Client to Peer Definitions * Linting and Renaming * Linting * Remove Transport Checks for now * Move to Client Side Flag * Remove check * Linting * Transport Client Version Intro * Adding Version Client to Transport Client * Add missing parameter * Adding Version Check, to set Allowed = true * Set Default to true, testing * Restructuring Code * Uplink Changes * Add more proper Defaults * Renaming of Version struct * Dont pass Service use Pointer * Set Defaults for Versioning Checks * Put HTTP Server in go routine * Add Versioncontrol to Storj-Sim * Testplanet Fixes * Linting * Add Error Handling and new Server Struct * Move Lock slightly * Reduce Race Potentials * Remove unnecessary files * Linting * Add Proper Transport Handling * small fixes * add fence for allowed check * Add Startup Version Check and Service Naming * make errormessage private * Add Comments about VersionedClient * Linting * Remove Checks that refuse outgoing connections * Remove release cmd * Add Release Script * Linting * Update to use correct Values * Move vars private and set minimum default versions for testing builds * Remove VersionedClient * Better Error Handling and naked return removal * Straighten the Regex and string conversion * Change Check to allows testplanet and storj-sim to run without the need to pass an LDFlag * Cosmetic Change to Dashboard * Cleanup Returns and remove commented code * Remove Version Check if no build options are passed in * Pass in Config Values instead of Pointers * Handle missed Error * Update Endpoint URL * Change Type of Release Flag * Add additional Logging * Remove Versions Logging of other Services * minor fixes Change-Id: I5cc04a410ea6b2008d14dffd63eb5f36dd348a8b
2019-04-03 20:13:39 +01:00
"--version.server-address", fmt.Sprintf("http://%s/", versioncontrol.Address),
2019-01-08 15:24:15 +00:00
},
2019-01-28 14:48:49 +00:00
"run": {},
2019-01-08 15:24:15 +00:00
})
2019-02-06 12:47:00 +00:00
process.ExecBefore["run"] = func(process *Process) error {
return readConfigString(&process.Address, process.Directory, "server.address")
}
2019-01-02 18:07:49 +00:00
}
2019-01-08 15:24:15 +00:00
// Create gateways for each satellite
for i, satellite := range satellites {
2019-02-06 12:47:00 +00:00
satellite := satellite
2019-01-08 15:24:15 +00:00
process := processes.New(Info{
Name: fmt.Sprintf("gateway/%d", i),
Executable: "gateway",
2019-03-19 09:10:23 +00:00
Directory: filepath.Join(processes.Directory, "gateway", fmt.Sprint(i)),
2019-01-08 15:24:15 +00:00
Address: net.JoinHostPort(host, strconv.Itoa(gatewayPort+i)),
Extra: []string{},
2019-01-08 15:24:15 +00:00
})
// gateway must wait for the corresponding satellite to start up
process.WaitForStart(satellite)
2019-03-19 09:10:23 +00:00
process.Arguments = withCommon(process.Directory, Arguments{
2019-01-08 15:24:15 +00:00
"setup": {
"--identity-dir", process.Directory,
2019-01-08 15:24:15 +00:00
"--satellite-addr", satellite.Address,
2019-01-28 14:48:49 +00:00
2019-01-08 15:24:15 +00:00
"--server.address", process.Address,
2019-03-22 09:01:49 +00:00
"--satellite-addr", satellite.Address,
2019-01-08 15:24:15 +00:00
"--rs.min-threshold", strconv.Itoa(1 * flags.StorageNodeCount / 5),
"--rs.repair-threshold", strconv.Itoa(2 * flags.StorageNodeCount / 5),
"--rs.success-threshold", strconv.Itoa(3 * flags.StorageNodeCount / 5),
"--rs.max-threshold", strconv.Itoa(4 * flags.StorageNodeCount / 5),
"--tls.extensions.revocation=false",
"--tls.use-peer-ca-whitelist=false",
2019-01-08 15:24:15 +00:00
},
"run": {},
2019-01-08 15:24:15 +00:00
})
process.ExecBefore["run"] = func(process *Process) error {
2019-02-06 12:47:00 +00:00
err := readConfigString(&process.Address, process.Directory, "server.address")
if err != nil {
return err
}
vip := viper.New()
vip.AddConfigPath(process.Directory)
if err := vip.ReadInConfig(); err != nil {
return err
}
// TODO: maybe all the config flags should be exposed for all processes?
// check if gateway config has an api key, if it's not
// create example project with key and add it to the config
// so that gateway can have access to the satellite
2019-03-22 09:01:49 +00:00
apiKey := vip.GetString("api-key")
if !flags.OnlyEnv && apiKey == "" {
2019-02-06 12:47:00 +00:00
var consoleAddress string
satelliteConfigErr := readConfigString(&consoleAddress, satellite.Directory, "console.address")
if satelliteConfigErr != nil {
return satelliteConfigErr
}
host := "http://" + consoleAddress
createRegistrationTokenAddress := host + "/registrationToken/?projectsLimit=1"
consoleActivationAddress := host + "/activation/?token="
consoleAPIAddress := host + "/api/graphql/v0"
// wait for console server to start
time.Sleep(3 * time.Second)
if err := addExampleProjectWithKey(&apiKey, createRegistrationTokenAddress, consoleActivationAddress, consoleAPIAddress); err != nil {
return err
}
2019-03-22 09:01:49 +00:00
vip.Set("api-key", apiKey)
if err := vip.WriteConfig(); err != nil {
return err
}
}
if apiKey != "" {
process.Extra = append(process.Extra, "API_KEY="+apiKey)
}
accessKey := vip.GetString("minio.access-key")
secretKey := vip.GetString("minio.secret-key")
process.Extra = append(process.Extra,
"ACCESS_KEY="+accessKey,
"SECRET_KEY="+secretKey,
)
return nil
}
2019-01-02 18:07:49 +00:00
}
2019-01-08 15:24:15 +00:00
// Create storage nodes
for i := 0; i < flags.StorageNodeCount; i++ {
process := processes.New(Info{
Name: fmt.Sprintf("storagenode/%d", i),
Executable: "storagenode",
2019-03-19 09:10:23 +00:00
Directory: filepath.Join(processes.Directory, "storagenode", fmt.Sprint(i)),
2019-01-08 15:24:15 +00:00
Address: net.JoinHostPort(host, strconv.Itoa(storageNodePort+i)),
})
// storage node must wait for bootstrap and satellites to start
process.WaitForStart(bootstrap)
for _, satellite := range satellites {
process.WaitForStart(satellite)
}
2019-01-08 15:24:15 +00:00
2019-03-19 09:10:23 +00:00
process.Arguments = withCommon(process.Directory, Arguments{
"setup": {
"--identity-dir", process.Directory,
2019-01-28 14:48:49 +00:00
"--server.address", process.Address,
"--server.private-address", net.JoinHostPort(host, strconv.Itoa(storageNodePrivatePort+i)),
2019-01-28 14:48:49 +00:00
"--kademlia.bootstrap-addr", bootstrap.Address,
2019-01-08 15:24:15 +00:00
"--kademlia.operator.email", fmt.Sprintf("storage%d@example.com", i),
"--kademlia.operator.wallet", "0x0123456789012345678901234567890123456789",
"--server.extensions.revocation=false",
"--server.use-peer-ca-whitelist=false",
"--storage.satellite-id-restriction=false",
Add Versioning Server (#1576) * Initial Webserver Draft for Version Controlling * Rename type to avoid confusion * Move Function Calls into Version Package * Fix Linting and Language Typos * Fix Linting and Spelling Mistakes * Include Copyright * Include Copyright * Adjust Version-Control Server to return list of Versions * Linting * Improve Request Handling and Readability * Add Configuration File Option Add Systemd Service file * Add Logging to File * Smaller Changes * Add Semantic Versioning and refuses outdated Software from Startup (#1612) * implements internal Semantic Version library * adds version logging + reporting to process * Advance SemVer struct for easier handling * Add Accepted Version Store * Fix Function * Restructure * Type Conversion * Handle Version String properly * Add Note about array index * Set temporary Default Version * Add Copyright * Adding Version to Dashboard * Adding Version Info Log * Renaming and adding CheckerProcess * Iteration Sync * Iteration V2 * linting * made LogAndReportVersion a go routine * Refactor to Go Routine * Add Context to Go Routine and allow Operation if Lookup to Control Server fails * Handle Unmarshal properly * Linting * Relocate Version Checks * Relocating Version Check and specified default Version for now * Linting Error Prevention * Refuse Startup on outdated Version * Add Startup Check Function * Straighten Logging * Dont force Shutdown if --dev flag is set * Create full Service/Peer Structure for ControlServer * Linting * Straighting Naming * Finish VersionControl Service Layout * Improve Error Handling * Change Listening Address * Move Checker Function * Remove VersionControl Peer * Linting * Linting * Create VersionClient Service * Renaming * Add Version Client to Peer Definitions * Linting and Renaming * Linting * Remove Transport Checks for now * Move to Client Side Flag * Remove check * Linting * Transport Client Version Intro * Adding Version Client to Transport Client * Add missing parameter * Adding Version Check, to set Allowed = true * Set Default to true, testing * Restructuring Code * Uplink Changes * Add more proper Defaults * Renaming of Version struct * Dont pass Service use Pointer * Set Defaults for Versioning Checks * Put HTTP Server in go routine * Add Versioncontrol to Storj-Sim * Testplanet Fixes * Linting * Add Error Handling and new Server Struct * Move Lock slightly * Reduce Race Potentials * Remove unnecessary files * Linting * Add Proper Transport Handling * small fixes * add fence for allowed check * Add Startup Version Check and Service Naming * make errormessage private * Add Comments about VersionedClient * Linting * Remove Checks that refuse outgoing connections * Remove release cmd * Add Release Script * Linting * Update to use correct Values * Move vars private and set minimum default versions for testing builds * Remove VersionedClient * Better Error Handling and naked return removal * Straighten the Regex and string conversion * Change Check to allows testplanet and storj-sim to run without the need to pass an LDFlag * Cosmetic Change to Dashboard * Cleanup Returns and remove commented code * Remove Version Check if no build options are passed in * Pass in Config Values instead of Pointers * Handle missed Error * Update Endpoint URL * Change Type of Release Flag * Add additional Logging * Remove Versions Logging of other Services * minor fixes Change-Id: I5cc04a410ea6b2008d14dffd63eb5f36dd348a8b
2019-04-03 20:13:39 +01:00
"--version.server-address", fmt.Sprintf("http://%s/", versioncontrol.Address),
2019-01-08 15:24:15 +00:00
},
2019-01-28 14:48:49 +00:00
"run": {},
2019-01-08 15:24:15 +00:00
})
2019-02-06 12:47:00 +00:00
process.ExecBefore["run"] = func(process *Process) error {
return readConfigString(&process.Address, process.Directory, "server.address")
}
2019-01-08 15:24:15 +00:00
}
2019-01-02 18:07:49 +00:00
{ // verify that we have all binaries
missing := map[string]bool{}
for _, process := range processes.List {
_, err := exec.LookPath(process.Executable)
if err != nil {
missing[process.Executable] = true
}
}
if len(missing) > 0 {
var list []string
for executable := range missing {
list = append(list, executable)
}
sort.Strings(list)
return nil, fmt.Errorf("some executables cannot be found: %v", list)
}
}
2019-01-08 15:24:15 +00:00
// Create directories for all processes
for _, process := range processes.List {
if err := os.MkdirAll(process.Directory, folderPermissions); err != nil {
2019-01-02 18:07:49 +00:00
return nil, err
}
}
return processes, nil
}
2019-01-08 15:24:15 +00:00
func identitySetup(network *Processes) (*Processes, error) {
2019-03-19 09:10:23 +00:00
processes := NewProcesses(network.Directory)
for _, process := range network.List {
identity := processes.New(Info{
Name: "identity/" + process.Info.Name,
Executable: "identity",
Directory: process.Directory,
Address: "",
})
identity.Arguments = Arguments{
"setup": {
"--identity-dir", process.Directory,
"--concurrency", "1",
"--difficulty", "8",
2019-01-24 15:41:16 +00:00
"create", ".",
},
}
}
// create directories for all processes
for _, process := range processes.List {
if err := os.MkdirAll(process.Directory, folderPermissions); err != nil {
return nil, err
}
}
return processes, nil
}
2019-02-06 12:47:00 +00:00
// readConfigString reads from dir/config.yaml flagName returns the value in `into`
func readConfigString(into *string, dir, flagName string) error {
vip := viper.New()
vip.AddConfigPath(dir)
if err := vip.ReadInConfig(); err != nil {
return err
}
if v := vip.GetString(flagName); v != "" {
*into = v
}
return nil
}