storj/cmd/storj-sim/network.go

715 lines
21 KiB
Go
Raw Normal View History

2019-01-24 20:15:10 +00:00
// Copyright (C) 2019 Storj Labs, Inc.
2019-01-02 18:07:49 +00:00
// See LICENSE for copying information.
package main
import (
"context"
"errors"
"fmt"
"io/ioutil"
2019-01-02 18:07:49 +00:00
"net"
"net/url"
2019-01-02 18:07:49 +00:00
"os"
"os/exec"
"path/filepath"
"runtime"
"sort"
2019-01-02 18:07:49 +00:00
"strconv"
"strings"
"time"
2019-01-02 18:07:49 +00:00
"github.com/alessio/shellescape"
"github.com/spf13/viper"
2019-01-02 18:07:49 +00:00
"github.com/zeebo/errs"
"golang.org/x/sync/errgroup"
"storj.io/common/fpath"
"storj.io/common/identity"
"storj.io/common/processgroup"
"storj.io/common/storj"
"storj.io/storj/lib/uplink"
"storj.io/storj/private/dbutil"
"storj.io/storj/private/dbutil/pgutil"
2019-01-02 18:07:49 +00:00
)
const (
maxInstanceCount = 100
maxStoragenodeCount = 200
folderPermissions = 0744
)
var (
defaultAPIKeyData = "13YqgH45XZLg7nm6KsQ72QgXfjbDu2uhTaeSdMVP2A85QuANthM9K58ww5Y4nhMowrZDoqdA4Kyqt1ioQghQcm9fT5uR2drPHpFEqeb"
defaultAPIKey, _ = uplink.ParseAPIKey(defaultAPIKeyData)
)
const (
// The following values of peer class and endpoints are used
// to create a port with a consistent format for storj-sim services.
// Peer class
satellitePeer = 0
gatewayPeer = 1
versioncontrolPeer = 2
2019-10-04 21:48:41 +01:00
storagenodePeer = 3
// Endpoint
publicGRPC = 0
privateGRPC = 1
publicHTTP = 2
privateHTTP = 3
debugHTTP = 9
2019-10-29 15:25:25 +00:00
// satellite specific constants
redisPort = 4
adminHTTP = 5
debugAdminHTTP = 6
debugPeerHTTP = 7
debugRepairerHTTP = 8
debugGCHTTP = 10
)
// port creates a port with a consistent format for storj-sim services.
// The port format is: "1PXXE", where P is the peer class, XX is the index of the instance, and E is the endpoint.
func port(peerclass, index, endpoint int) string {
port := 10000 + peerclass*1000 + index*10 + endpoint
return strconv.Itoa(port)
}
2019-01-02 18:07:49 +00:00
func networkExec(flags *Flags, args []string, command string) error {
2019-01-08 15:24:15 +00:00
processes, err := newNetwork(flags)
2019-01-02 18:07:49 +00:00
if err != nil {
return err
}
ctx, cancel := NewCLIContext(context.Background())
defer cancel()
if command == "setup" {
if flags.Postgres == "" {
return errors.New("postgres connection URL is required for running storj-sim. Example: `storj-sim network setup --postgres=<connection URL>`.\nSee docs for more details https://github.com/storj/docs/blob/master/Test-network.md#running-tests-with-postgres")
}
identities, err := identitySetup(processes)
if err != nil {
return err
}
err = identities.Exec(ctx, command)
if err != nil {
return err
}
}
2019-01-02 18:07:49 +00:00
err = processes.Exec(ctx, command)
closeErr := processes.Close()
return errs.Combine(err, closeErr)
}
func escapeEnv(env string) string {
// TODO(jeff): escape env variables appropriately on windows. perhaps the
// env output should be of the form `set KEY=VALUE` as well.
if runtime.GOOS == "windows" {
return env
}
parts := strings.SplitN(env, "=", 2)
if len(parts) != 2 {
return env
}
return parts[0] + "=" + shellescape.Quote(parts[1])
}
func networkEnv(flags *Flags, args []string) error {
flags.OnlyEnv = true
processes, err := newNetwork(flags)
if err != nil {
return err
}
// run exec before, since it will load env vars from configs
for _, process := range processes.List {
if exec := process.ExecBefore["run"]; exec != nil {
if err := exec(process); err != nil {
return err
}
}
}
if len(args) == 1 {
envprefix := strings.ToUpper(args[0] + "=")
// find the environment value that the environment variable is set to
for _, env := range processes.Env() {
if strings.HasPrefix(strings.ToUpper(env), envprefix) {
fmt.Println(escapeEnv(env[len(envprefix):]))
return nil
}
}
return nil
}
for _, env := range processes.Env() {
fmt.Println(escapeEnv(env))
}
return nil
}
2019-01-02 18:07:49 +00:00
func networkTest(flags *Flags, command string, args []string) error {
2019-01-08 15:24:15 +00:00
processes, err := newNetwork(flags)
2019-01-02 18:07:49 +00:00
if err != nil {
return err
}
ctx, cancel := NewCLIContext(context.Background())
var group errgroup.Group
processes.Start(ctx, &group, "run")
2019-01-08 15:24:15 +00:00
for _, process := range processes.List {
process.Status.Started.Wait(ctx)
}
if err := ctx.Err(); err != nil {
return err
2019-01-08 15:24:15 +00:00
}
2019-01-02 18:07:49 +00:00
cmd := exec.CommandContext(ctx, command, args...)
cmd.Env = append(os.Environ(), processes.Env()...)
stdout := processes.Output.Prefixed("test:out")
stderr := processes.Output.Prefixed("test:err")
cmd.Stdout, cmd.Stderr = stdout, stderr
processgroup.Setup(cmd)
if printCommands {
fmt.Fprintf(processes.Output, "exec: %v\n", strings.Join(cmd.Args, " "))
}
errRun := cmd.Run()
cancel()
return errs.Combine(errRun, processes.Close(), group.Wait())
}
func networkDestroy(flags *Flags, args []string) error {
if fpath.IsRoot(flags.Directory) {
return errors.New("safety check: disallowed to remove root directory " + flags.Directory)
}
if printCommands {
2019-01-16 23:09:57 +00:00
fmt.Println("sim | exec: rm -rf", flags.Directory)
2019-01-02 18:07:49 +00:00
}
return os.RemoveAll(flags.Directory)
}
// newNetwork creates a default network
2019-01-08 15:24:15 +00:00
func newNetwork(flags *Flags) (*Processes, error) {
_, filename, _, ok := runtime.Caller(0)
if !ok {
return nil, errs.New("no caller information")
}
storjRoot := strings.TrimSuffix(filename, "/cmd/storj-sim/network.go")
2019-01-08 15:24:15 +00:00
// with common adds all common arguments to the process
2019-03-19 09:10:23 +00:00
withCommon := func(dir string, all Arguments) Arguments {
common := []string{"--metrics.app-suffix", "sim", "--log.level", "debug", "--config-dir", dir}
if flags.IsDev {
common = append(common, "--defaults", "dev")
}
2019-01-08 15:24:15 +00:00
for command, args := range all {
all[command] = append(append(common, command), args...)
2019-01-08 15:24:15 +00:00
}
return all
}
2019-01-02 18:07:49 +00:00
2019-03-19 09:10:23 +00:00
processes := NewProcesses(flags.Directory)
var host = flags.Host
Add Versioning Server (#1576) * Initial Webserver Draft for Version Controlling * Rename type to avoid confusion * Move Function Calls into Version Package * Fix Linting and Language Typos * Fix Linting and Spelling Mistakes * Include Copyright * Include Copyright * Adjust Version-Control Server to return list of Versions * Linting * Improve Request Handling and Readability * Add Configuration File Option Add Systemd Service file * Add Logging to File * Smaller Changes * Add Semantic Versioning and refuses outdated Software from Startup (#1612) * implements internal Semantic Version library * adds version logging + reporting to process * Advance SemVer struct for easier handling * Add Accepted Version Store * Fix Function * Restructure * Type Conversion * Handle Version String properly * Add Note about array index * Set temporary Default Version * Add Copyright * Adding Version to Dashboard * Adding Version Info Log * Renaming and adding CheckerProcess * Iteration Sync * Iteration V2 * linting * made LogAndReportVersion a go routine * Refactor to Go Routine * Add Context to Go Routine and allow Operation if Lookup to Control Server fails * Handle Unmarshal properly * Linting * Relocate Version Checks * Relocating Version Check and specified default Version for now * Linting Error Prevention * Refuse Startup on outdated Version * Add Startup Check Function * Straighten Logging * Dont force Shutdown if --dev flag is set * Create full Service/Peer Structure for ControlServer * Linting * Straighting Naming * Finish VersionControl Service Layout * Improve Error Handling * Change Listening Address * Move Checker Function * Remove VersionControl Peer * Linting * Linting * Create VersionClient Service * Renaming * Add Version Client to Peer Definitions * Linting and Renaming * Linting * Remove Transport Checks for now * Move to Client Side Flag * Remove check * Linting * Transport Client Version Intro * Adding Version Client to Transport Client * Add missing parameter * Adding Version Check, to set Allowed = true * Set Default to true, testing * Restructuring Code * Uplink Changes * Add more proper Defaults * Renaming of Version struct * Dont pass Service use Pointer * Set Defaults for Versioning Checks * Put HTTP Server in go routine * Add Versioncontrol to Storj-Sim * Testplanet Fixes * Linting * Add Error Handling and new Server Struct * Move Lock slightly * Reduce Race Potentials * Remove unnecessary files * Linting * Add Proper Transport Handling * small fixes * add fence for allowed check * Add Startup Version Check and Service Naming * make errormessage private * Add Comments about VersionedClient * Linting * Remove Checks that refuse outgoing connections * Remove release cmd * Add Release Script * Linting * Update to use correct Values * Move vars private and set minimum default versions for testing builds * Remove VersionedClient * Better Error Handling and naked return removal * Straighten the Regex and string conversion * Change Check to allows testplanet and storj-sim to run without the need to pass an LDFlag * Cosmetic Change to Dashboard * Cleanup Returns and remove commented code * Remove Version Check if no build options are passed in * Pass in Config Values instead of Pointers * Handle missed Error * Update Endpoint URL * Change Type of Release Flag * Add additional Logging * Remove Versions Logging of other Services * minor fixes Change-Id: I5cc04a410ea6b2008d14dffd63eb5f36dd348a8b
2019-04-03 20:13:39 +01:00
versioncontrol := processes.New(Info{
Name: "versioncontrol/0",
Executable: "versioncontrol",
Directory: filepath.Join(processes.Directory, "versioncontrol", "0"),
Address: net.JoinHostPort(host, port(versioncontrolPeer, 0, publicGRPC)),
Add Versioning Server (#1576) * Initial Webserver Draft for Version Controlling * Rename type to avoid confusion * Move Function Calls into Version Package * Fix Linting and Language Typos * Fix Linting and Spelling Mistakes * Include Copyright * Include Copyright * Adjust Version-Control Server to return list of Versions * Linting * Improve Request Handling and Readability * Add Configuration File Option Add Systemd Service file * Add Logging to File * Smaller Changes * Add Semantic Versioning and refuses outdated Software from Startup (#1612) * implements internal Semantic Version library * adds version logging + reporting to process * Advance SemVer struct for easier handling * Add Accepted Version Store * Fix Function * Restructure * Type Conversion * Handle Version String properly * Add Note about array index * Set temporary Default Version * Add Copyright * Adding Version to Dashboard * Adding Version Info Log * Renaming and adding CheckerProcess * Iteration Sync * Iteration V2 * linting * made LogAndReportVersion a go routine * Refactor to Go Routine * Add Context to Go Routine and allow Operation if Lookup to Control Server fails * Handle Unmarshal properly * Linting * Relocate Version Checks * Relocating Version Check and specified default Version for now * Linting Error Prevention * Refuse Startup on outdated Version * Add Startup Check Function * Straighten Logging * Dont force Shutdown if --dev flag is set * Create full Service/Peer Structure for ControlServer * Linting * Straighting Naming * Finish VersionControl Service Layout * Improve Error Handling * Change Listening Address * Move Checker Function * Remove VersionControl Peer * Linting * Linting * Create VersionClient Service * Renaming * Add Version Client to Peer Definitions * Linting and Renaming * Linting * Remove Transport Checks for now * Move to Client Side Flag * Remove check * Linting * Transport Client Version Intro * Adding Version Client to Transport Client * Add missing parameter * Adding Version Check, to set Allowed = true * Set Default to true, testing * Restructuring Code * Uplink Changes * Add more proper Defaults * Renaming of Version struct * Dont pass Service use Pointer * Set Defaults for Versioning Checks * Put HTTP Server in go routine * Add Versioncontrol to Storj-Sim * Testplanet Fixes * Linting * Add Error Handling and new Server Struct * Move Lock slightly * Reduce Race Potentials * Remove unnecessary files * Linting * Add Proper Transport Handling * small fixes * add fence for allowed check * Add Startup Version Check and Service Naming * make errormessage private * Add Comments about VersionedClient * Linting * Remove Checks that refuse outgoing connections * Remove release cmd * Add Release Script * Linting * Update to use correct Values * Move vars private and set minimum default versions for testing builds * Remove VersionedClient * Better Error Handling and naked return removal * Straighten the Regex and string conversion * Change Check to allows testplanet and storj-sim to run without the need to pass an LDFlag * Cosmetic Change to Dashboard * Cleanup Returns and remove commented code * Remove Version Check if no build options are passed in * Pass in Config Values instead of Pointers * Handle missed Error * Update Endpoint URL * Change Type of Release Flag * Add additional Logging * Remove Versions Logging of other Services * minor fixes Change-Id: I5cc04a410ea6b2008d14dffd63eb5f36dd348a8b
2019-04-03 20:13:39 +01:00
})
versioncontrol.Arguments = withCommon(versioncontrol.Directory, Arguments{
"setup": {
"--address", versioncontrol.Address,
"--debug.addr", net.JoinHostPort(host, port(versioncontrolPeer, 0, debugHTTP)),
"--binary.gateway.rollout.seed", "0000000000000000000000000000000000000000000000000000000000000001",
"--binary.identity.rollout.seed", "0000000000000000000000000000000000000000000000000000000000000001",
"--binary.satellite.rollout.seed", "0000000000000000000000000000000000000000000000000000000000000001",
"--binary.storagenode-updater.rollout.seed", "0000000000000000000000000000000000000000000000000000000000000001",
"--binary.storagenode.rollout.seed", "0000000000000000000000000000000000000000000000000000000000000001",
"--binary.uplink.rollout.seed", "0000000000000000000000000000000000000000000000000000000000000001",
Add Versioning Server (#1576) * Initial Webserver Draft for Version Controlling * Rename type to avoid confusion * Move Function Calls into Version Package * Fix Linting and Language Typos * Fix Linting and Spelling Mistakes * Include Copyright * Include Copyright * Adjust Version-Control Server to return list of Versions * Linting * Improve Request Handling and Readability * Add Configuration File Option Add Systemd Service file * Add Logging to File * Smaller Changes * Add Semantic Versioning and refuses outdated Software from Startup (#1612) * implements internal Semantic Version library * adds version logging + reporting to process * Advance SemVer struct for easier handling * Add Accepted Version Store * Fix Function * Restructure * Type Conversion * Handle Version String properly * Add Note about array index * Set temporary Default Version * Add Copyright * Adding Version to Dashboard * Adding Version Info Log * Renaming and adding CheckerProcess * Iteration Sync * Iteration V2 * linting * made LogAndReportVersion a go routine * Refactor to Go Routine * Add Context to Go Routine and allow Operation if Lookup to Control Server fails * Handle Unmarshal properly * Linting * Relocate Version Checks * Relocating Version Check and specified default Version for now * Linting Error Prevention * Refuse Startup on outdated Version * Add Startup Check Function * Straighten Logging * Dont force Shutdown if --dev flag is set * Create full Service/Peer Structure for ControlServer * Linting * Straighting Naming * Finish VersionControl Service Layout * Improve Error Handling * Change Listening Address * Move Checker Function * Remove VersionControl Peer * Linting * Linting * Create VersionClient Service * Renaming * Add Version Client to Peer Definitions * Linting and Renaming * Linting * Remove Transport Checks for now * Move to Client Side Flag * Remove check * Linting * Transport Client Version Intro * Adding Version Client to Transport Client * Add missing parameter * Adding Version Check, to set Allowed = true * Set Default to true, testing * Restructuring Code * Uplink Changes * Add more proper Defaults * Renaming of Version struct * Dont pass Service use Pointer * Set Defaults for Versioning Checks * Put HTTP Server in go routine * Add Versioncontrol to Storj-Sim * Testplanet Fixes * Linting * Add Error Handling and new Server Struct * Move Lock slightly * Reduce Race Potentials * Remove unnecessary files * Linting * Add Proper Transport Handling * small fixes * add fence for allowed check * Add Startup Version Check and Service Naming * make errormessage private * Add Comments about VersionedClient * Linting * Remove Checks that refuse outgoing connections * Remove release cmd * Add Release Script * Linting * Update to use correct Values * Move vars private and set minimum default versions for testing builds * Remove VersionedClient * Better Error Handling and naked return removal * Straighten the Regex and string conversion * Change Check to allows testplanet and storj-sim to run without the need to pass an LDFlag * Cosmetic Change to Dashboard * Cleanup Returns and remove commented code * Remove Version Check if no build options are passed in * Pass in Config Values instead of Pointers * Handle missed Error * Update Endpoint URL * Change Type of Release Flag * Add additional Logging * Remove Versions Logging of other Services * minor fixes Change-Id: I5cc04a410ea6b2008d14dffd63eb5f36dd348a8b
2019-04-03 20:13:39 +01:00
},
"run": {},
})
versioncontrol.ExecBefore["run"] = func(process *Process) error {
return readConfigString(&versioncontrol.Address, versioncontrol.Directory, "address")
}
// gateway must wait for the versioncontrol to start up
2019-10-04 21:48:41 +01:00
// Create satellites
if flags.SatelliteCount > maxInstanceCount {
return nil, fmt.Errorf("exceeded the max instance count of %d with Satellite count of %d", maxInstanceCount, flags.SatelliteCount)
}
// set up redis servers
var redisServers []*Process
if flags.Redis == "" {
for i := 0; i < flags.SatelliteCount; i++ {
rp := port(satellitePeer, i, redisPort)
process := processes.New(Info{
Name: fmt.Sprintf("redis/%d", i),
Executable: "redis-server",
Directory: filepath.Join(processes.Directory, "satellite", fmt.Sprint(i), "redis"),
Address: net.JoinHostPort(host, rp),
})
redisServers = append(redisServers, process)
process.ExecBefore["setup"] = func(process *Process) error {
confpath := filepath.Join(process.Directory, "redis.conf")
arguments := []string{
"daemonize no",
"bind " + host,
"port " + rp,
"timeout 0",
"databases 2",
"dbfilename sim.rdb",
"dir ./",
}
conf := strings.Join(arguments, "\n") + "\n"
err := ioutil.WriteFile(confpath, []byte(conf), 0755)
return err
}
process.Arguments = Arguments{
"run": []string{filepath.Join(process.Directory, "redis.conf")},
}
}
}
var satellites []*Process
2019-01-08 15:24:15 +00:00
for i := 0; i < flags.SatelliteCount; i++ {
apiProcess := processes.New(Info{
2019-01-08 15:24:15 +00:00
Name: fmt.Sprintf("satellite/%d", i),
Executable: "satellite",
2019-03-19 09:10:23 +00:00
Directory: filepath.Join(processes.Directory, "satellite", fmt.Sprint(i)),
Address: net.JoinHostPort(host, port(satellitePeer, i, publicGRPC)),
2019-01-08 15:24:15 +00:00
})
satellites = append(satellites, apiProcess)
2019-01-08 15:24:15 +00:00
consoleAuthToken := "secure_token"
redisAddress := flags.Redis
redisPortBase := flags.RedisStartDB + i*2
if redisAddress == "" {
redisAddress = redisServers[i].Address
redisPortBase = 0
apiProcess.WaitForStart(redisServers[i])
}
apiProcess.Arguments = withCommon(apiProcess.Directory, Arguments{
"setup": {
"--identity-dir", apiProcess.Directory,
"--console.address", net.JoinHostPort(host, port(satellitePeer, i, publicHTTP)),
"--console.static-dir", filepath.Join(storjRoot, "web/satellite/"),
// TODO: remove console.auth-token after vanguard release
"--console.auth-token", consoleAuthToken,
"--marketing.base-url", "",
"--marketing.address", net.JoinHostPort(host, port(satellitePeer, i, privateHTTP)),
"--marketing.static-dir", filepath.Join(storjRoot, "web/marketing/"),
"--server.address", apiProcess.Address,
"--server.private-address", net.JoinHostPort(host, port(satellitePeer, i, privateGRPC)),
2019-01-08 23:41:01 +00:00
"--live-accounting.storage-backend", "redis://" + redisAddress + "?db=" + strconv.Itoa(redisPortBase),
"--server.revocation-dburl", "redis://" + redisAddress + "?db=" + strconv.Itoa(redisPortBase+1),
"--server.extensions.revocation=false",
"--server.use-peer-ca-whitelist=false",
"--mail.smtp-server-address", "smtp.gmail.com:587",
"--mail.from", "Storj <yaroslav-satellite-test@storj.io>",
"--mail.template-path", filepath.Join(storjRoot, "web/satellite/static/emails"),
Add Versioning Server (#1576) * Initial Webserver Draft for Version Controlling * Rename type to avoid confusion * Move Function Calls into Version Package * Fix Linting and Language Typos * Fix Linting and Spelling Mistakes * Include Copyright * Include Copyright * Adjust Version-Control Server to return list of Versions * Linting * Improve Request Handling and Readability * Add Configuration File Option Add Systemd Service file * Add Logging to File * Smaller Changes * Add Semantic Versioning and refuses outdated Software from Startup (#1612) * implements internal Semantic Version library * adds version logging + reporting to process * Advance SemVer struct for easier handling * Add Accepted Version Store * Fix Function * Restructure * Type Conversion * Handle Version String properly * Add Note about array index * Set temporary Default Version * Add Copyright * Adding Version to Dashboard * Adding Version Info Log * Renaming and adding CheckerProcess * Iteration Sync * Iteration V2 * linting * made LogAndReportVersion a go routine * Refactor to Go Routine * Add Context to Go Routine and allow Operation if Lookup to Control Server fails * Handle Unmarshal properly * Linting * Relocate Version Checks * Relocating Version Check and specified default Version for now * Linting Error Prevention * Refuse Startup on outdated Version * Add Startup Check Function * Straighten Logging * Dont force Shutdown if --dev flag is set * Create full Service/Peer Structure for ControlServer * Linting * Straighting Naming * Finish VersionControl Service Layout * Improve Error Handling * Change Listening Address * Move Checker Function * Remove VersionControl Peer * Linting * Linting * Create VersionClient Service * Renaming * Add Version Client to Peer Definitions * Linting and Renaming * Linting * Remove Transport Checks for now * Move to Client Side Flag * Remove check * Linting * Transport Client Version Intro * Adding Version Client to Transport Client * Add missing parameter * Adding Version Check, to set Allowed = true * Set Default to true, testing * Restructuring Code * Uplink Changes * Add more proper Defaults * Renaming of Version struct * Dont pass Service use Pointer * Set Defaults for Versioning Checks * Put HTTP Server in go routine * Add Versioncontrol to Storj-Sim * Testplanet Fixes * Linting * Add Error Handling and new Server Struct * Move Lock slightly * Reduce Race Potentials * Remove unnecessary files * Linting * Add Proper Transport Handling * small fixes * add fence for allowed check * Add Startup Version Check and Service Naming * make errormessage private * Add Comments about VersionedClient * Linting * Remove Checks that refuse outgoing connections * Remove release cmd * Add Release Script * Linting * Update to use correct Values * Move vars private and set minimum default versions for testing builds * Remove VersionedClient * Better Error Handling and naked return removal * Straighten the Regex and string conversion * Change Check to allows testplanet and storj-sim to run without the need to pass an LDFlag * Cosmetic Change to Dashboard * Cleanup Returns and remove commented code * Remove Version Check if no build options are passed in * Pass in Config Values instead of Pointers * Handle missed Error * Update Endpoint URL * Change Type of Release Flag * Add additional Logging * Remove Versions Logging of other Services * minor fixes Change-Id: I5cc04a410ea6b2008d14dffd63eb5f36dd348a8b
2019-04-03 20:13:39 +01:00
"--version.server-address", fmt.Sprintf("http://%s/", versioncontrol.Address),
"--debug.addr", net.JoinHostPort(host, port(satellitePeer, i, debugHTTP)),
"--admin.address", net.JoinHostPort(host, port(satellitePeer, i, adminHTTP)),
2019-01-08 15:24:15 +00:00
},
"run": {"api"},
2019-01-08 15:24:15 +00:00
})
2019-02-06 12:47:00 +00:00
if flags.Postgres != "" {
masterDBURL, err := namespacedDatabaseURL(flags.Postgres, fmt.Sprintf("satellite/%d", i))
if err != nil {
return nil, err
}
metainfoDBURL, err := namespacedDatabaseURL(flags.Postgres, fmt.Sprintf("satellite/%d/meta", i))
if err != nil {
return nil, err
}
apiProcess.Arguments["setup"] = append(apiProcess.Arguments["setup"],
"--database", masterDBURL,
"--metainfo.database-url", metainfoDBURL,
)
}
apiProcess.ExecBefore["run"] = func(process *Process) error {
2019-02-06 12:47:00 +00:00
return readConfigString(&process.Address, process.Directory, "server.address")
}
2019-01-02 18:07:49 +00:00
migrationProcess := processes.New(Info{
Name: fmt.Sprintf("satellite-migration/%d", i),
Executable: "satellite",
Directory: filepath.Join(processes.Directory, "satellite", fmt.Sprint(i)),
})
migrationProcess.Arguments = withCommon(apiProcess.Directory, Arguments{
"run": {
"migration",
"--debug.addr", net.JoinHostPort(host, port(satellitePeer, i, debugPeerHTTP)),
},
})
apiProcess.WaitForExited(migrationProcess)
coreProcess := processes.New(Info{
Name: fmt.Sprintf("satellite-core/%d", i),
Executable: "satellite",
Directory: filepath.Join(processes.Directory, "satellite", fmt.Sprint(i)),
Address: "",
})
coreProcess.Arguments = withCommon(apiProcess.Directory, Arguments{
"run": {
"--debug.addr", net.JoinHostPort(host, port(satellitePeer, i, debugPeerHTTP)),
},
})
coreProcess.WaitForExited(migrationProcess)
adminProcess := processes.New(Info{
Name: fmt.Sprintf("satellite-admin/%d", i),
Executable: "satellite",
Directory: filepath.Join(processes.Directory, "satellite", fmt.Sprint(i)),
Address: net.JoinHostPort(host, port(satellitePeer, i, adminHTTP)),
})
adminProcess.Arguments = withCommon(apiProcess.Directory, Arguments{
"run": {
"admin",
"--debug.addr", net.JoinHostPort(host, port(satellitePeer, i, debugAdminHTTP)),
},
})
adminProcess.WaitForExited(migrationProcess)
repairProcess := processes.New(Info{
Name: fmt.Sprintf("satellite-repairer/%d", i),
Executable: "satellite",
Directory: filepath.Join(processes.Directory, "satellite", fmt.Sprint(i)),
})
repairProcess.Arguments = withCommon(apiProcess.Directory, Arguments{
"run": {
"repair",
2019-10-29 15:25:25 +00:00
"--debug.addr", net.JoinHostPort(host, port(satellitePeer, i, debugRepairerHTTP)),
},
})
repairProcess.WaitForExited(migrationProcess)
garbageCollectionProcess := processes.New(Info{
Name: fmt.Sprintf("satellite-garbage-collection/%d", i),
Executable: "satellite",
Directory: filepath.Join(processes.Directory, "satellite", fmt.Sprint(i)),
})
garbageCollectionProcess.Arguments = withCommon(apiProcess.Directory, Arguments{
"run": {
"garbage-collection",
"--debug.addr", net.JoinHostPort(host, port(satellitePeer, i, debugGCHTTP)),
},
})
garbageCollectionProcess.WaitForExited(migrationProcess)
}
// Create gateways for each satellite
for i, satellite := range satellites {
2019-02-06 12:47:00 +00:00
satellite := satellite
2019-01-08 15:24:15 +00:00
process := processes.New(Info{
Name: fmt.Sprintf("gateway/%d", i),
Executable: "gateway",
2019-03-19 09:10:23 +00:00
Directory: filepath.Join(processes.Directory, "gateway", fmt.Sprint(i)),
Address: net.JoinHostPort(host, port(gatewayPeer, i, publicGRPC)),
2019-01-08 15:24:15 +00:00
})
encAccess := uplink.NewEncryptionAccessWithDefaultKey(storj.Key{})
encAccess.SetDefaultPathCipher(storj.EncAESGCM)
accessData, err := (&uplink.Scope{
SatelliteAddr: satellite.Address,
APIKey: defaultAPIKey,
EncryptionAccess: encAccess,
}).Serialize()
if err != nil {
return nil, err
}
2019-01-08 15:24:15 +00:00
// gateway must wait for the corresponding satellite to start up
process.WaitForStart(satellite)
2019-03-19 09:10:23 +00:00
process.Arguments = withCommon(process.Directory, Arguments{
2019-01-08 15:24:15 +00:00
"setup": {
uplink: enc.encryption-key flag is only available for setup command (#2090) * uplink: Mark encryption key config field for setup Set the "setup" property to the `EncryptionConfig.EncrptionKey` for avoiding to save it in the configuration file. This field is only meant for using in the command line parameters which need to use a different encryption key than the one present in the key file or use it when there is not set any encryption key file path. * cmd/uplink: Setup non-interactive accept enc key Change the uplink CLI setup command non-interactive to save the encryption key into a file when it's passed through the flag --enc.encryption-key Previous to this change it wasn't possible to create an key file despite of that the flag was provided, so it was useless on the setup command. * cmd/uplink: Reuse logic to read pwd from terminal Reuse the logic which is already implemented in the pkg/cfgstruct for reading a password from the terminal on interactive mode, rather than duplicating it in the setup command. * cmd/gateway: Use encryption key file flags The cmd/gateway was still using the `enc.key` configuration field which doesn't exist anymore and its setup command wasn't using the `enc.key-filepath` with combination of the `enc.encryption-key` for generating a file with the encryption key. This commit update the cmd/gateway appropriately and move to the uplink package the function used by cmd/uplink to save the encryption key for allowing to also be used by the cmd/gateway without duplicating the logic. * cmd/storj-sim: Adapt gateway config cmd changes Adapt the cmd/storj-sim to correctly pass the parameters to the cmd/gateway setup and run command. * scripts: Don't pass the --enc.encryption-key flag uplink configuration has changed to only support the `--enc.encryption-key` flag for setup commands and consequently the cmd/uplink and cmd/gateway don't accept this flag over other commands, hence the test for the uplink had to be updated for no passing the flag on the multiples calls that the test do to cmd/uplink. * uplink: Remove func which aren't useful anymore Remove the function which allows to user or load an encryption key because it isn't needed anymore since the `--enc.encryption-key` flag is only available for the setup command. Consequently remove its usage from cmd/uplink and cmd/gateway, because such flag will always be empty because in case that's passed Cobra will return an error due to a "unknown flag".
2019-06-07 17:14:40 +01:00
"--non-interactive",
"--access", accessData,
2019-01-08 15:24:15 +00:00
"--server.address", process.Address,
"--debug.addr", net.JoinHostPort(host, port(gatewayPeer, i, debugHTTP)),
2019-01-08 15:24:15 +00:00
},
uplink: enc.encryption-key flag is only available for setup command (#2090) * uplink: Mark encryption key config field for setup Set the "setup" property to the `EncryptionConfig.EncrptionKey` for avoiding to save it in the configuration file. This field is only meant for using in the command line parameters which need to use a different encryption key than the one present in the key file or use it when there is not set any encryption key file path. * cmd/uplink: Setup non-interactive accept enc key Change the uplink CLI setup command non-interactive to save the encryption key into a file when it's passed through the flag --enc.encryption-key Previous to this change it wasn't possible to create an key file despite of that the flag was provided, so it was useless on the setup command. * cmd/uplink: Reuse logic to read pwd from terminal Reuse the logic which is already implemented in the pkg/cfgstruct for reading a password from the terminal on interactive mode, rather than duplicating it in the setup command. * cmd/gateway: Use encryption key file flags The cmd/gateway was still using the `enc.key` configuration field which doesn't exist anymore and its setup command wasn't using the `enc.key-filepath` with combination of the `enc.encryption-key` for generating a file with the encryption key. This commit update the cmd/gateway appropriately and move to the uplink package the function used by cmd/uplink to save the encryption key for allowing to also be used by the cmd/gateway without duplicating the logic. * cmd/storj-sim: Adapt gateway config cmd changes Adapt the cmd/storj-sim to correctly pass the parameters to the cmd/gateway setup and run command. * scripts: Don't pass the --enc.encryption-key flag uplink configuration has changed to only support the `--enc.encryption-key` flag for setup commands and consequently the cmd/uplink and cmd/gateway don't accept this flag over other commands, hence the test for the uplink had to be updated for no passing the flag on the multiples calls that the test do to cmd/uplink. * uplink: Remove func which aren't useful anymore Remove the function which allows to user or load an encryption key because it isn't needed anymore since the `--enc.encryption-key` flag is only available for the setup command. Consequently remove its usage from cmd/uplink and cmd/gateway, because such flag will always be empty because in case that's passed Cobra will return an error due to a "unknown flag".
2019-06-07 17:14:40 +01:00
"run": {},
2019-01-08 15:24:15 +00:00
})
process.ExecBefore["run"] = func(process *Process) (err error) {
err = readConfigString(&process.Address, process.Directory, "server.address")
2019-02-06 12:47:00 +00:00
if err != nil {
return err
}
vip := viper.New()
vip.AddConfigPath(process.Directory)
if err := vip.ReadInConfig(); err != nil {
return err
}
// TODO: maybe all the config flags should be exposed for all processes?
// check if gateway config has an api key, if it's not
// create example project with key and add it to the config
// so that gateway can have access to the satellite
if runAccessData := vip.GetString("access"); !flags.OnlyEnv && runAccessData == accessData {
2019-02-06 12:47:00 +00:00
var consoleAddress string
err := readConfigString(&consoleAddress, satellite.Directory, "console.address")
if err != nil {
return err
2019-02-06 12:47:00 +00:00
}
// try with 100ms delays until we hit 3s
apiKey, start := "", time.Now()
for apiKey == "" {
apiKey, err = newConsoleEndpoints(consoleAddress).createOrGetAPIKey()
if err != nil && time.Since(start) > 3*time.Second {
return err
}
time.Sleep(100 * time.Millisecond)
}
access, err := uplink.ParseScope(runAccessData)
if err != nil {
return err
}
access.APIKey, err = uplink.ParseAPIKey(apiKey)
if err != nil {
return err
}
satNodeID, err := identity.NodeIDFromCertPath(filepath.Join(satellite.Directory, "identity.cert"))
if err != nil {
return err
}
nodeURL := storj.NodeURL{
ID: satNodeID,
Address: access.SatelliteAddr,
}
access.SatelliteAddr = nodeURL.String()
accessData, err := access.Serialize()
if err != nil {
return err
}
vip.Set("access", accessData)
if err := vip.WriteConfig(); err != nil {
return err
}
}
if runAccessData := vip.GetString("access"); runAccessData != accessData {
process.AddExtra("ACCESS", runAccessData)
if access, err := uplink.ParseScope(runAccessData); err == nil {
process.AddExtra("API_KEY", access.APIKey.Serialize())
}
}
process.AddExtra("ACCESS_KEY", vip.GetString("minio.access-key"))
process.AddExtra("SECRET_KEY", vip.GetString("minio.secret-key"))
return nil
}
2019-01-02 18:07:49 +00:00
}
2019-01-08 15:24:15 +00:00
// Create storage nodes
if flags.StorageNodeCount > maxStoragenodeCount {
return nil, fmt.Errorf("exceeded the max instance count of %d with Storage Node count of %d", maxStoragenodeCount, flags.StorageNodeCount)
}
2019-01-08 15:24:15 +00:00
for i := 0; i < flags.StorageNodeCount; i++ {
process := processes.New(Info{
Name: fmt.Sprintf("storagenode/%d", i),
Executable: "storagenode",
2019-03-19 09:10:23 +00:00
Directory: filepath.Join(processes.Directory, "storagenode", fmt.Sprint(i)),
Address: net.JoinHostPort(host, port(storagenodePeer, i, publicGRPC)),
2019-01-08 15:24:15 +00:00
})
for _, satellite := range satellites {
process.WaitForStart(satellite)
}
2019-01-08 15:24:15 +00:00
2019-03-19 09:10:23 +00:00
process.Arguments = withCommon(process.Directory, Arguments{
"setup": {
"--identity-dir", process.Directory,
"--console.address", net.JoinHostPort(host, port(storagenodePeer, i, publicHTTP)),
"--console.static-dir", filepath.Join(storjRoot, "web/storagenode/"),
2019-01-28 14:48:49 +00:00
"--server.address", process.Address,
"--server.private-address", net.JoinHostPort(host, port(storagenodePeer, i, privateGRPC)),
2019-01-28 14:48:49 +00:00
2019-10-04 21:48:41 +01:00
"--operator.email", fmt.Sprintf("storage%d@mail.test", i),
"--operator.wallet", "0x0123456789012345678901234567890123456789",
"--storage2.monitor.minimum-disk-space", "0",
"--server.extensions.revocation=false",
"--server.use-peer-ca-whitelist=false",
Add Versioning Server (#1576) * Initial Webserver Draft for Version Controlling * Rename type to avoid confusion * Move Function Calls into Version Package * Fix Linting and Language Typos * Fix Linting and Spelling Mistakes * Include Copyright * Include Copyright * Adjust Version-Control Server to return list of Versions * Linting * Improve Request Handling and Readability * Add Configuration File Option Add Systemd Service file * Add Logging to File * Smaller Changes * Add Semantic Versioning and refuses outdated Software from Startup (#1612) * implements internal Semantic Version library * adds version logging + reporting to process * Advance SemVer struct for easier handling * Add Accepted Version Store * Fix Function * Restructure * Type Conversion * Handle Version String properly * Add Note about array index * Set temporary Default Version * Add Copyright * Adding Version to Dashboard * Adding Version Info Log * Renaming and adding CheckerProcess * Iteration Sync * Iteration V2 * linting * made LogAndReportVersion a go routine * Refactor to Go Routine * Add Context to Go Routine and allow Operation if Lookup to Control Server fails * Handle Unmarshal properly * Linting * Relocate Version Checks * Relocating Version Check and specified default Version for now * Linting Error Prevention * Refuse Startup on outdated Version * Add Startup Check Function * Straighten Logging * Dont force Shutdown if --dev flag is set * Create full Service/Peer Structure for ControlServer * Linting * Straighting Naming * Finish VersionControl Service Layout * Improve Error Handling * Change Listening Address * Move Checker Function * Remove VersionControl Peer * Linting * Linting * Create VersionClient Service * Renaming * Add Version Client to Peer Definitions * Linting and Renaming * Linting * Remove Transport Checks for now * Move to Client Side Flag * Remove check * Linting * Transport Client Version Intro * Adding Version Client to Transport Client * Add missing parameter * Adding Version Check, to set Allowed = true * Set Default to true, testing * Restructuring Code * Uplink Changes * Add more proper Defaults * Renaming of Version struct * Dont pass Service use Pointer * Set Defaults for Versioning Checks * Put HTTP Server in go routine * Add Versioncontrol to Storj-Sim * Testplanet Fixes * Linting * Add Error Handling and new Server Struct * Move Lock slightly * Reduce Race Potentials * Remove unnecessary files * Linting * Add Proper Transport Handling * small fixes * add fence for allowed check * Add Startup Version Check and Service Naming * make errormessage private * Add Comments about VersionedClient * Linting * Remove Checks that refuse outgoing connections * Remove release cmd * Add Release Script * Linting * Update to use correct Values * Move vars private and set minimum default versions for testing builds * Remove VersionedClient * Better Error Handling and naked return removal * Straighten the Regex and string conversion * Change Check to allows testplanet and storj-sim to run without the need to pass an LDFlag * Cosmetic Change to Dashboard * Cleanup Returns and remove commented code * Remove Version Check if no build options are passed in * Pass in Config Values instead of Pointers * Handle missed Error * Update Endpoint URL * Change Type of Release Flag * Add additional Logging * Remove Versions Logging of other Services * minor fixes Change-Id: I5cc04a410ea6b2008d14dffd63eb5f36dd348a8b
2019-04-03 20:13:39 +01:00
"--version.server-address", fmt.Sprintf("http://%s/", versioncontrol.Address),
"--debug.addr", net.JoinHostPort(host, port(storagenodePeer, i, debugHTTP)),
"--tracing.app", fmt.Sprintf("storagenode/%d", i),
2019-01-08 15:24:15 +00:00
},
2019-01-28 14:48:49 +00:00
"run": {},
2019-01-08 15:24:15 +00:00
})
2019-02-06 12:47:00 +00:00
process.ExecBefore["setup"] = func(process *Process) error {
whitelisted := []string{}
for _, satellite := range satellites {
peer, err := identity.PeerConfig{
CertPath: filepath.Join(satellite.Directory, "identity.cert"),
}.Load()
if err != nil {
return err
}
whitelisted = append(whitelisted, peer.ID.String()+"@"+satellite.Address)
}
process.Arguments["setup"] = append(process.Arguments["setup"],
"--storage2.trust.sources", strings.Join(whitelisted, ","),
)
return nil
}
2019-02-06 12:47:00 +00:00
process.ExecBefore["run"] = func(process *Process) error {
return readConfigString(&process.Address, process.Directory, "server.address")
}
2019-01-08 15:24:15 +00:00
}
2019-01-02 18:07:49 +00:00
{ // verify that we have all binaries
missing := map[string]bool{}
for _, process := range processes.List {
_, err := exec.LookPath(process.Executable)
if err != nil {
missing[process.Executable] = true
}
}
if len(missing) > 0 {
var list []string
for executable := range missing {
list = append(list, executable)
}
sort.Strings(list)
return nil, fmt.Errorf("some executables cannot be found: %v", list)
}
}
2019-01-08 15:24:15 +00:00
// Create directories for all processes
for _, process := range processes.List {
if err := os.MkdirAll(process.Directory, folderPermissions); err != nil {
2019-01-02 18:07:49 +00:00
return nil, err
}
}
return processes, nil
}
2019-01-08 15:24:15 +00:00
func identitySetup(network *Processes) (*Processes, error) {
2019-03-19 09:10:23 +00:00
processes := NewProcesses(network.Directory)
for _, process := range network.List {
if process.Info.Executable == "gateway" || process.Info.Executable == "redis-server" {
// gateways and redis-servers don't need an identity
continue
}
if strings.Contains(process.Name, "satellite-") {
// we only need to create the identity once for the satellite system, we create the
// identity for the satellite process and share it with these other satellite processes
continue
}
identity := processes.New(Info{
Name: "identity/" + process.Info.Name,
Executable: "identity",
Directory: process.Directory,
Address: "",
})
identity.Arguments = Arguments{
"setup": {
"--identity-dir", process.Directory,
"--concurrency", "1",
"--difficulty", "8",
2019-01-24 15:41:16 +00:00
"create", ".",
},
}
}
// create directories for all processes
for _, process := range processes.List {
if err := os.MkdirAll(process.Directory, folderPermissions); err != nil {
return nil, err
}
}
return processes, nil
}
2019-02-06 12:47:00 +00:00
// readConfigString reads from dir/config.yaml flagName returns the value in `into`
func readConfigString(into *string, dir, flagName string) error {
vip := viper.New()
vip.AddConfigPath(dir)
if err := vip.ReadInConfig(); err != nil {
return err
}
if v := vip.GetString(flagName); v != "" {
*into = v
}
return nil
}
// namespacedDatabaseURL returns an equivalent database url with the given namespace
// so that a database opened with the url does not conflict with other databases
// opened with a different namespace.
func namespacedDatabaseURL(dbURL, namespace string) (string, error) {
parsed, err := url.Parse(dbURL)
if err != nil {
return "", err
}
switch dbutil.ImplementationForScheme(parsed.Scheme) {
case dbutil.Postgres:
return pgutil.ConnstrWithSchema(dbURL, namespace), nil
case dbutil.Cockroach:
parsed.Path += "/" + namespace
return parsed.String(), nil
default:
return "", errs.New("unable to namespace db url: %q", dbURL)
}
}