diff --git a/Makefile b/Makefile index 451144a52..283f43bbb 100644 --- a/Makefile +++ b/Makefile @@ -85,17 +85,24 @@ build-npm: ##@ Simulator +.PHONY: go-install-grpc-and-drpc +go-install-grpc-and-drpc: + @: $(if ${PACKAGE},,$(error PACKAGE must be defined for the go-install-grpc-and-drpc target)) + go build -race -v -tags=grpc -o "$(shell go list -f '{{.Target}}' ${PACKAGE})-grpc" "${PACKAGE}" + go build -race -v -tags=drpc -o "$(shell go list -f '{{.Target}}' ${PACKAGE})-drpc" "${PACKAGE}" + go install -race -v "${PACKAGE}" + .PHONY: install-sim install-sim: ## install storj-sim @echo "Running ${@}" - @go install -race -v -tags=grpc storj.io/storj/cmd/storj-sim - @go install -race -v -tags=grpc storj.io/storj/cmd/versioncontrol - @go install -race -v -tags=grpc storj.io/storj/cmd/satellite - @go install -race -v -tags=grpc storj.io/storj/cmd/storagenode - @go install -race -v storj.io/storj/cmd/uplink - @go install -race -v storj.io/storj/cmd/gateway - @go install -race -v -tags=grpc storj.io/storj/cmd/identity - @go install -race -v -tags=grpc storj.io/storj/cmd/certificates + $(MAKE) go-install-grpc-and-drpc PACKAGE=storj.io/storj/cmd/storagenode + $(MAKE) go-install-grpc-and-drpc PACKAGE=storj.io/storj/cmd/satellite + go install -race -v storj.io/storj/cmd/storj-sim + go install -race -v storj.io/storj/cmd/versioncontrol + go install -race -v storj.io/storj/cmd/uplink + go install -race -v storj.io/storj/cmd/gateway + go install -race -v storj.io/storj/cmd/identity + go install -race -v storj.io/storj/cmd/certificates ##@ Test @@ -269,34 +276,34 @@ binary-check: .PHONY: certificates_% certificates_%: - $(MAKE) binary-check COMPONENT=certificates GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) EXTRA_ARGS=-tags=grpc + $(MAKE) binary-check COMPONENT=certificates GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) .PHONY: gateway_% gateway_%: $(MAKE) binary-check COMPONENT=gateway GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) .PHONY: identity_% identity_%: - $(MAKE) binary-check COMPONENT=identity GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) EXTRA_ARGS=-tags=grpc + $(MAKE) binary-check COMPONENT=identity GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) .PHONY: inspector_% inspector_%: - $(MAKE) binary-check COMPONENT=inspector GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) EXTRA_ARGS=-tags=grpc + $(MAKE) binary-check COMPONENT=inspector GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) .PHONY: linksharing_% linksharing_%: $(MAKE) binary-check COMPONENT=linksharing GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) .PHONY: satellite_% satellite_%: - $(MAKE) binary-check COMPONENT=satellite GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) EXTRA_ARGS=-tags=grpc + $(MAKE) binary-check COMPONENT=satellite GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) .PHONY: storagenode_% storagenode_%: storagenode-console - $(MAKE) binary-check COMPONENT=storagenode GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) EXTRA_ARGS=-tags=grpc + $(MAKE) binary-check COMPONENT=storagenode GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) .PHONY: storagenode-updater_% storagenode-updater_%: - $(MAKE) binary-check COMPONENT=storagenode-updater GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) EXTRA_ARGS=-tags=grpc + $(MAKE) binary-check COMPONENT=storagenode-updater GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) .PHONY: uplink_% uplink_%: $(MAKE) binary-check COMPONENT=uplink GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) .PHONY: versioncontrol_% versioncontrol_%: - $(MAKE) binary-check COMPONENT=versioncontrol GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) EXTRA_ARGS=-tags=grpc + $(MAKE) binary-check COMPONENT=versioncontrol GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) COMPONENTLIST := certificates gateway identity inspector linksharing satellite storagenode storagenode-updater uplink versioncontrol diff --git a/cmd/storj-sim/network.go b/cmd/storj-sim/network.go index 19fe03aaf..abb081848 100644 --- a/cmd/storj-sim/network.go +++ b/cmd/storj-sim/network.go @@ -18,6 +18,7 @@ import ( "strings" "time" + "github.com/alessio/shellescape" "github.com/spf13/viper" "github.com/zeebo/errs" "golang.org/x/sync/errgroup" @@ -104,6 +105,20 @@ func networkExec(flags *Flags, args []string, command string) error { return errs.Combine(err, closeErr) } +func escapeEnv(env string) string { + // TODO(jeff): escape env variables appropriately on windows. perhaps the + // env output should be of the form `set KEY=VALUE` as well. + if runtime.GOOS == "windows" { + return env + } + + parts := strings.SplitN(env, "=", 2) + if len(parts) != 2 { + return env + } + return parts[0] + "=" + shellescape.Quote(parts[1]) +} + func networkEnv(flags *Flags, args []string) error { flags.OnlyEnv = true @@ -126,7 +141,7 @@ func networkEnv(flags *Flags, args []string) error { // find the environment value that the environment variable is set to for _, env := range processes.Env() { if strings.HasPrefix(strings.ToUpper(env), envprefix) { - fmt.Println(env[len(envprefix):]) + fmt.Println(escapeEnv(env[len(envprefix):])) return nil } } @@ -135,7 +150,7 @@ func networkEnv(flags *Flags, args []string) error { } for _, env := range processes.Env() { - fmt.Println(env) + fmt.Println(escapeEnv(env)) } return nil @@ -381,7 +396,6 @@ func newNetwork(flags *Flags) (*Processes, error) { Executable: "gateway", Directory: filepath.Join(processes.Directory, "gateway", fmt.Sprint(i)), Address: net.JoinHostPort(host, port(gatewayPeer, i, publicGRPC)), - Extra: []string{}, }) scopeData, err := (&uplink.Scope{ @@ -474,19 +488,14 @@ func newNetwork(flags *Flags) (*Processes, error) { } if runScopeData := vip.GetString("scope"); runScopeData != scopeData { - process.Extra = append(process.Extra, "SCOPE="+runScopeData) + process.AddExtra("SCOPE", runScopeData) if scope, err := uplink.ParseScope(runScopeData); err == nil { - process.Extra = append(process.Extra, "API_KEY="+scope.APIKey.Serialize()) + process.AddExtra("API_KEY", scope.APIKey.Serialize()) } } - accessKey := vip.GetString("minio.access-key") - secretKey := vip.GetString("minio.secret-key") - - process.Extra = append(process.Extra, - "ACCESS_KEY="+accessKey, - "SECRET_KEY="+secretKey, - ) + process.AddExtra("ACCESS_KEY", vip.GetString("minio.access-key")) + process.AddExtra("SECRET_KEY", vip.GetString("minio.secret-key")) return nil } diff --git a/cmd/storj-sim/process.go b/cmd/storj-sim/process.go index 2628bfbcc..10bf4d0af 100644 --- a/cmd/storj-sim/process.go +++ b/cmd/storj-sim/process.go @@ -82,7 +82,18 @@ type Info struct { Directory string ID string Pid int - Extra []string + Extra []EnvVar +} + +// EnvVar represents an environment variable like Key=Value +type EnvVar struct { + Key string + Value string +} + +// AddExtra appends an extra environment variable to the process info. +func (info *Info) AddExtra(key, value string) { + info.Extra = append(info.Extra, EnvVar{Key: key, Value: value}) } // Env returns process flags @@ -116,7 +127,7 @@ func (info *Info) Env() []string { env = append(env, name+"_PID="+strconv.Itoa(info.Pid)) } for _, extra := range info.Extra { - env = append(env, name+"_"+extra) + env = append(env, name+"_"+strings.ToUpper(extra.Key)+"="+extra.Value) } return env } diff --git a/go.mod b/go.mod index ddd778e11..5d760dac8 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ replace google.golang.org/grpc => github.com/storj/grpc-go v1.23.1-0.20190918084 require ( github.com/Shopify/go-lua v0.0.0-20181106184032-48449c60c0a9 + github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053 github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 // indirect github.com/alicebob/miniredis v0.0.0-20180911162847-3657542c8629 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect diff --git a/go.sum b/go.sum index d98b7d03f..3b5957d59 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdc github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053 h1:H/GMMKYPkEIC3DF/JWQz8Pdd+Feifov2EIgGfNpeogI= +github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053/go.mod h1:xW8sBma2LE3QxFSzCnH9qe6gAE2yO9GvQaWwX89HxbE= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 h1:45bxf7AZMwWcqkLzDAQugVEwedisr5nRJ1r+7LYnv0U= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis v0.0.0-20180911162847-3657542c8629 h1:gLoh8jzwIxdisBnHiWRIuReqtH9cpslSE2564UWXun0= diff --git a/scripts/test-sim.sh b/scripts/test-sim.sh index 03c3ec89b..4b34f2c61 100755 --- a/scripts/test-sim.sh +++ b/scripts/test-sim.sh @@ -27,6 +27,31 @@ else storj-sim -x --satellites 2 --host $STORJ_NETWORK_HOST4 network --postgres=$STORJ_SIM_POSTGRES setup fi +# explicitly set all the satellites and storagenodes to use mixed grpc and drpc +( + eval "$( storj-sim --satellites 2 network env )" + + N=0 + DIR="SATELLITE_${N}_DIR" + while [ -n "${!DIR:-""}" ]; do + [ $((N%2)) -eq 0 ] && KIND=drpc || KIND=grpc + BIN="$(which satellite-${KIND})" + ( set -x; cp "${BIN}" "${!DIR}/satellite" ) + let N=N+1 + DIR="SATELLITE_${N}_DIR" + done + + N=0 + DIR="STORAGENODE_${N}_DIR" + while [ -n "${!DIR:-""}" ]; do + [ $((N%2)) -eq 0 ] && KIND=drpc || KIND=grpc + BIN="$(which storagenode-${KIND})" + ( set -x; cp "${BIN}" "${!DIR}/storagenode" ) + let N=N+1 + DIR="STORAGENODE_${N}_DIR" + done +) + # set the segment size lower to make test run faster echo client.segment-size: "6 MiB" >> `storj-sim network env GATEWAY_0_DIR`/config.yaml