JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

all: build release fully dprc and test in mixed mode

Browse Source 
Change-Id: I3bded3edf25a0b113601c8b12ecf1337f596649b
This commit is contained in:
Jeff Wendling 2019-11-11 11:58:08 -07:00
parent 2524cc5d42
commit ecd2ef4a21
6 changed files with 84 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
Makefile
View File

@ -85,17 +85,24 @@ build-npm:
##@ Simulator
.PHONY: go-install-grpc-and-drpc
go-install-grpc-and-drpc:
@: $(if ${PACKAGE},,$(error PACKAGE must be defined for the go-install-grpc-and-drpc target))
go build -race -v -tags=grpc -o "$(shell go list -f '{{.Target}}' ${PACKAGE})-grpc" "${PACKAGE}"
go build -race -v -tags=drpc -o "$(shell go list -f '{{.Target}}' ${PACKAGE})-drpc" "${PACKAGE}"
go install -race -v "${PACKAGE}"
.PHONY: install-sim
install-sim: ## install storj-sim
@echo "Running ${@}"
@go install -race -v -tags=grpc storj.io/storj/cmd/storj-sim
@go install -race -v -tags=grpc storj.io/storj/cmd/versioncontrol
@go install -race -v -tags=grpc storj.io/storj/cmd/satellite
@go install -race -v -tags=grpc storj.io/storj/cmd/storagenode
@go install -race -v storj.io/storj/cmd/uplink
@go install -race -v storj.io/storj/cmd/gateway
@go install -race -v -tags=grpc storj.io/storj/cmd/identity
@go install -race -v -tags=grpc storj.io/storj/cmd/certificates
$(MAKE) go-install-grpc-and-drpc PACKAGE=storj.io/storj/cmd/storagenode
$(MAKE) go-install-grpc-and-drpc PACKAGE=storj.io/storj/cmd/satellite
go install -race -v storj.io/storj/cmd/storj-sim
go install -race -v storj.io/storj/cmd/versioncontrol
go install -race -v storj.io/storj/cmd/uplink
go install -race -v storj.io/storj/cmd/gateway
go install -race -v storj.io/storj/cmd/identity
go install -race -v storj.io/storj/cmd/certificates
##@ Test
@ -269,34 +276,34 @@ binary-check:
.PHONY: certificates_%
certificates_%:
$(MAKE) binary-check COMPONENT=certificates GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) EXTRA_ARGS=-tags=grpc
$(MAKE) binary-check COMPONENT=certificates GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@))
.PHONY: gateway_%
gateway_%:
$(MAKE) binary-check COMPONENT=gateway GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@))
.PHONY: identity_%
identity_%:
$(MAKE) binary-check COMPONENT=identity GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) EXTRA_ARGS=-tags=grpc
$(MAKE) binary-check COMPONENT=identity GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@))
.PHONY: inspector_%
inspector_%:
$(MAKE) binary-check COMPONENT=inspector GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) EXTRA_ARGS=-tags=grpc
$(MAKE) binary-check COMPONENT=inspector GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@))
.PHONY: linksharing_%
linksharing_%:
$(MAKE) binary-check COMPONENT=linksharing GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@))
.PHONY: satellite_%
satellite_%:
$(MAKE) binary-check COMPONENT=satellite GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) EXTRA_ARGS=-tags=grpc
$(MAKE) binary-check COMPONENT=satellite GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@))
.PHONY: storagenode_%
storagenode_%: storagenode-console
$(MAKE) binary-check COMPONENT=storagenode GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) EXTRA_ARGS=-tags=grpc
$(MAKE) binary-check COMPONENT=storagenode GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@))
.PHONY: storagenode-updater_%
storagenode-updater_%:
$(MAKE) binary-check COMPONENT=storagenode-updater GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) EXTRA_ARGS=-tags=grpc
$(MAKE) binary-check COMPONENT=storagenode-updater GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@))
.PHONY: uplink_%
uplink_%:
$(MAKE) binary-check COMPONENT=uplink GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@))
.PHONY: versioncontrol_%
versioncontrol_%:
$(MAKE) binary-check COMPONENT=versioncontrol GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@)) EXTRA_ARGS=-tags=grpc
$(MAKE) binary-check COMPONENT=versioncontrol GOARCH=$(word 3, $(subst _, ,$@)) GOOS=$(word 2, $(subst _, ,$@))
COMPONENTLIST := certificates gateway identity inspector linksharing satellite storagenode storagenode-updater uplink versioncontrol

33
cmd/storj-sim/network.go
View File

@ -18,6 +18,7 @@ import (
"strings"
"time"
"github.com/alessio/shellescape"
"github.com/spf13/viper"
"github.com/zeebo/errs"
"golang.org/x/sync/errgroup"
@ -104,6 +105,20 @@ func networkExec(flags *Flags, args []string, command string) error {
return errs.Combine(err, closeErr)
}
func escapeEnv(env string) string {
// TODO(jeff): escape env variables appropriately on windows. perhaps the
// env output should be of the form `set KEY=VALUE` as well.
if runtime.GOOS == "windows" {
return env
}
parts := strings.SplitN(env, "=", 2)
if len(parts) != 2 {
return env
}
return parts[0] + "=" + shellescape.Quote(parts[1])
}
func networkEnv(flags *Flags, args []string) error {
flags.OnlyEnv = true
@ -126,7 +141,7 @@ func networkEnv(flags *Flags, args []string) error {
// find the environment value that the environment variable is set to
for _, env := range processes.Env() {
if strings.HasPrefix(strings.ToUpper(env), envprefix) {
fmt.Println(env[len(envprefix):])
fmt.Println(escapeEnv(env[len(envprefix):]))
return nil
}
}
@ -135,7 +150,7 @@ func networkEnv(flags *Flags, args []string) error {
}
for _, env := range processes.Env() {
fmt.Println(env)
fmt.Println(escapeEnv(env))
}
return nil
@ -381,7 +396,6 @@ func newNetwork(flags *Flags) (*Processes, error) {
Executable: "gateway",
Directory: filepath.Join(processes.Directory, "gateway", fmt.Sprint(i)),
Address: net.JoinHostPort(host, port(gatewayPeer, i, publicGRPC)),
Extra: []string{},
})
scopeData, err := (&uplink.Scope{
@ -474,19 +488,14 @@ func newNetwork(flags *Flags) (*Processes, error) {
}
if runScopeData := vip.GetString("scope"); runScopeData != scopeData {
process.Extra = append(process.Extra, "SCOPE="+runScopeData)
process.AddExtra("SCOPE", runScopeData)
if scope, err := uplink.ParseScope(runScopeData); err == nil {
process.Extra = append(process.Extra, "API_KEY="+scope.APIKey.Serialize())
process.AddExtra("API_KEY", scope.APIKey.Serialize())
}
}
accessKey := vip.GetString("minio.access-key")
secretKey := vip.GetString("minio.secret-key")
process.Extra = append(process.Extra,
"ACCESS_KEY="+accessKey,
"SECRET_KEY="+secretKey,
)
process.AddExtra("ACCESS_KEY", vip.GetString("minio.access-key"))
process.AddExtra("SECRET_KEY", vip.GetString("minio.secret-key"))
return nil
}

15
cmd/storj-sim/process.go
View File

@ -82,7 +82,18 @@ type Info struct {
Directory string
ID string
Pid int
Extra []string
Extra []EnvVar
}
// EnvVar represents an environment variable like Key=Value
type EnvVar struct {
Key string
Value string
}
// AddExtra appends an extra environment variable to the process info.
func (info *Info) AddExtra(key, value string) {
info.Extra = append(info.Extra, EnvVar{Key: key, Value: value})
}
// Env returns process flags
@ -116,7 +127,7 @@ func (info *Info) Env() []string {
env = append(env, name+"_PID="+strconv.Itoa(info.Pid))
}
for _, extra := range info.Extra {
env = append(env, name+"_"+extra)
env = append(env, name+"_"+strings.ToUpper(extra.Key)+"="+extra.Value)
}
return env
}

1
go.mod
View File

@ -18,6 +18,7 @@ replace google.golang.org/grpc => github.com/storj/grpc-go v1.23.1-0.20190918084
require (
github.com/Shopify/go-lua v0.0.0-20181106184032-48449c60c0a9
github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053
github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 // indirect
github.com/alicebob/miniredis v0.0.0-20180911162847-3657542c8629
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect

2
go.sum
View File

@ -19,6 +19,8 @@ github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdc
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053 h1:H/GMMKYPkEIC3DF/JWQz8Pdd+Feifov2EIgGfNpeogI=
github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053/go.mod h1:xW8sBma2LE3QxFSzCnH9qe6gAE2yO9GvQaWwX89HxbE=
github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 h1:45bxf7AZMwWcqkLzDAQugVEwedisr5nRJ1r+7LYnv0U=
github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
github.com/alicebob/miniredis v0.0.0-20180911162847-3657542c8629 h1:gLoh8jzwIxdisBnHiWRIuReqtH9cpslSE2564UWXun0=

25
scripts/test-sim.sh
View File

@ -27,6 +27,31 @@ else
storj-sim -x --satellites 2 --host $STORJ_NETWORK_HOST4 network --postgres=$STORJ_SIM_POSTGRES setup
fi
# explicitly set all the satellites and storagenodes to use mixed grpc and drpc
(
eval "$( storj-sim --satellites 2 network env )"
N=0
DIR="SATELLITE_${N}_DIR"
while [ -n "${!DIR:-""}" ]; do
[ $((N%2)) -eq 0 ] && KIND=drpc || KIND=grpc
BIN="$(which satellite-${KIND})"
( set -x; cp "${BIN}" "${!DIR}/satellite" )
let N=N+1
DIR="SATELLITE_${N}_DIR"
done
N=0
DIR="STORAGENODE_${N}_DIR"
while [ -n "${!DIR:-""}" ]; do
[ $((N%2)) -eq 0 ] && KIND=drpc || KIND=grpc
BIN="$(which storagenode-${KIND})"
( set -x; cp "${BIN}" "${!DIR}/storagenode" )
let N=N+1
DIR="STORAGENODE_${N}_DIR"
done
)
# set the segment size lower to make test run faster
echo client.segment-size: "6 MiB" >> `storj-sim network env GATEWAY_0_DIR`/config.yaml