diff --git a/Jenkinsfile.public b/Jenkinsfile.public index 47bad1aee..e1a90d66a 100644 --- a/Jenkinsfile.public +++ b/Jenkinsfile.public @@ -35,10 +35,11 @@ pipeline { sh 'go run ./scripts/check-imports.go -race ./...' sh 'go run ./scripts/check-peer-constraints.go -race -fail=false' sh 'go run ./scripts/protobuf.go --protoc=$HOME/protoc/bin/protoc lint' - sh 'protolock status' + sh 'go run ./scripts/protobuf.go --protoc=$HOME/protoc/bin/protoc check-lock' sh 'bash ./scripts/check-dbx-version.sh' sh 'golangci-lint -j=4 run' sh 'go run scripts/check-mod-tidy.go -mod .build/go.mod.orig' + sh 'make check-satellite-config-lock' } } @@ -95,7 +96,6 @@ pipeline { steps { sh 'psql -U postgres -c \'create database teststorj2;\'' sh 'make test-sim' - sh 'make test-satellite-cfg-change' } } } diff --git a/Makefile b/Makefile index 129b9867d..5e8be09e2 100644 --- a/Makefile +++ b/Makefile @@ -90,11 +90,6 @@ test-sim: ## Test source with storj-sim (jenkins) @echo "Running ${@}" @./scripts/test-sim.sh -.PHONY: test-satellite-cfg-change -test-satellite-cfg-change: ## Test if the satellite config file has changed (jenkins) - @echo "Running ${@}" - @cd scripts; ./test-satellite-cfg-change.sh - .PHONY: test-certificate-signing test-certificate-signing: ## Test certificate signing service and storagenode setup (jenkins) @echo "Running ${@}" @@ -105,6 +100,11 @@ test-docker: ## Run tests in Docker docker-compose up -d --remove-orphans test docker-compose run test make test +.PHONY: check-satellite-config-lock +check-satellite-config-lock: ## Test if the satellite config file has changed (jenkins) + @echo "Running ${@}" + @cd scripts; ./check-satellite-config-lock.sh + .PHONY: all-in-one all-in-one: ## Deploy docker images with one storagenode locally export VERSION="${TAG}${CUSTOMTAG}" \ @@ -284,8 +284,8 @@ test-docker-clean: ## Clean up Docker environment used in test-docker target ##@ Tooling -.PHONY: update-satellite-cfg-lock -update-satellite-cfg-lock: ## Update the satellite config lock file +.PHONY: update-satellite-config-lock +update-satellite-config-lock: ## Update the satellite config lock file @docker run -ti --rm \ -v ${GOPATH}/pkg/mod:/go/pkg/mod \ -v $(shell pwd):/storj \ @@ -293,4 +293,4 @@ update-satellite-cfg-lock: ## Update the satellite config lock file -e "GOCACHE=/go-cache" \ -u root:root \ golang:${GO_VERSION} \ - /bin/bash -c "cd /storj/scripts; ./update-satellite-cfg-lock.sh" + /bin/bash -c "cd /storj/scripts; ./update-satellite-config-lock.sh" diff --git a/proto.lock b/proto.lock index 03cf56ac6..c541abe0d 100644 --- a/proto.lock +++ b/proto.lock @@ -1,5 +1,98 @@ { "definitions": [ + { + "protopath": "pkg:/:macaroon:/:types.proto", + "def": { + "messages": [ + { + "name": "Caveat", + "fields": [ + { + "id": 1, + "name": "disallow_reads", + "type": "bool" + }, + { + "id": 2, + "name": "disallow_writes", + "type": "bool" + }, + { + "id": 3, + "name": "disallow_lists", + "type": "bool" + }, + { + "id": 4, + "name": "disallow_deletes", + "type": "bool" + }, + { + "id": 10, + "name": "allowed_paths", + "type": "Path", + "is_repeated": true + }, + { + "id": 20, + "name": "not_after", + "type": "google.protobuf.Timestamp", + "options": [ + { + "name": "(gogoproto.stdtime)", + "value": "true" + } + ] + }, + { + "id": 21, + "name": "not_before", + "type": "google.protobuf.Timestamp", + "options": [ + { + "name": "(gogoproto.stdtime)", + "value": "true" + } + ] + }, + { + "id": 30, + "name": "nonce", + "type": "bytes" + } + ], + "messages": [ + { + "name": "Path", + "fields": [ + { + "id": 1, + "name": "bucket", + "type": "bytes" + }, + { + "id": 2, + "name": "encrypted_path_prefix", + "type": "bytes" + } + ] + } + ] + } + ], + "imports": [ + { + "path": "gogo.proto" + }, + { + "path": "google/protobuf/timestamp.proto" + } + ], + "package": { + "name": "macaroon" + } + } + }, { "protopath": "pkg:/:pb:/:bandwidth.proto", "def": { @@ -84,7 +177,13 @@ ], "package": { "name": "bandwidth" - } + }, + "options": [ + { + "name": "go_package", + "value": "pb" + } + ] } }, { @@ -137,7 +236,13 @@ ], "package": { "name": "node" - } + }, + "options": [ + { + "name": "go_package", + "value": "pb" + } + ] } }, { @@ -163,7 +268,13 @@ ], "package": { "name": "repair" - } + }, + "options": [ + { + "name": "go_package", + "value": "pb" + } + ] } }, { @@ -583,7 +694,21 @@ ], "package": { "name": "gogoproto" - } + }, + "options": [ + { + "name": "java_package", + "value": "com.google.protobuf" + }, + { + "name": "java_outer_classname", + "value": "GoGoProtos" + }, + { + "name": "go_package", + "value": "github.com/gogo/protobuf/gogoproto" + } + ] } }, { @@ -1402,7 +1527,13 @@ ], "package": { "name": "inspector" - } + }, + "options": [ + { + "name": "go_package", + "value": "pb" + } + ] } }, { @@ -1432,7 +1563,13 @@ ], "package": { "name": "objects" - } + }, + "options": [ + { + "name": "go_package", + "value": "pb" + } + ] } }, { @@ -1788,7 +1925,13 @@ ], "package": { "name": "metainfo" - } + }, + "options": [ + { + "name": "go_package", + "value": "pb" + } + ] } }, { @@ -2061,7 +2204,13 @@ ], "package": { "name": "node" - } + }, + "options": [ + { + "name": "go_package", + "value": "pb" + } + ] } }, { @@ -2348,7 +2497,13 @@ ], "package": { "name": "orders" - } + }, + "options": [ + { + "name": "go_package", + "value": "pb" + } + ] } }, { @@ -2521,7 +2676,13 @@ ], "package": { "name": "overlay" - } + }, + "options": [ + { + "name": "go_package", + "value": "pb" + } + ] } }, { @@ -2724,7 +2885,13 @@ ], "package": { "name": "piecestoreroutes" - } + }, + "options": [ + { + "name": "go_package", + "value": "pb" + } + ] } }, { @@ -2896,7 +3063,13 @@ ], "package": { "name": "piecestore" - } + }, + "options": [ + { + "name": "go_package", + "value": "pb" + } + ] } }, { @@ -3124,7 +3297,13 @@ ], "package": { "name": "pointerdb" - } + }, + "options": [ + { + "name": "go_package", + "value": "pb" + } + ] } }, { @@ -3199,7 +3378,97 @@ ], "package": { "name": "streams" - } + }, + "options": [ + { + "name": "go_package", + "value": "pb" + } + ] + } + }, + { + "protopath": "pkg:/:pb:/:vouchers.proto", + "def": { + "messages": [ + { + "name": "Voucher", + "fields": [ + { + "id": 1, + "name": "satellite_id", + "type": "bytes", + "options": [ + { + "name": "(gogoproto.customtype)", + "value": "NodeID" + }, + { + "name": "(gogoproto.nullable)", + "value": "false" + } + ] + }, + { + "id": 2, + "name": "storage_node_id", + "type": "bytes", + "options": [ + { + "name": "(gogoproto.customtype)", + "value": "NodeID" + }, + { + "name": "(gogoproto.nullable)", + "value": "false" + } + ] + }, + { + "id": 3, + "name": "expiration", + "type": "google.protobuf.Timestamp" + }, + { + "id": 4, + "name": "satellite_signature", + "type": "bytes" + } + ] + }, + { + "name": "VoucherRequest" + } + ], + "services": [ + { + "name": "Vouchers", + "rpcs": [ + { + "name": "Request", + "in_type": "VoucherRequest", + "out_type": "Voucher" + } + ] + } + ], + "imports": [ + { + "path": "gogo.proto" + }, + { + "path": "google/protobuf/timestamp.proto" + } + ], + "package": { + "name": "vouchers" + }, + "options": [ + { + "name": "go_package", + "value": "pb" + } + ] } } ] diff --git a/scripts/check-satellite-config-lock.sh b/scripts/check-satellite-config-lock.sh new file mode 100755 index 000000000..6baba10b1 --- /dev/null +++ b/scripts/check-satellite-config-lock.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# NOTE this script MUST BE EXECUTED from the same directory where it's located +# to always obtain the same paths in the satellite configuration file. + +set -uo pipefail + +#setup tmpdir for testfiles and cleanup +TMPDIR=$(mktemp -d -t tmp.XXXXXXXXXX) +cleanup(){ + rm -rf "$TMPDIR" +} +trap cleanup EXIT + +pushd "$TMPDIR" +satellite --config-dir "./testdata" --defaults release setup > /dev/null +popd + +diff "./testdata/satellite-config.yaml.lock" "$TMPDIR/testdata/config.yaml" +if [[ $? != 0 ]]; then + echo + echo "NOTIFY the Devops and PM when this test fails so they can plan for changing it in the release process before fixing it to merge your PR." + echo "Once you have notified them you can update the lock file through another Makefile target" + echo + exit 1 +fi diff --git a/scripts/protobuf.go b/scripts/protobuf.go index 4d6de9e28..789cfb74b 100644 --- a/scripts/protobuf.go +++ b/scripts/protobuf.go @@ -6,6 +6,7 @@ package main import ( + "bytes" "errors" "flag" "fmt" @@ -16,6 +17,8 @@ import ( "regexp" "sort" "strings" + + "github.com/kylelemons/godebug/diff" ) var ignoreProto = map[string]bool{ @@ -56,12 +59,14 @@ func run(command, root string) error { "github.com/ckaznocha/protoc-gen-lint@68a05858965b31eb872cbeb8d027507a94011acc", // See https://github.com/gogo/protobuf#most-speed-and-most-customization "github.com/gogo/protobuf/protoc-gen-gogo@"+gogoVersion, - "github.com/nilslice/protolock/cmd/protolock", + "github.com/nilslice/protolock/cmd/protolock@v0.12.0", ) case "generate": return walkdirs(root, generate) case "lint": return walkdirs(root, lint) + case "check-lock": + return walkdirs(root, checklock) default: return errors.New("unknown command " + command) } @@ -177,6 +182,57 @@ func lint(dir string, dirs []string, files []string) error { return err } +func checklock(dir string, dirs []string, files []string) error { + defer switchdir(dir)() + + local, err := os.Getwd() + if err != nil { + panic(err) + } + + protolockdir := findProtolockDir(local) + + tmpdir, err := ioutil.TempDir("", "protolock") + if err != nil { + return err + } + defer func() { + _ = os.RemoveAll(tmpdir) + }() + + original, err := ioutil.ReadFile(filepath.Join(protolockdir, "proto.lock")) + if err != nil { + return fmt.Errorf("unable to read proto.lock: %v", err) + } + + err = ioutil.WriteFile(filepath.Join(tmpdir, "proto.lock"), original, 0755) + if err != nil { + return fmt.Errorf("unable to read proto.lock: %v", err) + } + + cmd := exec.Command("protolock", "commit", "-lockdir", tmpdir) + cmd.Dir = protolockdir + out, err := cmd.CombinedOutput() + if len(out) > 0 { + fmt.Println(string(out)) + } + if err != nil { + return err + } + + changed, err := ioutil.ReadFile(filepath.Join(tmpdir, "proto.lock")) + if err != nil { + return fmt.Errorf("unable to read new proto.lock: %v", err) + } + + if !bytes.Equal(original, changed) { + diff, _ := difflines(string(original), string(changed)) + return fmt.Errorf("protolock is not up to date: %v", diff) + } + + return nil +} + func switchdir(to string) func() { local, err := os.Getwd() if err != nil { @@ -259,3 +315,22 @@ func findProtolockDir(dir string) string { return dir } + +func difflines(a, b string) (patch string, removed bool) { + alines, blines := strings.Split(a, "\n"), strings.Split(b, "\n") + + chunks := diff.DiffChunks(alines, blines) + + buf := new(bytes.Buffer) + for _, c := range chunks { + for _, line := range c.Added { + fmt.Fprintf(buf, "+%s\n", line) + } + for _, line := range c.Deleted { + fmt.Fprintf(buf, "-%s\n", line) + removed = true + } + } + + return strings.TrimRight(buf.String(), "\n"), removed +} diff --git a/scripts/test-satellite-cfg-change.sh b/scripts/test-satellite-cfg-change.sh deleted file mode 100755 index 3a9a245db..000000000 --- a/scripts/test-satellite-cfg-change.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -# NOTE this script MUST BE EXECUTED from the same directory where it's located -# to always obtain the same paths in the satellite configuration file. - -set -uo pipefail - -TESTDATA_DIR="./testdata" -cleanup(){ - rm "$TESTDATA_DIR/config.yaml" -} -trap cleanup EXIT - -satellite --config-dir "$TESTDATA_DIR" --defaults release setup > /dev/null - - -diff "$TESTDATA_DIR/satellite-config.yaml.lock" "$TESTDATA_DIR/config.yaml" -if [[ $? != 0 ]]; then - echo - echo "NOTIFY the Devops and PM when this test fails so they can plan for changing it in the release process before fixing it to merge your PR." - echo "Once you have notified them you can update the lock file through another Makefile target" - echo - exit 1 -fi diff --git a/scripts/update-satellite-cfg-lock.sh b/scripts/update-satellite-config-lock.sh similarity index 100% rename from scripts/update-satellite-cfg-lock.sh rename to scripts/update-satellite-config-lock.sh