JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

jenkins: check that lock file is updated properly (#1977)

Browse Source
This commit is contained in:
Egon Elbre 2019-06-04 20:12:27 +03:00 committed by Stefan Benten
parent d6c02fc657
commit 9ed8c96058
7 changed files with 395 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Jenkinsfile.public
View File

@ -35,10 +35,11 @@ pipeline {
sh 'go run ./scripts/check-imports.go -race ./...'
sh 'go run ./scripts/check-peer-constraints.go -race -fail=false'
sh 'go run ./scripts/protobuf.go --protoc=$HOME/protoc/bin/protoc lint'
sh 'protolock status'
sh 'go run ./scripts/protobuf.go --protoc=$HOME/protoc/bin/protoc check-lock'
sh 'bash ./scripts/check-dbx-version.sh'
sh 'golangci-lint -j=4 run'
sh 'go run scripts/check-mod-tidy.go -mod .build/go.mod.orig'
sh 'make check-satellite-config-lock'
}
}
@ -95,7 +96,6 @@ pipeline {
steps {
sh 'psql -U postgres -c \'create database teststorj2;\''
sh 'make test-sim'
sh 'make test-satellite-cfg-change'
}
}
}

16
Makefile
View File

@ -90,11 +90,6 @@ test-sim: ## Test source with storj-sim (jenkins)
@echo "Running ${@}"
@./scripts/test-sim.sh
.PHONY: test-satellite-cfg-change
test-satellite-cfg-change: ## Test if the satellite config file has changed (jenkins)
@echo "Running ${@}"
@cd scripts; ./test-satellite-cfg-change.sh
.PHONY: test-certificate-signing
test-certificate-signing: ## Test certificate signing service and storagenode setup (jenkins)
@echo "Running ${@}"
@ -105,6 +100,11 @@ test-docker: ## Run tests in Docker
docker-compose up -d --remove-orphans test
docker-compose run test make test
.PHONY: check-satellite-config-lock
check-satellite-config-lock: ## Test if the satellite config file has changed (jenkins)
@echo "Running ${@}"
@cd scripts; ./check-satellite-config-lock.sh
.PHONY: all-in-one
all-in-one: ## Deploy docker images with one storagenode locally
export VERSION="${TAG}${CUSTOMTAG}" \
@ -284,8 +284,8 @@ test-docker-clean: ## Clean up Docker environment used in test-docker target
##@ Tooling
.PHONY: update-satellite-cfg-lock
update-satellite-cfg-lock: ## Update the satellite config lock file
.PHONY: update-satellite-config-lock
update-satellite-config-lock: ## Update the satellite config lock file
@docker run -ti --rm \
-v ${GOPATH}/pkg/mod:/go/pkg/mod \
-v $(shell pwd):/storj \
@ -293,4 +293,4 @@ update-satellite-cfg-lock: ## Update the satellite config lock file
-e "GOCACHE=/go-cache" \
-u root:root \
golang:${GO_VERSION} \
/bin/bash -c "cd /storj/scripts; ./update-satellite-cfg-lock.sh"
/bin/bash -c "cd /storj/scripts; ./update-satellite-config-lock.sh"

297
proto.lock
View File

@ -1,5 +1,98 @@
{
"definitions": [
{
"protopath": "pkg:/:macaroon:/:types.proto",
"def": {
"messages": [
{
"name": "Caveat",
"fields": [
{
"id": 1,
"name": "disallow_reads",
"type": "bool"
},
{
"id": 2,
"name": "disallow_writes",
"type": "bool"
},
{
"id": 3,
"name": "disallow_lists",
"type": "bool"
},
{
"id": 4,
"name": "disallow_deletes",
"type": "bool"
},
{
"id": 10,
"name": "allowed_paths",
"type": "Path",
"is_repeated": true
},
{
"id": 20,
"name": "not_after",
"type": "google.protobuf.Timestamp",
"options": [
{
"name": "(gogoproto.stdtime)",
"value": "true"
}
]
},
{
"id": 21,
"name": "not_before",
"type": "google.protobuf.Timestamp",
"options": [
{
"name": "(gogoproto.stdtime)",
"value": "true"
}
]
},
{
"id": 30,
"name": "nonce",
"type": "bytes"
}
],
"messages": [
{
"name": "Path",
"fields": [
{
"id": 1,
"name": "bucket",
"type": "bytes"
},
{
"id": 2,
"name": "encrypted_path_prefix",
"type": "bytes"
}
]
}
]
}
],
"imports": [
{
"path": "gogo.proto"
},
{
"path": "google/protobuf/timestamp.proto"
}
],
"package": {
"name": "macaroon"
}
}
},
{
"protopath": "pkg:/:pb:/:bandwidth.proto",
"def": {
@ -84,7 +177,13 @@
],
"package": {
"name": "bandwidth"
}
},
"options": [
{
"name": "go_package",
"value": "pb"
}
]
}
},
{
@ -137,7 +236,13 @@
],
"package": {
"name": "node"
}
},
"options": [
{
"name": "go_package",
"value": "pb"
}
]
}
},
{
@ -163,7 +268,13 @@
],
"package": {
"name": "repair"
}
},
"options": [
{
"name": "go_package",
"value": "pb"
}
]
}
},
{
@ -583,7 +694,21 @@
],
"package": {
"name": "gogoproto"
}
},
"options": [
{
"name": "java_package",
"value": "com.google.protobuf"
},
{
"name": "java_outer_classname",
"value": "GoGoProtos"
},
{
"name": "go_package",
"value": "github.com/gogo/protobuf/gogoproto"
}
]
}
},
{
@ -1402,7 +1527,13 @@
],
"package": {
"name": "inspector"
}
},
"options": [
{
"name": "go_package",
"value": "pb"
}
]
}
},
{
@ -1432,7 +1563,13 @@
],
"package": {
"name": "objects"
}
},
"options": [
{
"name": "go_package",
"value": "pb"
}
]
}
},
{
@ -1788,7 +1925,13 @@
],
"package": {
"name": "metainfo"
}
},
"options": [
{
"name": "go_package",
"value": "pb"
}
]
}
},
{
@ -2061,7 +2204,13 @@
],
"package": {
"name": "node"
}
},
"options": [
{
"name": "go_package",
"value": "pb"
}
]
}
},
{
@ -2348,7 +2497,13 @@
],
"package": {
"name": "orders"
}
},
"options": [
{
"name": "go_package",
"value": "pb"
}
]
}
},
{
@ -2521,7 +2676,13 @@
],
"package": {
"name": "overlay"
}
},
"options": [
{
"name": "go_package",
"value": "pb"
}
]
}
},
{
@ -2724,7 +2885,13 @@
],
"package": {
"name": "piecestoreroutes"
}
},
"options": [
{
"name": "go_package",
"value": "pb"
}
]
}
},
{
@ -2896,7 +3063,13 @@
],
"package": {
"name": "piecestore"
}
},
"options": [
{
"name": "go_package",
"value": "pb"
}
]
}
},
{
@ -3124,7 +3297,13 @@
],
"package": {
"name": "pointerdb"
}
},
"options": [
{
"name": "go_package",
"value": "pb"
}
]
}
},
{
@ -3199,7 +3378,97 @@
],
"package": {
"name": "streams"
}
},
"options": [
{
"name": "go_package",
"value": "pb"
}
]
}
},
{
"protopath": "pkg:/:pb:/:vouchers.proto",
"def": {
"messages": [
{
"name": "Voucher",
"fields": [
{
"id": 1,
"name": "satellite_id",
"type": "bytes",
"options": [
{
"name": "(gogoproto.customtype)",
"value": "NodeID"
},
{
"name": "(gogoproto.nullable)",
"value": "false"
}
]
},
{
"id": 2,
"name": "storage_node_id",
"type": "bytes",
"options": [
{
"name": "(gogoproto.customtype)",
"value": "NodeID"
},
{
"name": "(gogoproto.nullable)",
"value": "false"
}
]
},
{
"id": 3,
"name": "expiration",
"type": "google.protobuf.Timestamp"
},
{
"id": 4,
"name": "satellite_signature",
"type": "bytes"
}
]
},
{
"name": "VoucherRequest"
}
],
"services": [
{
"name": "Vouchers",
"rpcs": [
{
"name": "Request",
"in_type": "VoucherRequest",
"out_type": "Voucher"
}
]
}
],
"imports": [
{
"path": "gogo.proto"
},
{
"path": "google/protobuf/timestamp.proto"
}
],
"package": {
"name": "vouchers"
},
"options": [
{
"name": "go_package",
"value": "pb"
}
]
}
}
]

26
scripts/check-satellite-config-lock.sh Executable file
View File

@ -0,0 +1,26 @@
#!/usr/bin/env bash
# NOTE this script MUST BE EXECUTED from the same directory where it's located
# to always obtain the same paths in the satellite configuration file.
set -uo pipefail
#setup tmpdir for testfiles and cleanup
TMPDIR=$(mktemp -d -t tmp.XXXXXXXXXX)
cleanup(){
rm -rf "$TMPDIR"
}
trap cleanup EXIT
pushd "$TMPDIR"
satellite --config-dir "./testdata" --defaults release setup > /dev/null
popd
diff "./testdata/satellite-config.yaml.lock" "$TMPDIR/testdata/config.yaml"
if [[ $? != 0 ]]; then
echo
echo "NOTIFY the Devops and PM when this test fails so they can plan for changing it in the release process before fixing it to merge your PR."
echo "Once you have notified them you can update the lock file through another Makefile target"
echo
exit 1
fi

77
scripts/protobuf.go
View File

@ -6,6 +6,7 @@
package main
import (
"bytes"
"errors"
"flag"
"fmt"
@ -16,6 +17,8 @@ import (
"regexp"
"sort"
"strings"
"github.com/kylelemons/godebug/diff"
)
var ignoreProto = map[string]bool{
@ -56,12 +59,14 @@ func run(command, root string) error {
"github.com/ckaznocha/protoc-gen-lint@68a05858965b31eb872cbeb8d027507a94011acc",
// See https://github.com/gogo/protobuf#most-speed-and-most-customization
"github.com/gogo/protobuf/protoc-gen-gogo@"+gogoVersion,
"github.com/nilslice/protolock/cmd/protolock",
"github.com/nilslice/protolock/cmd/protolock@v0.12.0",
)
case "generate":
return walkdirs(root, generate)
case "lint":
return walkdirs(root, lint)
case "check-lock":
return walkdirs(root, checklock)
default:
return errors.New("unknown command " + command)
}
@ -177,6 +182,57 @@ func lint(dir string, dirs []string, files []string) error {
return err
}
func checklock(dir string, dirs []string, files []string) error {
defer switchdir(dir)()
local, err := os.Getwd()
if err != nil {
panic(err)
}
protolockdir := findProtolockDir(local)
tmpdir, err := ioutil.TempDir("", "protolock")
if err != nil {
return err
}
defer func() {
_ = os.RemoveAll(tmpdir)
}()
original, err := ioutil.ReadFile(filepath.Join(protolockdir, "proto.lock"))
if err != nil {
return fmt.Errorf("unable to read proto.lock: %v", err)
}
err = ioutil.WriteFile(filepath.Join(tmpdir, "proto.lock"), original, 0755)
if err != nil {
return fmt.Errorf("unable to read proto.lock: %v", err)
}
cmd := exec.Command("protolock", "commit", "-lockdir", tmpdir)
cmd.Dir = protolockdir
out, err := cmd.CombinedOutput()
if len(out) > 0 {
fmt.Println(string(out))
}
if err != nil {
return err
}
changed, err := ioutil.ReadFile(filepath.Join(tmpdir, "proto.lock"))
if err != nil {
return fmt.Errorf("unable to read new proto.lock: %v", err)
}
if !bytes.Equal(original, changed) {
diff, _ := difflines(string(original), string(changed))
return fmt.Errorf("protolock is not up to date: %v", diff)
}
return nil
}
func switchdir(to string) func() {
local, err := os.Getwd()
if err != nil {
@ -259,3 +315,22 @@ func findProtolockDir(dir string) string {
return dir
}
func difflines(a, b string) (patch string, removed bool) {
alines, blines := strings.Split(a, "\n"), strings.Split(b, "\n")
chunks := diff.DiffChunks(alines, blines)
buf := new(bytes.Buffer)
for _, c := range chunks {
for _, line := range c.Added {
fmt.Fprintf(buf, "+%s\n", line)
}
for _, line := range c.Deleted {
fmt.Fprintf(buf, "-%s\n", line)
removed = true
}
}
return strings.TrimRight(buf.String(), "\n"), removed
}

24
scripts/test-satellite-cfg-change.sh
View File

@ -1,24 +0,0 @@
#!/usr/bin/env bash
# NOTE this script MUST BE EXECUTED from the same directory where it's located
# to always obtain the same paths in the satellite configuration file.
set -uo pipefail
TESTDATA_DIR="./testdata"
cleanup(){
rm "$TESTDATA_DIR/config.yaml"
}
trap cleanup EXIT
satellite --config-dir "$TESTDATA_DIR" --defaults release setup > /dev/null
diff "$TESTDATA_DIR/satellite-config.yaml.lock" "$TESTDATA_DIR/config.yaml"
if [[ $? != 0 ]]; then
echo
echo "NOTIFY the Devops and PM when this test fails so they can plan for changing it in the release process before fixing it to merge your PR."
echo "Once you have notified them you can update the lock file through another Makefile target"
echo
exit 1
fi

0
scripts/update-satellite-cfg-lock.sh → scripts/update-satellite-config-lock.sh
View File