jenkins: check that lock file is updated properly (#1977)

Jenkinsfile.public

@@ -35,10 +35,11 @@ pipeline {
                         sh 'go run ./scripts/check-imports.go -race ./...'
                         sh 'go run ./scripts/check-peer-constraints.go -race -fail=false'
                         sh 'go run ./scripts/protobuf.go --protoc=$HOME/protoc/bin/protoc lint'
-                        sh 'protolock status'
+                        sh 'go run ./scripts/protobuf.go --protoc=$HOME/protoc/bin/protoc check-lock'
                         sh 'bash ./scripts/check-dbx-version.sh'
                         sh 'golangci-lint -j=4 run'
                         sh 'go run scripts/check-mod-tidy.go -mod .build/go.mod.orig'
+                        sh 'make check-satellite-config-lock'
                     }
                 }
 
@@ -95,7 +96,6 @@ pipeline {
                     steps {
                         sh 'psql -U postgres -c \'create database teststorj2;\''
                         sh 'make test-sim'
-                        sh 'make test-satellite-cfg-change'
                     }
                 }
             }

Makefile

@@ -90,11 +90,6 @@ test-sim: ## Test source with storj-sim (jenkins)
 	@echo "Running ${@}"
 	@./scripts/test-sim.sh
 
-.PHONY: test-satellite-cfg-change
-test-satellite-cfg-change: ## Test if the satellite config file has changed (jenkins)
-	@echo "Running ${@}"
-	@cd scripts; ./test-satellite-cfg-change.sh
-
 .PHONY: test-certificate-signing
 test-certificate-signing: ## Test certificate signing service and storagenode setup (jenkins)
 	@echo "Running ${@}"
@@ -105,6 +100,11 @@ test-docker: ## Run tests in Docker
 	docker-compose up -d --remove-orphans test
 	docker-compose run test make test
 
+.PHONY: check-satellite-config-lock
+check-satellite-config-lock: ## Test if the satellite config file has changed (jenkins)
+	@echo "Running ${@}"
+	@cd scripts; ./check-satellite-config-lock.sh
+
 .PHONY: all-in-one
 all-in-one: ## Deploy docker images with one storagenode locally
 	export VERSION="${TAG}${CUSTOMTAG}" \
@@ -284,8 +284,8 @@ test-docker-clean: ## Clean up Docker environment used in test-docker target
 
 ##@ Tooling
 
-.PHONY: update-satellite-cfg-lock
+.PHONY: update-satellite-config-lock
-update-satellite-cfg-lock: ## Update the satellite config lock file
+update-satellite-config-lock: ## Update the satellite config lock file
 	@docker run -ti --rm \
 		-v ${GOPATH}/pkg/mod:/go/pkg/mod \
 		-v $(shell pwd):/storj \
@@ -293,4 +293,4 @@ update-satellite-cfg-lock: ## Update the satellite config lock file
 		-e "GOCACHE=/go-cache" \
 		-u root:root \
 		golang:${GO_VERSION} \
-		/bin/bash -c "cd /storj/scripts; ./update-satellite-cfg-lock.sh"
+		/bin/bash -c "cd /storj/scripts; ./update-satellite-config-lock.sh"

proto.lock

@@ -1,5 +1,98 @@
 {
   "definitions": [
+    {
+      "protopath": "pkg:/:macaroon:/:types.proto",
+      "def": {
+        "messages": [
+          {
+            "name": "Caveat",
+            "fields": [
+              {
+                "id": 1,
+                "name": "disallow_reads",
+                "type": "bool"
+              },
+              {
+                "id": 2,
+                "name": "disallow_writes",
+                "type": "bool"
+              },
+              {
+                "id": 3,
+                "name": "disallow_lists",
+                "type": "bool"
+              },
+              {
+                "id": 4,
+                "name": "disallow_deletes",
+                "type": "bool"
+              },
+              {
+                "id": 10,
+                "name": "allowed_paths",
+                "type": "Path",
+                "is_repeated": true
+              },
+              {
+                "id": 20,
+                "name": "not_after",
+                "type": "google.protobuf.Timestamp",
+                "options": [
+                  {
+                    "name": "(gogoproto.stdtime)",
+                    "value": "true"
+                  }
+                ]
+              },
+              {
+                "id": 21,
+                "name": "not_before",
+                "type": "google.protobuf.Timestamp",
+                "options": [
+                  {
+                    "name": "(gogoproto.stdtime)",
+                    "value": "true"
+                  }
+                ]
+              },
+              {
+                "id": 30,
+                "name": "nonce",
+                "type": "bytes"
+              }
+            ],
+            "messages": [
+              {
+                "name": "Path",
+                "fields": [
+                  {
+                    "id": 1,
+                    "name": "bucket",
+                    "type": "bytes"
+                  },
+                  {
+                    "id": 2,
+                    "name": "encrypted_path_prefix",
+                    "type": "bytes"
+                  }
+                ]
+              }
+            ]
+          }
+        ],
+        "imports": [
+          {
+            "path": "gogo.proto"
+          },
+          {
+            "path": "google/protobuf/timestamp.proto"
+          }
+        ],
+        "package": {
+          "name": "macaroon"
+        }
+      }
+    },
     {
       "protopath": "pkg:/:pb:/:bandwidth.proto",
       "def": {
@@ -84,7 +177,13 @@
         ],
         "package": {
           "name": "bandwidth"
+        },
+        "options": [
+          {
+            "name": "go_package",
+            "value": "pb"
           }
+        ]
       }
     },
     {
@@ -137,7 +236,13 @@
         ],
         "package": {
           "name": "node"
+        },
+        "options": [
+          {
+            "name": "go_package",
+            "value": "pb"
           }
+        ]
       }
     },
     {
@@ -163,7 +268,13 @@
         ],
         "package": {
           "name": "repair"
+        },
+        "options": [
+          {
+            "name": "go_package",
+            "value": "pb"
           }
+        ]
       }
     },
     {
@@ -583,7 +694,21 @@
         ],
         "package": {
           "name": "gogoproto"
+        },
+        "options": [
+          {
+            "name": "java_package",
+            "value": "com.google.protobuf"
+          },
+          {
+            "name": "java_outer_classname",
+            "value": "GoGoProtos"
+          },
+          {
+            "name": "go_package",
+            "value": "github.com/gogo/protobuf/gogoproto"
           }
+        ]
       }
     },
     {
@@ -1402,7 +1527,13 @@
         ],
         "package": {
           "name": "inspector"
+        },
+        "options": [
+          {
+            "name": "go_package",
+            "value": "pb"
           }
+        ]
       }
     },
     {
@@ -1432,7 +1563,13 @@
         ],
         "package": {
           "name": "objects"
+        },
+        "options": [
+          {
+            "name": "go_package",
+            "value": "pb"
           }
+        ]
       }
     },
     {
@@ -1788,7 +1925,13 @@
         ],
         "package": {
           "name": "metainfo"
+        },
+        "options": [
+          {
+            "name": "go_package",
+            "value": "pb"
           }
+        ]
       }
     },
     {
@@ -2061,7 +2204,13 @@
         ],
         "package": {
           "name": "node"
+        },
+        "options": [
+          {
+            "name": "go_package",
+            "value": "pb"
           }
+        ]
       }
     },
     {
@@ -2348,7 +2497,13 @@
         ],
         "package": {
           "name": "orders"
+        },
+        "options": [
+          {
+            "name": "go_package",
+            "value": "pb"
           }
+        ]
       }
     },
     {
@@ -2521,7 +2676,13 @@
         ],
         "package": {
           "name": "overlay"
+        },
+        "options": [
+          {
+            "name": "go_package",
+            "value": "pb"
           }
+        ]
       }
     },
     {
@@ -2724,7 +2885,13 @@
         ],
         "package": {
           "name": "piecestoreroutes"
+        },
+        "options": [
+          {
+            "name": "go_package",
+            "value": "pb"
           }
+        ]
       }
     },
     {
@@ -2896,7 +3063,13 @@
         ],
         "package": {
           "name": "piecestore"
+        },
+        "options": [
+          {
+            "name": "go_package",
+            "value": "pb"
           }
+        ]
       }
     },
     {
@@ -3124,7 +3297,13 @@
         ],
         "package": {
           "name": "pointerdb"
+        },
+        "options": [
+          {
+            "name": "go_package",
+            "value": "pb"
           }
+        ]
       }
     },
     {
@@ -3199,7 +3378,97 @@
         ],
         "package": {
           "name": "streams"
+        },
+        "options": [
+          {
+            "name": "go_package",
+            "value": "pb"
           }
+        ]
+      }
+    },
+    {
+      "protopath": "pkg:/:pb:/:vouchers.proto",
+      "def": {
+        "messages": [
+          {
+            "name": "Voucher",
+            "fields": [
+              {
+                "id": 1,
+                "name": "satellite_id",
+                "type": "bytes",
+                "options": [
+                  {
+                    "name": "(gogoproto.customtype)",
+                    "value": "NodeID"
+                  },
+                  {
+                    "name": "(gogoproto.nullable)",
+                    "value": "false"
+                  }
+                ]
+              },
+              {
+                "id": 2,
+                "name": "storage_node_id",
+                "type": "bytes",
+                "options": [
+                  {
+                    "name": "(gogoproto.customtype)",
+                    "value": "NodeID"
+                  },
+                  {
+                    "name": "(gogoproto.nullable)",
+                    "value": "false"
+                  }
+                ]
+              },
+              {
+                "id": 3,
+                "name": "expiration",
+                "type": "google.protobuf.Timestamp"
+              },
+              {
+                "id": 4,
+                "name": "satellite_signature",
+                "type": "bytes"
+              }
+            ]
+          },
+          {
+            "name": "VoucherRequest"
+          }
+        ],
+        "services": [
+          {
+            "name": "Vouchers",
+            "rpcs": [
+              {
+                "name": "Request",
+                "in_type": "VoucherRequest",
+                "out_type": "Voucher"
+              }
+            ]
+          }
+        ],
+        "imports": [
+          {
+            "path": "gogo.proto"
+          },
+          {
+            "path": "google/protobuf/timestamp.proto"
+          }
+        ],
+        "package": {
+          "name": "vouchers"
+        },
+        "options": [
+          {
+            "name": "go_package",
+            "value": "pb"
+          }
+        ]
       }
     }
   ]

scripts/check-satellite-config-lock.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+# NOTE this script MUST BE EXECUTED from the same directory where it's located
+# to always obtain the same paths in the satellite configuration file.
+
+set -uo pipefail
+
+#setup tmpdir for testfiles and cleanup
+TMPDIR=$(mktemp -d -t tmp.XXXXXXXXXX)
+cleanup(){
+	rm -rf "$TMPDIR"
+}
+trap cleanup EXIT
+
+pushd "$TMPDIR"
+satellite --config-dir "./testdata" --defaults release setup > /dev/null
+popd
+
+diff "./testdata/satellite-config.yaml.lock" "$TMPDIR/testdata/config.yaml"
+if [[ $? != 0 ]]; then
+    echo
+    echo "NOTIFY the Devops and PM when this test fails so they can plan for changing it in the release process before fixing it to merge your PR."
+    echo "Once you have notified them you can update the lock file through another Makefile target"
+    echo
+    exit 1
+fi

scripts/protobuf.go

@@ -6,6 +6,7 @@
 package main
 
 import (
+	"bytes"
 	"errors"
 	"flag"
 	"fmt"
@@ -16,6 +17,8 @@ import (
 	"regexp"
 	"sort"
 	"strings"
+
+	"github.com/kylelemons/godebug/diff"
 )
 
 var ignoreProto = map[string]bool{
@@ -56,12 +59,14 @@ func run(command, root string) error {
 			"github.com/ckaznocha/protoc-gen-lint@68a05858965b31eb872cbeb8d027507a94011acc",
 			// See https://github.com/gogo/protobuf#most-speed-and-most-customization
 			"github.com/gogo/protobuf/protoc-gen-gogo@"+gogoVersion,
-			"github.com/nilslice/protolock/cmd/protolock",
+			"github.com/nilslice/protolock/cmd/protolock@v0.12.0",
 		)
 	case "generate":
 		return walkdirs(root, generate)
 	case "lint":
 		return walkdirs(root, lint)
+	case "check-lock":
+		return walkdirs(root, checklock)
 	default:
 		return errors.New("unknown command " + command)
 	}
@@ -177,6 +182,57 @@ func lint(dir string, dirs []string, files []string) error {
 	return err
 }
 
+func checklock(dir string, dirs []string, files []string) error {
+	defer switchdir(dir)()
+
+	local, err := os.Getwd()
+	if err != nil {
+		panic(err)
+	}
+
+	protolockdir := findProtolockDir(local)
+
+	tmpdir, err := ioutil.TempDir("", "protolock")
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = os.RemoveAll(tmpdir)
+	}()
+
+	original, err := ioutil.ReadFile(filepath.Join(protolockdir, "proto.lock"))
+	if err != nil {
+		return fmt.Errorf("unable to read proto.lock: %v", err)
+	}
+
+	err = ioutil.WriteFile(filepath.Join(tmpdir, "proto.lock"), original, 0755)
+	if err != nil {
+		return fmt.Errorf("unable to read proto.lock: %v", err)
+	}
+
+	cmd := exec.Command("protolock", "commit", "-lockdir", tmpdir)
+	cmd.Dir = protolockdir
+	out, err := cmd.CombinedOutput()
+	if len(out) > 0 {
+		fmt.Println(string(out))
+	}
+	if err != nil {
+		return err
+	}
+
+	changed, err := ioutil.ReadFile(filepath.Join(tmpdir, "proto.lock"))
+	if err != nil {
+		return fmt.Errorf("unable to read new proto.lock: %v", err)
+	}
+
+	if !bytes.Equal(original, changed) {
+		diff, _ := difflines(string(original), string(changed))
+		return fmt.Errorf("protolock is not up to date: %v", diff)
+	}
+
+	return nil
+}
+
 func switchdir(to string) func() {
 	local, err := os.Getwd()
 	if err != nil {
@@ -259,3 +315,22 @@ func findProtolockDir(dir string) string {
 
 	return dir
 }
+
+func difflines(a, b string) (patch string, removed bool) {
+	alines, blines := strings.Split(a, "\n"), strings.Split(b, "\n")
+
+	chunks := diff.DiffChunks(alines, blines)
+
+	buf := new(bytes.Buffer)
+	for _, c := range chunks {
+		for _, line := range c.Added {
+			fmt.Fprintf(buf, "+%s\n", line)
+		}
+		for _, line := range c.Deleted {
+			fmt.Fprintf(buf, "-%s\n", line)
+			removed = true
+		}
+	}
+
+	return strings.TrimRight(buf.String(), "\n"), removed
+}

scripts/test-satellite-cfg-change.sh

@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-
-# NOTE this script MUST BE EXECUTED from the same directory where it's located
-# to always obtain the same paths in the satellite configuration file.
-
-set -uo pipefail
-
-TESTDATA_DIR="./testdata"
-cleanup(){
-  rm "$TESTDATA_DIR/config.yaml"
-}
-trap cleanup EXIT
-
-satellite --config-dir "$TESTDATA_DIR" --defaults release setup > /dev/null
-
-
-diff "$TESTDATA_DIR/satellite-config.yaml.lock" "$TESTDATA_DIR/config.yaml"
-if [[ $? != 0 ]]; then
-  echo
-  echo "NOTIFY the Devops and PM when this test fails so they can plan for changing it in the release process before fixing it to merge your PR."
-  echo "Once you have notified them you can update the lock file through another Makefile target"
-  echo
-  exit 1
-fi