From 96b88e286ef3f356f2fcd0e7529056727853dc45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Mon, 8 Aug 2022 12:24:00 +0200 Subject: [PATCH] build: fix conditional run of web builds Jenkins doesn't do a very good job with identifying what has been changed. While it has a syntax to defined patterns, it compares the current build with the previous build (in case of git-verify it can be a totally different branch) instead of checking the HEAD commit. This patch introduces shell scripts to do this better: * It doesn't depend on Jenkins any more * It can be executed locally * It can detect web changes properly (see the relation change as an example). Change-Id: I9d37775e3818c08c4aa96ffb78f84d57f28a2c95 --- Jenkinsfile.premerge | 39 +++++------- Jenkinsfile.public | 83 +++++--------------------- Jenkinsfile.verify | 65 ++------------------ satellite/admin/ui/build-if-changed.sh | 29 +++++++++ satellite/admin/ui/build.sh | 13 ++++ scripts/test-wasm.sh | 18 ++++++ web/multinode/build-if-changed.sh | 25 ++++++++ web/multinode/build.sh | 15 +++++ web/satellite/build-if-changed.sh | 29 +++++++++ web/satellite/build.sh | 14 +++++ web/satellite/package-lock.json | 4 +- web/storagenode/build-if-changed.sh | 29 +++++++++ web/storagenode/build.sh | 13 ++++ web/storagenode/package-lock.json | 4 +- 14 files changed, 222 insertions(+), 158 deletions(-) create mode 100755 satellite/admin/ui/build-if-changed.sh create mode 100755 satellite/admin/ui/build.sh create mode 100755 scripts/test-wasm.sh create mode 100755 web/multinode/build-if-changed.sh create mode 100755 web/multinode/build.sh create mode 100755 web/satellite/build-if-changed.sh create mode 100755 web/satellite/build.sh create mode 100755 web/storagenode/build-if-changed.sh create mode 100755 web/storagenode/build.sh diff --git a/Jenkinsfile.premerge b/Jenkinsfile.premerge index ed3f7cf82..51be3779d 100644 --- a/Jenkinsfile.premerge +++ b/Jenkinsfile.premerge @@ -49,51 +49,40 @@ pipeline { } } stage('Build Web') { - options { - timeout(time: 6, unit: "MINUTES") - } - // The build code depends on the following assets being loaded. parallel { stage('web/satellite') { steps { - dir('web/satellite') { - sh 'npm install --prefer-offline --no-audit --loglevel silly' - sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir - sh './scripts/build-wasm.sh' - sh 'npm run build' - } + sh './web/satellite/build.sh' } } + + stage('wasm') { + steps { + sh './scripts/test-wasm.sh' + } + } + stage('web/storagenode') { steps { - dir('web/storagenode') { - sh 'npm install --prefer-offline --no-audit --loglevel silly' - sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir - sh 'npm run build' - } + sh './web/storagenode/build.sh' } } + stage('web/multinode') { steps { - dir('web/multinode') { - sh 'npm install --prefer-offline --no-audit --loglevel silly' - sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir - sh 'npm run build' - } + sh './web/multinode/build.sh' } } + stage('satellite/admin/ui') { steps { - dir('satellite/admin/ui') { - sh 'npm install --prefer-offline --no-audit --loglevel silly' - sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir - sh 'npm run build' - } + sh './satellite/admin/ui/build.sh' } } } } + stage('Build') { parallel { stage('go') { diff --git a/Jenkinsfile.public b/Jenkinsfile.public index e7a3ef34e..4a11a85ad 100644 --- a/Jenkinsfile.public +++ b/Jenkinsfile.public @@ -42,39 +42,36 @@ pipeline { parallel { stage('web/satellite') { steps { - dir('web/satellite') { - sh 'npm install --prefer-offline --no-audit --loglevel verbose' - sh './scripts/build-wasm.sh' - sh 'npm run build' - } + sh './web/satellite/build.sh' } } + + stage('wasm') { + steps { + sh './scripts/test-wasm.sh' + } + } + stage('web/storagenode') { steps { - dir('web/storagenode') { - sh 'npm install --prefer-offline --no-audit --loglevel verbose' - sh 'npm run build' - } + sh './web/storagenode/build.sh' } } + stage('web/multinode') { steps { - dir('web/multinode') { - sh 'npm install --prefer-offline --no-audit --loglevel verbose' - sh 'npm run build' - } + sh './web/multinode/build.sh' } } + stage('satellite/admin/ui') { steps { - dir('satellite/admin/ui') { - sh 'npm install --prefer-offline --no-audit --loglevel verbose' - sh 'npm run build' - } + sh './satellite/admin/ui/build.sh' } } } } + stage('Build') { parallel { stage('go') { @@ -146,11 +143,6 @@ pipeline { sh 'golangci-lint --config /go/ci/.golangci.yml -j=2 run' } - dir("satellite/admin/ui") { - sh 'npm run check' - sh 'npm run lint' - } - sh './scripts/check-package-lock.sh' } } @@ -308,53 +300,6 @@ pipeline { } } - stage('wasm npm') { - steps { - dir(".build") { - sh 'cp -r ../satellite/console/wasm/tests/ .' - sh 'cd tests && cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" .' - sh 'cd tests && npm install && npm run test' - } - } - } - - stage('web/satellite') { - steps { - dir("web/satellite") { - sh 'npm run lint-ci' - sh script: 'npm audit', returnStatus: true - sh 'npm run test' - } - } - } - - stage('web/storagenode') { - steps { - dir("web/storagenode") { - sh 'npm run lint-ci' - sh script: 'npm audit', returnStatus: true - sh 'npm run test' - } - } - } - - stage('web/multinode') { - steps { - dir("web/multinode") { - sh 'npm run lint-ci' - sh script: 'npm audit', returnStatus: true - sh 'npm run test' - } - } - } - - stage('satellite/admin/ui') { - steps { - dir("satellite/admin/ui") { - sh script: 'npm audit', returnStatus: true - } - } - } /* stage('UI') { diff --git a/Jenkinsfile.verify b/Jenkinsfile.verify index 73611b5a5..9dba01ac2 100644 --- a/Jenkinsfile.verify +++ b/Jenkinsfile.verify @@ -90,82 +90,27 @@ pipeline { } } - stage('Build and Lint Web') { - options { - timeout(time: 6, unit: "MINUTES") - } - + stage('Build Web') { // The build code depends on the following assets being loaded. parallel { stage('web/satellite') { - when { - changeset "web/satellite/**" - } steps { - dir('web/satellite') { - sh 'npm install --prefer-offline --no-audit --loglevel silly' - sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir - sh './scripts/build-wasm.sh' - sh 'npm run build' - - sh 'npm run lint-ci' - sh script: 'npm audit', returnStatus: true - sh 'npm run test' - - } - dir(".build") { - sh 'cp -r ../satellite/console/wasm/tests/ .' - sh 'cd tests && cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" .' - sh 'cd tests && npm install && npm run test' - } + sh './web/satellite/build-if-changed.sh' } } - stage('web/storagenode') { - when { - changeset "web/storagenode/**" - } steps { - dir("web/storagenode") { - sh 'npm install --prefer-offline --no-audit --loglevel silly' - sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir - sh 'npm run build' - sh 'npm run lint-ci' - sh script: 'npm audit', returnStatus: true - sh 'npm run test' - } + sh './web/storagenode/build-if-changed.sh' } } - stage('web/multinode') { - when { - changeset "web/multinode/**" - } steps { - dir("web/multinode") { - sh 'npm install --prefer-offline --no-audit --loglevel silly' - sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir - sh 'npm run build' - sh 'npm run lint-ci' - sh script: 'npm audit', returnStatus: true - sh 'npm run test' - } + sh './web/multinode/build-if-changed.sh' } } - stage('satellite/admin/ui') { - when { - changeset "satellite/admin/ui/**" - } steps { - dir("satellite/admin/ui") { - sh 'npm install --prefer-offline --no-audit --loglevel silly' - sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir - sh 'npm run build' - sh script: 'npm audit', returnStatus: true - sh 'npm run check' - sh 'npm run lint' - } + sh './satellite/admin/ui/build-if-changed.sh' } } } diff --git a/satellite/admin/ui/build-if-changed.sh b/satellite/admin/ui/build-if-changed.sh new file mode 100755 index 000000000..6db6206d6 --- /dev/null +++ b/satellite/admin/ui/build-if-changed.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Copyright (C) 2022 Storj Labs, Inc. +# See LICENSE for copying information. + +cd "$(dirname "${BASH_SOURCE[0]}")" +set -xuo pipefail + +CHECK=satellite/admin/ui + +CHANGED=false + +#last commit +git diff HEAD HEAD~1 --name-only | grep $CHECK + +if [ $? -eq 0 ]; then + CHANGED=true +fi + +#working directory +git diff --name-only | grep $CHECK + +if [ $? -eq 0 ]; then + CHANGED=true +fi + +if [ $CHANGED == "true" ]; then + ./build.sh +fi + diff --git a/satellite/admin/ui/build.sh b/satellite/admin/ui/build.sh new file mode 100755 index 000000000..ab2822e2e --- /dev/null +++ b/satellite/admin/ui/build.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Copyright (C) 2022 Storj Labs, Inc. +# See LICENSE for copying information. +set -exuo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")" + +npm install --prefer-offline --no-audit --logleve verbose +touch ./node_modules/go.mod # prevent Go from scanning this dir +npm run build + +npm audit || true +npm run check +npm run lint diff --git a/scripts/test-wasm.sh b/scripts/test-wasm.sh new file mode 100755 index 000000000..a5d58ac54 --- /dev/null +++ b/scripts/test-wasm.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Copyright (C) 2022 Storj Labs, Inc. +# See LICENSE for copying information. +set -x +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +: ${TAG:=dev} + +mkdir -p .build +rm -rf .build/wasm || true +rm -rf release/$TAG/wasm || true + +./scripts/build-wasm.sh +cp -r satellite/console/wasm/tests/ .build/ +cd .build/tests/ +cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" . +npm install +npm run test diff --git a/web/multinode/build-if-changed.sh b/web/multinode/build-if-changed.sh new file mode 100755 index 000000000..f21316f16 --- /dev/null +++ b/web/multinode/build-if-changed.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Copyright (C) 2022 Storj Labs, Inc. +# See LICENSE for copying information. + +cd "$(dirname "${BASH_SOURCE[0]}")" +set -xuo pipefail + +CHECK=web/multinode + +CHANGED=false + +#last commit +git diff HEAD HEAD~1 --name-only | grep $CHECK + +if [ $? -eq 0 ]; then + ./build.sh +fi + +#working directory +git diff --name-only | grep $CHECK + +if [ $? -eq 0 ]; then + ./build.sh +fi + diff --git a/web/multinode/build.sh b/web/multinode/build.sh new file mode 100755 index 000000000..59c6ecfb8 --- /dev/null +++ b/web/multinode/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Copyright (C) 2022 Storj Labs, Inc. +# See LICENSE for copying information. +set -exuo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")" + +npm install --prefer-offline --no-audit --logleve verbose +touch ./node_modules/go.mod # prevent Go from scanning this dir +npm run build + +npm run lint-ci +npm audit || true +npm run test + + diff --git a/web/satellite/build-if-changed.sh b/web/satellite/build-if-changed.sh new file mode 100755 index 000000000..db14b5769 --- /dev/null +++ b/web/satellite/build-if-changed.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Copyright (C) 2022 Storj Labs, Inc. +# See LICENSE for copying information. + +cd "$(dirname "${BASH_SOURCE[0]}")" +set -xuo pipefail + +CHECK=web/satellite + +CHANGED=false + +#last commit +git diff HEAD HEAD~1 --name-only | grep $CHECK + +if [ $? -eq 0 ]; then + CHANGED=true +fi + +#working directory +git diff --name-only | grep $CHECK + +if [ $? -eq 0 ]; then + CHANGED=true +fi + +if [ $CHANGED == "true" ]; then + ./build.sh +fi + diff --git a/web/satellite/build.sh b/web/satellite/build.sh new file mode 100755 index 000000000..207bc55b7 --- /dev/null +++ b/web/satellite/build.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Copyright (C) 2022 Storj Labs, Inc. +# See LICENSE for copying information. + +cd "$(dirname "${BASH_SOURCE[0]}")" +set -euxo pipefail + +npm install --prefer-offline --no-audit --logleve verbose +touch ./node_modules/go.mod # prevent Go from scanning this dir +npm run build + +npm run lint-ci +npm audit || true +npm run test diff --git a/web/satellite/package-lock.json b/web/satellite/package-lock.json index 0988a5651..7f6b10ea1 100644 --- a/web/satellite/package-lock.json +++ b/web/satellite/package-lock.json @@ -65,7 +65,7 @@ "compression-webpack-plugin": "9.2.0", "eslint": "8.14.0", "eslint-plugin-import": "2.26.0", - "eslint-plugin-storj": "git+https://git@github.com/storj/eslint-storj.git", + "eslint-plugin-storj": "0.0.2", "eslint-plugin-vue": "8.7.1", "jest": "27.5.1", "jest-fetch-mock": "3.0.3", @@ -25212,7 +25212,7 @@ "eslint-plugin-storj": { "version": "git+https://git@github.com/storj/eslint-storj.git#5f952ffab7141e752cc095e5f024c39bab89679f", "dev": true, - "from": "eslint-plugin-storj@git+https://git@github.com/storj/eslint-storj.git" + "from": "eslint-plugin-storj@0.0.2" }, "eslint-plugin-vue": { "version": "8.7.1", diff --git a/web/storagenode/build-if-changed.sh b/web/storagenode/build-if-changed.sh new file mode 100755 index 000000000..0ff78bf67 --- /dev/null +++ b/web/storagenode/build-if-changed.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Copyright (C) 2022 Storj Labs, Inc. +# See LICENSE for copying information. + +cd "$(dirname "${BASH_SOURCE[0]}")" +set -xuo pipefail + +CHECK=web/storagenode + +CHANGED=false + +#last commit +git diff HEAD HEAD~1 --name-only | grep $CHECK + +if [ $? -eq 0 ]; then + CHANGED=true +fi + +#working directory +git diff --name-only | grep $CHECK + +if [ $? -eq 0 ]; then + CHANGED=true +fi + +if [ $CHANGED == "true" ]; then + ./build.sh +fi + diff --git a/web/storagenode/build.sh b/web/storagenode/build.sh new file mode 100755 index 000000000..44d1afc26 --- /dev/null +++ b/web/storagenode/build.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Copyright (C) 2022 Storj Labs, Inc. +# See LICENSE for copying information. +set -euxo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")" + +npm install --prefer-offline --no-audit --logleve verbose +touch ./node_modules/go.mod # prevent Go from scanning this dir +npm run build + +npm run lint-ci +npm audit || true +npm run test diff --git a/web/storagenode/package-lock.json b/web/storagenode/package-lock.json index afe1d3460..83877ab80 100644 --- a/web/storagenode/package-lock.json +++ b/web/storagenode/package-lock.json @@ -35,7 +35,7 @@ "compression-webpack-plugin": "9.2.0", "eslint": "8.14.0", "eslint-plugin-import": "2.25.4", - "eslint-plugin-storj": "git+https://git@github.com/storj/eslint-storj.git", + "eslint-plugin-storj": "0.0.2", "eslint-plugin-vue": "8.7.1", "jest": "27.5.1", "jest-fetch-mock": "3.0.3", @@ -23433,7 +23433,7 @@ "eslint-plugin-storj": { "version": "git+https://git@github.com/storj/eslint-storj.git#5f952ffab7141e752cc095e5f024c39bab89679f", "dev": true, - "from": "eslint-plugin-storj@git+https://git@github.com/storj/eslint-storj.git" + "from": "eslint-plugin-storj@0.0.2" }, "eslint-plugin-vue": { "version": "8.7.1",