build: fix conditional run of web builds

Jenkins doesn't do a very good job with identifying what has been changed.

While it has a syntax to defined patterns, it compares the current build with the previous build (in case of git-verify it can be a totally different branch) instead of checking the HEAD commit.

This patch introduces shell scripts to do this better:
 * It doesn't depend on Jenkins any more
 * It can be executed locally
 * It can detect web changes properly (see the relation change as an example).

Change-Id: I9d37775e3818c08c4aa96ffb78f84d57f28a2c95
This commit is contained in:
Márton Elek 2022-08-08 12:24:00 +02:00 committed by Storj Robot
parent cff8158054
commit 96b88e286e
14 changed files with 222 additions and 158 deletions

View File

@ -49,51 +49,40 @@ pipeline {
} }
} }
stage('Build Web') { stage('Build Web') {
options {
timeout(time: 6, unit: "MINUTES")
}
// The build code depends on the following assets being loaded. // The build code depends on the following assets being loaded.
parallel { parallel {
stage('web/satellite') { stage('web/satellite') {
steps { steps {
dir('web/satellite') { sh './web/satellite/build.sh'
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh './scripts/build-wasm.sh'
sh 'npm run build'
}
} }
} }
stage('wasm') {
steps {
sh './scripts/test-wasm.sh'
}
}
stage('web/storagenode') { stage('web/storagenode') {
steps { steps {
dir('web/storagenode') { sh './web/storagenode/build.sh'
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh 'npm run build'
}
} }
} }
stage('web/multinode') { stage('web/multinode') {
steps { steps {
dir('web/multinode') { sh './web/multinode/build.sh'
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh 'npm run build'
}
} }
} }
stage('satellite/admin/ui') { stage('satellite/admin/ui') {
steps { steps {
dir('satellite/admin/ui') { sh './satellite/admin/ui/build.sh'
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh 'npm run build'
}
} }
} }
} }
} }
stage('Build') { stage('Build') {
parallel { parallel {
stage('go') { stage('go') {

View File

@ -42,39 +42,36 @@ pipeline {
parallel { parallel {
stage('web/satellite') { stage('web/satellite') {
steps { steps {
dir('web/satellite') { sh './web/satellite/build.sh'
sh 'npm install --prefer-offline --no-audit --loglevel verbose'
sh './scripts/build-wasm.sh'
sh 'npm run build'
}
} }
} }
stage('wasm') {
steps {
sh './scripts/test-wasm.sh'
}
}
stage('web/storagenode') { stage('web/storagenode') {
steps { steps {
dir('web/storagenode') { sh './web/storagenode/build.sh'
sh 'npm install --prefer-offline --no-audit --loglevel verbose'
sh 'npm run build'
}
} }
} }
stage('web/multinode') { stage('web/multinode') {
steps { steps {
dir('web/multinode') { sh './web/multinode/build.sh'
sh 'npm install --prefer-offline --no-audit --loglevel verbose'
sh 'npm run build'
}
} }
} }
stage('satellite/admin/ui') { stage('satellite/admin/ui') {
steps { steps {
dir('satellite/admin/ui') { sh './satellite/admin/ui/build.sh'
sh 'npm install --prefer-offline --no-audit --loglevel verbose'
sh 'npm run build'
}
} }
} }
} }
} }
stage('Build') { stage('Build') {
parallel { parallel {
stage('go') { stage('go') {
@ -146,11 +143,6 @@ pipeline {
sh 'golangci-lint --config /go/ci/.golangci.yml -j=2 run' sh 'golangci-lint --config /go/ci/.golangci.yml -j=2 run'
} }
dir("satellite/admin/ui") {
sh 'npm run check'
sh 'npm run lint'
}
sh './scripts/check-package-lock.sh' sh './scripts/check-package-lock.sh'
} }
} }
@ -308,53 +300,6 @@ pipeline {
} }
} }
stage('wasm npm') {
steps {
dir(".build") {
sh 'cp -r ../satellite/console/wasm/tests/ .'
sh 'cd tests && cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" .'
sh 'cd tests && npm install && npm run test'
}
}
}
stage('web/satellite') {
steps {
dir("web/satellite") {
sh 'npm run lint-ci'
sh script: 'npm audit', returnStatus: true
sh 'npm run test'
}
}
}
stage('web/storagenode') {
steps {
dir("web/storagenode") {
sh 'npm run lint-ci'
sh script: 'npm audit', returnStatus: true
sh 'npm run test'
}
}
}
stage('web/multinode') {
steps {
dir("web/multinode") {
sh 'npm run lint-ci'
sh script: 'npm audit', returnStatus: true
sh 'npm run test'
}
}
}
stage('satellite/admin/ui') {
steps {
dir("satellite/admin/ui") {
sh script: 'npm audit', returnStatus: true
}
}
}
/* /*
stage('UI') { stage('UI') {

View File

@ -90,82 +90,27 @@ pipeline {
} }
} }
stage('Build and Lint Web') { stage('Build Web') {
options {
timeout(time: 6, unit: "MINUTES")
}
// The build code depends on the following assets being loaded. // The build code depends on the following assets being loaded.
parallel { parallel {
stage('web/satellite') { stage('web/satellite') {
when {
changeset "web/satellite/**"
}
steps { steps {
dir('web/satellite') { sh './web/satellite/build-if-changed.sh'
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh './scripts/build-wasm.sh'
sh 'npm run build'
sh 'npm run lint-ci'
sh script: 'npm audit', returnStatus: true
sh 'npm run test'
}
dir(".build") {
sh 'cp -r ../satellite/console/wasm/tests/ .'
sh 'cd tests && cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" .'
sh 'cd tests && npm install && npm run test'
}
} }
} }
stage('web/storagenode') { stage('web/storagenode') {
when {
changeset "web/storagenode/**"
}
steps { steps {
dir("web/storagenode") { sh './web/storagenode/build-if-changed.sh'
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh 'npm run build'
sh 'npm run lint-ci'
sh script: 'npm audit', returnStatus: true
sh 'npm run test'
}
} }
} }
stage('web/multinode') { stage('web/multinode') {
when {
changeset "web/multinode/**"
}
steps { steps {
dir("web/multinode") { sh './web/multinode/build-if-changed.sh'
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh 'npm run build'
sh 'npm run lint-ci'
sh script: 'npm audit', returnStatus: true
sh 'npm run test'
}
} }
} }
stage('satellite/admin/ui') { stage('satellite/admin/ui') {
when {
changeset "satellite/admin/ui/**"
}
steps { steps {
dir("satellite/admin/ui") { sh './satellite/admin/ui/build-if-changed.sh'
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh 'npm run build'
sh script: 'npm audit', returnStatus: true
sh 'npm run check'
sh 'npm run lint'
}
} }
} }
} }

View File

@ -0,0 +1,29 @@
#!/usr/bin/env bash
# Copyright (C) 2022 Storj Labs, Inc.
# See LICENSE for copying information.
cd "$(dirname "${BASH_SOURCE[0]}")"
set -xuo pipefail
CHECK=satellite/admin/ui
CHANGED=false
#last commit
git diff HEAD HEAD~1 --name-only | grep $CHECK
if [ $? -eq 0 ]; then
CHANGED=true
fi
#working directory
git diff --name-only | grep $CHECK
if [ $? -eq 0 ]; then
CHANGED=true
fi
if [ $CHANGED == "true" ]; then
./build.sh
fi

13
satellite/admin/ui/build.sh Executable file
View File

@ -0,0 +1,13 @@
#!/usr/bin/env bash
# Copyright (C) 2022 Storj Labs, Inc.
# See LICENSE for copying information.
set -exuo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
npm install --prefer-offline --no-audit --logleve verbose
touch ./node_modules/go.mod # prevent Go from scanning this dir
npm run build
npm audit || true
npm run check
npm run lint

18
scripts/test-wasm.sh Executable file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env bash
# Copyright (C) 2022 Storj Labs, Inc.
# See LICENSE for copying information.
set -x
cd "$(dirname "${BASH_SOURCE[0]}")/.."
: ${TAG:=dev}
mkdir -p .build
rm -rf .build/wasm || true
rm -rf release/$TAG/wasm || true
./scripts/build-wasm.sh
cp -r satellite/console/wasm/tests/ .build/
cd .build/tests/
cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" .
npm install
npm run test

View File

@ -0,0 +1,25 @@
#!/usr/bin/env bash
# Copyright (C) 2022 Storj Labs, Inc.
# See LICENSE for copying information.
cd "$(dirname "${BASH_SOURCE[0]}")"
set -xuo pipefail
CHECK=web/multinode
CHANGED=false
#last commit
git diff HEAD HEAD~1 --name-only | grep $CHECK
if [ $? -eq 0 ]; then
./build.sh
fi
#working directory
git diff --name-only | grep $CHECK
if [ $? -eq 0 ]; then
./build.sh
fi

15
web/multinode/build.sh Executable file
View File

@ -0,0 +1,15 @@
#!/usr/bin/env bash
# Copyright (C) 2022 Storj Labs, Inc.
# See LICENSE for copying information.
set -exuo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
npm install --prefer-offline --no-audit --logleve verbose
touch ./node_modules/go.mod # prevent Go from scanning this dir
npm run build
npm run lint-ci
npm audit || true
npm run test

View File

@ -0,0 +1,29 @@
#!/usr/bin/env bash
# Copyright (C) 2022 Storj Labs, Inc.
# See LICENSE for copying information.
cd "$(dirname "${BASH_SOURCE[0]}")"
set -xuo pipefail
CHECK=web/satellite
CHANGED=false
#last commit
git diff HEAD HEAD~1 --name-only | grep $CHECK
if [ $? -eq 0 ]; then
CHANGED=true
fi
#working directory
git diff --name-only | grep $CHECK
if [ $? -eq 0 ]; then
CHANGED=true
fi
if [ $CHANGED == "true" ]; then
./build.sh
fi

14
web/satellite/build.sh Executable file
View File

@ -0,0 +1,14 @@
#!/usr/bin/env bash
# Copyright (C) 2022 Storj Labs, Inc.
# See LICENSE for copying information.
cd "$(dirname "${BASH_SOURCE[0]}")"
set -euxo pipefail
npm install --prefer-offline --no-audit --logleve verbose
touch ./node_modules/go.mod # prevent Go from scanning this dir
npm run build
npm run lint-ci
npm audit || true
npm run test

View File

@ -65,7 +65,7 @@
"compression-webpack-plugin": "9.2.0", "compression-webpack-plugin": "9.2.0",
"eslint": "8.14.0", "eslint": "8.14.0",
"eslint-plugin-import": "2.26.0", "eslint-plugin-import": "2.26.0",
"eslint-plugin-storj": "git+https://git@github.com/storj/eslint-storj.git", "eslint-plugin-storj": "0.0.2",
"eslint-plugin-vue": "8.7.1", "eslint-plugin-vue": "8.7.1",
"jest": "27.5.1", "jest": "27.5.1",
"jest-fetch-mock": "3.0.3", "jest-fetch-mock": "3.0.3",
@ -25212,7 +25212,7 @@
"eslint-plugin-storj": { "eslint-plugin-storj": {
"version": "git+https://git@github.com/storj/eslint-storj.git#5f952ffab7141e752cc095e5f024c39bab89679f", "version": "git+https://git@github.com/storj/eslint-storj.git#5f952ffab7141e752cc095e5f024c39bab89679f",
"dev": true, "dev": true,
"from": "eslint-plugin-storj@git+https://git@github.com/storj/eslint-storj.git" "from": "eslint-plugin-storj@0.0.2"
}, },
"eslint-plugin-vue": { "eslint-plugin-vue": {
"version": "8.7.1", "version": "8.7.1",

View File

@ -0,0 +1,29 @@
#!/usr/bin/env bash
# Copyright (C) 2022 Storj Labs, Inc.
# See LICENSE for copying information.
cd "$(dirname "${BASH_SOURCE[0]}")"
set -xuo pipefail
CHECK=web/storagenode
CHANGED=false
#last commit
git diff HEAD HEAD~1 --name-only | grep $CHECK
if [ $? -eq 0 ]; then
CHANGED=true
fi
#working directory
git diff --name-only | grep $CHECK
if [ $? -eq 0 ]; then
CHANGED=true
fi
if [ $CHANGED == "true" ]; then
./build.sh
fi

13
web/storagenode/build.sh Executable file
View File

@ -0,0 +1,13 @@
#!/usr/bin/env bash
# Copyright (C) 2022 Storj Labs, Inc.
# See LICENSE for copying information.
set -euxo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
npm install --prefer-offline --no-audit --logleve verbose
touch ./node_modules/go.mod # prevent Go from scanning this dir
npm run build
npm run lint-ci
npm audit || true
npm run test

View File

@ -35,7 +35,7 @@
"compression-webpack-plugin": "9.2.0", "compression-webpack-plugin": "9.2.0",
"eslint": "8.14.0", "eslint": "8.14.0",
"eslint-plugin-import": "2.25.4", "eslint-plugin-import": "2.25.4",
"eslint-plugin-storj": "git+https://git@github.com/storj/eslint-storj.git", "eslint-plugin-storj": "0.0.2",
"eslint-plugin-vue": "8.7.1", "eslint-plugin-vue": "8.7.1",
"jest": "27.5.1", "jest": "27.5.1",
"jest-fetch-mock": "3.0.3", "jest-fetch-mock": "3.0.3",
@ -23433,7 +23433,7 @@
"eslint-plugin-storj": { "eslint-plugin-storj": {
"version": "git+https://git@github.com/storj/eslint-storj.git#5f952ffab7141e752cc095e5f024c39bab89679f", "version": "git+https://git@github.com/storj/eslint-storj.git#5f952ffab7141e752cc095e5f024c39bab89679f",
"dev": true, "dev": true,
"from": "eslint-plugin-storj@git+https://git@github.com/storj/eslint-storj.git" "from": "eslint-plugin-storj@0.0.2"
}, },
"eslint-plugin-vue": { "eslint-plugin-vue": {
"version": "8.7.1", "version": "8.7.1",