JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

build: fix conditional run of web builds

Browse Source 
Jenkins doesn't do a very good job with identifying what has been changed.

While it has a syntax to defined patterns, it compares the current build with the previous build (in case of git-verify it can be a totally different branch) instead of checking the HEAD commit.

This patch introduces shell scripts to do this better:
 * It doesn't depend on Jenkins any more
 * It can be executed locally
 * It can detect web changes properly (see the relation change as an example).

Change-Id: I9d37775e3818c08c4aa96ffb78f84d57f28a2c95
This commit is contained in:
Márton Elek 2022-08-08 12:24:00 +02:00 committed by Storj Robot
parent cff8158054
commit 96b88e286e
14 changed files with 222 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
Jenkinsfile.premerge
View File

@ -49,51 +49,40 @@ pipeline {
} }
} }
stage('Build Web') { stage('Build Web') {
options {
timeout(time: 6, unit: "MINUTES")
}
// The build code depends on the following assets being loaded. // The build code depends on the following assets being loaded.
parallel { parallel {
stage('web/satellite') { stage('web/satellite') {
steps { steps {
dir('web/satellite') { sh './web/satellite/build.sh'
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh './scripts/build-wasm.sh'
sh 'npm run build'
} }
} }
stage('wasm') {
steps {
sh './scripts/test-wasm.sh'
} }
}
stage('web/storagenode') { stage('web/storagenode') {
steps { steps {
dir('web/storagenode') { sh './web/storagenode/build.sh'
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh 'npm run build'
}
} }
} }
stage('web/multinode') { stage('web/multinode') {
steps { steps {
dir('web/multinode') { sh './web/multinode/build.sh'
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh 'npm run build'
}
} }
} }
stage('satellite/admin/ui') { stage('satellite/admin/ui') {
steps { steps {
dir('satellite/admin/ui') { sh './satellite/admin/ui/build.sh'
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh 'npm run build'
}
} }
} }
} }
} }
stage('Build') { stage('Build') {
parallel { parallel {
stage('go') { stage('go') {

81
Jenkinsfile.public
View File

@ -42,39 +42,36 @@ pipeline {
parallel { parallel {
stage('web/satellite') { stage('web/satellite') {
steps { steps {
dir('web/satellite') { sh './web/satellite/build.sh'
sh 'npm install --prefer-offline --no-audit --loglevel verbose'
sh './scripts/build-wasm.sh'
sh 'npm run build'
} }
} }
stage('wasm') {
steps {
sh './scripts/test-wasm.sh'
} }
}
stage('web/storagenode') { stage('web/storagenode') {
steps { steps {
dir('web/storagenode') { sh './web/storagenode/build.sh'
sh 'npm install --prefer-offline --no-audit --loglevel verbose'
sh 'npm run build'
}
} }
} }
stage('web/multinode') { stage('web/multinode') {
steps { steps {
dir('web/multinode') { sh './web/multinode/build.sh'
sh 'npm install --prefer-offline --no-audit --loglevel verbose'
sh 'npm run build'
}
} }
} }
stage('satellite/admin/ui') { stage('satellite/admin/ui') {
steps { steps {
dir('satellite/admin/ui') { sh './satellite/admin/ui/build.sh'
sh 'npm install --prefer-offline --no-audit --loglevel verbose'
sh 'npm run build'
}
} }
} }
} }
} }
stage('Build') { stage('Build') {
parallel { parallel {
stage('go') { stage('go') {
@ -146,11 +143,6 @@ pipeline {
sh 'golangci-lint --config /go/ci/.golangci.yml -j=2 run' sh 'golangci-lint --config /go/ci/.golangci.yml -j=2 run'
} }
dir("satellite/admin/ui") {
sh 'npm run check'
sh 'npm run lint'
}
sh './scripts/check-package-lock.sh' sh './scripts/check-package-lock.sh'
} }
} }
@ -308,53 +300,6 @@ pipeline {
} }
} }
stage('wasm npm') {
steps {
dir(".build") {
sh 'cp -r ../satellite/console/wasm/tests/ .'
sh 'cd tests && cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" .'
sh 'cd tests && npm install && npm run test'
}
}
}
stage('web/satellite') {
steps {
dir("web/satellite") {
sh 'npm run lint-ci'
sh script: 'npm audit', returnStatus: true
sh 'npm run test'
}
}
}
stage('web/storagenode') {
steps {
dir("web/storagenode") {
sh 'npm run lint-ci'
sh script: 'npm audit', returnStatus: true
sh 'npm run test'
}
}
}
stage('web/multinode') {
steps {
dir("web/multinode") {
sh 'npm run lint-ci'
sh script: 'npm audit', returnStatus: true
sh 'npm run test'
}
}
}
stage('satellite/admin/ui') {
steps {
dir("satellite/admin/ui") {
sh script: 'npm audit', returnStatus: true
}
}
}
/* /*
stage('UI') { stage('UI') {

65
Jenkinsfile.verify
View File

@ -90,82 +90,27 @@ pipeline {
} }
} }
stage('Build and Lint Web') { stage('Build Web') {
options {
timeout(time: 6, unit: "MINUTES")
}
// The build code depends on the following assets being loaded. // The build code depends on the following assets being loaded.
parallel { parallel {
stage('web/satellite') { stage('web/satellite') {
when {
changeset "web/satellite/**"
}
steps { steps {
dir('web/satellite') { sh './web/satellite/build-if-changed.sh'
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh './scripts/build-wasm.sh'
sh 'npm run build'
sh 'npm run lint-ci'
sh script: 'npm audit', returnStatus: true
sh 'npm run test'
}
dir(".build") {
sh 'cp -r ../satellite/console/wasm/tests/ .'
sh 'cd tests && cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" .'
sh 'cd tests && npm install && npm run test'
} }
} }
}
stage('web/storagenode') { stage('web/storagenode') {
when {
changeset "web/storagenode/**"
}
steps { steps {
dir("web/storagenode") { sh './web/storagenode/build-if-changed.sh'
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh 'npm run build'
sh 'npm run lint-ci'
sh script: 'npm audit', returnStatus: true
sh 'npm run test'
} }
} }
}
stage('web/multinode') { stage('web/multinode') {
when {
changeset "web/multinode/**"
}
steps { steps {
dir("web/multinode") { sh './web/multinode/build-if-changed.sh'
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh 'npm run build'
sh 'npm run lint-ci'
sh script: 'npm audit', returnStatus: true
sh 'npm run test'
} }
} }
}
stage('satellite/admin/ui') { stage('satellite/admin/ui') {
when {
changeset "satellite/admin/ui/**"
}
steps { steps {
dir("satellite/admin/ui") { sh './satellite/admin/ui/build-if-changed.sh'
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh 'npm run build'
sh script: 'npm audit', returnStatus: true
sh 'npm run check'
sh 'npm run lint'
}
} }
} }
} }

29
satellite/admin/ui/build-if-changed.sh Executable file
View File

@ -0,0 +1,29 @@
#!/usr/bin/env bash
# Copyright (C) 2022 Storj Labs, Inc.
# See LICENSE for copying information.
cd "$(dirname "${BASH_SOURCE[0]}")"
set -xuo pipefail
CHECK=satellite/admin/ui
CHANGED=false
#last commit
git diff HEAD HEAD~1 --name-only | grep $CHECK
if [ $? -eq 0 ]; then
CHANGED=true
fi
#working directory
git diff --name-only | grep $CHECK
if [ $? -eq 0 ]; then
CHANGED=true
fi
if [ $CHANGED == "true" ]; then
./build.sh
fi

13
satellite/admin/ui/build.sh Executable file
View File

@ -0,0 +1,13 @@
#!/usr/bin/env bash
# Copyright (C) 2022 Storj Labs, Inc.
# See LICENSE for copying information.
set -exuo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
npm install --prefer-offline --no-audit --logleve verbose
touch ./node_modules/go.mod # prevent Go from scanning this dir
npm run build
npm audit || true
npm run check
npm run lint

18
scripts/test-wasm.sh Executable file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env bash
# Copyright (C) 2022 Storj Labs, Inc.
# See LICENSE for copying information.
set -x
cd "$(dirname "${BASH_SOURCE[0]}")/.."
: ${TAG:=dev}
mkdir -p .build
rm -rf .build/wasm || true
rm -rf release/$TAG/wasm || true
./scripts/build-wasm.sh
cp -r satellite/console/wasm/tests/ .build/
cd .build/tests/
cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" .
npm install
npm run test

25
web/multinode/build-if-changed.sh Executable file
View File

@ -0,0 +1,25 @@
#!/usr/bin/env bash
# Copyright (C) 2022 Storj Labs, Inc.
# See LICENSE for copying information.
cd "$(dirname "${BASH_SOURCE[0]}")"
set -xuo pipefail
CHECK=web/multinode
CHANGED=false
#last commit
git diff HEAD HEAD~1 --name-only | grep $CHECK
if [ $? -eq 0 ]; then
./build.sh
fi
#working directory
git diff --name-only | grep $CHECK
if [ $? -eq 0 ]; then
./build.sh
fi

15
web/multinode/build.sh Executable file
View File

@ -0,0 +1,15 @@
#!/usr/bin/env bash
# Copyright (C) 2022 Storj Labs, Inc.
# See LICENSE for copying information.
set -exuo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
npm install --prefer-offline --no-audit --logleve verbose
touch ./node_modules/go.mod # prevent Go from scanning this dir
npm run build
npm run lint-ci
npm audit || true
npm run test

29
web/satellite/build-if-changed.sh Executable file
View File

@ -0,0 +1,29 @@
#!/usr/bin/env bash
# Copyright (C) 2022 Storj Labs, Inc.
# See LICENSE for copying information.
cd "$(dirname "${BASH_SOURCE[0]}")"
set -xuo pipefail
CHECK=web/satellite
CHANGED=false
#last commit
git diff HEAD HEAD~1 --name-only | grep $CHECK
if [ $? -eq 0 ]; then
CHANGED=true
fi
#working directory
git diff --name-only | grep $CHECK
if [ $? -eq 0 ]; then
CHANGED=true
fi
if [ $CHANGED == "true" ]; then
./build.sh
fi

14
web/satellite/build.sh Executable file
View File

@ -0,0 +1,14 @@
#!/usr/bin/env bash
# Copyright (C) 2022 Storj Labs, Inc.
# See LICENSE for copying information.
cd "$(dirname "${BASH_SOURCE[0]}")"
set -euxo pipefail
npm install --prefer-offline --no-audit --logleve verbose
touch ./node_modules/go.mod # prevent Go from scanning this dir
npm run build
npm run lint-ci
npm audit || true
npm run test

4
web/satellite/package-lock.json
View File

@ -65,7 +65,7 @@
"compression-webpack-plugin": "9.2.0", "compression-webpack-plugin": "9.2.0",
"eslint": "8.14.0", "eslint": "8.14.0",
"eslint-plugin-import": "2.26.0", "eslint-plugin-import": "2.26.0",
"eslint-plugin-storj": "git+https://git@github.com/storj/eslint-storj.git", "eslint-plugin-storj": "0.0.2",
"eslint-plugin-vue": "8.7.1", "eslint-plugin-vue": "8.7.1",
"jest": "27.5.1", "jest": "27.5.1",
"jest-fetch-mock": "3.0.3", "jest-fetch-mock": "3.0.3",
@ -25212,7 +25212,7 @@
"eslint-plugin-storj": { "eslint-plugin-storj": {
"version": "git+https://git@github.com/storj/eslint-storj.git#5f952ffab7141e752cc095e5f024c39bab89679f", "version": "git+https://git@github.com/storj/eslint-storj.git#5f952ffab7141e752cc095e5f024c39bab89679f",
"dev": true, "dev": true,
"from": "eslint-plugin-storj@git+https://git@github.com/storj/eslint-storj.git" "from": "eslint-plugin-storj@0.0.2"
}, },
"eslint-plugin-vue": { "eslint-plugin-vue": {
"version": "8.7.1", "version": "8.7.1",

29
web/storagenode/build-if-changed.sh Executable file
View File

@ -0,0 +1,29 @@
#!/usr/bin/env bash
# Copyright (C) 2022 Storj Labs, Inc.
# See LICENSE for copying information.
cd "$(dirname "${BASH_SOURCE[0]}")"
set -xuo pipefail
CHECK=web/storagenode
CHANGED=false
#last commit
git diff HEAD HEAD~1 --name-only | grep $CHECK
if [ $? -eq 0 ]; then
CHANGED=true
fi
#working directory
git diff --name-only | grep $CHECK
if [ $? -eq 0 ]; then
CHANGED=true
fi
if [ $CHANGED == "true" ]; then
./build.sh
fi

13
web/storagenode/build.sh Executable file
View File

@ -0,0 +1,13 @@
#!/usr/bin/env bash
# Copyright (C) 2022 Storj Labs, Inc.
# See LICENSE for copying information.
set -euxo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
npm install --prefer-offline --no-audit --logleve verbose
touch ./node_modules/go.mod # prevent Go from scanning this dir
npm run build
npm run lint-ci
npm audit || true
npm run test

4
web/storagenode/package-lock.json
View File

@ -35,7 +35,7 @@
"compression-webpack-plugin": "9.2.0", "compression-webpack-plugin": "9.2.0",
"eslint": "8.14.0", "eslint": "8.14.0",
"eslint-plugin-import": "2.25.4", "eslint-plugin-import": "2.25.4",
"eslint-plugin-storj": "git+https://git@github.com/storj/eslint-storj.git", "eslint-plugin-storj": "0.0.2",
"eslint-plugin-vue": "8.7.1", "eslint-plugin-vue": "8.7.1",
"jest": "27.5.1", "jest": "27.5.1",
"jest-fetch-mock": "3.0.3", "jest-fetch-mock": "3.0.3",
@ -23433,7 +23433,7 @@
"eslint-plugin-storj": { "eslint-plugin-storj": {
"version": "git+https://git@github.com/storj/eslint-storj.git#5f952ffab7141e752cc095e5f024c39bab89679f", "version": "git+https://git@github.com/storj/eslint-storj.git#5f952ffab7141e752cc095e5f024c39bab89679f",
"dev": true, "dev": true,
"from": "eslint-plugin-storj@git+https://git@github.com/storj/eslint-storj.git" "from": "eslint-plugin-storj@0.0.2"
}, },
"eslint-plugin-vue": { "eslint-plugin-vue": {
"version": "8.7.1", "version": "8.7.1",