storj/Jenkinsfile.premerge
Márton Elek 2a3c821bd2 build: turn off parallel Verification for premerge/public
Our tests depend on timing (timeouts / sleeps). Therefore we must not overcommit the CPU with highly parallel execution as builds become flaky.

go test can do parallel execution (that's fine as it doesn't overcommit CPU, if only one job is running), but we shouldn't execute different type of
integration tests at the same time.

This commit changes only public (PR + main build) and pre-merge (one build per patch), which are less build time sensitive.

Change-Id: Ib9aaabd301cfa9cd3515a8aa4dd8ae82f58eb811
2022-08-03 11:34:58 +00:00

325 lines
15 KiB
Plaintext

pipeline {
agent {
docker {
label 'main'
image 'storjlabs/ci:latest'
alwaysPull true
args '-u root:root --cap-add SYS_PTRACE -v "/tmp/gomod":/go/pkg/mod -v "/tmp/npm":/npm --tmpfs "/tmp:exec,mode=777"'
}
}
options {
timeout(time: 40, unit: 'MINUTES')
skipDefaultCheckout(true)
}
environment {
NPM_CONFIG_CACHE = '/npm/cache'
GOTRACEBACK = 'all'
COCKROACH_MEMPROF_INTERVAL=0
}
stages {
stage('Checkout') {
steps {
// Delete any content left over from a previous run.
sh "chmod -R 777 ."
// Bash requires extglob option to support !(.git) syntax,
// and we don't want to delete .git to have faster clones.
sh 'bash -O extglob -c "rm -rf !(.git)"'
checkout scm
sh 'mkdir -p .build'
// download dependencies
sh 'go mod download'
sh 'cd testsuite/ui && go mod download'
// pre-check that we cannot do at a later stage reliably
sh 'check-large-files'
// add a stub go.mod file to node projects to prevent go build ./... from scanning them
sh 'mkdir -p web/satellite/node_modules web/storagenode/node_modules web/multinode/node_modules'
sh 'touch web/satellite/node_modules/go.mod web/storagenode/node_modules/go.mod web/multinode/node_modules/go.mod'
}
}
stage('Gerrit status') {
steps {
withCredentials([sshUserPrivateKey(credentialsId: 'gerrit-trigger-ssh', keyFileVariable: 'SSH_KEY', usernameVariable: 'SSH_USER')]) {
sh './scripts/gerrit-status.sh pre-merge start +1'
}
}
}
stage('Build Web') {
options {
timeout(time: 6, unit: "MINUTES")
}
// The build code depends on the following assets being loaded.
parallel {
stage('web/satellite') {
steps {
dir('web/satellite') {
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh './scripts/build-wasm.sh'
sh 'npm run build'
}
}
}
stage('web/storagenode') {
steps {
dir('web/storagenode') {
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh 'npm run build'
}
}
}
stage('web/multinode') {
steps {
dir('web/multinode') {
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh 'npm run build'
}
}
}
stage('satellite/admin/ui') {
steps {
dir('satellite/admin/ui') {
sh 'npm install --prefer-offline --no-audit --loglevel silly'
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
sh 'npm run build'
}
}
}
}
}
stage('Build') {
parallel {
stage('go') {
steps {
// use go test to build all the packages, including tests
sh 'go test -v -p 16 -tags noembed -bench XYZXYZXYZXYZ -run XYZXYZXYZXYZ ./...'
}
}
stage('go -race') {
steps {
// use go test to build all the packages, including tests
sh 'go test -v -p 16 -tags noembed -bench XYZXYZXYZXYZ -run XYZXYZXYZXYZ -race ./...'
// install storj-sim
sh 'go install -race -v storj.io/storj/cmd/satellite '+
'storj.io/storj/cmd/storagenode ' +
'storj.io/storj/cmd/storj-sim ' +
'storj.io/storj/cmd/versioncontrol ' +
'storj.io/storj/cmd/uplink ' +
'storj.io/storj/cmd/identity ' +
'storj.io/storj/cmd/certificates ' +
'storj.io/storj/cmd/multinode'
}
}
stage('go -race gateway') {
steps {
// install gateway for storj-sim
sh 'go install -race -v storj.io/gateway@latest'
}
}
stage('db') {
steps {
sh 'service postgresql start'
dir('.build') {
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26256 --http-addr=localhost:8086 --cache 512MiB --max-sql-memory 512MiB --background'
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26257 --http-addr=localhost:8087 --cache 512MiB --max-sql-memory 512MiB --background'
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26258 --http-addr=localhost:8088 --cache 512MiB --max-sql-memory 512MiB --background'
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26259 --http-addr=localhost:8089 --cache 512MiB --max-sql-memory 512MiB --background'
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26260 --http-addr=localhost:8090 --cache 256MiB --max-sql-memory 256MiB --background'
}
}
}
}
}
stage('Cross Compile') {
steps {
// verify most of the commands, we cannot check everything since some of them
// have a C dependency and we don't have cross-compilation in storj/ci image
sh 'GOOS=linux GOARCH=386 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
sh 'GOOS=linux GOARCH=amd64 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
sh 'GOOS=linux GOARCH=arm go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
sh 'GOOS=linux GOARCH=arm64 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
sh 'GOOS=windows GOARCH=386 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
sh 'GOOS=windows GOARCH=amd64 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
sh 'GOOS=windows GOARCH=arm64 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
sh 'GOOS=darwin GOARCH=amd64 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
sh 'GOOS=darwin GOARCH=arm64 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
}
}
stage('Check Benchmark') {
environment {
STORJ_TEST_COCKROACH = 'omit'
STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/benchstorj?sslmode=disable'
}
steps {
sh 'psql -U postgres -c \'create database benchstorj;\''
sh 'go test -tags noembed -parallel 1 -p 1 -vet=off -timeout 20m -short -run XYZXYZXYZXYZ -bench . -benchtime 1x ./...'
}
}
stage('Integration') {
environment {
// use different hostname to avoid port conflicts
STORJ_NETWORK_HOST4 = '127.0.0.2'
STORJ_NETWORK_HOST6 = '127.0.0.2'
STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/teststorj2?sslmode=disable'
}
steps {
sh 'psql -U postgres -c \'create database teststorj2;\''
sh 'make test-sim'
// sh 'make test-certificates' // flaky
}
}
stage('Cockroach Integration') {
environment {
STORJ_NETWORK_HOST4 = '127.0.0.4'
STORJ_NETWORK_HOST6 = '127.0.0.4'
STORJ_SIM_POSTGRES = 'cockroach://root@localhost:26257/testcockroach4?sslmode=disable'
}
steps {
sh 'cockroach sql --insecure --host=localhost:26257 -e \'create database testcockroach4;\''
sh 'make test-sim'
sh 'cockroach sql --insecure --host=localhost:26257 -e \'drop database testcockroach4;\''
}
}
stage('Integration Redis unavailability') {
environment {
// use different hostname to avoid port conflicts
STORJ_NETWORK_HOST4 = '127.0.0.6'
STORJ_NETWORK_HOST6 = '127.0.0.6'
STORJ_REDIS_PORT = '7379'
STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/teststorj6?sslmode=disable'
}
steps {
sh 'psql -U postgres -c \'create database teststorj6;\''
sh 'make test-sim-redis-unavailability'
}
}
stage('Backwards Compatibility') {
environment {
STORJ_NETWORK_HOST4 = '127.0.0.3'
STORJ_NETWORK_HOST6 = '127.0.0.3'
STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/teststorj3?sslmode=disable'
STORJ_MIGRATION_DB = 'postgres://postgres@localhost/teststorj3?sslmode=disable&options=--search_path=satellite/0/meta'
}
steps {
sh 'psql -U postgres -c \'create database teststorj3;\''
sh 'make test-sim-backwards-compatible'
}
}
stage('Cockroach Backwards Compatibility') {
environment {
STORJ_NETWORK_HOST4 = '127.0.0.5'
STORJ_NETWORK_HOST6 = '127.0.0.5'
STORJ_SIM_POSTGRES = 'cockroach://root@localhost:26257/testcockroach5?sslmode=disable'
STORJ_MIGRATION_DB = 'postgres://root@localhost:26257/testcockroach5/satellite/0/meta?sslmode=disable'
}
steps {
sh 'cockroach sql --insecure --host=localhost:26257 -e \'create database testcockroach5;\''
sh 'make test-sim-backwards-compatible'
sh 'cockroach sql --insecure --host=localhost:26257 -e \'drop database testcockroach5;\''
}
}
stage('wasm npm') {
steps {
dir(".build") {
sh 'cp -r ../satellite/console/wasm/tests/ .'
sh 'cd tests && cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" .'
sh 'cd tests && npm install && npm run test'
}
}
}
stage('web/satellite') {
steps {
dir("web/satellite") {
sh 'npm run lint-ci'
sh script: 'npm audit', returnStatus: true
sh 'npm run test'
}
}
}
stage('web/storagenode') {
steps {
dir("web/storagenode") {
sh 'npm run lint-ci'
sh script: 'npm audit', returnStatus: true
sh 'npm run test'
}
}
}
stage('web/multinode') {
steps {
dir("web/multinode") {
sh 'npm run lint-ci'
sh script: 'npm audit', returnStatus: true
sh 'npm run test'
}
}
}
stage('satellite/admin/ui') {
steps {
dir("satellite/admin/ui") {
sh script: 'npm audit', returnStatus: true
}
}
}
stage('Post') {
parallel {
stage('Lint') {
steps {
sh 'check-clean-directory'
}
}
}
}
}
post {
success {
withCredentials([sshUserPrivateKey(credentialsId: 'gerrit-trigger-ssh', keyFileVariable: 'SSH_KEY', usernameVariable: 'SSH_USER')]) {
sh './scripts/gerrit-status.sh pre-merge success +2'
}
}
failure {
withCredentials([sshUserPrivateKey(credentialsId: 'gerrit-trigger-ssh', keyFileVariable: 'SSH_KEY', usernameVariable: 'SSH_USER')]) {
sh './scripts/gerrit-status.sh pre-merge failure -2'
}
}
aborted {
withCredentials([sshUserPrivateKey(credentialsId: 'gerrit-trigger-ssh', keyFileVariable: 'SSH_KEY', usernameVariable: 'SSH_USER')]) {
sh './scripts/gerrit-status.sh pre-merge failure -2'
}
}
}
}