96c3816313
Add timeout to npm install and increase logging level. npm install is still taking sometimes too long and it's not clear why, verbose logging is not sufficient. Change-Id: Ib72f9823f30c9744562e279c2a5481f096e38128
325 lines
14 KiB
Plaintext
325 lines
14 KiB
Plaintext
pipeline {
|
|
agent {
|
|
docker {
|
|
label 'main'
|
|
image 'storjlabs/ci:latest'
|
|
alwaysPull true
|
|
args '-u root:root --cap-add SYS_PTRACE -v "/tmp/gomod":/go/pkg/mod -v "/tmp/npm":/npm --tmpfs "/tmp:exec,mode=777"'
|
|
}
|
|
}
|
|
options {
|
|
timeout(time: 40, unit: 'MINUTES')
|
|
skipDefaultCheckout(true)
|
|
}
|
|
environment {
|
|
NPM_CONFIG_CACHE = '/npm/cache'
|
|
GOTRACEBACK = 'all'
|
|
COCKROACH_MEMPROF_INTERVAL=0
|
|
}
|
|
stages {
|
|
stage('Checkout') {
|
|
steps {
|
|
// Delete any content left over from a previous run.
|
|
sh "chmod -R 777 ."
|
|
// Bash requires extglob option to support !(.git) syntax,
|
|
// and we don't want to delete .git to have faster clones.
|
|
sh 'bash -O extglob -c "rm -rf !(.git)"'
|
|
|
|
checkout scm
|
|
|
|
sh 'mkdir -p .build'
|
|
|
|
// download dependencies
|
|
sh 'go mod download'
|
|
sh 'cd testsuite/ui && go mod download'
|
|
|
|
// pre-check that we cannot do at a later stage reliably
|
|
sh 'check-large-files'
|
|
|
|
// add a stub go.mod file to node projects to prevent go build ./... from scanning them
|
|
sh 'mkdir -p web/satellite/node_modules web/storagenode/node_modules web/multinode/node_modules'
|
|
sh 'touch web/satellite/node_modules/go.mod web/storagenode/node_modules/go.mod web/multinode/node_modules/go.mod'
|
|
}
|
|
}
|
|
stage('Gerrit status') {
|
|
steps {
|
|
withCredentials([sshUserPrivateKey(credentialsId: 'gerrit-trigger-ssh', keyFileVariable: 'SSH_KEY', usernameVariable: 'SSH_USER')]) {
|
|
sh './scripts/gerrit-status.sh pre-merge start +1'
|
|
}
|
|
}
|
|
}
|
|
stage('Build Web') {
|
|
options {
|
|
timeout(time: 6, unit: "MINUTES")
|
|
}
|
|
|
|
// The build code depends on the following assets being loaded.
|
|
parallel {
|
|
stage('web/satellite') {
|
|
steps {
|
|
dir('web/satellite') {
|
|
sh 'npm install --prefer-offline --no-audit --loglevel silly'
|
|
sh './scripts/build-wasm.sh'
|
|
sh 'npm run build'
|
|
}
|
|
}
|
|
}
|
|
stage('web/storagenode') {
|
|
steps {
|
|
dir('web/storagenode') {
|
|
sh 'npm install --prefer-offline --no-audit --loglevel silly'
|
|
sh 'npm run build'
|
|
}
|
|
}
|
|
}
|
|
stage('web/multinode') {
|
|
steps {
|
|
dir('web/multinode') {
|
|
sh 'npm install --prefer-offline --no-audit --loglevel silly'
|
|
sh 'npm run build'
|
|
}
|
|
}
|
|
}
|
|
stage('satellite/admin/ui') {
|
|
steps {
|
|
dir('satellite/admin/ui') {
|
|
sh 'npm install --prefer-offline --no-audit --loglevel silly'
|
|
sh 'npm run build'
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
stage('Build') {
|
|
parallel {
|
|
stage('go') {
|
|
steps {
|
|
// use go test to build all the packages, including tests
|
|
sh 'go test -v -p 16 -tags noembed -bench XYZXYZXYZXYZ -run XYZXYZXYZXYZ ./...'
|
|
}
|
|
}
|
|
stage('go -race') {
|
|
steps {
|
|
// use go test to build all the packages, including tests
|
|
sh 'go test -v -p 16 -tags noembed -bench XYZXYZXYZXYZ -run XYZXYZXYZXYZ -race ./...'
|
|
|
|
// install storj-sim
|
|
sh 'go install -race -v storj.io/storj/cmd/satellite '+
|
|
'storj.io/storj/cmd/storagenode ' +
|
|
'storj.io/storj/cmd/storj-sim ' +
|
|
'storj.io/storj/cmd/versioncontrol ' +
|
|
'storj.io/storj/cmd/uplink ' +
|
|
'storj.io/storj/cmd/identity ' +
|
|
'storj.io/storj/cmd/certificates ' +
|
|
'storj.io/storj/cmd/multinode'
|
|
}
|
|
}
|
|
stage('go -race gateway') {
|
|
steps {
|
|
// install gateway for storj-sim
|
|
sh 'go install -race -v storj.io/gateway@latest'
|
|
}
|
|
}
|
|
|
|
stage('db') {
|
|
steps {
|
|
sh 'service postgresql start'
|
|
dir('.build') {
|
|
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26256 --http-addr=localhost:8086 --cache 512MiB --max-sql-memory 512MiB --background'
|
|
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26257 --http-addr=localhost:8087 --cache 512MiB --max-sql-memory 512MiB --background'
|
|
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26258 --http-addr=localhost:8088 --cache 512MiB --max-sql-memory 512MiB --background'
|
|
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26259 --http-addr=localhost:8089 --cache 512MiB --max-sql-memory 512MiB --background'
|
|
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26260 --http-addr=localhost:8090 --cache 256MiB --max-sql-memory 256MiB --background'
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Verification') {
|
|
parallel {
|
|
stage('Cross Compile') {
|
|
steps {
|
|
// verify most of the commands, we cannot check everything since some of them
|
|
// have a C dependency and we don't have cross-compilation in storj/ci image
|
|
sh 'GOOS=linux GOARCH=386 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
|
|
sh 'GOOS=linux GOARCH=amd64 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
|
|
sh 'GOOS=linux GOARCH=arm go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
|
|
sh 'GOOS=linux GOARCH=arm64 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
|
|
sh 'GOOS=windows GOARCH=386 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
|
|
sh 'GOOS=windows GOARCH=amd64 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
|
|
sh 'GOOS=windows GOARCH=arm64 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
|
|
sh 'GOOS=darwin GOARCH=amd64 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
|
|
sh 'GOOS=darwin GOARCH=arm64 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
|
|
}
|
|
}
|
|
|
|
stage('Check Benchmark') {
|
|
environment {
|
|
STORJ_TEST_COCKROACH = 'omit'
|
|
STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/benchstorj?sslmode=disable'
|
|
}
|
|
steps {
|
|
sh 'psql -U postgres -c \'create database benchstorj;\''
|
|
sh 'go test -tags noembed -parallel 1 -p 1 -vet=off -timeout 20m -short -run XYZXYZXYZXYZ -bench . -benchtime 1x ./...'
|
|
}
|
|
}
|
|
|
|
stage('Integration') {
|
|
environment {
|
|
// use different hostname to avoid port conflicts
|
|
STORJ_NETWORK_HOST4 = '127.0.0.2'
|
|
STORJ_NETWORK_HOST6 = '127.0.0.2'
|
|
|
|
STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/teststorj2?sslmode=disable'
|
|
}
|
|
|
|
steps {
|
|
sh 'psql -U postgres -c \'create database teststorj2;\''
|
|
sh 'make test-sim'
|
|
|
|
// sh 'make test-certificates' // flaky
|
|
}
|
|
}
|
|
|
|
stage('Cockroach Integration') {
|
|
environment {
|
|
STORJ_NETWORK_HOST4 = '127.0.0.4'
|
|
STORJ_NETWORK_HOST6 = '127.0.0.4'
|
|
|
|
STORJ_SIM_POSTGRES = 'cockroach://root@localhost:26257/testcockroach4?sslmode=disable'
|
|
}
|
|
|
|
steps {
|
|
sh 'cockroach sql --insecure --host=localhost:26257 -e \'create database testcockroach4;\''
|
|
sh 'make test-sim'
|
|
sh 'cockroach sql --insecure --host=localhost:26257 -e \'drop database testcockroach4;\''
|
|
}
|
|
}
|
|
|
|
stage('Integration Redis unavailability') {
|
|
environment {
|
|
// use different hostname to avoid port conflicts
|
|
STORJ_NETWORK_HOST4 = '127.0.0.6'
|
|
STORJ_NETWORK_HOST6 = '127.0.0.6'
|
|
STORJ_REDIS_PORT = '7379'
|
|
|
|
STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/teststorj6?sslmode=disable'
|
|
}
|
|
|
|
steps {
|
|
sh 'psql -U postgres -c \'create database teststorj6;\''
|
|
sh 'make test-sim-redis-unavailability'
|
|
}
|
|
}
|
|
|
|
stage('Backwards Compatibility') {
|
|
environment {
|
|
STORJ_NETWORK_HOST4 = '127.0.0.3'
|
|
STORJ_NETWORK_HOST6 = '127.0.0.3'
|
|
|
|
STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/teststorj3?sslmode=disable'
|
|
STORJ_MIGRATION_DB = 'postgres://postgres@localhost/teststorj3?sslmode=disable&options=--search_path=satellite/0/meta'
|
|
}
|
|
|
|
steps {
|
|
sh 'psql -U postgres -c \'create database teststorj3;\''
|
|
sh 'make test-sim-backwards-compatible'
|
|
}
|
|
}
|
|
|
|
stage('Cockroach Backwards Compatibility') {
|
|
environment {
|
|
STORJ_NETWORK_HOST4 = '127.0.0.5'
|
|
STORJ_NETWORK_HOST6 = '127.0.0.5'
|
|
|
|
STORJ_SIM_POSTGRES = 'cockroach://root@localhost:26257/testcockroach5?sslmode=disable'
|
|
STORJ_MIGRATION_DB = 'postgres://root@localhost:26257/testcockroach5/satellite/0/meta?sslmode=disable'
|
|
}
|
|
|
|
steps {
|
|
sh 'cockroach sql --insecure --host=localhost:26257 -e \'create database testcockroach5;\''
|
|
sh 'make test-sim-backwards-compatible'
|
|
sh 'cockroach sql --insecure --host=localhost:26257 -e \'drop database testcockroach5;\''
|
|
}
|
|
}
|
|
|
|
stage('wasm npm') {
|
|
steps {
|
|
dir(".build") {
|
|
sh 'cp -r ../satellite/console/wasm/tests/ .'
|
|
sh 'cd tests && cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" .'
|
|
sh 'cd tests && npm install && npm run test'
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('web/satellite') {
|
|
steps {
|
|
dir("web/satellite") {
|
|
sh 'npm run lint-ci'
|
|
sh script: 'npm audit', returnStatus: true
|
|
sh 'npm run test'
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('web/storagenode') {
|
|
steps {
|
|
dir("web/storagenode") {
|
|
sh 'npm run lint-ci'
|
|
sh script: 'npm audit', returnStatus: true
|
|
sh 'npm run test'
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('web/multinode') {
|
|
steps {
|
|
dir("web/multinode") {
|
|
sh 'npm run lint-ci'
|
|
sh script: 'npm audit', returnStatus: true
|
|
sh 'npm run test'
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('satellite/admin/ui') {
|
|
steps {
|
|
dir("satellite/admin/ui") {
|
|
sh script: 'npm audit', returnStatus: true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Post') {
|
|
parallel {
|
|
stage('Lint') {
|
|
steps {
|
|
sh 'check-clean-directory'
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
post {
|
|
success {
|
|
withCredentials([sshUserPrivateKey(credentialsId: 'gerrit-trigger-ssh', keyFileVariable: 'SSH_KEY', usernameVariable: 'SSH_USER')]) {
|
|
sh './scripts/gerrit-status.sh pre-merge success +2'
|
|
}
|
|
}
|
|
failure {
|
|
withCredentials([sshUserPrivateKey(credentialsId: 'gerrit-trigger-ssh', keyFileVariable: 'SSH_KEY', usernameVariable: 'SSH_USER')]) {
|
|
sh './scripts/gerrit-status.sh pre-merge failure -2'
|
|
}
|
|
}
|
|
aborted {
|
|
withCredentials([sshUserPrivateKey(credentialsId: 'gerrit-trigger-ssh', keyFileVariable: 'SSH_KEY', usernameVariable: 'SSH_USER')]) {
|
|
sh './scripts/gerrit-status.sh pre-merge failure -2'
|
|
}
|
|
}
|
|
}
|
|
}
|