15dc797efd
Adding go.mod into node_modules is not sufficient, because npm install wipes them quite often out. Similarly, when running npm install locally it will remove it, causing the git state to be dirty. Rather than having them committed, add them after running npm install. Change-Id: Iaf21a9c6e198dc31fe50345ec5dee85b44617176
329 lines
15 KiB
Plaintext
329 lines
15 KiB
Plaintext
pipeline {
|
|
agent {
|
|
docker {
|
|
label 'main'
|
|
image 'storjlabs/ci:latest'
|
|
alwaysPull true
|
|
args '-u root:root --cap-add SYS_PTRACE -v "/tmp/gomod":/go/pkg/mod -v "/tmp/npm":/npm --tmpfs "/tmp:exec,mode=777"'
|
|
}
|
|
}
|
|
options {
|
|
timeout(time: 40, unit: 'MINUTES')
|
|
skipDefaultCheckout(true)
|
|
}
|
|
environment {
|
|
NPM_CONFIG_CACHE = '/npm/cache'
|
|
GOTRACEBACK = 'all'
|
|
COCKROACH_MEMPROF_INTERVAL=0
|
|
}
|
|
stages {
|
|
stage('Checkout') {
|
|
steps {
|
|
// Delete any content left over from a previous run.
|
|
sh "chmod -R 777 ."
|
|
// Bash requires extglob option to support !(.git) syntax,
|
|
// and we don't want to delete .git to have faster clones.
|
|
sh 'bash -O extglob -c "rm -rf !(.git)"'
|
|
|
|
checkout scm
|
|
|
|
sh 'mkdir -p .build'
|
|
|
|
// download dependencies
|
|
sh 'go mod download'
|
|
sh 'cd testsuite/ui && go mod download'
|
|
|
|
// pre-check that we cannot do at a later stage reliably
|
|
sh 'check-large-files'
|
|
|
|
// add a stub go.mod file to node projects to prevent go build ./... from scanning them
|
|
sh 'mkdir -p web/satellite/node_modules web/storagenode/node_modules web/multinode/node_modules'
|
|
sh 'touch web/satellite/node_modules/go.mod web/storagenode/node_modules/go.mod web/multinode/node_modules/go.mod'
|
|
}
|
|
}
|
|
stage('Gerrit status') {
|
|
steps {
|
|
withCredentials([sshUserPrivateKey(credentialsId: 'gerrit-trigger-ssh', keyFileVariable: 'SSH_KEY', usernameVariable: 'SSH_USER')]) {
|
|
sh './scripts/gerrit-status.sh pre-merge start +1'
|
|
}
|
|
}
|
|
}
|
|
stage('Build Web') {
|
|
options {
|
|
timeout(time: 6, unit: "MINUTES")
|
|
}
|
|
|
|
// The build code depends on the following assets being loaded.
|
|
parallel {
|
|
stage('web/satellite') {
|
|
steps {
|
|
dir('web/satellite') {
|
|
sh 'npm install --prefer-offline --no-audit --loglevel silly'
|
|
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
|
|
sh './scripts/build-wasm.sh'
|
|
sh 'npm run build'
|
|
}
|
|
}
|
|
}
|
|
stage('web/storagenode') {
|
|
steps {
|
|
dir('web/storagenode') {
|
|
sh 'npm install --prefer-offline --no-audit --loglevel silly'
|
|
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
|
|
sh 'npm run build'
|
|
}
|
|
}
|
|
}
|
|
stage('web/multinode') {
|
|
steps {
|
|
dir('web/multinode') {
|
|
sh 'npm install --prefer-offline --no-audit --loglevel silly'
|
|
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
|
|
sh 'npm run build'
|
|
}
|
|
}
|
|
}
|
|
stage('satellite/admin/ui') {
|
|
steps {
|
|
dir('satellite/admin/ui') {
|
|
sh 'npm install --prefer-offline --no-audit --loglevel silly'
|
|
sh 'touch ./node_modules/go.mod' // prevent Go from scanning this dir
|
|
sh 'npm run build'
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
stage('Build') {
|
|
parallel {
|
|
stage('go') {
|
|
steps {
|
|
// use go test to build all the packages, including tests
|
|
sh 'go test -v -p 16 -tags noembed -bench XYZXYZXYZXYZ -run XYZXYZXYZXYZ ./...'
|
|
}
|
|
}
|
|
stage('go -race') {
|
|
steps {
|
|
// use go test to build all the packages, including tests
|
|
sh 'go test -v -p 16 -tags noembed -bench XYZXYZXYZXYZ -run XYZXYZXYZXYZ -race ./...'
|
|
|
|
// install storj-sim
|
|
sh 'go install -race -v storj.io/storj/cmd/satellite '+
|
|
'storj.io/storj/cmd/storagenode ' +
|
|
'storj.io/storj/cmd/storj-sim ' +
|
|
'storj.io/storj/cmd/versioncontrol ' +
|
|
'storj.io/storj/cmd/uplink ' +
|
|
'storj.io/storj/cmd/identity ' +
|
|
'storj.io/storj/cmd/certificates ' +
|
|
'storj.io/storj/cmd/multinode'
|
|
}
|
|
}
|
|
stage('go -race gateway') {
|
|
steps {
|
|
// install gateway for storj-sim
|
|
sh 'go install -race -v storj.io/gateway@latest'
|
|
}
|
|
}
|
|
|
|
stage('db') {
|
|
steps {
|
|
sh 'service postgresql start'
|
|
dir('.build') {
|
|
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26256 --http-addr=localhost:8086 --cache 512MiB --max-sql-memory 512MiB --background'
|
|
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26257 --http-addr=localhost:8087 --cache 512MiB --max-sql-memory 512MiB --background'
|
|
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26258 --http-addr=localhost:8088 --cache 512MiB --max-sql-memory 512MiB --background'
|
|
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26259 --http-addr=localhost:8089 --cache 512MiB --max-sql-memory 512MiB --background'
|
|
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26260 --http-addr=localhost:8090 --cache 256MiB --max-sql-memory 256MiB --background'
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Verification') {
|
|
parallel {
|
|
stage('Cross Compile') {
|
|
steps {
|
|
// verify most of the commands, we cannot check everything since some of them
|
|
// have a C dependency and we don't have cross-compilation in storj/ci image
|
|
sh 'GOOS=linux GOARCH=386 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
|
|
sh 'GOOS=linux GOARCH=amd64 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
|
|
sh 'GOOS=linux GOARCH=arm go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
|
|
sh 'GOOS=linux GOARCH=arm64 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
|
|
sh 'GOOS=windows GOARCH=386 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
|
|
sh 'GOOS=windows GOARCH=amd64 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
|
|
sh 'GOOS=windows GOARCH=arm64 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
|
|
sh 'GOOS=darwin GOARCH=amd64 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
|
|
sh 'GOOS=darwin GOARCH=arm64 go vet ./cmd/uplink ./cmd/satellite ./cmd/storagenode-updater ./cmd/storj-sim'
|
|
}
|
|
}
|
|
|
|
stage('Check Benchmark') {
|
|
environment {
|
|
STORJ_TEST_COCKROACH = 'omit'
|
|
STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/benchstorj?sslmode=disable'
|
|
}
|
|
steps {
|
|
sh 'psql -U postgres -c \'create database benchstorj;\''
|
|
sh 'go test -tags noembed -parallel 1 -p 1 -vet=off -timeout 20m -short -run XYZXYZXYZXYZ -bench . -benchtime 1x ./...'
|
|
}
|
|
}
|
|
|
|
stage('Integration') {
|
|
environment {
|
|
// use different hostname to avoid port conflicts
|
|
STORJ_NETWORK_HOST4 = '127.0.0.2'
|
|
STORJ_NETWORK_HOST6 = '127.0.0.2'
|
|
|
|
STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/teststorj2?sslmode=disable'
|
|
}
|
|
|
|
steps {
|
|
sh 'psql -U postgres -c \'create database teststorj2;\''
|
|
sh 'make test-sim'
|
|
|
|
// sh 'make test-certificates' // flaky
|
|
}
|
|
}
|
|
|
|
stage('Cockroach Integration') {
|
|
environment {
|
|
STORJ_NETWORK_HOST4 = '127.0.0.4'
|
|
STORJ_NETWORK_HOST6 = '127.0.0.4'
|
|
|
|
STORJ_SIM_POSTGRES = 'cockroach://root@localhost:26257/testcockroach4?sslmode=disable'
|
|
}
|
|
|
|
steps {
|
|
sh 'cockroach sql --insecure --host=localhost:26257 -e \'create database testcockroach4;\''
|
|
sh 'make test-sim'
|
|
sh 'cockroach sql --insecure --host=localhost:26257 -e \'drop database testcockroach4;\''
|
|
}
|
|
}
|
|
|
|
stage('Integration Redis unavailability') {
|
|
environment {
|
|
// use different hostname to avoid port conflicts
|
|
STORJ_NETWORK_HOST4 = '127.0.0.6'
|
|
STORJ_NETWORK_HOST6 = '127.0.0.6'
|
|
STORJ_REDIS_PORT = '7379'
|
|
|
|
STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/teststorj6?sslmode=disable'
|
|
}
|
|
|
|
steps {
|
|
sh 'psql -U postgres -c \'create database teststorj6;\''
|
|
sh 'make test-sim-redis-unavailability'
|
|
}
|
|
}
|
|
|
|
stage('Backwards Compatibility') {
|
|
environment {
|
|
STORJ_NETWORK_HOST4 = '127.0.0.3'
|
|
STORJ_NETWORK_HOST6 = '127.0.0.3'
|
|
|
|
STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/teststorj3?sslmode=disable'
|
|
STORJ_MIGRATION_DB = 'postgres://postgres@localhost/teststorj3?sslmode=disable&options=--search_path=satellite/0/meta'
|
|
}
|
|
|
|
steps {
|
|
sh 'psql -U postgres -c \'create database teststorj3;\''
|
|
sh 'make test-sim-backwards-compatible'
|
|
}
|
|
}
|
|
|
|
stage('Cockroach Backwards Compatibility') {
|
|
environment {
|
|
STORJ_NETWORK_HOST4 = '127.0.0.5'
|
|
STORJ_NETWORK_HOST6 = '127.0.0.5'
|
|
|
|
STORJ_SIM_POSTGRES = 'cockroach://root@localhost:26257/testcockroach5?sslmode=disable'
|
|
STORJ_MIGRATION_DB = 'postgres://root@localhost:26257/testcockroach5/satellite/0/meta?sslmode=disable'
|
|
}
|
|
|
|
steps {
|
|
sh 'cockroach sql --insecure --host=localhost:26257 -e \'create database testcockroach5;\''
|
|
sh 'make test-sim-backwards-compatible'
|
|
sh 'cockroach sql --insecure --host=localhost:26257 -e \'drop database testcockroach5;\''
|
|
}
|
|
}
|
|
|
|
stage('wasm npm') {
|
|
steps {
|
|
dir(".build") {
|
|
sh 'cp -r ../satellite/console/wasm/tests/ .'
|
|
sh 'cd tests && cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" .'
|
|
sh 'cd tests && npm install && npm run test'
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('web/satellite') {
|
|
steps {
|
|
dir("web/satellite") {
|
|
sh 'npm run lint-ci'
|
|
sh script: 'npm audit', returnStatus: true
|
|
sh 'npm run test'
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('web/storagenode') {
|
|
steps {
|
|
dir("web/storagenode") {
|
|
sh 'npm run lint-ci'
|
|
sh script: 'npm audit', returnStatus: true
|
|
sh 'npm run test'
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('web/multinode') {
|
|
steps {
|
|
dir("web/multinode") {
|
|
sh 'npm run lint-ci'
|
|
sh script: 'npm audit', returnStatus: true
|
|
sh 'npm run test'
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('satellite/admin/ui') {
|
|
steps {
|
|
dir("satellite/admin/ui") {
|
|
sh script: 'npm audit', returnStatus: true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Post') {
|
|
parallel {
|
|
stage('Lint') {
|
|
steps {
|
|
sh 'check-clean-directory'
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
post {
|
|
success {
|
|
withCredentials([sshUserPrivateKey(credentialsId: 'gerrit-trigger-ssh', keyFileVariable: 'SSH_KEY', usernameVariable: 'SSH_USER')]) {
|
|
sh './scripts/gerrit-status.sh pre-merge success +2'
|
|
}
|
|
}
|
|
failure {
|
|
withCredentials([sshUserPrivateKey(credentialsId: 'gerrit-trigger-ssh', keyFileVariable: 'SSH_KEY', usernameVariable: 'SSH_USER')]) {
|
|
sh './scripts/gerrit-status.sh pre-merge failure -2'
|
|
}
|
|
}
|
|
aborted {
|
|
withCredentials([sshUserPrivateKey(credentialsId: 'gerrit-trigger-ssh', keyFileVariable: 'SSH_KEY', usernameVariable: 'SSH_USER')]) {
|
|
sh './scripts/gerrit-status.sh pre-merge failure -2'
|
|
}
|
|
}
|
|
}
|
|
}
|