From 97e6e5a6a360a99e8f5ec727855aa71f8b85efb7 Mon Sep 17 00:00:00 2001 From: Egon Elbre Date: Fri, 1 Jul 2022 11:42:22 +0300 Subject: [PATCH] build: improve npm build Add go.mod to node_modules folder, that way Go compiler doesn't need to scan the node_module directories for any Go code. Change-Id: I747909416490c847d6b4bfa3438fea66660fcd53 --- Jenkinsfile.public | 4 - Jenkinsfile.verify | 161 ++++++++++++------------- satellite/admin/ui/.gitignore | 3 + satellite/admin/ui/node_modules/go.mod | 5 + web/multinode/.gitignore | 3 + web/multinode/node_modules/go.mod | 5 + web/satellite/.gitignore | 3 + web/satellite/node_modules/go.mod | 5 + web/storagenode/.gitignore | 3 + web/storagenode/node_modules/go.mod | 5 + 10 files changed, 107 insertions(+), 90 deletions(-) create mode 100644 satellite/admin/ui/node_modules/go.mod create mode 100644 web/multinode/node_modules/go.mod create mode 100644 web/satellite/node_modules/go.mod create mode 100644 web/storagenode/node_modules/go.mod diff --git a/Jenkinsfile.public b/Jenkinsfile.public index b23a56326..b367c8db2 100644 --- a/Jenkinsfile.public +++ b/Jenkinsfile.public @@ -35,10 +35,6 @@ pipeline { // pre-check that we cannot do at a later stage reliably sh 'check-large-files' - - // add a stub go.mod file to node projects to prevent go build ./... from scanning them - sh 'mkdir -p web/satellite/node_modules web/storagenode/node_modules web/multinode/node_modules' - sh 'touch web/satellite/node_modules/go.mod web/storagenode/node_modules/go.mod web/multinode/node_modules/go.mod' } } stage('Build Web') { diff --git a/Jenkinsfile.verify b/Jenkinsfile.verify index 2b1987453..0e34f3ab0 100644 --- a/Jenkinsfile.verify +++ b/Jenkinsfile.verify @@ -35,10 +35,6 @@ pipeline { // pre-check that we cannot do at a later stage reliably sh 'check-large-files' - - // add a stub go.mod file to node projects to prevent go build ./... from scanning them - sh 'mkdir -p web/satellite/node_modules web/storagenode/node_modules web/multinode/node_modules' - sh 'touch web/satellite/node_modules/go.mod web/storagenode/node_modules/go.mod web/multinode/node_modules/go.mod' } } stage('Gerrit status') { @@ -93,7 +89,8 @@ pipeline { } } } - stage('Build Web') { + + stage('Build and Lint Web') { // The build code depends on the following assets being loaded. parallel { stage('web/satellite') { @@ -158,99 +155,91 @@ pipeline { sh 'npm ci --prefer-offline --no-audit' sh 'npm run build' sh script: 'npm audit', returnStatus: true + sh 'npm run check' + sh 'npm run lint' } } } } } - stage('Lint') { - steps { - sh 'check-mod-tidy' - sh 'check-copyright' - sh 'check-imports -race ./...' - sh 'check-peer-constraints -race' - sh 'check-atomic-align ./...' - sh 'check-monkit ./...' - sh 'check-errs ./...' - sh 'staticcheck ./...' - sh 'golangci-lint --config /go/ci/.golangci.yml -j=2 run' - sh 'make check-monitoring' - sh 'make test-wasm-size' + stage('Lint') { + steps { + sh 'check-mod-tidy' + sh 'check-copyright' + sh 'check-imports -race ./...' + sh 'check-peer-constraints -race' + sh 'check-atomic-align ./...' + sh 'check-monkit ./...' + sh 'check-errs ./...' + sh 'staticcheck ./...' + sh 'golangci-lint --config /go/ci/.golangci.yml -j=2 run' + sh 'make check-monitoring' + sh 'make test-wasm-size' - sh 'protolock status' + sh 'protolock status' - dir("testsuite/ui") { - sh 'check-imports ./...' - sh 'check-atomic-align ./...' - sh 'check-monkit ./...' - sh 'check-errs ./...' - sh 'staticcheck ./...' - sh 'golangci-lint --config /go/ci/.golangci.yml -j=2 run' - } - - sh './scripts/check-package-lock.sh' - } + dir("testsuite/ui") { + sh 'check-imports ./...' + sh 'check-atomic-align ./...' + sh 'check-monkit ./...' + sh 'check-errs ./...' + sh 'staticcheck ./...' + sh 'golangci-lint --config /go/ci/.golangci.yml -j=2 run' } - stage('Lint Admin UI') { - when { - changeset "satellite/admin/ui" - } - steps { - dir("satellite/admin/ui") { - sh 'npm run check' - sh 'npm run lint' - } - } - } - stage('Tests') { - environment { - STORJ_TEST_HOST = '127.0.0.20;127.0.0.21;127.0.0.22;127.0.0.23;127.0.0.24;127.0.0.25' - STORJ_TEST_COCKROACH = 'cockroach://root@localhost:26256/testcockroach?sslmode=disable;' + - 'cockroach://root@localhost:26257/testcockroach?sslmode=disable;' + - 'cockroach://root@localhost:26258/testcockroach?sslmode=disable;' + - 'cockroach://root@localhost:26259/testcockroach?sslmode=disable' - STORJ_TEST_COCKROACH_NODROP = 'true' - STORJ_TEST_COCKROACH_ALT = 'cockroach://root@localhost:26260/testcockroach?sslmode=disable' - STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/teststorj?sslmode=disable' - STORJ_TEST_LOG_LEVEL = 'info' - COVERFLAGS = "${ env.BRANCH_NAME == 'main' ? '-coverprofile=.build/coverprofile -coverpkg=storj.io/storj/private/...,storj.io/storj/satellite/...,storj.io/storj/storage/...,storj.io/storj/storagenode/...,storj.io/storj/versioncontrol/...' : ''}" - } - steps { - sh 'cockroach sql --insecure --host=localhost:26256 -e \'create database testcockroach;\'' - sh 'cockroach sql --insecure --host=localhost:26257 -e \'create database testcockroach;\'' - sh 'cockroach sql --insecure --host=localhost:26258 -e \'create database testcockroach;\'' - sh 'cockroach sql --insecure --host=localhost:26259 -e \'create database testcockroach;\'' - sh 'cockroach sql --insecure --host=localhost:26260 -e \'create database testcockroach;\'' - - sh 'psql -U postgres -c \'create database teststorj;\'' - - sh 'use-ports -from 1024 -to 10000 &' - - sh 'go test -tags noembed -parallel 4 -p 6 -vet=off $COVERFLAGS -timeout 32m -json -race ./... 2>&1 | tee .build/tests.json | xunit -out .build/tests.xml' - } - - post { - always { - archiveArtifacts artifacts: '.build/tests.json' - sh script: 'cat .build/tests.json | tparse -all -top -slow 100', returnStatus: true - junit '.build/tests.xml' - - script { - if(fileExists(".build/coverprofile")){ - sh script: 'filter-cover-profile < .build/coverprofile > .build/clean.coverprofile', returnStatus: true - sh script: 'gocov convert .build/clean.coverprofile > .build/cover.json', returnStatus: true - sh script: 'gocov-xml < .build/cover.json > .build/cobertura.xml', returnStatus: true - cobertura coberturaReportFile: '.build/cobertura.xml', - lineCoverageTargets: '70, 60, 50', - autoUpdateHealth: false, - autoUpdateStability: false, - failUnhealthy: true - } - } + + sh './scripts/check-package-lock.sh' + } + } + + stage('Tests') { + environment { + STORJ_TEST_HOST = '127.0.0.20;127.0.0.21;127.0.0.22;127.0.0.23;127.0.0.24;127.0.0.25' + STORJ_TEST_COCKROACH = 'cockroach://root@localhost:26256/testcockroach?sslmode=disable;' + + 'cockroach://root@localhost:26257/testcockroach?sslmode=disable;' + + 'cockroach://root@localhost:26258/testcockroach?sslmode=disable;' + + 'cockroach://root@localhost:26259/testcockroach?sslmode=disable' + STORJ_TEST_COCKROACH_NODROP = 'true' + STORJ_TEST_COCKROACH_ALT = 'cockroach://root@localhost:26260/testcockroach?sslmode=disable' + STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/teststorj?sslmode=disable' + STORJ_TEST_LOG_LEVEL = 'info' + COVERFLAGS = "${ env.BRANCH_NAME == 'main' ? '-coverprofile=.build/coverprofile -coverpkg=storj.io/storj/private/...,storj.io/storj/satellite/...,storj.io/storj/storage/...,storj.io/storj/storagenode/...,storj.io/storj/versioncontrol/...' : ''}" + } + steps { + sh 'cockroach sql --insecure --host=localhost:26256 -e \'create database testcockroach;\'' + sh 'cockroach sql --insecure --host=localhost:26257 -e \'create database testcockroach;\'' + sh 'cockroach sql --insecure --host=localhost:26258 -e \'create database testcockroach;\'' + sh 'cockroach sql --insecure --host=localhost:26259 -e \'create database testcockroach;\'' + sh 'cockroach sql --insecure --host=localhost:26260 -e \'create database testcockroach;\'' + + sh 'psql -U postgres -c \'create database teststorj;\'' + + sh 'use-ports -from 1024 -to 10000 &' + + sh 'go test -tags noembed -parallel 4 -p 6 -vet=off $COVERFLAGS -timeout 32m -json -race ./... 2>&1 | tee .build/tests.json | xunit -out .build/tests.xml' + } + + post { + always { + archiveArtifacts artifacts: '.build/tests.json' + sh script: 'cat .build/tests.json | tparse -all -top -slow 100', returnStatus: true + junit '.build/tests.xml' + + script { + if(fileExists(".build/coverprofile")){ + sh script: 'filter-cover-profile < .build/coverprofile > .build/clean.coverprofile', returnStatus: true + sh script: 'gocov convert .build/clean.coverprofile > .build/cover.json', returnStatus: true + sh script: 'gocov-xml < .build/cover.json > .build/cobertura.xml', returnStatus: true + cobertura coberturaReportFile: '.build/cobertura.xml', + lineCoverageTargets: '70, 60, 50', + autoUpdateHealth: false, + autoUpdateStability: false, + failUnhealthy: true } } } + } + } stage('Post') { parallel { diff --git a/satellite/admin/ui/.gitignore b/satellite/admin/ui/.gitignore index 483f84307..99f0a4840 100644 --- a/satellite/admin/ui/.gitignore +++ b/satellite/admin/ui/.gitignore @@ -6,3 +6,6 @@ node_modules /package .env .env.* + +# Stub for Go to avoid scanning node_modules. +!node_modules/go.mod \ No newline at end of file diff --git a/satellite/admin/ui/node_modules/go.mod b/satellite/admin/ui/node_modules/go.mod new file mode 100644 index 000000000..3ab96e155 --- /dev/null +++ b/satellite/admin/ui/node_modules/go.mod @@ -0,0 +1,5 @@ +module storj.io/storj/satellite/admin/ui/node_modules + +go 1.17 + +// Stub module to prevent Go from scanning node_modules directory. diff --git a/web/multinode/.gitignore b/web/multinode/.gitignore index f17f073f5..df91d8758 100644 --- a/web/multinode/.gitignore +++ b/web/multinode/.gitignore @@ -24,3 +24,6 @@ yarn-error.log* *.njsproj *.sln *.sw* + +# Stub for Go to avoid scanning node_modules. +!node_modules/go.mod \ No newline at end of file diff --git a/web/multinode/node_modules/go.mod b/web/multinode/node_modules/go.mod new file mode 100644 index 000000000..8205db366 --- /dev/null +++ b/web/multinode/node_modules/go.mod @@ -0,0 +1,5 @@ +module storj.io/storj/web/multinode/node_modules + +go 1.17 + +// Stub module to prevent Go from scanning node_modules directory. diff --git a/web/satellite/.gitignore b/web/satellite/.gitignore index f95433400..2e93f7c33 100644 --- a/web/satellite/.gitignore +++ b/web/satellite/.gitignore @@ -30,3 +30,6 @@ yarn-error.log* *.njsproj *.sln *.sw* + +# Stub for Go to avoid scanning node_modules. +!node_modules/go.mod \ No newline at end of file diff --git a/web/satellite/node_modules/go.mod b/web/satellite/node_modules/go.mod new file mode 100644 index 000000000..8098f1bc4 --- /dev/null +++ b/web/satellite/node_modules/go.mod @@ -0,0 +1,5 @@ +module storj.io/storj/web/satellite/node_modules + +go 1.17 + +// Stub module to prevent Go from scanning node_modules directory. diff --git a/web/storagenode/.gitignore b/web/storagenode/.gitignore index f17f073f5..df91d8758 100644 --- a/web/storagenode/.gitignore +++ b/web/storagenode/.gitignore @@ -24,3 +24,6 @@ yarn-error.log* *.njsproj *.sln *.sw* + +# Stub for Go to avoid scanning node_modules. +!node_modules/go.mod \ No newline at end of file diff --git a/web/storagenode/node_modules/go.mod b/web/storagenode/node_modules/go.mod new file mode 100644 index 000000000..8a6f46822 --- /dev/null +++ b/web/storagenode/node_modules/go.mod @@ -0,0 +1,5 @@ +module storj.io/storj/web/storagenode/node_modules + +go 1.17 + +// Stub module to prevent Go from scanning node_modules directory.