diff --git a/Dockerfile.jenkins b/Dockerfile.jenkins index 32df766ac..6d9942829 100644 --- a/Dockerfile.jenkins +++ b/Dockerfile.jenkins @@ -27,4 +27,8 @@ RUN go get github.com/nilslice/protolock/cmd/protolock RUN go get github.com/mfridman/tparse RUN go get github.com/josephspurrier/goversioninfo +RUN go get github.com/axw/gocov/gocov +RUN go get github.com/AlekSi/gocov-xml +RUN go get gopkg.in/matm/v1/gocov-html + RUN go version \ No newline at end of file diff --git a/Jenkinsfile.public b/Jenkinsfile.public index 4363565c0..fbb2e9d59 100644 --- a/Jenkinsfile.public +++ b/Jenkinsfile.public @@ -29,7 +29,7 @@ pipeline { stage('Lint') { steps { sh 'go run ./scripts/check-copyright.go' - sh 'go run ./scripts/check-imports.go' + sh 'go run ./scripts/check-imports.go ./...' sh 'go run ./scripts/protobuf.go --protoc=$HOME/protoc/bin/protoc lint' sh 'protolock status' sh 'bash ./scripts/check-dbx-version.sh' @@ -41,11 +41,12 @@ pipeline { stage('Tests') { environment { STORJ_POSTGRES_TEST = 'postgres://postgres@localhost/teststorj?sslmode=disable' + COVERFLAGS = "${ env.BRANCH_NAME != 'master' ? '' : '-coverprofile=.build/coverprofile -coverpkg=-coverpkg=storj.io/storj/bootstrap/...,storj.io/storj/internal/...,storj.io/storj/lib/...,storj.io/storj/pkg/...,storj.io/storj/satellite/...,storj.io/storj/storage/...,storj.io/storj/storagenode/...,storj.io/storj/uplink/...,storj.io/storj/versioncontrol/...'}" } steps { sh 'psql -U postgres -c \'create database teststorj;\'' sh 'go run scripts/use-ports.go -from 1024 -to 10000 &' - sh 'go test -vet=off -timeout 9m -json -race ./... 2>&1 | tee .build/tests.json | go run ./scripts/xunit.go -out .build/tests.xml' + sh 'go test -vet=off $COVERFLAGS -timeout 9m -json -race ./... 2>&1 | tee .build/tests.json | go run ./scripts/xunit.go -out .build/tests.xml' sh 'go run scripts/check-clean-directory.go' } @@ -54,6 +55,26 @@ pipeline { sh script: 'cat .build/tests.json | tparse -all -top -slow 100', returnStatus: true archiveArtifacts artifacts: '.build/tests.json' junit '.build/tests.xml' + + script { + if(fileExists(".build/coverprofile")){ + sh script: 'go run ./scripts/cover-remove-generated.go < .build/coverprofile > .build/clean.coverprofile', returnStatus: true + sh script: 'gocov convert .build/clean.coverprofile > .build/cover.json', returnStatus: true + + sh script: 'gocov-html < .build/cover.json > .build/cover.html', returnStatus: true + publishHTML target: [ + allowMissing: false, + alwaysLinkToLastBuild: false, + keepAll: false, + reportDir: '.build', + reportFiles: 'cover.html', + reportName: 'Coverage Details' + ] + + sh script: 'gocov-xml < .build/cover.json > .build/cobertura.xml', returnStatus: true + cobertura coberturaReportFile: '.build/cobertura.xml' + } + } } } } diff --git a/internal/dbutil/pgutil/pgtest/flag.go b/internal/dbutil/pgutil/pgtest/flag.go new file mode 100644 index 000000000..cdaf70f74 --- /dev/null +++ b/internal/dbutil/pgutil/pgtest/flag.go @@ -0,0 +1,17 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package pgtest + +import ( + "flag" + "os" +) + +// We need to define this in a separate package due to https://golang.org/issue/23910. + +// ConnStr is the test database connection string. +var ConnStr = flag.String("postgres-test-db", os.Getenv("STORJ_POSTGRES_TEST"), "PostgreSQL test database connection string") + +// DefaultConnStr is expected to work under the storj-test docker-compose instance +const DefaultConnStr = "postgres://storj:storj-pass@test-postgres/teststorj?sslmode=disable" diff --git a/internal/dbutil/pgutil/query_test.go b/internal/dbutil/pgutil/query_test.go index 44c626231..cf640db49 100644 --- a/internal/dbutil/pgutil/query_test.go +++ b/internal/dbutil/pgutil/query_test.go @@ -4,8 +4,6 @@ package pgutil_test import ( - "flag" - "os" "testing" _ "github.com/lib/pq" @@ -14,6 +12,7 @@ import ( "storj.io/storj/internal/dbutil/dbschema" "storj.io/storj/internal/dbutil/pgutil" + "storj.io/storj/internal/dbutil/pgutil/pgtest" "storj.io/storj/internal/testcontext" ) @@ -22,20 +21,15 @@ const ( DefaultPostgresConn = "postgres://storj:storj-pass@test-postgres/teststorj?sslmode=disable" ) -var ( - // TestPostgres is flag for the postgres test database - TestPostgres = flag.String("postgres-test-db", os.Getenv("STORJ_POSTGRES_TEST"), "PostgreSQL test database connection string") -) - func TestQuery(t *testing.T) { - if *TestPostgres == "" { + if *pgtest.ConnStr == "" { t.Skip("Postgres flag missing, example: -postgres-test-db=" + DefaultPostgresConn) } ctx := testcontext.New(t) defer ctx.Cleanup() - db, err := pgutil.Open(*TestPostgres, "pgutil-query") + db, err := pgutil.Open(*pgtest.ConnStr, "pgutil-query") require.NoError(t, err) defer ctx.Check(db.Close) diff --git a/internal/migrate/create_test.go b/internal/migrate/create_test.go index 8acdddefc..c7477c17f 100644 --- a/internal/migrate/create_test.go +++ b/internal/migrate/create_test.go @@ -5,8 +5,6 @@ package migrate_test import ( "database/sql" - "flag" - "os" "strconv" "testing" @@ -14,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "storj.io/storj/internal/dbutil/pgutil" + "storj.io/storj/internal/dbutil/pgutil/pgtest" "storj.io/storj/internal/migrate" _ "github.com/lib/pq" @@ -44,19 +43,14 @@ func TestCreate_Sqlite(t *testing.T) { assert.Error(t, err) } -// this connstring is expected to work under the storj-test docker-compose instance -const defaultPostgresConn = "postgres://storj:storj-pass@test-postgres/teststorj?sslmode=disable" - -var testPostgres = flag.String("postgres-test-db", os.Getenv("STORJ_POSTGRES_TEST"), "PostgreSQL test database connection string") - func TestCreate_Postgres(t *testing.T) { - if *testPostgres == "" { - t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", defaultPostgresConn) + if *pgtest.ConnStr == "" { + t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", pgtest.DefaultConnStr) } schema := "create-" + pgutil.CreateRandomTestingSchemaName(8) - db, err := sql.Open("postgres", pgutil.ConnstrWithSchema(*testPostgres, schema)) + db, err := sql.Open("postgres", pgutil.ConnstrWithSchema(*pgtest.ConnStr, schema)) if err != nil { t.Fatal(err) } diff --git a/internal/migrate/versions_test.go b/internal/migrate/versions_test.go index 0a0fe1b5c..4e923dc39 100644 --- a/internal/migrate/versions_test.go +++ b/internal/migrate/versions_test.go @@ -17,6 +17,7 @@ import ( "go.uber.org/zap" "storj.io/storj/internal/dbutil/pgutil" + "storj.io/storj/internal/dbutil/pgutil/pgtest" "storj.io/storj/internal/migrate" "storj.io/storj/internal/testcontext" ) @@ -30,13 +31,13 @@ func TestBasicMigrationSqlite(t *testing.T) { } func TestBasicMigrationPostgres(t *testing.T) { - if *testPostgres == "" { - t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", defaultPostgresConn) + if *pgtest.ConnStr == "" { + t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", pgtest.DefaultConnStr) } schema := "create-" + pgutil.CreateRandomTestingSchemaName(8) - db, err := sql.Open("postgres", pgutil.ConnstrWithSchema(*testPostgres, schema)) + db, err := sql.Open("postgres", pgutil.ConnstrWithSchema(*pgtest.ConnStr, schema)) if err != nil { t.Fatal(err) } @@ -112,11 +113,11 @@ func TestMultipleMigrationSqlite(t *testing.T) { } func TestMultipleMigrationPostgres(t *testing.T) { - if *testPostgres == "" { - t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", defaultPostgresConn) + if *pgtest.ConnStr == "" { + t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", pgtest.DefaultConnStr) } - db, err := sql.Open("postgres", *testPostgres) + db, err := sql.Open("postgres", *pgtest.ConnStr) require.NoError(t, err) defer func() { assert.NoError(t, db.Close()) }() @@ -185,11 +186,11 @@ func TestFailedMigrationSqlite(t *testing.T) { } func TestFailedMigrationPostgres(t *testing.T) { - if *testPostgres == "" { - t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", defaultPostgresConn) + if *pgtest.ConnStr == "" { + t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", pgtest.DefaultConnStr) } - db, err := sql.Open("postgres", *testPostgres) + db, err := sql.Open("postgres", *pgtest.ConnStr) require.NoError(t, err) defer func() { assert.NoError(t, db.Close()) }() diff --git a/pkg/datarepair/datarepair.go b/pkg/datarepair/datarepair.go new file mode 100644 index 000000000..be597dda8 --- /dev/null +++ b/pkg/datarepair/datarepair.go @@ -0,0 +1,6 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package datarepair + +// Empty file to workaround coverpkg issue https://golang.org/issue/27333 diff --git a/satellite/satellitedb/migrate_postgres_test.go b/satellite/satellitedb/migrate_postgres_test.go index 2c87aaed6..e210ec37a 100644 --- a/satellite/satellitedb/migrate_postgres_test.go +++ b/satellite/satellitedb/migrate_postgres_test.go @@ -17,8 +17,8 @@ import ( "storj.io/storj/internal/dbutil/dbschema" "storj.io/storj/internal/dbutil/pgutil" + "storj.io/storj/internal/dbutil/pgutil/pgtest" "storj.io/storj/satellite/satellitedb" - "storj.io/storj/satellite/satellitedb/satellitedbtest" ) // loadSnapshots loads all the dbschemas from testdata/postgres.* caching the result @@ -90,11 +90,11 @@ const ( ) func TestMigratePostgres(t *testing.T) { - if *satellitedbtest.TestPostgres == "" { - t.Skip("Postgres flag missing, example: -postgres-test-db=" + satellitedbtest.DefaultPostgresConn) + if *pgtest.ConnStr == "" { + t.Skip("Postgres flag missing, example: -postgres-test-db=" + pgtest.DefaultConnStr) } - snapshots, err := loadSnapshots(*satellitedbtest.TestPostgres) + snapshots, err := loadSnapshots(*pgtest.ConnStr) require.NoError(t, err) for _, base := range snapshots.List { @@ -106,7 +106,7 @@ func TestMigratePostgres(t *testing.T) { t.Run(strconv.Itoa(base.Version), func(t *testing.T) { log := zaptest.NewLogger(t) schemaName := "migrate/satellite/" + strconv.Itoa(base.Version) + pgutil.CreateRandomTestingSchemaName(8) - connstr := pgutil.ConnstrWithSchema(*satellitedbtest.TestPostgres, schemaName) + connstr := pgutil.ConnstrWithSchema(*pgtest.ConnStr, schemaName) // create a new satellitedb connection db, err := satellitedb.New(log, connstr) @@ -170,7 +170,7 @@ func TestMigratePostgres(t *testing.T) { } // verify that we also match the dbx version - dbxschema, err := loadDBXSchema(*satellitedbtest.TestPostgres, rawdb.Schema()) + dbxschema, err := loadDBXSchema(*pgtest.ConnStr, rawdb.Schema()) require.NoError(t, err) require.Equal(t, dbxschema, finalSchema, "dbx") diff --git a/satellite/satellitedb/satellitedbtest/run.go b/satellite/satellitedb/satellitedbtest/run.go index 0a1381560..3549bdf10 100644 --- a/satellite/satellitedb/satellitedbtest/run.go +++ b/satellite/satellitedb/satellitedbtest/run.go @@ -6,8 +6,6 @@ package satellitedbtest // This package should be referenced only in test files! import ( - "flag" - "os" "strings" "testing" @@ -15,22 +13,16 @@ import ( "go.uber.org/zap/zaptest" "storj.io/storj/internal/dbutil/pgutil" + "storj.io/storj/internal/dbutil/pgutil/pgtest" "storj.io/storj/satellite" "storj.io/storj/satellite/satellitedb" ) const ( - // DefaultPostgresConn is a connstring that works with docker-compose - DefaultPostgresConn = "postgres://storj:storj-pass@test-postgres/teststorj?sslmode=disable" // DefaultSqliteConn is a connstring that is inmemory DefaultSqliteConn = "sqlite3://file::memory:?mode=memory" ) -var ( - // TestPostgres is flag for the postgres test database - TestPostgres = flag.String("postgres-test-db", os.Getenv("STORJ_POSTGRES_TEST"), "PostgreSQL test database connection string") -) - // Database describes a test database type Database struct { Name string @@ -42,7 +34,7 @@ type Database struct { func Databases() []Database { return []Database{ {"Sqlite", DefaultSqliteConn, ""}, - {"Postgres", *TestPostgres, "Postgres flag missing, example: -postgres-test-db=" + DefaultPostgresConn}, + {"Postgres", *pgtest.ConnStr, "Postgres flag missing, example: -postgres-test-db=" + pgtest.DefaultConnStr}, } } diff --git a/scripts/cover-remove-generated.go b/scripts/cover-remove-generated.go new file mode 100644 index 000000000..12c4a597d --- /dev/null +++ b/scripts/cover-remove-generated.go @@ -0,0 +1,27 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +// +build ignore + +package main + +import ( + "bufio" + "fmt" + "os" + "strings" +) + +func main() { + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(line, ".pb.") { + continue + } + if strings.Contains(line, ".dbx.") { + continue + } + fmt.Println(line) + } +} diff --git a/scripts/xunit.go b/scripts/xunit.go index 114770c07..3fe3fd714 100644 --- a/scripts/xunit.go +++ b/scripts/xunit.go @@ -265,7 +265,7 @@ func ProcessWithEcho(r io.Reader) (parse.Packages, error) { return nil, parse.ErrNotParseable } if hasRace { - return nil, parse.ErrRaceDetected + return pkgs, parse.ErrRaceDetected } return pkgs, nil diff --git a/storage/postgreskv/alternateclient_test.go b/storage/postgreskv/alternateclient_test.go index ced866119..4bbe7d40d 100644 --- a/storage/postgreskv/alternateclient_test.go +++ b/storage/postgreskv/alternateclient_test.go @@ -7,6 +7,7 @@ import ( "flag" "testing" + "storj.io/storj/internal/dbutil/pgutil/pgtest" "storj.io/storj/storage" "storj.io/storj/storage/testsuite" ) @@ -19,11 +20,11 @@ func newTestAlternatePostgres(t testing.TB) (store *AlternateClient, cleanup fun if !*doAltTests { t.Skip("alternate-implementation PG tests not enabled.") } - if *testPostgres == "" { - t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", defaultPostgresConn) + if *pgtest.ConnStr == "" { + t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", pgtest.DefaultConnStr) } - pgdb, err := AltNew(*testPostgres) + pgdb, err := AltNew(*pgtest.ConnStr) if err != nil { t.Fatalf("init: %v", err) } diff --git a/storage/postgreskv/client_test.go b/storage/postgreskv/client_test.go index fe9479bd5..d1d42a558 100644 --- a/storage/postgreskv/client_test.go +++ b/storage/postgreskv/client_test.go @@ -5,32 +5,22 @@ package postgreskv import ( "database/sql" - "flag" - "os" "testing" "github.com/lib/pq" "github.com/zeebo/errs" + "storj.io/storj/internal/dbutil/pgutil/pgtest" "storj.io/storj/storage" "storj.io/storj/storage/testsuite" ) -const ( - // this connstring is expected to work under the storj-test docker-compose instance - defaultPostgresConn = "postgres://storj:storj-pass@test-postgres/teststorj?sslmode=disable" -) - -var ( - testPostgres = flag.String("postgres-test-db", os.Getenv("STORJ_POSTGRES_TEST"), "PostgreSQL test database connection string") -) - func newTestPostgres(t testing.TB) (store *Client, cleanup func()) { - if *testPostgres == "" { - t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", defaultPostgresConn) + if *pgtest.ConnStr == "" { + t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", pgtest.DefaultConnStr) } - pgdb, err := New(*testPostgres) + pgdb, err := New(*pgtest.ConnStr) if err != nil { t.Fatalf("init: %v", err) }