JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cover all the things (#1818)

Browse Source
This commit is contained in:
Egon Elbre 2019-04-26 16:39:11 +03:00 committed by GitHub
parent 8e09c30e2b
commit db939d37ec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 111 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Dockerfile.jenkins
View File

@ -27,4 +27,8 @@ RUN go get github.com/nilslice/protolock/cmd/protolock
RUN go get github.com/mfridman/tparse
RUN go get github.com/josephspurrier/goversioninfo
RUN go get github.com/axw/gocov/gocov
RUN go get github.com/AlekSi/gocov-xml
RUN go get gopkg.in/matm/v1/gocov-html
RUN go version

25
Jenkinsfile.public
View File

@ -29,7 +29,7 @@ pipeline {
stage('Lint') {
steps {
sh 'go run ./scripts/check-copyright.go'
sh 'go run ./scripts/check-imports.go'
sh 'go run ./scripts/check-imports.go ./...'
sh 'go run ./scripts/protobuf.go --protoc=$HOME/protoc/bin/protoc lint'
sh 'protolock status'
sh 'bash ./scripts/check-dbx-version.sh'
@ -41,11 +41,12 @@ pipeline {
stage('Tests') {
environment {
STORJ_POSTGRES_TEST = 'postgres://postgres@localhost/teststorj?sslmode=disable'
COVERFLAGS = "${ env.BRANCH_NAME != 'master' ? '' : '-coverprofile=.build/coverprofile -coverpkg=-coverpkg=storj.io/storj/bootstrap/...,storj.io/storj/internal/...,storj.io/storj/lib/...,storj.io/storj/pkg/...,storj.io/storj/satellite/...,storj.io/storj/storage/...,storj.io/storj/storagenode/...,storj.io/storj/uplink/...,storj.io/storj/versioncontrol/...'}"
}
steps {
sh 'psql -U postgres -c \'create database teststorj;\''
sh 'go run scripts/use-ports.go -from 1024 -to 10000 &'
sh 'go test -vet=off -timeout 9m -json -race ./... 2>&1 | tee .build/tests.json | go run ./scripts/xunit.go -out .build/tests.xml'
sh 'go test -vet=off $COVERFLAGS -timeout 9m -json -race ./... 2>&1 | tee .build/tests.json | go run ./scripts/xunit.go -out .build/tests.xml'
sh 'go run scripts/check-clean-directory.go'
}
@ -54,6 +55,26 @@ pipeline {
sh script: 'cat .build/tests.json | tparse -all -top -slow 100', returnStatus: true
archiveArtifacts artifacts: '.build/tests.json'
junit '.build/tests.xml'
script {
if(fileExists(".build/coverprofile")){
sh script: 'go run ./scripts/cover-remove-generated.go < .build/coverprofile > .build/clean.coverprofile', returnStatus: true
sh script: 'gocov convert .build/clean.coverprofile > .build/cover.json', returnStatus: true
sh script: 'gocov-html < .build/cover.json > .build/cover.html', returnStatus: true
publishHTML target: [
allowMissing: false,
alwaysLinkToLastBuild: false,
keepAll: false,
reportDir: '.build',
reportFiles: 'cover.html',
reportName: 'Coverage Details'
]
sh script: 'gocov-xml < .build/cover.json > .build/cobertura.xml', returnStatus: true
cobertura coberturaReportFile: '.build/cobertura.xml'
}
}
}
}
}

17
internal/dbutil/pgutil/pgtest/flag.go Normal file
View File

@ -0,0 +1,17 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package pgtest
import (
"flag"
"os"
)
// We need to define this in a separate package due to https://golang.org/issue/23910.
// ConnStr is the test database connection string.
var ConnStr = flag.String("postgres-test-db", os.Getenv("STORJ_POSTGRES_TEST"), "PostgreSQL test database connection string")
// DefaultConnStr is expected to work under the storj-test docker-compose instance
const DefaultConnStr = "postgres://storj:storj-pass@test-postgres/teststorj?sslmode=disable"

12
internal/dbutil/pgutil/query_test.go
View File

@ -4,8 +4,6 @@
package pgutil_test
import (
"flag"
"os"
"testing"
_ "github.com/lib/pq"
@ -14,6 +12,7 @@ import (
"storj.io/storj/internal/dbutil/dbschema"
"storj.io/storj/internal/dbutil/pgutil"
"storj.io/storj/internal/dbutil/pgutil/pgtest"
"storj.io/storj/internal/testcontext"
)
@ -22,20 +21,15 @@ const (
DefaultPostgresConn = "postgres://storj:storj-pass@test-postgres/teststorj?sslmode=disable"
)
var (
// TestPostgres is flag for the postgres test database
TestPostgres = flag.String("postgres-test-db", os.Getenv("STORJ_POSTGRES_TEST"), "PostgreSQL test database connection string")
)
func TestQuery(t *testing.T) {
if *TestPostgres == "" {
if *pgtest.ConnStr == "" {
t.Skip("Postgres flag missing, example: -postgres-test-db=" + DefaultPostgresConn)
}
ctx := testcontext.New(t)
defer ctx.Cleanup()
db, err := pgutil.Open(*TestPostgres, "pgutil-query")
db, err := pgutil.Open(*pgtest.ConnStr, "pgutil-query")
require.NoError(t, err)
defer ctx.Check(db.Close)

14
internal/migrate/create_test.go
View File

@ -5,8 +5,6 @@ package migrate_test
import (
"database/sql"
"flag"
"os"
"strconv"
"testing"
@ -14,6 +12,7 @@ import (
"github.com/stretchr/testify/require"
"storj.io/storj/internal/dbutil/pgutil"
"storj.io/storj/internal/dbutil/pgutil/pgtest"
"storj.io/storj/internal/migrate"
_ "github.com/lib/pq"
@ -44,19 +43,14 @@ func TestCreate_Sqlite(t *testing.T) {
assert.Error(t, err)
}
// this connstring is expected to work under the storj-test docker-compose instance
const defaultPostgresConn = "postgres://storj:storj-pass@test-postgres/teststorj?sslmode=disable"
var testPostgres = flag.String("postgres-test-db", os.Getenv("STORJ_POSTGRES_TEST"), "PostgreSQL test database connection string")
func TestCreate_Postgres(t *testing.T) {
if *testPostgres == "" {
t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", defaultPostgresConn)
if *pgtest.ConnStr == "" {
t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", pgtest.DefaultConnStr)
}
schema := "create-" + pgutil.CreateRandomTestingSchemaName(8)
db, err := sql.Open("postgres", pgutil.ConnstrWithSchema(*testPostgres, schema))
db, err := sql.Open("postgres", pgutil.ConnstrWithSchema(*pgtest.ConnStr, schema))
if err != nil {
t.Fatal(err)
}

19
internal/migrate/versions_test.go
View File

@ -17,6 +17,7 @@ import (
"go.uber.org/zap"
"storj.io/storj/internal/dbutil/pgutil"
"storj.io/storj/internal/dbutil/pgutil/pgtest"
"storj.io/storj/internal/migrate"
"storj.io/storj/internal/testcontext"
)
@ -30,13 +31,13 @@ func TestBasicMigrationSqlite(t *testing.T) {
}
func TestBasicMigrationPostgres(t *testing.T) {
if *testPostgres == "" {
t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", defaultPostgresConn)
if *pgtest.ConnStr == "" {
t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", pgtest.DefaultConnStr)
}
schema := "create-" + pgutil.CreateRandomTestingSchemaName(8)
db, err := sql.Open("postgres", pgutil.ConnstrWithSchema(*testPostgres, schema))
db, err := sql.Open("postgres", pgutil.ConnstrWithSchema(*pgtest.ConnStr, schema))
if err != nil {
t.Fatal(err)
}
@ -112,11 +113,11 @@ func TestMultipleMigrationSqlite(t *testing.T) {
}
func TestMultipleMigrationPostgres(t *testing.T) {
if *testPostgres == "" {
t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", defaultPostgresConn)
if *pgtest.ConnStr == "" {
t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", pgtest.DefaultConnStr)
}
db, err := sql.Open("postgres", *testPostgres)
db, err := sql.Open("postgres", *pgtest.ConnStr)
require.NoError(t, err)
defer func() { assert.NoError(t, db.Close()) }()
@ -185,11 +186,11 @@ func TestFailedMigrationSqlite(t *testing.T) {
}
func TestFailedMigrationPostgres(t *testing.T) {
if *testPostgres == "" {
t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", defaultPostgresConn)
if *pgtest.ConnStr == "" {
t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", pgtest.DefaultConnStr)
}
db, err := sql.Open("postgres", *testPostgres)
db, err := sql.Open("postgres", *pgtest.ConnStr)
require.NoError(t, err)
defer func() { assert.NoError(t, db.Close()) }()

6
pkg/datarepair/datarepair.go Normal file
View File

@ -0,0 +1,6 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package datarepair
// Empty file to workaround coverpkg issue https://golang.org/issue/27333

12
satellite/satellitedb/migrate_postgres_test.go
View File

@ -17,8 +17,8 @@ import (
"storj.io/storj/internal/dbutil/dbschema"
"storj.io/storj/internal/dbutil/pgutil"
"storj.io/storj/internal/dbutil/pgutil/pgtest"
"storj.io/storj/satellite/satellitedb"
"storj.io/storj/satellite/satellitedb/satellitedbtest"
)
// loadSnapshots loads all the dbschemas from testdata/postgres.* caching the result
@ -90,11 +90,11 @@ const (
)
func TestMigratePostgres(t *testing.T) {
if *satellitedbtest.TestPostgres == "" {
t.Skip("Postgres flag missing, example: -postgres-test-db=" + satellitedbtest.DefaultPostgresConn)
if *pgtest.ConnStr == "" {
t.Skip("Postgres flag missing, example: -postgres-test-db=" + pgtest.DefaultConnStr)
}
snapshots, err := loadSnapshots(*satellitedbtest.TestPostgres)
snapshots, err := loadSnapshots(*pgtest.ConnStr)
require.NoError(t, err)
for _, base := range snapshots.List {
@ -106,7 +106,7 @@ func TestMigratePostgres(t *testing.T) {
t.Run(strconv.Itoa(base.Version), func(t *testing.T) {
log := zaptest.NewLogger(t)
schemaName := "migrate/satellite/" + strconv.Itoa(base.Version) + pgutil.CreateRandomTestingSchemaName(8)
connstr := pgutil.ConnstrWithSchema(*satellitedbtest.TestPostgres, schemaName)
connstr := pgutil.ConnstrWithSchema(*pgtest.ConnStr, schemaName)
// create a new satellitedb connection
db, err := satellitedb.New(log, connstr)
@ -170,7 +170,7 @@ func TestMigratePostgres(t *testing.T) {
}
// verify that we also match the dbx version
dbxschema, err := loadDBXSchema(*satellitedbtest.TestPostgres, rawdb.Schema())
dbxschema, err := loadDBXSchema(*pgtest.ConnStr, rawdb.Schema())
require.NoError(t, err)
require.Equal(t, dbxschema, finalSchema, "dbx")

12
satellite/satellitedb/satellitedbtest/run.go
View File

@ -6,8 +6,6 @@ package satellitedbtest
// This package should be referenced only in test files!
import (
"flag"
"os"
"strings"
"testing"
@ -15,22 +13,16 @@ import (
"go.uber.org/zap/zaptest"
"storj.io/storj/internal/dbutil/pgutil"
"storj.io/storj/internal/dbutil/pgutil/pgtest"
"storj.io/storj/satellite"
"storj.io/storj/satellite/satellitedb"
)
const (
// DefaultPostgresConn is a connstring that works with docker-compose
DefaultPostgresConn = "postgres://storj:storj-pass@test-postgres/teststorj?sslmode=disable"
// DefaultSqliteConn is a connstring that is inmemory
DefaultSqliteConn = "sqlite3://file::memory:?mode=memory"
)
var (
// TestPostgres is flag for the postgres test database
TestPostgres = flag.String("postgres-test-db", os.Getenv("STORJ_POSTGRES_TEST"), "PostgreSQL test database connection string")
)
// Database describes a test database
type Database struct {
Name string
@ -42,7 +34,7 @@ type Database struct {
func Databases() []Database {
return []Database{
{"Sqlite", DefaultSqliteConn, ""},
{"Postgres", *TestPostgres, "Postgres flag missing, example: -postgres-test-db=" + DefaultPostgresConn},
{"Postgres", *pgtest.ConnStr, "Postgres flag missing, example: -postgres-test-db=" + pgtest.DefaultConnStr},
}
}

27
scripts/cover-remove-generated.go Normal file
View File

@ -0,0 +1,27 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
// +build ignore
package main
import (
"bufio"
"fmt"
"os"
"strings"
)
func main() {
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
line := scanner.Text()
if strings.Contains(line, ".pb.") {
continue
}
if strings.Contains(line, ".dbx.") {
continue
}
fmt.Println(line)
}
}

2
scripts/xunit.go
View File

@ -265,7 +265,7 @@ func ProcessWithEcho(r io.Reader) (parse.Packages, error) {
return nil, parse.ErrNotParseable
}
if hasRace {
return nil, parse.ErrRaceDetected
return pkgs, parse.ErrRaceDetected
}
return pkgs, nil

7
storage/postgreskv/alternateclient_test.go
View File

@ -7,6 +7,7 @@ import (
"flag"
"testing"
"storj.io/storj/internal/dbutil/pgutil/pgtest"
"storj.io/storj/storage"
"storj.io/storj/storage/testsuite"
)
@ -19,11 +20,11 @@ func newTestAlternatePostgres(t testing.TB) (store *AlternateClient, cleanup fun
if !*doAltTests {
t.Skip("alternate-implementation PG tests not enabled.")
}
if *testPostgres == "" {
t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", defaultPostgresConn)
if *pgtest.ConnStr == "" {
t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", pgtest.DefaultConnStr)
}
pgdb, err := AltNew(*testPostgres)
pgdb, err := AltNew(*pgtest.ConnStr)
if err != nil {
t.Fatalf("init: %v", err)
}

18
storage/postgreskv/client_test.go
View File

@ -5,32 +5,22 @@ package postgreskv
import (
"database/sql"
"flag"
"os"
"testing"
"github.com/lib/pq"
"github.com/zeebo/errs"
"storj.io/storj/internal/dbutil/pgutil/pgtest"
"storj.io/storj/storage"
"storj.io/storj/storage/testsuite"
)
const (
// this connstring is expected to work under the storj-test docker-compose instance
defaultPostgresConn = "postgres://storj:storj-pass@test-postgres/teststorj?sslmode=disable"
)
var (
testPostgres = flag.String("postgres-test-db", os.Getenv("STORJ_POSTGRES_TEST"), "PostgreSQL test database connection string")
)
func newTestPostgres(t testing.TB) (store *Client, cleanup func()) {
if *testPostgres == "" {
t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", defaultPostgresConn)
if *pgtest.ConnStr == "" {
t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", pgtest.DefaultConnStr)
}
pgdb, err := New(*testPostgres)
pgdb, err := New(*pgtest.ConnStr)
if err != nil {
t.Fatalf("init: %v", err)
}