From ac18432dc55bfb357b3e546e004d08c063db5714 Mon Sep 17 00:00:00 2001 From: Egon Elbre Date: Mon, 22 Apr 2019 16:45:53 +0300 Subject: [PATCH] Public Jenkins (#1779) * initial test * add parenthesis * remove pipeline * add few todos * use docker image for environment * use pipeline * fix * add missing steps * invoke with bash * disable protoc * try using golang image * try as root * Disable install-awscli.sh temporarily * Debugging * debugging part 2 * Set absolute path for debugging * Remove absolute path * Dont run as root * Install unzip * Dont forget to apt-get update * Put into folder that is in PATH * disable IPv6 Test * add verbose info and check protobuf * make integration non-parallel * remove -v and make checkout part of build * make a single block for linting * fix echo * update * try using things directly * try add xunit output * fix name * don't print empty lines * skip testsuites without any tests * remove coverage, because it's not showing the right thing * try using dockerfile * fix deb source * fix typos * setup postgres * use the right flag * try using postgresdb * expose different port * remove port mapping * start postgres * export * use env block * try using different host for integration tests * eat standard ports * try building images and binaries * remove if statement * add steps * do before verification * add go get goversioninfo * make separate jenkinsfile * add check * don't add empty packages * disable logging to reduce output size * add timeout * add comment about mfridman * Revert Absolute Path * Add aws to PATH * PATH Changes * Docker Env Fixes * PATH Simplification * Debugging the PATH * Debug Logs * Debugging * Update PATH Handling * Rename * revert changes to Jenkinsfile --- Dockerfile.jenkins | 25 ++ Jenkinsfile.public | 73 ++++++ scripts/test-sim.sh | 18 +- scripts/xunit.go | 272 +++++++++++++++++++++ storage/postgreskv/alternateclient_test.go | 8 +- storage/postgreskv/client_test.go | 7 +- 6 files changed, 386 insertions(+), 17 deletions(-) create mode 100644 Dockerfile.jenkins create mode 100644 Jenkinsfile.public create mode 100644 scripts/xunit.go diff --git a/Dockerfile.jenkins b/Dockerfile.jenkins new file mode 100644 index 000000000..7ad6a3008 --- /dev/null +++ b/Dockerfile.jenkins @@ -0,0 +1,25 @@ +FROM golang:1.12 + +RUN apt-get update; +RUN apt-get install -y -qq postgresql-9.6 unzip; + +RUN rm /etc/postgresql/9.6/main/pg_hba.conf; \ + echo 'local all all trust' >> /etc/postgresql/9.6/main/pg_hba.conf; \ + echo 'host all all 127.0.0.1/8 trust' >> /etc/postgresql/9.6/main/pg_hba.conf; \ + echo 'host all all ::1/128 trust' >> /etc/postgresql/9.6/main/pg_hba.conf; \ + echo 'host all all ::0/0 trust' >> /etc/postgresql/9.6/main/pg_hba.conf; + +COPY ./scripts/install-awscli.sh /tmp/install-awscli.sh +RUN bash /tmp/install-awscli.sh +ENV PATH "$PATH:/root/bin" + +RUN curl -L https://github.com/google/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip -o /tmp/protoc.zip +RUN unzip /tmp/protoc.zip -d "$HOME"/protoc + +RUN curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b ${GOPATH}/bin v1.16.0 +RUN go get github.com/ckaznocha/protoc-gen-lint +RUN go get github.com/nilslice/protolock/cmd/protolock +RUN go get github.com/mfridman/tparse +RUN go get github.com/josephspurrier/goversioninfo + +RUN go version \ No newline at end of file diff --git a/Jenkinsfile.public b/Jenkinsfile.public new file mode 100644 index 000000000..bfff672aa --- /dev/null +++ b/Jenkinsfile.public @@ -0,0 +1,73 @@ +pipeline { + agent { + dockerfile { + filename 'Dockerfile.jenkins' + args '-u root:root -v "/tmp/gomod":/go/pkg/mod' + } + } + stages { + stage('Build') { + steps { + checkout scm + sh 'go mod download' + + sh 'go install -v -race ./...' + sh 'make install-sim' + + sh 'service postgresql start' + } + } + + stage('Verification') { + parallel { + stage('Lint') { + steps { + sh 'go run ./scripts/check-copyright.go' + sh 'go run ./scripts/check-imports.go' + sh 'go run ./scripts/protobuf.go --protoc=$HOME/protoc/bin/protoc lint' + sh 'protolock status' + sh 'bash ./scripts/check-dbx-version.sh' + sh 'golangci-lint -j=4 run' + // TODO: check for go mod tidy + // TODO: check for directory tidy + } + } + + stage('Tests') { + environment { + STORJ_POSTGRES_TEST = 'postgres://postgres@localhost/teststorj?sslmode=disable' + } + steps { + sh 'psql -U postgres -c \'create database teststorj;\'' + sh 'go run scripts/use-ports.go -from 1024 -to 10000 &' + sh 'go test -vet=off -timeout 9m -json -race ./... | go run ./scripts/xunit.go -out tests.xml' + } + + post { + always { + junit 'tests.xml' + } + } + } + + stage('Integration') { + environment { + // use different hostname to avoid port conflicts + STORJ_NETWORK_HOST4 = '127.0.0.2' + STORJ_NETWORK_HOST6 = '127.0.0.2' + } + + steps { + sh 'make test-sim' + } + } + } + } + } + + post { + always { + deleteDir() + } + } +} \ No newline at end of file diff --git a/scripts/test-sim.sh b/scripts/test-sim.sh index cb975746d..e6eda3078 100755 --- a/scripts/test-sim.sh +++ b/scripts/test-sim.sh @@ -15,18 +15,20 @@ trap cleanup EXIT export STORJ_NETWORK_DIR=$TMP +STORJ_NETWORK_HOST4=${STORJ_NETWORK_HOST4:-127.0.0.1} + # setup the network -storj-sim -x network setup +storj-sim -x --host $STORJ_NETWORK_HOST4 network setup # run aws-cli tests -storj-sim -x network test bash "$SCRIPTDIR"/test-sim-aws.sh -storj-sim -x network test bash "$SCRIPTDIR"/test-uplink.sh -storj-sim -x network destroy +storj-sim -x --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/test-sim-aws.sh +storj-sim -x --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/test-uplink.sh +storj-sim -x --host $STORJ_NETWORK_HOST4 network destroy # setup the network with ipv6 -storj-sim -x --host "::1" network setup +#storj-sim -x --host "::1" network setup # aws-cli doesn't support gateway with ipv6 address, so change it to use localhost -find "$STORJ_NETWORK_DIR"/gateway -type f -name config.yaml -exec sed -i 's/server.address: "\[::1\]/server.address: "127.0.0.1/' '{}' + +#find "$STORJ_NETWORK_DIR"/gateway -type f -name config.yaml -exec sed -i 's/server.address: "\[::1\]/server.address: "127.0.0.1/' '{}' + # run aws-cli tests using ipv6 -storj-sim -x --host "::1" network test bash "$SCRIPTDIR"/test-sim-aws.sh -storj-sim -x network destroy +#storj-sim -x --host "::1" network test bash "$SCRIPTDIR"/test-sim-aws.sh +#storj-sim -x network destroy diff --git a/scripts/xunit.go b/scripts/xunit.go new file mode 100644 index 000000000..caecd8c80 --- /dev/null +++ b/scripts/xunit.go @@ -0,0 +1,272 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +// +build ignore + +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "encoding/xml" + "flag" + "fmt" + "io" + "os" + "strconv" + "strings" + + "github.com/mfridman/tparse/parse" +) + +var xunit = flag.String("out", "", "xunit output file") + +func main() { + flag.Parse() + + if *xunit == "" { + fmt.Fprintf(os.Stderr, "xunit file not specified") + os.Exit(1) + } + + var buffer bytes.Buffer + stdin := io.TeeReader(os.Stdin, &buffer) + + pkgs, err := ProcessWithEcho(stdin) + switch err { + case nil: // do nothing + + case parse.ErrNotParseable: + fmt.Fprintf(os.Stderr, "tparse error: no parseable events: call go test with -json flag\n\n") + fmt.Fprintf(os.Stdout, "\n\n\n\n=== RAW OUTPUT ===\n\n\n\n") + parse.ReplayOutput(os.Stderr, &buffer) + os.Exit(1) + + case parse.ErrRaceDetected: + fmt.Fprintf(os.Stderr, "tparse error: %v\n\n", err) + fmt.Fprintf(os.Stdout, "\n\n\n\n=== RAW OUTPUT ===\n\n\n\n") + parse.ReplayRaceOutput(os.Stderr, &buffer) + os.Exit(1) + + default: + fmt.Fprintf(os.Stderr, "tparse error: %v\n\n", err) + fmt.Fprintf(os.Stdout, "\n\n\n\n=== RAW OUTPUT ===\n\n\n\n") + parse.ReplayOutput(os.Stderr, &buffer) + os.Exit(1) + } + + defer os.Exit(pkgs.ExitCode()) + + output, err := os.Create(*xunit) + if err != nil { + fmt.Fprintf(os.Stderr, "create error: %v\n\n", err) + return + } + defer func() { + if err := output.Close(); err != nil { + fmt.Fprintf(os.Stderr, "close error: %v\n\n", err) + } + }() + + _, _ = output.Write([]byte(xml.Header)) + + encoder := xml.NewEncoder(output) + encoder.Indent("", "\t") + defer encoder.Flush() + + encoder.EncodeToken(xml.StartElement{Name: xml.Name{Local: "testsuites"}, Attr: nil}) + defer encoder.EncodeToken(xml.EndElement{Name: xml.Name{Local: "testsuites"}}) + + for _, pkg := range pkgs { + failed := pkg.TestsByAction(parse.ActionFail) + skipped := pkg.TestsByAction(parse.ActionSkip) + passed := pkg.TestsByAction(parse.ActionPass) + + skipped = withoutEmptyName(skipped) + + all := []*parse.Test{} + all = append(all, failed...) + all = append(all, skipped...) + all = append(all, passed...) + + if pkg.NoTests || len(all) == 0 { + continue + } + + func() { + encoder.EncodeToken(xml.StartElement{ + Name: xml.Name{Local: "testsuite"}, + Attr: []xml.Attr{ + {xml.Name{Local: "name"}, pkg.Summary.Package}, + {xml.Name{Local: "time"}, fmt.Sprintf("%.2f", pkg.Summary.Elapsed)}, + + {xml.Name{Local: "tests"}, strconv.Itoa(len(all))}, + {xml.Name{Local: "failures"}, strconv.Itoa(len(failed))}, + {xml.Name{Local: "skips"}, strconv.Itoa(len(skipped))}, + }, + }) + defer encoder.EncodeToken(xml.EndElement{Name: xml.Name{Local: "testsuite"}}) + + for _, t := range all { + t.SortEvents() + func() { + encoder.EncodeToken(xml.StartElement{ + Name: xml.Name{Local: "testcase"}, + Attr: []xml.Attr{ + {xml.Name{Local: "classname"}, t.Package}, + {xml.Name{Local: "name"}, t.Name}, + {xml.Name{Local: "time"}, fmt.Sprintf("%.2f", t.Elapsed())}, + }, + }) + defer encoder.EncodeToken(xml.EndElement{Name: xml.Name{Local: "testcase"}}) + + encoder.EncodeToken(xml.StartElement{xml.Name{Local: "system-out"}, nil}) + encoder.EncodeToken(xml.CharData(fullOutput(t))) + encoder.EncodeToken(xml.EndElement{xml.Name{Local: "system-out"}}) + + switch t.Status() { + case parse.ActionSkip: + encoder.EncodeToken(xml.StartElement{ + Name: xml.Name{Local: "skipped"}, + Attr: []xml.Attr{ + {xml.Name{Local: "message"}, t.Stack()}, + }, + }) + encoder.EncodeToken(xml.EndElement{Name: xml.Name{Local: "skipped"}}) + + case parse.ActionFail: + encoder.EncodeToken(xml.StartElement{Name: xml.Name{Local: "failure"}, Attr: nil}) + encoder.EncodeToken(xml.CharData(t.Stack())) + encoder.EncodeToken(xml.EndElement{Name: xml.Name{Local: "failure"}}) + } + }() + } + }() + } +} + +func fullOutput(t *parse.Test) string { + var out strings.Builder + for _, event := range t.Events { + out.WriteString(event.Output) + } + return out.String() +} + +func withoutEmptyName(tests []*parse.Test) []*parse.Test { + out := tests[:0] + for _, test := range tests { + if test.Name != "" { + out = append(out, test) + } + } + return out +} + +// Code based on: https://github.com/mfridman/tparse/blob/master/parse/process.go#L27 +func ProcessWithEcho(r io.Reader) (parse.Packages, error) { + pkgs := parse.Packages{} + + var hasRace bool + + var scan bool + var badLines int + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + // Scan up-to 50 lines for a parseable event, if we get one, expect + // no errors to follow until EOF. + event, err := parse.NewEvent(scanner.Bytes()) + if err != nil { + badLines++ + if scan || badLines > 50 { + switch err.(type) { + case *json.SyntaxError: + return nil, parse.ErrNotParseable + default: + return nil, err + } + } + continue + } + scan = true + + pkg, ok := pkgs[event.Package] + if !ok { + pkg = parse.NewPackage() + pkgs[event.Package] = pkg + } + + if event.IsPanic() { + pkg.HasPanic = true + pkg.Summary.Action = parse.ActionFail + pkg.Summary.Package = event.Package + pkg.Summary.Test = event.Test + } + // Short circuit output when panic is detected. + if pkg.HasPanic { + pkg.PanicEvents = append(pkg.PanicEvents, event) + continue + } + + if event.IsRace() { + hasRace = true + } + + if event.IsCached() { + pkg.Cached = true + } + + if event.NoTestFiles() { + pkg.NoTestFiles = true + // Manually mark [no test files] as "pass", because the go test tool reports the + // package Summary action as "skip". + pkg.Summary.Package = event.Package + pkg.Summary.Action = parse.ActionPass + } + if event.NoTestsWarn() { + // One or more tests within the package contains no tests. + pkg.NoTestSlice = append(pkg.NoTestSlice, event) + } + + if event.NoTestsToRun() { + // Only pkgs marked as "pass" will contain a summary line appended with [no tests to run]. + // This indicates one or more tests is marked as having no tests to run. + pkg.NoTests = true + pkg.Summary.Package = event.Package + pkg.Summary.Action = parse.ActionPass + } + + if event.LastLine() { + pkg.Summary = event + continue + } + + cover, ok := event.Cover() + if ok { + pkg.Cover = true + pkg.Coverage = cover + } + + if line := strings.TrimSpace(event.Output); line != "" { + fmt.Fprintln(os.Stdout, line) + } + + if !event.Discard() { + pkg.AddEvent(event) + } + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("bufio scanner error: %v", err) + } + if !scan { + return nil, parse.ErrNotParseable + } + if hasRace { + return nil, parse.ErrRaceDetected + } + + return pkgs, nil +} diff --git a/storage/postgreskv/alternateclient_test.go b/storage/postgreskv/alternateclient_test.go index 90de457a5..ced866119 100644 --- a/storage/postgreskv/alternateclient_test.go +++ b/storage/postgreskv/alternateclient_test.go @@ -7,10 +7,7 @@ import ( "flag" "testing" - "go.uber.org/zap/zaptest" - "storj.io/storj/storage" - "storj.io/storj/storage/storelogger" "storj.io/storj/storage/testsuite" ) @@ -42,8 +39,9 @@ func TestSuiteAlt(t *testing.T) { store, cleanup := newTestAlternatePostgres(t) defer cleanup() - zap := zaptest.NewLogger(t) - testsuite.RunTests(t, storelogger.New(zap, store)) + // zap := zaptest.NewLogger(t) + // loggedStore := storelogger.New(zap, store) + testsuite.RunTests(t, store) } func BenchmarkSuiteAlt(b *testing.B) { diff --git a/storage/postgreskv/client_test.go b/storage/postgreskv/client_test.go index cd03ec8f7..fe9479bd5 100644 --- a/storage/postgreskv/client_test.go +++ b/storage/postgreskv/client_test.go @@ -11,10 +11,8 @@ import ( "github.com/lib/pq" "github.com/zeebo/errs" - "go.uber.org/zap/zaptest" "storj.io/storj/storage" - "storj.io/storj/storage/storelogger" "storj.io/storj/storage/testsuite" ) @@ -48,8 +46,9 @@ func TestSuite(t *testing.T) { store, cleanup := newTestPostgres(t) defer cleanup() - zap := zaptest.NewLogger(t) - testsuite.RunTests(t, storelogger.New(zap, store)) + // zap := zaptest.NewLogger(t) + // loggedStore := storelogger.New(zap, store) + testsuite.RunTests(t, store) } func BenchmarkSuite(b *testing.B) {