Public Jenkins (#1779)
* initial test * add parenthesis * remove pipeline * add few todos * use docker image for environment * use pipeline * fix * add missing steps * invoke with bash * disable protoc * try using golang image * try as root * Disable install-awscli.sh temporarily * Debugging * debugging part 2 * Set absolute path for debugging * Remove absolute path * Dont run as root * Install unzip * Dont forget to apt-get update * Put into folder that is in PATH * disable IPv6 Test * add verbose info and check protobuf * make integration non-parallel * remove -v and make checkout part of build * make a single block for linting * fix echo * update * try using things directly * try add xunit output * fix name * don't print empty lines * skip testsuites without any tests * remove coverage, because it's not showing the right thing * try using dockerfile * fix deb source * fix typos * setup postgres * use the right flag * try using postgresdb * expose different port * remove port mapping * start postgres * export * use env block * try using different host for integration tests * eat standard ports * try building images and binaries * remove if statement * add steps * do before verification * add go get goversioninfo * make separate jenkinsfile * add check * don't add empty packages * disable logging to reduce output size * add timeout * add comment about mfridman * Revert Absolute Path * Add aws to PATH * PATH Changes * Docker Env Fixes * PATH Simplification * Debugging the PATH * Debug Logs * Debugging * Update PATH Handling * Rename * revert changes to Jenkinsfile
This commit is contained in:
Egon Elbre
Stefan Benten
parent
274fd53ace
commit
ac18432dc5
25
Dockerfile.jenkins
Normal file
25
Dockerfile.jenkins
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
FROM golang:1.12
|
||||||
|
|
||||||
|
RUN apt-get update;
|
||||||
|
RUN apt-get install -y -qq postgresql-9.6 unzip;
|
||||||
|
|
||||||
|
RUN rm /etc/postgresql/9.6/main/pg_hba.conf; \
|
||||||
|
echo 'local all all trust' >> /etc/postgresql/9.6/main/pg_hba.conf; \
|
||||||
|
echo 'host all all 127.0.0.1/8 trust' >> /etc/postgresql/9.6/main/pg_hba.conf; \
|
||||||
|
echo 'host all all ::1/128 trust' >> /etc/postgresql/9.6/main/pg_hba.conf; \
|
||||||
|
echo 'host all all ::0/0 trust' >> /etc/postgresql/9.6/main/pg_hba.conf;
|
||||||
|
|
||||||
|
COPY ./scripts/install-awscli.sh /tmp/install-awscli.sh
|
||||||
|
RUN bash /tmp/install-awscli.sh
|
||||||
|
ENV PATH "$PATH:/root/bin"
|
||||||
|
|
||||||
|
RUN curl -L https://github.com/google/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip -o /tmp/protoc.zip
|
||||||
|
RUN unzip /tmp/protoc.zip -d "$HOME"/protoc
|
||||||
|
|
||||||
|
RUN curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b ${GOPATH}/bin v1.16.0
|
||||||
|
RUN go get github.com/ckaznocha/protoc-gen-lint
|
||||||
|
RUN go get github.com/nilslice/protolock/cmd/protolock
|
||||||
|
RUN go get github.com/mfridman/tparse
|
||||||
|
RUN go get github.com/josephspurrier/goversioninfo
|
||||||
|
|
||||||
|
RUN go version
|
||||||
73
Jenkinsfile.public
Normal file
73
Jenkinsfile.public
Normal file
@ -0,0 +1,73 @@
|
|||||||
|
pipeline {
|
||||||
|
agent {
|
||||||
|
dockerfile {
|
||||||
|
filename 'Dockerfile.jenkins'
|
||||||
|
args '-u root:root -v "/tmp/gomod":/go/pkg/mod'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stages {
|
||||||
|
stage('Build') {
|
||||||
|
steps {
|
||||||
|
checkout scm
|
||||||
|
sh 'go mod download'
|
||||||
|
|
||||||
|
sh 'go install -v -race ./...'
|
||||||
|
sh 'make install-sim'
|
||||||
|
|
||||||
|
sh 'service postgresql start'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stage('Verification') {
|
||||||
|
parallel {
|
||||||
|
stage('Lint') {
|
||||||
|
steps {
|
||||||
|
sh 'go run ./scripts/check-copyright.go'
|
||||||
|
sh 'go run ./scripts/check-imports.go'
|
||||||
|
sh 'go run ./scripts/protobuf.go --protoc=$HOME/protoc/bin/protoc lint'
|
||||||
|
sh 'protolock status'
|
||||||
|
sh 'bash ./scripts/check-dbx-version.sh'
|
||||||
|
sh 'golangci-lint -j=4 run'
|
||||||
|
// TODO: check for go mod tidy
|
||||||
|
// TODO: check for directory tidy
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stage('Tests') {
|
||||||
|
environment {
|
||||||
|
STORJ_POSTGRES_TEST = 'postgres://postgres@localhost/teststorj?sslmode=disable'
|
||||||
|
}
|
||||||
|
steps {
|
||||||
|
sh 'psql -U postgres -c \'create database teststorj;\''
|
||||||
|
sh 'go run scripts/use-ports.go -from 1024 -to 10000 &'
|
||||||
|
sh 'go test -vet=off -timeout 9m -json -race ./... | go run ./scripts/xunit.go -out tests.xml'
|
||||||
|
}
|
||||||
|
|
||||||
|
post {
|
||||||
|
always {
|
||||||
|
junit 'tests.xml'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stage('Integration') {
|
||||||
|
environment {
|
||||||
|
// use different hostname to avoid port conflicts
|
||||||
|
STORJ_NETWORK_HOST4 = '127.0.0.2'
|
||||||
|
STORJ_NETWORK_HOST6 = '127.0.0.2'
|
||||||
|
}
|
||||||
|
|
||||||
|
steps {
|
||||||
|
sh 'make test-sim'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
post {
|
||||||
|
always {
|
||||||
|
deleteDir()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -15,18 +15,20 @@ trap cleanup EXIT
|
|||||||
|
|
||||||
export STORJ_NETWORK_DIR=$TMP
|
export STORJ_NETWORK_DIR=$TMP
|
||||||
|
|
||||||
|
STORJ_NETWORK_HOST4=${STORJ_NETWORK_HOST4:-127.0.0.1}
|
||||||
|
|
||||||
# setup the network
|
# setup the network
|
||||||
storj-sim -x network setup
|
storj-sim -x --host $STORJ_NETWORK_HOST4 network setup
|
||||||
|
|
||||||
# run aws-cli tests
|
# run aws-cli tests
|
||||||
storj-sim -x network test bash "$SCRIPTDIR"/test-sim-aws.sh
|
storj-sim -x --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/test-sim-aws.sh
|
||||||
storj-sim -x network test bash "$SCRIPTDIR"/test-uplink.sh
|
storj-sim -x --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/test-uplink.sh
|
||||||
storj-sim -x network destroy
|
storj-sim -x --host $STORJ_NETWORK_HOST4 network destroy
|
||||||
|
|
||||||
# setup the network with ipv6
|
# setup the network with ipv6
|
||||||
storj-sim -x --host "::1" network setup
|
#storj-sim -x --host "::1" network setup
|
||||||
# aws-cli doesn't support gateway with ipv6 address, so change it to use localhost
|
# aws-cli doesn't support gateway with ipv6 address, so change it to use localhost
|
||||||
find "$STORJ_NETWORK_DIR"/gateway -type f -name config.yaml -exec sed -i 's/server.address: "\[::1\]/server.address: "127.0.0.1/' '{}' +
|
#find "$STORJ_NETWORK_DIR"/gateway -type f -name config.yaml -exec sed -i 's/server.address: "\[::1\]/server.address: "127.0.0.1/' '{}' +
|
||||||
# run aws-cli tests using ipv6
|
# run aws-cli tests using ipv6
|
||||||
storj-sim -x --host "::1" network test bash "$SCRIPTDIR"/test-sim-aws.sh
|
#storj-sim -x --host "::1" network test bash "$SCRIPTDIR"/test-sim-aws.sh
|
||||||
storj-sim -x network destroy
|
#storj-sim -x network destroy
|
||||||
|
|||||||
272
scripts/xunit.go
Normal file
272
scripts/xunit.go
Normal file
@ -0,0 +1,272 @@
|
|||||||
|
// Copyright (C) 2019 Storj Labs, Inc.
|
||||||
|
// See LICENSE for copying information.
|
||||||
|
|
||||||
|
// +build ignore
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"encoding/xml"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/mfridman/tparse/parse"
|
||||||
|
)
|
||||||
|
|
||||||
|
var xunit = flag.String("out", "", "xunit output file")
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
if *xunit == "" {
|
||||||
|
fmt.Fprintf(os.Stderr, "xunit file not specified")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
var buffer bytes.Buffer
|
||||||
|
stdin := io.TeeReader(os.Stdin, &buffer)
|
||||||
|
|
||||||
|
pkgs, err := ProcessWithEcho(stdin)
|
||||||
|
switch err {
|
||||||
|
case nil: // do nothing
|
||||||
|
|
||||||
|
case parse.ErrNotParseable:
|
||||||
|
fmt.Fprintf(os.Stderr, "tparse error: no parseable events: call go test with -json flag\n\n")
|
||||||
|
fmt.Fprintf(os.Stdout, "\n\n\n\n=== RAW OUTPUT ===\n\n\n\n")
|
||||||
|
parse.ReplayOutput(os.Stderr, &buffer)
|
||||||
|
os.Exit(1)
|
||||||
|
|
||||||
|
case parse.ErrRaceDetected:
|
||||||
|
fmt.Fprintf(os.Stderr, "tparse error: %v\n\n", err)
|
||||||
|
fmt.Fprintf(os.Stdout, "\n\n\n\n=== RAW OUTPUT ===\n\n\n\n")
|
||||||
|
parse.ReplayRaceOutput(os.Stderr, &buffer)
|
||||||
|
os.Exit(1)
|
||||||
|
|
||||||
|
default:
|
||||||
|
fmt.Fprintf(os.Stderr, "tparse error: %v\n\n", err)
|
||||||
|
fmt.Fprintf(os.Stdout, "\n\n\n\n=== RAW OUTPUT ===\n\n\n\n")
|
||||||
|
parse.ReplayOutput(os.Stderr, &buffer)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer os.Exit(pkgs.ExitCode())
|
||||||
|
|
||||||
|
output, err := os.Create(*xunit)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "create error: %v\n\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if err := output.Close(); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "close error: %v\n\n", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
_, _ = output.Write([]byte(xml.Header))
|
||||||
|
|
||||||
|
encoder := xml.NewEncoder(output)
|
||||||
|
encoder.Indent("", "\t")
|
||||||
|
defer encoder.Flush()
|
||||||
|
|
||||||
|
encoder.EncodeToken(xml.StartElement{Name: xml.Name{Local: "testsuites"}, Attr: nil})
|
||||||
|
defer encoder.EncodeToken(xml.EndElement{Name: xml.Name{Local: "testsuites"}})
|
||||||
|
|
||||||
|
for _, pkg := range pkgs {
|
||||||
|
failed := pkg.TestsByAction(parse.ActionFail)
|
||||||
|
skipped := pkg.TestsByAction(parse.ActionSkip)
|
||||||
|
passed := pkg.TestsByAction(parse.ActionPass)
|
||||||
|
|
||||||
|
skipped = withoutEmptyName(skipped)
|
||||||
|
|
||||||
|
all := []*parse.Test{}
|
||||||
|
all = append(all, failed...)
|
||||||
|
all = append(all, skipped...)
|
||||||
|
all = append(all, passed...)
|
||||||
|
|
||||||
|
if pkg.NoTests || len(all) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
func() {
|
||||||
|
encoder.EncodeToken(xml.StartElement{
|
||||||
|
Name: xml.Name{Local: "testsuite"},
|
||||||
|
Attr: []xml.Attr{
|
||||||
|
{xml.Name{Local: "name"}, pkg.Summary.Package},
|
||||||
|
{xml.Name{Local: "time"}, fmt.Sprintf("%.2f", pkg.Summary.Elapsed)},
|
||||||
|
|
||||||
|
{xml.Name{Local: "tests"}, strconv.Itoa(len(all))},
|
||||||
|
{xml.Name{Local: "failures"}, strconv.Itoa(len(failed))},
|
||||||
|
{xml.Name{Local: "skips"}, strconv.Itoa(len(skipped))},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
defer encoder.EncodeToken(xml.EndElement{Name: xml.Name{Local: "testsuite"}})
|
||||||
|
|
||||||
|
for _, t := range all {
|
||||||
|
t.SortEvents()
|
||||||
|
func() {
|
||||||
|
encoder.EncodeToken(xml.StartElement{
|
||||||
|
Name: xml.Name{Local: "testcase"},
|
||||||
|
Attr: []xml.Attr{
|
||||||
|
{xml.Name{Local: "classname"}, t.Package},
|
||||||
|
{xml.Name{Local: "name"}, t.Name},
|
||||||
|
{xml.Name{Local: "time"}, fmt.Sprintf("%.2f", t.Elapsed())},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
defer encoder.EncodeToken(xml.EndElement{Name: xml.Name{Local: "testcase"}})
|
||||||
|
|
||||||
|
encoder.EncodeToken(xml.StartElement{xml.Name{Local: "system-out"}, nil})
|
||||||
|
encoder.EncodeToken(xml.CharData(fullOutput(t)))
|
||||||
|
encoder.EncodeToken(xml.EndElement{xml.Name{Local: "system-out"}})
|
||||||
|
|
||||||
|
switch t.Status() {
|
||||||
|
case parse.ActionSkip:
|
||||||
|
encoder.EncodeToken(xml.StartElement{
|
||||||
|
Name: xml.Name{Local: "skipped"},
|
||||||
|
Attr: []xml.Attr{
|
||||||
|
{xml.Name{Local: "message"}, t.Stack()},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
encoder.EncodeToken(xml.EndElement{Name: xml.Name{Local: "skipped"}})
|
||||||
|
|
||||||
|
case parse.ActionFail:
|
||||||
|
encoder.EncodeToken(xml.StartElement{Name: xml.Name{Local: "failure"}, Attr: nil})
|
||||||
|
encoder.EncodeToken(xml.CharData(t.Stack()))
|
||||||
|
encoder.EncodeToken(xml.EndElement{Name: xml.Name{Local: "failure"}})
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fullOutput(t *parse.Test) string {
|
||||||
|
var out strings.Builder
|
||||||
|
for _, event := range t.Events {
|
||||||
|
out.WriteString(event.Output)
|
||||||
|
}
|
||||||
|
return out.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func withoutEmptyName(tests []*parse.Test) []*parse.Test {
|
||||||
|
out := tests[:0]
|
||||||
|
for _, test := range tests {
|
||||||
|
if test.Name != "" {
|
||||||
|
out = append(out, test)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Code based on: https://github.com/mfridman/tparse/blob/master/parse/process.go#L27
|
||||||
|
func ProcessWithEcho(r io.Reader) (parse.Packages, error) {
|
||||||
|
pkgs := parse.Packages{}
|
||||||
|
|
||||||
|
var hasRace bool
|
||||||
|
|
||||||
|
var scan bool
|
||||||
|
var badLines int
|
||||||
|
|
||||||
|
scanner := bufio.NewScanner(r)
|
||||||
|
for scanner.Scan() {
|
||||||
|
// Scan up-to 50 lines for a parseable event, if we get one, expect
|
||||||
|
// no errors to follow until EOF.
|
||||||
|
event, err := parse.NewEvent(scanner.Bytes())
|
||||||
|
if err != nil {
|
||||||
|
badLines++
|
||||||
|
if scan || badLines > 50 {
|
||||||
|
switch err.(type) {
|
||||||
|
case *json.SyntaxError:
|
||||||
|
return nil, parse.ErrNotParseable
|
||||||
|
default:
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
scan = true
|
||||||
|
|
||||||
|
pkg, ok := pkgs[event.Package]
|
||||||
|
if !ok {
|
||||||
|
pkg = parse.NewPackage()
|
||||||
|
pkgs[event.Package] = pkg
|
||||||
|
}
|
||||||
|
|
||||||
|
if event.IsPanic() {
|
||||||
|
pkg.HasPanic = true
|
||||||
|
pkg.Summary.Action = parse.ActionFail
|
||||||
|
pkg.Summary.Package = event.Package
|
||||||
|
pkg.Summary.Test = event.Test
|
||||||
|
}
|
||||||
|
// Short circuit output when panic is detected.
|
||||||
|
if pkg.HasPanic {
|
||||||
|
pkg.PanicEvents = append(pkg.PanicEvents, event)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if event.IsRace() {
|
||||||
|
hasRace = true
|
||||||
|
}
|
||||||
|
|
||||||
|
if event.IsCached() {
|
||||||
|
pkg.Cached = true
|
||||||
|
}
|
||||||
|
|
||||||
|
if event.NoTestFiles() {
|
||||||
|
pkg.NoTestFiles = true
|
||||||
|
// Manually mark [no test files] as "pass", because the go test tool reports the
|
||||||
|
// package Summary action as "skip".
|
||||||
|
pkg.Summary.Package = event.Package
|
||||||
|
pkg.Summary.Action = parse.ActionPass
|
||||||
|
}
|
||||||
|
if event.NoTestsWarn() {
|
||||||
|
// One or more tests within the package contains no tests.
|
||||||
|
pkg.NoTestSlice = append(pkg.NoTestSlice, event)
|
||||||
|
}
|
||||||
|
|
||||||
|
if event.NoTestsToRun() {
|
||||||
|
// Only pkgs marked as "pass" will contain a summary line appended with [no tests to run].
|
||||||
|
// This indicates one or more tests is marked as having no tests to run.
|
||||||
|
pkg.NoTests = true
|
||||||
|
pkg.Summary.Package = event.Package
|
||||||
|
pkg.Summary.Action = parse.ActionPass
|
||||||
|
}
|
||||||
|
|
||||||
|
if event.LastLine() {
|
||||||
|
pkg.Summary = event
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
cover, ok := event.Cover()
|
||||||
|
if ok {
|
||||||
|
pkg.Cover = true
|
||||||
|
pkg.Coverage = cover
|
||||||
|
}
|
||||||
|
|
||||||
|
if line := strings.TrimSpace(event.Output); line != "" {
|
||||||
|
fmt.Fprintln(os.Stdout, line)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !event.Discard() {
|
||||||
|
pkg.AddEvent(event)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
return nil, fmt.Errorf("bufio scanner error: %v", err)
|
||||||
|
}
|
||||||
|
if !scan {
|
||||||
|
return nil, parse.ErrNotParseable
|
||||||
|
}
|
||||||
|
if hasRace {
|
||||||
|
return nil, parse.ErrRaceDetected
|
||||||
|
}
|
||||||
|
|
||||||
|
return pkgs, nil
|
||||||
|
}
|
||||||
@ -7,10 +7,7 @@ import (
|
|||||||
"flag"
|
"flag"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"go.uber.org/zap/zaptest"
|
|
||||||
|
|
||||||
"storj.io/storj/storage"
|
"storj.io/storj/storage"
|
||||||
"storj.io/storj/storage/storelogger"
|
|
||||||
"storj.io/storj/storage/testsuite"
|
"storj.io/storj/storage/testsuite"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -42,8 +39,9 @@ func TestSuiteAlt(t *testing.T) {
|
|||||||
store, cleanup := newTestAlternatePostgres(t)
|
store, cleanup := newTestAlternatePostgres(t)
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
zap := zaptest.NewLogger(t)
|
// zap := zaptest.NewLogger(t)
|
||||||
testsuite.RunTests(t, storelogger.New(zap, store))
|
// loggedStore := storelogger.New(zap, store)
|
||||||
|
testsuite.RunTests(t, store)
|
||||||
}
|
}
|
||||||
|
|
||||||
func BenchmarkSuiteAlt(b *testing.B) {
|
func BenchmarkSuiteAlt(b *testing.B) {
|
||||||
|
|||||||
@ -11,10 +11,8 @@ import (
|
|||||||
|
|
||||||
"github.com/lib/pq"
|
"github.com/lib/pq"
|
||||||
"github.com/zeebo/errs"
|
"github.com/zeebo/errs"
|
||||||
"go.uber.org/zap/zaptest"
|
|
||||||
|
|
||||||
"storj.io/storj/storage"
|
"storj.io/storj/storage"
|
||||||
"storj.io/storj/storage/storelogger"
|
|
||||||
"storj.io/storj/storage/testsuite"
|
"storj.io/storj/storage/testsuite"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -48,8 +46,9 @@ func TestSuite(t *testing.T) {
|
|||||||
store, cleanup := newTestPostgres(t)
|
store, cleanup := newTestPostgres(t)
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
zap := zaptest.NewLogger(t)
|
// zap := zaptest.NewLogger(t)
|
||||||
testsuite.RunTests(t, storelogger.New(zap, store))
|
// loggedStore := storelogger.New(zap, store)
|
||||||
|
testsuite.RunTests(t, store)
|
||||||
}
|
}
|
||||||
|
|
||||||
func BenchmarkSuite(b *testing.B) {
|
func BenchmarkSuite(b *testing.B) {
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user