JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Public Jenkins (#1779)

Browse Source 
* initial test

* add parenthesis

* remove pipeline

* add few todos

* use docker image for environment

* use pipeline

* fix

* add missing steps

* invoke with bash

* disable protoc

* try using golang image

* try as root

* Disable install-awscli.sh temporarily

* Debugging

* debugging part 2

* Set absolute path for debugging

* Remove absolute path

* Dont run as root

* Install unzip

* Dont forget to apt-get update

* Put into folder that is in PATH

* disable IPv6 Test

* add verbose info and check protobuf

* make integration non-parallel

* remove -v and make checkout part of build

* make a single block for linting

* fix echo

* update

* try using things directly

* try add xunit output

* fix name

* don't print empty lines

* skip testsuites without any tests

* remove coverage, because it's not showing the right thing

* try using dockerfile

* fix deb source

* fix typos

* setup postgres

* use the right flag

* try using postgresdb

* expose different port

* remove port mapping

* start postgres

* export

* use env block

* try using different host for integration tests

* eat standard ports

* try building images and binaries

* remove if statement

* add steps

* do before verification

* add go get goversioninfo

* make separate jenkinsfile

* add check

* don't add empty packages

* disable logging to reduce output size

* add timeout

* add comment about mfridman

* Revert Absolute Path

* Add aws to PATH

* PATH Changes

* Docker Env Fixes

* PATH Simplification

* Debugging the PATH

* Debug Logs

* Debugging

* Update PATH Handling

* Rename

* revert changes to Jenkinsfile
This commit is contained in:
Egon Elbre 2019-04-22 16:45:53 +03:00 committed by Stefan Benten
parent 274fd53ace
commit ac18432dc5
6 changed files with 386 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
Dockerfile.jenkins Normal file
View File

@ -0,0 +1,25 @@
FROM golang:1.12
RUN apt-get update;
RUN apt-get install -y -qq postgresql-9.6 unzip;
RUN rm /etc/postgresql/9.6/main/pg_hba.conf; \
echo 'local all all trust' >> /etc/postgresql/9.6/main/pg_hba.conf; \
echo 'host all all 127.0.0.1/8 trust' >> /etc/postgresql/9.6/main/pg_hba.conf; \
echo 'host all all ::1/128 trust' >> /etc/postgresql/9.6/main/pg_hba.conf; \
echo 'host all all ::0/0 trust' >> /etc/postgresql/9.6/main/pg_hba.conf;
COPY ./scripts/install-awscli.sh /tmp/install-awscli.sh
RUN bash /tmp/install-awscli.sh
ENV PATH "$PATH:/root/bin"
RUN curl -L https://github.com/google/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip -o /tmp/protoc.zip
RUN unzip /tmp/protoc.zip -d "$HOME"/protoc
RUN curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b ${GOPATH}/bin v1.16.0
RUN go get github.com/ckaznocha/protoc-gen-lint
RUN go get github.com/nilslice/protolock/cmd/protolock
RUN go get github.com/mfridman/tparse
RUN go get github.com/josephspurrier/goversioninfo
RUN go version

73
Jenkinsfile.public Normal file
View File

@ -0,0 +1,73 @@
pipeline {
agent {
dockerfile {
filename 'Dockerfile.jenkins'
args '-u root:root -v "/tmp/gomod":/go/pkg/mod'
}
}
stages {
stage('Build') {
steps {
checkout scm
sh 'go mod download'
sh 'go install -v -race ./...'
sh 'make install-sim'
sh 'service postgresql start'
}
}
stage('Verification') {
parallel {
stage('Lint') {
steps {
sh 'go run ./scripts/check-copyright.go'
sh 'go run ./scripts/check-imports.go'
sh 'go run ./scripts/protobuf.go --protoc=$HOME/protoc/bin/protoc lint'
sh 'protolock status'
sh 'bash ./scripts/check-dbx-version.sh'
sh 'golangci-lint -j=4 run'
// TODO: check for go mod tidy
// TODO: check for directory tidy
}
}
stage('Tests') {
environment {
STORJ_POSTGRES_TEST = 'postgres://postgres@localhost/teststorj?sslmode=disable'
}
steps {
sh 'psql -U postgres -c \'create database teststorj;\''
sh 'go run scripts/use-ports.go -from 1024 -to 10000 &'
sh 'go test -vet=off -timeout 9m -json -race ./... | go run ./scripts/xunit.go -out tests.xml'
}
post {
always {
junit 'tests.xml'
}
}
}
stage('Integration') {
environment {
// use different hostname to avoid port conflicts
STORJ_NETWORK_HOST4 = '127.0.0.2'
STORJ_NETWORK_HOST6 = '127.0.0.2'
}
steps {
sh 'make test-sim'
}
}
}
}
}
post {
always {
deleteDir()
}
}
}

18
scripts/test-sim.sh
View File

@ -15,18 +15,20 @@ trap cleanup EXIT
export STORJ_NETWORK_DIR=$TMP
STORJ_NETWORK_HOST4=${STORJ_NETWORK_HOST4:-127.0.0.1}
# setup the network
storj-sim -x network setup
storj-sim -x --host $STORJ_NETWORK_HOST4 network setup
# run aws-cli tests
storj-sim -x network test bash "$SCRIPTDIR"/test-sim-aws.sh
storj-sim -x network test bash "$SCRIPTDIR"/test-uplink.sh
storj-sim -x network destroy
storj-sim -x --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/test-sim-aws.sh
storj-sim -x --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/test-uplink.sh
storj-sim -x --host $STORJ_NETWORK_HOST4 network destroy
# setup the network with ipv6
storj-sim -x --host "::1" network setup
#storj-sim -x --host "::1" network setup
# aws-cli doesn't support gateway with ipv6 address, so change it to use localhost
find "$STORJ_NETWORK_DIR"/gateway -type f -name config.yaml -exec sed -i 's/server.address: "\[::1\]/server.address: "127.0.0.1/' '{}' +
#find "$STORJ_NETWORK_DIR"/gateway -type f -name config.yaml -exec sed -i 's/server.address: "\[::1\]/server.address: "127.0.0.1/' '{}' +
# run aws-cli tests using ipv6
storj-sim -x --host "::1" network test bash "$SCRIPTDIR"/test-sim-aws.sh
storj-sim -x network destroy
#storj-sim -x --host "::1" network test bash "$SCRIPTDIR"/test-sim-aws.sh
#storj-sim -x network destroy

272
scripts/xunit.go Normal file
View File

@ -0,0 +1,272 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
// +build ignore
package main
import (
"bufio"
"bytes"
"encoding/json"
"encoding/xml"
"flag"
"fmt"
"io"
"os"
"strconv"
"strings"
"github.com/mfridman/tparse/parse"
)
var xunit = flag.String("out", "", "xunit output file")
func main() {
flag.Parse()
if *xunit == "" {
fmt.Fprintf(os.Stderr, "xunit file not specified")
os.Exit(1)
}
var buffer bytes.Buffer
stdin := io.TeeReader(os.Stdin, &buffer)
pkgs, err := ProcessWithEcho(stdin)
switch err {
case nil: // do nothing
case parse.ErrNotParseable:
fmt.Fprintf(os.Stderr, "tparse error: no parseable events: call go test with -json flag\n\n")
fmt.Fprintf(os.Stdout, "\n\n\n\n=== RAW OUTPUT ===\n\n\n\n")
parse.ReplayOutput(os.Stderr, &buffer)
os.Exit(1)
case parse.ErrRaceDetected:
fmt.Fprintf(os.Stderr, "tparse error: %v\n\n", err)
fmt.Fprintf(os.Stdout, "\n\n\n\n=== RAW OUTPUT ===\n\n\n\n")
parse.ReplayRaceOutput(os.Stderr, &buffer)
os.Exit(1)
default:
fmt.Fprintf(os.Stderr, "tparse error: %v\n\n", err)
fmt.Fprintf(os.Stdout, "\n\n\n\n=== RAW OUTPUT ===\n\n\n\n")
parse.ReplayOutput(os.Stderr, &buffer)
os.Exit(1)
}
defer os.Exit(pkgs.ExitCode())
output, err := os.Create(*xunit)
if err != nil {
fmt.Fprintf(os.Stderr, "create error: %v\n\n", err)
return
}
defer func() {
if err := output.Close(); err != nil {
fmt.Fprintf(os.Stderr, "close error: %v\n\n", err)
}
}()
_, _ = output.Write([]byte(xml.Header))
encoder := xml.NewEncoder(output)
encoder.Indent("", "\t")
defer encoder.Flush()
encoder.EncodeToken(xml.StartElement{Name: xml.Name{Local: "testsuites"}, Attr: nil})
defer encoder.EncodeToken(xml.EndElement{Name: xml.Name{Local: "testsuites"}})
for _, pkg := range pkgs {
failed := pkg.TestsByAction(parse.ActionFail)
skipped := pkg.TestsByAction(parse.ActionSkip)
passed := pkg.TestsByAction(parse.ActionPass)
skipped = withoutEmptyName(skipped)
all := []*parse.Test{}
all = append(all, failed...)
all = append(all, skipped...)
all = append(all, passed...)
if pkg.NoTests || len(all) == 0 {
continue
}
func() {
encoder.EncodeToken(xml.StartElement{
Name: xml.Name{Local: "testsuite"},
Attr: []xml.Attr{
{xml.Name{Local: "name"}, pkg.Summary.Package},
{xml.Name{Local: "time"}, fmt.Sprintf("%.2f", pkg.Summary.Elapsed)},
{xml.Name{Local: "tests"}, strconv.Itoa(len(all))},
{xml.Name{Local: "failures"}, strconv.Itoa(len(failed))},
{xml.Name{Local: "skips"}, strconv.Itoa(len(skipped))},
},
})
defer encoder.EncodeToken(xml.EndElement{Name: xml.Name{Local: "testsuite"}})
for _, t := range all {
t.SortEvents()
func() {
encoder.EncodeToken(xml.StartElement{
Name: xml.Name{Local: "testcase"},
Attr: []xml.Attr{
{xml.Name{Local: "classname"}, t.Package},
{xml.Name{Local: "name"}, t.Name},
{xml.Name{Local: "time"}, fmt.Sprintf("%.2f", t.Elapsed())},
},
})
defer encoder.EncodeToken(xml.EndElement{Name: xml.Name{Local: "testcase"}})
encoder.EncodeToken(xml.StartElement{xml.Name{Local: "system-out"}, nil})
encoder.EncodeToken(xml.CharData(fullOutput(t)))
encoder.EncodeToken(xml.EndElement{xml.Name{Local: "system-out"}})
switch t.Status() {
case parse.ActionSkip:
encoder.EncodeToken(xml.StartElement{
Name: xml.Name{Local: "skipped"},
Attr: []xml.Attr{
{xml.Name{Local: "message"}, t.Stack()},
},
})
encoder.EncodeToken(xml.EndElement{Name: xml.Name{Local: "skipped"}})
case parse.ActionFail:
encoder.EncodeToken(xml.StartElement{Name: xml.Name{Local: "failure"}, Attr: nil})
encoder.EncodeToken(xml.CharData(t.Stack()))
encoder.EncodeToken(xml.EndElement{Name: xml.Name{Local: "failure"}})
}
}()
}
}()
}
}
func fullOutput(t *parse.Test) string {
var out strings.Builder
for _, event := range t.Events {
out.WriteString(event.Output)
}
return out.String()
}
func withoutEmptyName(tests []*parse.Test) []*parse.Test {
out := tests[:0]
for _, test := range tests {
if test.Name != "" {
out = append(out, test)
}
}
return out
}
// Code based on: https://github.com/mfridman/tparse/blob/master/parse/process.go#L27
func ProcessWithEcho(r io.Reader) (parse.Packages, error) {
pkgs := parse.Packages{}
var hasRace bool
var scan bool
var badLines int
scanner := bufio.NewScanner(r)
for scanner.Scan() {
// Scan up-to 50 lines for a parseable event, if we get one, expect
// no errors to follow until EOF.
event, err := parse.NewEvent(scanner.Bytes())
if err != nil {
badLines++
if scan || badLines > 50 {
switch err.(type) {
case *json.SyntaxError:
return nil, parse.ErrNotParseable
default:
return nil, err
}
}
continue
}
scan = true
pkg, ok := pkgs[event.Package]
if !ok {
pkg = parse.NewPackage()
pkgs[event.Package] = pkg
}
if event.IsPanic() {
pkg.HasPanic = true
pkg.Summary.Action = parse.ActionFail
pkg.Summary.Package = event.Package
pkg.Summary.Test = event.Test
}
// Short circuit output when panic is detected.
if pkg.HasPanic {
pkg.PanicEvents = append(pkg.PanicEvents, event)
continue
}
if event.IsRace() {
hasRace = true
}
if event.IsCached() {
pkg.Cached = true
}
if event.NoTestFiles() {
pkg.NoTestFiles = true
// Manually mark [no test files] as "pass", because the go test tool reports the
// package Summary action as "skip".
pkg.Summary.Package = event.Package
pkg.Summary.Action = parse.ActionPass
}
if event.NoTestsWarn() {
// One or more tests within the package contains no tests.
pkg.NoTestSlice = append(pkg.NoTestSlice, event)
}
if event.NoTestsToRun() {
// Only pkgs marked as "pass" will contain a summary line appended with [no tests to run].
// This indicates one or more tests is marked as having no tests to run.
pkg.NoTests = true
pkg.Summary.Package = event.Package
pkg.Summary.Action = parse.ActionPass
}
if event.LastLine() {
pkg.Summary = event
continue
}
cover, ok := event.Cover()
if ok {
pkg.Cover = true
pkg.Coverage = cover
}
if line := strings.TrimSpace(event.Output); line != "" {
fmt.Fprintln(os.Stdout, line)
}
if !event.Discard() {
pkg.AddEvent(event)
}
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("bufio scanner error: %v", err)
}
if !scan {
return nil, parse.ErrNotParseable
}
if hasRace {
return nil, parse.ErrRaceDetected
}
return pkgs, nil
}

8
storage/postgreskv/alternateclient_test.go
View File

@ -7,10 +7,7 @@ import (
"flag"
"testing"
"go.uber.org/zap/zaptest"
"storj.io/storj/storage"
"storj.io/storj/storage/storelogger"
"storj.io/storj/storage/testsuite"
)
@ -42,8 +39,9 @@ func TestSuiteAlt(t *testing.T) {
store, cleanup := newTestAlternatePostgres(t)
defer cleanup()
zap := zaptest.NewLogger(t)
testsuite.RunTests(t, storelogger.New(zap, store))
// zap := zaptest.NewLogger(t)
// loggedStore := storelogger.New(zap, store)
testsuite.RunTests(t, store)
}
func BenchmarkSuiteAlt(b *testing.B) {

7
storage/postgreskv/client_test.go
View File

@ -11,10 +11,8 @@ import (
"github.com/lib/pq"
"github.com/zeebo/errs"
"go.uber.org/zap/zaptest"
"storj.io/storj/storage"
"storj.io/storj/storage/storelogger"
"storj.io/storj/storage/testsuite"
)
@ -48,8 +46,9 @@ func TestSuite(t *testing.T) {
store, cleanup := newTestPostgres(t)
defer cleanup()
zap := zaptest.NewLogger(t)
testsuite.RunTests(t, storelogger.New(zap, store))
// zap := zaptest.NewLogger(t)
// loggedStore := storelogger.New(zap, store)
testsuite.RunTests(t, store)
}
func BenchmarkSuite(b *testing.B) {