jenkins: add tidiness checks (#1824)

Jenkinsfile.public

@@ -9,6 +9,12 @@ pipeline {
         stage('Build') {
             steps {
                 checkout scm
+
+                sh 'mkdir .build'
+
+                // make a backup of the mod file in case, for later linting
+                sh 'cp go.mod .build/go.mod.orig'
+                // download dependencies
                 sh 'go mod download'
 
                 sh 'go install -v -race ./...'
@@ -28,8 +34,7 @@ pipeline {
                         sh 'protolock status'
                         sh 'bash ./scripts/check-dbx-version.sh'
                         sh 'golangci-lint -j=4 run'
-                        // TODO: check for go mod tidy
+                        sh 'go run scripts/check-mod-tidy.go -mod .build/go.mod.orig'
-                        // TODO: check for directory tidy
                     }
                 }
 
@@ -41,13 +46,15 @@ pipeline {
                         sh 'psql -U postgres -c \'alter system set max_connections = 1000;\''
                         sh 'psql -U postgres -c \'create database teststorj;\''
                         sh 'go run scripts/use-ports.go -from 1024 -to 10000 &'
-                        sh 'go test -vet=off -timeout 9m -json -race ./... 2>&1 | tee tests.json | go run ./scripts/xunit.go -out tests.xml'
+                        sh 'go test -vet=off -timeout 9m -json -race ./... 2>&1 | tee .build/tests.json | go run ./scripts/xunit.go -out .build/tests.xml'
+                        sh 'go run scripts/check-clean-directory.go'
                     }
 
                     post {
                         always {
-                            archiveArtifacts artifacts: 'tests.json'
+                            sh script: 'cat .build/tests.json | tparse -all -top -slow 100', returnStatus: true
-                            junit 'tests.xml'
+                            archiveArtifacts artifacts: '.build/tests.json'
+                            junit '.build/tests.xml'
                         }
                     }
                 }
@@ -69,6 +76,7 @@ pipeline {
 
     post {
         always {
+            sh "chmod -R 777 ." // ensure Jenkins agent can delete the working directory
             deleteDir()
         }
     }

go.sum

@@ -219,8 +219,6 @@ github.com/minio/minio v0.0.0-20180508161510-54cd29b51c38 h1:F7p0ZU9AQuxlA6SWwhX
 github.com/minio/minio v0.0.0-20180508161510-54cd29b51c38/go.mod h1:lXcp05uxYaW99ebgI6ZKIGYU7tqZkM5xSsG0xRt4VIU=
 github.com/minio/minio-go v6.0.3+incompatible h1:yTq5mJOcWg6ot6STkEMnrNN896L0aDDu6njDB+8Ply0=
 github.com/minio/minio-go v6.0.3+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8=
-github.com/minio/sha256-simd v0.0.0-20171213220625-ad98a36ba0da h1:tazA5y1hWYJO8VSYbU36yBhXeIvruLXMUKu6WBtcJck=
-github.com/minio/sha256-simd v0.0.0-20171213220625-ad98a36ba0da/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
 github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5 h1:l16XLUUJ34wIz+RIvLhSwGvLvKyy+W598b135bJN6mg=
 github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
 github.com/minio/sio v0.0.0-20180327104954-6a41828a60f0 h1:ys4bbOlPvaUBlA0byjm6TqydsXZu614ZIUTfF+4MRY0=

scripts/check-clean-directory.go

@@ -22,9 +22,28 @@ func main() {
 		os.Exit(1)
 	}
 
-	if strings.TrimSpace(string(out)) != "" {
+	leftover := strings.Split(strings.TrimSpace(string(out)), "\n")
+	leftover = ignoreDir(leftover, ".build")
+
+	if len(leftover) != 0 {
 		fmt.Println("Files left-over after running tests:")
-		fmt.Println(string(out))
+		for _, file := range leftover {
+			fmt.Println(file)
+		}
 		os.Exit(1)
 	}
 }
+
+func ignoreDir(files []string, dir string) []string {
+	result := files[:0]
+	for _, file := range files {
+		if file == "" {
+			continue
+		}
+		if strings.HasPrefix(file, dir) {
+			continue
+		}
+		result = append(result, file)
+	}
+	return result
+}

scripts/check-mod-tidy.go

@@ -0,0 +1,112 @@
+// Copyright (C) 2019 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+// +build ignore
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/kylelemons/godebug/diff"
+)
+
+var modfile = flag.String("mod", "go.mod", "original mod file")
+
+func main() {
+	flag.Parse()
+
+	tempdir, err := ioutil.TempDir("", "check-mod-tidy")
+	checkf(err, "failed to create a temporary directory: %v\n", err)
+
+	defer func() {
+		err := os.RemoveAll(tempdir)
+		fmt.Fprintf(os.Stderr, "failed to delete temporary directory: %v\n", err)
+	}()
+
+	err = copyDir(".", tempdir)
+	checkf(err, "failed to copy directory: %v\n", err)
+
+	workingDir, err := os.Getwd()
+	checkf(err, "failed to get working directory: %v\n", err)
+
+	err = os.Chdir(tempdir)
+	checkf(err, "failed to change directory: %v\n", err)
+
+	defer os.Chdir(workingDir)
+
+	original, err := ioutil.ReadFile(*modfile)
+	checkf(err, "failed to read %q: %v\n", *modfile, err)
+
+	err = ioutil.WriteFile("go.mod", original, 0755)
+	checkf(err, "failed to write go.mod: %v\n", err)
+
+	err = tidy()
+	checkf(err, "failed to tidy go.mod: %v\n", err)
+
+	changed, err := ioutil.ReadFile("go.mod")
+	checkf(err, "failed to read go.mod: %v\n", err)
+
+	if !bytes.Equal(original, changed) {
+		diff, removed := difflines(string(original), string(changed))
+		fmt.Fprintln(os.Stderr, "go.mod is not tidy")
+		fmt.Fprintln(os.Stderr, diff)
+		if removed {
+			os.Exit(1)
+		}
+	}
+}
+
+func tidy() error {
+	var err error
+	for repeat := 2; repeat > 0; repeat-- {
+		cmd := exec.Command("go", "mod", "tidy")
+		cmd.Stdout, cmd.Stderr = os.Stderr, os.Stderr
+		err = cmd.Run()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "go mod tidy failed, retrying: %v", err)
+			continue
+		}
+		break
+	}
+	return err
+}
+
+func copyDir(src, dst string) error {
+	cmd := exec.Command("cp", "-a", src, dst)
+	cmd.Stdout, cmd.Stderr = os.Stderr, os.Stderr
+	return cmd.Run()
+}
+
+func checkf(err error, format string, args ...interface{}) {
+	if err == nil {
+		return
+	}
+	fmt.Fprintf(os.Stderr, format, args...)
+	os.Exit(1)
+}
+
+func difflines(a, b string) (patch string, removed bool) {
+	alines, blines := strings.Split(a, "\n"), strings.Split(b, "\n")
+
+	chunks := diff.DiffChunks(alines, blines)
+
+	buf := new(bytes.Buffer)
+	for _, c := range chunks {
+		for _, line := range c.Added {
+			fmt.Fprintf(buf, "+%s\n", line)
+		}
+		for _, line := range c.Deleted {
+			fmt.Fprintf(buf, "-%s\n", line)
+			removed = true
+		}
+	}
+
+	return strings.TrimRight(buf.String(), "\n"), removed
+}