Makefile: run lint locally in docker

Our linting process depends heavily on custom tools and linting
configuration. To help improve this process for running on local
developer machines, we can run the various tasks in our existing CI
container.

Change-Id: I60407686ce9233cc4f16e3724c5e8d44367aa200

DEVELOPING.md

@@ -260,6 +260,23 @@ git rebase -i HEAD~3
 ...
 ```
 
+## Linting locally
+
+Our code linting process requires several customized and company specific tools. These are all packaged and provided as
+part of our CI container. When running the lint target, we will spin up our CI container locally, and execute the
+various linters from inside the container.
+
+```sh
+make lint
+```
+
+By default, the linter runs on the entire code base but can be limited to specific packages. It is worth noting that
+some linters cannot run on specific packages and will therefore be unaffected by the provided packages.
+
+```sh
+make lint LINT_TARGET="./satellite/oidc/..."
+```
+
 ## Executing tests locally
 
 The Storj project has an extensive suite of integration tests. Many of these tests require several infrastructure

Makefile

@@ -51,11 +51,6 @@ build-dev-deps: ## Install dependencies for builds
 	go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo
 	go get github.com/github-release/github-release
 
-.PHONY: lint
-lint: ## Analyze and find programs in source code
-	@echo "Running ${@}"
-	@golangci-lint run
-
 .PHONY: goimports-fix
 goimports-fix: ## Applies goimports to every go file (excluding vendored files)
 	goimports -w -local storj.io $$(find . -type f -name '*.go' -not -path "*/vendor/*")
@@ -102,6 +97,59 @@ install-sim: ## install storj-sim
 	## install the latest stable version of Gateway-ST
 	go install -race -v storj.io/gateway@latest
 
+##@ Lint
+
+LINT_TARGET="./..."
+
+.PHONY: .lint
+.lint:
+	go run ./scripts/lint.go \
+		-parallel 4 \
+		-race \
+		-modules \
+		-copyright \
+		-imports \
+		-peer-constraints \
+		-atomic-align \
+		-monkit \
+		-errs \
+		-staticcheck \
+		-golangci \
+		-monitoring \
+		-wasm-size \
+		-protolock \
+		$(LINT_TARGET)
+
+.PHONY: lint
+lint:
+	docker run --rm -it \
+		-v ${GOPATH}/pkg:/go/pkg \
+		-v ${PWD}:/storj \
+		-w /storj \
+		storjlabs/ci \
+		make .lint LINT_TARGET="$(LINT_TARGET)"
+
+.PHONY: .lint/testsuite
+.lint/testsuite:
+	go run ./scripts/lint.go \
+		-work-dir testsuite \
+		-parallel 4 \
+		-imports \
+		-atomic-align \
+		-errs \
+		-staticcheck \
+		-golangci \
+		$(LINT_TARGET)
+
+.PHONY: lint/testsuite
+lint/testsuite:
+	docker run --rm -it \
+		-v ${GOPATH}/pkg:/go/pkg \
+		-v ${PWD}:/storj \
+		-w /storj \
+		storjlabs/ci \
+		make .lint/testsuite LINT_TARGET="$(LINT_TARGET)"
+
 ##@ Test
 
 TEST_TARGET ?= "./..."

scripts/lint.go

@@ -0,0 +1,179 @@
+// Copyright (C) 2022 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+//go:build ignore
+// +build ignore
+
+package main
+
+import (
+	"context"
+	"flag"
+	"log"
+	"os"
+	"os/exec"
+	"os/signal"
+	"runtime"
+	"strings"
+	"syscall"
+	"time"
+
+	"storj.io/common/sync2"
+)
+
+func newCommand(ctx context.Context, directory string, name string, args ...string) *exec.Cmd {
+	cmd := exec.CommandContext(ctx, name, args...)
+	cmd.Dir = directory
+
+	return cmd
+}
+
+type Checks struct {
+	Modules         bool
+	Copyright       bool
+	Imports         bool
+	PeerConstraints bool
+	AtomicAlign     bool
+	Monkit          bool
+	Errors          bool
+	Static          bool
+	Monitoring      bool
+	WASMSize        bool
+	Protolock       bool
+	GolangCI        bool
+}
+
+func main() {
+	workDir, err := os.Getwd()
+	if err != nil {
+		log.Fatalln("error", err)
+	}
+	checks := Checks{}
+
+	parallel := flag.Int("parallel", runtime.NumCPU(), "specify the number of tasks to run concurrently")
+	race := flag.Bool("race", false, "pass race to appropriate linters")
+	flag.StringVar(&workDir, "work-dir", workDir, "specify the working directory")
+
+	flag.BoolVar(&checks.Modules, "modules", checks.Modules, "check module tidiness")
+	flag.BoolVar(&checks.Copyright, "copyright", checks.Copyright, "ensure copyright")
+	flag.BoolVar(&checks.Imports, "imports", checks.Imports, "check import usage")
+	flag.BoolVar(&checks.PeerConstraints, "peer-constraints", checks.PeerConstraints, "check peer constraints")
+	flag.BoolVar(&checks.AtomicAlign, "atomic-align", checks.AtomicAlign, "ensure atomic alignment")
+	flag.BoolVar(&checks.Monkit, "monkit", checks.Monkit, "check monkit usage")
+	flag.BoolVar(&checks.Errors, "errs", checks.Errors, "check error usage")
+	flag.BoolVar(&checks.Static, "staticcheck", checks.Static, "perform static analysis checks against the code base")
+	flag.BoolVar(&checks.Monitoring, "monitoring", checks.Monitoring, "check monitoring")
+	flag.BoolVar(&checks.WASMSize, "wasm-size", checks.WASMSize, "check the wasm file size for optimal performance")
+	flag.BoolVar(&checks.Protolock, "protolock", checks.Protolock, "check the status of the protolock file")
+	flag.BoolVar(&checks.GolangCI, "golangci", checks.GolangCI, "run the golangci-lint tool")
+
+	flag.Parse()
+
+	target := []string{"./..."}
+	if args := flag.Args(); len(args) > 0 {
+		target = args
+	}
+
+	ctx, halt := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer halt()
+
+	limiter := sync2.NewLimiter(*parallel)
+
+	submit := func(cmd *exec.Cmd) bool {
+		prefix := "[" + cmd.Dir + " " + strings.Join(cmd.Args, " ") + "]"
+
+		return limiter.Go(ctx, func() {
+			start := time.Now()
+
+			log.Println(prefix, "running")
+			defer log.Println(prefix, "done", time.Since(start))
+
+			_ = cmd.Run()
+			exitCode := cmd.ProcessState.ExitCode()
+			if exitCode > 0 {
+				out, err := cmd.CombinedOutput()
+				log.Fatalln(prefix, "error", string(out), err)
+			}
+		})
+	}
+
+	// separate commands into two tiers to handle commands that can not be run in parallel (like staticcheck and
+	// golangci-lint).
+	commands := [][]*exec.Cmd{
+		make([]*exec.Cmd, 0, 10),
+		make([]*exec.Cmd, 0, 1),
+	}
+
+	if checks.Modules {
+		commands[0] = append(commands[0], newCommand(ctx, workDir, "check-mod-tidy"))
+	}
+
+	if checks.Copyright {
+		commands[0] = append(commands[0], newCommand(ctx, workDir, "check-copyright"))
+	}
+
+	if checks.Imports {
+		args := make([]string, 0, 2)
+		if *race {
+			args = append(args, "-race")
+		}
+
+		args = append(args, target...)
+		commands[0] = append(commands[0], newCommand(ctx, workDir, "check-imports", args...))
+	}
+
+	if checks.PeerConstraints {
+		args := make([]string, 0, 1)
+		if *race {
+			args = append(args, "-race")
+		}
+
+		commands[0] = append(commands[0], newCommand(ctx, workDir, "check-peer-constraints", args...))
+	}
+
+	if checks.AtomicAlign {
+		commands[0] = append(commands[0], newCommand(ctx, workDir, "check-atomic-align", target...))
+	}
+
+	if checks.Monkit {
+		commands[0] = append(commands[0], newCommand(ctx, workDir, "check-monkit", target...))
+	}
+
+	if checks.Errors {
+		commands[0] = append(commands[0], newCommand(ctx, workDir, "check-errs", target...))
+	}
+
+	if checks.Static {
+		commands[0] = append(commands[0], newCommand(ctx, workDir, "staticcheck", target...))
+	}
+
+	if checks.Monitoring {
+		commands[0] = append(commands[0], newCommand(ctx, workDir, "make", "check-monitoring"))
+	}
+
+	if checks.WASMSize {
+		commands[0] = append(commands[0], newCommand(ctx, workDir, "make", "test-wasm-size"))
+	}
+
+	if checks.Protolock {
+		commands[0] = append(commands[0], newCommand(ctx, workDir, "protolock", "status"))
+	}
+
+	if checks.GolangCI {
+		args := append([]string{"--config", "/go/ci/.golangci.yml", "--skip-dirs", "(^|/)node_modules($|/)", "-j=2", "run"}, target...)
+		commands[1] = append(commands[1], newCommand(ctx, workDir, "golangci-lint", args...))
+	}
+
+	for _, tier := range commands {
+		for _, cmd := range tier {
+			ok := submit(cmd)
+			if !ok {
+				log.Fatalln("error", "failed to submit task to queue")
+			}
+		}
+
+		limiter.Wait()
+	}
+
+	limiter.Wait()
+}