diff --git a/DEVELOPING.md b/DEVELOPING.md index dc7b78a97..c712957d6 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -260,6 +260,23 @@ git rebase -i HEAD~3 ... ``` +## Linting locally + +Our code linting process requires several customized and company specific tools. These are all packaged and provided as +part of our CI container. When running the lint target, we will spin up our CI container locally, and execute the +various linters from inside the container. + +```sh +make lint +``` + +By default, the linter runs on the entire code base but can be limited to specific packages. It is worth noting that +some linters cannot run on specific packages and will therefore be unaffected by the provided packages. + +```sh +make lint LINT_TARGET="./satellite/oidc/..." +``` + ## Executing tests locally The Storj project has an extensive suite of integration tests. Many of these tests require several infrastructure diff --git a/Makefile b/Makefile index 4a69cc094..89f4fb3f2 100644 --- a/Makefile +++ b/Makefile @@ -51,11 +51,6 @@ build-dev-deps: ## Install dependencies for builds go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo go get github.com/github-release/github-release -.PHONY: lint -lint: ## Analyze and find programs in source code - @echo "Running ${@}" - @golangci-lint run - .PHONY: goimports-fix goimports-fix: ## Applies goimports to every go file (excluding vendored files) goimports -w -local storj.io $$(find . -type f -name '*.go' -not -path "*/vendor/*") @@ -102,6 +97,59 @@ install-sim: ## install storj-sim ## install the latest stable version of Gateway-ST go install -race -v storj.io/gateway@latest +##@ Lint + +LINT_TARGET="./..." + +.PHONY: .lint +.lint: + go run ./scripts/lint.go \ + -parallel 4 \ + -race \ + -modules \ + -copyright \ + -imports \ + -peer-constraints \ + -atomic-align \ + -monkit \ + -errs \ + -staticcheck \ + -golangci \ + -monitoring \ + -wasm-size \ + -protolock \ + $(LINT_TARGET) + +.PHONY: lint +lint: + docker run --rm -it \ + -v ${GOPATH}/pkg:/go/pkg \ + -v ${PWD}:/storj \ + -w /storj \ + storjlabs/ci \ + make .lint LINT_TARGET="$(LINT_TARGET)" + +.PHONY: .lint/testsuite +.lint/testsuite: + go run ./scripts/lint.go \ + -work-dir testsuite \ + -parallel 4 \ + -imports \ + -atomic-align \ + -errs \ + -staticcheck \ + -golangci \ + $(LINT_TARGET) + +.PHONY: lint/testsuite +lint/testsuite: + docker run --rm -it \ + -v ${GOPATH}/pkg:/go/pkg \ + -v ${PWD}:/storj \ + -w /storj \ + storjlabs/ci \ + make .lint/testsuite LINT_TARGET="$(LINT_TARGET)" + ##@ Test TEST_TARGET ?= "./..." diff --git a/scripts/lint.go b/scripts/lint.go new file mode 100644 index 000000000..52cf3bbf9 --- /dev/null +++ b/scripts/lint.go @@ -0,0 +1,179 @@ +// Copyright (C) 2022 Storj Labs, Inc. +// See LICENSE for copying information. + +//go:build ignore +// +build ignore + +package main + +import ( + "context" + "flag" + "log" + "os" + "os/exec" + "os/signal" + "runtime" + "strings" + "syscall" + "time" + + "storj.io/common/sync2" +) + +func newCommand(ctx context.Context, directory string, name string, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = directory + + return cmd +} + +type Checks struct { + Modules bool + Copyright bool + Imports bool + PeerConstraints bool + AtomicAlign bool + Monkit bool + Errors bool + Static bool + Monitoring bool + WASMSize bool + Protolock bool + GolangCI bool +} + +func main() { + workDir, err := os.Getwd() + if err != nil { + log.Fatalln("error", err) + } + checks := Checks{} + + parallel := flag.Int("parallel", runtime.NumCPU(), "specify the number of tasks to run concurrently") + race := flag.Bool("race", false, "pass race to appropriate linters") + flag.StringVar(&workDir, "work-dir", workDir, "specify the working directory") + + flag.BoolVar(&checks.Modules, "modules", checks.Modules, "check module tidiness") + flag.BoolVar(&checks.Copyright, "copyright", checks.Copyright, "ensure copyright") + flag.BoolVar(&checks.Imports, "imports", checks.Imports, "check import usage") + flag.BoolVar(&checks.PeerConstraints, "peer-constraints", checks.PeerConstraints, "check peer constraints") + flag.BoolVar(&checks.AtomicAlign, "atomic-align", checks.AtomicAlign, "ensure atomic alignment") + flag.BoolVar(&checks.Monkit, "monkit", checks.Monkit, "check monkit usage") + flag.BoolVar(&checks.Errors, "errs", checks.Errors, "check error usage") + flag.BoolVar(&checks.Static, "staticcheck", checks.Static, "perform static analysis checks against the code base") + flag.BoolVar(&checks.Monitoring, "monitoring", checks.Monitoring, "check monitoring") + flag.BoolVar(&checks.WASMSize, "wasm-size", checks.WASMSize, "check the wasm file size for optimal performance") + flag.BoolVar(&checks.Protolock, "protolock", checks.Protolock, "check the status of the protolock file") + flag.BoolVar(&checks.GolangCI, "golangci", checks.GolangCI, "run the golangci-lint tool") + + flag.Parse() + + target := []string{"./..."} + if args := flag.Args(); len(args) > 0 { + target = args + } + + ctx, halt := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer halt() + + limiter := sync2.NewLimiter(*parallel) + + submit := func(cmd *exec.Cmd) bool { + prefix := "[" + cmd.Dir + " " + strings.Join(cmd.Args, " ") + "]" + + return limiter.Go(ctx, func() { + start := time.Now() + + log.Println(prefix, "running") + defer log.Println(prefix, "done", time.Since(start)) + + _ = cmd.Run() + exitCode := cmd.ProcessState.ExitCode() + if exitCode > 0 { + out, err := cmd.CombinedOutput() + log.Fatalln(prefix, "error", string(out), err) + } + }) + } + + // separate commands into two tiers to handle commands that can not be run in parallel (like staticcheck and + // golangci-lint). + commands := [][]*exec.Cmd{ + make([]*exec.Cmd, 0, 10), + make([]*exec.Cmd, 0, 1), + } + + if checks.Modules { + commands[0] = append(commands[0], newCommand(ctx, workDir, "check-mod-tidy")) + } + + if checks.Copyright { + commands[0] = append(commands[0], newCommand(ctx, workDir, "check-copyright")) + } + + if checks.Imports { + args := make([]string, 0, 2) + if *race { + args = append(args, "-race") + } + + args = append(args, target...) + commands[0] = append(commands[0], newCommand(ctx, workDir, "check-imports", args...)) + } + + if checks.PeerConstraints { + args := make([]string, 0, 1) + if *race { + args = append(args, "-race") + } + + commands[0] = append(commands[0], newCommand(ctx, workDir, "check-peer-constraints", args...)) + } + + if checks.AtomicAlign { + commands[0] = append(commands[0], newCommand(ctx, workDir, "check-atomic-align", target...)) + } + + if checks.Monkit { + commands[0] = append(commands[0], newCommand(ctx, workDir, "check-monkit", target...)) + } + + if checks.Errors { + commands[0] = append(commands[0], newCommand(ctx, workDir, "check-errs", target...)) + } + + if checks.Static { + commands[0] = append(commands[0], newCommand(ctx, workDir, "staticcheck", target...)) + } + + if checks.Monitoring { + commands[0] = append(commands[0], newCommand(ctx, workDir, "make", "check-monitoring")) + } + + if checks.WASMSize { + commands[0] = append(commands[0], newCommand(ctx, workDir, "make", "test-wasm-size")) + } + + if checks.Protolock { + commands[0] = append(commands[0], newCommand(ctx, workDir, "protolock", "status")) + } + + if checks.GolangCI { + args := append([]string{"--config", "/go/ci/.golangci.yml", "--skip-dirs", "(^|/)node_modules($|/)", "-j=2", "run"}, target...) + commands[1] = append(commands[1], newCommand(ctx, workDir, "golangci-lint", args...)) + } + + for _, tier := range commands { + for _, cmd := range tier { + ok := submit(cmd) + if !ok { + log.Fatalln("error", "failed to submit task to queue") + } + } + + limiter.Wait() + } + + limiter.Wait() +}