From 9153f3705555fee2e3c58e10c17102d7963d8cb5 Mon Sep 17 00:00:00 2001 From: Mya Date: Thu, 19 May 2022 17:46:37 -0500 Subject: [PATCH] Makefile: run lint locally in docker Our linting process depends heavily on custom tools and linting configuration. To help improve this process for running on local developer machines, we can run the various tasks in our existing CI container. Change-Id: I60407686ce9233cc4f16e3724c5e8d44367aa200 --- DEVELOPING.md | 17 +++++ Makefile | 58 ++++++++++++++-- scripts/lint.go | 179 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 249 insertions(+), 5 deletions(-) create mode 100644 scripts/lint.go diff --git a/DEVELOPING.md b/DEVELOPING.md index dc7b78a97..c712957d6 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -260,6 +260,23 @@ git rebase -i HEAD~3 ... ``` +## Linting locally + +Our code linting process requires several customized and company specific tools. These are all packaged and provided as +part of our CI container. When running the lint target, we will spin up our CI container locally, and execute the +various linters from inside the container. + +```sh +make lint +``` + +By default, the linter runs on the entire code base but can be limited to specific packages. It is worth noting that +some linters cannot run on specific packages and will therefore be unaffected by the provided packages. + +```sh +make lint LINT_TARGET="./satellite/oidc/..." +``` + ## Executing tests locally The Storj project has an extensive suite of integration tests. Many of these tests require several infrastructure diff --git a/Makefile b/Makefile index 4a69cc094..89f4fb3f2 100644 --- a/Makefile +++ b/Makefile @@ -51,11 +51,6 @@ build-dev-deps: ## Install dependencies for builds go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo go get github.com/github-release/github-release -.PHONY: lint -lint: ## Analyze and find programs in source code - @echo "Running ${@}" - @golangci-lint run - .PHONY: goimports-fix goimports-fix: ## Applies goimports to every go file (excluding vendored files) goimports -w -local storj.io $$(find . -type f -name '*.go' -not -path "*/vendor/*") @@ -102,6 +97,59 @@ install-sim: ## install storj-sim ## install the latest stable version of Gateway-ST go install -race -v storj.io/gateway@latest +##@ Lint + +LINT_TARGET="./..." + +.PHONY: .lint +.lint: + go run ./scripts/lint.go \ + -parallel 4 \ + -race \ + -modules \ + -copyright \ + -imports \ + -peer-constraints \ + -atomic-align \ + -monkit \ + -errs \ + -staticcheck \ + -golangci \ + -monitoring \ + -wasm-size \ + -protolock \ + $(LINT_TARGET) + +.PHONY: lint +lint: + docker run --rm -it \ + -v ${GOPATH}/pkg:/go/pkg \ + -v ${PWD}:/storj \ + -w /storj \ + storjlabs/ci \ + make .lint LINT_TARGET="$(LINT_TARGET)" + +.PHONY: .lint/testsuite +.lint/testsuite: + go run ./scripts/lint.go \ + -work-dir testsuite \ + -parallel 4 \ + -imports \ + -atomic-align \ + -errs \ + -staticcheck \ + -golangci \ + $(LINT_TARGET) + +.PHONY: lint/testsuite +lint/testsuite: + docker run --rm -it \ + -v ${GOPATH}/pkg:/go/pkg \ + -v ${PWD}:/storj \ + -w /storj \ + storjlabs/ci \ + make .lint/testsuite LINT_TARGET="$(LINT_TARGET)" + ##@ Test TEST_TARGET ?= "./..." diff --git a/scripts/lint.go b/scripts/lint.go new file mode 100644 index 000000000..52cf3bbf9 --- /dev/null +++ b/scripts/lint.go @@ -0,0 +1,179 @@ +// Copyright (C) 2022 Storj Labs, Inc. +// See LICENSE for copying information. + +//go:build ignore +// +build ignore + +package main + +import ( + "context" + "flag" + "log" + "os" + "os/exec" + "os/signal" + "runtime" + "strings" + "syscall" + "time" + + "storj.io/common/sync2" +) + +func newCommand(ctx context.Context, directory string, name string, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = directory + + return cmd +} + +type Checks struct { + Modules bool + Copyright bool + Imports bool + PeerConstraints bool + AtomicAlign bool + Monkit bool + Errors bool + Static bool + Monitoring bool + WASMSize bool + Protolock bool + GolangCI bool +} + +func main() { + workDir, err := os.Getwd() + if err != nil { + log.Fatalln("error", err) + } + checks := Checks{} + + parallel := flag.Int("parallel", runtime.NumCPU(), "specify the number of tasks to run concurrently") + race := flag.Bool("race", false, "pass race to appropriate linters") + flag.StringVar(&workDir, "work-dir", workDir, "specify the working directory") + + flag.BoolVar(&checks.Modules, "modules", checks.Modules, "check module tidiness") + flag.BoolVar(&checks.Copyright, "copyright", checks.Copyright, "ensure copyright") + flag.BoolVar(&checks.Imports, "imports", checks.Imports, "check import usage") + flag.BoolVar(&checks.PeerConstraints, "peer-constraints", checks.PeerConstraints, "check peer constraints") + flag.BoolVar(&checks.AtomicAlign, "atomic-align", checks.AtomicAlign, "ensure atomic alignment") + flag.BoolVar(&checks.Monkit, "monkit", checks.Monkit, "check monkit usage") + flag.BoolVar(&checks.Errors, "errs", checks.Errors, "check error usage") + flag.BoolVar(&checks.Static, "staticcheck", checks.Static, "perform static analysis checks against the code base") + flag.BoolVar(&checks.Monitoring, "monitoring", checks.Monitoring, "check monitoring") + flag.BoolVar(&checks.WASMSize, "wasm-size", checks.WASMSize, "check the wasm file size for optimal performance") + flag.BoolVar(&checks.Protolock, "protolock", checks.Protolock, "check the status of the protolock file") + flag.BoolVar(&checks.GolangCI, "golangci", checks.GolangCI, "run the golangci-lint tool") + + flag.Parse() + + target := []string{"./..."} + if args := flag.Args(); len(args) > 0 { + target = args + } + + ctx, halt := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer halt() + + limiter := sync2.NewLimiter(*parallel) + + submit := func(cmd *exec.Cmd) bool { + prefix := "[" + cmd.Dir + " " + strings.Join(cmd.Args, " ") + "]" + + return limiter.Go(ctx, func() { + start := time.Now() + + log.Println(prefix, "running") + defer log.Println(prefix, "done", time.Since(start)) + + _ = cmd.Run() + exitCode := cmd.ProcessState.ExitCode() + if exitCode > 0 { + out, err := cmd.CombinedOutput() + log.Fatalln(prefix, "error", string(out), err) + } + }) + } + + // separate commands into two tiers to handle commands that can not be run in parallel (like staticcheck and + // golangci-lint). + commands := [][]*exec.Cmd{ + make([]*exec.Cmd, 0, 10), + make([]*exec.Cmd, 0, 1), + } + + if checks.Modules { + commands[0] = append(commands[0], newCommand(ctx, workDir, "check-mod-tidy")) + } + + if checks.Copyright { + commands[0] = append(commands[0], newCommand(ctx, workDir, "check-copyright")) + } + + if checks.Imports { + args := make([]string, 0, 2) + if *race { + args = append(args, "-race") + } + + args = append(args, target...) + commands[0] = append(commands[0], newCommand(ctx, workDir, "check-imports", args...)) + } + + if checks.PeerConstraints { + args := make([]string, 0, 1) + if *race { + args = append(args, "-race") + } + + commands[0] = append(commands[0], newCommand(ctx, workDir, "check-peer-constraints", args...)) + } + + if checks.AtomicAlign { + commands[0] = append(commands[0], newCommand(ctx, workDir, "check-atomic-align", target...)) + } + + if checks.Monkit { + commands[0] = append(commands[0], newCommand(ctx, workDir, "check-monkit", target...)) + } + + if checks.Errors { + commands[0] = append(commands[0], newCommand(ctx, workDir, "check-errs", target...)) + } + + if checks.Static { + commands[0] = append(commands[0], newCommand(ctx, workDir, "staticcheck", target...)) + } + + if checks.Monitoring { + commands[0] = append(commands[0], newCommand(ctx, workDir, "make", "check-monitoring")) + } + + if checks.WASMSize { + commands[0] = append(commands[0], newCommand(ctx, workDir, "make", "test-wasm-size")) + } + + if checks.Protolock { + commands[0] = append(commands[0], newCommand(ctx, workDir, "protolock", "status")) + } + + if checks.GolangCI { + args := append([]string{"--config", "/go/ci/.golangci.yml", "--skip-dirs", "(^|/)node_modules($|/)", "-j=2", "run"}, target...) + commands[1] = append(commands[1], newCommand(ctx, workDir, "golangci-lint", args...)) + } + + for _, tier := range commands { + for _, cmd := range tier { + ok := submit(cmd) + if !ok { + log.Fatalln("error", "failed to submit task to queue") + } + } + + limiter.Wait() + } + + limiter.Wait() +}