Makefile: run lint locally in docker

Our linting process depends heavily on custom tools and linting
configuration. To help improve this process for running on local
developer machines, we can run the various tasks in our existing CI
container.

Change-Id: I60407686ce9233cc4f16e3724c5e8d44367aa200
This commit is contained in:
Mya 2022-05-19 17:46:37 -05:00
parent 4b61cc9e9c
commit 9153f37055
3 changed files with 249 additions and 5 deletions

View File

@ -260,6 +260,23 @@ git rebase -i HEAD~3
... ...
``` ```
## Linting locally
Our code linting process requires several customized and company specific tools. These are all packaged and provided as
part of our CI container. When running the lint target, we will spin up our CI container locally, and execute the
various linters from inside the container.
```sh
make lint
```
By default, the linter runs on the entire code base but can be limited to specific packages. It is worth noting that
some linters cannot run on specific packages and will therefore be unaffected by the provided packages.
```sh
make lint LINT_TARGET="./satellite/oidc/..."
```
## Executing tests locally ## Executing tests locally
The Storj project has an extensive suite of integration tests. Many of these tests require several infrastructure The Storj project has an extensive suite of integration tests. Many of these tests require several infrastructure

View File

@ -51,11 +51,6 @@ build-dev-deps: ## Install dependencies for builds
go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo
go get github.com/github-release/github-release go get github.com/github-release/github-release
.PHONY: lint
lint: ## Analyze and find programs in source code
@echo "Running ${@}"
@golangci-lint run
.PHONY: goimports-fix .PHONY: goimports-fix
goimports-fix: ## Applies goimports to every go file (excluding vendored files) goimports-fix: ## Applies goimports to every go file (excluding vendored files)
goimports -w -local storj.io $$(find . -type f -name '*.go' -not -path "*/vendor/*") goimports -w -local storj.io $$(find . -type f -name '*.go' -not -path "*/vendor/*")
@ -102,6 +97,59 @@ install-sim: ## install storj-sim
## install the latest stable version of Gateway-ST ## install the latest stable version of Gateway-ST
go install -race -v storj.io/gateway@latest go install -race -v storj.io/gateway@latest
##@ Lint
LINT_TARGET="./..."
.PHONY: .lint
.lint:
go run ./scripts/lint.go \
-parallel 4 \
-race \
-modules \
-copyright \
-imports \
-peer-constraints \
-atomic-align \
-monkit \
-errs \
-staticcheck \
-golangci \
-monitoring \
-wasm-size \
-protolock \
$(LINT_TARGET)
.PHONY: lint
lint:
docker run --rm -it \
-v ${GOPATH}/pkg:/go/pkg \
-v ${PWD}:/storj \
-w /storj \
storjlabs/ci \
make .lint LINT_TARGET="$(LINT_TARGET)"
.PHONY: .lint/testsuite
.lint/testsuite:
go run ./scripts/lint.go \
-work-dir testsuite \
-parallel 4 \
-imports \
-atomic-align \
-errs \
-staticcheck \
-golangci \
$(LINT_TARGET)
.PHONY: lint/testsuite
lint/testsuite:
docker run --rm -it \
-v ${GOPATH}/pkg:/go/pkg \
-v ${PWD}:/storj \
-w /storj \
storjlabs/ci \
make .lint/testsuite LINT_TARGET="$(LINT_TARGET)"
##@ Test ##@ Test
TEST_TARGET ?= "./..." TEST_TARGET ?= "./..."

179
scripts/lint.go Normal file
View File

@ -0,0 +1,179 @@
// Copyright (C) 2022 Storj Labs, Inc.
// See LICENSE for copying information.
//go:build ignore
// +build ignore
package main
import (
"context"
"flag"
"log"
"os"
"os/exec"
"os/signal"
"runtime"
"strings"
"syscall"
"time"
"storj.io/common/sync2"
)
func newCommand(ctx context.Context, directory string, name string, args ...string) *exec.Cmd {
cmd := exec.CommandContext(ctx, name, args...)
cmd.Dir = directory
return cmd
}
type Checks struct {
Modules bool
Copyright bool
Imports bool
PeerConstraints bool
AtomicAlign bool
Monkit bool
Errors bool
Static bool
Monitoring bool
WASMSize bool
Protolock bool
GolangCI bool
}
func main() {
workDir, err := os.Getwd()
if err != nil {
log.Fatalln("error", err)
}
checks := Checks{}
parallel := flag.Int("parallel", runtime.NumCPU(), "specify the number of tasks to run concurrently")
race := flag.Bool("race", false, "pass race to appropriate linters")
flag.StringVar(&workDir, "work-dir", workDir, "specify the working directory")
flag.BoolVar(&checks.Modules, "modules", checks.Modules, "check module tidiness")
flag.BoolVar(&checks.Copyright, "copyright", checks.Copyright, "ensure copyright")
flag.BoolVar(&checks.Imports, "imports", checks.Imports, "check import usage")
flag.BoolVar(&checks.PeerConstraints, "peer-constraints", checks.PeerConstraints, "check peer constraints")
flag.BoolVar(&checks.AtomicAlign, "atomic-align", checks.AtomicAlign, "ensure atomic alignment")
flag.BoolVar(&checks.Monkit, "monkit", checks.Monkit, "check monkit usage")
flag.BoolVar(&checks.Errors, "errs", checks.Errors, "check error usage")
flag.BoolVar(&checks.Static, "staticcheck", checks.Static, "perform static analysis checks against the code base")
flag.BoolVar(&checks.Monitoring, "monitoring", checks.Monitoring, "check monitoring")
flag.BoolVar(&checks.WASMSize, "wasm-size", checks.WASMSize, "check the wasm file size for optimal performance")
flag.BoolVar(&checks.Protolock, "protolock", checks.Protolock, "check the status of the protolock file")
flag.BoolVar(&checks.GolangCI, "golangci", checks.GolangCI, "run the golangci-lint tool")
flag.Parse()
target := []string{"./..."}
if args := flag.Args(); len(args) > 0 {
target = args
}
ctx, halt := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer halt()
limiter := sync2.NewLimiter(*parallel)
submit := func(cmd *exec.Cmd) bool {
prefix := "[" + cmd.Dir + " " + strings.Join(cmd.Args, " ") + "]"
return limiter.Go(ctx, func() {
start := time.Now()
log.Println(prefix, "running")
defer log.Println(prefix, "done", time.Since(start))
_ = cmd.Run()
exitCode := cmd.ProcessState.ExitCode()
if exitCode > 0 {
out, err := cmd.CombinedOutput()
log.Fatalln(prefix, "error", string(out), err)
}
})
}
// separate commands into two tiers to handle commands that can not be run in parallel (like staticcheck and
// golangci-lint).
commands := [][]*exec.Cmd{
make([]*exec.Cmd, 0, 10),
make([]*exec.Cmd, 0, 1),
}
if checks.Modules {
commands[0] = append(commands[0], newCommand(ctx, workDir, "check-mod-tidy"))
}
if checks.Copyright {
commands[0] = append(commands[0], newCommand(ctx, workDir, "check-copyright"))
}
if checks.Imports {
args := make([]string, 0, 2)
if *race {
args = append(args, "-race")
}
args = append(args, target...)
commands[0] = append(commands[0], newCommand(ctx, workDir, "check-imports", args...))
}
if checks.PeerConstraints {
args := make([]string, 0, 1)
if *race {
args = append(args, "-race")
}
commands[0] = append(commands[0], newCommand(ctx, workDir, "check-peer-constraints", args...))
}
if checks.AtomicAlign {
commands[0] = append(commands[0], newCommand(ctx, workDir, "check-atomic-align", target...))
}
if checks.Monkit {
commands[0] = append(commands[0], newCommand(ctx, workDir, "check-monkit", target...))
}
if checks.Errors {
commands[0] = append(commands[0], newCommand(ctx, workDir, "check-errs", target...))
}
if checks.Static {
commands[0] = append(commands[0], newCommand(ctx, workDir, "staticcheck", target...))
}
if checks.Monitoring {
commands[0] = append(commands[0], newCommand(ctx, workDir, "make", "check-monitoring"))
}
if checks.WASMSize {
commands[0] = append(commands[0], newCommand(ctx, workDir, "make", "test-wasm-size"))
}
if checks.Protolock {
commands[0] = append(commands[0], newCommand(ctx, workDir, "protolock", "status"))
}
if checks.GolangCI {
args := append([]string{"--config", "/go/ci/.golangci.yml", "--skip-dirs", "(^|/)node_modules($|/)", "-j=2", "run"}, target...)
commands[1] = append(commands[1], newCommand(ctx, workDir, "golangci-lint", args...))
}
for _, tier := range commands {
for _, cmd := range tier {
ok := submit(cmd)
if !ok {
log.Fatalln("error", "failed to submit task to queue")
}
}
limiter.Wait()
}
limiter.Wait()
}