Makefile: run lint locally in docker
Our linting process depends heavily on custom tools and linting configuration. To help improve this process for running on local developer machines, we can run the various tasks in our existing CI container. Change-Id: I60407686ce9233cc4f16e3724c5e8d44367aa200
This commit is contained in:
Mya
parent
4b61cc9e9c
commit
9153f37055
@ -260,6 +260,23 @@ git rebase -i HEAD~3
|
||||
...
|
||||
```
|
||||
|
||||
## Linting locally
|
||||
|
||||
Our code linting process requires several customized and company specific tools. These are all packaged and provided as
|
||||
part of our CI container. When running the lint target, we will spin up our CI container locally, and execute the
|
||||
various linters from inside the container.
|
||||
|
||||
```sh
|
||||
make lint
|
||||
```
|
||||
|
||||
By default, the linter runs on the entire code base but can be limited to specific packages. It is worth noting that
|
||||
some linters cannot run on specific packages and will therefore be unaffected by the provided packages.
|
||||
|
||||
```sh
|
||||
make lint LINT_TARGET="./satellite/oidc/..."
|
||||
```
|
||||
|
||||
## Executing tests locally
|
||||
|
||||
The Storj project has an extensive suite of integration tests. Many of these tests require several infrastructure
|
||||
|
||||
58
Makefile
58
Makefile
@ -51,11 +51,6 @@ build-dev-deps: ## Install dependencies for builds
|
||||
go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo
|
||||
go get github.com/github-release/github-release
|
||||
|
||||
.PHONY: lint
|
||||
lint: ## Analyze and find programs in source code
|
||||
@echo "Running ${@}"
|
||||
@golangci-lint run
|
||||
|
||||
.PHONY: goimports-fix
|
||||
goimports-fix: ## Applies goimports to every go file (excluding vendored files)
|
||||
goimports -w -local storj.io $$(find . -type f -name '*.go' -not -path "*/vendor/*")
|
||||
@ -102,6 +97,59 @@ install-sim: ## install storj-sim
|
||||
## install the latest stable version of Gateway-ST
|
||||
go install -race -v storj.io/gateway@latest
|
||||
|
||||
##@ Lint
|
||||
|
||||
LINT_TARGET="./..."
|
||||
|
||||
.PHONY: .lint
|
||||
.lint:
|
||||
go run ./scripts/lint.go \
|
||||
-parallel 4 \
|
||||
-race \
|
||||
-modules \
|
||||
-copyright \
|
||||
-imports \
|
||||
-peer-constraints \
|
||||
-atomic-align \
|
||||
-monkit \
|
||||
-errs \
|
||||
-staticcheck \
|
||||
-golangci \
|
||||
-monitoring \
|
||||
-wasm-size \
|
||||
-protolock \
|
||||
$(LINT_TARGET)
|
||||
|
||||
.PHONY: lint
|
||||
lint:
|
||||
docker run --rm -it \
|
||||
-v ${GOPATH}/pkg:/go/pkg \
|
||||
-v ${PWD}:/storj \
|
||||
-w /storj \
|
||||
storjlabs/ci \
|
||||
make .lint LINT_TARGET="$(LINT_TARGET)"
|
||||
|
||||
.PHONY: .lint/testsuite
|
||||
.lint/testsuite:
|
||||
go run ./scripts/lint.go \
|
||||
-work-dir testsuite \
|
||||
-parallel 4 \
|
||||
-imports \
|
||||
-atomic-align \
|
||||
-errs \
|
||||
-staticcheck \
|
||||
-golangci \
|
||||
$(LINT_TARGET)
|
||||
|
||||
.PHONY: lint/testsuite
|
||||
lint/testsuite:
|
||||
docker run --rm -it \
|
||||
-v ${GOPATH}/pkg:/go/pkg \
|
||||
-v ${PWD}:/storj \
|
||||
-w /storj \
|
||||
storjlabs/ci \
|
||||
make .lint/testsuite LINT_TARGET="$(LINT_TARGET)"
|
||||
|
||||
##@ Test
|
||||
|
||||
TEST_TARGET ?= "./..."
|
||||
|
||||
179
scripts/lint.go
Normal file
179
scripts/lint.go
Normal file
@ -0,0 +1,179 @@
|
||||
// Copyright (C) 2022 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
//go:build ignore
|
||||
// +build ignore
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"storj.io/common/sync2"
|
||||
)
|
||||
|
||||
func newCommand(ctx context.Context, directory string, name string, args ...string) *exec.Cmd {
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
cmd.Dir = directory
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
type Checks struct {
|
||||
Modules bool
|
||||
Copyright bool
|
||||
Imports bool
|
||||
PeerConstraints bool
|
||||
AtomicAlign bool
|
||||
Monkit bool
|
||||
Errors bool
|
||||
Static bool
|
||||
Monitoring bool
|
||||
WASMSize bool
|
||||
Protolock bool
|
||||
GolangCI bool
|
||||
}
|
||||
|
||||
func main() {
|
||||
workDir, err := os.Getwd()
|
||||
if err != nil {
|
||||
log.Fatalln("error", err)
|
||||
}
|
||||
checks := Checks{}
|
||||
|
||||
parallel := flag.Int("parallel", runtime.NumCPU(), "specify the number of tasks to run concurrently")
|
||||
race := flag.Bool("race", false, "pass race to appropriate linters")
|
||||
flag.StringVar(&workDir, "work-dir", workDir, "specify the working directory")
|
||||
|
||||
flag.BoolVar(&checks.Modules, "modules", checks.Modules, "check module tidiness")
|
||||
flag.BoolVar(&checks.Copyright, "copyright", checks.Copyright, "ensure copyright")
|
||||
flag.BoolVar(&checks.Imports, "imports", checks.Imports, "check import usage")
|
||||
flag.BoolVar(&checks.PeerConstraints, "peer-constraints", checks.PeerConstraints, "check peer constraints")
|
||||
flag.BoolVar(&checks.AtomicAlign, "atomic-align", checks.AtomicAlign, "ensure atomic alignment")
|
||||
flag.BoolVar(&checks.Monkit, "monkit", checks.Monkit, "check monkit usage")
|
||||
flag.BoolVar(&checks.Errors, "errs", checks.Errors, "check error usage")
|
||||
flag.BoolVar(&checks.Static, "staticcheck", checks.Static, "perform static analysis checks against the code base")
|
||||
flag.BoolVar(&checks.Monitoring, "monitoring", checks.Monitoring, "check monitoring")
|
||||
flag.BoolVar(&checks.WASMSize, "wasm-size", checks.WASMSize, "check the wasm file size for optimal performance")
|
||||
flag.BoolVar(&checks.Protolock, "protolock", checks.Protolock, "check the status of the protolock file")
|
||||
flag.BoolVar(&checks.GolangCI, "golangci", checks.GolangCI, "run the golangci-lint tool")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
target := []string{"./..."}
|
||||
if args := flag.Args(); len(args) > 0 {
|
||||
target = args
|
||||
}
|
||||
|
||||
ctx, halt := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer halt()
|
||||
|
||||
limiter := sync2.NewLimiter(*parallel)
|
||||
|
||||
submit := func(cmd *exec.Cmd) bool {
|
||||
prefix := "[" + cmd.Dir + " " + strings.Join(cmd.Args, " ") + "]"
|
||||
|
||||
return limiter.Go(ctx, func() {
|
||||
start := time.Now()
|
||||
|
||||
log.Println(prefix, "running")
|
||||
defer log.Println(prefix, "done", time.Since(start))
|
||||
|
||||
_ = cmd.Run()
|
||||
exitCode := cmd.ProcessState.ExitCode()
|
||||
if exitCode > 0 {
|
||||
out, err := cmd.CombinedOutput()
|
||||
log.Fatalln(prefix, "error", string(out), err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// separate commands into two tiers to handle commands that can not be run in parallel (like staticcheck and
|
||||
// golangci-lint).
|
||||
commands := [][]*exec.Cmd{
|
||||
make([]*exec.Cmd, 0, 10),
|
||||
make([]*exec.Cmd, 0, 1),
|
||||
}
|
||||
|
||||
if checks.Modules {
|
||||
commands[0] = append(commands[0], newCommand(ctx, workDir, "check-mod-tidy"))
|
||||
}
|
||||
|
||||
if checks.Copyright {
|
||||
commands[0] = append(commands[0], newCommand(ctx, workDir, "check-copyright"))
|
||||
}
|
||||
|
||||
if checks.Imports {
|
||||
args := make([]string, 0, 2)
|
||||
if *race {
|
||||
args = append(args, "-race")
|
||||
}
|
||||
|
||||
args = append(args, target...)
|
||||
commands[0] = append(commands[0], newCommand(ctx, workDir, "check-imports", args...))
|
||||
}
|
||||
|
||||
if checks.PeerConstraints {
|
||||
args := make([]string, 0, 1)
|
||||
if *race {
|
||||
args = append(args, "-race")
|
||||
}
|
||||
|
||||
commands[0] = append(commands[0], newCommand(ctx, workDir, "check-peer-constraints", args...))
|
||||
}
|
||||
|
||||
if checks.AtomicAlign {
|
||||
commands[0] = append(commands[0], newCommand(ctx, workDir, "check-atomic-align", target...))
|
||||
}
|
||||
|
||||
if checks.Monkit {
|
||||
commands[0] = append(commands[0], newCommand(ctx, workDir, "check-monkit", target...))
|
||||
}
|
||||
|
||||
if checks.Errors {
|
||||
commands[0] = append(commands[0], newCommand(ctx, workDir, "check-errs", target...))
|
||||
}
|
||||
|
||||
if checks.Static {
|
||||
commands[0] = append(commands[0], newCommand(ctx, workDir, "staticcheck", target...))
|
||||
}
|
||||
|
||||
if checks.Monitoring {
|
||||
commands[0] = append(commands[0], newCommand(ctx, workDir, "make", "check-monitoring"))
|
||||
}
|
||||
|
||||
if checks.WASMSize {
|
||||
commands[0] = append(commands[0], newCommand(ctx, workDir, "make", "test-wasm-size"))
|
||||
}
|
||||
|
||||
if checks.Protolock {
|
||||
commands[0] = append(commands[0], newCommand(ctx, workDir, "protolock", "status"))
|
||||
}
|
||||
|
||||
if checks.GolangCI {
|
||||
args := append([]string{"--config", "/go/ci/.golangci.yml", "--skip-dirs", "(^|/)node_modules($|/)", "-j=2", "run"}, target...)
|
||||
commands[1] = append(commands[1], newCommand(ctx, workDir, "golangci-lint", args...))
|
||||
}
|
||||
|
||||
for _, tier := range commands {
|
||||
for _, cmd := range tier {
|
||||
ok := submit(cmd)
|
||||
if !ok {
|
||||
log.Fatalln("error", "failed to submit task to queue")
|
||||
}
|
||||
}
|
||||
|
||||
limiter.Wait()
|
||||
}
|
||||
|
||||
limiter.Wait()
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user