Fix import groupings (#739)

cmd/inspector/main.go

@@ -16,11 +16,10 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/zeebo/errs"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/pb"
 	"storj.io/storj/pkg/process"
 	"storj.io/storj/pkg/provider"
+	"storj.io/storj/pkg/storj"
 	"storj.io/storj/pkg/transport"
 )
 

cmd/overlay/main.go

@@ -11,11 +11,10 @@ import (
 	"github.com/zeebo/errs"
 	"go.uber.org/zap"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/cfgstruct"
 	"storj.io/storj/pkg/pb"
 	"storj.io/storj/pkg/process"
+	"storj.io/storj/pkg/storj"
 )
 
 var (

cmd/satellite/main.go

@@ -16,8 +16,6 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/zeebo/errs"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/auth/grpcauth"
 	"storj.io/storj/pkg/bwagreement"
 	dbmanager "storj.io/storj/pkg/bwagreement/database-manager"
@@ -32,6 +30,7 @@ import (
 	"storj.io/storj/pkg/process"
 	"storj.io/storj/pkg/provider"
 	"storj.io/storj/pkg/statdb"
+	"storj.io/storj/pkg/storj"
 	"storj.io/storj/storage/redis"
 )
 

cmd/storagenode/main.go

@@ -14,8 +14,6 @@ import (
 	"github.com/gogo/protobuf/proto"
 	"github.com/spf13/cobra"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/cfgstruct"
 	"storj.io/storj/pkg/kademlia"
 	"storj.io/storj/pkg/pb"
@@ -23,6 +21,7 @@ import (
 	"storj.io/storj/pkg/piecestore/psserver/psdb"
 	"storj.io/storj/pkg/process"
 	"storj.io/storj/pkg/provider"
+	"storj.io/storj/pkg/storj"
 )
 
 var (

examples/grpc-debug/main.go

@@ -10,10 +10,9 @@ import (
 
 	"google.golang.org/grpc"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/cfgstruct"
 	"storj.io/storj/pkg/provider"
+	"storj.io/storj/pkg/storj"
 )
 
 var (

internal/testplanet/node.go

@@ -13,8 +13,6 @@ import (
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/auth/grpcauth"
 	"storj.io/storj/pkg/kademlia"
 	"storj.io/storj/pkg/overlay"
@@ -22,6 +20,7 @@ import (
 	"storj.io/storj/pkg/pointerdb/pdbclient"
 	"storj.io/storj/pkg/provider"
 	"storj.io/storj/pkg/statdb"
+	"storj.io/storj/pkg/storj"
 	"storj.io/storj/pkg/transport"
 	"storj.io/storj/pkg/utils"
 	"storj.io/storj/storage/teststore"

pkg/audit/verifier_test.go

@@ -13,7 +13,6 @@ import (
 	"github.com/vivint/infectious"
 
 	"storj.io/storj/internal/teststorj"
-
 	"storj.io/storj/pkg/pb"
 )
 

pkg/bwagreement/config.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/zeebo/errs"
 	"go.uber.org/zap"
-
 	monkit "gopkg.in/spacemonkeygo/monkit.v2"
 
 	"storj.io/storj/pkg/bwagreement/database-manager"

pkg/bwagreement/database-manager/database-manager.go

@@ -9,7 +9,6 @@ import (
 
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
-
 	monkit "gopkg.in/spacemonkeygo/monkit.v2"
 
 	"storj.io/storj/internal/migrate"

pkg/inspector/server.go

@@ -11,8 +11,6 @@ import (
 	"go.uber.org/zap"
 	"gopkg.in/spacemonkeygo/monkit.v2"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/dht"
 	"storj.io/storj/pkg/node"
 	"storj.io/storj/pkg/overlay"
@@ -20,6 +18,7 @@ import (
 	"storj.io/storj/pkg/provider"
 	"storj.io/storj/pkg/statdb"
 	statsproto "storj.io/storj/pkg/statdb/proto"
+	"storj.io/storj/pkg/storj"
 )
 
 var (

pkg/kademlia/kademlia_test.go

@@ -20,7 +20,6 @@ import (
 
 	"storj.io/storj/internal/identity"
 	"storj.io/storj/internal/teststorj"
-
 	"storj.io/storj/pkg/node"
 	"storj.io/storj/pkg/pb"
 	"storj.io/storj/pkg/provider"

pkg/kademlia/replacement_cache_test.go

@@ -9,7 +9,6 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"storj.io/storj/internal/teststorj"
-
 	"storj.io/storj/pkg/pb"
 )
 

pkg/kademlia/routing_helpers.go

@@ -12,10 +12,9 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 
+	"storj.io/storj/pkg/pb"
 	"storj.io/storj/pkg/storj"
 	"storj.io/storj/pkg/utils"
-
-	"storj.io/storj/pkg/pb"
 	"storj.io/storj/storage"
 )
 

pkg/kademlia/routing_helpers_test.go

@@ -17,7 +17,6 @@ import (
 	"go.uber.org/zap"
 
 	"storj.io/storj/internal/teststorj"
-
 	"storj.io/storj/pkg/pb"
 	"storj.io/storj/pkg/storj"
 	"storj.io/storj/storage"

pkg/kademlia/routing_test.go

@@ -11,9 +11,8 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"storj.io/storj/internal/teststorj"
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/pb"
+	"storj.io/storj/pkg/storj"
 	"storj.io/storj/storage"
 )
 

pkg/node/connection_pool.go

@@ -10,10 +10,9 @@ import (
 	"github.com/zeebo/errs"
 	"google.golang.org/grpc"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/pb"
 	"storj.io/storj/pkg/provider"
+	"storj.io/storj/pkg/storj"
 	"storj.io/storj/pkg/transport"
 	"storj.io/storj/pkg/utils"
 )

pkg/node/server_test.go

@@ -12,7 +12,6 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"storj.io/storj/internal/teststorj"
-
 	"storj.io/storj/pkg/dht"
 	"storj.io/storj/pkg/dht/mocks"
 	"storj.io/storj/pkg/pb"

pkg/overlay/cache.go

@@ -9,11 +9,10 @@ import (
 	"github.com/gogo/protobuf/proto"
 	"github.com/zeebo/errs"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/dht"
 	"storj.io/storj/pkg/pb"
 	"storj.io/storj/pkg/statdb"
+	"storj.io/storj/pkg/storj"
 	"storj.io/storj/storage"
 )
 

pkg/overlay/cache_test.go

@@ -9,13 +9,12 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
-	"storj.io/storj/internal/teststorj"
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/internal/testcontext"
 	"storj.io/storj/internal/testplanet"
+	"storj.io/storj/internal/teststorj"
 	"storj.io/storj/pkg/overlay"
 	"storj.io/storj/pkg/pb"
+	"storj.io/storj/pkg/storj"
 	"storj.io/storj/storage"
 	"storj.io/storj/storage/boltdb"
 	"storj.io/storj/storage/redis"

pkg/overlay/config.go

@@ -12,12 +12,11 @@ import (
 	"go.uber.org/zap"
 	"gopkg.in/spacemonkeygo/monkit.v2"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/kademlia"
 	"storj.io/storj/pkg/pb"
 	"storj.io/storj/pkg/provider"
 	"storj.io/storj/pkg/statdb"
+	"storj.io/storj/pkg/storj"
 	"storj.io/storj/pkg/utils"
 	"storj.io/storj/storage"
 	"storj.io/storj/storage/boltdb"

pkg/overlay/mocks/overlay.go

@@ -10,10 +10,9 @@ import (
 
 	"github.com/zeebo/errs"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/pb"
 	"storj.io/storj/pkg/provider"
+	"storj.io/storj/pkg/storj"
 )
 
 // Overlay is a mocked overlay implementation

pkg/overlay/server.go

@@ -15,10 +15,9 @@ import (
 	"google.golang.org/grpc/status"
 	"gopkg.in/spacemonkeygo/monkit.v2"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/dht"
 	"storj.io/storj/pkg/pb"
+	"storj.io/storj/pkg/storj"
 	"storj.io/storj/storage"
 )
 

pkg/piecestore/psclient/pieceranger_test.go

@@ -17,7 +17,6 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"storj.io/storj/internal/teststorj"
-
 	"storj.io/storj/pkg/pb"
 )
 

pkg/piecestore/psserver/agreementsender/agreementsender.go

@@ -12,12 +12,11 @@ import (
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/overlay"
 	"storj.io/storj/pkg/pb"
 	"storj.io/storj/pkg/piecestore/psserver/psdb"
 	"storj.io/storj/pkg/provider"
+	"storj.io/storj/pkg/storj"
 	"storj.io/storj/pkg/utils"
 )
 

pkg/piecestore/psserver/psdb/psdb.go

@@ -19,10 +19,9 @@ import (
 	"go.uber.org/zap"
 	monkit "gopkg.in/spacemonkeygo/monkit.v2"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/pb"
 	"storj.io/storj/pkg/piecestore"
+	"storj.io/storj/pkg/storj"
 	"storj.io/storj/pkg/utils"
 )
 

pkg/piecestore/psserver/psdb/psdb_test.go

@@ -17,9 +17,8 @@ import (
 	_ "github.com/mattn/go-sqlite3"
 
 	"storj.io/storj/internal/teststorj"
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/pb"
+	"storj.io/storj/pkg/storj"
 )
 
 var ctx = context.Background()

pkg/provider/identity.go

@@ -19,9 +19,8 @@ import (
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/peer"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/peertls"
+	"storj.io/storj/pkg/storj"
 	"storj.io/storj/pkg/utils"
 )
 

pkg/provider/utils.go

@@ -17,9 +17,8 @@ import (
 
 	"github.com/zeebo/errs"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/peertls"
+	"storj.io/storj/pkg/storj"
 )
 
 // TLSFilesStatus is the status of keys

pkg/satellite/satellitedb/companies.go

@@ -6,9 +6,8 @@ package satellitedb
 import (
 	"context"
 
-	"github.com/zeebo/errs"
-
 	"github.com/skyrings/skyring-common/tools/uuid"
+	"github.com/zeebo/errs"
 
 	"storj.io/storj/pkg/satellite"
 	"storj.io/storj/pkg/satellite/satellitedb/dbx"

pkg/satellite/satellitedb/db.go

@@ -6,7 +6,6 @@ package satellitedb
 import (
 	"storj.io/storj/internal/migrate"
 	"storj.io/storj/pkg/satellite"
-
 	"storj.io/storj/pkg/satellite/satellitedb/dbx"
 )
 

pkg/satellite/satellitedb/projects_test.go

@@ -7,13 +7,11 @@ import (
 	"testing"
 
 	"github.com/skyrings/skyring-common/tools/uuid"
-
-	"storj.io/storj/pkg/satellite/satellitedb/dbx"
-
 	"github.com/stretchr/testify/assert"
 
 	"storj.io/storj/internal/testcontext"
 	"storj.io/storj/pkg/satellite"
+	"storj.io/storj/pkg/satellite/satellitedb/dbx"
 )
 
 func TestProjectsRepository(t *testing.T) {

pkg/satellite/satellitedb/users_test.go

@@ -7,13 +7,12 @@ import (
 	"testing"
 	"time"
 
-	"storj.io/storj/internal/testcontext"
-	"storj.io/storj/pkg/satellite/satellitedb/dbx"
-
-	"storj.io/storj/pkg/satellite"
-
 	"github.com/skyrings/skyring-common/tools/uuid"
 	"github.com/stretchr/testify/assert"
+
+	"storj.io/storj/internal/testcontext"
+	"storj.io/storj/pkg/satellite"
+	"storj.io/storj/pkg/satellite/satellitedb/dbx"
 )
 
 func TestUserRepository(t *testing.T) {

pkg/satellite/satelliteweb/config.go

@@ -6,14 +6,12 @@ package satelliteweb
 import (
 	"context"
 
-	"storj.io/storj/pkg/satellite"
-	"storj.io/storj/pkg/satellite/satelliteauth"
-
+	"github.com/graphql-go/graphql"
 	"go.uber.org/zap"
 
-	"github.com/graphql-go/graphql"
-
 	"storj.io/storj/pkg/provider"
+	"storj.io/storj/pkg/satellite"
+	"storj.io/storj/pkg/satellite/satelliteauth"
 	"storj.io/storj/pkg/satellite/satellitedb"
 	"storj.io/storj/pkg/satellite/satelliteweb/satelliteql"
 	"storj.io/storj/pkg/utils"

pkg/satellite/satelliteweb/gateway.go

@@ -7,9 +7,8 @@ import (
 	"net/http"
 	"path/filepath"
 
-	"go.uber.org/zap"
-
 	"github.com/graphql-go/graphql"
+	"go.uber.org/zap"
 
 	"storj.io/storj/pkg/satellite"
 )

pkg/satellite/service.go

@@ -10,15 +10,13 @@ import (
 	"fmt"
 	"time"
 
-	"storj.io/storj/pkg/utils"
-
-	"go.uber.org/zap"
-
 	"github.com/skyrings/skyring-common/tools/uuid"
 	"github.com/zeebo/errs"
+	"go.uber.org/zap"
 
 	"storj.io/storj/pkg/auth"
 	"storj.io/storj/pkg/satellite/satelliteauth"
+	"storj.io/storj/pkg/utils"
 )
 
 // Service is handling accounts related logic

pkg/statdb/statdb.go

@@ -14,12 +14,11 @@ import (
 	"google.golang.org/grpc/status"
 	monkit "gopkg.in/spacemonkeygo/monkit.v2"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/internal/migrate"
 	"storj.io/storj/pkg/auth"
 	dbx "storj.io/storj/pkg/statdb/dbx"
 	pb "storj.io/storj/pkg/statdb/proto"
+	"storj.io/storj/pkg/storj"
 )
 
 var (

pkg/storage/ec/client.go

@@ -13,13 +13,12 @@ import (
 	"go.uber.org/zap"
 	"gopkg.in/spacemonkeygo/monkit.v2"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/eestream"
 	"storj.io/storj/pkg/pb"
 	"storj.io/storj/pkg/piecestore/psclient"
 	"storj.io/storj/pkg/provider"
 	"storj.io/storj/pkg/ranger"
+	"storj.io/storj/pkg/storj"
 	"storj.io/storj/pkg/transport"
 	"storj.io/storj/pkg/utils"
 )

pkg/storage/ec/client_test.go

@@ -20,7 +20,6 @@ import (
 	"github.com/vivint/infectious"
 
 	"storj.io/storj/internal/teststorj"
-
 	"storj.io/storj/pkg/eestream"
 	"storj.io/storj/pkg/pb"
 	"storj.io/storj/pkg/piecestore/psclient"

pkg/storage/segments/store_test.go

@@ -14,7 +14,6 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"storj.io/storj/internal/teststorj"
-
 	"storj.io/storj/pkg/eestream"
 	"storj.io/storj/pkg/eestream/mocks"
 	mock_overlay "storj.io/storj/pkg/overlay/mocks"

pkg/transport/transport.go

@@ -11,10 +11,9 @@ import (
 	"google.golang.org/grpc"
 	"gopkg.in/spacemonkeygo/monkit.v2"
 
-	"storj.io/storj/pkg/storj"
-
 	"storj.io/storj/pkg/pb"
 	"storj.io/storj/pkg/provider"
+	"storj.io/storj/pkg/storj"
 )
 
 var (

scripts/check-imports.go

@@ -0,0 +1,183 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+// +build ignore
+package main
+
+import (
+	"flag"
+	"fmt"
+	"go/ast"
+	"go/token"
+	"os"
+	"runtime"
+	"sort"
+	"strconv"
+	"strings"
+
+	"golang.org/x/tools/go/packages"
+)
+
+/*
+This tool verifies whether imports are divided into three blocks:
+
+	std packages
+	external packages
+	storj.io packages
+
+*/
+
+func main() {
+	flag.Parse()
+
+	pkgNames := flag.Args()
+	if len(pkgNames) == 0 {
+		pkgNames = []string{"."}
+	}
+
+	roots, err := packages.Load(&packages.Config{
+		Mode: packages.LoadAllSyntax,
+		Env:  os.Environ(),
+	}, pkgNames...)
+
+	if err != nil {
+		panic(err)
+	}
+
+	seen := map[*packages.Package]bool{}
+	pkgs := []*packages.Package{}
+
+	var visit func(*packages.Package)
+	visit = func(p *packages.Package) {
+		if seen[p] {
+			return
+		}
+		includeStd(p)
+
+		if strings.HasPrefix(p.ID, "storj.io") {
+			pkgs = append(pkgs, p)
+		}
+
+		seen[p] = true
+		for _, pkg := range p.Imports {
+			visit(pkg)
+		}
+	}
+	for _, pkg := range roots {
+		visit(pkg)
+	}
+
+	sort.Slice(pkgs, func(i, k int) bool { return pkgs[i].ID < pkgs[k].ID })
+	for _, pkg := range pkgs {
+		process(pkg)
+	}
+}
+
+func process(pkg *packages.Package) {
+	for i, file := range pkg.Syntax {
+		checkImports(pkg.Fset, pkg.CompiledGoFiles[i], file)
+	}
+}
+
+func checkImports(fset *token.FileSet, name string, f *ast.File) {
+	for _, d := range f.Decls {
+		d, ok := d.(*ast.GenDecl)
+		if !ok || d.Tok != token.IMPORT {
+			// Not an import declaration, so we're done.
+			// Imports are always first.
+			break
+		}
+
+		if !d.Lparen.IsValid() {
+			// Not a block: sorted by default.
+			continue
+		}
+
+		// Identify and sort runs of specs on successive lines.
+		lastGroup := 0
+		specgroups := [][]ast.Spec{}
+		for i, s := range d.Specs {
+			if i > lastGroup && fset.Position(s.Pos()).Line > 1+fset.Position(d.Specs[i-1].End()).Line {
+				// i begins a new run. End this one.
+				specgroups = append(specgroups, d.Specs[lastGroup:i])
+				lastGroup = i
+			}
+		}
+
+		specgroups = append(specgroups, d.Specs[lastGroup:])
+
+		if !correctOrder(specgroups) {
+			fmt.Println(name)
+		}
+	}
+}
+
+func correctOrder(specgroups [][]ast.Spec) bool {
+	if len(specgroups) == 0 {
+		return true
+	}
+
+	// remove std group from beginning
+	std, other, storj := countGroup(specgroups[0])
+	if std > 0 {
+		if other+storj != 0 {
+			return false
+		}
+		specgroups = specgroups[1:]
+	}
+	if len(specgroups) == 0 {
+		return true
+	}
+
+	// remove storj.io group from the end
+	std, other, storj = countGroup(specgroups[len(specgroups)-1])
+	if storj > 0 {
+		if std+other > 0 {
+			return false
+		}
+		specgroups = specgroups[:len(specgroups)-1]
+	}
+	if len(specgroups) == 0 {
+		return true
+	}
+
+	// check that we have a center group for misc stuff
+	if len(specgroups) != 1 {
+		return false
+	}
+
+	std, other, storj = countGroup(specgroups[0])
+	return other > 0 && std+storj == 0
+}
+
+func countGroup(p []ast.Spec) (std, other, storj int) {
+	for _, imp := range p {
+		imp := imp.(*ast.ImportSpec)
+		path, err := strconv.Unquote(imp.Path.Value)
+		if err != nil {
+			panic(err)
+		}
+		if strings.HasPrefix(path, "storj.io/") {
+			storj++
+		} else if stdlib[path] {
+			std++
+		} else {
+			other++
+		}
+	}
+	return std, other, storj
+}
+
+var root = runtime.GOROOT()
+var stdlib = map[string]bool{}
+
+func includeStd(p *packages.Package) {
+	if len(p.GoFiles) == 0 {
+		stdlib[p.ID] = true
+		return
+	}
+	if strings.HasPrefix(p.GoFiles[0], root) {
+		stdlib[p.ID] = true
+		return
+	}
+}