storj-sdk base code (#799)

cmd/storj-sdk/cancelable.go

@@ -0,0 +1,32 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package main
+
+import (
+	"context"
+	"os"
+	"os/signal"
+)
+
+// NewCLIContext creates a context that can be canceled with Ctrl-C
+func NewCLIContext(root context.Context) (context.Context, func()) {
+	// trap Ctrl+C and call cancel on the context
+	ctx, cancel := context.WithCancel(root)
+	signals := make(chan os.Signal, 1)
+
+	signal.Notify(signals, os.Interrupt)
+
+	go func() {
+		select {
+		case <-signals:
+			cancel()
+		case <-ctx.Done():
+		}
+	}()
+
+	return ctx, func() {
+		signal.Stop(signals)
+		cancel()
+	}
+}

cmd/storj-sdk/main.go

@@ -0,0 +1,102 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package main
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"storj.io/storj/internal/fpath"
+)
+
+// Flags contains different flags for commands
+type Flags struct {
+	Directory string
+
+	SatelliteCount   int
+	StorageNodeCount int
+	Identities       int
+}
+
+var printCommands bool
+
+func main() {
+	cobra.EnableCommandSorting = false
+
+	var flags Flags
+
+	rootCmd := &cobra.Command{
+		Use:   "storj-sdk",
+		Short: "Storj SDK",
+	}
+
+	defaultConfigDir := fpath.ApplicationDir("storj", "local-network")
+	configDir := defaultConfigDir
+	if os.Getenv("STORJ_NETWORK_DIR") != "" {
+		configDir = os.Getenv("STORJ_NETWORK_DIR")
+	}
+
+	rootCmd.PersistentFlags().StringVarP(&flags.Directory, "config-dir", "", configDir, "base project directory")
+
+	rootCmd.PersistentFlags().IntVarP(&flags.SatelliteCount, "satellites", "", 1, "number of satellites to start")
+	rootCmd.PersistentFlags().IntVarP(&flags.StorageNodeCount, "storage-nodes", "", 10, "number of storage nodes to start")
+	rootCmd.PersistentFlags().IntVarP(&flags.Identities, "identities", "", 10, "number of identities to create")
+
+	rootCmd.PersistentFlags().BoolVarP(&printCommands, "", "x", false, "print commands as they are run")
+
+	networkCmd := &cobra.Command{
+		Use:   "network",
+		Short: "local network for testing",
+	}
+
+	networkCmd.AddCommand(
+		&cobra.Command{
+			Use:   "run",
+			Short: "run network",
+			RunE: func(cmd *cobra.Command, args []string) (err error) {
+				return networkExec(&flags, args, "run")
+			},
+		}, &cobra.Command{
+			Use:   "setup",
+			Short: "setup network",
+			RunE: func(cmd *cobra.Command, args []string) (err error) {
+				return networkExec(&flags, args, "setup")
+			},
+		}, &cobra.Command{
+			Use:   "test <command>",
+			Short: "run command with an actual network",
+			Args:  cobra.MinimumNArgs(1),
+			RunE: func(cmd *cobra.Command, args []string) (err error) {
+				return networkTest(&flags, args[0], args[1:])
+			},
+		}, &cobra.Command{
+			Use:   "destroy",
+			Short: "destroys network if it exists",
+			RunE: func(cmd *cobra.Command, args []string) (err error) {
+				return networkDestroy(&flags, args)
+			},
+		},
+	)
+
+	testCmd := &cobra.Command{
+		Use:   "test <command>",
+		Short: "run command with an in-memory network",
+		Args:  cobra.MinimumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			return runTestPlanet(&flags, args[0], args[1:])
+		},
+	}
+
+	rootCmd.AddCommand(
+		networkCmd,
+		testCmd,
+	)
+
+	rootCmd.SilenceUsage = true
+	err := rootCmd.Execute()
+	if err != nil {
+		os.Exit(1)
+	}
+}

cmd/storj-sdk/network.go

@@ -0,0 +1,181 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/zeebo/errs"
+	"golang.org/x/sync/errgroup"
+
+	"storj.io/storj/internal/fpath"
+	"storj.io/storj/internal/processgroup"
+	"storj.io/storj/pkg/utils"
+)
+
+const folderPermissions = 0744
+
+func networkExec(flags *Flags, args []string, command string) error {
+	processes, err := newNetwork(flags.Directory, flags.SatelliteCount, flags.StorageNodeCount)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := NewCLIContext(context.Background())
+	defer cancel()
+
+	err = processes.Exec(ctx, command)
+	closeErr := processes.Close()
+
+	return errs.Combine(err, closeErr)
+}
+
+func networkTest(flags *Flags, command string, args []string) error {
+	processes, err := newNetwork(flags.Directory, flags.SatelliteCount, flags.StorageNodeCount)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := NewCLIContext(context.Background())
+
+	var group errgroup.Group
+	processes.Start(ctx, &group, "run")
+
+	time.Sleep(2 * time.Second)
+
+	cmd := exec.CommandContext(ctx, command, args...)
+	cmd.Env = append(os.Environ(), processes.Env()...)
+	stdout := processes.Output.Prefixed("test:out")
+	stderr := processes.Output.Prefixed("test:err")
+	cmd.Stdout, cmd.Stderr = stdout, stderr
+	processgroup.Setup(cmd)
+
+	if printCommands {
+		fmt.Fprintf(processes.Output, "exec: %v\n", strings.Join(cmd.Args, " "))
+	}
+	errRun := cmd.Run()
+
+	cancel()
+	return errs.Combine(errRun, processes.Close(), group.Wait())
+}
+
+func networkDestroy(flags *Flags, args []string) error {
+	if fpath.IsRoot(flags.Directory) {
+		return errors.New("safety check: disallowed to remove root directory " + flags.Directory)
+	}
+	if printCommands {
+		fmt.Println("sdk | exec: rm -rf", flags.Directory)
+	}
+	return os.RemoveAll(flags.Directory)
+}
+
+// newNetwork creates a default network
+func newNetwork(dir string, satelliteCount, storageNodeCount int) (*Processes, error) {
+	processes := NewProcesses()
+
+	const (
+		host            = "127.0.0.1"
+		gatewayPort     = 9000
+		satellitePort   = 10000
+		storageNodePort = 11000
+	)
+
+	defaultSatellite := net.JoinHostPort(host, strconv.Itoa(satellitePort+0))
+
+	arguments := func(name, command, addr string, rest ...string) []string {
+		return append([]string{
+			"--log.level", "debug",
+			"--config-dir", ".",
+			command,
+			"--server.address", addr,
+		}, rest...)
+	}
+
+	for i := 0; i < satelliteCount; i++ {
+		name := fmt.Sprintf("satellite/%d", i)
+
+		dir := filepath.Join(dir, "satellite", fmt.Sprint(i))
+		if err := os.MkdirAll(dir, folderPermissions); err != nil {
+			return nil, err
+		}
+
+		process, err := processes.New(name, "satellite", dir)
+		if err != nil {
+			return nil, utils.CombineErrors(err, processes.Close())
+		}
+		process.Info.Address = net.JoinHostPort(host, strconv.Itoa(satellitePort+i))
+
+		process.Arguments["setup"] = arguments(name, "setup", process.Info.Address)
+		process.Arguments["run"] = arguments(name, "run", process.Info.Address,
+			"--kademlia.bootstrap-addr", defaultSatellite,
+		)
+	}
+
+	gatewayArguments := func(name, command string, addr string, rest ...string) []string {
+		return append([]string{
+			"--log.level", "debug",
+			"--config-dir", ".",
+			command,
+			"--server.address", addr,
+		}, rest...)
+	}
+
+	for i := 0; i < satelliteCount; i++ {
+		name := fmt.Sprintf("gateway/%d", i)
+
+		dir := filepath.Join(dir, "gateway", fmt.Sprint(i))
+		if err := os.MkdirAll(dir, folderPermissions); err != nil {
+			return nil, err
+		}
+
+		satellite := processes.List[i]
+
+		process, err := processes.New(name, "gateway", dir)
+		if err != nil {
+			return nil, utils.CombineErrors(err, processes.Close())
+		}
+		process.Info.Address = net.JoinHostPort(host, strconv.Itoa(gatewayPort+i))
+
+		process.Arguments["setup"] = gatewayArguments(name, "setup", process.Info.Address,
+			"--satellite-addr", satellite.Info.Address,
+		)
+		process.Arguments["run"] = gatewayArguments(name, "run", process.Info.Address)
+	}
+
+	for i := 0; i < storageNodeCount; i++ {
+		name := fmt.Sprintf("storage/%d", i)
+
+		dir := filepath.Join(dir, "storage", fmt.Sprint(i))
+		if err := os.MkdirAll(dir, folderPermissions); err != nil {
+			return nil, err
+		}
+
+		process, err := processes.New(name, "storagenode", dir)
+		if err != nil {
+			return nil, utils.CombineErrors(err, processes.Close())
+		}
+		process.Info.Address = net.JoinHostPort(host, strconv.Itoa(storageNodePort+i))
+
+		process.Arguments["setup"] = arguments(name, "setup", process.Info.Address,
+			"--piecestore.agreementsender.overlay-addr", defaultSatellite,
+		)
+		process.Arguments["run"] = arguments(name, "run", process.Info.Address,
+			"--piecestore.agreementsender.overlay-addr", defaultSatellite,
+			"--kademlia.bootstrap-addr", defaultSatellite,
+			"--kademlia.operator.email", fmt.Sprintf("storage%d@example.com", i),
+			"--kademlia.operator.wallet", "0x0123456789012345678901234567890123456789",
+		)
+	}
+
+	return processes, nil
+}

cmd/storj-sdk/prefix.go

@@ -0,0 +1,124 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"sync"
+)
+
+// PrefixWriter writes to the specified output with prefixes.
+type PrefixWriter struct {
+	root    *prefixWriter
+	maxline int
+
+	mu  sync.Mutex
+	len int
+	dst io.Writer
+}
+
+// NewPrefixWriter creates a writer than can prefix all lines written to it.
+func NewPrefixWriter(defaultPrefix string, dst io.Writer) *PrefixWriter {
+	writer := &PrefixWriter{
+		maxline: 10000, // disable maxline cutting
+		dst:     dst,
+	}
+	writer.root = writer.Prefixed(defaultPrefix).(*prefixWriter)
+	return writer
+}
+
+// prefixWriter is the implementation that handles buffering and prefixing.
+type prefixWriter struct {
+	*PrefixWriter
+	prefix string
+	buffer []byte
+}
+
+// Prefixed returns a new writer that has writes with specified prefix.
+func (writer *PrefixWriter) Prefixed(prefix string) io.Writer {
+	writer.mu.Lock()
+	if len(prefix) > writer.len {
+		writer.len = len(prefix)
+	}
+	writer.mu.Unlock()
+
+	return &prefixWriter{writer, prefix, make([]byte, 0, writer.maxline)}
+}
+
+// Write implements io.Writer that prefixes lines.
+func (writer *PrefixWriter) Write(data []byte) (int, error) {
+	return writer.root.Write(data)
+}
+
+// Write implements io.Writer that prefixes lines
+func (writer *prefixWriter) Write(data []byte) (int, error) {
+	if len(data) == 0 {
+		return 0, nil
+	}
+
+	buffer := data
+
+	// buffer everything that hasn't been written yet
+	if len(writer.buffer) > 0 {
+		buffer = append(writer.buffer, data...)
+		defer func() {
+			writer.buffer = buffer
+		}()
+	} else {
+		defer func() {
+			if len(buffer) > 0 {
+				writer.buffer = append(writer.buffer, buffer...)
+			}
+		}()
+	}
+
+	writer.mu.Lock()
+	defer writer.mu.Unlock()
+
+	prefix := writer.prefix
+	for len(buffer) > 0 {
+		pos := bytes.IndexByte(buffer, '\n') + 1
+		breakline := false
+		if pos <= 0 {
+			if len(buffer) < writer.maxline {
+				return len(data), nil
+			}
+		}
+		if pos < 0 || pos > writer.maxline {
+			pos = writer.maxline
+			for p := pos; p >= writer.maxline*2/3; p-- {
+				if buffer[p] == ' ' {
+					pos = p
+					break
+				}
+			}
+			breakline = true
+		}
+
+		_, err := fmt.Fprintf(writer.dst, "%-*s | ", writer.len, prefix)
+		if err != nil {
+			return len(data), err
+		}
+
+		_, err = writer.dst.Write(buffer[:pos])
+		buffer = buffer[pos:]
+
+		if err != nil {
+			return len(data), err
+		}
+
+		if breakline {
+			_, err = writer.dst.Write([]byte{'\n'})
+			if err != nil {
+				return len(data), err
+			}
+		}
+
+		prefix = ""
+	}
+
+	return len(data), nil
+}

cmd/storj-sdk/process.go

@@ -0,0 +1,187 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/sync/errgroup"
+
+	"storj.io/storj/internal/processgroup"
+	"storj.io/storj/pkg/utils"
+)
+
+// Processes contains list of processes
+type Processes struct {
+	Output *PrefixWriter
+	List   []*Process
+}
+
+// NewProcesses returns a group of processes
+func NewProcesses() *Processes {
+	return &Processes{
+		Output: NewPrefixWriter("sdk", os.Stdout),
+		List:   nil,
+	}
+}
+
+// Exec executes a command on all processes
+func (processes *Processes) Exec(ctx context.Context, command string) error {
+	var group errgroup.Group
+	processes.Start(ctx, &group, command)
+	return group.Wait()
+}
+
+// Start executes all processes using specified errgroup.Group
+func (processes *Processes) Start(ctx context.Context, group *errgroup.Group, command string) {
+	for _, p := range processes.List {
+		process := p
+		group.Go(func() error {
+			return process.Exec(ctx, command)
+		})
+	}
+}
+
+// Env returns environment flags for other nodes
+func (processes *Processes) Env() []string {
+	var env []string
+	for _, process := range processes.List {
+		env = append(env, process.Info.Env()...)
+	}
+	return env
+}
+
+// Close closes all the processes and their resources
+func (processes *Processes) Close() error {
+	var errs []error
+	for _, process := range processes.List {
+		err := process.Close()
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+	return utils.CombineErrors(errs...)
+}
+
+// ProcessInfo represents public information about the process
+type ProcessInfo struct {
+	Name      string
+	ID        string
+	Address   string
+	Directory string
+}
+
+// Env returns process flags
+func (info *ProcessInfo) Env() []string {
+	name := strings.ToUpper(info.Name)
+
+	name = strings.Map(func(r rune) rune {
+		switch {
+		case '0' <= r && r <= '9':
+			return r
+		case 'a' <= r && r <= 'z':
+			return r
+		case 'A' <= r && r <= 'Z':
+			return r
+		default:
+			return '_'
+		}
+	}, name)
+
+	var env []string
+	if info.ID != "" {
+		env = append(env, name+"_ID="+info.ID)
+	}
+	if info.Address != "" {
+		env = append(env, name+"_ADDR="+info.Address)
+	}
+	if info.Directory != "" {
+		env = append(env, name+"_DIR="+info.Directory)
+	}
+	return env
+}
+
+// Process is a type for monitoring the process
+type Process struct {
+	processes *Processes
+
+	Name       string
+	Directory  string
+	Executable string
+
+	Info ProcessInfo
+
+	Arguments map[string][]string
+
+	stdout io.Writer
+	stderr io.Writer
+
+	outfile *os.File
+	errfile *os.File
+}
+
+// New creates a process which can be run in the specified directory
+func (processes *Processes) New(name, executable, directory string) (*Process, error) {
+	outfile, err1 := os.OpenFile(filepath.Join(directory, "stderr.log"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	errfile, err2 := os.OpenFile(filepath.Join(directory, "stdout.log"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+
+	err := utils.CombineErrors(err1, err2)
+	if err != nil {
+		return nil, err
+	}
+
+	output := processes.Output.Prefixed(name)
+
+	process := &Process{
+		processes: processes,
+
+		Name:       name,
+		Directory:  directory,
+		Executable: executable,
+
+		Info: ProcessInfo{
+			Name:      name,
+			Directory: directory,
+		},
+		Arguments: map[string][]string{},
+
+		stdout: io.MultiWriter(output, outfile),
+		stderr: io.MultiWriter(output, errfile),
+
+		outfile: outfile,
+		errfile: errfile,
+	}
+
+	processes.List = append(processes.List, process)
+
+	return process, nil
+}
+
+// Exec runs the process using the arguments for a given command
+func (process *Process) Exec(ctx context.Context, command string) error {
+	cmd := exec.CommandContext(ctx, process.Executable, process.Arguments[command]...)
+	cmd.Dir = process.Directory
+	cmd.Stdout, cmd.Stderr = process.stdout, process.stderr
+
+	processgroup.Setup(cmd)
+
+	if printCommands {
+		fmt.Fprintf(process.processes.Output, "exec: %v\n", strings.Join(cmd.Args, " "))
+	}
+	return cmd.Run()
+}
+
+// Close closes process resources
+func (process *Process) Close() error {
+	return utils.CombineErrors(
+		process.outfile.Close(),
+		process.errfile.Close(),
+	)
+}

cmd/storj-sdk/testplanet.go

@@ -0,0 +1,94 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/pem"
+	"fmt"
+	"os"
+	"os/exec"
+	"strconv"
+	"time"
+
+	"go.uber.org/zap"
+
+	"storj.io/storj/internal/processgroup"
+	"storj.io/storj/internal/testplanet"
+	"storj.io/storj/pkg/peertls"
+	"storj.io/storj/pkg/utils"
+)
+
+func runTestPlanet(flags *Flags, command string, args []string) error {
+	ctx, cancel := NewCLIContext(context.Background())
+	defer cancel()
+
+	planet, err := testplanet.NewWithLogger(zap.L(), flags.SatelliteCount, flags.StorageNodeCount, 0)
+	if err != nil {
+		return err
+	}
+
+	planet.Start(ctx)
+	// wait a bit for kademlia to start
+	time.Sleep(time.Second * 2)
+
+	var env = os.Environ()
+
+	// add satellites to environment
+	for i, satellite := range planet.Satellites {
+		env = append(env, (&ProcessInfo{
+			Name:    "satellite/" + strconv.Itoa(i),
+			ID:      satellite.ID().String(),
+			Address: satellite.Addr(),
+		}).Env()...)
+	}
+
+	// add storage nodes to environment
+	for i, storage := range planet.StorageNodes {
+		env = append(env, (&ProcessInfo{
+			Name:    "storage/" + strconv.Itoa(i),
+			ID:      storage.ID().String(),
+			Address: storage.Addr(),
+		}).Env()...)
+	}
+
+	// add additional identities to the environment
+	for i := 0; i < flags.Identities; i++ {
+		identity, err := planet.NewIdentity()
+		if err != nil {
+			return utils.CombineErrors(err, planet.Shutdown())
+		}
+
+		var chainPEM bytes.Buffer
+		errLeaf := pem.Encode(&chainPEM, peertls.NewCertBlock(identity.Leaf.Raw))
+		errCA := pem.Encode(&chainPEM, peertls.NewCertBlock(identity.CA.Raw))
+		if errLeaf != nil || errCA != nil {
+			return utils.CombineErrors(errLeaf, errCA, planet.Shutdown())
+		}
+
+		var key bytes.Buffer
+		errKey := peertls.WriteKey(&key, identity.Key)
+		if errKey != nil {
+			return utils.CombineErrors(errKey, planet.Shutdown())
+		}
+
+		env = append(env,
+			fmt.Sprintf("IDENTITY_%d_ID=%v", i, identity.ID.String()),
+			fmt.Sprintf("IDENTITY_%d_KEY=%v", i, base64.StdEncoding.EncodeToString(key.Bytes())),
+			fmt.Sprintf("IDENTITY_%d_CHAIN=%v", i, base64.StdEncoding.EncodeToString(chainPEM.Bytes())),
+		)
+	}
+
+	// run the specified program
+	cmd := exec.CommandContext(ctx, command, args...)
+	cmd.Env = env
+	cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
+	processgroup.Setup(cmd)
+
+	errRun := cmd.Run()
+
+	return utils.CombineErrors(errRun, planet.Shutdown())
+}

internal/fpath/os.go

@@ -0,0 +1,101 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package fpath
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"storj.io/storj/pkg/utils"
+)
+
+// IsRoot returns whether path is the root directory
+func IsRoot(path string) bool {
+	abs, err := filepath.Abs(path)
+	if err == nil {
+		path = abs
+	}
+
+	return filepath.Dir(path) == path
+}
+
+// ApplicationDir returns best base directory for specific OS
+func ApplicationDir(subdir ...string) string {
+	for i := range subdir {
+		if runtime.GOOS == "windows" || runtime.GOOS == "darwin" {
+			subdir[i] = strings.Title(subdir[i])
+		} else {
+			subdir[i] = strings.ToLower(subdir[i])
+		}
+	}
+	var appdir string
+	home := os.Getenv("HOME")
+
+	switch runtime.GOOS {
+	case "windows":
+		// Windows standards: https://msdn.microsoft.com/en-us/library/windows/apps/hh465094.aspx?f=255&MSPPError=-2147217396
+		for _, env := range []string{"AppData", "AppDataLocal", "UserProfile", "Home"} {
+			val := os.Getenv(env)
+			if val != "" {
+				appdir = val
+				break
+			}
+		}
+	case "darwin":
+		// Mac standards: https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/MacOSXDirectories/MacOSXDirectories.html
+		appdir = filepath.Join(home, "Library", "Application Support")
+	case "linux":
+		fallthrough
+	default:
+		// Linux standards: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html
+		appdir = os.Getenv("XDG_DATA_HOME")
+		if appdir == "" && home != "" {
+			appdir = filepath.Join(home, ".local", "share")
+		}
+	}
+	return filepath.Join(append([]string{appdir}, subdir...)...)
+}
+
+// IsValidSetupDir checks if directory is valid for setup configuration
+func IsValidSetupDir(name string) (ok bool, err error) {
+	_, err = os.Stat(name)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return true, err
+		}
+		return false, err
+	}
+
+	f, err := os.Open(name)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		err = utils.CombineErrors(err, f.Close())
+	}()
+
+	for {
+		var filenames []string
+		filenames, err = f.Readdirnames(100)
+		if err == io.EOF {
+			// nothing more
+			return true, nil
+		} else if err != nil {
+			// something went wrong
+			return false, err
+		}
+
+		for _, filename := range filenames {
+			// allow log files to exist in the folder
+			if strings.EqualFold(filepath.Ext(filename), ".log") {
+				continue
+			}
+
+			return false, nil
+		}
+	}
+}

internal/fpath/path.go

@@ -6,12 +6,9 @@ package fpath
 import (
 	"errors"
 	"fmt"
-	"io"
 	"net/url"
-	"os"
 	"path"
 	"path/filepath"
-	"runtime"
 	"strings"
 )
 
@@ -123,64 +120,3 @@ func (p FPath) IsLocal() bool {
 func (p FPath) String() string {
 	return p.original
 }
-
-// ApplicationDir returns best base directory for specific OS
-func ApplicationDir(subdir ...string) string {
-	for i := range subdir {
-		if runtime.GOOS == "windows" || runtime.GOOS == "darwin" {
-			subdir[i] = strings.Title(subdir[i])
-		} else {
-			subdir[i] = strings.ToLower(subdir[i])
-		}
-	}
-	var appdir string
-	home := os.Getenv("HOME")
-
-	switch runtime.GOOS {
-	case "windows":
-		// Windows standards: https://msdn.microsoft.com/en-us/library/windows/apps/hh465094.aspx?f=255&MSPPError=-2147217396
-		for _, env := range []string{"AppData", "AppDataLocal", "UserProfile", "Home"} {
-			val := os.Getenv(env)
-			if val != "" {
-				appdir = val
-				break
-			}
-		}
-	case "darwin":
-		// Mac standards: https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/MacOSXDirectories/MacOSXDirectories.html
-		appdir = filepath.Join(home, "Library", "Application Support")
-	case "linux":
-		fallthrough
-	default:
-		// Linux standards: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html
-		appdir = os.Getenv("XDG_DATA_HOME")
-		if appdir == "" && home != "" {
-			appdir = filepath.Join(home, ".local", "share")
-		}
-	}
-	return filepath.Join(append([]string{appdir}, subdir...)...)
-}
-
-// IsValidSetupDir checks if directory is valid for setup configuration
-func IsValidSetupDir(name string) (bool, error) {
-	_, err := os.Stat(name)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return true, err
-		}
-		return false, err
-	}
-
-	f, err := os.Open(name)
-	if err != nil {
-		return false, err
-	}
-	defer func() { _ = f.Close() }()
-
-	_, err = f.Readdir(1)
-	if err == io.EOF {
-		// is empty
-		return true, nil
-	}
-	return false, err
-}

internal/testplanet/node.go

@@ -47,7 +47,7 @@ type Node struct {
 
 // newNode creates a new node.
 func (planet *Planet) newNode(name string, nodeType pb.NodeType) (*Node, error) {
-	identity, err := planet.newIdentity()
+	identity, err := planet.NewIdentity()
 	if err != nil {
 		return nil, err
 	}

internal/testplanet/planet.go

@@ -56,6 +56,11 @@ func New(t zaptest.TestingT, satelliteCount, storageNodeCount, uplinkCount int)
 		log = zaptest.NewLogger(t)
 	}
 
+	return NewWithLogger(log, satelliteCount, storageNodeCount, uplinkCount)
+}
+
+// NewWithLogger creates a new full system with the given number of nodes.
+func NewWithLogger(log *zap.Logger, satelliteCount, storageNodeCount, uplinkCount int) (*Planet, error) {
 	planet := &Planet{
 		log:        log,
 		identities: NewPregeneratedIdentities(),
@@ -232,8 +237,8 @@ func (planet *Planet) newNodes(prefix string, count int, nodeType pb.NodeType) (
 	return xs, nil
 }
 
-// newIdentity creates a new identity for a node
+// NewIdentity creates a new identity for a node
-func (planet *Planet) newIdentity() (*provider.FullIdentity, error) {
+func (planet *Planet) NewIdentity() (*provider.FullIdentity, error) {
 	return planet.identities.NewIdentity()
 }
 

pkg/audit/service.go

@@ -25,7 +25,7 @@ type Service struct {
 
 // Config contains configurable values for audit service
 type Config struct {
-	APIKey           string        `help:"APIKey to access the statdb" default:"abc123"`
+	APIKey           string        `help:"APIKey to access the statdb" default:""`
 	SatelliteAddr    string        `help:"address to contact services on the satellite"`
 	MaxRetriesStatDB int           `help:"max number of times to attempt updating a statdb batch" default:"3"`
 	Interval         time.Duration `help:"how frequently segments are audited" default:"30s"`

pkg/kademlia/peer_discovery.go

@@ -96,7 +96,12 @@ func (lookup *peerDiscovery) Run(ctx context.Context) (target *pb.Node, err erro
 					// ok := lookup.queue.Reinsert(lookup.target, next, lookup.opts.retries)
 					ok := false
 					if !ok {
-						lookup.log.Debug("connecting to node failed", zap.Any("target", lookup.target), zap.Any("dial", next.Id), zap.Error(err))
+						lookup.log.Debug("connecting to node failed",
+							zap.Any("target", lookup.target),
+							zap.Any("dial", next.Id),
+							zap.Any("dial-address", next.Address.Address),
+							zap.Error(err),
+						)
 					}
 				}
 

pkg/process/logging.go

@@ -20,8 +20,7 @@ var (
 	logCaller   = flag.Bool("log.caller", false, "if true, log function filename and line number")
 	logStack    = flag.Bool("log.stack", false, "if true, log stack traces")
 	logEncoding = flag.String("log.encoding", "console", "configures log encoding. can either be 'console' or 'json'")
-	logOutput   = flag.String("log.output", "stderr",
+	logOutput   = flag.String("log.output", "stderr", "can be stdout, stderr, or a filename")
-		"can be stdout, stderr, or a filename")
 )
 
 func newLogger() (*zap.Logger, error) {