From 89db0fe9f5d9ec624db7dfc59243b6e36cc6e440 Mon Sep 17 00:00:00 2001 From: Egon Elbre Date: Wed, 2 Jan 2019 20:07:49 +0200 Subject: [PATCH] storj-sdk base code (#799) --- cmd/storj-sdk/cancelable.go | 32 ++++++ cmd/storj-sdk/main.go | 102 ++++++++++++++++++ cmd/storj-sdk/network.go | 181 +++++++++++++++++++++++++++++++ cmd/storj-sdk/prefix.go | 124 ++++++++++++++++++++++ cmd/storj-sdk/process.go | 187 +++++++++++++++++++++++++++++++++ cmd/storj-sdk/testplanet.go | 94 +++++++++++++++++ internal/fpath/os.go | 101 ++++++++++++++++++ internal/fpath/path.go | 64 ----------- internal/testplanet/node.go | 2 +- internal/testplanet/planet.go | 9 +- pkg/audit/service.go | 2 +- pkg/kademlia/peer_discovery.go | 7 +- pkg/process/logging.go | 3 +- 13 files changed, 837 insertions(+), 71 deletions(-) create mode 100644 cmd/storj-sdk/cancelable.go create mode 100644 cmd/storj-sdk/main.go create mode 100644 cmd/storj-sdk/network.go create mode 100644 cmd/storj-sdk/prefix.go create mode 100644 cmd/storj-sdk/process.go create mode 100644 cmd/storj-sdk/testplanet.go create mode 100644 internal/fpath/os.go diff --git a/cmd/storj-sdk/cancelable.go b/cmd/storj-sdk/cancelable.go new file mode 100644 index 000000000..f0ed2dbf3 --- /dev/null +++ b/cmd/storj-sdk/cancelable.go @@ -0,0 +1,32 @@ +// Copyright (C) 2018 Storj Labs, Inc. +// See LICENSE for copying information. + +package main + +import ( + "context" + "os" + "os/signal" +) + +// NewCLIContext creates a context that can be canceled with Ctrl-C +func NewCLIContext(root context.Context) (context.Context, func()) { + // trap Ctrl+C and call cancel on the context + ctx, cancel := context.WithCancel(root) + signals := make(chan os.Signal, 1) + + signal.Notify(signals, os.Interrupt) + + go func() { + select { + case <-signals: + cancel() + case <-ctx.Done(): + } + }() + + return ctx, func() { + signal.Stop(signals) + cancel() + } +} diff --git a/cmd/storj-sdk/main.go b/cmd/storj-sdk/main.go new file mode 100644 index 000000000..c2ce5e52e --- /dev/null +++ b/cmd/storj-sdk/main.go @@ -0,0 +1,102 @@ +// Copyright (C) 2018 Storj Labs, Inc. +// See LICENSE for copying information. + +package main + +import ( + "os" + + "github.com/spf13/cobra" + + "storj.io/storj/internal/fpath" +) + +// Flags contains different flags for commands +type Flags struct { + Directory string + + SatelliteCount int + StorageNodeCount int + Identities int +} + +var printCommands bool + +func main() { + cobra.EnableCommandSorting = false + + var flags Flags + + rootCmd := &cobra.Command{ + Use: "storj-sdk", + Short: "Storj SDK", + } + + defaultConfigDir := fpath.ApplicationDir("storj", "local-network") + configDir := defaultConfigDir + if os.Getenv("STORJ_NETWORK_DIR") != "" { + configDir = os.Getenv("STORJ_NETWORK_DIR") + } + + rootCmd.PersistentFlags().StringVarP(&flags.Directory, "config-dir", "", configDir, "base project directory") + + rootCmd.PersistentFlags().IntVarP(&flags.SatelliteCount, "satellites", "", 1, "number of satellites to start") + rootCmd.PersistentFlags().IntVarP(&flags.StorageNodeCount, "storage-nodes", "", 10, "number of storage nodes to start") + rootCmd.PersistentFlags().IntVarP(&flags.Identities, "identities", "", 10, "number of identities to create") + + rootCmd.PersistentFlags().BoolVarP(&printCommands, "", "x", false, "print commands as they are run") + + networkCmd := &cobra.Command{ + Use: "network", + Short: "local network for testing", + } + + networkCmd.AddCommand( + &cobra.Command{ + Use: "run", + Short: "run network", + RunE: func(cmd *cobra.Command, args []string) (err error) { + return networkExec(&flags, args, "run") + }, + }, &cobra.Command{ + Use: "setup", + Short: "setup network", + RunE: func(cmd *cobra.Command, args []string) (err error) { + return networkExec(&flags, args, "setup") + }, + }, &cobra.Command{ + Use: "test ", + Short: "run command with an actual network", + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) (err error) { + return networkTest(&flags, args[0], args[1:]) + }, + }, &cobra.Command{ + Use: "destroy", + Short: "destroys network if it exists", + RunE: func(cmd *cobra.Command, args []string) (err error) { + return networkDestroy(&flags, args) + }, + }, + ) + + testCmd := &cobra.Command{ + Use: "test ", + Short: "run command with an in-memory network", + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) (err error) { + return runTestPlanet(&flags, args[0], args[1:]) + }, + } + + rootCmd.AddCommand( + networkCmd, + testCmd, + ) + + rootCmd.SilenceUsage = true + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} diff --git a/cmd/storj-sdk/network.go b/cmd/storj-sdk/network.go new file mode 100644 index 000000000..8c63b76b4 --- /dev/null +++ b/cmd/storj-sdk/network.go @@ -0,0 +1,181 @@ +// Copyright (C) 2018 Storj Labs, Inc. +// See LICENSE for copying information. + +package main + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/zeebo/errs" + "golang.org/x/sync/errgroup" + + "storj.io/storj/internal/fpath" + "storj.io/storj/internal/processgroup" + "storj.io/storj/pkg/utils" +) + +const folderPermissions = 0744 + +func networkExec(flags *Flags, args []string, command string) error { + processes, err := newNetwork(flags.Directory, flags.SatelliteCount, flags.StorageNodeCount) + if err != nil { + return err + } + + ctx, cancel := NewCLIContext(context.Background()) + defer cancel() + + err = processes.Exec(ctx, command) + closeErr := processes.Close() + + return errs.Combine(err, closeErr) +} + +func networkTest(flags *Flags, command string, args []string) error { + processes, err := newNetwork(flags.Directory, flags.SatelliteCount, flags.StorageNodeCount) + if err != nil { + return err + } + + ctx, cancel := NewCLIContext(context.Background()) + + var group errgroup.Group + processes.Start(ctx, &group, "run") + + time.Sleep(2 * time.Second) + + cmd := exec.CommandContext(ctx, command, args...) + cmd.Env = append(os.Environ(), processes.Env()...) + stdout := processes.Output.Prefixed("test:out") + stderr := processes.Output.Prefixed("test:err") + cmd.Stdout, cmd.Stderr = stdout, stderr + processgroup.Setup(cmd) + + if printCommands { + fmt.Fprintf(processes.Output, "exec: %v\n", strings.Join(cmd.Args, " ")) + } + errRun := cmd.Run() + + cancel() + return errs.Combine(errRun, processes.Close(), group.Wait()) +} + +func networkDestroy(flags *Flags, args []string) error { + if fpath.IsRoot(flags.Directory) { + return errors.New("safety check: disallowed to remove root directory " + flags.Directory) + } + if printCommands { + fmt.Println("sdk | exec: rm -rf", flags.Directory) + } + return os.RemoveAll(flags.Directory) +} + +// newNetwork creates a default network +func newNetwork(dir string, satelliteCount, storageNodeCount int) (*Processes, error) { + processes := NewProcesses() + + const ( + host = "127.0.0.1" + gatewayPort = 9000 + satellitePort = 10000 + storageNodePort = 11000 + ) + + defaultSatellite := net.JoinHostPort(host, strconv.Itoa(satellitePort+0)) + + arguments := func(name, command, addr string, rest ...string) []string { + return append([]string{ + "--log.level", "debug", + "--config-dir", ".", + command, + "--server.address", addr, + }, rest...) + } + + for i := 0; i < satelliteCount; i++ { + name := fmt.Sprintf("satellite/%d", i) + + dir := filepath.Join(dir, "satellite", fmt.Sprint(i)) + if err := os.MkdirAll(dir, folderPermissions); err != nil { + return nil, err + } + + process, err := processes.New(name, "satellite", dir) + if err != nil { + return nil, utils.CombineErrors(err, processes.Close()) + } + process.Info.Address = net.JoinHostPort(host, strconv.Itoa(satellitePort+i)) + + process.Arguments["setup"] = arguments(name, "setup", process.Info.Address) + process.Arguments["run"] = arguments(name, "run", process.Info.Address, + "--kademlia.bootstrap-addr", defaultSatellite, + ) + } + + gatewayArguments := func(name, command string, addr string, rest ...string) []string { + return append([]string{ + "--log.level", "debug", + "--config-dir", ".", + command, + "--server.address", addr, + }, rest...) + } + + for i := 0; i < satelliteCount; i++ { + name := fmt.Sprintf("gateway/%d", i) + + dir := filepath.Join(dir, "gateway", fmt.Sprint(i)) + if err := os.MkdirAll(dir, folderPermissions); err != nil { + return nil, err + } + + satellite := processes.List[i] + + process, err := processes.New(name, "gateway", dir) + if err != nil { + return nil, utils.CombineErrors(err, processes.Close()) + } + process.Info.Address = net.JoinHostPort(host, strconv.Itoa(gatewayPort+i)) + + process.Arguments["setup"] = gatewayArguments(name, "setup", process.Info.Address, + "--satellite-addr", satellite.Info.Address, + ) + process.Arguments["run"] = gatewayArguments(name, "run", process.Info.Address) + } + + for i := 0; i < storageNodeCount; i++ { + name := fmt.Sprintf("storage/%d", i) + + dir := filepath.Join(dir, "storage", fmt.Sprint(i)) + if err := os.MkdirAll(dir, folderPermissions); err != nil { + return nil, err + } + + process, err := processes.New(name, "storagenode", dir) + if err != nil { + return nil, utils.CombineErrors(err, processes.Close()) + } + process.Info.Address = net.JoinHostPort(host, strconv.Itoa(storageNodePort+i)) + + process.Arguments["setup"] = arguments(name, "setup", process.Info.Address, + "--piecestore.agreementsender.overlay-addr", defaultSatellite, + ) + process.Arguments["run"] = arguments(name, "run", process.Info.Address, + "--piecestore.agreementsender.overlay-addr", defaultSatellite, + "--kademlia.bootstrap-addr", defaultSatellite, + "--kademlia.operator.email", fmt.Sprintf("storage%d@example.com", i), + "--kademlia.operator.wallet", "0x0123456789012345678901234567890123456789", + ) + } + + return processes, nil +} diff --git a/cmd/storj-sdk/prefix.go b/cmd/storj-sdk/prefix.go new file mode 100644 index 000000000..83034c5f3 --- /dev/null +++ b/cmd/storj-sdk/prefix.go @@ -0,0 +1,124 @@ +// Copyright (C) 2018 Storj Labs, Inc. +// See LICENSE for copying information. + +package main + +import ( + "bytes" + "fmt" + "io" + "sync" +) + +// PrefixWriter writes to the specified output with prefixes. +type PrefixWriter struct { + root *prefixWriter + maxline int + + mu sync.Mutex + len int + dst io.Writer +} + +// NewPrefixWriter creates a writer than can prefix all lines written to it. +func NewPrefixWriter(defaultPrefix string, dst io.Writer) *PrefixWriter { + writer := &PrefixWriter{ + maxline: 10000, // disable maxline cutting + dst: dst, + } + writer.root = writer.Prefixed(defaultPrefix).(*prefixWriter) + return writer +} + +// prefixWriter is the implementation that handles buffering and prefixing. +type prefixWriter struct { + *PrefixWriter + prefix string + buffer []byte +} + +// Prefixed returns a new writer that has writes with specified prefix. +func (writer *PrefixWriter) Prefixed(prefix string) io.Writer { + writer.mu.Lock() + if len(prefix) > writer.len { + writer.len = len(prefix) + } + writer.mu.Unlock() + + return &prefixWriter{writer, prefix, make([]byte, 0, writer.maxline)} +} + +// Write implements io.Writer that prefixes lines. +func (writer *PrefixWriter) Write(data []byte) (int, error) { + return writer.root.Write(data) +} + +// Write implements io.Writer that prefixes lines +func (writer *prefixWriter) Write(data []byte) (int, error) { + if len(data) == 0 { + return 0, nil + } + + buffer := data + + // buffer everything that hasn't been written yet + if len(writer.buffer) > 0 { + buffer = append(writer.buffer, data...) + defer func() { + writer.buffer = buffer + }() + } else { + defer func() { + if len(buffer) > 0 { + writer.buffer = append(writer.buffer, buffer...) + } + }() + } + + writer.mu.Lock() + defer writer.mu.Unlock() + + prefix := writer.prefix + for len(buffer) > 0 { + pos := bytes.IndexByte(buffer, '\n') + 1 + breakline := false + if pos <= 0 { + if len(buffer) < writer.maxline { + return len(data), nil + } + } + if pos < 0 || pos > writer.maxline { + pos = writer.maxline + for p := pos; p >= writer.maxline*2/3; p-- { + if buffer[p] == ' ' { + pos = p + break + } + } + breakline = true + } + + _, err := fmt.Fprintf(writer.dst, "%-*s | ", writer.len, prefix) + if err != nil { + return len(data), err + } + + _, err = writer.dst.Write(buffer[:pos]) + buffer = buffer[pos:] + + if err != nil { + return len(data), err + } + + if breakline { + _, err = writer.dst.Write([]byte{'\n'}) + if err != nil { + return len(data), err + } + } + + prefix = "" + } + + return len(data), nil +} diff --git a/cmd/storj-sdk/process.go b/cmd/storj-sdk/process.go new file mode 100644 index 000000000..ed1a0e89f --- /dev/null +++ b/cmd/storj-sdk/process.go @@ -0,0 +1,187 @@ +// Copyright (C) 2018 Storj Labs, Inc. +// See LICENSE for copying information. + +package main + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + + "golang.org/x/sync/errgroup" + + "storj.io/storj/internal/processgroup" + "storj.io/storj/pkg/utils" +) + +// Processes contains list of processes +type Processes struct { + Output *PrefixWriter + List []*Process +} + +// NewProcesses returns a group of processes +func NewProcesses() *Processes { + return &Processes{ + Output: NewPrefixWriter("sdk", os.Stdout), + List: nil, + } +} + +// Exec executes a command on all processes +func (processes *Processes) Exec(ctx context.Context, command string) error { + var group errgroup.Group + processes.Start(ctx, &group, command) + return group.Wait() +} + +// Start executes all processes using specified errgroup.Group +func (processes *Processes) Start(ctx context.Context, group *errgroup.Group, command string) { + for _, p := range processes.List { + process := p + group.Go(func() error { + return process.Exec(ctx, command) + }) + } +} + +// Env returns environment flags for other nodes +func (processes *Processes) Env() []string { + var env []string + for _, process := range processes.List { + env = append(env, process.Info.Env()...) + } + return env +} + +// Close closes all the processes and their resources +func (processes *Processes) Close() error { + var errs []error + for _, process := range processes.List { + err := process.Close() + if err != nil { + errs = append(errs, err) + } + } + return utils.CombineErrors(errs...) +} + +// ProcessInfo represents public information about the process +type ProcessInfo struct { + Name string + ID string + Address string + Directory string +} + +// Env returns process flags +func (info *ProcessInfo) Env() []string { + name := strings.ToUpper(info.Name) + + name = strings.Map(func(r rune) rune { + switch { + case '0' <= r && r <= '9': + return r + case 'a' <= r && r <= 'z': + return r + case 'A' <= r && r <= 'Z': + return r + default: + return '_' + } + }, name) + + var env []string + if info.ID != "" { + env = append(env, name+"_ID="+info.ID) + } + if info.Address != "" { + env = append(env, name+"_ADDR="+info.Address) + } + if info.Directory != "" { + env = append(env, name+"_DIR="+info.Directory) + } + return env +} + +// Process is a type for monitoring the process +type Process struct { + processes *Processes + + Name string + Directory string + Executable string + + Info ProcessInfo + + Arguments map[string][]string + + stdout io.Writer + stderr io.Writer + + outfile *os.File + errfile *os.File +} + +// New creates a process which can be run in the specified directory +func (processes *Processes) New(name, executable, directory string) (*Process, error) { + outfile, err1 := os.OpenFile(filepath.Join(directory, "stderr.log"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + errfile, err2 := os.OpenFile(filepath.Join(directory, "stdout.log"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + + err := utils.CombineErrors(err1, err2) + if err != nil { + return nil, err + } + + output := processes.Output.Prefixed(name) + + process := &Process{ + processes: processes, + + Name: name, + Directory: directory, + Executable: executable, + + Info: ProcessInfo{ + Name: name, + Directory: directory, + }, + Arguments: map[string][]string{}, + + stdout: io.MultiWriter(output, outfile), + stderr: io.MultiWriter(output, errfile), + + outfile: outfile, + errfile: errfile, + } + + processes.List = append(processes.List, process) + + return process, nil +} + +// Exec runs the process using the arguments for a given command +func (process *Process) Exec(ctx context.Context, command string) error { + cmd := exec.CommandContext(ctx, process.Executable, process.Arguments[command]...) + cmd.Dir = process.Directory + cmd.Stdout, cmd.Stderr = process.stdout, process.stderr + + processgroup.Setup(cmd) + + if printCommands { + fmt.Fprintf(process.processes.Output, "exec: %v\n", strings.Join(cmd.Args, " ")) + } + return cmd.Run() +} + +// Close closes process resources +func (process *Process) Close() error { + return utils.CombineErrors( + process.outfile.Close(), + process.errfile.Close(), + ) +} diff --git a/cmd/storj-sdk/testplanet.go b/cmd/storj-sdk/testplanet.go new file mode 100644 index 000000000..0711df53b --- /dev/null +++ b/cmd/storj-sdk/testplanet.go @@ -0,0 +1,94 @@ +// Copyright (C) 2018 Storj Labs, Inc. +// See LICENSE for copying information. + +package main + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/pem" + "fmt" + "os" + "os/exec" + "strconv" + "time" + + "go.uber.org/zap" + + "storj.io/storj/internal/processgroup" + "storj.io/storj/internal/testplanet" + "storj.io/storj/pkg/peertls" + "storj.io/storj/pkg/utils" +) + +func runTestPlanet(flags *Flags, command string, args []string) error { + ctx, cancel := NewCLIContext(context.Background()) + defer cancel() + + planet, err := testplanet.NewWithLogger(zap.L(), flags.SatelliteCount, flags.StorageNodeCount, 0) + if err != nil { + return err + } + + planet.Start(ctx) + // wait a bit for kademlia to start + time.Sleep(time.Second * 2) + + var env = os.Environ() + + // add satellites to environment + for i, satellite := range planet.Satellites { + env = append(env, (&ProcessInfo{ + Name: "satellite/" + strconv.Itoa(i), + ID: satellite.ID().String(), + Address: satellite.Addr(), + }).Env()...) + } + + // add storage nodes to environment + for i, storage := range planet.StorageNodes { + env = append(env, (&ProcessInfo{ + Name: "storage/" + strconv.Itoa(i), + ID: storage.ID().String(), + Address: storage.Addr(), + }).Env()...) + } + + // add additional identities to the environment + for i := 0; i < flags.Identities; i++ { + identity, err := planet.NewIdentity() + if err != nil { + return utils.CombineErrors(err, planet.Shutdown()) + } + + var chainPEM bytes.Buffer + errLeaf := pem.Encode(&chainPEM, peertls.NewCertBlock(identity.Leaf.Raw)) + errCA := pem.Encode(&chainPEM, peertls.NewCertBlock(identity.CA.Raw)) + if errLeaf != nil || errCA != nil { + return utils.CombineErrors(errLeaf, errCA, planet.Shutdown()) + } + + var key bytes.Buffer + errKey := peertls.WriteKey(&key, identity.Key) + if errKey != nil { + return utils.CombineErrors(errKey, planet.Shutdown()) + } + + env = append(env, + fmt.Sprintf("IDENTITY_%d_ID=%v", i, identity.ID.String()), + fmt.Sprintf("IDENTITY_%d_KEY=%v", i, base64.StdEncoding.EncodeToString(key.Bytes())), + fmt.Sprintf("IDENTITY_%d_CHAIN=%v", i, base64.StdEncoding.EncodeToString(chainPEM.Bytes())), + ) + } + + // run the specified program + cmd := exec.CommandContext(ctx, command, args...) + cmd.Env = env + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + processgroup.Setup(cmd) + + errRun := cmd.Run() + + return utils.CombineErrors(errRun, planet.Shutdown()) +} diff --git a/internal/fpath/os.go b/internal/fpath/os.go new file mode 100644 index 000000000..83c33b2ef --- /dev/null +++ b/internal/fpath/os.go @@ -0,0 +1,101 @@ +// Copyright (C) 2018 Storj Labs, Inc. +// See LICENSE for copying information. + +package fpath + +import ( + "io" + "os" + "path/filepath" + "runtime" + "strings" + + "storj.io/storj/pkg/utils" +) + +// IsRoot returns whether path is the root directory +func IsRoot(path string) bool { + abs, err := filepath.Abs(path) + if err == nil { + path = abs + } + + return filepath.Dir(path) == path +} + +// ApplicationDir returns best base directory for specific OS +func ApplicationDir(subdir ...string) string { + for i := range subdir { + if runtime.GOOS == "windows" || runtime.GOOS == "darwin" { + subdir[i] = strings.Title(subdir[i]) + } else { + subdir[i] = strings.ToLower(subdir[i]) + } + } + var appdir string + home := os.Getenv("HOME") + + switch runtime.GOOS { + case "windows": + // Windows standards: https://msdn.microsoft.com/en-us/library/windows/apps/hh465094.aspx?f=255&MSPPError=-2147217396 + for _, env := range []string{"AppData", "AppDataLocal", "UserProfile", "Home"} { + val := os.Getenv(env) + if val != "" { + appdir = val + break + } + } + case "darwin": + // Mac standards: https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/MacOSXDirectories/MacOSXDirectories.html + appdir = filepath.Join(home, "Library", "Application Support") + case "linux": + fallthrough + default: + // Linux standards: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + appdir = os.Getenv("XDG_DATA_HOME") + if appdir == "" && home != "" { + appdir = filepath.Join(home, ".local", "share") + } + } + return filepath.Join(append([]string{appdir}, subdir...)...) +} + +// IsValidSetupDir checks if directory is valid for setup configuration +func IsValidSetupDir(name string) (ok bool, err error) { + _, err = os.Stat(name) + if err != nil { + if os.IsNotExist(err) { + return true, err + } + return false, err + } + + f, err := os.Open(name) + if err != nil { + return false, err + } + defer func() { + err = utils.CombineErrors(err, f.Close()) + }() + + for { + var filenames []string + filenames, err = f.Readdirnames(100) + if err == io.EOF { + // nothing more + return true, nil + } else if err != nil { + // something went wrong + return false, err + } + + for _, filename := range filenames { + // allow log files to exist in the folder + if strings.EqualFold(filepath.Ext(filename), ".log") { + continue + } + + return false, nil + } + } +} diff --git a/internal/fpath/path.go b/internal/fpath/path.go index 11f724169..0adab6a03 100644 --- a/internal/fpath/path.go +++ b/internal/fpath/path.go @@ -6,12 +6,9 @@ package fpath import ( "errors" "fmt" - "io" "net/url" - "os" "path" "path/filepath" - "runtime" "strings" ) @@ -123,64 +120,3 @@ func (p FPath) IsLocal() bool { func (p FPath) String() string { return p.original } - -// ApplicationDir returns best base directory for specific OS -func ApplicationDir(subdir ...string) string { - for i := range subdir { - if runtime.GOOS == "windows" || runtime.GOOS == "darwin" { - subdir[i] = strings.Title(subdir[i]) - } else { - subdir[i] = strings.ToLower(subdir[i]) - } - } - var appdir string - home := os.Getenv("HOME") - - switch runtime.GOOS { - case "windows": - // Windows standards: https://msdn.microsoft.com/en-us/library/windows/apps/hh465094.aspx?f=255&MSPPError=-2147217396 - for _, env := range []string{"AppData", "AppDataLocal", "UserProfile", "Home"} { - val := os.Getenv(env) - if val != "" { - appdir = val - break - } - } - case "darwin": - // Mac standards: https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/MacOSXDirectories/MacOSXDirectories.html - appdir = filepath.Join(home, "Library", "Application Support") - case "linux": - fallthrough - default: - // Linux standards: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html - appdir = os.Getenv("XDG_DATA_HOME") - if appdir == "" && home != "" { - appdir = filepath.Join(home, ".local", "share") - } - } - return filepath.Join(append([]string{appdir}, subdir...)...) -} - -// IsValidSetupDir checks if directory is valid for setup configuration -func IsValidSetupDir(name string) (bool, error) { - _, err := os.Stat(name) - if err != nil { - if os.IsNotExist(err) { - return true, err - } - return false, err - } - - f, err := os.Open(name) - if err != nil { - return false, err - } - defer func() { _ = f.Close() }() - - _, err = f.Readdir(1) - if err == io.EOF { - // is empty - return true, nil - } - return false, err -} diff --git a/internal/testplanet/node.go b/internal/testplanet/node.go index 54f7a40dc..caed36763 100644 --- a/internal/testplanet/node.go +++ b/internal/testplanet/node.go @@ -47,7 +47,7 @@ type Node struct { // newNode creates a new node. func (planet *Planet) newNode(name string, nodeType pb.NodeType) (*Node, error) { - identity, err := planet.newIdentity() + identity, err := planet.NewIdentity() if err != nil { return nil, err } diff --git a/internal/testplanet/planet.go b/internal/testplanet/planet.go index d839ae88a..5dbff9c2e 100644 --- a/internal/testplanet/planet.go +++ b/internal/testplanet/planet.go @@ -56,6 +56,11 @@ func New(t zaptest.TestingT, satelliteCount, storageNodeCount, uplinkCount int) log = zaptest.NewLogger(t) } + return NewWithLogger(log, satelliteCount, storageNodeCount, uplinkCount) +} + +// NewWithLogger creates a new full system with the given number of nodes. +func NewWithLogger(log *zap.Logger, satelliteCount, storageNodeCount, uplinkCount int) (*Planet, error) { planet := &Planet{ log: log, identities: NewPregeneratedIdentities(), @@ -232,8 +237,8 @@ func (planet *Planet) newNodes(prefix string, count int, nodeType pb.NodeType) ( return xs, nil } -// newIdentity creates a new identity for a node -func (planet *Planet) newIdentity() (*provider.FullIdentity, error) { +// NewIdentity creates a new identity for a node +func (planet *Planet) NewIdentity() (*provider.FullIdentity, error) { return planet.identities.NewIdentity() } diff --git a/pkg/audit/service.go b/pkg/audit/service.go index 86b3264fe..17631fa1e 100644 --- a/pkg/audit/service.go +++ b/pkg/audit/service.go @@ -25,7 +25,7 @@ type Service struct { // Config contains configurable values for audit service type Config struct { - APIKey string `help:"APIKey to access the statdb" default:"abc123"` + APIKey string `help:"APIKey to access the statdb" default:""` SatelliteAddr string `help:"address to contact services on the satellite"` MaxRetriesStatDB int `help:"max number of times to attempt updating a statdb batch" default:"3"` Interval time.Duration `help:"how frequently segments are audited" default:"30s"` diff --git a/pkg/kademlia/peer_discovery.go b/pkg/kademlia/peer_discovery.go index 92eb8b4d2..5ad945ba3 100644 --- a/pkg/kademlia/peer_discovery.go +++ b/pkg/kademlia/peer_discovery.go @@ -96,7 +96,12 @@ func (lookup *peerDiscovery) Run(ctx context.Context) (target *pb.Node, err erro // ok := lookup.queue.Reinsert(lookup.target, next, lookup.opts.retries) ok := false if !ok { - lookup.log.Debug("connecting to node failed", zap.Any("target", lookup.target), zap.Any("dial", next.Id), zap.Error(err)) + lookup.log.Debug("connecting to node failed", + zap.Any("target", lookup.target), + zap.Any("dial", next.Id), + zap.Any("dial-address", next.Address.Address), + zap.Error(err), + ) } } diff --git a/pkg/process/logging.go b/pkg/process/logging.go index 6b1d61a7e..b48858386 100644 --- a/pkg/process/logging.go +++ b/pkg/process/logging.go @@ -20,8 +20,7 @@ var ( logCaller = flag.Bool("log.caller", false, "if true, log function filename and line number") logStack = flag.Bool("log.stack", false, "if true, log stack traces") logEncoding = flag.String("log.encoding", "console", "configures log encoding. can either be 'console' or 'json'") - logOutput = flag.String("log.output", "stderr", - "can be stdout, stderr, or a filename") + logOutput = flag.String("log.output", "stderr", "can be stdout, stderr, or a filename") ) func newLogger() (*zap.Logger, error) {