// Copyright (C) 2021 Storj Labs, Inc. // See LICENSE for copying information. package main import ( "bytes" "context" "errors" "fmt" "io" "os" "path/filepath" "runtime" "runtime/pprof" "runtime/trace" "strconv" "strings" "sync" "time" "github.com/jtolio/eventkit" "github.com/spacemonkeygo/monkit/v3" "github.com/zeebo/clingy" "github.com/zeebo/errs" "go.uber.org/zap" "golang.org/x/term" "storj.io/common/experiment" "storj.io/common/rpc/rpctracing" "storj.io/common/tracing" jaeger "storj.io/monkit-jaeger" "storj.io/private/version" ) type external struct { interactive bool // controls if interactive input is allowed analytics *bool // enables sending analytics dirs struct { loaded bool // true if Setup has been called current string // current config directory legacy string // old config directory } migration struct { migrated bool // true if a migration has been attempted err error // any error from the migration attempt } config struct { loaded bool // true if the existing config file is successfully loaded values map[string][]string // the existing configuration } access struct { loaded bool // true if we've successfully loaded access.json defaultName string // default access name to use from accesses accesses map[string]string // map of all of the stored accesses } tracing struct { traceID int64 // if non-zero, sets outgoing traces to the given id traceAddress string // if non-zero, sampled spans are sent to this trace collector address. tags map[string]string // coma separated k=v pairs to be added to the trace sample float64 // the chance (number between 0 and 1.0) to send samples to the server. verbose bool // flag to print out tracing information (like the used trace id) } debug struct { pprofFile string traceFile string } events struct { address string // if non-zero, events are sent to this address. } } func newExternal() *external { return &external{} } func (ex *external) Setup(f clingy.Flags) { ex.interactive = f.Flag( "interactive", "Controls if interactive input is allowed", true, clingy.Transform(strconv.ParseBool), clingy.Boolean, clingy.Advanced, ).(bool) ex.dirs.current = f.Flag( "config-dir", "Directory that stores the configuration", appDir(false, defaultUplinkSubdir()...), ).(string) ex.dirs.legacy = f.Flag( "legacy-config-dir", "Directory that stores legacy configuration. Only used during migration", appDir(true, defaultUplinkSubdir()...), clingy.Advanced, ).(string) ex.tracing.traceID = f.Flag( "trace-id", "Specify a trace id manually. This should be globally unique. "+ "Usually you don't need to set it, and it will be automatically generated.", int64(0), clingy.Transform(transformInt64), clingy.Advanced, ).(int64) ex.tracing.sample = f.Flag( "trace-sample", "The chance (between 0 and 1.0) to report tracing information. Set to 1 to always send it.", float64(0), clingy.Transform(transformFloat64), clingy.Advanced, ).(float64) ex.tracing.verbose = f.Flag( "trace-verbose", "Flag to print out used trace ID", false, clingy.Transform(strconv.ParseBool), clingy.Advanced, ).(bool) ex.tracing.traceAddress = f.Flag( "trace-addr", "Specify where to send traces", "agent.tracing.datasci.storj.io:5775", clingy.Advanced, ).(string) ex.tracing.tags = f.Flag( "trace-tags", "coma separated k=v pairs to be added to distributed traces", map[string]string{}, clingy.Advanced, clingy.Transform(func(val string) (map[string]string, error) { res := map[string]string{} for _, kv := range strings.Split(val, ",") { parts := strings.SplitN(kv, "=", 2) res[parts[0]] = parts[1] } return res, nil }), ).(map[string]string) ex.events.address = f.Flag( "events-addr", "Specify where to send events", "eventkitd.datasci.storj.io:9002", clingy.Advanced, ).(string) ex.debug.pprofFile = f.Flag( "debug-pprof", "File to collect Golang pprof profiling data", "", clingy.Advanced, ).(string) ex.debug.traceFile = f.Flag( "debug-trace", "File to collect Golang trace data", "", clingy.Advanced, ).(string) ex.analytics = f.Flag( "analytics", "Whether to send usage information to Storj", nil, clingy.Transform(strconv.ParseBool), clingy.Optional, clingy.Boolean, clingy.Advanced, ).(*bool) ex.dirs.loaded = true } func transformInt64(x string) (int64, error) { return strconv.ParseInt(x, 0, 64) } func transformFloat64(x string) (float64, error) { return strconv.ParseFloat(x, 64) } func (ex *external) AccessInfoFile() string { return filepath.Join(ex.dirs.current, "access.json") } func (ex *external) ConfigFile() string { return filepath.Join(ex.dirs.current, "config.ini") } func (ex *external) legacyConfigFile() string { return filepath.Join(ex.dirs.legacy, "config.yaml") } // Dynamic is called by clingy to look up values for global flags not specified on the command // line. This call lets us fill in values from config files or environment variables. func (ex *external) Dynamic(name string) (vals []string, err error) { key := "UPLINK_" + strings.ToUpper(strings.ReplaceAll(name, "-", "_")) if val, ok := os.LookupEnv(key); ok { return []string{val}, nil } // if we have not yet loaded the directories, we should not try to migrate // and load the current config. if !ex.dirs.loaded { return nil, nil } // allow errors from migration and configuration loading so that calls to // `uplink setup` can happen and write out a new configuration. if err := ex.migrate(); err != nil { return nil, nil //nolint } if err := ex.loadConfig(); err != nil { return nil, nil //nolint } return ex.config.values[name], nil } func (ex *external) analyticsEnabled() bool { if ex.analytics != nil { return *ex.analytics } // N.B.: saveInitialConfig prompts the user if they want analytics enabled. // In the past, even after prompting for this, we did not write out their // answer in the config. Instead, what has historically happened is that // if the user said yes, we wrote out an empty config, and if the user // said no, we wrote out: // // [metrics] // addr = // // So, if the new value (analytics.enabled) exists at all, we prefer that. // Otherwise, we need to check for the existence of metrics.addr and if it // is an empty value to determine if analytics are disabled. At some point // in the future after enough upgrades have happened, perhaps we can switch // to just analytics.enabled. Unfortunately, an entirely empty config file // is precisely the config file we've been writing out if a user opts in // to analytics, so we are only going to have analytics disabled if (a) // analytics.enabled says so, or absent that, if the config file's final // specification for metrics.addr is the empty string. val, err := ex.Dynamic("analytics.enabled") if err != nil { return false } if len(val) > 0 { enabled, err := strconv.ParseBool(val[len(val)-1]) if err != nil { return false } return enabled } val, err = ex.Dynamic("metrics.addr") if err != nil { return false } return len(val) == 0 || val[len(val)-1] != "" } // Wrap is called by clingy with the command to be executed. func (ex *external) Wrap(ctx context.Context, cmd clingy.Command) (err error) { if err = ex.migrate(); err != nil { return err } if err = ex.loadConfig(); err != nil { return err } if !ex.config.loaded { if err = saveInitialConfig(ctx, ex, ex.interactive, ex.analytics); err != nil { return err } } exp := os.Getenv("STORJ_EXPERIMENTAL") if exp != "" { ctx = experiment.With(ctx, exp) } if ex.debug.pprofFile != "" { var output *os.File output, err = os.Create(ex.debug.pprofFile) if err != nil { return errs.Wrap(err) } defer func() { err = errs.Combine(err, output.Close()) }() err = pprof.StartCPUProfile(output) if err != nil { return errs.Wrap(err) } defer pprof.StopCPUProfile() } if ex.debug.traceFile != "" { var output *os.File output, err = os.Create(ex.debug.traceFile) if err != nil { return errs.Wrap(err) } defer func() { err = errs.Combine(err, output.Close()) }() err = trace.Start(output) if err != nil { return errs.Wrap(err) } defer trace.Stop() } // N.B.: Tracing is currently disabled by default (sample == 0, traceID == 0) and is // something a user can only opt into. as a result, we don't check ex.analyticsEnabled() // in this if statement. If we do ever start turning on trace samples by default, we // will need to make sure we only do so if ex.analyticsEnabled(). if ex.tracing.traceAddress != "" && (ex.tracing.sample > 0 || ex.tracing.traceID > 0) { versionName := fmt.Sprintf("uplink-release-%s", version.Build.Version.String()) if !version.Build.Release { versionName = "uplink-dev" } collector, err := jaeger.NewUDPCollector(zap.L(), ex.tracing.traceAddress, versionName, nil, 0, 0, 0) if err != nil { return err } defer func() { _ = collector.Close() }() defer tracked(ctx, collector.Run)() cancel := jaeger.RegisterJaeger(monkit.Default, collector, jaeger.Options{ Fraction: ex.tracing.sample, Excluded: tracing.IsExcluded, }, ) defer cancel() if ex.tracing.traceID == 0 { if ex.tracing.verbose { var printedFirst bool monkit.Default.ObserveTraces(func(trace *monkit.Trace) { // workaround to hide the traceID of tlsopts.verifyIndentity called from a separated goroutine if !printedFirst { _, _ = fmt.Fprintf(clingy.Stdout(ctx), "New traceID %x\n", trace.Id()) printedFirst = true } }) } } else { trace := monkit.NewTrace(ex.tracing.traceID) trace.Set(rpctracing.Sampled, true) defer mon.Func().RemoteTrace(&ctx, monkit.NewId(), trace)(&err) } monkit.Default.ObserveTraces(func(trace *monkit.Trace) { if hn, err := os.Hostname(); err == nil { trace.Set("hostname", hn) } for k, v := range ex.tracing.tags { trace.Set(k, v) } }) } if ex.analyticsEnabled() && ex.events.address != "" { var appname string var appversion string if version.Build.Release { // TODO: eventkit should probably think through // application and application version more carefully. appname = "uplink-release" appversion = version.Build.Version.String() } else { appname = "uplink-dev" appversion = version.Build.Timestamp.Format(time.RFC3339) } client := eventkit.NewUDPClient( appname, appversion, "", ex.events.address, ) defer tracked(ctx, client.Run)() eventkit.DefaultRegistry.AddDestination(client) eventkit.DefaultRegistry.Scope("init").Event("init") } defer mon.Task()(&ctx)(&err) return cmd.Execute(ctx) } func tracked(ctx context.Context, cb func(context.Context)) (done func()) { ctx, cancel := context.WithCancel(ctx) var wg sync.WaitGroup wg.Add(1) go func() { cb(ctx) wg.Done() }() return func() { cancel() wg.Wait() } } // PromptInput gets a line of input text from the user and returns an error if // interactive mode is disabled. func (ex *external) PromptInput(ctx context.Context, prompt string) (input string, err error) { if !ex.interactive { return "", errs.New("required user input in non-interactive setting") } fmt.Fprint(clingy.Stdout(ctx), prompt, " ") var buf []byte var tmp [1]byte for { _, err := clingy.Stdin(ctx).Read(tmp[:]) if errors.Is(err, io.EOF) { break } else if err != nil { return "", errs.Wrap(err) } else if tmp[0] == '\n' { break } buf = append(buf, tmp[0]) } return string(bytes.TrimSpace(buf)), nil } // PromptInput gets a line of secret input from the user twice to ensure that // it is the same value, and returns an error if interactive mode is disabled // or if the prompt cannot be put into a mode where the typing is not echoed. func (ex *external) PromptSecret(ctx context.Context, prompt string) (secret string, err error) { if !ex.interactive { return "", errs.New("required secret input in non-interactive setting") } fh, ok := clingy.Stdin(ctx).(interface{ Fd() uintptr }) if !ok { return "", errs.New("unable to request secret from stdin") } fd := int(fh.Fd()) for { fmt.Fprint(clingy.Stdout(ctx), prompt, " ") first, err := term.ReadPassword(fd) if err != nil { return "", errs.New("unable to request secret from stdin: %w", err) } fmt.Fprintln(clingy.Stdout(ctx)) fmt.Fprint(clingy.Stdout(ctx), "Again: ") second, err := term.ReadPassword(fd) if err != nil { return "", errs.New("unable to request secret from stdin: %w", err) } fmt.Fprintln(clingy.Stdout(ctx)) if string(first) != string(second) { fmt.Fprintln(clingy.Stdout(ctx), "Values did not match. Try again.") fmt.Fprintln(clingy.Stdout(ctx)) continue } return string(first), nil } } func defaultUplinkSubdir() []string { switch runtime.GOOS { case "windows", "darwin": return []string{"Storj", "Uplink"} default: return []string{"storj", "uplink"} } } // appDir returns best base directory for the currently running operating system. It // has a legacy bool to have it return the same values that storj.io/common/fpath.ApplicationDir // would have returned. func appDir(legacy bool, subdir ...string) string { var appdir string home := os.Getenv("HOME") switch runtime.GOOS { case "windows": // Windows standards: https://msdn.microsoft.com/en-us/library/windows/apps/hh465094.aspx?f=255&MSPPError=-2147217396 for _, env := range []string{"AppData", "AppDataLocal", "UserProfile", "Home"} { val := os.Getenv(env) if val != "" { appdir = val break } } case "darwin": // Mac standards: https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/MacOSXDirectories/MacOSXDirectories.html appdir = filepath.Join(home, "Library", "Application Support") case "linux": fallthrough default: // Linux standards: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html if legacy { appdir = os.Getenv("XDG_DATA_HOME") if appdir == "" && home != "" { appdir = filepath.Join(home, ".local", "share") } } else { appdir = os.Getenv("XDG_CONFIG_HOME") if appdir == "" && home != "" { appdir = filepath.Join(home, ".config") } } } return filepath.Join(append([]string{appdir}, subdir...)...) }