JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

storj: remove from storj/pkg packages moved to storj/private repo

Browse Source 
* debug
* traces
* cfgstruct
* process

Package `storj/private/version` will be removed as a separate change.

Change-Id: Iadc40faa782e6225513b28218952f02d9c240a9f
This commit is contained in:
Michal Niewrzal 2020-03-23 20:18:20 +01:00
parent ddf87286c5
commit f0aeda3091
71 changed files with 105 additions and 2334 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/certificates/auth.go
View File

@ -18,8 +18,8 @@ import (
"github.com/spf13/cobra"
"github.com/zeebo/errs"
"storj.io/private/process"
"storj.io/storj/certificate/authorization"
"storj.io/storj/pkg/process"
)
var (

2
cmd/certificates/claims.go
View File

@ -11,8 +11,8 @@ import (
"github.com/spf13/cobra"
"github.com/zeebo/errs"
"storj.io/private/process"
"storj.io/storj/certificate/authorization"
"storj.io/storj/pkg/process"
)
var (

4
cmd/certificates/main.go
View File

@ -9,10 +9,10 @@ import (
"go.uber.org/zap"
"storj.io/common/fpath"
"storj.io/private/cfgstruct"
"storj.io/private/process"
"storj.io/storj/certificate"
"storj.io/storj/certificate/authorization"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/revocation"
)

2
cmd/certificates/setup.go
View File

@ -11,8 +11,8 @@ import (
"github.com/spf13/cobra"
"storj.io/common/fpath"
"storj.io/private/process"
"storj.io/storj/certificate/authorization"
"storj.io/storj/pkg/process"
)
var (

4
cmd/identity/batch.go
View File

@ -23,8 +23,8 @@ import (
"storj.io/common/peertls"
"storj.io/common/pkcrypto"
"storj.io/common/storj"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/private/cfgstruct"
"storj.io/private/process"
"storj.io/storj/private/cui"
)

4
cmd/identity/certificate_authority.go
View File

@ -13,8 +13,8 @@ import (
"storj.io/common/identity"
"storj.io/common/peertls/extensions"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/private/cfgstruct"
"storj.io/private/process"
"storj.io/storj/pkg/revocation"
)

4
cmd/identity/identity.go
View File

@ -10,8 +10,8 @@ import (
"github.com/zeebo/errs"
"storj.io/common/identity"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/private/cfgstruct"
"storj.io/private/process"
)
var (

4
cmd/identity/main.go
View File

@ -20,9 +20,9 @@ import (
"storj.io/common/peertls/tlsopts"
"storj.io/common/pkcrypto"
"storj.io/common/rpc"
"storj.io/private/cfgstruct"
"storj.io/private/process"
"storj.io/storj/certificate/certificateclient"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/revocation"
"storj.io/storj/private/version"
"storj.io/storj/private/version/checker"

4
cmd/identity/revocations.go
View File

@ -12,8 +12,8 @@ import (
"github.com/spf13/cobra"
"github.com/zeebo/errs"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/private/cfgstruct"
"storj.io/private/process"
"storj.io/storj/pkg/revocation"
)

2
cmd/inspector/main.go
View File

@ -21,7 +21,7 @@ import (
"storj.io/common/pb"
"storj.io/common/rpc"
"storj.io/common/storj"
"storj.io/storj/pkg/process"
"storj.io/private/process"
"storj.io/storj/private/prompt"
"storj.io/uplink/private/eestream"
)

2
cmd/satellite/admin.go
View File

@ -8,7 +8,7 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap"
"storj.io/storj/pkg/process"
"storj.io/private/process"
"storj.io/storj/pkg/revocation"
"storj.io/storj/private/version"
"storj.io/storj/satellite"

2
cmd/satellite/api.go
View File

@ -9,7 +9,7 @@ import (
"go.uber.org/zap"
"storj.io/common/context2"
"storj.io/storj/pkg/process"
"storj.io/private/process"
"storj.io/storj/pkg/revocation"
"storj.io/storj/private/version"
"storj.io/storj/satellite"

2
cmd/satellite/gc.go
View File

@ -8,7 +8,7 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap"
"storj.io/storj/pkg/process"
"storj.io/private/process"
"storj.io/storj/pkg/revocation"
"storj.io/storj/private/version"
"storj.io/storj/satellite"

4
cmd/satellite/main.go
View File

@ -19,10 +19,10 @@ import (
"storj.io/common/context2"
"storj.io/common/fpath"
"storj.io/common/storj"
"storj.io/private/cfgstruct"
"storj.io/private/process"
"storj.io/storj/cmd/satellite/reports"
"storj.io/storj/pkg/cache"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/revocation"
"storj.io/storj/private/version"
"storj.io/storj/satellite"

2
cmd/satellite/repairer.go
View File

@ -9,7 +9,7 @@ import (
"go.uber.org/zap"
"storj.io/common/context2"
"storj.io/storj/pkg/process"
"storj.io/private/process"
"storj.io/storj/pkg/revocation"
"storj.io/storj/private/version"
"storj.io/storj/satellite"

2
cmd/segment-reaper/delete.go
View File

@ -18,7 +18,7 @@ import (
"storj.io/common/pb"
"storj.io/common/storj"
"storj.io/storj/pkg/process"
"storj.io/private/process"
"storj.io/storj/satellite/metainfo"
"storj.io/storj/storage"
)

4
cmd/segment-reaper/detect.go
View File

@ -12,8 +12,8 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/private/cfgstruct"
"storj.io/private/process"
"storj.io/storj/satellite/metainfo"
)

2
cmd/segment-reaper/main.go
View File

@ -6,7 +6,7 @@ package main
import (
"github.com/spf13/cobra"
"storj.io/storj/pkg/process"
"storj.io/private/process"
)
var (

4
cmd/storagenode-updater/main.go
View File

@ -31,8 +31,8 @@ import (
"storj.io/common/identity"
"storj.io/common/storj"
"storj.io/common/sync2"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/private/cfgstruct"
"storj.io/private/process"
"storj.io/storj/private/version"
"storj.io/storj/private/version/checker"
)

10
cmd/storagenode-updater/main_test.go
View File

@ -27,7 +27,7 @@ import (
"storj.io/common/storj"
"storj.io/common/testcontext"
"storj.io/common/testrand"
"storj.io/storj/private/version"
"storj.io/private/version"
"storj.io/storj/versioncontrol"
)
@ -134,10 +134,10 @@ func TestAutoUpdater(t *testing.T) {
// to the passed version info values and returns the executable name.
func CompileWithVersion(ctx *testcontext.Context, pkg string, info version.Info) string {
ldFlagsX := map[string]string{
"storj.io/storj/private/version.buildTimestamp": strconv.Itoa(int(info.Timestamp.Unix())),
"storj.io/storj/private/version.buildCommitHash": info.CommitHash,
"storj.io/storj/private/version.buildVersion": info.Version.String(),
"storj.io/storj/private/version.buildRelease": strconv.FormatBool(info.Release),
"storj.io/private/version.buildTimestamp": strconv.Itoa(int(info.Timestamp.Unix())),
"storj.io/private/version.buildCommitHash": info.CommitHash,
"storj.io/private/version.buildVersion": info.Version.String(),
"storj.io/private/version.buildRelease": strconv.FormatBool(info.Release),
}
return ctx.CompileWithLDFlagsX(pkg, ldFlagsX)
}

2
cmd/storagenode-updater/windows.go
View File

@ -19,7 +19,7 @@ import (
"golang.org/x/sync/errgroup"
"golang.org/x/sys/windows/svc"
"storj.io/storj/pkg/process"
"storj.io/private/process"
)
func init() {

2
cmd/storagenode/dashboard.go
View File

@ -21,7 +21,7 @@ import (
"storj.io/common/memory"
"storj.io/common/pb"
"storj.io/common/rpc"
"storj.io/storj/pkg/process"
"storj.io/private/process"
"storj.io/storj/private/version"
)

2
cmd/storagenode/gracefulexit.go
View File

@ -20,7 +20,7 @@ import (
"storj.io/common/pb"
"storj.io/common/rpc"
"storj.io/common/storj"
"storj.io/storj/pkg/process"
"storj.io/private/process"
"storj.io/storj/private/prompt"
)

4
cmd/storagenode/main.go
View File

@ -18,8 +18,8 @@ import (
"storj.io/common/fpath"
"storj.io/common/memory"
"storj.io/common/storj"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/private/cfgstruct"
"storj.io/private/process"
"storj.io/storj/pkg/revocation"
"storj.io/storj/private/version"
"storj.io/storj/storagenode"

2
cmd/storagenode/windows.go
View File

@ -22,7 +22,7 @@ import (
"golang.org/x/sync/errgroup"
"golang.org/x/sys/windows/svc"
"storj.io/storj/pkg/process"
"storj.io/private/process"
)
func init() {

4
cmd/uplink/cmd/access.go
View File

@ -8,9 +8,9 @@ import (
"github.com/spf13/cobra"
"storj.io/private/cfgstruct"
"storj.io/private/process"
libuplink "storj.io/storj/lib/uplink"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
)
var (

4
cmd/uplink/cmd/common.go
View File

@ -9,8 +9,8 @@ import (
"github.com/spf13/cobra"
"storj.io/common/fpath"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/private/cfgstruct"
"storj.io/private/process"
"storj.io/uplink/telemetry"
)

4
cmd/uplink/cmd/import.go
View File

@ -13,8 +13,8 @@ import (
"github.com/spf13/cobra"
"github.com/zeebo/errs"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/private/cfgstruct"
"storj.io/private/process"
)
var importCfg struct {

4
cmd/uplink/cmd/root.go
View File

@ -23,8 +23,8 @@ import (
"storj.io/common/fpath"
"storj.io/common/storj"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/private/cfgstruct"
"storj.io/private/process"
"storj.io/storj/private/version/checker"
"storj.io/uplink"
privateAccess "storj.io/uplink/private/access"

4
cmd/uplink/cmd/setup.go
View File

@ -12,9 +12,9 @@ import (
"github.com/spf13/cobra"
"storj.io/private/cfgstruct"
"storj.io/private/process"
"storj.io/storj/cmd/internal/wizard"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/uplink"
"storj.io/uplink/backcomp"
)

4
cmd/uplink/cmd/share.go
View File

@ -14,8 +14,8 @@ import (
"github.com/zeebo/errs"
"storj.io/common/fpath"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/private/cfgstruct"
"storj.io/private/process"
"storj.io/uplink"
)

2
cmd/uplink/main.go
View File

@ -7,8 +7,8 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/viper"
"storj.io/private/process"
"storj.io/storj/cmd/uplink/cmd"
"storj.io/storj/pkg/process"
)
func main() {

4
cmd/versioncontrol/main.go
View File

@ -12,8 +12,8 @@ import (
"go.uber.org/zap"
"storj.io/common/fpath"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/process"
"storj.io/private/cfgstruct"
"storj.io/private/process"
"storj.io/storj/versioncontrol"
)

2
examples/grpc-debug/main.go
View File

@ -12,7 +12,7 @@ import (
"storj.io/common/identity"
"storj.io/common/peertls/tlsopts"
"storj.io/storj/pkg/cfgstruct"
"storj.io/private/cfgstruct"
"storj.io/storj/private/grpctlsopts"
)

2
examples/metric-receiver/main.go
View File

@ -10,7 +10,7 @@ import (
"github.com/spf13/cobra"
"storj.io/common/telemetry"
"storj.io/storj/pkg/process"
"storj.io/private/process"
)
var (

17
go.mod
View File

@ -3,7 +3,6 @@ module storj.io/storj
go 1.13
require (
cloud.google.com/go v0.52.0
github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053
github.com/alicebob/miniredis/v2 v2.11.1
github.com/blang/semver v3.5.1+incompatible
@ -21,8 +20,6 @@ require (
github.com/gorilla/schema v1.1.0
github.com/graphql-go/graphql v0.7.9
github.com/jackc/pgx v3.2.0+incompatible
github.com/jtolds/monkit-hw/v2 v2.0.0-20191108235325-141a0da276b3
github.com/jtolds/tracetagger/v2 v2.0.0-rc5
github.com/lib/pq v1.3.0
github.com/mattn/go-isatty v0.0.9 // indirect
github.com/mattn/go-sqlite3 v2.0.2+incompatible
@ -30,25 +27,23 @@ require (
github.com/nsf/termbox-go v0.0.0-20190121233118-02980233997d
github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114
github.com/skyrings/skyring-common v0.0.0-20160929130248-d1c0bb1cbd5e
github.com/spacemonkeygo/monkit/v3 v3.0.4
github.com/spacemonkeygo/monkit/v3 v3.0.5
github.com/spf13/cast v1.3.0
github.com/spf13/cobra v0.0.5
github.com/spf13/pflag v1.0.3
github.com/spf13/cobra v0.0.6
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.4.0
github.com/stretchr/testify v1.4.0
github.com/stripe/stripe-go v63.1.1+incompatible
github.com/vivint/infectious v0.0.0-20190108171102-2455b059135b
github.com/zeebo/admission/v3 v3.0.1
github.com/zeebo/errs v1.2.2
github.com/zeebo/structs v1.0.2
go.uber.org/zap v1.10.0
go.uber.org/zap v1.14.1
golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1
golang.org/x/time v0.0.0-20191024005414-555d28b269f0
google.golang.org/grpc v1.27.1
gopkg.in/yaml.v2 v2.2.4
storj.io/common v0.0.0-20200320083002-07faa8a64ad8
storj.io/common v0.0.0-20200323134045-2bd4d6e2dd7d
storj.io/drpc v0.0.8
storj.io/private v0.0.0-20200323175212-c3ef21e873a9
storj.io/uplink v1.0.0
)

40
go.sum
View File

@ -79,11 +79,10 @@ github.com/containerd/containerd v1.2.7 h1:8lqLbl7u1j3MmiL9cJ/O275crSq7bfwUayvva
github.com/containerd/containerd v1.2.7/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8=
github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f/go.mod h1:8S58EK26zhXSxzv7NQFpnliaOQsmDUxvoQO3rt154Vg=
github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc=
@ -173,6 +172,8 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc h1:DLpL8pWq0v4JYoRpEhDfsJhhJyGKCcQM2WPW2TJs31c=
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3 h1:SRgJV+IoxM5MKyFdlSUeNy6/ycRUF2yBAKdAQswoHUk=
github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
@ -300,12 +301,13 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114 h1:Pm6R878vxWWWR+Sa3ppsLce/Zq+JNTs6aVvRu13jv9A=
github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
@ -318,6 +320,8 @@ github.com/spacemonkeygo/monkit/v3 v3.0.0-20191108235033-eacca33b3037/go.mod h1:
github.com/spacemonkeygo/monkit/v3 v3.0.1/go.mod h1:JcK1pCbReQsOsMKF/POFSZCq7drXFybgGmbc27tuwes=
github.com/spacemonkeygo/monkit/v3 v3.0.4 h1:Ay+PZirv+qfd4sqcT+X/U3BnC7AcIaqp/IXh0oV36k8=
github.com/spacemonkeygo/monkit/v3 v3.0.4/go.mod h1:JcK1pCbReQsOsMKF/POFSZCq7drXFybgGmbc27tuwes=
github.com/spacemonkeygo/monkit/v3 v3.0.5 h1:vMW8Ne6WAUU/OMYaSv7KGW9h/sRNgeh6TyBapOSuMhM=
github.com/spacemonkeygo/monkit/v3 v3.0.5/go.mod h1:JcK1pCbReQsOsMKF/POFSZCq7drXFybgGmbc27tuwes=
github.com/spacemonkeygo/monotime v0.0.0-20180824235756-e3f48a95f98a h1:8+cCjxhToanKmxLIbuyBNe2EnpgwhiivsIaRJstDRFA=
github.com/spacemonkeygo/monotime v0.0.0-20180824235756-e3f48a95f98a/go.mod h1:ul4bvvnCOPZgq8w0nTkSmWVg/hauVpFS97Am1YM1XXo=
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU=
@ -327,13 +331,14 @@ github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/cobra v0.0.6 h1:breEStsVwemnKh2/s6gMvSdMEkwW0sK8vGStnlVBMCs=
github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -347,7 +352,6 @@ github.com/stripe/stripe-go v63.1.1+incompatible/go.mod h1:A1dQZmO/QypXmsL0T8axY
github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/vivint/infectious v0.0.0-20190108171102-2455b059135b h1:dLkqBELopfQNhe8S9ucnSf+HhiUCgK/hPIjVG0f9GlY=
github.com/vivint/infectious v0.0.0-20190108171102-2455b059135b/go.mod h1:5oyMAv4hrBEKqBwORFsiqIrCNCmL2qcZLQTdJLYeYIc=
github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs=
@ -384,13 +388,20 @@ go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A=
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo=
go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@ -414,6 +425,7 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
@ -461,7 +473,6 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190102155601-82a175fd1598/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -505,6 +516,8 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200117161641-43d50277825c h1:2EA2K0k9bcvvEDlqD8xdlOhCOqq+O/p9Voqi4x9W1YU=
golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@ -520,6 +533,8 @@ google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.15.0 h1:yzlyyDW/J0w8yNFJIhiAJy4kq74S+1DOLdawELNxFMA=
google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.20.0 h1:jz2KixHX7EcCPiQrySzPdnYT7DbINAypCqKZ1Z7GM40=
google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@ -545,6 +560,7 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
@ -575,10 +591,12 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
storj.io/common v0.0.0-20200318103328-b7e942ff9304 h1:KQ1ITzbT32bAyBx5gPYQFkMs86iUGA1Sl+QvsMQuJZc=
storj.io/common v0.0.0-20200318103328-b7e942ff9304/go.mod h1:I0QTs7z1rI+ZEN95GGY2LKMzP5OZqu0Udga3WhyQfO0=
storj.io/common v0.0.0-20200320083002-07faa8a64ad8 h1:DgWXFCldvyu1drZKKL0D3vlHSkjmBsvBqr8nXg59SFs=
storj.io/common v0.0.0-20200320083002-07faa8a64ad8/go.mod h1:I0QTs7z1rI+ZEN95GGY2LKMzP5OZqu0Udga3WhyQfO0=
storj.io/common v0.0.0-20200323134045-2bd4d6e2dd7d h1:R4qel+wubcKJP8HvWromOGOplbAlpVNwfIkDkFD4RY0=
storj.io/common v0.0.0-20200323134045-2bd4d6e2dd7d/go.mod h1:I0QTs7z1rI+ZEN95GGY2LKMzP5OZqu0Udga3WhyQfO0=
storj.io/drpc v0.0.7-0.20191115031725-2171c57838d2/go.mod h1:/ascUDbzNAv0A3Jj7wUIKFBH2JdJ2uJIBO/b9+2yHgQ=
storj.io/drpc v0.0.8 h1:wu68cMmtoT0vSWIAZz29RpJkWdi4o0S8BIrLslpH5FQ=
storj.io/drpc v0.0.8/go.mod h1:v39uWro/EbXXk+gNnrM9FQuVVS2zUBWBfeduydgeXUA=
storj.io/private v0.0.0-20200323175212-c3ef21e873a9 h1:dFwvi7xBYIHpqmpfLddibHpXexiN4gaAN1uo4Xx5890=
storj.io/private v0.0.0-20200323175212-c3ef21e873a9/go.mod h1:I09FR2ZU1Dy1vaQ2SkUbfGpwVtJrwafjYWGw8W808WE=
storj.io/uplink v1.0.0 h1:2tj09e88/2CuGMguS9qyVIEPX2NDE4W/1vMf4bdUCyA=
storj.io/uplink v1.0.0/go.mod h1:2VBdNnx7zZoCx91ETUVM56mDRIFDToRyKBWxH/Usrek=

442
pkg/cfgstruct/bind.go
View File

@ -1,442 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package cfgstruct
import (
"fmt"
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"time"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"go.uber.org/zap"
"storj.io/storj/private/version"
)
const (
// AnySource is a source annotation for config values that can come from
// a flag or file.
AnySource = "any"
// FlagSource is a source annotation for config values that just come from
// flags (i.e. are never persisted to file)
FlagSource = "flag"
// BasicHelpAnnotationName is the name of the annotation used to indicate
// a flag should be included in basic usage/help.
BasicHelpAnnotationName = "basic-help"
)
var (
allSources = []string{
AnySource,
FlagSource,
}
)
// BindOpt is an option for the Bind method
type BindOpt struct {
isDev *bool
isSetup *bool
varfn func(vars map[string]confVar)
}
// ConfDir sets variables for default options called $CONFDIR and $CONFNAME.
func ConfDir(path string) BindOpt {
val := filepath.Clean(os.ExpandEnv(path))
return BindOpt{varfn: func(vars map[string]confVar) {
vars["CONFDIR"] = confVar{val: val, nested: false}
}}
}
// IdentityDir sets a variable for the default option called $IDENTITYDIR.
func IdentityDir(path string) BindOpt {
val := filepath.Clean(os.ExpandEnv(path))
return BindOpt{varfn: func(vars map[string]confVar) {
vars["IDENTITYDIR"] = confVar{val: val, nested: false}
}}
}
// SetupMode issues the bind in a mode where it does not ignore fields with the
// `setup:"true"` tag.
func SetupMode() BindOpt {
setup := true
return BindOpt{isSetup: &setup}
}
// UseDevDefaults forces the bind call to use development defaults unless
// UseReleaseDefaults is provided as a subsequent option.
// Without either, Bind will default to determining which defaults to use
// based on version.Build.Release
func UseDevDefaults() BindOpt {
dev := true
return BindOpt{isDev: &dev}
}
// UseReleaseDefaults forces the bind call to use release defaults unless
// UseDevDefaults is provided as a subsequent option.
// Without either, Bind will default to determining which defaults to use
// based on version.Build.Release
func UseReleaseDefaults() BindOpt {
dev := false
return BindOpt{isDev: &dev}
}
type confVar struct {
val string
nested bool
}
// Bind sets flags on a FlagSet that match the configuration struct
// 'config'. This works by traversing the config struct using the 'reflect'
// package.
func Bind(flags FlagSet, config interface{}, opts ...BindOpt) {
bind(flags, config, opts...)
}
func bind(flags FlagSet, config interface{}, opts ...BindOpt) {
ptrtype := reflect.TypeOf(config)
if ptrtype.Kind() != reflect.Ptr {
panic(fmt.Sprintf("invalid config type: %#v. Expecting pointer to struct.", config))
}
isDev := !version.Build.Release
setupCommand := false
vars := map[string]confVar{}
for _, opt := range opts {
if opt.varfn != nil {
opt.varfn(vars)
}
if opt.isDev != nil {
isDev = *opt.isDev
}
if opt.isSetup != nil {
setupCommand = *opt.isSetup
}
}
bindConfig(flags, "", reflect.ValueOf(config).Elem(), vars, setupCommand, false, isDev)
}
func bindConfig(flags FlagSet, prefix string, val reflect.Value, vars map[string]confVar, setupCommand, setupStruct bool, isDev bool) {
if val.Kind() != reflect.Struct {
panic(fmt.Sprintf("invalid config type: %#v. Expecting struct.", val.Interface()))
}
typ := val.Type()
resolvedVars := make(map[string]string, len(vars))
{
structpath := strings.Replace(prefix, ".", string(filepath.Separator), -1)
for k, v := range vars {
if !v.nested {
resolvedVars[k] = v.val
continue
}
resolvedVars[k] = filepath.Join(v.val, structpath)
}
}
for i := 0; i < typ.NumField(); i++ {
field := typ.Field(i)
fieldval := val.Field(i)
flagname := hyphenate(snakeCase(field.Name))
if field.Tag.Get("noprefix") != "true" {
flagname = prefix + flagname
}
if field.Tag.Get("internal") == "true" {
continue
}
onlyForSetup := (field.Tag.Get("setup") == "true") || setupStruct
// ignore setup params for non setup commands
if !setupCommand && onlyForSetup {
continue
}
if !fieldval.CanAddr() {
panic(fmt.Sprintf("cannot addr field %s in %s", field.Name, typ))
}
fieldref := fieldval.Addr()
if !fieldref.CanInterface() {
panic(fmt.Sprintf("cannot get interface of field %s in %s", field.Name, typ))
}
fieldaddr := fieldref.Interface()
if fieldvalue, ok := fieldaddr.(pflag.Value); ok {
help := field.Tag.Get("help")
var def string
if isDev {
def = getDefault(field.Tag, "devDefault", "releaseDefault", "default", flagname)
} else {
def = getDefault(field.Tag, "releaseDefault", "devDefault", "default", flagname)
}
err := fieldvalue.Set(def)
if err != nil {
panic(fmt.Sprintf("invalid default value for %s: %#v, %v", flagname, def, err))
}
flags.Var(fieldvalue, flagname, help)
markHidden := false
if onlyForSetup {
SetBoolAnnotation(flags, flagname, "setup", true)
}
if field.Tag.Get("user") == "true" {
SetBoolAnnotation(flags, flagname, "user", true)
}
if field.Tag.Get("hidden") == "true" {
markHidden = true
SetBoolAnnotation(flags, flagname, "hidden", true)
}
if field.Tag.Get("deprecated") == "true" {
markHidden = true
SetBoolAnnotation(flags, flagname, "deprecated", true)
}
if source := field.Tag.Get("source"); source != "" {
setSourceAnnotation(flags, flagname, source)
}
if markHidden {
err := flags.MarkHidden(flagname)
if err != nil {
panic(fmt.Sprintf("mark hidden failed %s: %v", flagname, err))
}
}
continue
}
switch field.Type.Kind() {
case reflect.Struct:
if field.Anonymous {
bindConfig(flags, prefix, fieldval, vars, setupCommand, onlyForSetup, isDev)
} else {
bindConfig(flags, flagname+".", fieldval, vars, setupCommand, onlyForSetup, isDev)
}
case reflect.Array:
digits := len(fmt.Sprint(fieldval.Len()))
for j := 0; j < fieldval.Len(); j++ {
padding := strings.Repeat("0", digits-len(fmt.Sprint(j)))
bindConfig(flags, fmt.Sprintf("%s.%s%d.", flagname, padding, j), fieldval.Index(j), vars, setupCommand, onlyForSetup, isDev)
}
default:
help := field.Tag.Get("help")
var def string
if isDev {
def = getDefault(field.Tag, "devDefault", "releaseDefault", "default", flagname)
} else {
def = getDefault(field.Tag, "releaseDefault", "devDefault", "default", flagname)
}
fieldaddr := fieldval.Addr().Interface()
check := func(err error) {
if err != nil {
panic(fmt.Sprintf("invalid default value for %s: %#v", flagname, def))
}
}
switch field.Type {
case reflect.TypeOf(int(0)):
val, err := strconv.ParseInt(def, 0, strconv.IntSize)
check(err)
flags.IntVar(fieldaddr.(*int), flagname, int(val), help)
case reflect.TypeOf(int64(0)):
val, err := strconv.ParseInt(def, 0, 64)
check(err)
flags.Int64Var(fieldaddr.(*int64), flagname, val, help)
case reflect.TypeOf(uint(0)):
val, err := strconv.ParseUint(def, 0, strconv.IntSize)
check(err)
flags.UintVar(fieldaddr.(*uint), flagname, uint(val), help)
case reflect.TypeOf(uint64(0)):
val, err := strconv.ParseUint(def, 0, 64)
check(err)
flags.Uint64Var(fieldaddr.(*uint64), flagname, val, help)
case reflect.TypeOf(time.Duration(0)):
val, err := time.ParseDuration(def)
check(err)
flags.DurationVar(fieldaddr.(*time.Duration), flagname, val, help)
case reflect.TypeOf(float64(0)):
val, err := strconv.ParseFloat(def, 64)
check(err)
flags.Float64Var(fieldaddr.(*float64), flagname, val, help)
case reflect.TypeOf(string("")):
if field.Tag.Get("path") == "true" {
// NB: conventionally unix path separators are used in default values
def = filepath.FromSlash(def)
}
flags.StringVar(
fieldaddr.(*string), flagname, expand(resolvedVars, def), help)
case reflect.TypeOf(bool(false)):
val, err := strconv.ParseBool(def)
check(err)
flags.BoolVar(fieldaddr.(*bool), flagname, val, help)
case reflect.TypeOf([]string(nil)):
flags.StringArrayVar(fieldaddr.(*[]string), flagname, nil, help)
default:
panic(fmt.Sprintf("invalid field type: %s", field.Type.String()))
}
if onlyForSetup {
SetBoolAnnotation(flags, flagname, "setup", true)
}
if field.Tag.Get("user") == "true" {
SetBoolAnnotation(flags, flagname, "user", true)
}
if field.Tag.Get(BasicHelpAnnotationName) == "true" {
SetBoolAnnotation(flags, flagname, BasicHelpAnnotationName, true)
}
markHidden := false
if field.Tag.Get("hidden") == "true" {
markHidden = true
SetBoolAnnotation(flags, flagname, "hidden", true)
}
if field.Tag.Get("deprecated") == "true" {
markHidden = true
SetBoolAnnotation(flags, flagname, "deprecated", true)
}
if source := field.Tag.Get("source"); source != "" {
setSourceAnnotation(flags, flagname, source)
}
if markHidden {
err := flags.MarkHidden(flagname)
if err != nil {
panic(fmt.Sprintf("mark hidden failed %s: %v", flagname, err))
}
}
}
}
}
func getDefault(tag reflect.StructTag, preferred, opposite, fallback, flagname string) string {
if val, ok := tag.Lookup(preferred); ok {
if _, oppositeExists := tag.Lookup(opposite); !oppositeExists {
panic(fmt.Sprintf("%q defined but %q missing for %v", preferred, opposite, flagname))
}
if _, fallbackExists := tag.Lookup(fallback); fallbackExists {
panic(fmt.Sprintf("%q defined along with %q fallback for %v", preferred, fallback, flagname))
}
return val
}
if _, oppositeExists := tag.Lookup(opposite); oppositeExists {
panic(fmt.Sprintf("%q missing but %q defined for %v", preferred, opposite, flagname))
}
return tag.Get(fallback)
}
func setSourceAnnotation(flagset interface{}, name, source string) {
switch source {
case AnySource:
case FlagSource:
default:
panic(fmt.Sprintf("invalid source annotation %q for %s: must be one of %q", source, name, allSources))
}
setStringAnnotation(flagset, name, "source", source)
}
func setStringAnnotation(flagset interface{}, name, key, value string) {
flags, ok := flagset.(*pflag.FlagSet)
if !ok {
return
}
err := flags.SetAnnotation(name, key, []string{value})
if err != nil {
panic(fmt.Sprintf("unable to set %s annotation for %s: %v", key, name, err))
}
}
// SetBoolAnnotation sets an annotation (if it can) on flagset with a value of []string{"true|false"}.
func SetBoolAnnotation(flagset interface{}, name, key string, value bool) {
flags, ok := flagset.(*pflag.FlagSet)
if !ok {
return
}
err := flags.SetAnnotation(name, key, []string{strconv.FormatBool(value)})
if err != nil {
panic(fmt.Sprintf("unable to set %s annotation for %s: %v", key, name, err))
}
}
func expand(vars map[string]string, val string) string {
return os.Expand(val, func(key string) string { return vars[key] })
}
// FindConfigDirParam returns '--config-dir' param from os.Args (if exists)
func FindConfigDirParam() string {
return FindFlagEarly("config-dir")
}
// FindIdentityDirParam returns '--identity-dir' param from os.Args (if exists)
func FindIdentityDirParam() string {
return FindFlagEarly("identity-dir")
}
// FindDefaultsParam returns '--defaults' param from os.Args (if it exists)
func FindDefaultsParam() string {
return FindFlagEarly("defaults")
}
// FindFlagEarly retrieves the value of a flag before `flag.Parse` has been called
func FindFlagEarly(flagName string) string {
// workaround to have early access to 'dir' param
for i, arg := range os.Args {
if strings.HasPrefix(arg, fmt.Sprintf("--%s=", flagName)) {
return strings.TrimPrefix(arg, fmt.Sprintf("--%s=", flagName))
} else if arg == fmt.Sprintf("--%s", flagName) && i < len(os.Args)-1 {
return os.Args[i+1]
}
}
return ""
}
// SetupFlag sets up flags that are needed before `flag.Parse` has been called
func SetupFlag(log *zap.Logger, cmd *cobra.Command, dest *string, name, value, usage string) {
if foundValue := FindFlagEarly(name); foundValue != "" {
value = foundValue
}
cmd.PersistentFlags().StringVar(dest, name, value, usage)
if cmd.PersistentFlags().SetAnnotation(name, "setup", []string{"true"}) != nil {
log.Sugar().Errorf("Failed to set 'setup' annotation for '%s'", name)
}
}
// DefaultsType returns the type of defaults (release/dev) this binary should use
func DefaultsType() string {
// define a flag so that the flag parsing system will be happy.
defaults := strings.ToLower(FindDefaultsParam())
if defaults != "" {
return defaults
}
if version.Build.Release {
return "release"
}
return "dev"
}
// DefaultsFlag sets up the defaults=dev/release flag options, which is needed
// before `flag.Parse` has been called
func DefaultsFlag(cmd *cobra.Command) BindOpt {
// define a flag so that the flag parsing system will be happy.
defaults := DefaultsType()
// we're actually going to ignore this flag entirely and parse the commandline
// arguments early instead
_ = cmd.PersistentFlags().String("defaults", defaults,
"determines which set of configuration defaults to use. can either be 'dev' or 'release'")
setSourceAnnotation(cmd.PersistentFlags(), "defaults", FlagSource)
switch defaults {
case "dev":
return UseDevDefaults()
case "release":
return UseReleaseDefaults()
default:
panic(fmt.Sprintf("unsupported defaults value %q", FindDefaultsParam()))
}
}

273
pkg/cfgstruct/bind_test.go
View File

@ -1,273 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package cfgstruct
import (
"fmt"
"reflect"
"testing"
"time"
"github.com/spf13/pflag"
"github.com/stretchr/testify/require"
"storj.io/common/memory"
"storj.io/common/storj"
)
func assertEqual(actual, expected interface{}) {
if !reflect.DeepEqual(actual, expected) {
panic(fmt.Sprintf("expected %v, got %v", expected, actual))
}
}
func TestBind(t *testing.T) {
f := pflag.NewFlagSet("test", pflag.PanicOnError)
var c struct {
String string `default:""`
Bool bool `releaseDefault:"false" devDefault:"true"`
Int64 int64 `releaseDefault:"0" devDefault:"1"`
Int int `default:"0"`
Uint64 uint64 `default:"0"`
Uint uint `default:"0"`
Float64 float64 `default:"0"`
Size memory.Size `default:"0"`
Duration time.Duration `default:"0"`
NodeURL storj.NodeURL `releaseDefault:"" devDefault:""`
NodeURLs storj.NodeURLs `releaseDefault:"" devDefault:""`
Struct struct {
AnotherString string `default:""`
}
Fields [10]struct {
AnotherInt int `default:"0"`
}
}
Bind(f, &c, UseReleaseDefaults())
assertEqual(c.String, string(""))
assertEqual(c.Bool, bool(false))
assertEqual(c.Int64, int64(0))
assertEqual(c.Int, int(0))
assertEqual(c.Uint64, uint64(0))
assertEqual(c.Uint, uint(0))
assertEqual(c.Float64, float64(0))
assertEqual(c.Size, memory.Size(0))
assertEqual(c.Duration, time.Duration(0))
assertEqual(c.NodeURL, storj.NodeURL{})
assertEqual(c.NodeURLs, storj.NodeURLs(nil))
assertEqual(c.Struct.AnotherString, string(""))
assertEqual(c.Fields[0].AnotherInt, int(0))
assertEqual(c.Fields[3].AnotherInt, int(0))
node1, err := storj.NodeIDFromString("12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S")
require.NoError(t, err)
node2, err := storj.NodeIDFromString("12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs")
require.NoError(t, err)
err = f.Parse([]string{
"--string=1",
"--bool=true",
"--int64=1",
"--int=1",
"--uint64=1",
"--uint=1",
"--float64=1",
"--size=1MiB",
"--duration=1h",
"--node-url=12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S@mars.tardigrade.io:7777",
"--node-ur-ls=12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S@mars.tardigrade.io:7777,12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs@jupiter.tardigrade.io:7777",
"--struct.another-string=1",
"--fields.03.another-int=1"})
if err != nil {
panic(err)
}
assertEqual(c.String, string("1"))
assertEqual(c.Bool, bool(true))
assertEqual(c.Int64, int64(1))
assertEqual(c.Int, int(1))
assertEqual(c.Uint64, uint64(1))
assertEqual(c.Uint, uint(1))
assertEqual(c.Float64, float64(1))
assertEqual(c.Size, memory.MiB)
assertEqual(c.Duration, time.Hour)
assertEqual(c.NodeURL, storj.NodeURL{ID: node1, Address: "mars.tardigrade.io:7777"})
assertEqual(c.NodeURLs, storj.NodeURLs{
storj.NodeURL{ID: node1, Address: "mars.tardigrade.io:7777"},
storj.NodeURL{ID: node2, Address: "jupiter.tardigrade.io:7777"},
})
assertEqual(c.Struct.AnotherString, string("1"))
assertEqual(c.Fields[0].AnotherInt, int(0))
assertEqual(c.Fields[3].AnotherInt, int(1))
}
func TestConfDir(t *testing.T) {
f := pflag.NewFlagSet("test", pflag.PanicOnError)
var c struct {
String string `default:"-$CONFDIR+"`
MyStruct1 struct {
String string `default:"1${CONFDIR}2"`
MyStruct2 struct {
String string `default:"2${CONFDIR}3"`
}
}
}
Bind(f, &c, UseReleaseDefaults(), ConfDir("confpath"))
assertEqual(f.Lookup("string").DefValue, "-confpath+")
assertEqual(f.Lookup("my-struct1.string").DefValue, "1confpath2")
assertEqual(f.Lookup("my-struct1.my-struct2.string").DefValue, "2confpath3")
}
func TestBindDevDefaults(t *testing.T) {
f := pflag.NewFlagSet("test", pflag.PanicOnError)
var c struct {
String string `default:"dev"`
Bool bool `releaseDefault:"false" devDefault:"true"`
Int64 int64 `releaseDefault:"0" devDefault:"1"`
Int int `default:"2"`
Uint64 uint64 `default:"3"`
Uint uint `releaseDefault:"0" devDefault:"4"`
Float64 float64 `default:"5.5"`
Duration time.Duration `default:"1h"`
NodeURL storj.NodeURL `releaseDefault:"" devDefault:"12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S@mars.tardigrade.io:7777"`
NodeURLs storj.NodeURLs `releaseDefault:"" devDefault:"12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S@mars.tardigrade.io:7777,12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs@jupiter.tardigrade.io:7777"`
Struct struct {
AnotherString string `default:"dev2"`
}
Fields [10]struct {
AnotherInt int `default:"6"`
}
}
Bind(f, &c, UseDevDefaults())
node1, err := storj.NodeIDFromString("12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S")
require.NoError(t, err)
node2, err := storj.NodeIDFromString("12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs")
require.NoError(t, err)
assertEqual(c.String, string("dev"))
assertEqual(c.Bool, bool(true))
assertEqual(c.Int64, int64(1))
assertEqual(c.Int, int(2))
assertEqual(c.Uint64, uint64(3))
assertEqual(c.Uint, uint(4))
assertEqual(c.Float64, float64(5.5))
assertEqual(c.Duration, time.Hour)
assertEqual(c.NodeURL, storj.NodeURL{ID: node1, Address: "mars.tardigrade.io:7777"})
assertEqual(c.NodeURLs, storj.NodeURLs{
storj.NodeURL{ID: node1, Address: "mars.tardigrade.io:7777"},
storj.NodeURL{ID: node2, Address: "jupiter.tardigrade.io:7777"},
})
assertEqual(c.Struct.AnotherString, string("dev2"))
assertEqual(c.Fields[0].AnotherInt, int(6))
assertEqual(c.Fields[3].AnotherInt, int(6))
node3, err := storj.NodeIDFromString("121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6")
require.NoError(t, err)
err = f.Parse([]string{
"--string=1",
"--bool=true",
"--int64=1",
"--int=1",
"--uint64=1",
"--uint=1",
"--float64=1",
"--duration=1h",
"--node-url=121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6@saturn.tardigrade.io:7777",
"--node-ur-ls=121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6@saturn.tardigrade.io:7777",
"--struct.another-string=1",
"--fields.03.another-int=1"})
if err != nil {
panic(err)
}
assertEqual(c.String, string("1"))
assertEqual(c.Bool, bool(true))
assertEqual(c.Int64, int64(1))
assertEqual(c.Int, int(1))
assertEqual(c.Uint64, uint64(1))
assertEqual(c.Uint, uint(1))
assertEqual(c.Float64, float64(1))
assertEqual(c.Duration, time.Hour)
assertEqual(c.NodeURL, storj.NodeURL{ID: node3, Address: "saturn.tardigrade.io:7777"})
assertEqual(c.NodeURLs, storj.NodeURLs{storj.NodeURL{ID: node3, Address: "saturn.tardigrade.io:7777"}})
assertEqual(c.Struct.AnotherString, string("1"))
assertEqual(c.Fields[0].AnotherInt, int(6))
assertEqual(c.Fields[3].AnotherInt, int(1))
}
func TestHiddenDev(t *testing.T) {
f := pflag.NewFlagSet("test", pflag.PanicOnError)
var c struct {
String string `default:"dev" hidden:"true"`
String2 string `default:"dev" hidden:"false"`
Bool bool `releaseDefault:"false" devDefault:"true" hidden:"true"`
Int64 int64 `releaseDefault:"0" devDefault:"1"`
Int int `default:"2"`
Size memory.Size `default:"0" hidden:"true"`
}
Bind(f, &c, UseDevDefaults())
flagString := f.Lookup("string")
flagStringHide := f.Lookup("string2")
flagBool := f.Lookup("bool")
flagInt64 := f.Lookup("int64")
flagInt := f.Lookup("int")
flagSize := f.Lookup("size")
assertEqual(flagString.Hidden, true)
assertEqual(flagStringHide.Hidden, false)
assertEqual(flagBool.Hidden, true)
assertEqual(flagInt64.Hidden, false)
assertEqual(flagInt.Hidden, false)
assertEqual(flagSize.Hidden, true)
}
func TestHiddenRelease(t *testing.T) {
f := pflag.NewFlagSet("test", pflag.PanicOnError)
var c struct {
String string `default:"dev" hidden:"false"`
String2 string `default:"dev" hidden:"true"`
Bool bool `releaseDefault:"false" devDefault:"true" hidden:"true"`
Int64 int64 `releaseDefault:"0" devDefault:"1"`
Int int `default:"2"`
}
Bind(f, &c, UseReleaseDefaults())
flagString := f.Lookup("string")
flagStringHide := f.Lookup("string2")
flagBool := f.Lookup("bool")
flagInt64 := f.Lookup("int64")
flagInt := f.Lookup("int")
assertEqual(flagString.Hidden, false)
assertEqual(flagStringHide.Hidden, true)
assertEqual(flagBool.Hidden, true)
assertEqual(flagInt64.Hidden, false)
assertEqual(flagInt.Hidden, false)
}
func TestSource(t *testing.T) {
var c struct {
Unset string
Any string `source:"any"`
Flag string `source:"flag"`
}
f := pflag.NewFlagSet("test", pflag.PanicOnError)
Bind(f, &c, UseReleaseDefaults())
unset := f.Lookup("unset")
require.NotNil(t, unset)
require.Empty(t, unset.Annotations)
any := f.Lookup("any")
require.NotNil(t, any)
require.Equal(t, map[string][]string{
"source": {"any"},
}, any.Annotations)
flag := f.Lookup("flag")
require.NotNil(t, flag)
require.Equal(t, map[string][]string{
"source": {"flag"},
}, flag.Annotations)
}

39
pkg/cfgstruct/case.go
View File

@ -1,39 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package cfgstruct
import (
"strings"
"unicode"
)
func hyphenate(val string) string {
return strings.Replace(val, "_", "-", -1)
}
func snakeCase(val string) string {
// don't you think this function should be in the standard library?
// seems useful
if len(val) <= 1 {
return strings.ToLower(val)
}
runes := []rune(val)
rv := make([]rune, 0, len(runes))
for i := 0; i < len(runes); i++ {
rv = append(rv, unicode.ToLower(runes[i]))
if i < len(runes)-1 &&
unicode.IsLower(runes[i]) &&
unicode.IsUpper(runes[i+1]) {
// lower-to-uppercase case
rv = append(rv, '_')
} else if i < len(runes)-2 &&
unicode.IsUpper(runes[i]) &&
unicode.IsUpper(runes[i+1]) &&
unicode.IsLower(runes[i+2]) {
// end-of-acronym case
rv = append(rv, '_')
}
}
return string(rv)
}

30
pkg/cfgstruct/case_test.go
View File

@ -1,30 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package cfgstruct
import (
"testing"
)
func TestSnakeCase(t *testing.T) {
for _, test := range []struct {
input, expected string
}{
{"CoolBeans", "cool_beans"},
{"coolBeans", "cool_beans"},
{"JSONBeans", "json_beans"},
{"JSONBeAns", "json_be_ans"},
{"JSONBeANS", "json_be_ans"},
{"coolbeans", "coolbeans"},
{"COOLBEANS", "coolbeans"},
{"CoolJSON", "cool_json"},
{"CoolJSONBeans", "cool_json_beans"},
} {
actual := snakeCase(test.input)
if actual != test.expected {
t.Logf("expected %#v but got %#v", test.expected, actual)
t.Fail()
}
}
}

27
pkg/cfgstruct/flags.go
View File

@ -1,27 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package cfgstruct
import (
"time"
"github.com/spf13/pflag"
)
// FlagSet is an interface that matches *pflag.FlagSet
type FlagSet interface {
BoolVar(p *bool, name string, value bool, usage string)
IntVar(p *int, name string, value int, usage string)
Int64Var(p *int64, name string, value int64, usage string)
UintVar(p *uint, name string, value uint, usage string)
Uint64Var(p *uint64, name string, value uint64, usage string)
DurationVar(p *time.Duration, name string, value time.Duration, usage string)
Float64Var(p *float64, name string, value float64, usage string)
StringVar(p *string, name string, value string, usage string)
StringArrayVar(p *[]string, name string, value []string, usage string)
Var(val pflag.Value, name string, usage string)
MarkHidden(name string) error
}
var _ FlagSet = (*pflag.FlagSet)(nil)

9
pkg/debug/common.go
View File

@ -1,9 +0,0 @@
// Copyright (C) 2020 Storj Labs, Inc.
// See LICENSE for copying information.
package debug
import "github.com/zeebo/errs"
// Error is default error class for debug package.
var Error = errs.Class("debug")

18
pkg/debug/error.go
View File

@ -1,18 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package debug
import (
"github.com/spacemonkeygo/monkit/v3"
"google.golang.org/grpc/status"
)
func init() {
monkit.AddErrorNameHandler(func(err error) (string, bool) {
if s, ok := status.FromError(err); ok {
return "grpc_" + s.Code().String(), true
}
return "", false
})
}

197
pkg/debug/panel.go
View File

@ -1,197 +0,0 @@
// Copyright (C) 2020 Storj Labs, Inc.
// See LICENSE for copying information.
package debug
import (
"fmt"
"io"
"net/http"
"path"
"sort"
"strings"
"sync"
"text/template"
"unicode"
"go.uber.org/zap"
"storj.io/common/sync2"
)
// Panel implements a serving of customized callbacks.
type Panel struct {
log *zap.Logger
url string
name string
mu sync.RWMutex
lookup map[string]*Button
categories []*ButtonGroup
}
// NewPanel creates a new panel.
func NewPanel(log *zap.Logger, url, name string) *Panel {
return &Panel{
log: log,
url: url,
name: name,
lookup: map[string]*Button{},
}
}
// ButtonGroup contains description of a collection of buttons.
type ButtonGroup struct {
Slug string
Name string
Buttons []*Button
}
// Button defines a clickable button.
type Button struct {
Slug string
Name string
Call func(progress io.Writer) error
}
// Add adds a button group to the panel.
func (panel *Panel) Add(cats ...*ButtonGroup) {
panel.mu.Lock()
defer panel.mu.Unlock()
for _, cat := range cats {
if cat.Slug == "" {
cat.Slug = slugify(cat.Name)
}
for _, but := range cat.Buttons {
but.Slug = slugify(but.Name)
panel.lookup["/"+path.Join(cat.Slug, but.Slug)] = but
}
panel.categories = append(panel.categories, cat)
}
sort.Slice(panel.categories, func(i, k int) bool {
return panel.categories[i].Name < panel.categories[k].Name
})
}
// ServeHTTP serves buttons on the prefix and on
// other endpoints calls the specified call.
func (panel *Panel) ServeHTTP(w http.ResponseWriter, r *http.Request) {
panel.mu.RLock()
defer panel.mu.RUnlock()
url := strings.TrimPrefix(r.URL.Path, panel.url)
if len(url) >= len(r.URL.Path) {
http.Error(w, "not found", http.StatusNotFound)
return
}
if url == "/" {
err := buttonsTemplate.Execute(w, map[string]interface{}{
"Name": panel.name,
"URL": panel.url,
"Categories": panel.categories,
})
if err != nil {
panel.log.Error("buttons template failed", zap.Error(err))
}
return
}
button, ok := panel.lookup[url]
if !ok {
http.Error(w, "control not found", http.StatusNotFound)
return
}
panel.log.Debug("calling", zap.String("url", url))
w.Header().Add("Content-Type", "text/plain")
w.Header().Add("Cache-Control", "no-cache")
w.Header().Add("Cache-Control", "max-age=0")
err := button.Call(w)
if err != nil {
panel.log.Error("failed to run button", zap.String("url", url), zap.Error(err))
}
}
var buttonsTemplate = template.Must(template.New("").Parse(`<!DOCTYPE html>
<html>
<head>
<title>Control Panel</title>
<style>
.button {
padding: 0.8rem 1rem;
border: 1px solid #ccc;
margin: 0.1px;
}
.button:hover {
background: #eee;
}
</style>
</head>
<body>
<h1>{{ .Name }} Control Panel</h1>
{{ $url := .URL }}
{{ range $cat := .Categories }}
<div class="category">
<h2>{{$cat.Name}}</h2>
{{ range $but := $cat.Buttons }}
<a class="button" href="{{$url}}/{{$cat.Slug}}/{{$but.Slug}}">{{.Name}}</a>
{{ end }}
</div>
{{ end }}
</body>
</html>`))
// slugify converts text to a slug
func slugify(s string) string {
return strings.Map(func(r rune) rune {
switch {
case 'a' <= r && r <= 'z':
return r
case 'A' <= r && r <= 'Z':
return unicode.ToLower(r)
case '0' <= r && r <= '9':
return r
default:
return '-'
}
}, s)
}
// Cycle returns button group for a cycle
func Cycle(name string, cycle *sync2.Cycle) *ButtonGroup {
return &ButtonGroup{
Name: name,
Buttons: []*Button{
{
Name: "Trigger",
Call: func(w io.Writer) error {
_, _ = fmt.Fprintln(w, "Triggering")
cycle.TriggerWait()
_, _ = fmt.Fprintln(w, "Done")
return nil
},
}, {
Name: "Pause",
Call: func(w io.Writer) error {
cycle.Pause()
_, _ = fmt.Fprintln(w, "Paused")
return nil
},
}, {
Name: "Resume",
Call: func(w io.Writer) error {
cycle.Restart()
_, _ = fmt.Fprintln(w, "Resumed")
return nil
},
},
},
}
}

162
pkg/debug/server.go
View File

@ -1,162 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
// Package debug implements debug server for satellite and storage node.
package debug
import (
"context"
"fmt"
"net"
"net/http"
"net/http/pprof"
"strings"
"time"
"github.com/spacemonkeygo/monkit/v3"
"github.com/spacemonkeygo/monkit/v3/present"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"
"storj.io/storj/pkg/traces"
"storj.io/storj/private/version/checker"
)
func init() {
// zero out the http.DefaultServeMux net/http/pprof so unhelpfully
// side-effected.
*http.DefaultServeMux = http.ServeMux{}
}
// Config defines configuration for debug server.
type Config struct {
Address string `internal:"true"`
ControlTitle string `internal:"true"`
Control bool `help:"expose control panel" releaseDefault:"false" devDefault:"true"`
}
// Server provides endpoints for debugging.
type Server struct {
log *zap.Logger
listener net.Listener
server http.Server
mux http.ServeMux
Panel *Panel
registry *monkit.Registry
}
// NewServer returns a new debug.Server.
func NewServer(log *zap.Logger, listener net.Listener, registry *monkit.Registry, config Config) *Server {
server := &Server{log: log}
server.listener = listener
server.server.Handler = &server.mux
server.registry = registry
server.Panel = NewPanel(log.Named("control"), "/control", config.ControlTitle)
if config.Control {
server.mux.Handle("/control/", server.Panel)
}
server.mux.Handle("/version/", http.StripPrefix("/version", checker.NewDebugHandler(log.Named("version"))))
server.mux.HandleFunc("/debug/pprof/", pprof.Index)
server.mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
server.mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
server.mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
server.mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
server.mux.HandleFunc("/debug/run/trace/db", server.collectTraces)
server.mux.Handle("/mon/", http.StripPrefix("/mon", present.HTTP(server.registry)))
server.mux.HandleFunc("/metrics", server.metrics)
server.mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
_, _ = fmt.Fprintln(w, "OK")
})
return server
}
// Run starts the debug endpoint.
func (server *Server) Run(ctx context.Context) error {
if server.listener == nil {
return nil
}
ctx, cancel := context.WithCancel(ctx)
var group errgroup.Group
group.Go(func() error {
<-ctx.Done()
return Error.Wrap(server.server.Shutdown(context.Background()))
})
group.Go(func() error {
defer cancel()
return Error.Wrap(server.server.Serve(server.listener))
})
return group.Wait()
}
// Close closes server and underlying listener.
func (server *Server) Close() error {
return Error.Wrap(server.server.Close())
}
// metrics writes https://prometheus.io/docs/instrumenting/exposition_formats/
func (server *Server) metrics(w http.ResponseWriter, r *http.Request) {
// writes https://prometheus.io/docs/instrumenting/exposition_formats/
// (https://prometheus.io/docs/concepts/metric_types/)
server.registry.Stats(func(key monkit.SeriesKey, field string, val float64) {
measurement := sanitize(key.Measurement)
var metrics []string
for tag, tagVal := range key.Tags.All() {
metric := sanitize(tag) + "=\"" + sanitize(tagVal) + "\""
metrics = append(metrics, metric)
}
fieldMetric := "field=\"" + sanitize(field) + "\""
metrics = append(metrics, fieldMetric)
_, _ = fmt.Fprintf(w, "# TYPE %s gauge\n%s{"+
strings.Join(metrics, ",")+"} %g\n", measurement, measurement, val)
})
}
// collectTraces collects traces until request is canceled.
func (server *Server) collectTraces(w http.ResponseWriter, r *http.Request) {
cancel := traces.CollectTraces()
defer cancel()
for {
_, err := w.Write([]byte{0})
if err != nil {
return
}
time.Sleep(time.Second)
}
}
// sanitize formats val to be suitable for prometheus.
func sanitize(val string) string {
// https://prometheus.io/docs/concepts/data_model/
// specifies all metric names must match [a-zA-Z_:][a-zA-Z0-9_:]*
// Note: The colons are reserved for user defined recording rules.
// They should not be used by exporters or direct instrumentation.
if '0' <= val[0] && val[0] <= '9' {
val = "_" + val
}
return strings.Map(func(r rune) rune {
switch {
case 'a' <= r && r <= 'z':
return r
case 'A' <= r && r <= 'Z':
return r
case '0' <= r && r <= '9':
return r
default:
return '_'
}
}, val)
}

218
pkg/process/config.go
View File

@ -1,218 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package process
import (
"bytes"
"flag"
"sort"
"github.com/spf13/cast"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/zeebo/errs"
yaml "gopkg.in/yaml.v2"
"storj.io/common/fpath"
"storj.io/storj/pkg/cfgstruct"
)
// SaveConfigOption is a function that updates the options for SaveConfig.
type SaveConfigOption func(*SaveConfigOptions)
// SaveConfigOptions controls the behavior of SaveConfig.
type SaveConfigOptions struct {
Overrides map[string]interface{}
RemoveDeprecated bool
}
// SaveConfigWithOverrides sets the overrides to the provided map.
func SaveConfigWithOverrides(overrides map[string]interface{}) SaveConfigOption {
return func(opts *SaveConfigOptions) {
opts.Overrides = overrides
}
}
// SaveConfigWithOverride adds a single override to SaveConfig.
func SaveConfigWithOverride(name string, value interface{}) SaveConfigOption {
return func(opts *SaveConfigOptions) {
if opts.Overrides == nil {
opts.Overrides = make(map[string]interface{})
}
opts.Overrides[name] = value
}
}
// SaveConfigRemovingDeprecated tells SaveConfig to not store deprecated flags.
func SaveConfigRemovingDeprecated() SaveConfigOption {
return func(opts *SaveConfigOptions) {
opts.RemoveDeprecated = true
}
}
// SaveConfig will save only the user-specific flags with default values to
// outfile with specific values specified in 'overrides' overridden.
func SaveConfig(cmd *cobra.Command, outfile string, opts ...SaveConfigOption) error {
// step 0. apply any options to change the behavior
//
var options SaveConfigOptions
for _, opt := range opts {
opt(&options)
}
// step 1. load all of the configuration settings we are going to save
//
flags := cmd.Flags()
vip, err := Viper(cmd)
if err != nil {
return errs.Wrap(err)
}
if err := vip.MergeConfigMap(options.Overrides); err != nil {
return errs.Wrap(err)
}
settings := vip.AllSettings()
// step 2. construct some data describing what exactly we're saving to the
// config file, and how they're saved.
//
type configValue struct {
value interface{}
comment string
set bool
}
flat := make(map[string]configValue)
flatKeys := make([]string, 0)
// N.B. we have to pre-declare the function so that it can make recursive calls.
var filterAndFlatten func(string, map[string]interface{})
filterAndFlatten = func(base string, settings map[string]interface{}) {
for key, value := range settings {
if value, ok := value.(map[string]interface{}); ok {
filterAndFlatten(base+key+".", value)
continue
}
fullKey := base + key
// the help key should not be persisted to the config file but
// cannot use the FlagSource source annotation since it is a
// standard flags and not pflags.
// TODO: figure out a better way to do for standard flags than
// hardcoding in pkg/process.
if fullKey == "help" {
continue
}
// gather information about the flag under consideration
var (
changed bool
setup bool
hidden bool
user bool
deprecated bool
source string
comment string
typ string
_, overrideExists = options.Overrides[fullKey]
)
if f := flags.Lookup(fullKey); f != nil { // first check pflags
// When a value is loaded from the file, the flag won't be
// "changed" but we still need to persist it. Therefore, for
// the following code, "changed" means that either a flag has
// been explicitly set or a value that differs from the
// default.
changed = f.Changed || f.Value.String() != f.DefValue
setup = readBoolAnnotation(f, "setup")
hidden = readBoolAnnotation(f, "hidden")
user = readBoolAnnotation(f, "user")
deprecated = readBoolAnnotation(f, "deprecated")
source = readSourceAnnotation(f)
comment = f.Usage
typ = f.Value.Type()
} else if f := flag.Lookup(fullKey); f != nil { // then stdlib flags
changed = f.Value.String() != f.DefValue
comment = f.Usage
} else {
// by default we store config values we know nothing about. we
// absue the meaning of "changed" to include this case.
changed = true
}
// in any of these cases, don't store the key in the config file
if setup ||
hidden ||
options.RemoveDeprecated && deprecated ||
source == cfgstruct.FlagSource {
continue
}
// viper is super cool and doesn't cast floats automatically, so we
// handle that ourselves.
if typ == "float64" {
value = cast.ToFloat64(value)
}
flatKeys = append(flatKeys, fullKey)
flat[fullKey] = configValue{
value: value,
comment: comment,
set: user || changed || overrideExists,
}
}
}
filterAndFlatten("", settings)
sort.Strings(flatKeys)
// step 3. write out the configuration file
//
var nl = []byte("\n")
var lines [][]byte
for _, key := range flatKeys {
config := flat[key]
if config.comment != "" {
lines = append(lines, []byte("# "+config.comment))
}
data, err := yaml.Marshal(map[string]interface{}{key: config.value})
if err != nil {
return errs.Wrap(err)
}
dataLines := bytes.Split(bytes.TrimSpace(data), nl)
// if the config value is set, concat in the yaml lines
if config.set {
lines = append(lines, dataLines...)
} else {
// otherwise, add them in but commented out
for _, line := range dataLines {
lines = append(lines, append([]byte("# "), line...))
}
}
// add a blank line separator
lines = append(lines, nil)
}
return errs.Wrap(fpath.AtomicWriteFile(outfile, bytes.Join(lines, nl), 0600))
}
// readSourceAnnotation is a helper to return the source annotation or cfgstruct.AnySource if unset.
func readSourceAnnotation(flag *pflag.Flag) string {
annotation := flag.Annotations["source"]
if len(annotation) == 0 {
return cfgstruct.AnySource
}
return annotation[0]
}
// readBoolAnnotation is a helper to see if a boolean annotation is set to true on the flag.
func readBoolAnnotation(flag *pflag.Flag, key string) bool {
annotation := flag.Annotations[key]
return len(annotation) > 0 && annotation[0] == "true"
}

45
pkg/process/debug.go
View File

@ -1,45 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package process
import (
"context"
"flag"
"fmt"
"net"
"github.com/spacemonkeygo/monkit/v3"
"go.uber.org/zap"
"storj.io/storj/pkg/debug"
)
var (
// DebugAddrFlag for --debug.addr
DebugAddrFlag = flag.String("debug.addr", "127.0.0.1:0", "address to listen on for debug endpoints")
)
func initDebug(log *zap.Logger, r *monkit.Registry) (err error) {
if *DebugAddrFlag == "" {
return nil
}
ln, err := net.Listen("tcp", *DebugAddrFlag)
if err != nil {
return err
}
go func() {
server := debug.NewServer(log, ln, r, debug.Config{
Address: *DebugAddrFlag,
})
log.Debug(fmt.Sprintf("debug server listening on %s", ln.Addr().String()))
err := server.Run(context.TODO())
if err != nil {
log.Error("debug server died", zap.Error(err))
}
}()
return nil
}

39
pkg/process/exec.go
View File

@ -1,39 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package process // import "storj.io/storj/pkg/process"
import (
"fmt"
"log"
"os"
"github.com/spf13/cobra"
// We use a blank import here to get the side effects from the init function in version
_ "storj.io/storj/private/version"
)
func init() {
cobra.MousetrapHelpText = "This is a command line tool.\n\n" +
"This needs to be run from a Command Prompt.\n"
// Figure out the executable name.
exe, err := os.Executable()
if err == nil {
cobra.MousetrapHelpText += fmt.Sprintf(
"Try running \"%s help\" for more information\n", exe)
}
}
// check if file exists, handle error correctly if it doesn't
func fileExists(path string) bool {
_, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return false
}
log.Fatalf("failed to check for file existence: %v", err)
}
return true
}

361
pkg/process/exec_conf.go
View File

@ -1,361 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package process
import (
"context"
"errors"
"flag"
"fmt"
"log"
"os"
"os/signal"
"path/filepath"
"strings"
"sync"
"syscall"
"time"
"github.com/spacemonkeygo/monkit/v3"
"github.com/spacemonkeygo/monkit/v3/collect"
"github.com/spacemonkeygo/monkit/v3/present"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/spf13/viper"
"github.com/zeebo/errs"
"github.com/zeebo/structs"
"go.uber.org/zap"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/private/version"
)
// DefaultCfgFilename is the default filename used for storing a configuration.
const DefaultCfgFilename = "config.yaml"
var (
mon = monkit.Package()
commandMtx sync.Mutex
contexts = map[*cobra.Command]context.Context{}
cancels = map[*cobra.Command]context.CancelFunc{}
configs = map[*cobra.Command][]interface{}{}
vipers = map[*cobra.Command]*viper.Viper{}
)
// Bind sets flags on a command that match the configuration struct
// 'config'. It ensures that the config has all of the values loaded into it
// when the command runs.
func Bind(cmd *cobra.Command, config interface{}, opts ...cfgstruct.BindOpt) {
commandMtx.Lock()
defer commandMtx.Unlock()
cfgstruct.Bind(cmd.Flags(), config, opts...)
configs[cmd] = append(configs[cmd], config)
}
// Exec runs a Cobra command. If a "config-dir" flag is defined it will be parsed
// and loaded using viper.
func Exec(cmd *cobra.Command) {
ExecWithCustomConfig(cmd, true, LoadConfig)
}
// ExecCustomDebug runs default configuration except the default debug is disabled.
func ExecCustomDebug(cmd *cobra.Command) {
ExecWithCustomConfig(cmd, false, LoadConfig)
}
// ExecWithCustomConfig runs a Cobra command. Custom configuration can be loaded.
func ExecWithCustomConfig(cmd *cobra.Command, debugEnabled bool, loadConfig func(cmd *cobra.Command, vip *viper.Viper) error) {
cmd.AddCommand(&cobra.Command{
Use: "version",
Short: "output the version's build information, if any",
RunE: cmdVersion,
Annotations: map[string]string{"type": "setup"}})
exe, err := os.Executable()
if err == nil {
cmd.Use = exe
}
pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
cleanup(cmd, debugEnabled, loadConfig)
err = cmd.Execute()
if err != nil {
os.Exit(1)
}
}
// Ctx returns the appropriate context.Context for ExecuteWithConfig commands
func Ctx(cmd *cobra.Command) (context.Context, context.CancelFunc) {
commandMtx.Lock()
defer commandMtx.Unlock()
ctx := contexts[cmd]
if ctx == nil {
ctx = context.Background()
contexts[cmd] = ctx
}
cancel := cancels[cmd]
if cancel == nil {
ctx, cancel = context.WithCancel(ctx)
contexts[cmd] = ctx
cancels[cmd] = cancel
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
go func() {
sig := <-c
log.Printf("Got a signal from the OS: %q", sig)
signal.Stop(c)
cancel()
}()
}
return ctx, cancel
}
// Viper returns the appropriate *viper.Viper for the command, creating if necessary.
func Viper(cmd *cobra.Command) (*viper.Viper, error) {
return ViperWithCustomConfig(cmd, LoadConfig)
}
// ViperWithCustomConfig returns the appropriate *viper.Viper for the command, creating if necessary. Custom
// config load logic can be defined with "loadConfig" parameter.
func ViperWithCustomConfig(cmd *cobra.Command, loadConfig func(cmd *cobra.Command, vip *viper.Viper) error) (*viper.Viper, error) {
commandMtx.Lock()
defer commandMtx.Unlock()
if vip := vipers[cmd]; vip != nil {
return vip, nil
}
vip := viper.New()
if err := vip.BindPFlags(cmd.Flags()); err != nil {
return nil, err
}
vip.SetEnvPrefix("storj")
vip.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))
vip.AutomaticEnv()
err := loadConfig(cmd, vip)
if err != nil {
return nil, err
}
vipers[cmd] = vip
return vip, nil
}
// LoadConfig loads configuration into *viper.Viper from file specified with "config-dir" flag.
func LoadConfig(cmd *cobra.Command, vip *viper.Viper) error {
cfgFlag := cmd.Flags().Lookup("config-dir")
if cfgFlag != nil && cfgFlag.Value.String() != "" {
path := filepath.Join(os.ExpandEnv(cfgFlag.Value.String()), DefaultCfgFilename)
if fileExists(path) {
setupCommand := cmd.Annotations["type"] == "setup"
vip.SetConfigFile(path)
if err := vip.ReadInConfig(); err != nil && !setupCommand {
return err
}
}
}
return nil
}
var traceOut = flag.String("debug.trace-out", "", "If set, a path to write a process trace SVG to")
func cleanup(cmd *cobra.Command, debugEnabled bool, loadConfig func(cmd *cobra.Command, vip *viper.Viper) error) {
for _, ccmd := range cmd.Commands() {
cleanup(ccmd, debugEnabled, loadConfig)
}
if cmd.Run != nil {
panic("Please use cobra's RunE instead of Run")
}
internalRun := cmd.RunE
if internalRun == nil {
return
}
cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
ctx := context.Background()
defer mon.TaskNamed("root")(&ctx)(&err)
vip, err := ViperWithCustomConfig(cmd, loadConfig)
if err != nil {
return err
}
commandMtx.Lock()
configValues := configs[cmd]
commandMtx.Unlock()
var (
brokenKeys = map[string]struct{}{}
missingKeys = map[string]struct{}{}
usedKeys = map[string]struct{}{}
allKeys = map[string]struct{}{}
allSettings = vip.AllSettings()
)
// Hacky hack: these two keys are noprefix which breaks all scoping
if val, ok := allSettings["api-key"]; ok {
allSettings["legacy.client.api-key"] = val
delete(allSettings, "api-key")
}
if val, ok := allSettings["satellite-addr"]; ok {
allSettings["legacy.client.satellite-addr"] = val
delete(allSettings, "satellite-addr")
}
for _, config := range configValues {
// Decode and all of the resulting keys into our sets
res := structs.Decode(allSettings, config)
for key := range res.Used {
usedKeys[key] = struct{}{}
allKeys[key] = struct{}{}
}
for key := range res.Missing {
missingKeys[key] = struct{}{}
allKeys[key] = struct{}{}
}
for key := range res.Broken {
brokenKeys[key] = struct{}{}
allKeys[key] = struct{}{}
}
}
// Propagate keys that are missing to flags, and remove any used keys
// from the missing set.
for key := range missingKeys {
if f := cmd.Flags().Lookup(key); f != nil {
val := vip.GetString(key)
err := f.Value.Set(val)
f.Changed = val != f.DefValue
if err != nil {
brokenKeys[key] = struct{}{}
} else {
usedKeys[key] = struct{}{}
}
} else if f := flag.Lookup(key); f != nil {
err := f.Value.Set(vip.GetString(key))
if err != nil {
brokenKeys[key] = struct{}{}
} else {
usedKeys[key] = struct{}{}
}
}
}
for key := range missingKeys {
if _, ok := usedKeys[key]; ok {
delete(missingKeys, key)
}
}
logger, err := NewLogger()
if err != nil {
return err
}
if vip.ConfigFileUsed() != "" {
path, err := filepath.Abs(vip.ConfigFileUsed())
if err != nil {
path = vip.ConfigFileUsed()
logger.Debug("unable to resolve path", zap.Error(err))
}
logger.Sugar().Info("Configuration loaded from: ", path)
}
defer func() { _ = logger.Sync() }()
defer zap.ReplaceGlobals(logger)()
defer zap.RedirectStdLog(logger)()
// okay now that logging is working, inform about the broken keys
if cmd.Annotations["type"] != "helper" {
for key := range missingKeys {
logger.Sugar().Infof("Invalid configuration file key: %s", key)
}
}
for key := range brokenKeys {
logger.Sugar().Infof("Invalid configuration file value for key: %s", key)
}
if debugEnabled {
err = initDebug(logger, monkit.Default)
if err != nil {
withoutStack := errors.New(err.Error())
logger.Debug("failed to start debug endpoints", zap.Error(withoutStack))
err = nil
}
}
if err = initProfiler(logger); err != nil {
logger.Debug("failed to init debug profiler", zap.Error(err))
err = nil
}
var workErr error
work := func(ctx context.Context) {
commandMtx.Lock()
contexts[cmd] = ctx
commandMtx.Unlock()
defer func() {
commandMtx.Lock()
delete(contexts, cmd)
delete(cancels, cmd)
commandMtx.Unlock()
}()
workErr = internalRun(cmd, args)
}
if *traceOut != "" {
fh, err := os.Create(*traceOut)
if err != nil {
return err
}
if strings.HasSuffix(*traceOut, ".json") {
err = present.SpansToJSON(fh, collect.CollectSpans(ctx, work))
} else {
err = present.SpansToSVG(fh, collect.CollectSpans(ctx, work))
}
err = errs.Combine(err, fh.Close())
if err != nil {
logger.Error("failed to write svg", zap.Error(err))
}
} else {
work(ctx)
}
err = workErr
if err != nil {
logger.Debug("Unrecoverable error", zap.Error(err))
fmt.Println("Error:", err.Error())
os.Exit(1)
}
return nil
}
}
func cmdVersion(cmd *cobra.Command, args []string) (err error) {
if version.Build.Release {
fmt.Println("Release build")
} else {
fmt.Println("Development build")
}
if !version.Build.Version.IsZero() {
fmt.Println("Version:", version.Build.Version.String())
}
if !version.Build.Timestamp.IsZero() {
fmt.Println("Build timestamp:", version.Build.Timestamp.Format(time.RFC822))
}
if version.Build.CommitHash != "" {
fmt.Println("Git commit:", version.Build.CommitHash)
}
return err
}

86
pkg/process/exec_conf_test.go
View File

@ -1,86 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package process
import (
"flag"
"io/ioutil"
"os"
"testing"
"github.com/spf13/cobra"
"github.com/stretchr/testify/require"
"storj.io/common/testcontext"
)
func setenv(key, value string) func() {
old := os.Getenv(key)
_ = os.Setenv(key, value)
return func() { _ = os.Setenv(key, old) }
}
var testZ = flag.Int("z", 0, "z flag (stdlib)")
func TestExec_PropagatesSettings(t *testing.T) {
// Set up a command that does nothing.
cmd := &cobra.Command{RunE: func(cmd *cobra.Command, args []string) error { return nil }}
// Define a config struct and some flags.
var config struct {
X int `default:"0"`
}
Bind(cmd, &config)
y := cmd.Flags().Int("y", 0, "y flag (command)")
// Set some environment variables for viper.
defer setenv("STORJ_X", "1")()
defer setenv("STORJ_Y", "2")()
defer setenv("STORJ_Z", "3")()
// Run the command through the exec call.
Exec(cmd)
// Check that the variables are now bound.
require.Equal(t, 1, config.X)
require.Equal(t, 2, *y)
require.Equal(t, 3, *testZ)
}
func TestHidden(t *testing.T) {
// Set up a command that does nothing.
cmd := &cobra.Command{RunE: func(cmd *cobra.Command, args []string) error { return nil }}
// Define a config struct with a hidden field.
var config struct {
W int `default:"0" hidden:"false"`
X int `default:"0" hidden:"true"`
Y int `releaseDefault:"1" devDefault:"0" hidden:"true"`
Z int `default:"1"`
}
Bind(cmd, &config)
// Setup test config file
ctx := testcontext.New(t)
defer ctx.Cleanup()
testConfigFile := ctx.File("testconfig.yaml")
// Run the command through the exec call.
Exec(cmd)
// Ensure that the file saves only the necessary data.
err := SaveConfig(cmd, testConfigFile)
require.NoError(t, err)
actualConfigFile, err := ioutil.ReadFile(testConfigFile)
require.NoError(t, err)
expectedConfigW := "# w: 0"
expectedConfigZ := "# z: 1"
require.Contains(t, string(actualConfigFile), expectedConfigW)
require.Contains(t, string(actualConfigFile), expectedConfigZ)
require.NotContains(t, string(actualConfigFile), "# y: ")
require.NotContains(t, string(actualConfigFile), "# x: ")
}

89
pkg/process/logging.go
View File

@ -1,89 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package process
import (
"flag"
"net/url"
"os"
"runtime"
"github.com/zeebo/errs"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"storj.io/storj/pkg/cfgstruct"
)
var (
// Error is a process error class
Error = errs.Class("process error")
logLevel = zap.LevelFlag("log.level", func() zapcore.Level {
if isDev() {
return zapcore.DebugLevel
}
return zapcore.InfoLevel
}(), "the minimum log level to log")
logDev = flag.Bool("log.development", isDev(), "if true, set logging to development mode")
logCaller = flag.Bool("log.caller", isDev(), "if true, log function filename and line number")
logStack = flag.Bool("log.stack", isDev(), "if true, log stack traces")
logEncoding = flag.String("log.encoding", "console", "configures log encoding. can either be 'console' or 'json'")
logOutput = flag.String("log.output", "stderr", "can be stdout, stderr, or a filename")
)
func init() {
winFileSink := func(u *url.URL) (zap.Sink, error) {
// Remove leading slash left by url.Parse()
return os.OpenFile(u.Path[1:], os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
}
err := zap.RegisterSink("winfile", winFileSink)
if err != nil {
panic("Unable to register winfile sink: " + err.Error())
}
}
func isDev() bool { return cfgstruct.DefaultsType() != "release" }
// NewLogger creates new logger configured by the process flags.
func NewLogger() (*zap.Logger, error) {
return NewLoggerWithOutputPaths(*logOutput)
}
// NewLoggerWithOutputPaths is the same as NewLogger, but overrides the log output paths.
func NewLoggerWithOutputPaths(outputPaths ...string) (*zap.Logger, error) {
levelEncoder := zapcore.CapitalColorLevelEncoder
if runtime.GOOS == "windows" {
levelEncoder = zapcore.CapitalLevelEncoder
}
timeKey := "T"
if os.Getenv("STORJ_LOG_NOTIME") != "" {
// using environment variable STORJ_LOG_NOTIME to avoid additional flags
timeKey = ""
}
return zap.Config{
Level: zap.NewAtomicLevelAt(*logLevel),
Development: *logDev,
DisableCaller: !*logCaller,
DisableStacktrace: !*logStack,
Encoding: *logEncoding,
EncoderConfig: zapcore.EncoderConfig{
TimeKey: timeKey,
LevelKey: "L",
NameKey: "N",
CallerKey: "C",
MessageKey: "M",
StacktraceKey: "S",
LineEnding: zapcore.DefaultLineEnding,
EncodeLevel: levelEncoder,
EncodeTime: zapcore.ISO8601TimeEncoder,
EncodeDuration: zapcore.StringDurationEncoder,
EncodeCaller: zapcore.ShortCallerEncoder,
},
OutputPaths: outputPaths,
ErrorOutputPaths: outputPaths,
}.Build()
}

106
pkg/process/metrics.go
View File

@ -1,106 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package process
import (
"context"
"flag"
"os"
"path/filepath"
"strings"
hw "github.com/jtolds/monkit-hw/v2"
"github.com/spacemonkeygo/monkit/v3"
"github.com/spacemonkeygo/monkit/v3/environment"
"github.com/zeebo/admission/v3/admproto"
"go.uber.org/zap"
"storj.io/common/identity"
"storj.io/common/telemetry"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/private/version"
)
var (
metricInterval = flag.Duration("metrics.interval", telemetry.DefaultInterval, "how frequently to send up telemetry")
metricCollector = flag.String("metrics.addr", flagDefault("", "collectora.storj.io:9000"), "address to send telemetry to")
metricApp = flag.String("metrics.app", filepath.Base(os.Args[0]), "application name for telemetry identification")
metricAppSuffix = flag.String("metrics.app-suffix", flagDefault("-dev", "-release"), "application suffix")
metricInstancePrefix = flag.String("metrics.instance-prefix", "", "instance id prefix")
)
const (
maxInstanceLength = 52
)
func flagDefault(dev, release string) string {
if cfgstruct.DefaultsType() == "release" {
return release
}
return dev
}
// InitMetrics initializes telemetry reporting. Makes a telemetry.Client and calls
// its Run() method in a goroutine.
func InitMetrics(ctx context.Context, log *zap.Logger, r *monkit.Registry, instanceID string) (err error) {
if r == nil {
r = monkit.Default
}
environment.Register(r)
hw.Register(r)
r.ScopeNamed("env").Chain(monkit.StatSourceFunc(version.Build.Stats))
log = log.Named("telemetry")
if *metricCollector == "" || *metricInterval == 0 {
log.Info("disabled")
return nil
}
if instanceID == "" {
instanceID = telemetry.DefaultInstanceID()
}
instanceID = *metricInstancePrefix + instanceID
if len(instanceID) > maxInstanceLength {
instanceID = instanceID[:maxInstanceLength]
}
c, err := telemetry.NewClient(log, *metricCollector, telemetry.ClientOpts{
Interval: *metricInterval,
Application: *metricApp + *metricAppSuffix,
Instance: instanceID,
Registry: r,
FloatEncoding: admproto.Float32Encoding,
})
if err != nil {
return err
}
go c.Run(ctx)
return nil
}
// InitMetricsWithCertPath initializes telemetry reporting, using the node ID
// corresponding to the given certificate as the telemetry instance ID.
func InitMetricsWithCertPath(ctx context.Context, log *zap.Logger, r *monkit.Registry, certPath string) error {
var metricsID string
nodeID, err := identity.NodeIDFromCertPath(certPath)
if err != nil {
log.Sugar().Errorf("Could not read identity for telemetry setup: %v", err)
metricsID = "" // InitMetrics() will fill in a default value
} else {
metricsID = nodeID.String()
}
return InitMetrics(ctx, log, r, metricsID)
}
// InitMetricsWithHostname initializes telemetry reporting, using the hostname as the telemetry instance ID.
func InitMetricsWithHostname(ctx context.Context, log *zap.Logger, r *monkit.Registry) error {
var metricsID string
hostname, err := os.Hostname()
if err != nil {
log.Sugar().Errorf("Could not read hostname for telemetry setup: %v", err)
metricsID = "" // InitMetrics() will fill in a default value
} else {
metricsID = strings.ReplaceAll(hostname, ".", "_")
}
return InitMetrics(ctx, log, r, metricsID)
}

32
pkg/process/profiler.go
View File

@ -1,32 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package process
import (
"flag"
"cloud.google.com/go/profiler"
"github.com/zeebo/errs"
"go.uber.org/zap"
)
var (
initProfilerError = errs.Class("initializing profiler")
debugProfilerName = flag.String("debug.profilername", "", "provide the name of the peer to enable continuous cpu/mem profiling for")
)
func initProfiler(log *zap.Logger) error {
name := *debugProfilerName
if name != "" {
if err := profiler.Start(profiler.Config{
Service: name,
ServiceVersion: "",
}); err != nil {
return initProfilerError.Wrap(err)
}
log.Debug("success debug profiler init")
}
return nil
}

14
pkg/process/rand.go
View File

@ -1,14 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package process
import (
"math/rand"
"time"
)
func init() {
// Initialize the seed for the math/rand default source
rand.Seed(time.Now().UnixNano())
}

54
pkg/traces/traces.go
View File

@ -1,54 +0,0 @@
// Copyright (C) 2020 Storj Labs, Inc.
// See LICENSE for copying information.
package traces
import (
"context"
"fmt"
"log"
"os"
"path/filepath"
tracetagger "github.com/jtolds/tracetagger/v2"
monkit "github.com/spacemonkeygo/monkit/v3"
"github.com/spacemonkeygo/monkit/v3/collect"
)
var (
// TagDB tags a trace as hitting the database
TagDB = tracetagger.NewTagRef()
)
// CollectTraces starts storing all known tagged traces on disk, until cancel
// is called
func CollectTraces() (cancel func()) {
path := filepath.Join(os.TempDir(), "storj-traces", fmt.Sprint(os.Getpid()))
disable := TagDB.Enable()
cancelCollect := tracetagger.TracesWithTag(TagDB, 10000, func(spans []*collect.FinishedSpan, capped bool) {
name := tracetagger.TracePathPrefix(spans, capped)
err := tracetagger.SaveTrace(spans, capped, filepath.Join(path, "unfiltered", name))
if err != nil {
log.Print(err)
}
spans = tracetagger.JustTaggedSpans(spans, TagDB)
err = tracetagger.SaveTrace(spans, capped, filepath.Join(path, "filtered", name))
if err != nil {
log.Print(err)
}
})
return func() {
cancelCollect()
disable()
}
}
// Tag tags a trace with the given tag
func Tag(ctx context.Context, tag *tracetagger.TagRef) {
tracetagger.Tag(ctx, tag)
}
// TagScope tags all functions on an entire monkit Scope with a given tag
func TagScope(tag *tracetagger.TagRef, scope *monkit.Scope) {
tracetagger.TagScope(tag, scope)
}

2
private/tagsql/conn.go
View File

@ -9,7 +9,7 @@ import (
"errors"
"storj.io/common/context2"
"storj.io/storj/pkg/traces"
"storj.io/private/traces"
)
// Conn is an interface for *sql.Conn-like connections.

2
private/tagsql/db.go
View File

@ -15,7 +15,7 @@ import (
"time"
"storj.io/common/context2"
"storj.io/storj/pkg/traces"
"storj.io/private/traces"
)
// Open opens *sql.DB and wraps the implementation with tagging.

2
private/tagsql/stmt.go
View File

@ -7,7 +7,7 @@ import (
"context"
"database/sql"
"storj.io/storj/pkg/traces"
"storj.io/private/traces"
)
// Stmt is an interface for *sql.Stmt.

2
private/tagsql/tx.go
View File

@ -7,7 +7,7 @@ import (
"context"
"database/sql"
"storj.io/storj/pkg/traces"
"storj.io/private/traces"
)
// Tx is an interface for *sql.Tx-like transactions.

2
private/testplanet/satellite.go
View File

@ -22,7 +22,7 @@ import (
"storj.io/common/peertls/tlsopts"
"storj.io/common/rpc"
"storj.io/common/storj"
"storj.io/storj/pkg/debug"
"storj.io/private/debug"
"storj.io/storj/pkg/revocation"
"storj.io/storj/pkg/server"
"storj.io/storj/private/version"

2
private/testplanet/storagenode.go
View File

@ -18,7 +18,7 @@ import (
"storj.io/common/peertls/extensions"
"storj.io/common/peertls/tlsopts"
"storj.io/common/storj"
"storj.io/storj/pkg/debug"
"storj.io/private/debug"
"storj.io/storj/pkg/revocation"
"storj.io/storj/pkg/server"
"storj.io/storj/storagenode"

2
private/testplanet/uplink.go
View File

@ -23,8 +23,8 @@ import (
"storj.io/common/peertls/tlsopts"
"storj.io/common/rpc"
"storj.io/common/storj"
"storj.io/private/cfgstruct"
libuplink "storj.io/storj/lib/uplink"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/satellite/console"
"storj.io/uplink/private/metainfo"
"storj.io/uplink/private/piecestore"

2
satellite/admin.go
View File

@ -16,7 +16,7 @@ import (
"storj.io/common/identity"
"storj.io/common/peertls/extensions"
"storj.io/common/storj"
"storj.io/storj/pkg/debug"
"storj.io/private/debug"
"storj.io/storj/private/lifecycle"
"storj.io/storj/private/version"
"storj.io/storj/private/version/checker"

2
satellite/api.go
View File

@ -23,8 +23,8 @@ import (
"storj.io/common/rpc"
"storj.io/common/signing"
"storj.io/common/storj"
"storj.io/private/debug"
"storj.io/storj/pkg/auth/grpcauth"
"storj.io/storj/pkg/debug"
"storj.io/storj/pkg/server"
"storj.io/storj/private/lifecycle"
"storj.io/storj/private/post"

2
satellite/core.go
View File

@ -20,7 +20,7 @@ import (
"storj.io/common/rpc"
"storj.io/common/signing"
"storj.io/common/storj"
"storj.io/storj/pkg/debug"
"storj.io/private/debug"
"storj.io/storj/private/lifecycle"
"storj.io/storj/private/version"
version_checker "storj.io/storj/private/version/checker"

2
satellite/gc.go
View File

@ -18,7 +18,7 @@ import (
"storj.io/common/peertls/tlsopts"
"storj.io/common/rpc"
"storj.io/common/storj"
"storj.io/storj/pkg/debug"
"storj.io/private/debug"
"storj.io/storj/private/lifecycle"
"storj.io/storj/private/version"
version_checker "storj.io/storj/private/version/checker"

2
satellite/peer.go
View File

@ -9,7 +9,7 @@ import (
"github.com/spacemonkeygo/monkit/v3"
"storj.io/common/identity"
"storj.io/storj/pkg/debug"
"storj.io/private/debug"
"storj.io/storj/pkg/server"
version_checker "storj.io/storj/private/version/checker"
"storj.io/storj/satellite/accounting"

2
satellite/repairer.go
View File

@ -20,7 +20,7 @@ import (
"storj.io/common/rpc"
"storj.io/common/signing"
"storj.io/common/storj"
"storj.io/storj/pkg/debug"
"storj.io/private/debug"
"storj.io/storj/private/lifecycle"
"storj.io/storj/private/version"
version_checker "storj.io/storj/private/version/checker"

2
storagenode/peer.go
View File

@ -23,7 +23,7 @@ import (
"storj.io/common/rpc"
"storj.io/common/signing"
"storj.io/common/storj"
"storj.io/storj/pkg/debug"
"storj.io/private/debug"
"storj.io/storj/pkg/server"
"storj.io/storj/private/lifecycle"
"storj.io/storj/private/version"

1
storagenode/storagenodedb/schema.go
View File

@ -671,4 +671,3 @@ func Schema() map[string]*dbschema.Schema {
},
}
}