393 lines
10 KiB
Go
393 lines
10 KiB
Go
// Copyright (C) 2019 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"fmt"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
base58 "github.com/jbenet/go-base58"
|
|
"github.com/minio/cli"
|
|
minio "github.com/minio/minio/cmd"
|
|
"github.com/spf13/cobra"
|
|
"github.com/zeebo/errs"
|
|
"go.uber.org/zap"
|
|
|
|
"storj.io/storj/cmd/internal/wizard"
|
|
"storj.io/storj/internal/fpath"
|
|
"storj.io/storj/internal/version"
|
|
libuplink "storj.io/storj/lib/uplink"
|
|
"storj.io/storj/pkg/cfgstruct"
|
|
"storj.io/storj/pkg/miniogw"
|
|
"storj.io/storj/pkg/process"
|
|
"storj.io/storj/pkg/storj"
|
|
"storj.io/storj/uplink"
|
|
"storj.io/storj/uplink/setup"
|
|
)
|
|
|
|
// GatewayFlags configuration flags
|
|
type GatewayFlags struct {
|
|
NonInteractive bool `help:"disable interactive mode" default:"false" setup:"true"`
|
|
|
|
Server miniogw.ServerConfig
|
|
Minio miniogw.MinioConfig
|
|
|
|
uplink.Config
|
|
|
|
Version version.Config
|
|
}
|
|
|
|
var (
|
|
// Error is the default gateway setup errs class
|
|
Error = errs.Class("gateway setup error")
|
|
// rootCmd represents the base gateway command when called without any subcommands
|
|
rootCmd = &cobra.Command{
|
|
Use: "gateway",
|
|
Short: "The Storj client-side S3 gateway",
|
|
Args: cobra.OnlyValidArgs,
|
|
}
|
|
setupCmd = &cobra.Command{
|
|
Use: "setup",
|
|
Short: "Create a gateway config file",
|
|
RunE: cmdSetup,
|
|
Annotations: map[string]string{"type": "setup"},
|
|
}
|
|
runCmd = &cobra.Command{
|
|
Use: "run",
|
|
Short: "Run the S3 gateway",
|
|
RunE: cmdRun,
|
|
}
|
|
|
|
setupCfg GatewayFlags
|
|
runCfg GatewayFlags
|
|
|
|
confDir string
|
|
identityDir string
|
|
)
|
|
|
|
func init() {
|
|
defaultConfDir := fpath.ApplicationDir("storj", "gateway")
|
|
defaultIdentityDir := fpath.ApplicationDir("storj", "identity", "gateway")
|
|
cfgstruct.SetupFlag(zap.L(), rootCmd, &confDir, "config-dir", defaultConfDir, "main directory for gateway configuration")
|
|
cfgstruct.SetupFlag(zap.L(), rootCmd, &identityDir, "identity-dir", defaultIdentityDir, "main directory for gateway identity credentials")
|
|
defaults := cfgstruct.DefaultsFlag(rootCmd)
|
|
|
|
rootCmd.AddCommand(runCmd)
|
|
rootCmd.AddCommand(setupCmd)
|
|
process.Bind(runCmd, &runCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
|
|
process.Bind(setupCmd, &setupCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir), cfgstruct.SetupMode())
|
|
}
|
|
|
|
func cmdSetup(cmd *cobra.Command, args []string) (err error) {
|
|
setupDir, err := filepath.Abs(confDir)
|
|
if err != nil {
|
|
return Error.Wrap(err)
|
|
}
|
|
|
|
valid, _ := fpath.IsValidSetupDir(setupDir)
|
|
if !valid {
|
|
return Error.New("gateway configuration already exists (%v)", setupDir)
|
|
}
|
|
|
|
err = os.MkdirAll(setupDir, 0700)
|
|
if err != nil {
|
|
return Error.Wrap(err)
|
|
}
|
|
|
|
overrides := map[string]interface{}{}
|
|
|
|
accessKeyFlag := cmd.Flag("minio.access-key")
|
|
if !accessKeyFlag.Changed {
|
|
accessKey, err := generateKey()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
overrides[accessKeyFlag.Name] = accessKey
|
|
}
|
|
|
|
secretKeyFlag := cmd.Flag("minio.secret-key")
|
|
if !secretKeyFlag.Changed {
|
|
secretKey, err := generateKey()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
overrides[secretKeyFlag.Name] = secretKey
|
|
}
|
|
|
|
// override is required because the default value of Enc.KeyFilepath is ""
|
|
// and setting the value directly in setupCfg.Enc.KeyFiletpath will set the
|
|
// value in the config file but commented out.
|
|
encryptionKeyFilepath := setupCfg.Enc.KeyFilepath
|
|
if encryptionKeyFilepath == "" {
|
|
encryptionKeyFilepath = filepath.Join(setupDir, ".encryption.key")
|
|
overrides["enc.key-filepath"] = encryptionKeyFilepath
|
|
}
|
|
|
|
if setupCfg.NonInteractive {
|
|
return setupCfg.nonInteractive(cmd, setupDir, encryptionKeyFilepath, overrides)
|
|
}
|
|
|
|
return setupCfg.interactive(cmd, setupDir, encryptionKeyFilepath, overrides)
|
|
}
|
|
|
|
func cmdRun(cmd *cobra.Command, args []string) (err error) {
|
|
address := runCfg.Server.Address
|
|
host, port, err := net.SplitHostPort(address)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if host == "" {
|
|
address = net.JoinHostPort("127.0.0.1", port)
|
|
}
|
|
|
|
ctx := process.Ctx(cmd)
|
|
|
|
if err := process.InitMetrics(ctx, zap.L(), nil, ""); err != nil {
|
|
zap.S().Error("Failed to initialize telemetry batcher: ", err)
|
|
}
|
|
|
|
err = version.CheckProcessVersion(ctx, zap.L(), runCfg.Version, version.Build, "Gateway")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
zap.S().Infof("Starting Storj S3-compatible gateway!\n\n")
|
|
zap.S().Infof("Endpoint: %s\n", address)
|
|
zap.S().Infof("Access key: %s\n", runCfg.Minio.AccessKey)
|
|
zap.S().Infof("Secret key: %s\n", runCfg.Minio.SecretKey)
|
|
|
|
err = checkCfg(ctx)
|
|
if err != nil {
|
|
zap.S().Warn("Failed to contact Satellite. Perhaps your configuration is invalid?")
|
|
return err
|
|
}
|
|
|
|
return runCfg.Run(ctx)
|
|
}
|
|
|
|
func generateKey() (key string, err error) {
|
|
var buf [20]byte
|
|
_, err = rand.Read(buf[:])
|
|
if err != nil {
|
|
return "", Error.Wrap(err)
|
|
}
|
|
return base58.Encode(buf[:]), nil
|
|
}
|
|
|
|
func checkCfg(ctx context.Context) (err error) {
|
|
proj, err := runCfg.openProject(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { err = errs.Combine(err, proj.Close()) }()
|
|
|
|
_, err = proj.ListBuckets(ctx, &storj.BucketListOptions{Direction: storj.Forward})
|
|
return err
|
|
}
|
|
|
|
// Run starts a Minio Gateway given proper config
|
|
func (flags GatewayFlags) Run(ctx context.Context) (err error) {
|
|
err = minio.RegisterGatewayCommand(cli.Command{
|
|
Name: "storj",
|
|
Usage: "Storj",
|
|
Action: func(cliCtx *cli.Context) error {
|
|
return flags.action(ctx, cliCtx)
|
|
},
|
|
HideHelpCommand: true,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// TODO(jt): Surely there is a better way. This is so upsetting
|
|
err = os.Setenv("MINIO_ACCESS_KEY", flags.Minio.AccessKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = os.Setenv("MINIO_SECRET_KEY", flags.Minio.SecretKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
minio.Main([]string{"storj", "gateway", "storj",
|
|
"--address", flags.Server.Address, "--config-dir", flags.Minio.Dir, "--quiet"})
|
|
return errs.New("unexpected minio exit")
|
|
}
|
|
|
|
func (flags GatewayFlags) action(ctx context.Context, cliCtx *cli.Context) (err error) {
|
|
gw, err := flags.NewGateway(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
minio.StartGateway(cliCtx, miniogw.Logging(gw, zap.L()))
|
|
return errs.New("unexpected minio exit")
|
|
}
|
|
|
|
// NewGateway creates a new minio Gateway
|
|
func (flags GatewayFlags) NewGateway(ctx context.Context) (gw minio.Gateway, err error) {
|
|
access, err := setup.LoadEncryptionAccess(ctx, flags.Enc)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
project, err := flags.openProject(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return miniogw.NewStorjGateway(
|
|
project,
|
|
access,
|
|
storj.CipherSuite(flags.Enc.PathType),
|
|
flags.GetEncryptionParameters(),
|
|
flags.GetRedundancyScheme(),
|
|
flags.Client.SegmentSize,
|
|
), nil
|
|
}
|
|
|
|
func (flags *GatewayFlags) newUplink(ctx context.Context) (*libuplink.Uplink, error) {
|
|
// Transform the gateway config flags to the libuplink config object
|
|
libuplinkCfg := &libuplink.Config{}
|
|
libuplinkCfg.Volatile.Log = zap.L()
|
|
libuplinkCfg.Volatile.MaxInlineSize = flags.Client.MaxInlineSize
|
|
libuplinkCfg.Volatile.MaxMemory = flags.RS.MaxBufferMem
|
|
libuplinkCfg.Volatile.PeerIDVersion = flags.TLS.PeerIDVersions
|
|
libuplinkCfg.Volatile.TLS.SkipPeerCAWhitelist = !flags.TLS.UsePeerCAWhitelist
|
|
libuplinkCfg.Volatile.TLS.PeerCAWhitelistPath = flags.TLS.PeerCAWhitelistPath
|
|
libuplinkCfg.Volatile.DialTimeout = flags.Client.DialTimeout
|
|
libuplinkCfg.Volatile.RequestTimeout = flags.Client.RequestTimeout
|
|
|
|
return libuplink.NewUplink(ctx, libuplinkCfg)
|
|
}
|
|
|
|
func (flags GatewayFlags) openProject(ctx context.Context) (*libuplink.Project, error) {
|
|
apiKey, err := libuplink.ParseAPIKey(flags.Client.APIKey)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
uplk, err := flags.newUplink(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return uplk.OpenProject(ctx, flags.Client.SatelliteAddr, apiKey)
|
|
}
|
|
|
|
// interactive creates the configuration of the gateway interactively.
|
|
//
|
|
// encryptionKeyFilepath should be set to the filepath indicated by the user or
|
|
// or to a default path whose directory tree exists.
|
|
func (flags GatewayFlags) interactive(
|
|
cmd *cobra.Command, setupDir string, encryptionKeyFilepath string, overrides map[string]interface{},
|
|
) error {
|
|
ctx := process.Ctx(cmd)
|
|
|
|
satelliteAddress, err := wizard.PromptForSatellite(cmd)
|
|
if err != nil {
|
|
return Error.Wrap(err)
|
|
}
|
|
|
|
apiKeyString, err := wizard.PromptForAPIKey()
|
|
if err != nil {
|
|
return Error.Wrap(err)
|
|
}
|
|
|
|
apiKey, err := libuplink.ParseAPIKey(apiKeyString)
|
|
if err != nil {
|
|
return Error.Wrap(err)
|
|
}
|
|
|
|
passphrase, err := wizard.PromptForEncryptionPassphrase()
|
|
if err != nil {
|
|
return Error.Wrap(err)
|
|
}
|
|
|
|
uplk, err := flags.newUplink(ctx)
|
|
if err != nil {
|
|
return Error.Wrap(err)
|
|
}
|
|
defer func() { err = errs.Combine(err, uplk.Close()) }()
|
|
|
|
project, err := uplk.OpenProject(ctx, satelliteAddress, apiKey)
|
|
if err != nil {
|
|
return Error.Wrap(err)
|
|
}
|
|
defer func() { err = errs.Combine(err, project.Close()) }()
|
|
|
|
key, err := project.SaltedKeyFromPassphrase(ctx, passphrase)
|
|
if err != nil {
|
|
return Error.Wrap(err)
|
|
}
|
|
|
|
err = setup.SaveEncryptionKey(string(key[:]), encryptionKeyFilepath)
|
|
if err != nil {
|
|
return Error.Wrap(err)
|
|
}
|
|
|
|
overrides["satellite-addr"] = satelliteAddress
|
|
overrides["api-key"] = apiKeyString
|
|
overrides["enc.key-filepath"] = encryptionKeyFilepath
|
|
|
|
err = process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), overrides)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
_, err = fmt.Printf(`
|
|
Your encryption key is saved to: %s
|
|
|
|
Your S3 Gateway is configured and ready to use!
|
|
|
|
Some things to try next:
|
|
|
|
* Run 'gateway --help' to see the operations that can be performed
|
|
|
|
* See https://github.com/storj/docs/blob/master/S3-Gateway.md#using-the-aws-s3-commandline-interface for some example commands
|
|
`, encryptionKeyFilepath)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
// nonInteractive creates the configuration of the gateway non-interactively.
|
|
//
|
|
// encryptionKeyFilepath should be set to the filepath indicated by the user or
|
|
// or to a default path whose directory tree exists.
|
|
func (flags GatewayFlags) nonInteractive(
|
|
cmd *cobra.Command, setupDir string, encryptionKeyFilepath string, overrides map[string]interface{},
|
|
) error {
|
|
if setupCfg.Enc.EncryptionKey != "" {
|
|
err := setup.SaveEncryptionKey(setupCfg.Enc.EncryptionKey, encryptionKeyFilepath)
|
|
if err != nil {
|
|
return Error.Wrap(err)
|
|
}
|
|
}
|
|
|
|
err := process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), overrides)
|
|
if err != nil {
|
|
return Error.Wrap(err)
|
|
}
|
|
|
|
if setupCfg.Enc.EncryptionKey != "" {
|
|
_, _ = fmt.Printf("Your encryption key is saved to: %s\n", encryptionKeyFilepath)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func main() {
|
|
process.Exec(rootCmd)
|
|
}
|