JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Divide uplink and gateway params set (#1026)

Browse Source 
* Divide uplink and gateway params set

* attempt to fix docker

* attempt to fix all in one

* test

* more reorganization

* fix compilation error

* fix imports order

* fix dependency

* rename structs

* keep minio params for now

* review comments

* remove manual flag check
This commit is contained in:
Michal Niewrzal 2019-02-04 16:50:06 +01:00 committed by GitHub
parent d1cc0dd85b
commit 383cb05e8e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 235 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/gateway/Dockerfile
View File

@ -5,12 +5,12 @@ FROM storjlabs/golang as build-env
# final stage
FROM alpine
ENV CONF_PATH=/root/.local/share/storj/uplink \
ENV CONF_PATH=/root/.local/share/storj/gateway \
API_KEY= \
SATELLITE_ADDR=
EXPOSE 7777
WORKDIR /app
VOLUME /root/.local/share/storj/uplink
VOLUME /root/.local/share/storj/gateway
COPY --from=build-env /app /app/gateway
COPY cmd/gateway/entrypoint /entrypoint
ENTRYPOINT ["/entrypoint"]

201
cmd/gateway/main.go
View File

@ -4,10 +4,205 @@
package main
import (
"storj.io/storj/cmd/uplink/cmd"
"context"
"crypto/rand"
"fmt"
"net"
"os"
"path/filepath"
base58 "github.com/jbenet/go-base58"
"github.com/spf13/cobra"
"go.uber.org/zap"
"storj.io/storj/internal/fpath"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/miniogw"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/storage/streams"
"storj.io/storj/pkg/storj"
)
func main() {
process.Exec(cmd.GWCmd)
// GatewayFlags configuration flags
type GatewayFlags struct {
APIKey string `default:"" help:"the api key to use for the satellite" setup:"true"`
GenerateTestCerts bool `default:"false" help:"generate sample TLS certs for Minio GW" setup:"true"`
SatelliteAddr string `default:"localhost:7778" help:"the address to use for the satellite" setup:"true"`
miniogw.Config
}
var (
// rootCmd represents the base gateway command when called without any subcommands
rootCmd = &cobra.Command{
Use: "gateway",
Short: "The Storj client-side S3 gateway",
Args: cobra.OnlyValidArgs,
}
setupCmd = &cobra.Command{
Use: "setup",
Short: "Create a gateway config file",
RunE: cmdSetup,
Annotations: map[string]string{"type": "setup"},
}
runCmd = &cobra.Command{
Use: "run",
Short: "Run the S3 gateway",
RunE: cmdRun,
}
defaultConfDir = fpath.ApplicationDir("storj", "gateway")
defaultIdentityDir = fpath.ApplicationDir("storj", "identity", "gateway")
setupCfg GatewayFlags
runCfg GatewayFlags
confDir string
identityDir string
)
func init() {
dirParam := cfgstruct.FindConfigDirParam()
if dirParam != "" {
defaultConfDir = dirParam
}
identityDirParam := cfgstruct.FindIdentityDirParam()
if identityDirParam != "" {
defaultIdentityDir = identityDirParam
}
rootCmd.PersistentFlags().StringVar(&confDir, "config-dir", defaultConfDir, "main directory for setup configuration")
err := rootCmd.PersistentFlags().SetAnnotation("config-dir", "setup", []string{"true"})
if err != nil {
zap.S().Error("Failed to set 'setup' annotation for 'config-dir'")
}
rootCmd.PersistentFlags().StringVar(&identityDir, "identity-dir", defaultIdentityDir, "main directory for gateway identity credentials")
err = rootCmd.PersistentFlags().SetAnnotation("identity-dir", "setup", []string{"true"})
if err != nil {
zap.S().Error("Failed to set 'setup' annotation for 'config-dir'")
}
rootCmd.AddCommand(runCmd)
rootCmd.AddCommand(setupCmd)
cfgstruct.Bind(runCmd.Flags(), &runCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.IdentityDir(defaultIdentityDir))
cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.IdentityDir(defaultIdentityDir))
}
func cmdSetup(cmd *cobra.Command, args []string) (err error) {
setupDir, err := filepath.Abs(confDir)
if err != nil {
return err
}
valid, _ := fpath.IsValidSetupDir(setupDir)
if !valid {
return fmt.Errorf("gateway configuration already exists (%v)", setupDir)
}
err = os.MkdirAll(setupDir, 0700)
if err != nil {
return err
}
if setupCfg.GenerateTestCerts {
minioCerts := filepath.Join(setupDir, "minio", "certs")
if err := os.MkdirAll(minioCerts, 0744); err != nil {
return err
}
if err := os.Link(setupCfg.Identity.CertPath, filepath.Join(minioCerts, "public.crt")); err != nil {
return err
}
if err := os.Link(setupCfg.Identity.KeyPath, filepath.Join(minioCerts, "private.key")); err != nil {
return err
}
}
overrides := map[string]interface{}{
"client.api-key": setupCfg.APIKey,
"client.pointer-db-addr": setupCfg.SatelliteAddr,
"client.overlay-addr": setupCfg.SatelliteAddr,
}
accessKeyFlag := cmd.Flag("minio.access-key")
if !accessKeyFlag.Changed {
accessKey, err := generateKey()
if err != nil {
return err
}
overrides[accessKeyFlag.Name] = accessKey
}
secretKeyFlag := cmd.Flag("minio.secret-key")
if !secretKeyFlag.Changed {
secretKey, err := generateKey()
if err != nil {
return err
}
overrides[secretKeyFlag.Name] = secretKey
}
return process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), overrides)
}
func cmdRun(cmd *cobra.Command, args []string) (err error) {
if _, err := runCfg.Identity.Load(); err != nil {
zap.S().Fatal(err)
}
address := runCfg.Server.Address
host, port, err := net.SplitHostPort(address)
if err != nil {
return err
}
if host == "" {
address = net.JoinHostPort("localhost", port)
}
fmt.Printf("Starting Storj S3-compatible gateway!\n\n")
fmt.Printf("Endpoint: %s\n", address)
fmt.Printf("Access key: %s\n", runCfg.Minio.AccessKey)
fmt.Printf("Secret key: %s\n", runCfg.Minio.SecretKey)
ctx := process.Ctx(cmd)
metainfo, _, err := runCfg.Metainfo(ctx)
if err != nil {
return err
}
if err := process.InitMetricsWithCertPath(ctx, nil, runCfg.Identity.CertPath); err != nil {
zap.S().Error("Failed to initialize telemetry batcher: ", err)
}
_, err = metainfo.ListBuckets(ctx, storj.BucketListOptions{Direction: storj.After})
if err != nil {
return fmt.Errorf("Failed to contact Satellite.\n"+
"Perhaps your configuration is invalid?\n%s", err)
}
return runCfg.Run(ctx)
}
func generateKey() (key string, err error) {
var buf [20]byte
_, err = rand.Read(buf[:])
if err != nil {
return "", err
}
return base58.Encode(buf[:]), nil
}
// Metainfo loads the storj.Metainfo
//
// Temporarily it also returns an instance of streams.Store until we improve
// the metainfo and streas implementations.
func (flags *GatewayFlags) Metainfo(ctx context.Context) (storj.Metainfo, streams.Store, error) {
identity, err := flags.Identity.Load()
if err != nil {
return nil, nil, err
}
return flags.GetMetainfo(ctx, identity)
}
func main() {
process.Exec(rootCmd)
}

2
cmd/uplink/cmd/cat.go
View File

@ -17,7 +17,7 @@ func init() {
Use: "cat",
Short: "Copies a Storj object to standard out",
RunE: catMain,
}, CLICmd)
}, RootCmd)
}
// catMain is the function executed when catCmd is called

2
cmd/uplink/cmd/cp.go
View File

@ -32,7 +32,7 @@ func init() {
Use: "cp",
Short: "Copies a local file or Storj object to another location locally or in Storj",
RunE: copyMain,
}, CLICmd)
}, RootCmd)
progress = cpCmd.Flags().Bool("progress", true, "if true, show progress")
}

2
cmd/uplink/cmd/ls.go
View File

@ -24,7 +24,7 @@ func init() {
Use: "ls",
Short: "List objects and prefixes or all buckets",
RunE: list,
}, CLICmd)
}, RootCmd)
recursiveFlag = lsCmd.Flags().Bool("recursive", false, "if true, list recursively")
}

2
cmd/uplink/cmd/mb.go
View File

@ -18,7 +18,7 @@ func init() {
Use: "mb",
Short: "Create a new bucket",
RunE: makeBucket,
}, CLICmd)
}, RootCmd)
}
func makeBucket(cmd *cobra.Command, args []string) error {

2
cmd/uplink/cmd/mount.go
View File

@ -32,7 +32,7 @@ func init() {
Use: "mount",
Short: "Mount a bucket",
RunE: mountBucket,
}, CLICmd)
}, RootCmd)
}
func mountBucket(cmd *cobra.Command, args []string) (err error) {

2
cmd/uplink/cmd/put.go
View File

@ -17,7 +17,7 @@ func init() {
Use: "put",
Short: "Copies data from standard in to a Storj object",
RunE: putMain,
}, CLICmd)
}, RootCmd)
}
// putMain is the function executed when putCmd is called

2
cmd/uplink/cmd/rb.go
View File

@ -18,7 +18,7 @@ func init() {
Use: "rb",
Short: "Remove an empty bucket",
RunE: deleteBucket,
}, CLICmd)
}, RootCmd)
}
func deleteBucket(cmd *cobra.Command, args []string) error {

2
cmd/uplink/cmd/rm.go
View File

@ -17,7 +17,7 @@ func init() {
Use: "rm",
Short: "Delete an object",
RunE: deleteObject,
}, CLICmd)
}, RootCmd)
}
func deleteObject(cmd *cobra.Command, args []string) error {

21
cmd/uplink/cmd/root.go
View File

@ -11,28 +11,17 @@ import (
"storj.io/storj/internal/fpath"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/miniogw"
"storj.io/storj/pkg/storage/streams"
"storj.io/storj/pkg/storj"
)
// Config is miniogw.Config configuration
type Config struct {
miniogw.Config
}
var cfg UplinkFlags
var cfg Config
// CLICmd represents the base CLI command when called without any subcommands
var CLICmd = &cobra.Command{
// RootCmd represents the base CLI command when called without any subcommands
var RootCmd = &cobra.Command{
Use: "uplink",
Short: "The Storj client-side CLI",
}
// GWCmd represents the base gateway command when called without any subcommands
var GWCmd = &cobra.Command{
Use: "gateway",
Short: "The Storj client-side S3 gateway",
Args: cobra.OnlyValidArgs,
}
func addCmd(cmd *cobra.Command, root *cobra.Command) *cobra.Command {
@ -58,7 +47,7 @@ func addCmd(cmd *cobra.Command, root *cobra.Command) *cobra.Command {
//
// Temporarily it also returns an instance of streams.Store until we improve
// the metainfo and streas implementations.
func (c *Config) Metainfo(ctx context.Context) (storj.Metainfo, streams.Store, error) {
func (c *UplinkFlags) Metainfo(ctx context.Context) (storj.Metainfo, streams.Store, error) {
identity, err := c.Identity.Load()
if err != nil {
return nil, nil, err

64
cmd/uplink/cmd/run.go
View File

@ -1,64 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package cmd
import (
"fmt"
"net"
"github.com/spf13/cobra"
"go.uber.org/zap"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/storj"
)
func init() {
addCmd(&cobra.Command{
Use: "run",
Short: "Run the S3 gateway",
RunE: cmdRun,
}, GWCmd)
}
func cmdRun(cmd *cobra.Command, args []string) (err error) {
if _, err := cfg.Identity.Load(); err != nil {
zap.S().Fatal(err)
}
for _, flagname := range args {
return fmt.Errorf("Invalid argument %#v. Try 'uplink run'", flagname)
}
address := cfg.Server.Address
host, port, err := net.SplitHostPort(address)
if err != nil {
return err
}
if host == "" {
address = net.JoinHostPort("localhost", port)
}
fmt.Printf("Starting Storj S3-compatible gateway!\n\n")
fmt.Printf("Endpoint: %s\n", address)
fmt.Printf("Access key: %s\n", cfg.Minio.AccessKey)
fmt.Printf("Secret key: %s\n", cfg.Minio.SecretKey)
ctx := process.Ctx(cmd)
metainfo, _, err := cfg.Metainfo(ctx)
if err != nil {
return err
}
if err := process.InitMetricsWithCertPath(ctx, nil, cfg.Identity.CertPath); err != nil {
zap.S().Error("Failed to initialize telemetry batcher: ", err)
}
_, err = metainfo.ListBuckets(ctx, storj.BucketListOptions{Direction: storj.After})
if err != nil {
return fmt.Errorf("Failed to contact Satellite.\n"+
"Perhaps your configuration is invalid?\n%s", err)
}
return cfg.Run(ctx)
}

93
cmd/uplink/cmd/setup.go
View File

@ -4,22 +4,27 @@
package cmd
import (
"crypto/rand"
"fmt"
"os"
"path/filepath"
base58 "github.com/jbenet/go-base58"
"github.com/spf13/cobra"
"go.uber.org/zap"
"storj.io/storj/internal/fpath"
"storj.io/storj/pkg/cfgstruct"
"storj.io/storj/pkg/identity"
"storj.io/storj/pkg/miniogw"
"storj.io/storj/pkg/process"
)
// UplinkFlags configuration flags
type UplinkFlags struct {
APIKey string `default:"" help:"the api key to use for the satellite" setup:"true"`
SatelliteAddr string `default:"localhost:7778" help:"the address to use for the satellite" setup:"true"`
miniogw.Config // TODO remove when GetMetainfo will be moved from minio
}
var (
setupCmd = &cobra.Command{
Use: "setup",
@ -27,25 +32,13 @@ var (
RunE: cmdSetup,
Annotations: map[string]string{"type": "setup"},
}
setupCfg struct {
Identity identity.SetupConfig
APIKey string `default:"" help:"the api key to use for the satellite" setup:"true"`
EncKey string `default:"" help:"your root encryption key" setup:"true"`
GenerateMinioCerts bool `default:"false" help:"generate sample TLS certs for Minio GW" setup:"true"`
SatelliteAddr string `default:"localhost:7778" help:"the address to use for the satellite" setup:"true"`
Server miniogw.ServerConfig
Minio miniogw.MinioConfig
Client miniogw.ClientConfig
RS miniogw.RSConfig
Enc miniogw.EncryptionConfig
}
setupCfg UplinkFlags
defaultConfDir = fpath.ApplicationDir("storj", "uplink")
defaultIdentityDir = fpath.ApplicationDir("storj", "identity", "uplink")
cliConfDir string
gwConfDir string
confDir string
identityDir string
)
@ -59,34 +52,28 @@ func init() {
defaultIdentityDir = identityDirParam
}
CLICmd.PersistentFlags().StringVar(&cliConfDir, "config-dir", defaultConfDir, "main directory for setup configuration")
GWCmd.PersistentFlags().StringVar(&gwConfDir, "config-dir", defaultConfDir, "main directory for setup configuration")
CLICmd.PersistentFlags().StringVar(&identityDir, "identity-dir", defaultIdentityDir, "main directory for uplink identity credentials")
err := CLICmd.PersistentFlags().SetAnnotation("identity-dir", "setup", []string{"true"})
if err != nil {
zap.S().Error("Failed to set 'setup' annotation for 'config-dir'")
}
GWCmd.PersistentFlags().StringVar(&identityDir, "identity-dir", defaultIdentityDir, "main directory for gateway identity credentials")
err = GWCmd.PersistentFlags().SetAnnotation("identity-dir", "setup", []string{"true"})
RootCmd.PersistentFlags().StringVar(&confDir, "config-dir", defaultConfDir, "main directory for setup configuration")
err := RootCmd.PersistentFlags().SetAnnotation("config-dir", "setup", []string{"true"})
if err != nil {
zap.S().Error("Failed to set 'setup' annotation for 'config-dir'")
}
CLICmd.AddCommand(setupCmd)
GWCmd.AddCommand(setupCmd)
RootCmd.PersistentFlags().StringVar(&identityDir, "identity-dir", defaultIdentityDir, "main directory for uplink identity credentials")
err = RootCmd.PersistentFlags().SetAnnotation("identity-dir", "setup", []string{"true"})
if err != nil {
zap.S().Error("Failed to set 'setup' annotation for 'config-dir'")
}
RootCmd.AddCommand(setupCmd)
cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, cfgstruct.ConfDir(defaultConfDir), cfgstruct.IdentityDir(defaultIdentityDir))
}
func cmdSetup(cmd *cobra.Command, args []string) (err error) {
setupDir, err := filepath.Abs(cliConfDir)
setupDir, err := filepath.Abs(confDir)
if err != nil {
return err
}
for _, flagname := range args {
return fmt.Errorf("%s - Invalid flag. Pleas see --help", flagname)
}
valid, _ := fpath.IsValidSetupDir(setupDir)
if !valid {
return fmt.Errorf("uplink configuration already exists (%v)", setupDir)
@ -97,51 +84,11 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
return err
}
if setupCfg.GenerateMinioCerts {
minioCerts := filepath.Join(setupDir, "minio", "certs")
if err := os.MkdirAll(minioCerts, 0744); err != nil {
return err
}
if err := os.Link(setupCfg.Identity.CertPath, filepath.Join(minioCerts, "public.crt")); err != nil {
return err
}
if err := os.Link(setupCfg.Identity.KeyPath, filepath.Join(minioCerts, "private.key")); err != nil {
return err
}
}
overrides := map[string]interface{}{
"client.api-key": setupCfg.APIKey,
"client.pointer-db-addr": setupCfg.SatelliteAddr,
"client.overlay-addr": setupCfg.SatelliteAddr,
"enc.key": setupCfg.EncKey,
}
accessKeyFlag := cmd.Flag("minio.access-key")
if !accessKeyFlag.Changed {
accessKey, err := generateAWSKey()
if err != nil {
return err
}
overrides[accessKeyFlag.Name] = accessKey
}
secretKeyFlag := cmd.Flag("minio.secret-key")
if !secretKeyFlag.Changed {
secretKey, err := generateAWSKey()
if err != nil {
return err
}
overrides[secretKeyFlag.Name] = secretKey
}
return process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), overrides)
}
func generateAWSKey() (key string, err error) {
var buf [20]byte
_, err = rand.Read(buf[:])
if err != nil {
return "", err
}
return base58.Encode(buf[:]), nil
}

2
cmd/uplink/main.go
View File

@ -9,5 +9,5 @@ import (
)
func main() {
process.Exec(cmd.CLICmd)
process.Exec(cmd.RootCmd)
}

3
satellite/satellitedb/dbx/satellitedb.dbx.go
View File

@ -9,6 +9,7 @@ import (
"database/sql"
"errors"
"fmt"
"math/rand"
"reflect"
"regexp"
"strconv"
@ -18,9 +19,7 @@ import (
"unicode"
"github.com/lib/pq"
"github.com/mattn/go-sqlite3"
"math/rand"
)
// Prevent conditional imports from causing build failures