JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change where the encryption key is being stored for uplink (#1967)

Browse Source 
* uplink: Add a new flag to set the filepath of the file which is used for 
  saving the encryption key and rename the one that hold the encryption key and 
  establish that it has priority over the key stored in the file to make the 
  configuration usable without having a huge refactoring in test-sim.
* cmd/uplink: Adapt the setup subcommand for storing the user input key to a file 
  and adapt the rest of the subcommands for reading the key from the key-file when 
  the key isn't explicitly set with a command line flag.
* cmd/gateway: Adapt it to read the encryption key from the key-file or use the 
  one passed by a command line flag.
* pkg/process: Export the default configuration filename so other packages which 
  use the same value can reference to it rather than having it hardcoded.
* Adapt several integrations (scripts, etc.) to consider the changes applied in uplink and cmd packages.
This commit is contained in:
Ivan Fraixedes 2019-05-22 15:57:12 +02:00 committed by GitHub
parent a6c4019288
commit 69d8b9f828
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 781 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
cmd/gateway/main.go
View File

@ -211,8 +211,10 @@ func (flags GatewayFlags) action(ctx context.Context, cliCtx *cli.Context) (err
// NewGateway creates a new minio Gateway
func (flags GatewayFlags) NewGateway(ctx context.Context) (gw minio.Gateway, err error) {
encKey := new(storj.Key)
copy(encKey[:], flags.Enc.Key)
encKey, err := uplink.UseOrLoadEncryptionKey(flags.Enc.EncryptionKey, flags.Enc.KeyFilepath)
if err != nil {
return nil, err
}
project, err := flags.openProject(ctx)
if err != nil {
@ -241,23 +243,25 @@ func (flags GatewayFlags) openProject(ctx context.Context) (*libuplink.Project,
cfg.Volatile.MaxInlineSize = flags.Client.MaxInlineSize
cfg.Volatile.MaxMemory = flags.RS.MaxBufferMem
uplink, err := libuplink.NewUplink(ctx, &cfg)
if err != nil {
return nil, err
}
apiKey, err := libuplink.ParseAPIKey(flags.Client.APIKey)
if err != nil {
return nil, err
}
encKey := new(storj.Key)
copy(encKey[:], flags.Enc.Key)
encKey, err := uplink.UseOrLoadEncryptionKey(flags.Enc.EncryptionKey, flags.Enc.KeyFilepath)
if err != nil {
return nil, err
}
var opts libuplink.ProjectOptions
opts.Volatile.EncryptionKey = encKey
return uplink.OpenProject(ctx, flags.Client.SatelliteAddr, apiKey, &opts)
uplk, err := libuplink.NewUplink(ctx, &cfg)
if err != nil {
return nil, err
}
return uplk.OpenProject(ctx, flags.Client.SatelliteAddr, apiKey, &opts)
}
func (flags GatewayFlags) interactive(cmd *cobra.Command, setupDir string, overrides map[string]interface{}) error {

6
cmd/storj-sim/network.go
View File

@ -321,8 +321,6 @@ func newNetwork(flags *Flags) (*Processes, error) {
"--satellite-addr", satellite.Address,
"--enc.key=TestEncryptionKey",
"--rs.min-threshold", strconv.Itoa(1 * flags.StorageNodeCount / 5),
"--rs.repair-threshold", strconv.Itoa(2 * flags.StorageNodeCount / 5),
"--rs.success-threshold", strconv.Itoa(3 * flags.StorageNodeCount / 5),
@ -333,7 +331,9 @@ func newNetwork(flags *Flags) (*Processes, error) {
"--debug.addr", net.JoinHostPort(host, port(gatewayPeer, i, debugHTTP)),
},
"run": {},
"run": {
"--enc.encryption-key=TestEncryptionKey",
},
})
process.ExecBefore["run"] = func(process *Process) error {

41
cmd/uplink/cmd/common.go Normal file
View File

@ -0,0 +1,41 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package cmd
import (
libuplink "storj.io/storj/lib/uplink"
"storj.io/storj/pkg/storj"
"storj.io/storj/uplink"
)
// loadEncryptionAccess loads the encryption key stored in the file pointed by
// filepath and creates an EncryptionAccess with it.
func loadEncryptionAccess(filepath string) (libuplink.EncryptionAccess, error) {
key, err := uplink.LoadEncryptionKey(filepath)
if err != nil {
return libuplink.EncryptionAccess{}, err
}
return libuplink.EncryptionAccess{
Key: *key,
}, nil
}
// useOrLoadEncryptionAccess creates an encryption key from humanReadableKey
// when it isn't empty otherwise try to load the key from the file pointed by
// filepath and creates an EnryptionAccess with it.
func useOrLoadEncryptionAccess(humanReadableKey string, filepath string) (libuplink.EncryptionAccess, error) {
if humanReadableKey != "" {
key, err := storj.NewKey([]byte(humanReadableKey))
if err != nil {
return libuplink.EncryptionAccess{}, err
}
return libuplink.EncryptionAccess{
Key: *key,
}, nil
}
return loadEncryptionAccess(filepath)
}

109
cmd/uplink/cmd/common_test.go Normal file
View File

@ -0,0 +1,109 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package cmd
import (
"io/ioutil"
"math/rand"
"os"
"testing"
"github.com/stretchr/testify/require"
"storj.io/storj/internal/testcontext"
"storj.io/storj/pkg/storj"
)
func TestLoadEncryptionKeyIntoEncryptionAccess(t *testing.T) {
t.Run("ok", func(t *testing.T) {
passphrase := make([]byte, rand.Intn(100)+1)
_, err := rand.Read(passphrase)
require.NoError(t, err)
expectedKey, err := storj.NewKey(passphrase)
require.NoError(t, err)
ctx := testcontext.New(t)
filename := ctx.File("encryption.key")
err = ioutil.WriteFile(filename, expectedKey[:], os.FileMode(0400))
require.NoError(t, err)
defer ctx.Cleanup()
access, err := loadEncryptionAccess(filename)
require.NoError(t, err)
require.Equal(t, *expectedKey, access.Key)
})
t.Run("error", func(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
filename := ctx.File("encryption.key")
_, err := loadEncryptionAccess(filename)
require.Error(t, err)
})
}
func TestUseOrLoadEncryptionKeyIntoEncryptionAccess(t *testing.T) {
t.Run("ok: load", func(t *testing.T) {
passphrase := make([]byte, rand.Intn(100)+1)
_, err := rand.Read(passphrase)
require.NoError(t, err)
expectedKey, err := storj.NewKey(passphrase)
require.NoError(t, err)
ctx := testcontext.New(t)
filename := ctx.File("encryption.key")
err = ioutil.WriteFile(filename, expectedKey[:], os.FileMode(0400))
require.NoError(t, err)
defer ctx.Cleanup()
access, err := useOrLoadEncryptionAccess("", filename)
require.NoError(t, err)
require.Equal(t, *expectedKey, access.Key)
})
t.Run("ok: use", func(t *testing.T) {
rawKey := make([]byte, rand.Intn(100)+1)
_, err := rand.Read(rawKey)
require.NoError(t, err)
access, err := useOrLoadEncryptionAccess(string(rawKey), "")
require.NoError(t, err)
require.Equal(t, rawKey[:storj.KeySize], access.Key[:])
})
t.Run("error", func(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
filename := ctx.File("encryption.key")
_, err := useOrLoadEncryptionAccess("", filename)
require.Error(t, err)
})
}
func TestSaveLoadEncryptionKey(t *testing.T) {
var inputKey []byte
{
inputKey = make([]byte, rand.Intn(storj.KeySize)*3+1)
_, err := rand.Read(inputKey)
require.NoError(t, err)
}
ctx := testcontext.New(t)
defer ctx.Cleanup()
filename := ctx.File("storj-test-cmd-uplink", "encryption.key")
err := saveEncryptionKey(inputKey, filename)
require.NoError(t, err)
access, err := useOrLoadEncryptionAccess("", filename)
require.NoError(t, err)
if len(inputKey) > storj.KeySize {
require.Equal(t, inputKey[:storj.KeySize], access.Key[:])
} else {
require.Equal(t, inputKey, access.Key[:len(inputKey)])
}
}

18
cmd/uplink/cmd/cp.go
View File

@ -82,8 +82,10 @@ func upload(ctx context.Context, src fpath.FPath, dst fpath.FPath, showProgress
return fmt.Errorf("source cannot be a directory: %s", src)
}
var access libuplink.EncryptionAccess
copy(access.Key[:], []byte(cfg.Enc.Key))
access, err := useOrLoadEncryptionAccess(cfg.Enc.EncryptionKey, cfg.Enc.KeyFilepath)
if err != nil {
return err
}
project, bucket, err := cfg.GetProjectAndBucket(ctx, dst.Bucket(), access)
if err != nil {
@ -132,8 +134,10 @@ func download(ctx context.Context, src fpath.FPath, dst fpath.FPath, showProgres
return fmt.Errorf("destination must be local path: %s", dst)
}
var access libuplink.EncryptionAccess
copy(access.Key[:], []byte(cfg.Enc.Key))
access, err := useOrLoadEncryptionAccess(cfg.Enc.EncryptionKey, cfg.Enc.KeyFilepath)
if err != nil {
return err
}
project, bucket, err := cfg.GetProjectAndBucket(ctx, src.Bucket(), access)
if err != nil {
@ -208,8 +212,10 @@ func copyObject(ctx context.Context, src fpath.FPath, dst fpath.FPath) (err erro
return fmt.Errorf("destination must be Storj URL: %s", dst)
}
var access libuplink.EncryptionAccess
copy(access.Key[:], []byte(cfg.Enc.Key))
access, err := useOrLoadEncryptionAccess(cfg.Enc.EncryptionKey, cfg.Enc.KeyFilepath)
if err != nil {
return err
}
project, bucket, err := cfg.GetProjectAndBucket(ctx, dst.Bucket(), access)
if err != nil {

6
cmd/uplink/cmd/ls.go
View File

@ -42,8 +42,10 @@ func list(cmd *cobra.Command, args []string) error {
}
}()
var access libuplink.EncryptionAccess
copy(access.Key[:], []byte(cfg.Enc.Key))
access, err := useOrLoadEncryptionAccess(cfg.Enc.EncryptionKey, cfg.Enc.KeyFilepath)
if err != nil {
return err
}
if len(args) > 0 {
src, err := fpath.New(args[0])

3
cmd/uplink/cmd/mb.go
View File

@ -7,6 +7,7 @@ import (
"fmt"
"github.com/spf13/cobra"
"github.com/zeebo/errs"
"storj.io/storj/internal/fpath"
"storj.io/storj/internal/memory"
@ -45,7 +46,7 @@ func makeBucket(cmd *cobra.Command, args []string) error {
project, err := cfg.GetProject(ctx)
if err != nil {
return fmt.Errorf("error setting up project: %+v", err)
return errs.New("error setting up project: %+v", err)
}
defer func() {
if err := project.Close(); err != nil {

7
cmd/uplink/cmd/rb.go
View File

@ -9,7 +9,6 @@ import (
"github.com/spf13/cobra"
"storj.io/storj/internal/fpath"
libuplink "storj.io/storj/lib/uplink"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/storj"
)
@ -42,8 +41,10 @@ func deleteBucket(cmd *cobra.Command, args []string) error {
return fmt.Errorf("Nested buckets not supported, use format sj://bucket/")
}
var access libuplink.EncryptionAccess
copy(access.Key[:], []byte(cfg.Enc.Key))
access, err := useOrLoadEncryptionAccess(cfg.Enc.EncryptionKey, cfg.Enc.KeyFilepath)
if err != nil {
return err
}
project, bucket, err := cfg.GetProjectAndBucket(ctx, dst.Bucket(), access)
if err != nil {

7
cmd/uplink/cmd/rm.go
View File

@ -9,7 +9,6 @@ import (
"github.com/spf13/cobra"
"storj.io/storj/internal/fpath"
libuplink "storj.io/storj/lib/uplink"
"storj.io/storj/pkg/process"
)
@ -37,8 +36,10 @@ func deleteObject(cmd *cobra.Command, args []string) error {
return fmt.Errorf("No bucket specified, use format sj://bucket/")
}
var access libuplink.EncryptionAccess
copy(access.Key[:], []byte(cfg.Enc.Key))
access, err := useOrLoadEncryptionAccess(cfg.Enc.EncryptionKey, cfg.Enc.KeyFilepath)
if err != nil {
return err
}
project, bucket, err := cfg.GetProjectAndBucket(ctx, dst.Bucket(), access)
if err != nil {

13
cmd/uplink/cmd/root.go
View File

@ -82,21 +82,24 @@ func (c *UplinkFlags) GetProject(ctx context.Context) (*libuplink.Project, error
cfg.Volatile.MaxInlineSize = c.Client.MaxInlineSize
cfg.Volatile.MaxMemory = c.RS.MaxBufferMem
uplink, err := c.NewUplink(ctx, cfg)
uplk, err := c.NewUplink(ctx, cfg)
if err != nil {
return nil, err
}
opts := &libuplink.ProjectOptions{}
encKey := new(storj.Key)
copy(encKey[:], c.Enc.Key)
encKey, err := uplink.UseOrLoadEncryptionKey(c.Enc.EncryptionKey, c.Enc.KeyFilepath)
if err != nil {
return nil, err
}
opts.Volatile.EncryptionKey = encKey
project, err := uplink.OpenProject(ctx, satelliteAddr, apiKey, opts)
project, err := uplk.OpenProject(ctx, satelliteAddr, apiKey, opts)
if err != nil {
if err := uplink.Close(); err != nil {
if err := uplk.Close(); err != nil {
fmt.Printf("error closing uplink: %+v\n", err)
}
}

253
cmd/uplink/cmd/setup.go
View File

@ -29,9 +29,10 @@ var (
Annotations: map[string]string{"type": "setup"},
}
setupCfg UplinkFlags
confDir string
defaults cfgstruct.BindOpt
setupCfg UplinkFlags
confDir string
encryptionKeyFilepath string
defaults cfgstruct.BindOpt
// Error is the default uplink setup errs class
Error = errs.Class("uplink setup error")
@ -43,6 +44,9 @@ func init() {
defaults = cfgstruct.DefaultsFlag(RootCmd)
RootCmd.AddCommand(setupCmd)
cfgstruct.BindSetup(setupCmd.Flags(), &setupCfg, defaults, cfgstruct.ConfDir(confDir))
defaultEncryptionKeyFilepath := filepath.Join(defaultConfDir, ".encryption.key")
cfgstruct.SetupFlag(zap.L(), setupCmd, &encryptionKeyFilepath, "enc.key-filepath", defaultEncryptionKeyFilepath, "path to the file which contains the encryption key")
}
func cmdSetup(cmd *cobra.Command, args []string) (err error) {
@ -67,111 +71,137 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
return err
}
var override map[string]interface{}
if !setupCfg.NonInteractive {
_, err = fmt.Print(`
// override is required because the default value of Enc.KeyFilepath is ""
// and setting the value directly in setupCfg.Enc.KeyFiletpathon will set the
// value in the config file but commented out.
usedEncryptionKeyFilepath := setupCfg.Enc.KeyFilepath
if usedEncryptionKeyFilepath == "" {
usedEncryptionKeyFilepath, err = filepath.Abs(encryptionKeyFilepath)
if err != nil {
return err
}
}
if setupCfg.NonInteractive {
override := map[string]interface{}{
"enc.key-filepath": usedEncryptionKeyFilepath,
}
return process.SaveConfigWithAllDefaults(
cmd.Flags(), filepath.Join(setupDir, process.DefaultCfgFilename), override)
}
_, err = fmt.Print(`
Pick satellite to use:
[1] mars.tardigrade.io
[2] jupiter.tardigrade.io
[3] saturn.tardigrade.io
[1] mars.tardigrade.io
[2] jupiter.tardigrade.io
[3] saturn.tardigrade.io
Please enter numeric choice or enter satellite address manually [1]: `)
if err != nil {
if err != nil {
return err
}
satellites := []string{"mars.tardigrade.io", "jupiter.tardigrade.io", "saturn.tardigrade.io"}
var satelliteAddress string
n, err := fmt.Scanln(&satelliteAddress)
if err != nil {
if n == 0 {
// fmt.Scanln cannot handle empty input
satelliteAddress = satellites[0]
} else {
return err
}
satellites := []string{"mars.tardigrade.io", "jupiter.tardigrade.io", "saturn.tardigrade.io"}
var satelliteAddress string
n, err := fmt.Scanln(&satelliteAddress)
if err != nil {
if n == 0 {
// fmt.Scanln cannot handle empty input
satelliteAddress = satellites[0]
} else {
return err
}
}
}
// TODO add better validation
if satelliteAddress == "" {
return errs.New("satellite address cannot be empty")
} else if len(satelliteAddress) == 1 {
switch satelliteAddress {
case "1":
satelliteAddress = satellites[0]
case "2":
satelliteAddress = satellites[1]
case "3":
satelliteAddress = satellites[2]
default:
return errs.New("Satellite address cannot be one character")
}
// TODO add better validation
if satelliteAddress == "" {
return errs.New("satellite address cannot be empty")
} else if len(satelliteAddress) == 1 {
switch satelliteAddress {
case "1":
satelliteAddress = satellites[0]
case "2":
satelliteAddress = satellites[1]
case "3":
satelliteAddress = satellites[2]
default:
return errs.New("Satellite address cannot be one character")
}
}
satelliteAddress, err = ApplyDefaultHostAndPortToAddr(satelliteAddress, cmd.Flags().Lookup("satellite-addr").Value.String())
if err != nil {
return err
}
satelliteAddress, err = ApplyDefaultHostAndPortToAddr(
satelliteAddress, cmd.Flags().Lookup("satellite-addr").Value.String())
if err != nil {
return err
}
_, err = fmt.Print("Enter your API key: ")
if err != nil {
return err
}
var apiKey string
n, err = fmt.Scanln(&apiKey)
if err != nil && n != 0 {
return err
}
_, err = fmt.Print("Enter your API key: ")
if err != nil {
return err
}
var apiKey string
n, err = fmt.Scanln(&apiKey)
if err != nil && n != 0 {
return err
}
if apiKey == "" {
return errs.New("API key cannot be empty")
}
if apiKey == "" {
return errs.New("API key cannot be empty")
}
_, err = fmt.Print("Enter your encryption passphrase: ")
if err != nil {
return err
}
encKey, err := terminal.ReadPassword(int(os.Stdin.Fd()))
if err != nil {
return err
}
_, err = fmt.Print("Enter your encryption key: ")
if err != nil {
return err
}
humanReadableKey, err := terminal.ReadPassword(int(os.Stdin.Fd()))
if err != nil {
return err
}
_, err = fmt.Println()
if err != nil {
return err
}
_, err = fmt.Println()
if err != nil {
return err
}
if len(encKey) == 0 {
return errs.New("Encryption passphrase cannot be empty")
}
if len(humanReadableKey) == 0 {
return errs.New("Encryption key cannot be empty")
}
_, err = fmt.Print("Enter your encryption passphrase again: ")
if err != nil {
return err
}
repeatedEncKey, err := terminal.ReadPassword(int(os.Stdin.Fd()))
if err != nil {
return err
}
_, err = fmt.Println()
if err != nil {
return err
}
_, err = fmt.Print("Enter your encryption key again: ")
if err != nil {
return err
}
repeatedHumanReadableKey, err := terminal.ReadPassword(int(os.Stdin.Fd()))
if err != nil {
return err
}
_, err = fmt.Println()
if err != nil {
return err
}
if !bytes.Equal(encKey, repeatedEncKey) {
return errs.New("encryption passphrases doesn't match")
}
if !bytes.Equal(humanReadableKey, repeatedHumanReadableKey) {
return errs.New("encryption keys don't match")
}
override = map[string]interface{}{
"satellite-addr": satelliteAddress,
"api-key": apiKey,
"enc.key": string(encKey),
}
err = saveEncryptionKey(humanReadableKey, usedEncryptionKeyFilepath)
if err != nil {
return err
}
err = process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), override)
if err != nil {
return nil
}
var override = map[string]interface{}{
"api-key": apiKey,
"satellite-addr": satelliteAddress,
"enc.key-filepath": usedEncryptionKeyFilepath,
}
_, err = fmt.Println(`
err = process.SaveConfigWithAllDefaults(
cmd.Flags(), filepath.Join(setupDir, process.DefaultCfgFilename), override)
if err != nil {
return nil
}
// if there is an error with this we cannot do that much and the setup process
// has ended OK, so we ignore it.
_, _ = fmt.Println(`
Your Uplink CLI is configured and ready to use!
Some things to try next:
@ -179,15 +209,9 @@ Some things to try next:
* Run 'uplink --help' to see the operations that can be performed
* See https://github.com/storj/docs/blob/master/Uplink-CLI.md#usage for some example commands
`)
if err != nil {
return nil
}
`)
return nil
}
return process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), nil)
return nil
}
// ApplyDefaultHostAndPortToAddrFlag applies the default host and/or port if either is missing in the specified flag name.
@ -247,3 +271,34 @@ func ApplyDefaultHostAndPortToAddr(address, defaultAddress string) (string, erro
// address is host:
return net.JoinHostPort(addressParts[0], defaultPort), nil
}
// saveEncryptionKey generates a Storj key from the inputKey and save it into a
// new file created in filepath.
func saveEncryptionKey(inputKey []byte, filepath string) error {
switch {
case len(inputKey) == 0:
return Error.New("inputKey is empty")
case filepath == "":
return Error.New("filepath is empty")
}
file, err := os.OpenFile(filepath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err != nil {
if os.IsNotExist(err) {
return Error.New("directory path doesn't exist. %+v", err)
}
if os.IsExist(err) {
return Error.New("file key already exists. %+v", err)
}
return Error.Wrap(err)
}
defer func() {
err = Error.Wrap(errs.Combine(err, file.Close()))
}()
_, err = file.Write(inputKey)
return err
}

87
cmd/uplink/cmd/setup_unexported_test.go Normal file
View File

@ -0,0 +1,87 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package cmd
import (
"io/ioutil"
"math/rand"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"storj.io/storj/internal/testcontext"
"storj.io/storj/pkg/storj"
)
func TestSaveEncryptionKey(t *testing.T) {
generateInputKey := func() []byte {
inputKey := make([]byte, rand.Intn(storj.KeySize*3)+1)
_, err := rand.Read(inputKey)
require.NoError(t, err)
return inputKey
}
t.Run("ok", func(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
inputKey := generateInputKey()
filename := ctx.File("storj-test-cmd-uplink", "encryption.key")
err := saveEncryptionKey(inputKey, filename)
require.NoError(t, err)
savedKey, err := ioutil.ReadFile(filename)
require.NoError(t, err)
assert.Equal(t, inputKey, savedKey)
})
t.Run("error: empty input key", func(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
filename := ctx.File("storj-test-cmd-uplink", "encryption.key")
err := saveEncryptionKey(nil, filename)
require.Error(t, err)
err = saveEncryptionKey([]byte{}, filename)
require.Error(t, err)
})
t.Run("error: empty filepath", func(t *testing.T) {
inputKey := generateInputKey()
err := saveEncryptionKey(inputKey, "")
require.Error(t, err)
})
t.Run("error: unexisting dir", func(t *testing.T) {
// Create a directory and remove it for making sure that the path doesn't
// exist
ctx := testcontext.New(t)
dir := ctx.Dir("storj-test-cmd-uplink")
ctx.Cleanup()
inputKey := generateInputKey()
filename := filepath.Join(dir, "enc.key")
err := saveEncryptionKey(inputKey, filename)
require.Errorf(t, err, "directory path doesn't exist")
})
t.Run("error: file already exists", func(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
inputKey := generateInputKey()
filename := ctx.File("encryption.key")
require.NoError(t, ioutil.WriteFile(filename, nil, os.FileMode(0600)))
err := saveEncryptionKey(inputKey, filename)
require.Errorf(t, err, "file key already exists")
})
}

8
cmd/uplink/cmd/share.go
View File

@ -101,8 +101,12 @@ func shareMain(cmd *cobra.Command, args []string) (err error) {
caveat.NotAfter = notAfter
var project *libuplink.Project
var access libuplink.EncryptionAccess
copy(access.Key[:], []byte(cfg.Enc.Key))
access, err := useOrLoadEncryptionAccess(cfg.Enc.EncryptionKey, cfg.Enc.KeyFilepath)
if err != nil {
return err
}
cache := make(map[string]*libuplink.BucketConfig)
for _, path := range shareCfg.AllowedPathPrefix {

32
pkg/miniogw/integration_test.go
View File

@ -7,6 +7,8 @@ import (
"context"
"errors"
"flag"
"io/ioutil"
"math/rand"
"os"
"testing"
"time"
@ -15,6 +17,7 @@ import (
minio "github.com/minio/minio/cmd"
"github.com/spf13/pflag"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
"go.uber.org/zap/zaptest"
@ -79,7 +82,18 @@ func TestUploadDownload(t *testing.T) {
// keys
uplinkCfg.Client.APIKey = "apiKey"
uplinkCfg.Enc.Key = "encKey"
// Encryption key
passphrase := make([]byte, rand.Intn(100)+1)
_, err = rand.Read(passphrase)
require.NoError(t, err)
encryptionKey, err := storj.NewKey(passphrase)
require.NoError(t, err)
filename := ctx.File("encryption.key")
err = ioutil.WriteFile(filename, encryptionKey[:], os.FileMode(0400))
require.NoError(t, err)
uplinkCfg.Enc.KeyFilepath = filename
// redundancy
uplinkCfg.RS.MinThreshold = 7
@ -112,7 +126,7 @@ func TestUploadDownload(t *testing.T) {
AccessKey: gwCfg.Minio.AccessKey,
SecretKey: gwCfg.Minio.SecretKey,
APIKey: uplinkCfg.Client.APIKey,
EncryptionKey: uplinkCfg.Enc.Key,
EncryptionKey: string(encryptionKey[:]),
NoSSL: true,
})
assert.NoError(t, err)
@ -190,8 +204,18 @@ func runGateway(ctx context.Context, gwCfg config, uplinkCfg uplink.Config, log
return err
}
encKey := new(storj.Key)
copy(encKey[:], uplinkCfg.Enc.Key)
var encKey *storj.Key
{
rawKey, err := ioutil.ReadFile(uplinkCfg.Enc.KeyFilepath)
if err != nil {
return err
}
encKey, err = storj.NewKey(rawKey)
if err != nil {
return err
}
}
var projectOptions libuplink.ProjectOptions
projectOptions.Volatile.EncryptionKey = encKey

14
pkg/pkcrypto/doc.go Normal file
View File

@ -0,0 +1,14 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
/*Package pkcrypto contains a set of helper functions and constants to perform
common cryptographic operations like:
* Signing and verification
* Public and private key generation
* Certification generation
*/
package pkcrypto

5
pkg/process/exec_conf.go
View File

@ -30,6 +30,9 @@ import (
"storj.io/storj/internal/version"
)
// DefaultCfgFilename is the default filename used for storing a configuration.
const DefaultCfgFilename = "config.yaml"
var (
mon = monkit.Package()
@ -182,7 +185,7 @@ func cleanup(cmd *cobra.Command) {
cfgFlag := cmd.Flags().Lookup("config-dir")
if cfgFlag != nil && cfgFlag.Value.String() != "" {
path := filepath.Join(os.ExpandEnv(cfgFlag.Value.String()), "config.yaml")
path := filepath.Join(os.ExpandEnv(cfgFlag.Value.String()), DefaultCfgFilename)
if cmd.Annotations["type"] != "setup" || fileExists(path) {
vip.SetConfigFile(path)
err = vip.ReadInConfig()

7
pkg/storj/doc.go Normal file
View File

@ -0,0 +1,7 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
/*Package storj contains the types which represent the main entities of the
Storj domain.
*/
package storj

16
pkg/storj/encryption.go
View File

@ -131,6 +131,17 @@ const (
NonceSize = 24
)
// NewKey creates a new Storj key from humanReadableKey.
func NewKey(humanReadableKey []byte) (*Key, error) {
var key Key
// Because of backward compatibility the key is filled with 0 or truncated if
// humanReadableKey isn't of the same size that KeySize.
// See https://github.com/storj/storj/pull/1967#discussion_r285544849
copy(key[:], humanReadableKey)
return &key, nil
}
// Key represents the largest key used by any encryption protocol
type Key [KeySize]byte
@ -139,6 +150,11 @@ func (key *Key) Raw() *[KeySize]byte {
return (*[KeySize]byte)(key)
}
// IsZero returns true if key is nil or it points to its zero value
func (key *Key) IsZero() bool {
return key == nil || *key == (Key{})
}
// Nonce represents the largest nonce used by any encryption protocol
type Nonce [NonceSize]byte

105
pkg/storj/encryption_test.go Normal file
View File

@ -0,0 +1,105 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package storj_test
import (
"math/rand"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"storj.io/storj/pkg/storj"
)
func TestNewKey(t *testing.T) {
t.Run("nil humanReadableKey", func(t *testing.T) {
t.Parallel()
key, err := storj.NewKey(nil)
require.NoError(t, err)
require.True(t, key.IsZero(), "key isn't zero value")
})
t.Run("empty humanReadableKey", func(t *testing.T) {
t.Parallel()
key, err := storj.NewKey([]byte{})
require.NoError(t, err)
require.True(t, key.IsZero(), "key isn't zero value")
})
t.Run("humanReadableKey is of KeySize length", func(t *testing.T) {
t.Parallel()
humanReadableKey := make([]byte, storj.KeySize)
_, err := rand.Read(humanReadableKey)
require.NoError(t, err)
key, err := storj.NewKey(humanReadableKey)
require.NoError(t, err)
require.Equal(t, humanReadableKey, key[:])
})
t.Run("humanReadableKey is shorter than KeySize", func(t *testing.T) {
t.Parallel()
humanReadableKey := make([]byte, rand.Intn(storj.KeySize))
_, err := rand.Read(humanReadableKey)
require.NoError(t, err)
key, err := storj.NewKey(humanReadableKey)
require.NoError(t, err)
require.Equal(t, humanReadableKey, key[:len(humanReadableKey)])
})
t.Run("humanReadableKey is larger than KeySize", func(t *testing.T) {
t.Parallel()
humanReadableKey := make([]byte, rand.Intn(10)+storj.KeySize+1)
_, err := rand.Read(humanReadableKey)
require.NoError(t, err)
key, err := storj.NewKey(humanReadableKey)
require.NoError(t, err)
assert.Equal(t, humanReadableKey[:storj.KeySize], key[:])
})
t.Run("same human readable key produce the same key", func(t *testing.T) {
t.Parallel()
humanReadableKey := make([]byte, rand.Intn(storj.KeySize)+10)
_, err := rand.Read(humanReadableKey)
require.NoError(t, err)
key1, err := storj.NewKey(humanReadableKey)
require.NoError(t, err)
key2, err := storj.NewKey(humanReadableKey)
require.NoError(t, err)
assert.Equal(t, key1, key2, "keys are equal")
})
}
func TestKey_IsZero(t *testing.T) {
t.Run("nil", func(t *testing.T) {
var key *storj.Key
require.True(t, key.IsZero())
wrapperFn := func(key *storj.Key) bool {
return key.IsZero()
}
require.True(t, wrapperFn(nil))
})
t.Run("zero", func(t *testing.T) {
key := &storj.Key{}
require.True(t, key.IsZero())
})
t.Run("no nil/zero", func(t *testing.T) {
key := &storj.Key{'k'}
require.False(t, key.IsZero())
})
}

18
scripts/test-uplink.sh
View File

@ -25,20 +25,20 @@ random_bytes_file () {
random_bytes_file 2x1024 "$SRC_DIR/small-upload-testfile" # create 2kb file of random bytes (inline)
random_bytes_file 5x1024x1024 "$SRC_DIR/big-upload-testfile" # create 5mb file of random bytes (remote)
uplink --config-dir "$GATEWAY_0_DIR" mb "sj://$BUCKET/"
uplink --config-dir "$GATEWAY_0_DIR" --enc.encryption-key "test-uplink" mb "sj://$BUCKET/"
uplink --config-dir "$GATEWAY_0_DIR" cp "$SRC_DIR/small-upload-testfile" "sj://$BUCKET/"
uplink --config-dir "$GATEWAY_0_DIR" cp "$SRC_DIR/big-upload-testfile" "sj://$BUCKET/"
uplink --config-dir "$GATEWAY_0_DIR" --enc.encryption-key "test-uplink" cp "$SRC_DIR/small-upload-testfile" "sj://$BUCKET/"
uplink --config-dir "$GATEWAY_0_DIR" --enc.encryption-key "test-uplink" cp "$SRC_DIR/big-upload-testfile" "sj://$BUCKET/"
uplink --config-dir "$GATEWAY_0_DIR" cp "sj://$BUCKET/small-upload-testfile" "$DST_DIR"
uplink --config-dir "$GATEWAY_0_DIR" cp "sj://$BUCKET/big-upload-testfile" "$DST_DIR"
uplink --config-dir "$GATEWAY_0_DIR" --enc.encryption-key "test-uplink" cp "sj://$BUCKET/small-upload-testfile" "$DST_DIR"
uplink --config-dir "$GATEWAY_0_DIR" --enc.encryption-key "test-uplink" cp "sj://$BUCKET/big-upload-testfile" "$DST_DIR"
uplink --config-dir "$GATEWAY_0_DIR" rm "sj://$BUCKET/small-upload-testfile"
uplink --config-dir "$GATEWAY_0_DIR" rm "sj://$BUCKET/big-upload-testfile"
uplink --config-dir "$GATEWAY_0_DIR" --enc.encryption-key "test-uplink" rm "sj://$BUCKET/small-upload-testfile"
uplink --config-dir "$GATEWAY_0_DIR" --enc.encryption-key "test-uplink" rm "sj://$BUCKET/big-upload-testfile"
uplink --config-dir "$GATEWAY_0_DIR" ls "sj://$BUCKET"
uplink --config-dir "$GATEWAY_0_DIR" --enc.encryption-key "test-uplink" ls "sj://$BUCKET"
uplink --config-dir "$GATEWAY_0_DIR" rb "sj://$BUCKET"
uplink --config-dir "$GATEWAY_0_DIR" --enc.encryption-key "test-uplink" rb "sj://$BUCKET"
if cmp "$SRC_DIR/small-upload-testfile" "$DST_DIR/small-upload-testfile"
then

49
uplink/config.go
View File

@ -6,6 +6,7 @@ package uplink
import (
"context"
"errors"
"io/ioutil"
"time"
"github.com/vivint/infectious"
@ -41,10 +42,11 @@ type RSConfig struct {
// EncryptionConfig is a configuration struct that keeps details about
// encrypting segments
type EncryptionConfig struct {
Key string `help:"root key for encrypting the data"`
BlockSize memory.Size `help:"size (in bytes) of encrypted blocks" default:"1KiB"`
DataType int `help:"Type of encryption to use for content and metadata (1=AES-GCM, 2=SecretBox)" default:"1"`
PathType int `help:"Type of encryption to use for paths (0=Unencrypted, 1=AES-GCM, 2=SecretBox)" default:"1"`
EncryptionKey string `help:"the root key for encrypting the data; when set, it overrides the key stored in the file indicated by the key-filepath flag"`
KeyFilepath string `help:"the path to the file which contains the root key for encrypting the data"`
BlockSize memory.Size `help:"size (in bytes) of encrypted blocks" default:"1KiB"`
DataType int `help:"Type of encryption to use for content and metadata (1=AES-GCM, 2=SecretBox)" default:"1"`
PathType int `help:"Type of encryption to use for paths (0=Unencrypted, 1=AES-GCM, 2=SecretBox)" default:"1"`
}
// ClientConfig is a configuration struct for the uplink that controls how
@ -119,8 +121,10 @@ func (c Config) GetMetainfo(ctx context.Context, identity *identity.FullIdentity
return nil, nil, err
}
key := new(storj.Key)
copy(key[:], c.Enc.Key)
key, err := UseOrLoadEncryptionKey(c.Enc.EncryptionKey, c.Enc.KeyFilepath)
if err != nil {
return nil, nil, Error.Wrap(err)
}
streams, err := streams.NewStreamStore(segments, c.Client.SegmentSize.Int64(), key, c.Enc.BlockSize.Int(), storj.Cipher(c.Enc.DataType))
if err != nil {
@ -150,3 +154,36 @@ func (c Config) GetEncryptionScheme() storj.EncryptionScheme {
BlockSize: int32(c.Enc.BlockSize),
}
}
// LoadEncryptionKey loads the encryption key stored in the file pointed by
// filepath.
//
// An error is file is not found or there is an I/O error.
func LoadEncryptionKey(filepath string) (key *storj.Key, error error) {
if filepath == "" {
return &storj.Key{}, nil
}
rawKey, err := ioutil.ReadFile(filepath)
if err != nil {
return nil, err
}
return storj.NewKey(rawKey)
}
// UseOrLoadEncryptionKey return an encryption key from humanReadableKey when
// it isn't empty otherwise try to load the key from the file pointed by
// filepath calling LoadEncryptionKey function.
func UseOrLoadEncryptionKey(humanReadableKey string, filepath string) (*storj.Key, error) {
if humanReadableKey != "" {
key, err := storj.NewKey([]byte(humanReadableKey))
if err != nil {
return nil, err
}
return key, nil
}
return LoadEncryptionKey(filepath)
}

107
uplink/config_test.go Normal file
View File

@ -0,0 +1,107 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package uplink_test
import (
"io/ioutil"
"math/rand"
"os"
"testing"
"github.com/stretchr/testify/require"
"storj.io/storj/internal/testcontext"
"storj.io/storj/pkg/storj"
"storj.io/storj/uplink"
)
func TestLoadEncryptionKey(t *testing.T) {
saveRawKey := func(key []byte) (filepath string, clenaup func()) {
t.Helper()
ctx := testcontext.New(t)
filename := ctx.File("encryption.key")
err := ioutil.WriteFile(filename, key, os.FileMode(0400))
require.NoError(t, err)
return filename, ctx.Cleanup
}
t.Run("ok: reading from file", func(t *testing.T) {
passphrase := make([]byte, rand.Intn(100)+1)
_, err := rand.Read(passphrase)
require.NoError(t, err)
expectedKey, err := storj.NewKey(passphrase)
require.NoError(t, err)
filename, cleanup := saveRawKey(expectedKey[:])
defer cleanup()
key, err := uplink.LoadEncryptionKey(filename)
require.NoError(t, err)
require.Equal(t, expectedKey, key)
})
t.Run("ok: empty filepath", func(t *testing.T) {
key, err := uplink.LoadEncryptionKey("")
require.NoError(t, err)
require.Equal(t, &storj.Key{}, key)
})
t.Run("error: file not found", func(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
filename := ctx.File("encryption.key")
_, err := uplink.LoadEncryptionKey(filename)
require.Error(t, err)
})
}
func TestUseOrLoadEncryptionKey(t *testing.T) {
saveRawKey := func(key []byte) (filepath string, clenaup func()) {
t.Helper()
ctx := testcontext.New(t)
filename := ctx.File("encryption.key")
err := ioutil.WriteFile(filename, key, os.FileMode(0400))
require.NoError(t, err)
return filename, ctx.Cleanup
}
t.Run("ok: load", func(t *testing.T) {
passphrase := make([]byte, rand.Intn(100)+1)
_, err := rand.Read(passphrase)
require.NoError(t, err)
expectedKey, err := storj.NewKey(passphrase)
require.NoError(t, err)
filename, cleanup := saveRawKey(expectedKey[:])
defer cleanup()
key, err := uplink.UseOrLoadEncryptionKey("", filename)
require.NoError(t, err)
require.Equal(t, expectedKey, key)
})
t.Run("ok: use", func(t *testing.T) {
rawKey := make([]byte, rand.Intn(100)+1)
_, err := rand.Read(rawKey)
require.NoError(t, err)
key, err := uplink.UseOrLoadEncryptionKey(string(rawKey), "")
require.NoError(t, err)
require.Equal(t, rawKey[:storj.KeySize], key[:])
})
t.Run("error", func(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
filename := ctx.File("encryption.key")
_, err := uplink.UseOrLoadEncryptionKey("", filename)
require.Error(t, err)
})
}