d70f6e3760
What: This change moves project-level bucket metadata encryption information to the volatile section, because it is unlikely to remain in future releases Why: Ultimately, the web user interface will allow bucket management (creation, removal, etc), but not object management as that requires an encryption key for sure and we don't want to have users give the satellite their encryption keys. At a high level, a (*Project) type should map to all of the things you can do inside the web user interface within a project, which by necessity cannot have an encryption key. So, we really don't want an encryption key in the non-volatile section of this library.
216 lines
5.3 KiB
Go
216 lines
5.3 KiB
Go
// Copyright (C) 2019 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package miniogw_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"flag"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/minio/cli"
|
|
minio "github.com/minio/minio/cmd"
|
|
"github.com/spf13/pflag"
|
|
"github.com/stretchr/testify/assert"
|
|
"go.uber.org/zap"
|
|
"go.uber.org/zap/zaptest"
|
|
|
|
"storj.io/storj/internal/s3client"
|
|
"storj.io/storj/internal/testcontext"
|
|
"storj.io/storj/internal/testidentity"
|
|
"storj.io/storj/internal/testplanet"
|
|
libuplink "storj.io/storj/lib/uplink"
|
|
"storj.io/storj/pkg/cfgstruct"
|
|
"storj.io/storj/pkg/identity"
|
|
"storj.io/storj/pkg/miniogw"
|
|
"storj.io/storj/pkg/storj"
|
|
"storj.io/storj/satellite/console"
|
|
"storj.io/storj/uplink"
|
|
)
|
|
|
|
type config struct {
|
|
Server miniogw.ServerConfig
|
|
Minio miniogw.MinioConfig
|
|
}
|
|
|
|
func TestUploadDownload(t *testing.T) {
|
|
t.Skip("disable because, keeps stalling Travis intermittently")
|
|
|
|
ctx := testcontext.New(t)
|
|
defer ctx.Cleanup()
|
|
|
|
planet, err := testplanet.New(t, 1, 30, 0)
|
|
assert.NoError(t, err)
|
|
|
|
defer ctx.Check(planet.Shutdown)
|
|
|
|
// add project to satisfy constraint
|
|
project, err := planet.Satellites[0].DB.Console().Projects().Insert(context.Background(), &console.Project{
|
|
Name: "testProject",
|
|
})
|
|
|
|
assert.NoError(t, err)
|
|
|
|
apiKey := console.APIKey{}
|
|
apiKeyInfo := console.APIKeyInfo{
|
|
ProjectID: project.ID,
|
|
Name: "testKey",
|
|
}
|
|
|
|
// add api key to db
|
|
_, err = planet.Satellites[0].DB.Console().APIKeys().Create(context.Background(), apiKey, apiKeyInfo)
|
|
assert.NoError(t, err)
|
|
|
|
// bind default values to config
|
|
var gwCfg config
|
|
cfgstruct.Bind(&pflag.FlagSet{}, &gwCfg, true)
|
|
var uplinkCfg uplink.Config
|
|
cfgstruct.Bind(&pflag.FlagSet{}, &uplinkCfg, true)
|
|
|
|
// minio config directory
|
|
gwCfg.Minio.Dir = ctx.Dir("minio")
|
|
|
|
// addresses
|
|
gwCfg.Server.Address = "127.0.0.1:7777"
|
|
uplinkCfg.Client.SatelliteAddr = planet.Satellites[0].Addr()
|
|
|
|
// keys
|
|
uplinkCfg.Client.APIKey = "apiKey"
|
|
uplinkCfg.Enc.Key = "encKey"
|
|
|
|
// redundancy
|
|
uplinkCfg.RS.MinThreshold = 7
|
|
uplinkCfg.RS.RepairThreshold = 8
|
|
uplinkCfg.RS.SuccessThreshold = 9
|
|
uplinkCfg.RS.MaxThreshold = 10
|
|
|
|
planet.Start(ctx)
|
|
|
|
// create identity for gateway
|
|
ca, err := testidentity.NewTestCA(ctx)
|
|
assert.NoError(t, err)
|
|
identity, err := ca.NewIdentity()
|
|
assert.NoError(t, err)
|
|
|
|
// setup and start gateway
|
|
go func() {
|
|
// TODO: this leaks the gateway server, however it shouldn't
|
|
err := runGateway(ctx, gwCfg, uplinkCfg, zaptest.NewLogger(t), identity)
|
|
if err != nil {
|
|
t.Log(err)
|
|
}
|
|
}()
|
|
|
|
time.Sleep(100 * time.Millisecond)
|
|
|
|
client, err := s3client.NewMinio(s3client.Config{
|
|
S3Gateway: gwCfg.Server.Address,
|
|
Satellite: planet.Satellites[0].Addr(),
|
|
AccessKey: gwCfg.Minio.AccessKey,
|
|
SecretKey: gwCfg.Minio.SecretKey,
|
|
APIKey: uplinkCfg.Client.APIKey,
|
|
EncryptionKey: uplinkCfg.Enc.Key,
|
|
NoSSL: true,
|
|
})
|
|
assert.NoError(t, err)
|
|
|
|
bucket := "bucket"
|
|
|
|
err = client.MakeBucket(bucket, "")
|
|
assert.NoError(t, err)
|
|
|
|
// generate enough data for a remote segment
|
|
data := []byte{}
|
|
for i := 0; i < 5000; i++ {
|
|
data = append(data, 'a')
|
|
}
|
|
|
|
objectName := "testdata"
|
|
|
|
err = client.Upload(bucket, objectName, data)
|
|
assert.NoError(t, err)
|
|
|
|
buffer := make([]byte, len(data))
|
|
|
|
bytes, err := client.Download(bucket, objectName, buffer)
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, string(data), string(bytes))
|
|
}
|
|
|
|
// runGateway creates and starts a gateway
|
|
func runGateway(ctx context.Context, gwCfg config, uplinkCfg uplink.Config, log *zap.Logger, ident *identity.FullIdentity) (err error) {
|
|
|
|
// set gateway flags
|
|
flags := flag.NewFlagSet("gateway", flag.ExitOnError)
|
|
flags.String("address", gwCfg.Server.Address, "")
|
|
flags.String("config-dir", gwCfg.Minio.Dir, "")
|
|
flags.Bool("quiet", true, "")
|
|
|
|
// create *cli.Context with gateway flags
|
|
cliCtx := cli.NewContext(cli.NewApp(), flags, nil)
|
|
|
|
// TODO: setting the flag on flagset and cliCtx seems redundant, but output is not quiet otherwise
|
|
err = cliCtx.Set("quiet", "true")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = os.Setenv("MINIO_ACCESS_KEY", gwCfg.Minio.AccessKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = os.Setenv("MINIO_SECRET_KEY", gwCfg.Minio.SecretKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
cfg := libuplink.Config{}
|
|
cfg.Volatile.TLS = struct {
|
|
SkipPeerCAWhitelist bool
|
|
PeerCAWhitelistPath string
|
|
}{
|
|
SkipPeerCAWhitelist: !uplinkCfg.TLS.UsePeerCAWhitelist,
|
|
PeerCAWhitelistPath: uplinkCfg.TLS.PeerCAWhitelistPath,
|
|
}
|
|
cfg.Volatile.UseIdentity = ident
|
|
cfg.Volatile.MaxInlineSize = uplinkCfg.Client.MaxInlineSize
|
|
cfg.Volatile.MaxMemory = uplinkCfg.RS.MaxBufferMem
|
|
|
|
uplink, err := libuplink.NewUplink(ctx, &cfg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
apiKey, err := libuplink.ParseAPIKey(uplinkCfg.Client.APIKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
encKey := new(storj.Key)
|
|
copy(encKey[:], uplinkCfg.Enc.Key)
|
|
|
|
var projectOptions libuplink.ProjectOptions
|
|
projectOptions.Volatile.EncryptionKey = encKey
|
|
project, err := uplink.OpenProject(ctx, uplinkCfg.Client.SatelliteAddr, apiKey, &projectOptions)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
gw := miniogw.NewStorjGateway(
|
|
project,
|
|
encKey,
|
|
storj.Cipher(uplinkCfg.Enc.PathType).ToCipherSuite(),
|
|
uplinkCfg.GetEncryptionScheme().ToEncryptionParameters(),
|
|
uplinkCfg.GetRedundancyScheme(),
|
|
uplinkCfg.Client.SegmentSize,
|
|
)
|
|
|
|
minio.StartGateway(cliCtx, miniogw.Logging(gw, log))
|
|
return errors.New("unexpected minio exit")
|
|
}
|