efcdaa43a3
* lib/uplink: encryption context Change-Id: I5c23dca3286a46b713b30c4997e9ae6e630b2280 * lib/uplink: bucket operation examples Change-Id: Ia0f6e69f365dcff0cf11c731f51b30842bce053b * lib/uplink: encryption key sharing test cases Change-Id: I3a172d565f33f4e591402cdcb9460664a7cc7fbe * fix encrypted path prefix restriction issue Change-Id: I8f3921f9d52aaf4b84039de608b8cbbc88769554 * implement panics in libuplink encryption code todo on cipher suite selection as well as an api concern Change-Id: Ifa39eb3cc4b3443f7d96f9304df9b2ac4ec4085d * implement GetProjectInfo api call to get salt Change-Id: Ic5f6b3be9ea35df48c1aa214ab5d355fb328e2cf * some fixes and accessors for encryption store Change-Id: I3bb61f6712a037900e2a96e72ad4029ec1d3f718 * general fixes to builds/tests/etc Change-Id: I9930fa96acb3b221d9a001f8e274af5729cc8a47 * java bindings changes Change-Id: Ia2bd4c9c69739c8d3154d79616cff1f36fb403b6 * get libuplink examples passing Change-Id: I828f09a144160e0a5dd932324f78491ae2ec8a07 * fix proto.lock file Change-Id: I2fbbf4d0976a7d0473c2645e6dcb21aaa3be7651 * fix proto.lock again Change-Id: I92702cf49e1a340eef6379c2be4f7c4a268112a9 * fix golint issues Change-Id: I631ff9f43307a58e3b25a58cbb4a4cc2495f5eb6 * more linting fixes Change-Id: I51f8f30b367b5bca14c94b15417b9a4c9e7aa0ce * bug fixed by structs bump Change-Id: Ibb03c691fce7606c35c08721b3ef0781ab48a38a * retrigger Change-Id: Ieee0470b6a2d07168a1578552e8e7f271ae93a13 * retrigger Change-Id: I753d63853171e6a436c104ce176048892eb974c5 * semantic merge conflict Change-Id: I9419448496de90340569047a6a16a1b858a7978a * update total to match prod defaults Change-Id: I693d55c1ebb28b5803ee1d26e9e198decf82308b * retrigger Change-Id: I28b74d5d6202f61aa3866fe407d423f6a0a14b9e * retrigger Change-Id: I6fd054885c715f602e2cef623fd464c42e88742c * retrigger Change-Id: I6a01bae88c72406d4ed5a8f13bf8a2b3c650bd2d
225 lines
5.7 KiB
Go
225 lines
5.7 KiB
Go
// Copyright (C) 2019 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package miniogw_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"flag"
|
|
"io/ioutil"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/minio/cli"
|
|
minio "github.com/minio/minio/cmd"
|
|
"github.com/spf13/pflag"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"go.uber.org/zap"
|
|
"go.uber.org/zap/zaptest"
|
|
|
|
"storj.io/storj/internal/s3client"
|
|
"storj.io/storj/internal/testcontext"
|
|
"storj.io/storj/internal/testidentity"
|
|
"storj.io/storj/internal/testplanet"
|
|
"storj.io/storj/internal/testrand"
|
|
libuplink "storj.io/storj/lib/uplink"
|
|
"storj.io/storj/pkg/cfgstruct"
|
|
"storj.io/storj/pkg/identity"
|
|
"storj.io/storj/pkg/macaroon"
|
|
"storj.io/storj/pkg/miniogw"
|
|
"storj.io/storj/pkg/storj"
|
|
"storj.io/storj/satellite/console"
|
|
"storj.io/storj/uplink"
|
|
)
|
|
|
|
type config struct {
|
|
Server miniogw.ServerConfig
|
|
Minio miniogw.MinioConfig
|
|
}
|
|
|
|
func TestUploadDownload(t *testing.T) {
|
|
t.Skip("disable because, keeps stalling CI intermittently")
|
|
|
|
ctx := testcontext.New(t)
|
|
defer ctx.Cleanup()
|
|
|
|
planet, err := testplanet.New(t, 1, 30, 0)
|
|
assert.NoError(t, err)
|
|
|
|
defer ctx.Check(planet.Shutdown)
|
|
|
|
// add project to satisfy constraint
|
|
project, err := planet.Satellites[0].DB.Console().Projects().Insert(context.Background(), &console.Project{
|
|
Name: "testProject",
|
|
})
|
|
assert.NoError(t, err)
|
|
|
|
apiKey, err := macaroon.NewAPIKey([]byte("testSecret"))
|
|
assert.NoError(t, err)
|
|
|
|
apiKeyInfo := console.APIKeyInfo{
|
|
ProjectID: project.ID,
|
|
Name: "testKey",
|
|
Secret: []byte("testSecret"),
|
|
}
|
|
|
|
// add api key to db
|
|
_, err = planet.Satellites[0].DB.Console().APIKeys().Create(context.Background(), apiKey.Head(), apiKeyInfo)
|
|
assert.NoError(t, err)
|
|
|
|
// bind default values to config
|
|
var gwCfg config
|
|
cfgstruct.Bind(&pflag.FlagSet{}, &gwCfg, cfgstruct.UseDevDefaults())
|
|
var uplinkCfg uplink.Config
|
|
cfgstruct.Bind(&pflag.FlagSet{}, &uplinkCfg, cfgstruct.UseDevDefaults())
|
|
|
|
// minio config directory
|
|
gwCfg.Minio.Dir = ctx.Dir("minio")
|
|
|
|
// addresses
|
|
gwCfg.Server.Address = "127.0.0.1:7777"
|
|
uplinkCfg.Client.SatelliteAddr = planet.Satellites[0].Addr()
|
|
|
|
// keys
|
|
uplinkCfg.Client.APIKey = "apiKey"
|
|
|
|
// Encryption key
|
|
passphrase := testrand.BytesInt(testrand.Intn(100) + 1)
|
|
|
|
encryptionKey, err := storj.NewKey(passphrase)
|
|
require.NoError(t, err)
|
|
filename := ctx.File("encryption.key")
|
|
err = ioutil.WriteFile(filename, encryptionKey[:], os.FileMode(0400))
|
|
require.NoError(t, err)
|
|
uplinkCfg.Enc.KeyFilepath = filename
|
|
|
|
// redundancy
|
|
uplinkCfg.RS.MinThreshold = 7
|
|
uplinkCfg.RS.RepairThreshold = 8
|
|
uplinkCfg.RS.SuccessThreshold = 9
|
|
uplinkCfg.RS.MaxThreshold = 10
|
|
|
|
planet.Start(ctx)
|
|
|
|
// create identity for gateway
|
|
ca, err := testidentity.NewTestCA(ctx)
|
|
assert.NoError(t, err)
|
|
identity, err := ca.NewIdentity()
|
|
assert.NoError(t, err)
|
|
|
|
// setup and start gateway
|
|
go func() {
|
|
// TODO: this leaks the gateway server, however it shouldn't
|
|
err := runGateway(ctx, gwCfg, uplinkCfg, zaptest.NewLogger(t), identity)
|
|
if err != nil {
|
|
t.Log(err)
|
|
}
|
|
}()
|
|
|
|
time.Sleep(100 * time.Millisecond)
|
|
|
|
client, err := s3client.NewMinio(s3client.Config{
|
|
S3Gateway: gwCfg.Server.Address,
|
|
Satellite: planet.Satellites[0].Addr(),
|
|
AccessKey: gwCfg.Minio.AccessKey,
|
|
SecretKey: gwCfg.Minio.SecretKey,
|
|
APIKey: uplinkCfg.Client.APIKey,
|
|
EncryptionKey: string(encryptionKey[:]),
|
|
NoSSL: true,
|
|
})
|
|
assert.NoError(t, err)
|
|
|
|
bucket := "bucket"
|
|
|
|
err = client.MakeBucket(bucket, "")
|
|
assert.NoError(t, err)
|
|
|
|
// generate enough data for a remote segment
|
|
data := []byte{}
|
|
for i := 0; i < 5000; i++ {
|
|
data = append(data, 'a')
|
|
}
|
|
|
|
objectName := "testdata"
|
|
|
|
err = client.Upload(bucket, objectName, data)
|
|
assert.NoError(t, err)
|
|
|
|
buffer := make([]byte, len(data))
|
|
|
|
bytes, err := client.Download(bucket, objectName, buffer)
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, string(data), string(bytes))
|
|
}
|
|
|
|
// runGateway creates and starts a gateway
|
|
func runGateway(ctx context.Context, gwCfg config, uplinkCfg uplink.Config, log *zap.Logger, ident *identity.FullIdentity) (err error) {
|
|
|
|
// set gateway flags
|
|
flags := flag.NewFlagSet("gateway", flag.ExitOnError)
|
|
flags.String("address", gwCfg.Server.Address, "")
|
|
flags.String("config-dir", gwCfg.Minio.Dir, "")
|
|
flags.Bool("quiet", true, "")
|
|
|
|
// create *cli.Context with gateway flags
|
|
cliCtx := cli.NewContext(cli.NewApp(), flags, nil)
|
|
|
|
// TODO: setting the flag on flagset and cliCtx seems redundant, but output is not quiet otherwise
|
|
err = cliCtx.Set("quiet", "true")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = os.Setenv("MINIO_ACCESS_KEY", gwCfg.Minio.AccessKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = os.Setenv("MINIO_SECRET_KEY", gwCfg.Minio.SecretKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
cfg := libuplink.Config{}
|
|
cfg.Volatile.TLS = struct {
|
|
SkipPeerCAWhitelist bool
|
|
PeerCAWhitelistPath string
|
|
}{
|
|
SkipPeerCAWhitelist: !uplinkCfg.TLS.UsePeerCAWhitelist,
|
|
PeerCAWhitelistPath: uplinkCfg.TLS.PeerCAWhitelistPath,
|
|
}
|
|
cfg.Volatile.MaxInlineSize = uplinkCfg.Client.MaxInlineSize
|
|
cfg.Volatile.MaxMemory = uplinkCfg.RS.MaxBufferMem
|
|
|
|
uplink, err := libuplink.NewUplink(ctx, &cfg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
apiKey, err := libuplink.ParseAPIKey(uplinkCfg.Client.APIKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
project, err := uplink.OpenProject(ctx, uplinkCfg.Client.SatelliteAddr, apiKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
gw := miniogw.NewStorjGateway(
|
|
project,
|
|
libuplink.NewEncryptionCtxWithDefaultKey(storj.Key{}),
|
|
storj.Cipher(uplinkCfg.Enc.PathType).ToCipherSuite(),
|
|
uplinkCfg.GetEncryptionScheme().ToEncryptionParameters(),
|
|
uplinkCfg.GetRedundancyScheme(),
|
|
uplinkCfg.Client.SegmentSize,
|
|
)
|
|
|
|
minio.StartGateway(cliCtx, miniogw.Logging(gw, log))
|
|
return errors.New("unexpected minio exit")
|
|
}
|