JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

aws s3 performance tests (#2060)

Browse Source 
* add aws s3 benchmark script

* add s3 benchmark tests

* rearrange so smaller diff, fix spelling

* add configurable uplink config for s3-benchmark

* make new bucket w/unique name for each s3 test

* changes per CR
This commit is contained in:
Jess G 2019-05-28 11:46:58 -07:00 committed by GitHub
parent dc2e6b9370
commit fb86238acc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 178 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/s3-benchmark/main.go
View File

@ -30,6 +30,7 @@ func main() {
flag.StringVar(&conf.APIKey, "apikey", "abc123", "api key")
flag.StringVar(&conf.EncryptionKey, "encryptionkey", "abc123", "encryption key")
flag.BoolVar(&conf.NoSSL, "no-ssl", false, "disable ssl")
flag.StringVar(&conf.ConfigDir, "config-dir", "", "path of config dir to use. If empty, a config will be created.")
clientName := flag.String("client", "minio", "client to use for requests (supported: minio, aws-cli, uplink)")

178
cmd/uplink/cmd/cp_test.go
View File

@ -15,10 +15,6 @@ import (
"storj.io/storj/internal/s3client"
)
const (
bucket = "testbucket"
)
var benchmarkCases = []struct {
name string
objectsize memory.Size
@ -48,7 +44,7 @@ func createObjects() map[string][]byte {
}
// uplinkSetup setups an uplink to use for testing uploads/downloads
func uplinkSetup() s3client.Client {
func uplinkSetup(bucket string) s3client.Client {
conf, err := setupConfig()
if err != nil {
log.Fatalf("failed to setup s3client config: %+v\n", err)
@ -72,7 +68,7 @@ func setupConfig() (s3client.Config, error) {
var conf s3client.Config
conf.EncryptionKey = uplinkEncryptionKey
conf.Satellite = getEnvOrDefault("SATELLITE_0_ADDR", defaultSatelliteAddr)
conf.APIKey = getEnvOrDefault(os.Getenv("GATEWAY_0_API_KEY"), os.Getenv("apiKey"))
conf.APIKey = getEnvOrDefault("GATEWAY_0_API_KEY", os.Getenv("apiKey"))
if conf.APIKey == "" {
return conf, errors.New("no api key provided. Expecting an env var $GATEWAY_0_API_KEY or $apiKey")
}
@ -86,13 +82,72 @@ func getEnvOrDefault(key, fallback string) string {
return fallback
}
func BenchmarkUpload(b *testing.B) {
var client = uplinkSetup()
func BenchmarkUpload_Uplink(b *testing.B) {
bucket := "testbucket"
client := uplinkSetup(bucket)
// uploadedObjects is used to store the names of all objects that are uploaded
// so that we can make sure to delete them all during cleanup
var uploadedObjects = map[string][]string{}
uploadedObjects = benchmarkUpload(b, client, bucket, uploadedObjects)
teardown(client, bucket, uploadedObjects)
}
func teardown(client s3client.Client, bucket string, uploadedObjects map[string][]string) {
for _, bm := range benchmarkCases {
for _, objectPath := range uploadedObjects[bm.name] {
err := client.Delete(bucket, objectPath)
if err != nil {
log.Printf("failed to delete object %q: %+v\n", objectPath, err)
}
}
}
err := client.RemoveBucket(bucket)
if err != nil {
log.Fatalf("failed to remove bucket %q: %+v\n", bucket, err)
}
}
func BenchmarkDownload_Uplink(b *testing.B) {
bucket := "testbucket"
client := uplinkSetup(bucket)
// upload some test objects so that there is something to download
uploadTestObjects(client, bucket)
benchmarkDownload(b, bucket, client)
teardownTestObjects(client, bucket)
}
func uploadTestObjects(client s3client.Client, bucket string) {
for name, data := range testObjects {
objectName := "folder/data_" + name
err := client.Upload(bucket, objectName, data)
if err != nil {
log.Fatalf("failed to upload object %q: %+v\n", objectName, err)
}
}
}
func teardownTestObjects(client s3client.Client, bucket string) {
for name := range testObjects {
objectName := "folder/data_" + name
err := client.Delete(bucket, objectName)
if err != nil {
log.Fatalf("failed to delete object %q: %+v\n", objectName, err)
}
}
err := client.RemoveBucket(bucket)
if err != nil {
log.Fatalf("failed to remove bucket %q: %+v\n", bucket, err)
}
}
func benchmarkUpload(b *testing.B, client s3client.Client, bucket string, uploadedObjects map[string][]string) map[string][]string {
for _, bm := range benchmarkCases {
b.Run(bm.name, func(b *testing.B) {
b.SetBytes(bm.objectsize.Int64())
@ -114,32 +169,10 @@ func BenchmarkUpload(b *testing.B) {
}
})
}
teardown(client, uploadedObjects)
return uploadedObjects
}
func teardown(client s3client.Client, uploadedObjects map[string][]string) {
for _, bm := range benchmarkCases {
for _, objectPath := range uploadedObjects[bm.name] {
err := client.Delete(bucket, objectPath)
if err != nil {
log.Printf("failed to delete object %q: %+v\n", objectPath, err)
}
}
}
err := client.RemoveBucket(bucket)
if err != nil {
log.Fatalf("failed to remove bucket %q: %+v\n", bucket, err)
}
}
func BenchmarkDownload(b *testing.B) {
var client = uplinkSetup()
// upload some test objects so that there is something to download
uploadTestObjects(client)
func benchmarkDownload(b *testing.B, bucket string, client s3client.Client) {
for _, bm := range benchmarkCases {
b.Run(bm.name, func(b *testing.B) {
buf := make([]byte, bm.objectsize)
@ -147,37 +180,72 @@ func BenchmarkDownload(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
objectName := "folder/data_" + bm.name
_, err := client.Download(bucket, objectName, buf)
out, err := client.Download(bucket, objectName, buf)
if err != nil {
log.Fatalf("failed to download object %q: %+v\n", objectName, err)
}
expectedBytes := bm.objectsize.Int()
actualBytes := len(out)
if actualBytes != expectedBytes {
log.Fatalf("err downloading object %q: Expected %d bytes, but got actual bytes: %d\n",
objectName, expectedBytes, actualBytes,
)
}
}
})
}
teardownTestObjects(client)
}
func uploadTestObjects(client s3client.Client) {
for name, data := range testObjects {
objectName := "folder/data_" + name
err := client.Upload(bucket, objectName, data)
if err != nil {
log.Fatalf("failed to upload object %q: %+v\n", objectName, err)
}
}
}
func teardownTestObjects(client s3client.Client) {
for name := range testObjects {
objectName := "folder/data_" + name
err := client.Delete(bucket, objectName)
if err != nil {
log.Fatalf("failed to delete object %q: %+v\n", objectName, err)
}
}
err := client.RemoveBucket(bucket)
func s3ClientSetup(bucket string) s3client.Client {
conf, err := s3ClientConfigSetup()
if err != nil {
log.Fatalf("failed to remove bucket %q: %+v\n", bucket, err)
log.Fatalf("failed to setup s3client config: %+v\n", err)
}
client, err := s3client.NewAWSCLI(conf)
if err != nil {
log.Fatalf("failed to create s3client NewUplink: %+v\n", err)
}
const region = "us-east-1"
err = client.MakeBucket(bucket, region)
if err != nil {
log.Fatalf("failed to create bucket with s3client %q: %+v\n", bucket, err)
}
return client
}
func s3ClientConfigSetup() (s3client.Config, error) {
const s3gateway = "https://s3.amazonaws.com/"
var conf s3client.Config
conf.S3Gateway = s3gateway
conf.AccessKey = os.Getenv("AWS_ACCESS_KEY_ID")
conf.SecretKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
if conf.AccessKey == "" || conf.SecretKey == "" {
return conf, errors.New("expecting environment variables $AWS_ACCESS_KEY_ID and $AWS_SECRET_ACCESS_KEY to be set")
}
return conf, nil
}
func BenchmarkUpload_S3(b *testing.B) {
bucket := "testbucket3bgdp2xbkkflxc2tallstvh6pb824r"
var client = s3ClientSetup(bucket)
// uploadedObjects is used to store the names of all objects that are uploaded
// so that we can make sure to delete them all during cleanup
var uploadedObjects = map[string][]string{}
uploadedObjects = benchmarkUpload(b, client, bucket, uploadedObjects)
teardown(client, bucket, uploadedObjects)
}
func BenchmarkDownload_S3(b *testing.B) {
bucket := "testbucket3bgdp2xbkkflxc2tallstvh6pb826a"
var client = s3ClientSetup(bucket)
// upload some test objects so that there is something to download
uploadTestObjects(client, bucket)
benchmarkDownload(b, bucket, client)
teardownTestObjects(client, bucket)
}

1
internal/s3client/common.go
View File

@ -12,6 +12,7 @@ type Config struct {
APIKey string
EncryptionKey string
NoSSL bool
ConfigDir string
}
// Client is the common interface for different implementations

59
internal/s3client/uplink.go
View File

@ -8,7 +8,6 @@ import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/zeebo/errs"
@ -28,37 +27,31 @@ type Uplink struct {
func NewUplink(conf Config) (Client, error) {
client := &Uplink{conf}
defaultConfDir := fpath.ApplicationDir("storj", "uplink")
setupDir, err := filepath.Abs(defaultConfDir)
if client.conf.ConfigDir != "" {
fmt.Printf("Using existing uplink config at %s\n", client.conf.ConfigDir)
return client, nil
}
client.conf.ConfigDir = fpath.ApplicationDir("storj", "s3-client", "uplink")
// remove existing s3client uplink config so that
// we can create a new one with up-to-date settings
err := os.RemoveAll(client.conf.ConfigDir)
if err != nil {
return nil, UplinkError.Wrap(fullExitError(err))
}
validForSetup, _ := fpath.IsValidSetupDir(setupDir)
// uplink configuration doesn't exists
if validForSetup {
fmt.Printf(`No existing uplink configuration located at (%v)...
Creating uplink configuration with the following settings:
"--non-interactive: true",
"--api-key: %s",
"--enc.key: %s",
"--satellite-addr: %s
`,
setupDir, client.conf.APIKey, client.conf.EncryptionKey, client.conf.Satellite,
)
cmd := client.cmd("setup",
"--non-interactive", "true",
"--api-key", client.conf.APIKey,
"--enc.key", client.conf.EncryptionKey,
"--satellite-addr", client.conf.Satellite)
cmd := client.cmd("--config-dir", client.conf.ConfigDir,
"setup",
"--non-interactive", "true",
"--api-key", client.conf.APIKey,
"--enc.encryption-key", client.conf.EncryptionKey,
"--satellite-addr", client.conf.Satellite,
)
_, err := cmd.Output()
if err != nil {
return nil, UplinkError.Wrap(fullExitError(err))
}
} else {
// if uplink config file already exists, use the current config
fmt.Printf("Using existing uplink configuration from (%v). To pass in new settings, delete existing configs first\n", setupDir)
_, err = cmd.Output()
if err != nil {
return nil, UplinkError.Wrap(fullExitError(err))
}
return client, nil
@ -67,6 +60,8 @@ func NewUplink(conf Config) (Client, error) {
func (client *Uplink) cmd(subargs ...string) *exec.Cmd {
args := []string{}
configArgs := []string{"--config-dir", client.conf.ConfigDir}
args = append(args, configArgs...)
args = append(args, subargs...)
cmd := exec.Command("uplink", args...)
@ -120,17 +115,11 @@ func (client *Uplink) Upload(bucket, objectName string, data []byte) error {
// Download downloads object data
func (client *Uplink) Download(bucket, objectName string, buffer []byte) ([]byte, error) {
cmd := client.cmd("cat", "s3://"+bucket+"/"+objectName)
buf := &bufferWriter{buffer[:0]}
cmd.Stdout = buf
cmd.Stderr = os.Stderr
err := cmd.Run()
out, err := cmd.Output()
if err != nil {
return nil, UplinkError.Wrap(fullExitError(err))
}
return buf.data, nil
return out, nil
}
// Delete deletes object

24
scripts/test-aws-benchmark.sh Executable file
View File

@ -0,0 +1,24 @@
#!/bin/bash
set -ueo pipefail
# Purpose: This script executes upload and download benchmark tests against aws s3 to compare with storj performance.
# Setup: Assumes the awscli is installed. Assumes $AWS_ACCESS_KEY_ID and $AWS_SECRET_ACCESS_KEY environment
# variables are set with valid aws credentials with permissions to read/write to aws s3.
# Usage: from root of storj repo, run
# $ ./scripts/test-aws-benchmark.sh
aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID"
aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY"
aws configure set default.region us-east-1
# run aws s3 benchmark tests
echo
echo "Executing upload/download benchmark tests for aws s3..."
go test -bench=S3 -benchmem ./cmd/uplink/cmd/
# run s3-benchmark with aws s3
echo
echo "Executing s3-benchmark tests with aws s3 client..."
s3-benchmark --client=aws-cli --accesskey="$AWS_ACCESS_KEY_ID" --secretkey="$AWS_SECRET_ACCESS_KEY" --location="us-east-1" --s3-gateway="https://s3.amazonaws.com/"

10
scripts/test-sim-benchmark.sh
View File

@ -8,19 +8,19 @@ set -ueo pipefail
# To run and filter out storj-sim logs, run:
# $ storj-sim -x network test bash ./scripts/test-sim-benchmark.sh | grep -i "test.out"
SATELLITE_0_ADDR=${SATELLITE_0_ADDR:-127.0.0.1}
SATELLITE_0_ADDR=${SATELLITE_0_ADDR:-127.0.0.1:10000}
apiKey=$(storj-sim network env GATEWAY_0_API_KEY)
export apiKey=$(storj-sim network env GATEWAY_0_API_KEY)
echo "apiKey:"
echo "$apiKey"
# run benchmark tests normally
# run benchmark tests
echo
echo "Executing benchmark tests locally"
go test -bench . -benchmem ./cmd/uplink/cmd/
echo "Executing benchmark tests with uplink client against storj-sim..."
go test -bench=Uplink -benchmem ./cmd/uplink/cmd/
# run s3-benchmark with uplink
echo
echo "Executing s3-benchmark tests with uplink client..."
echo "Executing s3-benchmark tests with uplink client against storj-sim..."
s3-benchmark --client=uplink --satellite="$SATELLITE_0_ADDR" --apikey="$apiKey"