From fb86238acc9586f954e5c23fe05668da5bc01b58 Mon Sep 17 00:00:00 2001 From: Jess G Date: Tue, 28 May 2019 11:46:58 -0700 Subject: [PATCH] aws s3 performance tests (#2060) * add aws s3 benchmark script * add s3 benchmark tests * rearrange so smaller diff, fix spelling * add configurable uplink config for s3-benchmark * make new bucket w/unique name for each s3 test * changes per CR --- cmd/s3-benchmark/main.go | 1 + cmd/uplink/cmd/cp_test.go | 178 +++++++++++++++++++++++----------- internal/s3client/common.go | 1 + internal/s3client/uplink.go | 59 +++++------ scripts/test-aws-benchmark.sh | 24 +++++ scripts/test-sim-benchmark.sh | 10 +- 6 files changed, 178 insertions(+), 95 deletions(-) create mode 100755 scripts/test-aws-benchmark.sh diff --git a/cmd/s3-benchmark/main.go b/cmd/s3-benchmark/main.go index 8688090dd..53199cf40 100644 --- a/cmd/s3-benchmark/main.go +++ b/cmd/s3-benchmark/main.go @@ -30,6 +30,7 @@ func main() { flag.StringVar(&conf.APIKey, "apikey", "abc123", "api key") flag.StringVar(&conf.EncryptionKey, "encryptionkey", "abc123", "encryption key") flag.BoolVar(&conf.NoSSL, "no-ssl", false, "disable ssl") + flag.StringVar(&conf.ConfigDir, "config-dir", "", "path of config dir to use. If empty, a config will be created.") clientName := flag.String("client", "minio", "client to use for requests (supported: minio, aws-cli, uplink)") diff --git a/cmd/uplink/cmd/cp_test.go b/cmd/uplink/cmd/cp_test.go index 2ed89da0b..6f7be5b7d 100644 --- a/cmd/uplink/cmd/cp_test.go +++ b/cmd/uplink/cmd/cp_test.go @@ -15,10 +15,6 @@ import ( "storj.io/storj/internal/s3client" ) -const ( - bucket = "testbucket" -) - var benchmarkCases = []struct { name string objectsize memory.Size @@ -48,7 +44,7 @@ func createObjects() map[string][]byte { } // uplinkSetup setups an uplink to use for testing uploads/downloads -func uplinkSetup() s3client.Client { +func uplinkSetup(bucket string) s3client.Client { conf, err := setupConfig() if err != nil { log.Fatalf("failed to setup s3client config: %+v\n", err) @@ -72,7 +68,7 @@ func setupConfig() (s3client.Config, error) { var conf s3client.Config conf.EncryptionKey = uplinkEncryptionKey conf.Satellite = getEnvOrDefault("SATELLITE_0_ADDR", defaultSatelliteAddr) - conf.APIKey = getEnvOrDefault(os.Getenv("GATEWAY_0_API_KEY"), os.Getenv("apiKey")) + conf.APIKey = getEnvOrDefault("GATEWAY_0_API_KEY", os.Getenv("apiKey")) if conf.APIKey == "" { return conf, errors.New("no api key provided. Expecting an env var $GATEWAY_0_API_KEY or $apiKey") } @@ -86,13 +82,72 @@ func getEnvOrDefault(key, fallback string) string { return fallback } -func BenchmarkUpload(b *testing.B) { - var client = uplinkSetup() +func BenchmarkUpload_Uplink(b *testing.B) { + bucket := "testbucket" + client := uplinkSetup(bucket) // uploadedObjects is used to store the names of all objects that are uploaded // so that we can make sure to delete them all during cleanup var uploadedObjects = map[string][]string{} + uploadedObjects = benchmarkUpload(b, client, bucket, uploadedObjects) + + teardown(client, bucket, uploadedObjects) +} + +func teardown(client s3client.Client, bucket string, uploadedObjects map[string][]string) { + for _, bm := range benchmarkCases { + for _, objectPath := range uploadedObjects[bm.name] { + err := client.Delete(bucket, objectPath) + if err != nil { + log.Printf("failed to delete object %q: %+v\n", objectPath, err) + } + } + } + + err := client.RemoveBucket(bucket) + if err != nil { + log.Fatalf("failed to remove bucket %q: %+v\n", bucket, err) + } +} + +func BenchmarkDownload_Uplink(b *testing.B) { + bucket := "testbucket" + client := uplinkSetup(bucket) + + // upload some test objects so that there is something to download + uploadTestObjects(client, bucket) + + benchmarkDownload(b, bucket, client) + + teardownTestObjects(client, bucket) +} + +func uploadTestObjects(client s3client.Client, bucket string) { + for name, data := range testObjects { + objectName := "folder/data_" + name + err := client.Upload(bucket, objectName, data) + if err != nil { + log.Fatalf("failed to upload object %q: %+v\n", objectName, err) + } + } +} + +func teardownTestObjects(client s3client.Client, bucket string) { + for name := range testObjects { + objectName := "folder/data_" + name + err := client.Delete(bucket, objectName) + if err != nil { + log.Fatalf("failed to delete object %q: %+v\n", objectName, err) + } + } + err := client.RemoveBucket(bucket) + if err != nil { + log.Fatalf("failed to remove bucket %q: %+v\n", bucket, err) + } +} + +func benchmarkUpload(b *testing.B, client s3client.Client, bucket string, uploadedObjects map[string][]string) map[string][]string { for _, bm := range benchmarkCases { b.Run(bm.name, func(b *testing.B) { b.SetBytes(bm.objectsize.Int64()) @@ -114,32 +169,10 @@ func BenchmarkUpload(b *testing.B) { } }) } - - teardown(client, uploadedObjects) + return uploadedObjects } -func teardown(client s3client.Client, uploadedObjects map[string][]string) { - for _, bm := range benchmarkCases { - for _, objectPath := range uploadedObjects[bm.name] { - err := client.Delete(bucket, objectPath) - if err != nil { - log.Printf("failed to delete object %q: %+v\n", objectPath, err) - } - } - } - - err := client.RemoveBucket(bucket) - if err != nil { - log.Fatalf("failed to remove bucket %q: %+v\n", bucket, err) - } -} - -func BenchmarkDownload(b *testing.B) { - var client = uplinkSetup() - - // upload some test objects so that there is something to download - uploadTestObjects(client) - +func benchmarkDownload(b *testing.B, bucket string, client s3client.Client) { for _, bm := range benchmarkCases { b.Run(bm.name, func(b *testing.B) { buf := make([]byte, bm.objectsize) @@ -147,37 +180,72 @@ func BenchmarkDownload(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { objectName := "folder/data_" + bm.name - _, err := client.Download(bucket, objectName, buf) + out, err := client.Download(bucket, objectName, buf) if err != nil { log.Fatalf("failed to download object %q: %+v\n", objectName, err) } + expectedBytes := bm.objectsize.Int() + actualBytes := len(out) + if actualBytes != expectedBytes { + log.Fatalf("err downloading object %q: Expected %d bytes, but got actual bytes: %d\n", + objectName, expectedBytes, actualBytes, + ) + } } }) } - - teardownTestObjects(client) } -func uploadTestObjects(client s3client.Client) { - for name, data := range testObjects { - objectName := "folder/data_" + name - err := client.Upload(bucket, objectName, data) - if err != nil { - log.Fatalf("failed to upload object %q: %+v\n", objectName, err) - } - } -} - -func teardownTestObjects(client s3client.Client) { - for name := range testObjects { - objectName := "folder/data_" + name - err := client.Delete(bucket, objectName) - if err != nil { - log.Fatalf("failed to delete object %q: %+v\n", objectName, err) - } - } - err := client.RemoveBucket(bucket) +func s3ClientSetup(bucket string) s3client.Client { + conf, err := s3ClientConfigSetup() if err != nil { - log.Fatalf("failed to remove bucket %q: %+v\n", bucket, err) + log.Fatalf("failed to setup s3client config: %+v\n", err) } + client, err := s3client.NewAWSCLI(conf) + if err != nil { + log.Fatalf("failed to create s3client NewUplink: %+v\n", err) + } + const region = "us-east-1" + err = client.MakeBucket(bucket, region) + if err != nil { + log.Fatalf("failed to create bucket with s3client %q: %+v\n", bucket, err) + } + return client +} + +func s3ClientConfigSetup() (s3client.Config, error) { + const s3gateway = "https://s3.amazonaws.com/" + var conf s3client.Config + conf.S3Gateway = s3gateway + conf.AccessKey = os.Getenv("AWS_ACCESS_KEY_ID") + conf.SecretKey = os.Getenv("AWS_SECRET_ACCESS_KEY") + if conf.AccessKey == "" || conf.SecretKey == "" { + return conf, errors.New("expecting environment variables $AWS_ACCESS_KEY_ID and $AWS_SECRET_ACCESS_KEY to be set") + } + return conf, nil +} + +func BenchmarkUpload_S3(b *testing.B) { + bucket := "testbucket3bgdp2xbkkflxc2tallstvh6pb824r" + var client = s3ClientSetup(bucket) + + // uploadedObjects is used to store the names of all objects that are uploaded + // so that we can make sure to delete them all during cleanup + var uploadedObjects = map[string][]string{} + + uploadedObjects = benchmarkUpload(b, client, bucket, uploadedObjects) + + teardown(client, bucket, uploadedObjects) +} + +func BenchmarkDownload_S3(b *testing.B) { + bucket := "testbucket3bgdp2xbkkflxc2tallstvh6pb826a" + var client = s3ClientSetup(bucket) + + // upload some test objects so that there is something to download + uploadTestObjects(client, bucket) + + benchmarkDownload(b, bucket, client) + + teardownTestObjects(client, bucket) } diff --git a/internal/s3client/common.go b/internal/s3client/common.go index 28b36516c..8a9a4f538 100644 --- a/internal/s3client/common.go +++ b/internal/s3client/common.go @@ -12,6 +12,7 @@ type Config struct { APIKey string EncryptionKey string NoSSL bool + ConfigDir string } // Client is the common interface for different implementations diff --git a/internal/s3client/uplink.go b/internal/s3client/uplink.go index 6c515b087..d52949871 100755 --- a/internal/s3client/uplink.go +++ b/internal/s3client/uplink.go @@ -8,7 +8,6 @@ import ( "fmt" "os" "os/exec" - "path/filepath" "strings" "github.com/zeebo/errs" @@ -28,37 +27,31 @@ type Uplink struct { func NewUplink(conf Config) (Client, error) { client := &Uplink{conf} - defaultConfDir := fpath.ApplicationDir("storj", "uplink") - setupDir, err := filepath.Abs(defaultConfDir) + if client.conf.ConfigDir != "" { + fmt.Printf("Using existing uplink config at %s\n", client.conf.ConfigDir) + return client, nil + } + + client.conf.ConfigDir = fpath.ApplicationDir("storj", "s3-client", "uplink") + + // remove existing s3client uplink config so that + // we can create a new one with up-to-date settings + err := os.RemoveAll(client.conf.ConfigDir) if err != nil { return nil, UplinkError.Wrap(fullExitError(err)) } - validForSetup, _ := fpath.IsValidSetupDir(setupDir) - // uplink configuration doesn't exists - if validForSetup { - fmt.Printf(`No existing uplink configuration located at (%v)... - Creating uplink configuration with the following settings: - "--non-interactive: true", - "--api-key: %s", - "--enc.key: %s", - "--satellite-addr: %s - `, - setupDir, client.conf.APIKey, client.conf.EncryptionKey, client.conf.Satellite, - ) - cmd := client.cmd("setup", - "--non-interactive", "true", - "--api-key", client.conf.APIKey, - "--enc.key", client.conf.EncryptionKey, - "--satellite-addr", client.conf.Satellite) + cmd := client.cmd("--config-dir", client.conf.ConfigDir, + "setup", + "--non-interactive", "true", + "--api-key", client.conf.APIKey, + "--enc.encryption-key", client.conf.EncryptionKey, + "--satellite-addr", client.conf.Satellite, + ) - _, err := cmd.Output() - if err != nil { - return nil, UplinkError.Wrap(fullExitError(err)) - } - } else { - // if uplink config file already exists, use the current config - fmt.Printf("Using existing uplink configuration from (%v). To pass in new settings, delete existing configs first\n", setupDir) + _, err = cmd.Output() + if err != nil { + return nil, UplinkError.Wrap(fullExitError(err)) } return client, nil @@ -67,6 +60,8 @@ func NewUplink(conf Config) (Client, error) { func (client *Uplink) cmd(subargs ...string) *exec.Cmd { args := []string{} + configArgs := []string{"--config-dir", client.conf.ConfigDir} + args = append(args, configArgs...) args = append(args, subargs...) cmd := exec.Command("uplink", args...) @@ -120,17 +115,11 @@ func (client *Uplink) Upload(bucket, objectName string, data []byte) error { // Download downloads object data func (client *Uplink) Download(bucket, objectName string, buffer []byte) ([]byte, error) { cmd := client.cmd("cat", "s3://"+bucket+"/"+objectName) - - buf := &bufferWriter{buffer[:0]} - cmd.Stdout = buf - cmd.Stderr = os.Stderr - - err := cmd.Run() + out, err := cmd.Output() if err != nil { return nil, UplinkError.Wrap(fullExitError(err)) } - - return buf.data, nil + return out, nil } // Delete deletes object diff --git a/scripts/test-aws-benchmark.sh b/scripts/test-aws-benchmark.sh new file mode 100755 index 000000000..76d7ccd9c --- /dev/null +++ b/scripts/test-aws-benchmark.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -ueo pipefail + +# Purpose: This script executes upload and download benchmark tests against aws s3 to compare with storj performance. +# Setup: Assumes the awscli is installed. Assumes $AWS_ACCESS_KEY_ID and $AWS_SECRET_ACCESS_KEY environment +# variables are set with valid aws credentials with permissions to read/write to aws s3. +# Usage: from root of storj repo, run +# $ ./scripts/test-aws-benchmark.sh + +aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID" +aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY" +aws configure set default.region us-east-1 + + +# run aws s3 benchmark tests +echo +echo "Executing upload/download benchmark tests for aws s3..." +go test -bench=S3 -benchmem ./cmd/uplink/cmd/ + + +# run s3-benchmark with aws s3 +echo +echo "Executing s3-benchmark tests with aws s3 client..." +s3-benchmark --client=aws-cli --accesskey="$AWS_ACCESS_KEY_ID" --secretkey="$AWS_SECRET_ACCESS_KEY" --location="us-east-1" --s3-gateway="https://s3.amazonaws.com/" diff --git a/scripts/test-sim-benchmark.sh b/scripts/test-sim-benchmark.sh index 56d3b2701..ed72aab74 100755 --- a/scripts/test-sim-benchmark.sh +++ b/scripts/test-sim-benchmark.sh @@ -8,19 +8,19 @@ set -ueo pipefail # To run and filter out storj-sim logs, run: # $ storj-sim -x network test bash ./scripts/test-sim-benchmark.sh | grep -i "test.out" -SATELLITE_0_ADDR=${SATELLITE_0_ADDR:-127.0.0.1} +SATELLITE_0_ADDR=${SATELLITE_0_ADDR:-127.0.0.1:10000} apiKey=$(storj-sim network env GATEWAY_0_API_KEY) export apiKey=$(storj-sim network env GATEWAY_0_API_KEY) echo "apiKey:" echo "$apiKey" -# run benchmark tests normally +# run benchmark tests echo -echo "Executing benchmark tests locally" -go test -bench . -benchmem ./cmd/uplink/cmd/ +echo "Executing benchmark tests with uplink client against storj-sim..." +go test -bench=Uplink -benchmem ./cmd/uplink/cmd/ # run s3-benchmark with uplink echo -echo "Executing s3-benchmark tests with uplink client..." +echo "Executing s3-benchmark tests with uplink client against storj-sim..." s3-benchmark --client=uplink --satellite="$SATELLITE_0_ADDR" --apikey="$apiKey"