JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

s3-benchmark: add aws-cli support (#386)

Browse Source
This commit is contained in:
Egon Elbre 2018-09-28 15:40:08 +03:00 committed by GitHub
parent c9cabaf73a
commit e83502f33d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 844 additions and 272 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

183
cmd/s3-benchmark/awscli.go Normal file
View File

@ -0,0 +1,183 @@
package main
import (
"bytes"
"encoding/json"
"os"
"os/exec"
"strings"
"github.com/zeebo/errs"
)
// AWSCLIError is class for minio errors
var AWSCLIError = errs.Class("aws-cli error")
// AWSCLI implements basic S3 Client with aws-cli
type AWSCLI struct {
conf Config
}
// NewAWSCLI creates new Client
func NewAWSCLI(conf Config) (Client, error) {
if !strings.HasPrefix(conf.Endpoint, "https://") &&
!strings.HasPrefix(conf.Endpoint, "http://") {
conf.Endpoint = "http://" + conf.Endpoint
}
return &AWSCLI{conf}, nil
}
func (client *AWSCLI) cmd(subargs ...string) *exec.Cmd {
args := []string{
"--endpoint", client.conf.Endpoint,
}
if client.conf.NoSSL {
args = append(args, "--no-verify-ssl")
}
args = append(args, subargs...)
cmd := exec.Command("aws", args...)
cmd.Env = append(os.Environ(),
"AWS_ACCESS_KEY_ID="+client.conf.AccessKey,
"AWS_SECRET_ACCESS_KEY="+client.conf.SecretKey,
)
return cmd
}
// MakeBucket makes a new bucket
func (client *AWSCLI) MakeBucket(bucket, location string) error {
cmd := client.cmd("s3", "mb", "s3://"+bucket, "--region", location)
_, err := cmd.Output()
if err != nil {
return AWSCLIError.Wrap(err)
}
return nil
}
// RemoveBucket removes a bucket
func (client *AWSCLI) RemoveBucket(bucket string) error {
cmd := client.cmd("s3", "rb", "s3://"+bucket)
_, err := cmd.Output()
if err != nil {
return AWSCLIError.Wrap(err)
}
return nil
}
// ListBuckets lists all buckets
func (client *AWSCLI) ListBuckets() ([]string, error) {
cmd := client.cmd("s3api", "list-buckets", "--output", "json")
jsondata, err := cmd.Output()
if err != nil {
return nil, AWSCLIError.Wrap(err)
}
var response struct {
Buckets []struct {
Name string `json:"Name"`
} `json:"Buckets"`
}
err = json.Unmarshal(jsondata, &response)
if err != nil {
return nil, AWSCLIError.Wrap(err)
}
names := []string{}
for _, bucket := range response.Buckets {
names = append(names, bucket.Name)
}
return names, nil
}
// Upload uploads object data to the specified path
func (client *AWSCLI) Upload(bucket, objectName string, data []byte) error {
// TODO: add upload threshold
cmd := client.cmd("s3", "cp", "-", "s3://"+bucket+"/"+objectName)
cmd.Stdin = bytes.NewReader(data)
_, err := cmd.Output()
if err != nil {
return AWSCLIError.Wrap(err)
}
return nil
}
// UploadMultipart uses multipart uploads, has hardcoded threshold
func (client *AWSCLI) UploadMultipart(bucket, objectName string, data []byte, threshold int) error {
// TODO: add upload threshold
cmd := client.cmd("s3", "cp", "-", "s3://"+bucket+"/"+objectName)
cmd.Stdin = bytes.NewReader(data)
_, err := cmd.Output()
if err != nil {
return AWSCLIError.Wrap(err)
}
return nil
}
// Download downloads object data
func (client *AWSCLI) Download(bucket, objectName string, buffer []byte) ([]byte, error) {
cmd := client.cmd("s3", "cp", "s3://"+bucket+"/"+objectName, "-")
buf := &bufferWriter{buffer[:0]}
cmd.Stdout = buf
cmd.Stderr = os.Stderr
err := cmd.Run()
if err != nil {
return nil, AWSCLIError.Wrap(err)
}
return buf.data, nil
}
type bufferWriter struct {
data []byte
}
func (b *bufferWriter) Write(data []byte) (n int, err error) {
b.data = append(b.data, data...)
return len(data), nil
}
// Delete deletes object
func (client *AWSCLI) Delete(bucket, objectName string) error {
cmd := client.cmd("s3", "rm", "s3://"+bucket+"/"+objectName)
_, err := cmd.Output()
if err != nil {
return AWSCLIError.Wrap(err)
}
return nil
}
// ListObjects lists objects
func (client *AWSCLI) ListObjects(bucket, prefix string) ([]string, error) {
cmd := client.cmd("s3api", "list-objects",
"--output", "json",
"--bucket", bucket,
"--prefix", prefix)
jsondata, err := cmd.Output()
if err != nil {
return nil, AWSCLIError.Wrap(err)
}
var response struct {
Contents []struct {
Key string `json:"Key"`
} `json:"Contents"`
}
err = json.Unmarshal(jsondata, &response)
if err != nil {
return nil, AWSCLIError.Wrap(err)
}
names := []string{}
for _, object := range response.Contents {
names = append(names, object.Key)
}
return names, nil
}

293
cmd/s3-benchmark/main.go
View File

@ -7,26 +7,27 @@ import (
"bytes" "bytes"
"flag" "flag"
"fmt" "fmt"
"image/color"
"io" "io"
"io/ioutil"
"log" "log"
"math/rand"
"os" "os"
"text/tabwriter" "text/tabwriter"
"time" "time"
"github.com/loov/hrtime" "github.com/loov/hrtime"
"github.com/loov/plot"
minio "github.com/minio/minio-go" "storj.io/storj/internal/memory"
) )
func main() { func main() {
endpoint := flag.String("endpoint", "127.0.0.1:7777", "endpoint address") var conf Config
accesskey := flag.String("accesskey", "insecure-dev-access-key", "access key")
secretkey := flag.String("secretkey", "insecure-dev-secret-key", "secret key") flag.StringVar(&conf.Endpoint, "endpoint", "127.0.0.1:7777", "endpoint address")
useSSL := flag.Bool("use-ssl", true, "use ssl") flag.StringVar(&conf.AccessKey, "accesskey", "insecure-dev-access-key", "access key")
flag.StringVar(&conf.SecretKey, "secretkey", "insecure-dev-secret-key", "secret key")
flag.BoolVar(&conf.NoSSL, "no-ssl", false, "disable ssl")
clientName := flag.String("client", "minio", "client to use for requests (supported: minio, aws-cli)")
location := flag.String("location", "", "bucket location") location := flag.String("location", "", "bucket location")
count := flag.Int("count", 50, "benchmark count") count := flag.Int("count", 50, "benchmark count")
duration := flag.Duration("time", 2*time.Minute, "maximum benchmark time per size") duration := flag.Duration("time", 2*time.Minute, "maximum benchmark time per size")
@ -35,14 +36,32 @@ func main() {
plotname := flag.String("plot", "plot"+suffix+".svg", "plot results") plotname := flag.String("plot", "plot"+suffix+".svg", "plot results")
sizes := &Sizes{ sizes := &memory.Sizes{
Default: []Size{{1 * KB}, {256 * KB}, {1 * MB}, {32 * MB}, {64 * MB}, {256 * MB}}, Default: []memory.Size{
1 * memory.KB,
256 * memory.KB,
1 * memory.MB,
32 * memory.MB,
64 * memory.MB,
256 * memory.MB,
},
} }
flag.Var(sizes, "size", "sizes to test with") flag.Var(sizes, "size", "sizes to test with")
flag.Parse() flag.Parse()
client, err := minio.New(*endpoint, *accesskey, *secretkey, *useSSL) var client Client
var err error
switch *clientName {
default:
log.Println("unknown client name ", *clientName, " defaulting to minio")
fallthrough
case "minio":
client, err = NewMinio(conf)
case "aws-cli":
client, err = NewAWSCLI(conf)
}
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
@ -51,14 +70,14 @@ func main() {
log.Println("Creating bucket", bucket) log.Println("Creating bucket", bucket)
err = client.MakeBucket(bucket, *location) err = client.MakeBucket(bucket, *location)
if err != nil { if err != nil {
log.Fatal("failed to create bucket: ", bucket, ": ", err) log.Fatalf("failed to create bucket %q: %+v\n", bucket, err)
} }
defer func() { defer func() {
log.Println("Removing bucket") log.Println("Removing bucket")
err := client.RemoveBucket(bucket) err := client.RemoveBucket(bucket)
if err != nil { if err != nil {
log.Fatal("failed to remove bucket: ", bucket) log.Fatalf("failed to remove bucket %q", bucket)
} }
}() }()
@ -91,163 +110,107 @@ func main() {
_ = w.Flush() _ = w.Flush()
if *plotname != "" { if *plotname != "" {
p := plot.New() err := Plot(*plotname, measurements)
p.X.Min = 0
p.X.Max = 10
p.X.MajorTicks = 10
p.X.MinorTicks = 10
speed := plot.NewAxisGroup()
speed.Y.Min = 0
speed.Y.Max = 1
speed.X.Min = 0
speed.X.Max = 30
speed.X.MajorTicks = 10
speed.X.MinorTicks = 10
rows := plot.NewVStack()
rows.Margin = plot.R(5, 5, 5, 5)
p.Add(rows)
for _, m := range measurements {
row := plot.NewHFlex()
rows.Add(row)
row.Add(35, plot.NewTextbox(m.Size.String()))
plots := plot.NewVStack()
row.Add(0, plots)
{ // time plotting
uploadTime := plot.NewDensity("s", asSeconds(m.Upload))
uploadTime.Stroke = color.NRGBA{0, 200, 0, 255}
downloadTime := plot.NewDensity("s", asSeconds(m.Download))
downloadTime.Stroke = color.NRGBA{0, 0, 200, 255}
deleteTime := plot.NewDensity("s", asSeconds(m.Delete))
deleteTime.Stroke = color.NRGBA{200, 0, 0, 255}
flexTime := plot.NewHFlex()
plots.Add(flexTime)
flexTime.Add(70, plot.NewTextbox("time (s)"))
flexTime.AddGroup(0,
plot.NewGrid(),
uploadTime,
downloadTime,
deleteTime,
plot.NewTickLabels(),
)
}
{ // speed plotting
uploadSpeed := plot.NewDensity("MB/s", asSpeed(m.Upload, m.Size.bytes))
uploadSpeed.Stroke = color.NRGBA{0, 200, 0, 255}
downloadSpeed := plot.NewDensity("MB/s", asSpeed(m.Download, m.Size.bytes))
downloadSpeed.Stroke = color.NRGBA{0, 0, 200, 255}
flexSpeed := plot.NewHFlex()
plots.Add(flexSpeed)
speedGroup := plot.NewAxisGroup()
speedGroup.X, speedGroup.Y = speed.X, speed.Y
speedGroup.AddGroup(
plot.NewGrid(),
uploadSpeed,
downloadSpeed,
plot.NewTickLabels(),
)
flexSpeed.Add(70, plot.NewTextbox("speed (MB/s)"))
flexSpeed.AddGroup(0, speedGroup)
}
}
svgCanvas := plot.NewSVG(1500, 150*float64(len(measurements)))
p.Draw(svgCanvas)
err := ioutil.WriteFile(*plotname, svgCanvas.Bytes(), 0755)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
} }
} }
func asSeconds(durations []time.Duration) []float64 {
xs := make([]float64, 0, len(durations))
for _, dur := range durations {
xs = append(xs, dur.Seconds())
}
return xs
}
func asSpeed(durations []time.Duration, size int64) []float64 {
const MB = 1 << 20
xs := make([]float64, 0, len(durations))
for _, dur := range durations {
xs = append(xs, (float64(size)/MB)/dur.Seconds())
}
return xs
}
// Measurement contains measurements for different requests // Measurement contains measurements for different requests
type Measurement struct { type Measurement struct {
Size Size Size memory.Size
Upload []time.Duration Results []*Result
Download []time.Duration }
Delete []time.Duration
// Result contains durations for specific tests
type Result struct {
Name string
WithSpeed bool
Durations []time.Duration
}
// Result finds or creates a result with the specified name
func (m *Measurement) Result(name string) *Result {
for _, x := range m.Results {
if x.Name == name {
return x
}
}
r := &Result{}
r.Name = name
m.Results = append(m.Results, r)
return r
}
// Record records a time measurement
func (m *Measurement) Record(name string, duration time.Duration) {
r := m.Result(name)
r.WithSpeed = false
r.Durations = append(r.Durations, duration)
}
// RecordSpeed records a time measurement that can be expressed in speed
func (m *Measurement) RecordSpeed(name string, duration time.Duration) {
r := m.Result(name)
r.WithSpeed = true
r.Durations = append(r.Durations, duration)
} }
// PrintStats prints important valueas about the measurement // PrintStats prints important valueas about the measurement
func (m *Measurement) PrintStats(w io.Writer) { func (m *Measurement) PrintStats(w io.Writer) {
const binCount = 10 const binCount = 10
upload := hrtime.NewDurationHistogram(m.Upload, binCount) type Hist struct {
download := hrtime.NewDurationHistogram(m.Download, binCount) *Result
delete := hrtime.NewDurationHistogram(m.Delete, binCount) *hrtime.Histogram
}
hists := []struct { hists := []Hist{}
L string for _, result := range m.Results {
H *hrtime.Histogram hists = append(hists, Hist{
}{ Result: result,
{"Upload", upload}, Histogram: hrtime.NewDurationHistogram(result.Durations, binCount),
{"Download", download}, })
{"Delete", delete},
} }
sec := func(ns float64) string { sec := func(ns float64) string {
return fmt.Sprintf("%.2f", ns/1e9) return fmt.Sprintf("%.2f", ns/1e9)
} }
speed := func(ns float64) string { speed := func(ns float64) string {
return fmt.Sprintf("%.2f", (float64(m.Size.bytes)/(1<<20))/(ns/1e9)) return fmt.Sprintf("%.2f", m.Size.MB()/(ns/1e9))
} }
for _, hist := range hists { for _, hist := range hists {
if hist.L == "Delete" { if !hist.WithSpeed {
fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n",
m.Size, hist.L, m.Size, hist.Name,
sec(hist.H.Average), "", sec(hist.Average), "",
sec(hist.H.Maximum), "", sec(hist.Maximum), "",
sec(hist.H.P50), "", sec(hist.P50), "",
sec(hist.H.P90), "", sec(hist.P90), "",
sec(hist.H.P99), "", sec(hist.P99), "",
) )
continue continue
} }
fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n",
m.Size, hist.L, m.Size, hist.Name,
sec(hist.H.Average), speed(hist.H.Average), sec(hist.Average), speed(hist.Average),
sec(hist.H.Maximum), speed(hist.H.Maximum), sec(hist.Maximum), speed(hist.Maximum),
sec(hist.H.P50), speed(hist.H.P50), sec(hist.P50), speed(hist.P50),
sec(hist.H.P90), speed(hist.H.P90), sec(hist.P90), speed(hist.P90),
sec(hist.H.P99), speed(hist.H.P99), sec(hist.P99), speed(hist.P99),
) )
} }
} }
// Benchmark runs benchmarks on bucket with given size // Benchmark runs benchmarks on bucket with given size
func Benchmark(client *minio.Client, bucket string, size Size, count int, duration time.Duration) (Measurement, error) { func Benchmark(client Client, bucket string, size memory.Size, count int, duration time.Duration) (Measurement, error) {
log.Print("Benchmarking size ", size.String(), " ") log.Print("Benchmarking size ", size.String(), " ")
data := make([]byte, size.bytes) data := make([]byte, size.Int())
result := make([]byte, size.bytes) result := make([]byte, size.Int())
defer fmt.Println() defer fmt.Println()
@ -260,50 +223,70 @@ func Benchmark(client *minio.Client, bucket string, size Size, count int, durati
} }
fmt.Print(".") fmt.Print(".")
rand.Read(data[:]) // rand.Read(data[:])
for i := range data {
data[i] = 'a' + byte(i%26)
}
{ // uploading { // uploading
start := hrtime.Now() start := hrtime.Now()
_, err := client.PutObject(bucket, "data", bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ err := client.Upload(bucket, "data", data)
ContentType: "application/octet-stream",
})
finish := hrtime.Now() finish := hrtime.Now()
if err != nil { if err != nil {
return measurement, fmt.Errorf("upload failed: %v", err) return measurement, fmt.Errorf("upload failed: %+v", err)
} }
measurement.Upload = append(measurement.Upload, (finish - start))
measurement.RecordSpeed("Upload", finish-start)
} }
{ // downloading { // downloading
start := hrtime.Now() start := hrtime.Now()
reader, err := client.GetObject(bucket, "data", minio.GetObjectOptions{}) var err error
result, err = client.Download(bucket, "data", result)
if err != nil { if err != nil {
return measurement, fmt.Errorf("get object failed: %v", err) return measurement, fmt.Errorf("get object failed: %+v", err)
}
var n int
n, err = reader.Read(result)
if err != nil && err != io.EOF {
return measurement, fmt.Errorf("download failed: %v", err)
} }
finish := hrtime.Now() finish := hrtime.Now()
if !bytes.Equal(data, result[:n]) { if !bytes.Equal(data, result) {
return measurement, fmt.Errorf("upload/download do not match: length %d != %d", len(data), n) return measurement, fmt.Errorf("upload/download do not match: lengths %d and %d", len(data), len(result))
} }
measurement.Download = append(measurement.Download, (finish - start)) measurement.RecordSpeed("Download", finish-start)
} }
{ // deleting { // deleting
start := hrtime.Now() start := hrtime.Now()
err := client.RemoveObject(bucket, "data") err := client.Delete(bucket, "data")
if err != nil { if err != nil {
return measurement, fmt.Errorf("delete failed: %v", err) return measurement, fmt.Errorf("delete failed: %+v", err)
} }
finish := hrtime.Now() finish := hrtime.Now()
measurement.Delete = append(measurement.Delete, (finish - start))
measurement.Record("Delete", finish-start)
} }
} }
return measurement, nil return measurement, nil
} }
// Config is the setup for a particular client
type Config struct {
Endpoint string
AccessKey string
SecretKey string
NoSSL bool
}
// Client is the common interface for different implementations
type Client interface {
MakeBucket(bucket, location string) error
RemoveBucket(bucket string) error
ListBuckets() ([]string, error)
Upload(bucket, objectName string, data []byte) error
UploadMultipart(bucket, objectName string, data []byte, multipartThreshold int) error
Download(bucket, objectName string, buffer []byte) ([]byte, error)
Delete(bucket, objectName string) error
ListObjects(bucket, prefix string) ([]string, error)
}

130
cmd/s3-benchmark/minio.go Normal file
View File

@ -0,0 +1,130 @@
package main
import (
"bytes"
"io"
"io/ioutil"
minio "github.com/minio/minio-go"
"github.com/zeebo/errs"
)
// MinioError is class for minio errors
var MinioError = errs.Class("minio error")
// Minio implements basic S3 Client with minio
type Minio struct {
api *minio.Client
}
// NewMinio creates new Client
func NewMinio(conf Config) (Client, error) {
api, err := minio.New(conf.Endpoint, conf.AccessKey, conf.SecretKey, !conf.NoSSL)
if err != nil {
return nil, MinioError.Wrap(err)
}
return &Minio{api}, nil
}
// MakeBucket makes a new bucket
func (client *Minio) MakeBucket(bucket, location string) error {
err := client.api.MakeBucket(bucket, location)
if err != nil {
return MinioError.Wrap(err)
}
return nil
}
// RemoveBucket removes a bucket
func (client *Minio) RemoveBucket(bucket string) error {
err := client.api.RemoveBucket(bucket)
if err != nil {
return MinioError.Wrap(err)
}
return nil
}
// ListBuckets lists all buckets
func (client *Minio) ListBuckets() ([]string, error) {
buckets, err := client.api.ListBuckets()
if err != nil {
return nil, MinioError.Wrap(err)
}
names := []string{}
for _, bucket := range buckets {
names = append(names, bucket.Name)
}
return names, nil
}
// Upload uploads object data to the specified path
func (client *Minio) Upload(bucket, objectName string, data []byte) error {
_, err := client.api.PutObject(
bucket, objectName,
bytes.NewReader(data), int64(len(data)),
minio.PutObjectOptions{ContentType: "application/octet-stream"})
if err != nil {
return MinioError.Wrap(err)
}
return nil
}
// UploadMultipart uses multipart uploads, has hardcoded threshold
func (client *Minio) UploadMultipart(bucket, objectName string, data []byte, threshold int) error {
_, err := client.api.PutObject(
bucket, objectName,
bytes.NewReader(data), -1,
minio.PutObjectOptions{ContentType: "application/octet-stream"})
if err != nil {
return MinioError.Wrap(err)
}
return nil
}
// Download downloads object data
func (client *Minio) Download(bucket, objectName string, buffer []byte) ([]byte, error) {
reader, err := client.api.GetObject(bucket, objectName, minio.GetObjectOptions{})
if err != nil {
return nil, MinioError.Wrap(err)
}
defer func() { _ = reader.Close() }()
n, err := reader.Read(buffer[:cap(buffer)])
if err != io.EOF {
rest, err := ioutil.ReadAll(reader)
if err == io.EOF {
err = nil
}
if err != nil {
return nil, MinioError.Wrap(err)
}
buffer = append(buffer, rest...)
n = len(buffer)
}
buffer = buffer[:n]
return buffer, nil
}
// Delete deletes object
func (client *Minio) Delete(bucket, objectName string) error {
err := client.api.RemoveObject(bucket, objectName)
if err != nil {
return MinioError.Wrap(err)
}
return nil
}
// ListObjects lists objects
func (client *Minio) ListObjects(bucket, prefix string) ([]string, error) {
doneCh := make(chan struct{})
defer close(doneCh)
names := []string{}
for message := range client.api.ListObjects(bucket, prefix, true, doneCh) {
names = append(names, message.Key)
}
return names, nil
}

109
cmd/s3-benchmark/plot.go Normal file
View File

@ -0,0 +1,109 @@
package main
import (
"image/color"
"io/ioutil"
"time"
"github.com/loov/plot"
)
var palette = []color.Color{
color.NRGBA{0, 200, 0, 255},
color.NRGBA{0, 0, 200, 255},
color.NRGBA{200, 0, 0, 255},
}
// Plot plots measurements into filename as an svg
func Plot(filename string, measurements []Measurement) error {
p := plot.New()
p.X.Min = 0
p.X.Max = 10
p.X.MajorTicks = 10
p.X.MinorTicks = 10
speed := plot.NewAxisGroup()
speed.Y.Min = 0
speed.Y.Max = 1
speed.X.Min = 0
speed.X.Max = 30
speed.X.MajorTicks = 10
speed.X.MinorTicks = 10
rows := plot.NewVStack()
rows.Margin = plot.R(5, 5, 5, 5)
p.Add(rows)
for _, m := range measurements {
row := plot.NewHFlex()
rows.Add(row)
row.Add(35, plot.NewTextbox(m.Size.String()))
plots := plot.NewVStack()
row.Add(0, plots)
{ // time plotting
group := []plot.Element{plot.NewGrid()}
for i, result := range m.Results {
time := plot.NewDensity("s", asSeconds(result.Durations))
time.Stroke = palette[i%len(palette)]
group = append(group, time)
}
group = append(group, plot.NewTickLabels())
flexTime := plot.NewHFlex()
plots.Add(flexTime)
flexTime.Add(70, plot.NewTextbox("time (s)"))
flexTime.AddGroup(0, group...)
}
{ // speed plotting
group := []plot.Element{plot.NewGrid()}
for i, result := range m.Results {
if !result.WithSpeed {
continue
}
speed := plot.NewDensity("MB/s", asSpeed(result.Durations, m.Size.Int64()))
speed.Stroke = palette[i%len(palette)]
}
group = append(group, plot.NewTickLabels())
flexSpeed := plot.NewHFlex()
plots.Add(flexSpeed)
speedGroup := plot.NewAxisGroup()
speedGroup.X, speedGroup.Y = speed.X, speed.Y
speedGroup.AddGroup(group...)
flexSpeed.Add(70, plot.NewTextbox("speed (MB/s)"))
flexSpeed.AddGroup(0, speedGroup)
}
}
svgCanvas := plot.NewSVG(1500, 150*float64(len(measurements)))
p.Draw(svgCanvas)
return ioutil.WriteFile(filename, svgCanvas.Bytes(), 0755)
}
func asSeconds(durations []time.Duration) []float64 {
xs := make([]float64, 0, len(durations))
for _, dur := range durations {
xs = append(xs, dur.Seconds())
}
return xs
}
func asSpeed(durations []time.Duration, size int64) []float64 {
const MB = 1 << 20
xs := make([]float64, 0, len(durations))
for _, dur := range durations {
xs = append(xs, (float64(size)/MB)/dur.Seconds())
}
return xs
}

117
cmd/s3-benchmark/size.go
View File

@ -1,117 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package main
import (
"errors"
"strconv"
"strings"
)
// Sizes implements flag.Value for collecting byte counts
type Sizes struct {
Default []Size
Custom []Size
}
// Sizes returns the loaded values
func (sizes Sizes) Sizes() []Size {
if len(sizes.Custom) > 0 {
return sizes.Custom
}
return sizes.Default
}
// String converts values to a string
func (sizes Sizes) String() string {
sz := sizes.Sizes()
xs := make([]string, len(sz))
for i, size := range sz {
xs[i] = size.String()
}
return strings.Join(xs, " ")
}
// Set adds values from byte values
func (sizes *Sizes) Set(s string) error {
for _, x := range strings.Fields(s) {
var size Size
if err := size.Set(x); err != nil {
return err
}
sizes.Custom = append(sizes.Custom, size)
}
return nil
}
// Size represents a value of bytes
type Size struct {
bytes int64
}
type unit struct {
suffix string
scale float64
}
// different byte-size suffixes
const (
TB = 1 << 40
GB = 1 << 40
MB = 1 << 40
KB = 1 << 40
B = 1
)
var units = []unit{
{"T", TB},
{"G", GB},
{"M", MB},
{"K", KB},
{"B", B},
{"", 0},
}
// String converts size to a string
func (size Size) String() string {
if size.bytes <= 0 {
return "0"
}
v := float64(size.bytes)
for _, unit := range units {
if v >= unit.scale {
r := strconv.FormatFloat(v/unit.scale, 'f', 1, 64)
r = strings.TrimSuffix(r, "0")
r = strings.TrimSuffix(r, ".")
return r + unit.suffix
}
}
return strconv.Itoa(int(size.bytes)) + "B"
}
// Set updates value from string
func (size *Size) Set(s string) error {
if s == "" {
return errors.New("empty size")
}
value, suffix := s[:len(s)-1], s[len(s)-1]
if '0' <= suffix && suffix <= '9' {
suffix = 'B'
value = s
}
for _, unit := range units {
if unit.suffix == string(suffix) {
v, err := strconv.ParseFloat(value, 64)
if err != nil {
return err
}
size.bytes = int64(v * unit.scale)
return nil
}
}
return errors.New("unknown suffix " + string(suffix))
}

128
internal/memory/size.go Normal file
View File

@ -0,0 +1,128 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package memory
import (
"errors"
"fmt"
"strconv"
"strings"
)
// different sizes
const (
B Size = 1 << (10 * iota)
KB
MB
GB
TB
PB
EB
)
// Size implements flag.Value for collecting memory size in bytes
type Size int64
// Int returns bytes size as int
func (size Size) Int() int { return int(size) }
// Int64 returns bytes size as int64
func (size Size) Int64() int64 { return int64(size) }
// Float64 returns bytes size as float64
func (size Size) Float64() float64 { return float64(size) }
// KB returns size in kilobytes
func (size Size) KB() float64 { return size.Float64() / KB.Float64() }
// MB returns size in megabytes
func (size Size) MB() float64 { return size.Float64() / MB.Float64() }
// GB returns size in gigabytes
func (size Size) GB() float64 { return size.Float64() / GB.Float64() }
// TB returns size in terabytes
func (size Size) TB() float64 { return size.Float64() / TB.Float64() }
// PB returns size in petabytes
func (size Size) PB() float64 { return size.Float64() / PB.Float64() }
// EB returns size in etabytes
func (size Size) EB() float64 { return size.Float64() / EB.Float64() }
// String converts size to a string
func (size Size) String() string {
if size == 0 {
return "0"
}
switch {
case size >= EB*2/3:
return fmt.Sprintf("%.1f EB", size.EB())
case size >= PB*2/3:
return fmt.Sprintf("%.1f PB", size.PB())
case size >= TB*2/3:
return fmt.Sprintf("%.1f TB", size.TB())
case size >= GB*2/3:
return fmt.Sprintf("%.1f GB", size.GB())
case size >= MB*2/3:
return fmt.Sprintf("%.1f MB", size.MB())
case size >= KB*2/3:
return fmt.Sprintf("%.1f KB", size.KB())
}
return strconv.Itoa(size.Int()) + " B"
}
func isLetter(b byte) bool {
return ('a' <= b && b <= 'z') || ('A' <= b && b <= 'Z')
}
// Set updates value from string
func (size *Size) Set(s string) error {
if s == "" {
return errors.New("empty size")
}
p := len(s)
if isLetter(s[len(s)-1]) {
p--
if len(s)-2 >= 0 && isLetter(s[len(s)-2]) {
p--
}
}
value, suffix := s[:p], s[p:]
suffix = strings.ToUpper(suffix)
if suffix == "" || suffix[len(suffix)-1] != 'B' {
suffix += "B"
}
value = strings.TrimSpace(value)
v, err := strconv.ParseFloat(value, 64)
if err != nil {
return err
}
switch suffix {
case "EB":
*size = Size(v * EB.Float64())
case "PB":
*size = Size(v * PB.Float64())
case "TB":
*size = Size(v * TB.Float64())
case "GB":
*size = Size(v * GB.Float64())
case "MB":
*size = Size(v * MB.Float64())
case "KB":
*size = Size(v * KB.Float64())
case "B", "":
*size = Size(v)
default:
return fmt.Errorf("unknown suffix %q", suffix)
}
return nil
}

104
internal/memory/size_test.go Normal file
View File

@ -0,0 +1,104 @@
package memory_test
import (
"testing"
"storj.io/storj/internal/memory"
)
const (
tb = 1 << 40
gb = 1 << 30
mb = 1 << 20
kb = 1 << 10
)
func TestSize(t *testing.T) {
var tests = []struct {
size memory.Size
text string
}{
// basics
{1 * tb, "1.0 TB"},
{1 * gb, "1.0 GB"},
{1 * mb, "1.0 MB"},
{1 * kb, "1.0 KB"},
{1, "1 B"},
// complicated
{68 * tb, "68.0 TB"},
{256 * mb, "256.0 MB"},
{500, "500 B"},
{5, "5 B"},
{1, "1 B"},
{0, "0"},
}
for i, test := range tests {
if test.size.String() != test.text {
t.Errorf("%d. invalid text got %v expected %v", i, test.size.String(), test.text)
}
var size memory.Size
err := size.Set(test.text)
if err != nil {
t.Errorf("%d. got error %v", i, err)
}
if test.size != size {
t.Errorf("%d. invalid size got %d expected %d", i, size, test.size)
}
}
}
func TestParse(t *testing.T) {
var tests = []struct {
size memory.Size
text string
}{
// case insensitivity
{1 * tb, "1.00TB"},
{1 * gb, "1.00gB"},
{1 * mb, "1.00Mb"},
{1 * kb, "1.00kb"},
{1, "1.00"},
{1 * tb, "1.0 TB"},
{1 * gb, "1.0 gB"},
{1 * mb, "1.0 Mb"},
{1 * kb, "1.0 kb"},
{1, "1.00"},
// without B suffix
{1 * tb, "1.00T"},
{1 * gb, "1.00g"},
{1 * mb, "1.00M"},
{1 * kb, "1.00k"},
}
for i, test := range tests {
var size memory.Size
err := size.Set(test.text)
if err != nil {
t.Errorf("%d. got error %v", i, err)
}
if test.size != size {
t.Errorf("%d. invalid size got %d expected %d", i, size, test.size)
}
}
}
func TestInvalidParse(t *testing.T) {
var tests = []string{
"1.0Q",
"1.0QB",
"z1.0KB",
"z1.0Q",
"1.0zQ",
"1.0zQB",
}
for i, test := range tests {
var size memory.Size
err := size.Set(test)
if err == nil {
t.Errorf("%d. didn't get error", i)
}
}
}

39
internal/memory/sizes.go Normal file
View File

@ -0,0 +1,39 @@
package memory
import "strings"
// Sizes implements flag.Value for collecting memory size
type Sizes struct {
Default []Size
Custom []Size
}
// Sizes returns the loaded values
func (sizes Sizes) Sizes() []Size {
if len(sizes.Custom) > 0 {
return sizes.Custom
}
return sizes.Default
}
// String converts values to a string
func (sizes Sizes) String() string {
sz := sizes.Sizes()
xs := make([]string, len(sz))
for i, size := range sz {
xs[i] = size.String()
}
return strings.Join(xs, " ")
}
// Set adds values from byte values
func (sizes *Sizes) Set(s string) error {
for _, x := range strings.Fields(s) {
var size Size
if err := size.Set(x); err != nil {
return err
}
sizes.Custom = append(sizes.Custom, size)
}
return nil
}

13
internal/memory/string.go Normal file
View File

@ -0,0 +1,13 @@
package memory
// FormatBytes converts number of bytes to appropriately sized string
func FormatBytes(bytes int64) string {
return Size(bytes).String()
}
// ParseString converts string to number of bytes
func ParseString(s string) (int64, error) {
var size Size
err := size.Set(s)
return size.Int64(), err
}